From 5b74dde6424c6392f787148e7b9330dc28f8d516 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 28 Sep 2020 11:14:43 +0100 Subject: [PATCH 1/8] WIP --- .../mapper/AbstractGeometryFieldMapper.java | 30 +++++------ .../index/mapper/DateFieldMapper.java | 49 +++++++++-------- .../index/mapper/FieldMapper.java | 8 --- .../index/mapper/FieldNamesFieldMapper.java | 6 +++ .../index/mapper/MappedFieldType.java | 6 +++ .../index/mapper/MetadataFieldMapper.java | 5 -- .../index/mapper/NestedPathFieldMapper.java | 6 +++ .../index/mapper/NumberFieldMapper.java | 52 +++++++++++-------- .../index/mapper/RoutingFieldMapper.java | 6 +++ .../search/fetch/subphase/FieldFetcher.java | 21 ++------ 10 files changed, 99 insertions(+), 90 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index f782eea59dfe3..c90242f0ea368 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -181,22 +181,6 @@ public Builder ignoreZValue(final boolean ignoreZValue) { } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - String geoFormat = format != null ? format : GeoJsonGeometryFormat.NAME; - - AbstractGeometryFieldType mappedFieldType = fieldType(); - Parser geometryParser = mappedFieldType.geometryParser(); - Function valueParser = value -> geometryParser.parseAndFormatObject(value, this, geoFormat); - - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { - @Override - protected Object parseSourceValue(Object value) { - return valueParser.apply(value); - } - }; - } - public abstract static class TypeParser implements Mapper.TypeParser { protected abstract T newBuilder(String name, Map params); @@ -287,6 +271,20 @@ public Query termQuery(Object value, QueryShardContext context) { "Geometry fields do not support exact searching, use dedicated geometry queries instead: [" + name() + "]"); } + + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + String geoFormat = format != null ? format : GeoJsonGeometryFormat.NAME; + + Function valueParser = value -> geometryParser.parseAndFormatObject(value, this, geoFormat); + + return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { + @Override + protected Object parseSourceValue(Object value) { + return valueParser.apply(value); + } + }; + } } protected Explicit ignoreMalformed; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 0508cd1def3f4..e3af2834ffd43 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -247,7 +247,7 @@ private Long parseNullValue(DateFieldType fieldType) { @Override public DateFieldMapper build(BuilderContext context) { DateFieldType ft = new DateFieldType(buildFullName(context), index.getValue(), store.getValue(), docValues.getValue(), - buildFormatter(), resolution, meta.getValue()); + buildFormatter(), resolution, nullValue.getValue(), meta.getValue()); Long nullTimestamp = parseNullValue(ft); return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), nullTimestamp, resolution, this); @@ -268,25 +268,28 @@ public static final class DateFieldType extends MappedFieldType { protected final DateFormatter dateTimeFormatter; protected final DateMathParser dateMathParser; protected final Resolution resolution; + protected final String nullValue; public DateFieldType(String name, boolean isSearchable, boolean isStored, boolean hasDocValues, - DateFormatter dateTimeFormatter, Resolution resolution, Map meta) { + DateFormatter dateTimeFormatter, Resolution resolution, String nullValue, + Map meta) { super(name, isSearchable, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.dateTimeFormatter = dateTimeFormatter; this.dateMathParser = dateTimeFormatter.toDateMathParser(); this.resolution = resolution; + this.nullValue = nullValue; } public DateFieldType(String name) { - this(name, true, false, true, DEFAULT_DATE_TIME_FORMATTER, Resolution.MILLISECONDS, Collections.emptyMap()); + this(name, true, false, true, DEFAULT_DATE_TIME_FORMATTER, Resolution.MILLISECONDS, null, Collections.emptyMap()); } public DateFieldType(String name, DateFormatter dateFormatter) { - this(name, true, false, true, dateFormatter, Resolution.MILLISECONDS, Collections.emptyMap()); + this(name, true, false, true, dateFormatter, Resolution.MILLISECONDS, null, Collections.emptyMap()); } public DateFieldType(String name, Resolution resolution) { - this(name, true, false, true, DEFAULT_DATE_TIME_FORMATTER, resolution, Collections.emptyMap()); + this(name, true, false, true, DEFAULT_DATE_TIME_FORMATTER, resolution, null, Collections.emptyMap()); } @Override @@ -311,6 +314,24 @@ public long parse(String value) { return resolution.convert(DateFormatters.from(dateTimeFormatter().parse(value), dateTimeFormatter().locale()).toInstant()); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + DateFormatter defaultFormatter = dateTimeFormatter(); + DateFormatter formatter = format != null + ? DateFormatter.forPattern(format).withLocale(defaultFormatter.locale()) + : defaultFormatter; + + return new SourceValueFetcher(name(), mapperService, false, nullValue) { + @Override + public String parseSourceValue(Object value) { + String date = value.toString(); + long timestamp = parse(date); + ZonedDateTime dateTime = resolution().toInstant(timestamp).atZone(ZoneOffset.UTC); + return formatter.format(dateTime); + } + }; + } + @Override public Query termQuery(Object value, @Nullable QueryShardContext context) { return rangeQuery(value, value, true, true, ShapeRelation.INTERSECTS, null, null, context); @@ -600,24 +621,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - DateFormatter defaultFormatter = fieldType().dateTimeFormatter(); - DateFormatter formatter = format != null - ? DateFormatter.forPattern(format).withLocale(defaultFormatter.locale()) - : defaultFormatter; - - return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValueAsString) { - @Override - public String parseSourceValue(Object value) { - String date = value.toString(); - long timestamp = fieldType().parse(date); - ZonedDateTime dateTime = fieldType().resolution().toInstant(timestamp).atZone(ZoneOffset.UTC); - return formatter.format(dateTime); - } - }; - } - public boolean getIgnoreMalformed() { return ignoreMalformed; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 0d8baf16acf56..7e242fb7668ac 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -24,7 +24,6 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -34,8 +33,6 @@ import org.elasticsearch.common.xcontent.support.AbstractXContentParser; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldNamesFieldMapper.FieldNamesFieldType; -import org.elasticsearch.search.fetch.subphase.FetchFieldsPhase; -import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; @@ -270,11 +267,6 @@ public void parse(ParseContext context) throws IOException { */ protected abstract void parseCreateField(ParseContext context) throws IOException; - /** - * Create a helper class to fetch field values during the {@link FetchFieldsPhase}. - */ - public abstract ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, @Nullable String format); - protected final void createFieldNamesField(ParseContext context) { assert fieldType().hasDocValues() == false : "_field_names should only be used when doc_values are turned off"; FieldNamesFieldType fieldNamesFieldType = context.docMapper().metadataMapper(FieldNamesFieldMapper.class).fieldType(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java index 7fd340e06f903..5525a8c551f9d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.util.Collections; import java.util.Iterator; @@ -129,6 +130,11 @@ public boolean isEnabled() { return enabled; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { + throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); + } + @Override public Query existsQuery(QueryShardContext context) { throw new UnsupportedOperationException("Cannot run exists query on _field_names"); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 7d43216d62728..e3a68370401d1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -50,6 +50,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.fetch.subphase.FetchFieldsPhase; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; @@ -97,6 +98,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S throw new IllegalArgumentException("Fielddata is not supported on field [" + name() + "] of type [" + typeName() + "]"); } + /** + * Create a helper class to fetch field values during the {@link FetchFieldsPhase}. + */ + public abstract ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, @Nullable String format); + /** Returns the name of this type, as would be specified in mapping properties */ public abstract String typeName(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 1a56b92d7e417..bbba108e2e803 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Map; @@ -173,8 +172,4 @@ public void postParse(ParseContext context) throws IOException { // do nothing } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { - throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java index 6fba5ca6817b6..fa9ea5f5460d9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.util.Collections; @@ -86,6 +87,11 @@ public String typeName() { public Query existsQuery(QueryShardContext context) { throw new UnsupportedOperationException("Cannot run exists() query against the nested field path"); } + + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { + throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); + } } private NestedPathFieldMapper(Settings settings) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index e3c3a98a94a18..87544a4a1e7db 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -126,9 +126,8 @@ protected List> getParameters() { @Override public NumberFieldMapper build(BuilderContext context) { - return new NumberFieldMapper(name, - new NumberFieldType(buildFullName(context), type, indexed.getValue(), stored.getValue(), hasDocValues.getValue(), - meta.getValue()), multiFieldsBuilder.build(this, context), copyTo.build(), this); + MappedFieldType ft = new NumberFieldType(buildFullName(context), this); + return new NumberFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this); } } @@ -890,16 +889,25 @@ public static Query longRangeQuery( public static final class NumberFieldType extends SimpleMappedFieldType { private final NumberType type; + private final boolean coerce; + private final Number nullValue; public NumberFieldType(String name, NumberType type, boolean isSearchable, boolean isStored, - boolean hasDocValues, Map meta) { + boolean hasDocValues, boolean coerce, Number nullValue, Map meta) { super(name, isSearchable, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.type = Objects.requireNonNull(type); + this.coerce = coerce; + this.nullValue = nullValue; this.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); // allows number fields in significant text aggs - do we need this? } + NumberFieldType(String name, Builder builder) { + this(name, builder.type, builder.indexed.getValue(), builder.stored.getValue(), builder.hasDocValues.getValue(), + builder.coerce.getValue().value(), builder.nullValue.getValue(), builder.meta.getValue()); + } + public NumberFieldType(String name, NumberType type) { - this(name, type, true, false, true, Collections.emptyMap()); + this(name, type, true, false, true, true, null, Collections.emptyMap()); } @Override @@ -951,6 +959,23 @@ public Object valueForDisplay(Object value) { return type.valueForSearch((Number) value); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + + return new SourceValueFetcher(name(), mapperService, false, nullValue) { + @Override + protected Object parseSourceValue(Object value) { + if (value.equals("")) { + return nullValue; + } + return type.parse(value, coerce); + } + }; + } + @Override public DocValueFormat docValueFormat(String format, ZoneId timeZone) { if (timeZone != null) { @@ -1069,23 +1094,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - - return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValue) { - @Override - protected Object parseSourceValue(Object value) { - if (value.equals("")) { - return nullValue; - } - return fieldType().type.parse(value, coerce.value()); - } - }; - } - @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), type, ignoreMalformedByDefault, coerceByDefault).init(this); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index 4f7407c87b29c..54bc2f2230e6a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -23,6 +23,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.search.lookup.SearchLookup; import java.util.Collections; import java.util.List; @@ -92,6 +93,11 @@ private RoutingFieldType() { public String typeName() { return CONTENT_TYPE; } + + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { + throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); + } } private final boolean required; diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java index 53765f250c04c..4e3a0dbebcd19 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java @@ -21,11 +21,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.document.DocumentField; -import org.elasticsearch.index.mapper.FieldAliasMapper; -import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; @@ -46,7 +43,7 @@ public class FieldFetcher { public static FieldFetcher create(MapperService mapperService, SearchLookup searchLookup, Collection fieldAndFormats) { - MappingLookup fieldMappers = mapperService.documentMapper().mappers(); + List fieldContexts = new ArrayList<>(); for (FieldAndFormat fieldAndFormat : fieldAndFormats) { @@ -55,19 +52,11 @@ public static FieldFetcher create(MapperService mapperService, Collection concreteFields = mapperService.simpleMatchToFullName(fieldPattern); for (String field : concreteFields) { - Mapper mapper = fieldMappers.getMapper(field); - if (mapper == null || mapperService.isMetadataField(field)) { + MappedFieldType ft = mapperService.fieldType(field); + if (ft == null || mapperService.isMetadataField(field)) { continue; } - - if (mapper instanceof FieldAliasMapper) { - String target = ((FieldAliasMapper) mapper).path(); - mapper = fieldMappers.getMapper(target); - assert mapper instanceof FieldMapper; - } - - FieldMapper fieldMapper = (FieldMapper) mapper; - ValueFetcher valueFetcher = fieldMapper.valueFetcher(mapperService, searchLookup, format); + ValueFetcher valueFetcher = ft.valueFetcher(mapperService, searchLookup, format); fieldContexts.add(new FieldContext(field, valueFetcher)); } } From 5b13f4c56c2177ff581933a150a4652e2ad4766d Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 28 Sep 2020 16:15:22 +0100 Subject: [PATCH 2/8] Move valueFetcher() from FieldMapper to MappedFieldType --- .../index/mapper/RankFeatureFieldMapper.java | 28 ++--- .../mapper/RankFeatureMetaFieldMapper.java | 6 ++ .../index/mapper/RankFeaturesFieldMapper.java | 26 ++--- .../index/mapper/ScaledFloatFieldMapper.java | 57 +++++----- .../mapper/SearchAsYouTypeFieldMapper.java | 38 ++++--- .../index/mapper/TokenCountFieldMapper.java | 15 --- .../mapper/RankFeatureFieldMapperTests.java | 12 --- .../mapper/RankFeatureFieldTypeTests.java | 15 +++ .../mapper/ScaledFloatFieldMapperTests.java | 20 ---- .../mapper/ScaledFloatFieldTypeTests.java | 26 ++++- .../join/mapper/MetaJoinFieldMapper.java | 10 +- .../join/mapper/ParentIdFieldMapper.java | 10 +- .../join/mapper/ParentJoinFieldMapper.java | 26 ++--- .../percolator/PercolatorFieldMapper.java | 26 ++--- .../ICUCollationKeywordFieldMapper.java | 54 +++++----- .../index/mapper/CollationFieldTypeTests.java | 7 +- .../ICUCollationKeywordFieldMapperTests.java | 23 ---- .../ICUCollationKeywordFieldTypeTests.java | 51 +++++++++ .../AnnotatedTextFieldMapperTests.java | 22 ---- .../AnnotatedTextFieldMapper.java | 17 --- .../AnnotatedTextFieldTypeTests.java | 23 ++++ .../mapper/murmur3/Murmur3FieldMapper.java | 26 ++--- .../mapper/AbstractGeometryFieldMapper.java | 4 +- .../index/mapper/BinaryFieldMapper.java | 26 ++--- .../index/mapper/BooleanFieldMapper.java | 51 +++++---- .../index/mapper/CompletionFieldMapper.java | 36 +++---- .../index/mapper/DateFieldMapper.java | 4 + .../index/mapper/IdFieldMapper.java | 5 + .../index/mapper/IgnoredFieldMapper.java | 6 ++ .../index/mapper/IndexFieldMapper.java | 5 + .../index/mapper/IpFieldMapper.java | 49 +++++---- .../index/mapper/KeywordFieldMapper.java | 100 ++++++++++-------- .../index/mapper/RangeFieldMapper.java | 80 +++++++------- .../index/mapper/SeqNoFieldMapper.java | 6 ++ .../index/mapper/SourceFieldMapper.java | 6 ++ .../index/mapper/TextFieldMapper.java | 46 ++++---- .../index/mapper/TypeFieldMapper.java | 5 + .../index/mapper/ValueFetcher.java | 4 +- .../index/mapper/VersionFieldMapper.java | 7 ++ .../MetadataIndexTemplateServiceTests.java | 7 ++ .../fielddata/IndexFieldDataServiceTests.java | 6 +- .../index/mapper/BooleanFieldMapperTests.java | 22 ---- .../index/mapper/BooleanFieldTypeTests.java | 17 ++- .../mapper/CompletionFieldMapperTests.java | 19 ---- .../mapper/CompletionFieldTypeTests.java | 46 ++++++++ .../index/mapper/DateFieldMapperTests.java | 66 ------------ .../index/mapper/DateFieldTypeTests.java | 57 ++++++++-- .../mapper/DocumentFieldMapperTests.java | 10 +- .../index/mapper/ExternalMapper.java | 20 ++-- .../index/mapper/FakeStringFieldMapper.java | 20 ++-- .../mapper/GeoPointFieldMapperTests.java | 39 ------- .../index/mapper/GeoPointFieldTypeTests.java | 63 +++++++++++ .../mapper/GeoShapeFieldMapperTests.java | 37 ------- .../index/mapper/GeoShapeFieldTypeTests.java | 64 +++++++++++ .../index/mapper/IpFieldMapperTests.java | 18 ---- .../index/mapper/IpFieldTypeTests.java | 26 ++++- .../index/mapper/IpRangeFieldMapperTests.java | 16 --- .../index/mapper/IpRangeFieldTypeTests.java | 41 +++++++ .../index/mapper/KeywordFieldMapperTests.java | 34 ------ .../index/mapper/KeywordFieldTypeTests.java | 87 ++++++++++++++- .../LegacyGeoShapeFieldMapperTests.java | 39 ------- .../mapper/LegacyGeoShapeFieldTypeTests.java | 40 +++++++ .../index/mapper/NumberFieldMapperTests.java | 18 ---- .../index/mapper/NumberFieldTypeTests.java | 27 ++++- .../index/mapper/ParametrizedMapperTests.java | 6 -- .../index/mapper/RangeFieldMapperTests.java | 37 ------- .../index/mapper/RangeFieldTypeTests.java | 55 ++++++++-- .../index/mapper/TextFieldMapperTests.java | 16 --- .../index/mapper/TextFieldTypeTests.java | 15 +++ .../aggregations/AggregatorBaseTests.java | 5 +- .../DateHistogramAggregatorTestCase.java | 2 +- .../range/DateRangeAggregatorTests.java | 2 +- .../bucket/range/RangeAggregatorTests.java | 4 +- .../search/collapse/CollapseBuilderTests.java | 12 ++- .../search/slice/SliceBuilderTests.java | 8 ++ .../index/mapper/FieldMapperTestCase.java | 20 ---- .../index/mapper/FieldTypeTestCase.java | 23 ++++ .../index/mapper/MapperTestCase.java | 22 +--- .../index/mapper/MockFieldMapper.java | 10 +- .../mapper/HistogramFieldMapper.java | 26 ++--- .../analytics/rate/RateAggregatorTests.java | 1 + .../DataStreamTimestampFieldMapper.java | 8 ++ .../ConstantKeywordFieldMapperTests.java | 24 ----- .../mapper/ConstantKeywordFieldTypeTests.java | 36 +++++++ .../mapper/ConstantKeywordFieldMapper.java | 22 ++-- .../mapper/FlatObjectFieldMapperTests.java | 21 ---- .../mapper/FlatObjectFieldMapper.java | 38 ++++--- .../mapper/FlatObjectFieldTypeTests.java | 37 +++++++ .../mapper/RootFlatObjectFieldTypeTests.java | 12 +-- .../unsignedlong/UnsignedLongFieldMapper.java | 58 +++++----- .../UnsignedLongFieldMapperTests.java | 21 ---- .../UnsignedLongFieldTypeTests.java | 30 +++++- .../VersionStringFieldMapper.java | 28 ++--- .../VersionStringFieldMapperTests.java | 19 ---- .../VersionStringFieldTypeTests.java | 34 ++++++ .../mapper/AbstractScriptMappedFieldType.java | 15 ++- .../mapper/RuntimeFieldMapper.java | 9 -- .../index/mapper/PointFieldMapperTests.java | 39 ------- .../index/mapper/PointFieldTypeTests.java | 54 ++++++++++ .../index/mapper/ShapeFieldMapperTests.java | 50 --------- .../index/mapper/ShapeFieldTypeTests.java | 55 ++++++++++ .../mapper/SparseVectorFieldMapperTests.java | 3 +- .../mapper/DenseVectorFieldMapper.java | 26 ++--- .../mapper/SparseVectorFieldMapper.java | 10 +- .../wildcard/mapper/WildcardFieldMapper.java | 49 +++++---- .../mapper/WildcardFieldMapperTests.java | 45 ++------ .../mapper/WildcardFieldTypeTests.java | 45 ++++++++ 107 files changed, 1606 insertions(+), 1263 deletions(-) create mode 100644 plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldTypeTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldTypeTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldTypeTests.java create mode 100644 x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java create mode 100644 x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldTypeTests.java create mode 100644 x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTypeTests.java create mode 100644 x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldTypeTests.java create mode 100644 x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldTypeTests.java create mode 100644 x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index 24c8c1b3b1615..34c4238a13ba4 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -124,6 +124,19 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S throw new IllegalArgumentException("[rank_feature] fields do not support sorting, scripting or aggregating"); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return new SourceValueFetcher(name(), mapperService, false) { + @Override + protected Float parseSourceValue(Object value) { + return objectToFloat(value); + } + }; + } + @Override public Query termQuery(Object value, QueryShardContext context) { throw new IllegalArgumentException("Queries on [rank_feature] fields are not supported"); @@ -174,7 +187,7 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().addWithKey(name(), new FeatureField("_feature", name(), value)); } - private Float objectToFloat(Object value) { + private static Float objectToFloat(Object value) { if (value instanceof Number) { return ((Number) value).floatValue(); } else { @@ -182,19 +195,6 @@ private Float objectToFloat(Object value) { } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { - @Override - protected Float parseSourceValue(Object value) { - return objectToFloat(value); - } - }; - } - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java index 3a8e81342bd72..1227c498130ac 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java @@ -21,6 +21,7 @@ import org.apache.lucene.search.Query; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.util.Collections; @@ -50,6 +51,11 @@ public String typeName() { return CONTENT_TYPE; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException("Cannot fetch values for internal field [" + typeName() + "]."); + } + @Override public Query existsQuery(QueryShardContext context) { throw new UnsupportedOperationException("Cannot run exists query on [_feature]"); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index 0a40d2a21d01a..333258c376c58 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -97,6 +97,19 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S throw new IllegalArgumentException("[rank_features] fields do not support sorting, scripting or aggregating"); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return new SourceValueFetcher(name(), mapperService, false) { + @Override + protected Object parseSourceValue(Object value) { + return value; + } + }; + } + @Override public Query termQuery(Object value, QueryShardContext context) { throw new IllegalArgumentException("Queries on [rank_features] fields are not supported"); @@ -161,19 +174,6 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new AssertionError("parse is implemented directly"); } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { - @Override - protected Object parseSourceValue(Object value) { - return value; - } - }; - } - @Override protected boolean indexedByDefault() { return false; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index 7f136ed3601f2..5080e5241ddf3 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -122,7 +122,7 @@ protected List> getParameters() { @Override public ScaledFloatFieldMapper build(BuilderContext context) { ScaledFloatFieldType type = new ScaledFloatFieldType(buildFullName(context), indexed.getValue(), stored.getValue(), - hasDocValues.getValue(), meta.getValue(), scalingFactor.getValue()); + hasDocValues.getValue(), meta.getValue(), scalingFactor.getValue(), nullValue.getValue()); return new ScaledFloatFieldMapper(name, type, multiFieldsBuilder.build(this, context), copyTo.build(), this); } } @@ -132,15 +132,17 @@ public ScaledFloatFieldMapper build(BuilderContext context) { public static final class ScaledFloatFieldType extends SimpleMappedFieldType { private final double scalingFactor; + private final Double nullValue; public ScaledFloatFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues, - Map meta, double scalingFactor) { + Map meta, double scalingFactor, Double nullValue) { super(name, indexed, stored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.scalingFactor = scalingFactor; + this.nullValue = nullValue; } public ScaledFloatFieldType(String name, double scalingFactor) { - this(name, true, false, true, Collections.emptyMap(), scalingFactor); + this(name, true, false, true, Collections.emptyMap(), scalingFactor, null); } public double getScalingFactor() { @@ -204,6 +206,30 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S }; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return new SourceValueFetcher(name(), mapperService, false) { + @Override + protected Double parseSourceValue(Object value) { + double doubleValue; + if (value.equals("")) { + if (nullValue == null) { + return null; + } + doubleValue = nullValue; + } else { + doubleValue = objectToDouble(value); + } + + double scalingFactor = getScalingFactor(); + return Math.round(doubleValue * scalingFactor) / scalingFactor; + } + }; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { @@ -380,31 +406,6 @@ private static double objectToDouble(Object value) { return doubleValue; } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { - @Override - protected Double parseSourceValue(Object value) { - double doubleValue; - if (value.equals("")) { - if (nullValue == null) { - return null; - } - doubleValue = nullValue; - } else { - doubleValue = objectToDouble(value); - } - - double scalingFactor = fieldType().getScalingFactor(); - return Math.round(doubleValue * scalingFactor) / scalingFactor; - } - }; - } - - private static class ScaledFloatIndexFieldData extends IndexNumericFieldData { private final IndexNumericFieldData scaledFieldData; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index 3d99777905095..78bc37c52d23b 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -270,6 +270,19 @@ private ShingleFieldType shingleFieldForPositions(int positions) { return shingleFields[Math.min(indexFromShingleSize, shingleFields.length - 1)]; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return new SourceValueFetcher(name(), mapperService, false) { + @Override + protected Object parseSourceValue(Object value) { + return value.toString(); + } + }; + } + @Override public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) { if (prefixField == null || prefixField.termLengthWithinBounds(value.length()) == false) { @@ -376,6 +389,11 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, bool .build(); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); + } + @Override public String typeName() { return "prefix"; @@ -412,11 +430,6 @@ protected void parseCreateField(ParseContext context) { throw new UnsupportedOperationException(); } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException(); - } - @Override protected void mergeOptions(FieldMapper other, List conflicts) { @@ -458,11 +471,6 @@ protected void mergeOptions(FieldMapper other, List conflicts) { } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException(); - } - @Override protected String contentType() { return "shingle"; @@ -485,6 +493,11 @@ void setPrefixFieldType(PrefixFieldType prefixFieldType) { this.prefixFieldType = prefixFieldType; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); + } + @Override public String typeName() { return CONTENT_TYPE; @@ -572,11 +585,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException(); - } - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index 087b828cdb232..fb6a41a0f7608 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -25,7 +25,6 @@ import org.apache.lucene.document.FieldType; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Iterator; @@ -159,20 +158,6 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().addAll(NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, indexed, docValued, stored)); } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - - return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValue) { - @Override - protected String parseSourceValue(Object value) { - return value.toString(); - } - }; - } - /** * Count position increments in a token stream. Package private for testing. * @param analyzer analyzer to create token stream diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java index aa8b4eed03b03..b675c7743be7f 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java @@ -25,10 +25,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.plugins.Plugin; import org.hamcrest.Matchers; @@ -161,13 +158,4 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep assertEquals("[rank_feature] fields do not support indexing multiple values for the same field [foo.field] in the same document", e.getCause().getMessage()); } - - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - RankFeatureFieldMapper mapper = new RankFeatureFieldMapper.Builder("field").build(context); - - assertEquals(List.of(3.14f), fetchSourceValue(mapper, 3.14)); - assertEquals(List.of(42.9f), fetchSourceValue(mapper, "42.9")); - } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldTypeTests.java index d25a32aa92f9a..a65b59682c6aa 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldTypeTests.java @@ -19,7 +19,13 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; + +import java.io.IOException; import java.util.Collections; +import java.util.List; public class RankFeatureFieldTypeTests extends FieldTypeTestCase { @@ -27,4 +33,13 @@ public void testIsNotAggregatable() { MappedFieldType fieldType = new RankFeatureFieldMapper.RankFeatureFieldType("field", Collections.emptyMap(), true); assertFalse(fieldType.isAggregatable()); } + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + MappedFieldType mapper = new RankFeatureFieldMapper.Builder("field").build(context).fieldType(); + + assertEquals(List.of(3.14f), fetchSourceValue(mapper, 3.14)); + assertEquals(List.of(42.9f), fetchSourceValue(mapper, "42.9")); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java index f099a9d39e435..57699c7cc8aa0 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -21,11 +21,8 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; @@ -285,21 +282,4 @@ public void testRejectIndexOptions() { containsString("Failed to parse mapping: unknown parameter [index_options] on mapper [field] of type [scaled_float]")); } - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - ScaledFloatFieldMapper mapper = new ScaledFloatFieldMapper.Builder("field", false, false) - .scalingFactor(100) - .build(context); - assertEquals(List.of(3.14), fetchSourceValue(mapper, 3.1415926)); - assertEquals(List.of(3.14), fetchSourceValue(mapper, "3.1415")); - assertEquals(List.of(), fetchSourceValue(mapper, "")); - - ScaledFloatFieldMapper nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false) - .scalingFactor(100) - .nullValue(2.71) - .build(context); - assertEquals(List.of(2.71), fetchSourceValue(nullValueMapper, "")); - } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java index f324d71d7f576..976c0582c9c7f 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java @@ -29,6 +29,9 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.LeafNumericFieldData; @@ -37,6 +40,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.List; public class ScaledFloatFieldTypeTests extends FieldTypeTestCase { @@ -65,7 +69,7 @@ public void testRangeQuery() throws IOException { // this test checks that searching scaled floats yields the same results as // searching doubles that are rounded to the closest half float ScaledFloatFieldMapper.ScaledFloatFieldType ft = new ScaledFloatFieldMapper.ScaledFloatFieldType( - "scaled_float", true, false, false, Collections.emptyMap(), 0.1 + randomDouble() * 100); + "scaled_float", true, false, false, Collections.emptyMap(), 0.1 + randomDouble() * 100, null); Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null)); final int numDocs = 1000; @@ -175,4 +179,24 @@ public void testFieldData() throws IOException { } IOUtils.close(w, dir); } + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false) + .scalingFactor(100) + .build(context) + .fieldType(); + assertEquals(List.of(3.14), fetchSourceValue(mapper, 3.1415926)); + assertEquals(List.of(3.14), fetchSourceValue(mapper, "3.1415")); + assertEquals(List.of(), fetchSourceValue(mapper, "")); + + MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false) + .scalingFactor(100) + .nullValue(2.71) + .build(context) + .fieldType(); + assertEquals(List.of(2.71), fetchSourceValue(nullValueMapper, "")); + } } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java index 37a7af7e2d0b0..18c3dda29542c 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java @@ -97,6 +97,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException("Cannot fetch values for metadata field [" + typeName() + "]."); + } + @Override public Object valueForDisplay(Object value) { if (value == null) { @@ -139,11 +144,6 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new IllegalStateException("Should never be called"); } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException("Cannot fetch values for metadata field [" + typeName() + "]."); - } - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index 22acb8d270c5f..d7e5f170355fa 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -113,6 +113,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException("Cannot fetch values for internal field [" + typeName() + "]."); + } + @Override public Object valueForDisplay(Object value) { if (value == null) { @@ -182,11 +187,6 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue)); } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException("Cannot fetch values for internal field [" + typeName() + "]."); - } - @Override protected void mergeOptions(FieldMapper other, List conflicts) { ParentIdFieldMapper parentMergeWith = (ParentIdFieldMapper) other; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index d6a5117dd043a..82e8e57f91e2b 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -222,6 +222,19 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return new SourceValueFetcher(name(), mapperService, false) { + @Override + protected Object parseSourceValue(Object value) { + return value; + } + }; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { @@ -343,19 +356,6 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException("parsing is implemented in parse(), this method should NEVER be called"); } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { - @Override - protected Object parseSourceValue(Object value) { - return value; - } - }; - } - @Override public void parse(ParseContext context) throws IOException { context.path().add(simpleName()); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 70c011b7cdb44..ab2304ece7ac7 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -206,6 +206,19 @@ public Query termQuery(Object value, QueryShardContext context) { throw new QueryShardException(context, "Percolator fields are not searchable directly, use a percolate query instead"); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return new SourceValueFetcher(name(), mapperService, false) { + @Override + protected Object parseSourceValue(Object value) { + return value; + } + }; + } + Query percolateQuery(String name, PercolateQuery.QueryStore queryStore, List documents, IndexSearcher searcher, boolean excludeNestedDocuments, Version indexVersion) throws IOException { IndexReader indexReader = searcher.getIndexReader(); @@ -355,19 +368,6 @@ public void parse(ParseContext context) throws IOException { processQuery(query, context); } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { - @Override - protected Object parseSourceValue(Object value) { - return value; - } - }; - } - static void createQueryBuilderField(Version indexVersion, BinaryFieldMapper qbField, QueryBuilder queryBuilder, ParseContext context) throws IOException { try (ByteArrayOutputStream stream = new ByteArrayOutputStream()) { diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index 7166aa62b5e01..d27f4c9034500 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -68,22 +68,29 @@ public static class Defaults { FIELD_TYPE.freeze(); } - public static final String NULL_VALUE = null; public static final int IGNORE_ABOVE = Integer.MAX_VALUE; } public static final class CollationFieldType extends StringFieldType { private final Collator collator; + private final String nullValue; + private final int ignoreAbove; public CollationFieldType(String name, boolean isSearchable, boolean isStored, boolean hasDocValues, - Collator collator, Map meta) { + Collator collator, String nullValue, int ignoreAbove, Map meta) { super(name, isSearchable, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.collator = collator; + this.nullValue = nullValue; + this.ignoreAbove = ignoreAbove; + } + + public CollationFieldType(String name, boolean searchable, Collator collator) { + this(name, searchable, false, true, collator, null, Integer.MAX_VALUE, Collections.emptyMap()); } public CollationFieldType(String name, Collator collator) { - this(name, true, false, true, collator, Collections.emptyMap()); + this(name, true, false, true, collator, null, Integer.MAX_VALUE, Collections.emptyMap()); } @Override @@ -91,8 +98,22 @@ public String typeName() { return CONTENT_TYPE; } - public Collator collator() { - return collator; + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + + return new SourceValueFetcher(name(), mapperService, false, nullValue) { + @Override + protected String parseSourceValue(Object value) { + String keywordValue = value.toString(); + if (keywordValue.length() > ignoreAbove) { + return null; + } + return keywordValue; + } + }; } @Override @@ -429,8 +450,9 @@ public Collator buildCollator() { @Override public ICUCollationKeywordFieldMapper build(BuilderContext context) { final Collator collator = buildCollator(); - CollationFieldType ft = new CollationFieldType(buildFullName(context), indexed, fieldType.stored(), hasDocValues, - collator, meta); + CollationFieldType ft + = new CollationFieldType(buildFullName(context), indexed, fieldType.stored(), + hasDocValues, collator, nullValue, ignoreAbove, meta); return new ICUCollationKeywordFieldMapper(name, fieldType, ft, multiFieldsBuilder.build(this, context), copyTo, rules, language, country, variant, strength, decomposition, alternate, caseLevel, caseFirst, numeric, variableTop, hiraganaQuaternaryMode, ignoreAbove, collator, nullValue); @@ -726,22 +748,4 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - - return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValue) { - @Override - protected String parseSourceValue(Object value) { - String keywordValue = value.toString(); - if (keywordValue.length() > ignoreAbove) { - return null; - } - return keywordValue; - } - }; - } - } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/CollationFieldTypeTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/CollationFieldTypeTests.java index c551fb087ef4b..14fefdf4382d3 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/CollationFieldTypeTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/CollationFieldTypeTests.java @@ -35,7 +35,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; public class CollationFieldTypeTests extends FieldTypeTestCase{ @@ -66,7 +65,7 @@ public void testTermQuery() { assertEquals(new TermQuery(new Term("field", expected)), ft.termQuery("I WİLL USE TURKİSH CASING", null)); - MappedFieldType unsearchable = new CollationFieldType("field", false, false, true, collator, Collections.emptyMap()); + MappedFieldType unsearchable = new CollationFieldType("field", false, collator); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("bar", null)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); @@ -86,7 +85,7 @@ public void testTermsQuery() { assertEquals(new TermInSetQuery("field", terms), ft.termsQuery(Arrays.asList("foo", "bar"), null)); - MappedFieldType unsearchable = new CollationFieldType("field", false, false, true, collator, Collections.emptyMap()); + MappedFieldType unsearchable = new CollationFieldType("field", false, collator); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termsQuery(Arrays.asList("foo", "bar"), null)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); @@ -135,7 +134,7 @@ public void testRangeQuery() { assertEquals("[range] queries on [text] or [keyword] fields cannot be executed when " + "'search.allow_expensive_queries' is set to false.", ee.getMessage()); - MappedFieldType unsearchable = new CollationFieldType("field", false, false, true, DEFAULT_COLLATOR, Collections.emptyMap()); + MappedFieldType unsearchable = new CollationFieldType("field", false, DEFAULT_COLLATOR); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.rangeQuery("a", "b", false, false, null, null, null, MOCK_QSC)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index 352d2574d7726..3fdd5239324a5 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -26,11 +26,8 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; @@ -305,24 +302,4 @@ public void testUpdateIgnoreAbove() throws IOException { assertEquals(0, fields.length); } - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context); - assertEquals(List.of("42"), fetchSourceValue(mapper, 42L)); - assertEquals(List.of("true"), fetchSourceValue(mapper, true)); - - ICUCollationKeywordFieldMapper ignoreAboveMapper = new ICUCollationKeywordFieldMapper.Builder("field") - .ignoreAbove(4) - .build(context); - assertEquals(List.of(), fetchSourceValue(ignoreAboveMapper, "value")); - assertEquals(List.of("42"), fetchSourceValue(ignoreAboveMapper, 42L)); - assertEquals(List.of("true"), fetchSourceValue(ignoreAboveMapper, true)); - - ICUCollationKeywordFieldMapper nullValueMapper = new ICUCollationKeywordFieldMapper.Builder("field") - .nullValue("NULL") - .build(context); - assertEquals(List.of("NULL"), fetchSourceValue(nullValueMapper, null)); - } } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldTypeTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldTypeTests.java new file mode 100644 index 0000000000000..dde0ce0052570 --- /dev/null +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldTypeTests.java @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; + +import java.io.IOException; +import java.util.List; + +public class ICUCollationKeywordFieldTypeTests extends FieldTypeTestCase { + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context); + assertEquals(List.of("42"), fetchSourceValue(mapper.fieldType(), 42L)); + assertEquals(List.of("true"), fetchSourceValue(mapper.fieldType(), true)); + + ICUCollationKeywordFieldMapper ignoreAboveMapper = new ICUCollationKeywordFieldMapper.Builder("field") + .ignoreAbove(4) + .build(context); + assertEquals(List.of(), fetchSourceValue(ignoreAboveMapper.fieldType(), "value")); + assertEquals(List.of("42"), fetchSourceValue(ignoreAboveMapper.fieldType(), 42L)); + assertEquals(List.of("true"), fetchSourceValue(ignoreAboveMapper.fieldType(), true)); + + ICUCollationKeywordFieldMapper nullValueMapper = new ICUCollationKeywordFieldMapper.Builder("field") + .nullValue("NULL") + .build(context); + assertEquals(List.of("NULL"), fetchSourceValue(nullValueMapper.fieldType(), null)); + } +} diff --git a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index ed84512c27c6a..e52b6fe47e227 100644 --- a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -28,12 +28,10 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; @@ -46,11 +44,8 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; -import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.ParsedDocument; @@ -76,7 +71,6 @@ import java.util.Map; import java.util.Set; -import static org.elasticsearch.index.mapper.FieldMapperTestCase.fetchSourceValue; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -677,20 +671,4 @@ public void testEmptyName() throws IOException { ); assertThat(e.getMessage(), containsString("name cannot be empty string")); } - - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - FieldMapper fieldMapper = new AnnotatedTextFieldMapper.Builder("field") - .indexAnalyzer(indexService.getIndexAnalyzers().getDefaultIndexAnalyzer()) - .searchAnalyzer(indexService.getIndexAnalyzers().getDefaultSearchAnalyzer()) - .searchQuoteAnalyzer(indexService.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer()) - .build(context); - AnnotatedTextFieldMapper mapper = (AnnotatedTextFieldMapper) fieldMapper; - - assertEquals(List.of("value"), fetchSourceValue(mapper, "value")); - assertEquals(List.of("42"), fetchSourceValue(mapper, 42L)); - assertEquals(List.of("true"), fetchSourceValue(mapper, true)); - } } diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 777bf7d248619..736db61abb292 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -39,14 +39,10 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextFieldMapper; -import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText.AnnotationToken; import org.elasticsearch.index.similarity.SimilarityProvider; -import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.io.Reader; @@ -586,19 +582,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { - @Override - protected Object parseSourceValue(Object value) { - return value.toString(); - } - }; - } - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java index 3671c21678b96..214a24e60d8be 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java @@ -22,13 +22,20 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.queries.intervals.Intervals; import org.apache.lucene.queries.intervals.IntervalsSource; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; import java.io.IOException; import java.util.Collections; +import java.util.List; public class AnnotatedTextFieldTypeTests extends FieldTypeTestCase { @@ -38,4 +45,20 @@ public void testIntervals() throws IOException { IntervalsSource source = ft.intervals("Donald Trump", 0, true, a, false); assertEquals(Intervals.phrase(Intervals.term("donald"), Intervals.term("trump")), source); } + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + MappedFieldType fieldType = new AnnotatedTextFieldMapper.Builder("field") + .indexAnalyzer(Lucene.STANDARD_ANALYZER) + .searchAnalyzer(Lucene.STANDARD_ANALYZER) + .searchQuoteAnalyzer(Lucene.STANDARD_ANALYZER) + .build(context) + .fieldType(); + + assertEquals(List.of("value"), fetchSourceValue(fieldType, "value")); + assertEquals(List.of("42"), fetchSourceValue(fieldType, 42L)); + assertEquals(List.of("true"), fetchSourceValue(fieldType, true)); + } } diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 4f4991552f9bc..99f35b6761398 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -111,6 +111,19 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return new SourceValueFetcher(name(), mapperService, false) { + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + }; + } + @Override public Query termQuery(Object value, QueryShardContext context) { throw new QueryShardException(context, "Murmur3 fields are not searchable: [" + name() + "]"); @@ -146,19 +159,6 @@ protected void parseCreateField(ParseContext context) } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { - @Override - protected String parseSourceValue(Object value) { - return value.toString(); - } - }; - } - @Override protected boolean indexedByDefault() { return false; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index c90242f0ea368..3fa3f11922478 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -276,9 +276,9 @@ public Query termQuery(Object value, QueryShardContext context) { public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { String geoFormat = format != null ? format : GeoJsonGeometryFormat.NAME; - Function valueParser = value -> geometryParser.parseAndFormatObject(value, this, geoFormat); + Function valueParser = value -> geometryParser.parseAndFormatObject(value, null, geoFormat); - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { + return new SourceValueFetcher(name(), mapperService, true) { @Override protected Object parseSourceValue(Object value) { return valueParser.apply(value); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 8c203a6555fa3..9e833339976fe 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -97,6 +97,19 @@ public String typeName() { return CONTENT_TYPE; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return new SourceValueFetcher(name(), mapperService, false) { + @Override + protected Object parseSourceValue(Object value) { + return value; + } + }; + } + @Override public DocValueFormat docValueFormat(String format, ZoneId timeZone) { return DocValueFormat.BINARY; @@ -179,19 +192,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { - @Override - protected Object parseSourceValue(Object value) { - return value; - } - }; - } - @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new BinaryFieldMapper.Builder(simpleName()).init(this); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 7dced24507ec8..a12fc06c7cc34 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -96,7 +96,7 @@ protected List> getParameters() { @Override public BooleanFieldMapper build(BuilderContext context) { MappedFieldType ft = new BooleanFieldType(buildFullName(context), indexed.getValue(), stored.getValue(), - docValues.getValue(), meta.getValue()); + docValues.getValue(), nullValue.getValue(), meta.getValue()); return new BooleanFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this); } } @@ -105,13 +105,20 @@ public BooleanFieldMapper build(BuilderContext context) { public static final class BooleanFieldType extends TermBasedFieldType { + private final Boolean nullValue; + public BooleanFieldType(String name, boolean isSearchable, boolean isStored, boolean hasDocValues, - Map meta) { + Boolean nullValue, Map meta) { super(name, isSearchable, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + this.nullValue = nullValue; } public BooleanFieldType(String name) { - this(name, true, false, true, Collections.emptyMap()); + this(name, true, false, true, false, Collections.emptyMap()); + } + + public BooleanFieldType(String name, boolean searchable) { + this(name, searchable, false, true, false, Collections.emptyMap()); } @Override @@ -119,6 +126,25 @@ public String typeName() { return CONTENT_TYPE; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + + return new SourceValueFetcher(name(), mapperService, false, nullValue) { + @Override + protected Boolean parseSourceValue(Object value) { + if (value instanceof Boolean) { + return (Boolean) value; + } else { + String textValue = value.toString(); + return Booleans.parseBoolean(textValue.toCharArray(), 0, textValue.length(), false); + } + } + }; + } + @Override public BytesRef indexedValueForSearch(Object value) { if (value == null) { @@ -240,25 +266,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - - return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValue) { - @Override - protected Boolean parseSourceValue(Object value) { - if (value instanceof Boolean) { - return (Boolean) value; - } else { - String textValue = value.toString(); - return Booleans.parseBoolean(textValue.toCharArray(), 0, textValue.length(), false); - } - } - }; - } - @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index dc5921f23805e..e7227a848de76 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -321,6 +321,24 @@ public String typeName() { return CONTENT_TYPE; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + + return new SourceValueFetcher(name(), mapperService, true) { + @Override + protected List parseSourceValue(Object value) { + if (value instanceof List) { + return (List) value; + } else { + return List.of(value); + } + } + }; + } + } private final int maxInputLength; @@ -535,24 +553,6 @@ private void parse(ParseContext parseContext, Token token, } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { - @Override - protected List parseSourceValue(Object value) { - if (value instanceof List) { - return (List) value; - } else { - return List.of(value); - } - } - }; - } - static class CompletionInputMetadata { public final String input; public final Map> contexts; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index e3af2834ffd43..71d6458de655a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -292,6 +292,10 @@ public DateFieldType(String name, Resolution resolution) { this(name, true, false, true, DEFAULT_DATE_TIME_FORMATTER, resolution, null, Collections.emptyMap()); } + public DateFieldType(String name, Resolution resolution, DateFormatter dateFormatter) { + this(name, true, false, true, dateFormatter, resolution, null, Collections.emptyMap()); + } + @Override public String typeName() { return resolution.type(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index 8471567c1ce24..770d7e0ea85e8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -117,6 +117,11 @@ public boolean isSearchable() { return true; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { + throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); + } + @Override public Query termQuery(Object value, QueryShardContext context) { return termsQuery(Arrays.asList(value), context); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java index 659d808458b6a..612944e6047ae 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java @@ -25,6 +25,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermRangeQuery; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.util.Collections; @@ -74,6 +75,11 @@ public Query existsQuery(QueryShardContext context) { // field is bounded by the number of fields in the mappings. return new TermRangeQuery(name(), null, null, true, true); } + + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { + throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); + } } private IgnoredFieldMapper() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index f22cefc99e3f2..3c3bf52b0047c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -71,6 +71,11 @@ public Query existsQuery(QueryShardContext context) { public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { return new ConstantIndexFieldData.Builder(mapperService -> fullyQualifiedIndexName, name(), CoreValuesSourceType.BYTES); } + + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + return lookup -> Collections.singletonList(mapperService.index().getName()); + } } public IndexFieldMapper() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 30b3d8804bcb9..30277a7ea0a3a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -115,7 +115,8 @@ protected List> getParameters() { @Override public IpFieldMapper build(BuilderContext context) { return new IpFieldMapper(name, - new IpFieldType(buildFullName(context), indexed.getValue(), stored.getValue(), hasDocValues.getValue(), meta.getValue()), + new IpFieldType(buildFullName(context), indexed.getValue(), stored.getValue(), + hasDocValues.getValue(), parseNullValue(), meta.getValue()), multiFieldsBuilder.build(this, context), copyTo.build(), this); } @@ -128,12 +129,16 @@ public IpFieldMapper build(BuilderContext context) { public static final class IpFieldType extends SimpleMappedFieldType { - public IpFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues, Map meta) { + private final InetAddress nullValue; + + public IpFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues, + InetAddress nullValue, Map meta) { super(name, indexed, stored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + this.nullValue = nullValue; } public IpFieldType(String name) { - this(name, true, false, true, Collections.emptyMap()); + this(name, true, false, true, null, Collections.emptyMap()); } @Override @@ -152,6 +157,25 @@ private static InetAddress parse(Object value) { } } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return new SourceValueFetcher(name(), mapperService, false, nullValue) { + @Override + protected Object parseSourceValue(Object value) { + InetAddress address; + if (value instanceof InetAddress) { + address = (InetAddress) value; + } else { + address = InetAddresses.forString(value.toString()); + } + return InetAddresses.toAddrString(address); + } + }; + } + @Override public Query termQuery(Object value, @Nullable QueryShardContext context) { failIfNotIndexed(); @@ -415,25 +439,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValue) { - @Override - protected Object parseSourceValue(Object value) { - InetAddress address; - if (value instanceof InetAddress) { - address = (InetAddress) value; - } else { - address = InetAddresses.forString(value.toString()); - } - return InetAddresses.toAddrString(address); - } - }; - } - @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), ignoreMalformedByDefault, indexCreatedVersion).init(this); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 6bab523d8522e..33234ac68fa8d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -175,9 +175,7 @@ private KeywordFieldType buildFieldType(BuilderContext context, FieldType fieldT else if (splitQueriesOnWhitespace.getValue()) { searchAnalyzer = Lucene.WHITESPACE_ANALYZER; } - return new KeywordFieldType(buildFullName(context), hasDocValues.getValue(), fieldType, - eagerGlobalOrdinals.getValue(), normalizer, searchAnalyzer, - similarity.getValue(), meta.getValue()); + return new KeywordFieldType(buildFullName(context), fieldType, normalizer, searchAnalyzer, this); } @Override @@ -196,27 +194,45 @@ public KeywordFieldMapper build(BuilderContext context) { public static final class KeywordFieldType extends StringFieldType { - public KeywordFieldType(String name, boolean hasDocValues, FieldType fieldType, - boolean eagerGlobalOrdinals, NamedAnalyzer normalizer, NamedAnalyzer searchAnalyzer, - SimilarityProvider similarity, Map meta) { - super(name, fieldType.indexOptions() != IndexOptions.NONE, fieldType.stored(), - hasDocValues, new TextSearchInfo(fieldType, similarity, searchAnalyzer, searchAnalyzer), meta); - setEagerGlobalOrdinals(eagerGlobalOrdinals); + private final int ignoreAbove; + private final String nullValue; + + public KeywordFieldType(String name, FieldType fieldType, + NamedAnalyzer normalizer, NamedAnalyzer searchAnalyzer, Builder builder) { + super(name, + fieldType.indexOptions() != IndexOptions.NONE, + fieldType.stored(), + builder.hasDocValues.getValue(), + new TextSearchInfo(fieldType, builder.similarity.getValue(), searchAnalyzer, searchAnalyzer), + builder.meta.getValue()); + setEagerGlobalOrdinals(builder.eagerGlobalOrdinals.getValue()); setIndexAnalyzer(normalizer); + this.ignoreAbove = builder.ignoreAbove.getValue(); + this.nullValue = builder.nullValue.getValue(); } public KeywordFieldType(String name, boolean isSearchable, boolean hasDocValues, Map meta) { super(name, isSearchable, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); + this.ignoreAbove = Integer.MAX_VALUE; + this.nullValue = null; } public KeywordFieldType(String name) { - this(name, true, Defaults.FIELD_TYPE, false, - Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, null, Collections.emptyMap()); + this(name, true, true, Collections.emptyMap()); + } + + public KeywordFieldType(String name, FieldType fieldType) { + super(name, fieldType.indexOptions() != IndexOptions.NONE, + false, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + this.ignoreAbove = Integer.MAX_VALUE; + this.nullValue = null; } public KeywordFieldType(String name, NamedAnalyzer analyzer) { super(name, true, false, true, new TextSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); + this.ignoreAbove = Integer.MAX_VALUE; + this.nullValue = null; } @Override @@ -234,6 +250,34 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + + return new SourceValueFetcher(name(), mapperService, false, nullValue) { + @Override + protected String parseSourceValue(Object value) { + String keywordValue = value.toString(); + if (keywordValue.length() > ignoreAbove) { + return null; + } + + NamedAnalyzer normalizer = normalizer(); + if (normalizer == null) { + return keywordValue; + } + + try { + return normalizeValue(normalizer, name(), keywordValue); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + }; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { @@ -332,7 +376,7 @@ protected void parseCreateField(ParseContext context) throws IOException { NamedAnalyzer normalizer = fieldType().normalizer(); if (normalizer != null) { - value = normalizeValue(normalizer, value); + value = normalizeValue(normalizer, name(), value); } // convert to utf8 only once before feeding postings/dv/stored fields @@ -351,8 +395,8 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - private String normalizeValue(NamedAnalyzer normalizer, String value) throws IOException { - try (TokenStream ts = normalizer.tokenStream(name(), value)) { + private static String normalizeValue(NamedAnalyzer normalizer, String field, String value) throws IOException { + try (TokenStream ts = normalizer.tokenStream(field, value)) { final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); ts.reset(); if (ts.incrementToken() == false) { @@ -371,34 +415,6 @@ private String normalizeValue(NamedAnalyzer normalizer, String value) throws IOE } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - - return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValue) { - @Override - protected String parseSourceValue(Object value) { - String keywordValue = value.toString(); - if (keywordValue.length() > ignoreAbove) { - return null; - } - - NamedAnalyzer normalizer = fieldType().normalizer(); - if (normalizer == null) { - return keywordValue; - } - - try { - return normalizeValue(normalizer, keywordValue); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - }; - } - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 5a9160d5db46e..521357085cf13 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -127,14 +127,15 @@ protected RangeFieldType setupFieldType(BuilderContext context) { + " should not define a dateTimeFormatter unless it is a " + RangeType.DATE + " type"); } return new RangeFieldType(buildFullName(context), index.getValue(), store.getValue(), hasDocValues.getValue(), - DateFormatter.forPattern(format.getValue()).withLocale(locale.getValue()), meta.getValue()); + DateFormatter.forPattern(format.getValue()).withLocale(locale.getValue()), + coerce.getValue().value(), meta.getValue()); } if (type == RangeType.DATE) { return new RangeFieldType(buildFullName(context), index.getValue(), store.getValue(), hasDocValues.getValue(), - Defaults.DATE_FORMATTER, meta.getValue()); + Defaults.DATE_FORMATTER, coerce.getValue().value(), meta.getValue()); } return new RangeFieldType(buildFullName(context), type, index.getValue(), store.getValue(), hasDocValues.getValue(), - meta.getValue()); + coerce.getValue().value(), meta.getValue()); } @Override @@ -148,32 +149,35 @@ public static final class RangeFieldType extends MappedFieldType { protected final RangeType rangeType; protected final DateFormatter dateTimeFormatter; protected final DateMathParser dateMathParser; + protected final boolean coerce; public RangeFieldType(String name, RangeType type, boolean indexed, boolean stored, - boolean hasDocValues, Map meta) { + boolean hasDocValues, boolean coerce, Map meta) { super(name, indexed, stored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); assert type != RangeType.DATE; this.rangeType = Objects.requireNonNull(type); dateTimeFormatter = null; dateMathParser = null; setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); + this.coerce = coerce; } public RangeFieldType(String name, RangeType type) { - this(name, type, true, false, true, Collections.emptyMap()); + this(name, type, true, false, true, false, Collections.emptyMap()); } public RangeFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues, DateFormatter formatter, - Map meta) { + boolean coerce, Map meta) { super(name, indexed, stored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.rangeType = RangeType.DATE; this.dateTimeFormatter = Objects.requireNonNull(formatter); this.dateMathParser = dateTimeFormatter.toDateMathParser(); setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); + this.coerce = coerce; } public RangeFieldType(String name, DateFormatter formatter) { - this(name, true, false, true, formatter, Collections.emptyMap()); + this(name, true, false, true, formatter, false, Collections.emptyMap()); } public RangeType rangeType() { return rangeType; } @@ -184,6 +188,37 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S return new BinaryIndexFieldData.Builder(name(), CoreValuesSourceType.RANGE); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + DateFormatter defaultFormatter = dateTimeFormatter(); + DateFormatter formatter = format != null + ? DateFormatter.forPattern(format).withLocale(defaultFormatter.locale()) + : defaultFormatter; + + return new SourceValueFetcher(name(), mapperService, false) { + + @Override + @SuppressWarnings("unchecked") + protected Object parseSourceValue(Object value) { + RangeType rangeType = rangeType(); + if (!(value instanceof Map)) { + assert rangeType == RangeType.IP; + Tuple ipRange = InetAddresses.parseCidr(value.toString()); + return InetAddresses.toCidrString(ipRange.v1(), ipRange.v2()); + } + + Map range = (Map) value; + Map parsedRange = new HashMap<>(); + for (Map.Entry entry : range.entrySet()) { + Object parsedValue = rangeType.parseValue(entry.getValue(), coerce, dateMathParser); + Object formattedValue = rangeType.formatValue(parsedValue, formatter); + parsedRange.put(entry.getKey(), formattedValue); + } + return parsedRange; + } + }; + } + @Override public String typeName() { return rangeType.name; @@ -347,37 +382,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - DateFormatter defaultFormatter = fieldType().dateTimeFormatter(); - DateFormatter formatter = format != null - ? DateFormatter.forPattern(format).withLocale(defaultFormatter.locale()) - : defaultFormatter; - - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { - - @Override - @SuppressWarnings("unchecked") - protected Object parseSourceValue(Object value) { - RangeType rangeType = fieldType().rangeType(); - if (!(value instanceof Map)) { - assert rangeType == RangeType.IP; - Tuple ipRange = InetAddresses.parseCidr(value.toString()); - return InetAddresses.toCidrString(ipRange.v1(), ipRange.v2()); - } - - Map range = (Map) value; - Map parsedRange = new HashMap<>(); - for (Map.Entry entry : range.entrySet()) { - Object parsedValue = rangeType.parseValue(entry.getValue(), coerce.value(), fieldType().dateMathParser); - Object formattedValue = rangeType.formatValue(parsedValue, formatter); - parsedRange.put(entry.getKey(), formattedValue); - } - return parsedRange; - } - }; - } - private static Range parseIpRangeFromCidr(final XContentParser parser) throws IOException { final Tuple cidr = InetAddresses.parseCidr(parser.text()); // create the lower value by zeroing out the host portion, upper value by filling it with all ones. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index 12c100ba3f3fa..6c43cb2a0d051 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.seqno.SequenceNumbers; +import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; @@ -121,6 +122,11 @@ private long parse(Object value) { return Long.parseLong(value.toString()); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { + return new DocValueFetcher(DocValueFormat.RAW, lookup.doc().getForField(this)); + } + @Override public Query termQuery(Object value, @Nullable QueryShardContext context) { long v = parse(value); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 6748dbdcf08a6..a68a6c8ce1635 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Arrays; @@ -110,6 +111,11 @@ public String typeName() { return CONTENT_TYPE; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { + throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); + } + @Override public Query existsQuery(QueryShardContext context) { throw new QueryShardException(context, "The _source field is not searchable"); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 7b1b789d7d2b6..ded2ec5dcf850 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -394,6 +394,11 @@ public String typeName() { return "phrase"; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + return parent.valueFetcher(mapperService, searchLookup, format); + } + @Override public Query existsQuery(QueryShardContext context) { throw new UnsupportedOperationException(); @@ -415,6 +420,11 @@ static final class PrefixFieldType extends StringFieldType { this.hasPositions = hasPositions; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + return parentField.valueFetcher(mapperService, searchLookup, format); + } + static boolean canMerge(PrefixFieldType first, PrefixFieldType second) { if (first == null) { return second == null; @@ -506,11 +516,6 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException(); } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException(); - } - @Override protected void mergeOptions(FieldMapper other, List conflicts) { @@ -537,11 +542,6 @@ protected void parseCreateField(ParseContext context) { throw new UnsupportedOperationException(); } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException(); - } - @Override protected void mergeOptions(FieldMapper other, List conflicts) { @@ -636,6 +636,19 @@ public String typeName() { return CONTENT_TYPE; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return new SourceValueFetcher(name(), mapperService, false) { + @Override + protected Object parseSourceValue(Object value) { + return value.toString(); + } + }; + } + @Override public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) { if (prefixFieldType == null || prefixFieldType.accept(value.length()) == false) { @@ -832,19 +845,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { - @Override - protected Object parseSourceValue(Object value) { - return value.toString(); - } - }; - } - @Override public Iterator iterator() { List subIterators = new ArrayList<>(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java index 6b8e2fe7607b7..691675692c1e0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java @@ -76,6 +76,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S return new ConstantIndexFieldData.Builder(typeFunction, name(), CoreValuesSourceType.BYTES); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + return lookup -> Collections.singletonList(mapperService.documentMapper().type()); + } + @Override protected boolean matches(String pattern, boolean caseInsensitive, QueryShardContext context) { if (caseInsensitive) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java index 7f8a5b6149613..980fc6b7a6d65 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java @@ -27,8 +27,8 @@ import java.util.List; /** - * A helper class for fetching field values during the {@link FetchFieldsPhase}. Each {@link FieldMapper} - * is in charge of defining a value fetcher through {@link FieldMapper#valueFetcher}. + * A helper class for fetching field values during the {@link FetchFieldsPhase}. Each {@link MappedFieldType} + * is in charge of defining a value fetcher through {@link MappedFieldType#valueFetcher}. */ public interface ValueFetcher { /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java index 3bb78b07c776e..ab0d1ea60b35d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java @@ -25,6 +25,8 @@ import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SearchLookup; import java.util.Collections; @@ -53,6 +55,11 @@ public String typeName() { public Query termQuery(Object value, QueryShardContext context) { throw new QueryShardException(context, "The _version field is not searchable"); } + + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { + return new DocValueFetcher(DocValueFormat.RAW, lookup.doc().getForField(this)); + } } private VersionFieldMapper() { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index ace8a9733df79..abff1f557c9c6 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.indices.IndexTemplateMissingException; import org.elasticsearch.indices.IndicesService; @@ -55,6 +56,7 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESSingleNodeTestCase; import java.io.IOException; @@ -1530,6 +1532,11 @@ public static class MetadataTimestampFieldMapper extends MetadataFieldMapper { public MetadataTimestampFieldMapper(boolean enabled) { super(new MappedFieldType("_data_stream_timestamp", false, false, false, TextSearchInfo.NONE, Map.of()) { + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); + } + @Override public String typeName() { return "_data_stream_timestamp"; diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 8394e71a5318b..099be8fb10fcd 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -312,17 +312,17 @@ private void doTestRequireDocValues(MappedFieldType ft) { public void testRequireDocValuesOnLongs() { doTestRequireDocValues(new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.LONG)); doTestRequireDocValues(new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.LONG, - true, false, false, Collections.emptyMap())); + true, false, false, false, null, Collections.emptyMap())); } public void testRequireDocValuesOnDoubles() { doTestRequireDocValues(new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.DOUBLE)); doTestRequireDocValues(new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.DOUBLE, - true, false, false, Collections.emptyMap())); + true, false, false, false, null, Collections.emptyMap())); } public void testRequireDocValuesOnBools() { doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field")); - doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field", true, false, false, Collections.emptyMap())); + doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field", true, false, false, null, Collections.emptyMap())); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index af01b6b512d12..2d2fa27b76d1b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -24,19 +24,13 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.ParseContext.Document; import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; public class BooleanFieldMapperTests extends MapperTestCase { @@ -179,20 +173,4 @@ public void testDocValues() throws Exception { assertEquals(DocValuesType.NONE, fields[0].fieldType().docValuesType()); assertEquals(DocValuesType.SORTED_NUMERIC, fields[1].fieldType().docValuesType()); } - - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - BooleanFieldMapper mapper = new BooleanFieldMapper.Builder("field").build(context); - assertEquals(List.of(true), fetchSourceValue(mapper, true)); - assertEquals(List.of(false), fetchSourceValue(mapper, "false")); - assertEquals(List.of(false), fetchSourceValue(mapper, "")); - - Map mapping = Map.of("type", "boolean", "null_value", true); - BooleanFieldMapper.Builder builder = new BooleanFieldMapper.Builder("field"); - builder.parse("field", null, new HashMap<>(mapping)); - BooleanFieldMapper nullValueMapper = builder.build(context); - assertEquals(List.of(true), fetchSourceValue(nullValueMapper, null)); - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java index b86dd346228a6..c4a92fe0d3896 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java @@ -21,7 +21,9 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.TermQuery; +import java.io.IOException; import java.util.Collections; +import java.util.List; public class BooleanFieldTypeTests extends FieldTypeTestCase { @@ -45,9 +47,22 @@ public void testTermQuery() { assertEquals(new TermQuery(new Term("field", "T")), ft.termQuery("true", null)); assertEquals(new TermQuery(new Term("field", "F")), ft.termQuery("false", null)); - MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false, false, true, Collections.emptyMap()); + MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("true", null)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); } + + public void testFetchSourceValue() throws IOException { + + MappedFieldType fieldType = new BooleanFieldMapper.BooleanFieldType("field"); + assertEquals(List.of(true), fetchSourceValue(fieldType, true)); + assertEquals(List.of(false), fetchSourceValue(fieldType, "false")); + assertEquals(List.of(false), fetchSourceValue(fieldType, "")); + + MappedFieldType nullFieldType = new BooleanFieldMapper.BooleanFieldType( + "field", true, false, true, false, Collections.emptyMap() + ); + assertEquals(List.of(true), fetchSourceValue(nullFieldType, null)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index c25245600fec5..945e6ab0f03e9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -33,11 +33,8 @@ import org.apache.lucene.util.CharsRefBuilder; import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -53,7 +50,6 @@ import org.hamcrest.core.CombinableMatcher; import java.io.IOException; -import java.util.List; import java.util.Map; import java.util.function.Function; @@ -765,21 +761,6 @@ public void testLimitOfContextMappings() throws Throwable { CompletionFieldMapper.COMPLETION_CONTEXTS_LIMIT + "] has been exceeded")); } - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - NamedAnalyzer defaultAnalyzer = new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer()); - CompletionFieldMapper mapper = new CompletionFieldMapper.Builder("completion", defaultAnalyzer, Version.CURRENT).build(context); - - assertEquals(List.of("value"), fetchSourceValue(mapper, "value")); - - List list = List.of("first", "second"); - assertEquals(list, fetchSourceValue(mapper, list)); - - Map object = Map.of("input", List.of("first", "second"), "weight", "2.718"); - assertEquals(List.of(object), fetchSourceValue(mapper, object)); - } - private Matcher suggestField(String value) { return Matchers.allOf(hasProperty(IndexableField::stringValue, equalTo(value)), Matchers.instanceOf(SuggestField.class)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldTypeTests.java new file mode 100644 index 0000000000000..9d1dc50795429 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldTypeTests.java @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.elasticsearch.index.analysis.AnalyzerScope; +import org.elasticsearch.index.analysis.NamedAnalyzer; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class CompletionFieldTypeTests extends FieldTypeTestCase { + + public void testFetchSourceValue() throws IOException { + NamedAnalyzer defaultAnalyzer = new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer()); + + MappedFieldType fieldType = new CompletionFieldMapper.CompletionFieldType("name", defaultAnalyzer, Collections.emptyMap()); + + assertEquals(List.of("value"), fetchSourceValue(fieldType, "value")); + + List list = List.of("first", "second"); + assertEquals(list, fetchSourceValue(fieldType, list)); + + Map object = Map.of("input", List.of("first", "second"), "weight", "2.718"); + assertEquals(List.of(object), fetchSourceValue(fieldType, object)); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index 8672c6ac09b33..75b6d038f8708 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -23,11 +23,8 @@ import org.apache.lucene.index.IndexableField; import org.elasticsearch.Version; import org.elasticsearch.bootstrap.JavaVersion; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.mapper.DateFieldMapper.Resolution; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.search.DocValueFormat; @@ -35,9 +32,7 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -315,45 +310,7 @@ public void testIllegalFormatField() { assertThat(e.getMessage(), containsString("Error parsing [format] on field [field]: Invalid")); } - public void testFetchSourceValue() throws IOException { - DateFieldMapper mapper = createMapper(Resolution.MILLISECONDS, null); - String date = "2020-05-15T21:33:02.000Z"; - assertEquals(List.of(date), fetchSourceValue(mapper, date)); - assertEquals(List.of(date), fetchSourceValue(mapper, 1589578382000L)); - - DateFieldMapper mapperWithFormat = createMapper(Resolution.MILLISECONDS, "yyyy/MM/dd||epoch_millis"); - String dateInFormat = "1990/12/29"; - assertEquals(List.of(dateInFormat), fetchSourceValue(mapperWithFormat, dateInFormat)); - assertEquals(List.of(dateInFormat), fetchSourceValue(mapperWithFormat, 662428800000L)); - - DateFieldMapper mapperWithMillis = createMapper(Resolution.MILLISECONDS, "epoch_millis"); - String dateInMillis = "662428800000"; - assertEquals(List.of(dateInMillis), fetchSourceValue(mapperWithMillis, dateInMillis)); - assertEquals(List.of(dateInMillis), fetchSourceValue(mapperWithMillis, 662428800000L)); - - String nullValueDate = "2020-05-15T21:33:02.000Z"; - DateFieldMapper nullValueMapper = createMapper(Resolution.MILLISECONDS, null, nullValueDate); - assertEquals(List.of(nullValueDate), fetchSourceValue(nullValueMapper, null)); - } - - public void testParseSourceValueWithFormat() throws IOException { - DateFieldMapper mapper = createMapper(Resolution.NANOSECONDS, "strict_date_time", "1970-12-29T00:00:00.000Z"); - String date = "1990-12-29T00:00:00.000Z"; - assertEquals(List.of("1990/12/29"), fetchSourceValue(mapper, date, "yyyy/MM/dd")); - assertEquals(List.of("662428800000"), fetchSourceValue(mapper, date, "epoch_millis")); - assertEquals(List.of("1970/12/29"), fetchSourceValue(mapper, null, "yyyy/MM/dd")); - } - - public void testParseSourceValueNanos() throws IOException { - DateFieldMapper mapper = createMapper(Resolution.NANOSECONDS, "strict_date_time||epoch_millis"); - String date = "2020-05-15T21:33:02.123456789Z"; - assertEquals(List.of("2020-05-15T21:33:02.123456789Z"), fetchSourceValue(mapper, date)); - assertEquals(List.of("2020-05-15T21:33:02.123Z"), fetchSourceValue(mapper, 1589578382123L)); - String nullValueDate = "2020-05-15T21:33:02.123456789Z"; - DateFieldMapper nullValueMapper = createMapper(Resolution.NANOSECONDS, "strict_date_time||epoch_millis", nullValueDate); - assertEquals(List.of(nullValueDate), fetchSourceValue(nullValueMapper, null)); - } public void testFetchDocValuesMillis() throws IOException { MapperService mapperService = createMapperService( @@ -376,27 +333,4 @@ public void testFetchDocValuesNanos() throws IOException { assertEquals(List.of(date), fetchFromDocValues(mapperService, ft, format, date)); assertEquals(List.of("2020-05-15T21:33:02.123Z"), fetchFromDocValues(mapperService, ft, format, 1589578382123L)); } - - private DateFieldMapper createMapper(Resolution resolution, String format) { - return createMapper(resolution, format, null); - } - - private DateFieldMapper createMapper(Resolution resolution, String format, String nullValue) { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - Map mapping = new HashMap<>(); - mapping.put("type", "date_nanos"); - if (format != null) { - mapping.put("format", format); - } - if (nullValue != null) { - mapping.put("null_value", nullValue); - } - - DateFieldMapper.Builder builder - = new DateFieldMapper.Builder("field", resolution, null, false, Version.CURRENT); - builder.parse("field", null, mapping); - return builder.build(context); - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index 1f2e15f0aae0a..95e09f229256c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.util.BigArrays; @@ -56,6 +57,7 @@ import java.time.Instant; import java.time.ZoneOffset; import java.util.Collections; +import java.util.List; public class DateFieldTypeTests extends FieldTypeTestCase { @@ -70,14 +72,12 @@ public void testIsFieldWithinRangeEmptyReader() throws IOException { } public void testIsFieldWithinQueryDateMillis() throws IOException { - DateFieldType ft = new DateFieldType("my_date", true, false, true, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, Resolution.MILLISECONDS, Collections.emptyMap()); + DateFieldType ft = new DateFieldType("my_date", Resolution.MILLISECONDS); isFieldWithinRangeTestCase(ft); } public void testIsFieldWithinQueryDateNanos() throws IOException { - DateFieldType ft = new DateFieldType("my_date", true, false, true, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, Resolution.NANOSECONDS, Collections.emptyMap()); + DateFieldType ft = new DateFieldType("my_date", Resolution.NANOSECONDS); isFieldWithinRangeTestCase(ft); } @@ -176,7 +176,7 @@ public void testTermQuery() { assertEquals(expected, ft.termQuery(date, context)); MappedFieldType unsearchable = new DateFieldType("field", false, false, true, DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, - Resolution.MILLISECONDS, Collections.emptyMap()); + Resolution.MILLISECONDS, null, Collections.emptyMap()); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery(date, context)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); @@ -211,7 +211,7 @@ BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry ft.rangeQuery("now", instant2, true, true, null, null, null, context)); MappedFieldType unsearchable = new DateFieldType("field", false, false, true, DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, - Resolution.MILLISECONDS, Collections.emptyMap()); + Resolution.MILLISECONDS, null, Collections.emptyMap()); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.rangeQuery(date1, date2, true, true, null, null, null, context)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); @@ -276,4 +276,49 @@ public void testDateNanoDocValues() throws IOException { private Instant instant(String str) { return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(str)).toInstant(); } + + private static DateFieldType fieldType(Resolution resolution, String format, String nullValue) { + DateFormatter formatter = DateFormatter.forPattern(format); + return new DateFieldType("field", true, false, true, formatter, resolution, nullValue, Collections.emptyMap()); + } + + public void testFetchSourceValue() throws IOException { + MappedFieldType fieldType = new DateFieldType("field", Resolution.MILLISECONDS); + String date = "2020-05-15T21:33:02.000Z"; + assertEquals(List.of(date), fetchSourceValue(fieldType, date)); + assertEquals(List.of(date), fetchSourceValue(fieldType, 1589578382000L)); + + MappedFieldType fieldWithFormat = fieldType(Resolution.MILLISECONDS, "yyyy/MM/dd||epoch_millis", null); + String dateInFormat = "1990/12/29"; + assertEquals(List.of(dateInFormat), fetchSourceValue(fieldWithFormat, dateInFormat)); + assertEquals(List.of(dateInFormat), fetchSourceValue(fieldWithFormat, 662428800000L)); + + MappedFieldType millis = fieldType(Resolution.MILLISECONDS, "epoch_millis", null); + String dateInMillis = "662428800000"; + assertEquals(List.of(dateInMillis), fetchSourceValue(millis, dateInMillis)); + assertEquals(List.of(dateInMillis), fetchSourceValue(millis, 662428800000L)); + + String nullValueDate = "2020-05-15T21:33:02.000Z"; + MappedFieldType nullFieldType = fieldType(Resolution.MILLISECONDS, "strict_date_time", nullValueDate); + assertEquals(List.of(nullValueDate), fetchSourceValue(nullFieldType, null)); + } + + public void testParseSourceValueWithFormat() throws IOException { + MappedFieldType mapper = fieldType(Resolution.NANOSECONDS, "strict_date_time", "1970-12-29T00:00:00.000Z"); + String date = "1990-12-29T00:00:00.000Z"; + assertEquals(List.of("1990/12/29"), fetchSourceValue(mapper, date, "yyyy/MM/dd")); + assertEquals(List.of("662428800000"), fetchSourceValue(mapper, date, "epoch_millis")); + assertEquals(List.of("1970/12/29"), fetchSourceValue(mapper, null, "yyyy/MM/dd")); + } + + public void testParseSourceValueNanos() throws IOException { + MappedFieldType mapper = fieldType(Resolution.NANOSECONDS, "strict_date_time||epoch_millis", null); + String date = "2020-05-15T21:33:02.123456789Z"; + assertEquals(List.of("2020-05-15T21:33:02.123456789Z"), fetchSourceValue(mapper, date)); + assertEquals(List.of("2020-05-15T21:33:02.123Z"), fetchSourceValue(mapper, 1589578382123L)); + + String nullValueDate = "2020-05-15T21:33:02.123456789Z"; + MappedFieldType nullValueMapper = fieldType(Resolution.NANOSECONDS, "strict_date_time||epoch_millis", nullValueDate); + assertEquals(List.of(nullValueDate), fetchSourceValue(nullValueMapper, null)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index da030c7252708..56f6db35334f1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -72,6 +72,11 @@ private FakeFieldType(String name) { super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); + } + @Override public String typeName() { return "fake"; @@ -88,11 +93,6 @@ static class FakeFieldMapper extends FieldMapper { protected void parseCreateField(ParseContext context) throws IOException { } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException(); - } - @Override protected void mergeOptions(FieldMapper other, List conflicts) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index b3611b1153f7a..00acdd5ad48ac 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -125,6 +125,16 @@ private ExternalFieldType(String name, boolean indexed, boolean stored, boolean public String typeName() { return "faketype"; } + + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + return new SourceValueFetcher(name(), mapperService, false) { + @Override + protected Object parseSourceValue(Object value) { + return value; + } + }; + } } private final String generatedValue; @@ -186,16 +196,6 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException(); } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { - @Override - protected Object parseSourceValue(Object value) { - return value; - } - }; - } - @Override public Iterator iterator() { return Iterators.concat(super.iterator(), Arrays.asList(binMapper, boolMapper, pointMapper, shapeMapper, stringMapper).iterator()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index 20ff0c1e5b98b..640fdcddad70d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -92,6 +92,16 @@ private FakeStringFieldType(String name, boolean stored, TextSearchInfo textSear public String typeName() { return CONTENT_TYPE; } + + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + return new SourceValueFetcher(name(), mapperService, false) { + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + }; + } } protected FakeStringFieldMapper(FieldType fieldType, MappedFieldType mappedFieldType, @@ -121,16 +131,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { - @Override - protected String parseSourceValue(Object value) { - return value.toString(); - } - }; - } - @Override protected void mergeOptions(FieldMapper other, List conflicts) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index 5fe285e49e265..32028a7d82752 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -19,19 +19,13 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.search.lookup.SourceLookup; import org.hamcrest.CoreMatchers; import java.io.IOException; -import java.util.List; -import java.util.Map; import java.util.Set; import static org.elasticsearch.geometry.utils.Geohash.stringEncode; @@ -331,39 +325,6 @@ public void testInvalidGeopointValuesIgnored() throws Exception { ); } - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - AbstractGeometryFieldMapper mapper = new GeoPointFieldMapper.Builder("field").build(context); - SourceLookup sourceLookup = new SourceLookup(); - - Map jsonPoint = Map.of("type", "Point", "coordinates", List.of(42.0, 27.1)); - Map otherJsonPoint = Map.of("type", "Point", "coordinates", List.of(30.0, 50.0)); - String wktPoint = "POINT (42.0 27.1)"; - String otherWktPoint = "POINT (30.0 50.0)"; - - // Test a single point in [lon, lat] array format. - Object sourceValue = List.of(42.0, 27.1); - assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); - - // Test a single point in "lat, lon" string format. - sourceValue = "27.1,42.0"; - assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); - - // Test a list of points in [lon, lat] array format. - sourceValue = List.of(List.of(42.0, 27.1), List.of(30.0, 50.0)); - assertEquals(List.of(jsonPoint, otherJsonPoint), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktPoint, otherWktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); - - // Test a single point in well-known text format. - sourceValue = "POINT (42.0 27.1)"; - assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); - } - @Override protected GeoPointFieldMapper.Builder newBuilder() { return new GeoPointFieldMapper.Builder("geo"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java new file mode 100644 index 0000000000000..c174be328100a --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class GeoPointFieldTypeTests extends FieldTypeTestCase { + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + MappedFieldType mapper = new GeoPointFieldMapper.Builder("field").build(context).fieldType(); + + Map jsonPoint = Map.of("type", "Point", "coordinates", List.of(42.0, 27.1)); + Map otherJsonPoint = Map.of("type", "Point", "coordinates", List.of(30.0, 50.0)); + String wktPoint = "POINT (42.0 27.1)"; + String otherWktPoint = "POINT (30.0 50.0)"; + + // Test a single point in [lon, lat] array format. + Object sourceValue = List.of(42.0, 27.1); + assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a single point in "lat, lon" string format. + sourceValue = "27.1,42.0"; + assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a list of points in [lon, lat] array format. + sourceValue = List.of(List.of(42.0, 27.1), List.of(30.0, 50.0)); + assertEquals(List.of(jsonPoint, otherJsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktPoint, otherWktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a single point in well-known text format. + sourceValue = "POINT (42.0 27.1)"; + assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java index 30fcfcd5dc134..6bc08d95601b5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java @@ -18,12 +18,9 @@ */ package org.elasticsearch.index.mapper; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.plugins.Plugin; @@ -34,7 +31,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Set; import static org.hamcrest.Matchers.containsString; @@ -237,39 +233,6 @@ public void testGeoShapeArrayParsing() throws Exception { assertThat(document.docs().get(0).getFields("field").length, equalTo(2)); } - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - GeoShapeFieldMapper mapper = new GeoShapeFieldMapper.Builder("field").build(context); - - Map jsonLineString = Map.of("type", "LineString", "coordinates", - List.of(List.of(42.0, 27.1), List.of(30.0, 50.0))); - Map jsonPoint = Map.of("type", "Point", "coordinates", List.of(14.0, 15.0)); - String wktLineString = "LINESTRING (42.0 27.1, 30.0 50.0)"; - String wktPoint = "POINT (14.0 15.0)"; - - // Test a single shape in geojson format. - Object sourceValue = jsonLineString; - assertEquals(List.of(jsonLineString), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktLineString), fetchSourceValue(mapper, sourceValue, "wkt")); - - // Test a list of shapes in geojson format. - sourceValue = List.of(jsonLineString, jsonPoint); - assertEquals(List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); - - // Test a single shape in wkt format. - sourceValue = wktLineString; - assertEquals(List.of(jsonLineString), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktLineString), fetchSourceValue(mapper, sourceValue, "wkt")); - - // Test a list of shapes in wkt format. - sourceValue = List.of(wktLineString, wktPoint); - assertEquals(List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); - } - @Override protected boolean supportsMeta() { return false; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java new file mode 100644 index 0000000000000..a4ac04ccfba6b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java @@ -0,0 +1,64 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class GeoShapeFieldTypeTests extends FieldTypeTestCase { + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + MappedFieldType mapper = new GeoShapeFieldMapper.Builder("field").build(context).fieldType(); + + Map jsonLineString = Map.of("type", "LineString", "coordinates", + List.of(List.of(42.0, 27.1), List.of(30.0, 50.0))); + Map jsonPoint = Map.of("type", "Point", "coordinates", List.of(14.0, 15.0)); + String wktLineString = "LINESTRING (42.0 27.1, 30.0 50.0)"; + String wktPoint = "POINT (14.0 15.0)"; + + // Test a single shape in geojson format. + Object sourceValue = jsonLineString; + assertEquals(List.of(jsonLineString), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktLineString), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a list of shapes in geojson format. + sourceValue = List.of(jsonLineString, jsonPoint); + assertEquals(List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a single shape in wkt format. + sourceValue = wktLineString; + assertEquals(List.of(jsonLineString), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktLineString), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a list of shapes in wkt format. + sourceValue = List.of(wktLineString, wktPoint); + assertEquals(List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 54c2f9c74409b..ccbd2fcc79d8d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -27,15 +27,12 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.network.InetAddresses; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.termvectors.TermVectorsService; import java.io.IOException; import java.net.InetAddress; -import java.util.List; import static org.hamcrest.Matchers.containsString; @@ -216,19 +213,4 @@ public void testNullValue() throws IOException { })); assertWarnings("Error parsing [:1] as IP in [null_value] on field [field]); [null_value] will be ignored"); } - - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - IpFieldMapper mapper = new IpFieldMapper.Builder("field", true, Version.CURRENT).build(context); - assertEquals(List.of("2001:db8::2:1"), fetchSourceValue(mapper, "2001:db8::2:1")); - assertEquals(List.of("2001:db8::2:1"), fetchSourceValue(mapper, "2001:db8:0:0:0:0:2:1")); - assertEquals(List.of("::1"), fetchSourceValue(mapper, "0:0:0:0:0:0:0:1")); - - IpFieldMapper nullValueMapper = new IpFieldMapper.Builder("field", true, Version.CURRENT) - .nullValue("2001:db8:0:0:0:0:2:7") - .build(context); - assertEquals(List.of("2001:db8::2:7"), fetchSourceValue(nullValueMapper, null)); - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java index 56d26bb06c8e3..28e976d7db600 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java @@ -24,11 +24,16 @@ import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.settings.Settings; +import java.io.IOException; import java.net.InetAddress; import java.util.Arrays; import java.util.Collections; +import java.util.List; public class IpFieldTypeTests extends FieldTypeTestCase { @@ -71,7 +76,7 @@ public void testTermQuery() { prefix = ip + "/16"; assertEquals(InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 16), ft.termQuery(prefix, null)); - MappedFieldType unsearchable = new IpFieldMapper.IpFieldType("field", false, false, true, Collections.emptyMap()); + MappedFieldType unsearchable = new IpFieldMapper.IpFieldType("field", false, false, true, null, Collections.emptyMap()); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("::1", null)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); @@ -174,9 +179,26 @@ public void testRangeQuery() { InetAddresses.forString("2001:db8::")), ft.rangeQuery("::ffff:c0a8:107", "2001:db8::", true, true, null, null, null, null)); - MappedFieldType unsearchable = new IpFieldMapper.IpFieldType("field", false, false, true, Collections.emptyMap()); + MappedFieldType unsearchable = new IpFieldMapper.IpFieldType("field", false, false, true, null, Collections.emptyMap()); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.rangeQuery("::1", "2001::", true, true, null, null, null, null)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); } + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + MappedFieldType mapper + = new IpFieldMapper.Builder("field", true, Version.CURRENT).build(context).fieldType(); + assertEquals(List.of("2001:db8::2:1"), fetchSourceValue(mapper, "2001:db8::2:1")); + assertEquals(List.of("2001:db8::2:1"), fetchSourceValue(mapper, "2001:db8:0:0:0:0:2:1")); + assertEquals(List.of("::1"), fetchSourceValue(mapper, "0:0:0:0:0:0:0:1")); + + MappedFieldType nullValueMapper = new IpFieldMapper.Builder("field", true, Version.CURRENT) + .nullValue("2001:db8:0:0:0:0:2:7") + .build(context) + .fieldType(); + assertEquals(List.of("2001:db8::2:7"), fetchSourceValue(nullValueMapper, null)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java index f3b05abf942ce..2f861ca6fc79c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java @@ -20,13 +20,10 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.network.InetAddresses; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; @@ -34,12 +31,9 @@ import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.Before; -import java.io.IOException; import java.util.HashMap; -import java.util.List; import java.util.Map; -import static org.elasticsearch.index.mapper.FieldMapperTestCase.fetchSourceValue; import static org.hamcrest.Matchers.containsString; public class IpRangeFieldMapperTests extends ESSingleNodeTestCase { @@ -86,14 +80,4 @@ public void testStoreCidr() throws Exception { assertThat(storedField.stringValue(), containsString(strVal)); } } - - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - RangeFieldMapper mapper = new RangeFieldMapper.Builder("field", RangeType.IP, true).build(context); - Map range = Map.of("gte", "2001:db8:0:0:0:0:2:1"); - assertEquals(List.of(Map.of("gte", "2001:db8::2:1")), fetchSourceValue(mapper, range)); - assertEquals(List.of("2001:db8::2:1/32"), fetchSourceValue(mapper, "2001:db8:0:0:0:0:2:1/32")); - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldTypeTests.java new file mode 100644 index 0000000000000..6203597cacaa6 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldTypeTests.java @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class IpRangeFieldTypeTests extends FieldTypeTestCase { + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + RangeFieldMapper mapper = new RangeFieldMapper.Builder("field", RangeType.IP, true).build(context); + Map range = Map.of("gte", "2001:db8:0:0:0:0:2:1"); + assertEquals(List.of(Map.of("gte", "2001:db8::2:1")), fetchSourceValue(mapper.fieldType(), range)); + assertEquals(List.of("2001:db8::2:1/32"), fetchSourceValue(mapper.fieldType(), "2001:db8:0:0:0:0:2:1/32")); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 3496f3ac9bbb5..33a886c131757 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -31,10 +31,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalyzerScope; @@ -455,35 +452,4 @@ public void testSplitQueriesOnWhitespace() throws IOException { new String[] { "hello world" } ); } - - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - KeywordFieldMapper mapper = new KeywordFieldMapper.Builder("field").build(context); - assertEquals(List.of("value"), fetchSourceValue(mapper, "value")); - assertEquals(List.of("42"), fetchSourceValue(mapper, 42L)); - assertEquals(List.of("true"), fetchSourceValue(mapper, true)); - - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> fetchSourceValue(mapper, "value", "format")); - assertEquals("Field [field] of type [keyword] doesn't support formats.", e.getMessage()); - - KeywordFieldMapper ignoreAboveMapper = new KeywordFieldMapper.Builder("field") - .ignoreAbove(4) - .build(context); - assertEquals(List.of(), fetchSourceValue(ignoreAboveMapper, "value")); - assertEquals(List.of("42"), fetchSourceValue(ignoreAboveMapper, 42L)); - assertEquals(List.of("true"), fetchSourceValue(ignoreAboveMapper, true)); - - KeywordFieldMapper normalizerMapper = new KeywordFieldMapper.Builder("field", createIndexAnalyzers(null)).normalizer("lowercase") - .build(context); - assertEquals(List.of("value"), fetchSourceValue(normalizerMapper, "VALUE")); - assertEquals(List.of("42"), fetchSourceValue(normalizerMapper, 42L)); - assertEquals(List.of("value"), fetchSourceValue(normalizerMapper, "value")); - - KeywordFieldMapper nullValueMapper = new KeywordFieldMapper.Builder("field") - .nullValue("NULL") - .build(context); - assertEquals(List.of("NULL"), fetchSourceValue(nullValueMapper, null)); - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index 340fffa1f54a4..2632753ae1b58 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -25,6 +25,7 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocValuesFieldExistsQuery; @@ -36,11 +37,20 @@ import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.analysis.AnalyzerScope; +import org.elasticsearch.index.analysis.CharFilterFactory; +import org.elasticsearch.index.analysis.CustomAnalyzer; +import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.analysis.LowercaseNormalizer; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.analysis.TokenFilterFactory; +import org.elasticsearch.index.analysis.TokenizerFactory; import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.elasticsearch.index.mapper.MappedFieldType.Relation; @@ -49,6 +59,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; public class KeywordFieldTypeTests extends FieldTypeTestCase { @@ -56,9 +67,9 @@ public void testIsFieldWithinQuery() throws IOException { KeywordFieldType ft = new KeywordFieldType("field"); // current impl ignores args and should always return INTERSECTS assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(null, - RandomStrings.randomAsciiOfLengthBetween(random(), 0, 5), - RandomStrings.randomAsciiOfLengthBetween(random(), 0, 5), - randomBoolean(), randomBoolean(), null, null, null)); + RandomStrings.randomAsciiLettersOfLengthBetween(random(), 0, 5), + RandomStrings.randomAsciiLettersOfLengthBetween(random(), 0, 5), + randomBoolean(), randomBoolean(), null, null, null)); } public void testTermQuery() { @@ -110,8 +121,7 @@ public void testExistsQuery() { { FieldType fieldType = new FieldType(); fieldType.setOmitNorms(false); - KeywordFieldType ft - = new KeywordFieldType("field", false, fieldType, randomBoolean(), null, null, null, Collections.emptyMap()); + KeywordFieldType ft = new KeywordFieldType("field", fieldType); assertEquals(new NormsFieldExistsQuery("field"), ft.existsQuery(null)); } { @@ -170,4 +180,71 @@ public void testNormalizeQueries() { ft = new KeywordFieldType("field", Lucene.STANDARD_ANALYZER); assertEquals(new TermQuery(new Term("field", new BytesRef("foo"))), ft.termQuery("FOO", null)); } + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + MappedFieldType mapper = new KeywordFieldMapper.Builder("field").build(context).fieldType(); + assertEquals(List.of("value"), fetchSourceValue(mapper, "value")); + assertEquals(List.of("42"), fetchSourceValue(mapper, 42L)); + assertEquals(List.of("true"), fetchSourceValue(mapper, true)); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> fetchSourceValue(mapper, "value", "format")); + assertEquals("Field [field] of type [keyword] doesn't support formats.", e.getMessage()); + + MappedFieldType ignoreAboveMapper = new KeywordFieldMapper.Builder("field") + .ignoreAbove(4) + .build(context) + .fieldType(); + assertEquals(List.of(), fetchSourceValue(ignoreAboveMapper, "value")); + assertEquals(List.of("42"), fetchSourceValue(ignoreAboveMapper, 42L)); + assertEquals(List.of("true"), fetchSourceValue(ignoreAboveMapper, true)); + + MappedFieldType normalizerMapper = new KeywordFieldMapper.Builder("field", createIndexAnalyzers()).normalizer("lowercase") + .build(context) + .fieldType(); + assertEquals(List.of("value"), fetchSourceValue(normalizerMapper, "VALUE")); + assertEquals(List.of("42"), fetchSourceValue(normalizerMapper, 42L)); + assertEquals(List.of("value"), fetchSourceValue(normalizerMapper, "value")); + + MappedFieldType nullValueMapper = new KeywordFieldMapper.Builder("field") + .nullValue("NULL") + .build(context) + .fieldType(); + assertEquals(List.of("NULL"), fetchSourceValue(nullValueMapper, null)); + } + + private static IndexAnalyzers createIndexAnalyzers() { + return new IndexAnalyzers( + Map.of("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer())), + Map.ofEntries( + Map.entry("lowercase", new NamedAnalyzer("lowercase", AnalyzerScope.INDEX, new LowercaseNormalizer())), + Map.entry("other_lowercase", new NamedAnalyzer("other_lowercase", AnalyzerScope.INDEX, new LowercaseNormalizer())) + ), + Map.of( + "lowercase", + new NamedAnalyzer( + "lowercase", + AnalyzerScope.INDEX, + new CustomAnalyzer( + TokenizerFactory.newFactory("lowercase", WhitespaceTokenizer::new), + new CharFilterFactory[0], + new TokenFilterFactory[] { new TokenFilterFactory() { + + @Override + public String name() { + return "lowercase"; + } + + @Override + public TokenStream create(TokenStream tokenStream) { + return new org.apache.lucene.analysis.core.LowerCaseFilter(tokenStream); + } + } } + ) + ) + ) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java index 3fb376670fbd3..cd0232c67e386 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java @@ -24,27 +24,22 @@ import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.geometry.Point; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import java.io.IOException; import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.Set; import static java.util.Collections.singletonMap; @@ -636,38 +631,4 @@ public void testGeoShapeArrayParsing() throws Exception { assertThat(fields.length, equalTo(2)); assertFieldWarnings("tree", "strategy"); } - - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - LegacyGeoShapeFieldMapper mapper = new LegacyGeoShapeFieldMapper.Builder("field").build(context); - SourceLookup sourceLookup = new SourceLookup(); - - Map jsonLineString = Map.of("type", "LineString", "coordinates", - List.of(List.of(42.0, 27.1), List.of(30.0, 50.0))); - Map jsonPoint = Map.of("type", "Point", "coordinates", List.of(14.0, 15.0)); - String wktLineString = "LINESTRING (42.0 27.1, 30.0 50.0)"; - String wktPoint = "POINT (14.0 15.0)"; - - // Test a single shape in geojson format. - Object sourceValue = jsonLineString; - assertEquals(List.of(jsonLineString), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktLineString), fetchSourceValue(mapper, sourceValue, "wkt")); - - // Test a list of shapes in geojson format. - sourceValue = List.of(jsonLineString, jsonPoint); - assertEquals(List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); - - // Test a single shape in wkt format. - sourceValue = wktLineString; - assertEquals(List.of(jsonLineString), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktLineString), fetchSourceValue(mapper, sourceValue, "wkt")); - - // Test a list of shapes in wkt format. - sourceValue = List.of(wktLineString, wktPoint); - assertEquals(List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java index b34c998656262..dc97fdb9f6680 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java @@ -18,9 +18,16 @@ */ package org.elasticsearch.index.mapper; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.geo.SpatialStrategy; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper.GeoShapeFieldType; +import java.io.IOException; +import java.util.List; +import java.util.Map; + public class LegacyGeoShapeFieldTypeTests extends FieldTypeTestCase { /** @@ -35,4 +42,37 @@ public void testSetStrategyName() { fieldType.setStrategy(SpatialStrategy.TERM); assertTrue(fieldType.pointsOnly()); } + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + MappedFieldType mapper = new LegacyGeoShapeFieldMapper.Builder("field").build(context).fieldType(); + + Map jsonLineString = Map.of("type", "LineString", "coordinates", + List.of(List.of(42.0, 27.1), List.of(30.0, 50.0))); + Map jsonPoint = Map.of("type", "Point", "coordinates", List.of(14.0, 15.0)); + String wktLineString = "LINESTRING (42.0 27.1, 30.0 50.0)"; + String wktPoint = "POINT (14.0 15.0)"; + + // Test a single shape in geojson format. + Object sourceValue = jsonLineString; + assertEquals(List.of(jsonLineString), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktLineString), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a list of shapes in geojson format. + sourceValue = List.of(jsonLineString, jsonPoint); + assertEquals(List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a single shape in wkt format. + sourceValue = wktLineString; + assertEquals(List.of(jsonLineString), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktLineString), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a list of shapes in wkt format. + sourceValue = List.of(wktLineString, wktPoint); + assertEquals(List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 220368d741959..d22c899945549 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -21,11 +21,8 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; @@ -229,21 +226,6 @@ protected void doTestNullValue(String type) throws IOException { assertFalse(dvField.fieldType().stored()); } - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - NumberFieldMapper mapper = new NumberFieldMapper.Builder("field", NumberType.INTEGER, false, true).build(context); - assertEquals(List.of(3), fetchSourceValue(mapper, 3.14)); - assertEquals(List.of(42), fetchSourceValue(mapper, "42.9")); - - NumberFieldMapper nullValueMapper = new NumberFieldMapper.Builder("field", NumberType.FLOAT, false, true) - .nullValue(2.71f) - .build(context); - assertEquals(List.of(2.71f), fetchSourceValue(nullValueMapper, "")); - assertEquals(List.of(2.71f), fetchSourceValue(nullValueMapper, null)); - } - public void testOutOfRangeValues() throws IOException { final List inputs = Arrays.asList( OutOfRangeSpec.of(NumberType.BYTE, "128", "is out of range for a byte"), diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index acea9a9f31e37..5a91fe3d8754c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -131,12 +131,15 @@ public void testLongTermQueryWithDecimalPart() { assertTrue(ft.termQuery(42.1, null) instanceof MatchNoDocsQuery); } + private static MappedFieldType unsearchable() { + return new NumberFieldType("field", NumberType.LONG, false, false, true, true, null, Collections.emptyMap()); + } + public void testTermQuery() { MappedFieldType ft = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.LONG); assertEquals(LongPoint.newExactQuery("field", 42), ft.termQuery("42", null)); - MappedFieldType unsearchable - = new NumberFieldType("field", NumberType.LONG, false, false, true, Collections.emptyMap()); + MappedFieldType unsearchable = unsearchable(); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("42", null)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); @@ -253,7 +256,7 @@ public void testRangeQuery() { SortedNumericDocValuesField.newSlowRangeQuery("field", 1, 3)); assertEquals(expected, ft.rangeQuery("1", "3", true, true, null, null, null, MOCK_QSC)); - MappedFieldType unsearchable = new NumberFieldType("field", NumberType.LONG, false, false, true, Collections.emptyMap()); + MappedFieldType unsearchable = unsearchable(); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.rangeQuery("1", "3", true, true, null, null, null, MOCK_QSC)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); @@ -639,4 +642,22 @@ public void testParsePoint() { assertThat(NumberType.HALF_FLOAT.parsePoint(bytes), equalTo(value)); } } + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + MappedFieldType mapper = new NumberFieldMapper.Builder("field", NumberType.INTEGER, false, true) + .build(context) + .fieldType(); + assertEquals(List.of(3), fetchSourceValue(mapper, 3.14)); + assertEquals(List.of(42), fetchSourceValue(mapper, "42.9")); + + MappedFieldType nullValueMapper = new NumberFieldMapper.Builder("field", NumberType.FLOAT, false, true) + .nullValue(2.71f) + .build(context) + .fieldType(); + assertEquals(List.of(2.71f), fetchSourceValue(nullValueMapper, "")); + assertEquals(List.of(2.71f), fetchSourceValue(nullValueMapper, null)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index be8e5df3cc57c..ed095c644a84b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -35,7 +35,6 @@ import org.elasticsearch.index.mapper.ParametrizedFieldMapper.Parameter; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collection; @@ -192,11 +191,6 @@ protected void parseCreateField(ParseContext context) { } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - return null; - } - @Override protected String contentType() { return "test_mapper"; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index cdc7953ead01d..d0d004506035c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -22,21 +22,16 @@ import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.network.InetAddresses; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import java.io.IOException; import java.net.InetAddress; -import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Set; import static org.elasticsearch.index.query.RangeQueryBuilder.GTE_FIELD; @@ -354,36 +349,4 @@ public void testIllegalFormatField() throws Exception { ); assertThat(e.getMessage(), containsString("Invalid format: [[test_format]]: Unknown pattern letter: t")); } - - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - RangeFieldMapper longMapper = new RangeFieldMapper.Builder("field", RangeType.LONG, true).build(context); - Map longRange = Map.of("gte", 3.14, "lt", "42.9"); - assertEquals(List.of(Map.of("gte", 3L, "lt", 42L)), fetchSourceValue(longMapper, longRange)); - - RangeFieldMapper dateMapper = new RangeFieldMapper.Builder("field", RangeType.DATE, true) - .format("yyyy/MM/dd||epoch_millis") - .build(context); - Map dateRange = Map.of("lt", "1990/12/29", "gte", 597429487111L); - assertEquals(List.of(Map.of("lt", "1990/12/29", "gte", "1988/12/06")), - fetchSourceValue(dateMapper, dateRange)); - } - - public void testParseSourceValueWithFormat() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - RangeFieldMapper longMapper = new RangeFieldMapper.Builder("field", RangeType.LONG, true).build(context); - Map longRange = Map.of("gte", 3.14, "lt", "42.9"); - assertEquals(List.of(Map.of("gte", 3L, "lt", 42L)), fetchSourceValue(longMapper, longRange)); - - RangeFieldMapper dateMapper = new RangeFieldMapper.Builder("field", RangeType.DATE, true) - .format("strict_date_time") - .build(context); - Map dateRange = Map.of("lt", "1990-12-29T00:00:00.000Z"); - assertEquals(List.of(Map.of("lt", "1990/12/29")), fetchSourceValue(dateMapper, dateRange, "yyy/MM/dd")); - assertEquals(List.of(Map.of("lt", "662428800000")), fetchSourceValue(dateMapper, dateRange,"epoch_millis")); - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index 7d0da2db408ec..ef17a3bc3371f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -47,8 +47,10 @@ import org.joda.time.DateTime; import org.junit.Before; +import java.io.IOException; import java.net.InetAddress; -import java.util.Collections; +import java.util.List; +import java.util.Map; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; @@ -66,9 +68,9 @@ public void setupProperties() { private RangeFieldType createDefaultFieldType() { if (type == RangeType.DATE) { - return new RangeFieldType("field", true, false, true, RangeFieldMapper.Defaults.DATE_FORMATTER, Collections.emptyMap()); + return new RangeFieldType("field", RangeFieldMapper.Defaults.DATE_FORMATTER); } - return new RangeFieldType("field", type, true, false, true, Collections.emptyMap()); + return new RangeFieldType("field", type); } public void testRangeQuery() throws Exception { @@ -216,8 +218,7 @@ private QueryShardContext createContext() { public void testDateRangeQueryUsingMappingFormat() { QueryShardContext context = createContext(); - RangeFieldType strict - = new RangeFieldType("field", true, false, false, RangeFieldMapper.Defaults.DATE_FORMATTER, Collections.emptyMap()); + RangeFieldType strict = new RangeFieldType("field", RangeFieldMapper.Defaults.DATE_FORMATTER); // don't use DISJOINT here because it doesn't work on date fields which we want to compare bounds with ShapeRelation relation = randomValueOtherThan(ShapeRelation.DISJOINT,() -> randomFrom(ShapeRelation.values())); @@ -236,13 +237,13 @@ public void testDateRangeQueryUsingMappingFormat() { assertEquals(1465975790000L, formatter.parseMillis(from)); assertEquals(1466062190000L, formatter.parseMillis(to)); - RangeFieldType fieldType = new RangeFieldType("field", true, false, true, formatter, Collections.emptyMap()); + RangeFieldType fieldType = new RangeFieldType("field", formatter); final Query query = fieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context); assertEquals("field:", query.toString()); // compare lower and upper bounds with what we would get on a `date` field DateFieldType dateFieldType - = new DateFieldType("field", true, false, true, formatter, DateFieldMapper.Resolution.MILLISECONDS, Collections.emptyMap()); + = new DateFieldType("field", DateFieldMapper.Resolution.MILLISECONDS, formatter); final Query queryOnDateField = dateFieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context); assertEquals("field:[1465975790000 TO 1466062190999]", queryOnDateField.toString()); } @@ -259,7 +260,7 @@ public void testDateVsDateRangeBounds() { long lower = randomLongBetween(formatter.parseMillis("2000-01-01T00:00"), formatter.parseMillis("2010-01-01T00:00")); long upper = randomLongBetween(formatter.parseMillis("2011-01-01T00:00"), formatter.parseMillis("2020-01-01T00:00")); - RangeFieldType fieldType = new RangeFieldType("field", true, false, false, formatter, Collections.emptyMap()); + RangeFieldType fieldType = new RangeFieldType("field", formatter); String lowerAsString = formatter.formatMillis(lower); String upperAsString = formatter.formatMillis(upper); // also add date math rounding to days occasionally @@ -491,4 +492,42 @@ public void testCaseInsensitiveQuery() throws Exception { () -> ft.termQueryCaseInsensitive(value, context)); assertTrue(ex.getMessage().contains("does not support case insensitive term queries")); } + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + MappedFieldType longMapper = new RangeFieldMapper.Builder("field", RangeType.LONG, true) + .build(context) + .fieldType(); + Map longRange = Map.of("gte", 3.14, "lt", "42.9"); + assertEquals(List.of(Map.of("gte", 3L, "lt", 42L)), fetchSourceValue(longMapper, longRange)); + + MappedFieldType dateMapper = new RangeFieldMapper.Builder("field", RangeType.DATE, true) + .format("yyyy/MM/dd||epoch_millis") + .build(context) + .fieldType(); + Map dateRange = Map.of("lt", "1990/12/29", "gte", 597429487111L); + assertEquals(List.of(Map.of("lt", "1990/12/29", "gte", "1988/12/06")), + fetchSourceValue(dateMapper, dateRange)); + } + + public void testParseSourceValueWithFormat() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + MappedFieldType longMapper = new RangeFieldMapper.Builder("field", RangeType.LONG, true) + .build(context) + .fieldType(); + Map longRange = Map.of("gte", 3.14, "lt", "42.9"); + assertEquals(List.of(Map.of("gte", 3L, "lt", 42L)), fetchSourceValue(longMapper, longRange)); + + MappedFieldType dateMapper = new RangeFieldMapper.Builder("field", RangeType.DATE, true) + .format("strict_date_time") + .build(context) + .fieldType(); + Map dateRange = Map.of("lt", "1990-12-29T00:00:00.000Z"); + assertEquals(List.of(Map.of("lt", "1990/12/29")), fetchSourceValue(dateMapper, dateRange, "yyy/MM/dd")); + assertEquals(List.of(Map.of("lt", "662428800000")), fetchSourceValue(dateMapper, dateRange,"epoch_millis")); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 7cf43131f667a..62cf6eba733e7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -50,11 +50,8 @@ import org.apache.lucene.search.spans.SpanOrQuery; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -76,7 +73,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Set; @@ -992,16 +988,4 @@ public void testSimpleMerge() throws IOException { assertThat(mapperService.documentMapper().mappers().getMapper("field"), instanceOf(TextFieldMapper.class)); assertThat(mapperService.documentMapper().mappers().getMapper("other_field"), instanceOf(KeywordFieldMapper.class)); } - - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - FieldMapper fieldMapper = newBuilder().build(context); - TextFieldMapper mapper = (TextFieldMapper) fieldMapper; - - assertEquals(List.of("value"), fetchSourceValue(mapper, "value")); - assertEquals(List.of("42"), fetchSourceValue(mapper, 42L)); - assertEquals(List.of("true"), fetchSourceValue(mapper, true)); - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index 9ed513577460d..4978962ee4961 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -35,11 +35,15 @@ import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.search.AutomatonQueries; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -160,4 +164,15 @@ public void testIndexPrefixes() { assertThat(q, equalTo(expected)); } + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + MappedFieldType mapper = new TextFieldMapper.Builder("field").build(context).fieldType(); + + assertEquals(List.of("value"), fetchSourceValue(mapper, "value")); + assertEquals(List.of("42"), fetchSourceValue(mapper, 42L)); + assertEquals(List.of("true"), fetchSourceValue(mapper, true)); + } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java index fdbe41e9eaa94..7a990921a0f77 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java @@ -100,7 +100,8 @@ private ValuesSourceConfig getVSConfig( boolean indexed, QueryShardContext context ) { - MappedFieldType ft = new NumberFieldMapper.NumberFieldType(fieldName, numType, indexed, false, true, Collections.emptyMap()); + MappedFieldType ft + = new NumberFieldMapper.NumberFieldType(fieldName, numType, indexed, false, true, false, null, Collections.emptyMap()); return ValuesSourceConfig.resolveFieldOnly(ft, context); } @@ -111,7 +112,7 @@ private ValuesSourceConfig getVSConfig( QueryShardContext context ) { MappedFieldType ft = new DateFieldMapper.DateFieldType(fieldName, indexed, false, true, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, resolution, Collections.emptyMap()); + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, resolution, null, Collections.emptyMap()); return ValuesSourceConfig.resolveFieldOnly(ft, context); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java index 6760fc59e6840..e0d644c9fdf18 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java @@ -100,6 +100,6 @@ protected final DateFieldMapper.DateFieldType aggregableDateFieldType(boolean us return new DateFieldMapper.DateFieldType(AGGREGABLE_DATE, isSearchable, false, true, DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, useNanosecondResolution ? DateFieldMapper.Resolution.NANOSECONDS : DateFieldMapper.Resolution.MILLISECONDS, - Collections.emptyMap()); + null, Collections.emptyMap()); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java index 4acb6ed6546bc..3c14aa75af93a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java @@ -272,7 +272,7 @@ private void testCase(Query query, Consumer> verify, DateFieldMapper.Resolution resolution) throws IOException { DateFieldMapper.DateFieldType fieldType = new DateFieldMapper.DateFieldType(DATE_FIELD_NAME, true, false, true, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, resolution, Collections.emptyMap()); + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, resolution, null, Collections.emptyMap()); DateRangeAggregationBuilder aggregationBuilder = new DateRangeAggregationBuilder("test_range_agg"); aggregationBuilder.field(DATE_FIELD_NAME); aggregationBuilder.addRange("2015-01-01", "2015-12-31"); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java index fc0fbb48b2a08..dda30646a6ca5 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java @@ -120,7 +120,7 @@ public void testDateFieldMillisecondResolution() throws IOException { @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/57651") public void testDateFieldNanosecondResolution() throws IOException { DateFieldMapper.DateFieldType fieldType = new DateFieldMapper.DateFieldType(DATE_FIELD_NAME, true, false, true, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, DateFieldMapper.Resolution.NANOSECONDS, Collections.emptyMap()); + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, DateFieldMapper.Resolution.NANOSECONDS, null, Collections.emptyMap()); // These values should work because aggs scale nanosecond up to millisecond always. long milli1 = ZonedDateTime.of(2015, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); @@ -144,7 +144,7 @@ public void testDateFieldNanosecondResolution() throws IOException { @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/57651") public void testMissingDateWithDateField() throws IOException { DateFieldMapper.DateFieldType fieldType = new DateFieldMapper.DateFieldType(DATE_FIELD_NAME, true, false, true, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, DateFieldMapper.Resolution.NANOSECONDS, Collections.emptyMap()); + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, DateFieldMapper.Resolution.NANOSECONDS, null, Collections.emptyMap()); // These values should work because aggs scale nanosecond up to millisecond always. long milli1 = ZonedDateTime.of(2015, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); diff --git a/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java b/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java index 414374110be5f..e9f96d1d57031 100644 --- a/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java @@ -32,12 +32,15 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.InnerHitBuilderTests; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.AbstractSerializingTestCase; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -155,14 +158,14 @@ public void testBuild() throws IOException { numberFieldType = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.LONG, true, false, - false, Collections.emptyMap()); + false, false, null, Collections.emptyMap()); when(shardContext.fieldMapper("field")).thenReturn(numberFieldType); IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> builder.build(shardContext)); assertEquals(exc.getMessage(), "cannot collapse on field `field` without `doc_values`"); numberFieldType = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.LONG, false, false, - true, Collections.emptyMap()); + true, false, null, Collections.emptyMap()); when(shardContext.fieldMapper("field")).thenReturn(numberFieldType); builder.setInnerHits(new InnerHitBuilder()); exc = expectThrows(IllegalArgumentException.class, () -> builder.build(shardContext)); @@ -205,6 +208,11 @@ public String typeName() { return null; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); + } + @Override public Query termQuery(Object value, QueryShardContext context) { return null; diff --git a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java index f719c73cc9698..7130f411ef549 100644 --- a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java @@ -51,10 +51,13 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.internal.ShardSearchRequest; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; @@ -122,6 +125,11 @@ private QueryShardContext createShardContext(Version indexVersionCreated, IndexR MappedFieldType fieldType = new MappedFieldType(fieldName, true, false, dvType != null, TextSearchInfo.NONE, Collections.emptyMap()) { + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); + } + @Override public String typeName() { return null; diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java index 94216e018616d..d4304b412e532 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java @@ -32,7 +32,6 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESSingleNodeTestCase; import java.io.IOException; @@ -46,8 +45,6 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * Base class for testing {@link FieldMapper}s. @@ -250,21 +247,4 @@ private String mappingsToString(ToXContent builder, boolean includeDefaults) thr x.endObject().endObject(); return Strings.toString(x); } - - public static List fetchSourceValue(FieldMapper mapper, Object sourceValue) throws IOException { - return fetchSourceValue(mapper, sourceValue, null); - } - - public static List fetchSourceValue(FieldMapper mapper, Object sourceValue, String format) throws IOException { - String field = mapper.name(); - - MapperService mapperService = mock(MapperService.class); - when(mapperService.sourcePath(field)).thenReturn(Set.of(field)); - - ValueFetcher fetcher = mapper.valueFetcher(mapperService, null, format); - SourceLookup lookup = new SourceLookup(); - lookup.setSource(Collections.singletonMap(field, sourceValue)); - return fetcher.fetchValues(lookup); - } - } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java index c0078da3615aa..4442da6c5370b 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java @@ -19,8 +19,14 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Set; + import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -39,4 +45,21 @@ static QueryShardContext createMockQueryShardContext(boolean allowExpensiveQueri when(queryShardContext.allowExpensiveQueries()).thenReturn(allowExpensiveQueries); return queryShardContext; } + + public static List fetchSourceValue(MappedFieldType fieldType, Object sourceValue) throws IOException { + return fetchSourceValue(fieldType, sourceValue, null); + } + + public static List fetchSourceValue(MappedFieldType fieldType, Object sourceValue, String format) throws IOException { + String field = fieldType.name(); + + MapperService mapperService = mock(MapperService.class); + when(mapperService.sourcePath(field)).thenReturn(Set.of(field)); + + ValueFetcher fetcher = fieldType.valueFetcher(mapperService, null, format); + SourceLookup lookup = new SourceLookup(); + lookup.setSource(Collections.singletonMap(field, sourceValue)); + return fetcher.fetchValues(lookup); + } + } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 3d722e058df04..e28a856a61e79 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -44,7 +44,6 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.lookup.SearchLookup; -import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.ArrayList; @@ -52,7 +51,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Supplier; @@ -60,8 +58,6 @@ import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * Base class for testing {@link Mapper}s. @@ -276,24 +272,8 @@ public final void testDeprecatedBoost() throws IOException { assertParseMinimalWarnings(); } - public static List fetchSourceValue(FieldMapper mapper, Object sourceValue) throws IOException { - return fetchSourceValue(mapper, sourceValue, null); - } - - public static List fetchSourceValue(FieldMapper mapper, Object sourceValue, String format) throws IOException { - String field = mapper.name(); - - MapperService mapperService = mock(MapperService.class); - when(mapperService.sourcePath(field)).thenReturn(Set.of(field)); - - ValueFetcher fetcher = mapper.valueFetcher(mapperService, null, format); - SourceLookup lookup = new SourceLookup(); - lookup.setSource(Collections.singletonMap(field, sourceValue)); - return fetcher.fetchValues(lookup); - } - /** - * Use a {@linkplain FieldMapper} to extract values from doc values. + * Use a {@linkplain ValueFetcher} to extract values from doc values. */ protected final List fetchFromDocValues(MapperService mapperService, MappedFieldType ft, DocValueFormat format, Object sourceValue) throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index 9c7e62844daa4..078c0820ffde2 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -65,6 +65,11 @@ public FakeFieldType(String name) { public String typeName() { return "faketype"; } + + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); + } } @Override @@ -76,11 +81,6 @@ protected String contentType() { protected void parseCreateField(ParseContext context) throws IOException { } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException(); - } - @Override protected void mergeOptions(FieldMapper other, List conflicts) { diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 6e299ac441baf..57eea7fee7552 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -170,19 +170,6 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException("Parsing is implemented in parse(), this method should NEVER be called"); } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { - @Override - protected Object parseSourceValue(Object value) { - return value; - } - }; - } - public static class HistogramFieldType extends MappedFieldType { public HistogramFieldType(String name, boolean hasDocValues, Map meta) { @@ -194,6 +181,19 @@ public String typeName() { return CONTENT_TYPE; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return new SourceValueFetcher(name(), mapperService, false) { + @Override + protected Object parseSourceValue(Object value) { + return value; + } + }; + } + @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java index 641035012ecbc..988c6c673bb07 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java @@ -432,6 +432,7 @@ private DateFieldMapper.DateFieldType dateFieldType(String name) { true, DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, DateFieldMapper.Resolution.MILLISECONDS, + null, Collections.emptyMap() ); } diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java index a2255dbc238f6..4b0c8a8bf4ea6 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java @@ -17,12 +17,15 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.io.UncheckedIOException; @@ -59,6 +62,11 @@ public Query termQuery(Object value, QueryShardContext context) { public Query existsQuery(QueryShardContext context) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support exists queries"); } + + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); + } } private static DataStreamTimestampFieldMapper toType(FieldMapper in) { diff --git a/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index 2ee052214b6c6..1fed2847531b8 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; @@ -20,15 +19,12 @@ import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.ParsedDocument; -import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin; import org.elasticsearch.xpack.constantkeyword.mapper.ConstantKeywordFieldMapper.ConstantKeywordFieldType; import java.io.IOException; import java.util.Collection; -import java.util.Collections; import java.util.List; import static org.hamcrest.Matchers.instanceOf; @@ -160,24 +156,4 @@ protected void registerParameters(ParameterChecker checker) throws IOException { b.field("value", "bar"); })); } - - public void testFetchValue() throws Exception { - MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "constant_keyword"))); - FieldMapper fieldMapper = (FieldMapper) mapperService.documentMapper().mappers().getMapper("field"); - ValueFetcher fetcher = fieldMapper.valueFetcher(mapperService, null, null); - - SourceLookup missingValueLookup = new SourceLookup(); - SourceLookup nullValueLookup = new SourceLookup(); - nullValueLookup.setSource(Collections.singletonMap("field", null)); - - assertTrue(fetcher.fetchValues(missingValueLookup).isEmpty()); - assertTrue(fetcher.fetchValues(nullValueLookup).isEmpty()); - - merge(mapperService, fieldMapping(b -> b.field("type", "constant_keyword").field("value", "foo"))); - fieldMapper = (FieldMapper) mapperService.documentMapper().mappers().getMapper("field"); - fetcher = fieldMapper.valueFetcher(mapperService, null, null); - - assertEquals(List.of("foo"), fetcher.fetchValues(missingValueLookup)); - assertEquals(List.of("foo"), fetcher.fetchValues(nullValueLookup)); - } } diff --git a/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java b/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java new file mode 100644 index 0000000000000..5250f10456e46 --- /dev/null +++ b/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.constantkeyword.mapper; + +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.util.Collections; +import java.util.List; + +public class ConstantKeywordFieldTypeTests extends FieldTypeTestCase { + + public void testFetchValue() throws Exception { + MappedFieldType fieldType = new ConstantKeywordFieldMapper.ConstantKeywordFieldType("field", null); + ValueFetcher fetcher = fieldType.valueFetcher(null, null, null); + + SourceLookup missingValueLookup = new SourceLookup(); + SourceLookup nullValueLookup = new SourceLookup(); + nullValueLookup.setSource(Collections.singletonMap("field", null)); + + assertTrue(fetcher.fetchValues(missingValueLookup).isEmpty()); + assertTrue(fetcher.fetchValues(nullValueLookup).isEmpty()); + + MappedFieldType valued = new ConstantKeywordFieldMapper.ConstantKeywordFieldType("field", null); + fetcher = valued.valueFetcher(null, null, null); + + assertEquals(List.of("foo"), fetcher.fetchValues(missingValueLookup)); + assertEquals(List.of("foo"), fetcher.fetchValues(nullValueLookup)); + } +} diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 37a83cc0eddbc..ad3d4c52ec8be 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -130,6 +130,17 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S return new ConstantIndexFieldData.Builder(mapperService -> value, name(), CoreValuesSourceType.BYTES); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + + return value == null + ? lookup -> List.of() + : lookup -> List.of(value); + } + @Override protected boolean matches(String pattern, boolean caseInsensitive, QueryShardContext context) { if (value == null) { @@ -250,17 +261,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - - return fieldType().value == null - ? lookup -> List.of() - : lookup -> List.of(fieldType().value); - } - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java b/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java index b65ecc2b5c89a..053674043e28b 100644 --- a/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java +++ b/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java @@ -9,23 +9,18 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldMapperTestCase; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; -import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService.MergeReason; @@ -41,8 +36,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; -import java.util.List; -import java.util.Map; import java.util.Set; import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; @@ -512,18 +505,4 @@ public void testSplitQueriesOnWhitespace() throws IOException { assertTokenStreamContents(keyedFieldType.getTextSearchInfo().getSearchAnalyzer().analyzer().tokenStream("", "Hello World"), new String[] {"Hello", "World"}); } - - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - Map sourceValue = Map.of("key", "value"); - FlatObjectFieldMapper mapper = new FlatObjectFieldMapper.Builder("field").build(context); - assertEquals(List.of(sourceValue), fetchSourceValue(mapper, sourceValue)); - - FlatObjectFieldMapper nullValueMapper = new FlatObjectFieldMapper.Builder("field") - .nullValue("NULL") - .build(context); - assertEquals(List.of("NULL"), fetchSourceValue(nullValueMapper, null)); - } } diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java index c84d8076bf12f..bc7c8b7801ad5 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java @@ -177,7 +177,8 @@ public Builder store(boolean store) { @Override public FlatObjectFieldMapper build(BuilderContext context) { - MappedFieldType ft = new RootFlatObjectFieldType(buildFullName(context), indexed, hasDocValues, meta, splitQueriesOnWhitespace); + MappedFieldType ft + = new RootFlatObjectFieldType(buildFullName(context), indexed, hasDocValues, meta, splitQueriesOnWhitespace, nullValue); if (eagerGlobalOrdinals) { ft.setEagerGlobalOrdinals(true); } @@ -319,6 +320,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S failIfNoDocValues(); return new KeyedFlatObjectFieldData.Builder(name(), key, CoreValuesSourceType.BYTES); } + + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); // TODO can we implement this? + } } /** @@ -432,13 +438,15 @@ public IndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService */ public static final class RootFlatObjectFieldType extends StringFieldType { private final boolean splitQueriesOnWhitespace; + private final String nullValue; public RootFlatObjectFieldType(String name, boolean indexed, boolean hasDocValues, Map meta, - boolean splitQueriesOnWhitespace) { + boolean splitQueriesOnWhitespace, String nullValue) { super(name, indexed, false, hasDocValues, splitQueriesOnWhitespace ? TextSearchInfo.WHITESPACE_MATCH_ONLY : TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.splitQueriesOnWhitespace = splitQueriesOnWhitespace; setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); + this.nullValue = nullValue; } @Override @@ -460,6 +468,19 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } + + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return new SourceValueFetcher(name(), mapperService, false, nullValue) { + @Override + protected Object parseSourceValue(Object value) { + return value; + } + }; + } } private FlatObjectFieldParser fieldParser; @@ -538,19 +559,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValue) { - @Override - protected Object parseSourceValue(Object value) { - return value; - } - }; - } - @Override protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { super.doXContentBody(builder, includeDefaults, params); diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldTypeTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldTypeTests.java new file mode 100644 index 0000000000000..155b7561131a5 --- /dev/null +++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldTypeTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.flattened.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class FlatObjectFieldTypeTests extends FieldTypeTestCase { + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + Map sourceValue = Map.of("key", "value"); + MappedFieldType mapper = new FlatObjectFieldMapper.Builder("field").build(context).fieldType(); + assertEquals(List.of(sourceValue), fetchSourceValue(mapper, sourceValue)); + + MappedFieldType nullValueMapper = new FlatObjectFieldMapper.Builder("field") + .nullValue("NULL") + .build(context) + .fieldType(); + assertEquals(List.of("NULL"), fetchSourceValue(nullValueMapper, null)); + } +} diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/RootFlatObjectFieldTypeTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/RootFlatObjectFieldTypeTests.java index 361fba64d9a45..8c590bf5da9b3 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/RootFlatObjectFieldTypeTests.java +++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/RootFlatObjectFieldTypeTests.java @@ -27,7 +27,7 @@ public class RootFlatObjectFieldTypeTests extends FieldTypeTestCase { private static RootFlatObjectFieldType createDefaultFieldType() { - return new RootFlatObjectFieldType("field", true, true, Collections.emptyMap(), false); + return new RootFlatObjectFieldType("field", true, true, Collections.emptyMap(), false, null); } public void testValueForDisplay() { @@ -46,22 +46,22 @@ public void testTermQuery() { expected = AutomatonQueries.caseInsensitiveTermQuery(new Term("field", "Value")); assertEquals(expected, ft.termQueryCaseInsensitive("Value", null)); - - + + RootFlatObjectFieldType unsearchable = new RootFlatObjectFieldType("field", false, true, - Collections.emptyMap(), false); + Collections.emptyMap(), false, null); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("field", null)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); } public void testExistsQuery() { - RootFlatObjectFieldType ft = new RootFlatObjectFieldType("field", true, false, Collections.emptyMap(), false); + RootFlatObjectFieldType ft = new RootFlatObjectFieldType("field", true, false, Collections.emptyMap(), false, null); assertEquals( new TermQuery(new Term(FieldNamesFieldMapper.NAME, new BytesRef("field"))), ft.existsQuery(null)); - RootFlatObjectFieldType withDv = new RootFlatObjectFieldType("field", true, true, Collections.emptyMap(), false); + RootFlatObjectFieldType withDv = new RootFlatObjectFieldType("field", true, true, Collections.emptyMap(), false, null); assertEquals(new DocValuesFieldExistsQuery("field"), withDv.existsQuery(null)); } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 0d06df9054a5a..c9b2923c6c618 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -109,11 +109,14 @@ protected List> getParameters() { @Override public UnsignedLongFieldMapper build(BuilderContext context) { + long parsed = parseUnsignedLong(nullValue); + Number nullValueFormatted = parsed >= 0 ? parsed : BigInteger.valueOf(parsed).and(BIGINTEGER_2_64_MINUS_ONE); UnsignedLongFieldType fieldType = new UnsignedLongFieldType( buildFullName(context), indexed.getValue(), stored.getValue(), hasDocValues.getValue(), + nullValueFormatted, meta.getValue() ); return new UnsignedLongFieldMapper(name, fieldType, multiFieldsBuilder.build(this, context), copyTo.build(), this); @@ -124,12 +127,16 @@ public UnsignedLongFieldMapper build(BuilderContext context) { public static final class UnsignedLongFieldType extends SimpleMappedFieldType { - public UnsignedLongFieldType(String name, boolean indexed, boolean isStored, boolean hasDocValues, Map meta) { + private final Number nullValueFormatted; + + public UnsignedLongFieldType(String name, boolean indexed, boolean isStored, boolean hasDocValues, + Number nullValueFormatted, Map meta) { super(name, indexed, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + this.nullValueFormatted = nullValueFormatted; } public UnsignedLongFieldType(String name) { - this(name, true, false, true, Collections.emptyMap()); + this(name, true, false, true, null, Collections.emptyMap()); } @Override @@ -208,6 +215,28 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S }; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + + return new SourceValueFetcher(name(), mapperService, false, nullValueFormatted) { + @Override + protected Object parseSourceValue(Object value) { + if (value.equals("")) { + return nullValueFormatted; + } + long ulValue = parseUnsignedLong(value); + if (ulValue >= 0) { + return ulValue; + } else { + return BigInteger.valueOf(ulValue).and(BIGINTEGER_2_64_MINUS_ONE); + } + } + }; + } + @Override public Object valueForDisplay(Object value) { if (value == null) { @@ -345,7 +374,6 @@ protected static Long parseUpperRangeTerm(Object value, boolean include) { private final boolean ignoreMalformedByDefault; private final String nullValue; private final Long nullValueIndexed; // null value to use for indexing, represented as shifted to signed long range - private final Number nullValueFormatted; // null value to use in place of a {@code null} value in the document source private UnsignedLongFieldMapper( String simpleName, @@ -363,11 +391,9 @@ private UnsignedLongFieldMapper( this.nullValue = builder.nullValue.getValue(); if (nullValue == null) { this.nullValueIndexed = null; - this.nullValueFormatted = null; } else { long parsed = parseUnsignedLong(nullValue); this.nullValueIndexed = unsignedToSortableSignedLong(parsed); - this.nullValueFormatted = parsed >= 0 ? parsed : BigInteger.valueOf(parsed).and(BIGINTEGER_2_64_MINUS_ONE); } } @@ -430,28 +456,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - - return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValueFormatted) { - @Override - protected Object parseSourceValue(Object value) { - if (value.equals("")) { - return nullValueFormatted; - } - long ulValue = parseUnsignedLong(value); - if (ulValue >= 0) { - return ulValue; - } else { - return BigInteger.valueOf(ulValue).and(BIGINTEGER_2_64_MINUS_ONE); - } - } - }; - } - @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), ignoreMalformedByDefault).init(this); diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index c03979337634e..d6409e1ae3f22 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -8,17 +8,12 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; @@ -33,7 +28,6 @@ import java.util.List; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.BIGINTEGER_2_64_MINUS_ONE; import static org.hamcrest.Matchers.containsString; public class UnsignedLongFieldMapperTests extends MapperTestCase { @@ -310,21 +304,6 @@ public void testIndexingOutOfRangeValues() throws Exception { } } - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - UnsignedLongFieldMapper mapper = new UnsignedLongFieldMapper.Builder("field", settings).build(context); - assertEquals(List.of(0L), fetchSourceValue(mapper, 0L)); - assertEquals(List.of(9223372036854775807L), fetchSourceValue(mapper, 9223372036854775807L)); - assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(mapper, "18446744073709551615")); - assertEquals(List.of(), fetchSourceValue(mapper, "")); - - UnsignedLongFieldMapper nullValueMapper = new UnsignedLongFieldMapper.Builder("field", settings).nullValue("18446744073709551615") - .build(context); - assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(nullValueMapper, "")); - } - public void testExistsQueryDocValuesDisabled() throws IOException { MapperService mapperService = createMapperService(fieldMapping(b -> { minimalMapping(b); diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java index 83fbcc98fdd68..364ecf6f49a91 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java @@ -8,12 +8,21 @@ import org.apache.lucene.document.LongPoint; import org.apache.lucene.search.MatchNoDocsQuery; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.UnsignedLongFieldType; + +import java.io.IOException; import java.util.List; -import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.UnsignedLongFieldType.parseTerm; +import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.BIGINTEGER_2_64_MINUS_ONE; import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.UnsignedLongFieldType.parseLowerRangeTerm; +import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.UnsignedLongFieldType.parseTerm; import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.UnsignedLongFieldType.parseUpperRangeTerm; public class UnsignedLongFieldTypeTests extends FieldTypeTestCase { @@ -47,7 +56,7 @@ public void testTermsQuery() { } public void testRangeQuery() { - UnsignedLongFieldType ft = new UnsignedLongFieldType("my_unsigned_long", true, false, false, null); + UnsignedLongFieldType ft = new UnsignedLongFieldType("my_unsigned_long", true, false, false, null, null); assertEquals( LongPoint.newRangeQuery("my_unsigned_long", -9223372036854775808L, -9223372036854775808L), @@ -148,4 +157,21 @@ public void testParseUpperTermForRangeQuery() { // wrongly formatted numbers expectThrows(NumberFormatException.class, () -> parseUpperRangeTerm("18incorrectnumber", true)); } + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + MappedFieldType mapper = new UnsignedLongFieldMapper.Builder("field", settings).build(context).fieldType(); + assertEquals(List.of(0L), fetchSourceValue(mapper, 0L)); + assertEquals(List.of(9223372036854775807L), fetchSourceValue(mapper, 9223372036854775807L)); + assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(mapper, "18446744073709551615")); + assertEquals(List.of(), fetchSourceValue(mapper, "")); + + MappedFieldType nullValueMapper = new UnsignedLongFieldMapper.Builder("field", settings) + .nullValue("18446744073709551615") + .build(context) + .fieldType(); + assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(nullValueMapper, "")); + } } diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index b297341dea38a..0c11354183e7d 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -132,6 +132,20 @@ public String typeName() { return CONTENT_TYPE; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + + return new SourceValueFetcher(name(), mapperService, false, null) { + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + }; + } + @Override public Query existsQuery(QueryShardContext context) { return new DocValuesFieldExistsQuery(name()); @@ -315,20 +329,6 @@ public VersionStringFieldType fieldType() { return (VersionStringFieldType) super.fieldType(); } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - - return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), null) { - @Override - protected String parseSourceValue(Object value) { - return value.toString(); - } - }; - } - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index 746dc680f6d21..94ef21682d598 100644 --- a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -10,17 +10,12 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; @@ -30,7 +25,6 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; -import java.util.List; import static org.hamcrest.Matchers.equalTo; @@ -133,17 +127,4 @@ public void testFailsParsingNestedList() throws IOException { ex.getMessage() ); } - - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - VersionStringFieldMapper mapper = new VersionStringFieldMapper.Builder("field").build(context); - assertEquals(List.of("value"), fetchSourceValue(mapper, "value")); - assertEquals(List.of("42"), fetchSourceValue(mapper, 42L)); - assertEquals(List.of("true"), fetchSourceValue(mapper, true)); - - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> fetchSourceValue(mapper, "value", "format")); - assertEquals("Field [field] of type [version] doesn't support formats.", e.getMessage()); - } } diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTypeTests.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTypeTests.java new file mode 100644 index 0000000000000..7e4dc90538e65 --- /dev/null +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTypeTests.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; + +import java.io.IOException; +import java.util.List; + +public class VersionStringFieldTypeTests extends FieldTypeTestCase { + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + MappedFieldType mapper = new VersionStringFieldMapper.Builder("field").build(context).fieldType(); + assertEquals(List.of("value"), fetchSourceValue(mapper, "value")); + assertEquals(List.of("42"), fetchSourceValue(mapper, 42L)); + assertEquals(List.of("true"), fetchSourceValue(mapper, true)); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> fetchSourceValue(mapper, "value", "format")); + assertEquals("Field [field] of type [version] doesn't support formats.", e.getMessage()); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldType.java index cdfee2cd92bc4..663b78398257e 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldType.java @@ -16,13 +16,15 @@ import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.index.mapper.DocValueFetcher; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.Script; import org.elasticsearch.search.lookup.SearchLookup; -import java.io.IOException; import java.time.ZoneId; import java.util.Locale; import java.util.Map; @@ -151,17 +153,17 @@ public Query regexpQuery( } @Override - public Query phraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) throws IOException { + public Query phraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) { throw new IllegalArgumentException(unsupported("phrase", "text")); } @Override - public Query multiPhraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) throws IOException { + public Query multiPhraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) { throw new IllegalArgumentException(unsupported("phrase", "text")); } @Override - public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions) throws IOException { + public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions) { throw new IllegalArgumentException(unsupported("phrase prefix", "text")); } @@ -210,4 +212,9 @@ protected String format() { protected Locale formatLocale() { return null; } + + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { + return new DocValueFetcher(docValueFormat(format, null), lookup.doc().getForField(this)); + } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java index 5f8f3e102ad25..91e1a06148d67 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java @@ -10,20 +10,16 @@ import org.elasticsearch.common.util.LocaleUtils; import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.DocValueFetcher; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptType; -import org.elasticsearch.search.lookup.SearchLookup; import java.util.List; import java.util.Locale; @@ -78,11 +74,6 @@ protected void parseCreateField(ParseContext context) { // there is no lucene field } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { - return new DocValueFetcher(fieldType().docValueFormat(format, null), lookup.doc().getForField(fieldType())); - } - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java index 1dcf312c8c6d9..4e896ebdac557 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java @@ -6,17 +6,12 @@ package org.elasticsearch.xpack.spatial.index.mapper; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.mapper.AbstractGeometryFieldMapper; -import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.ParsedDocument; @@ -25,12 +20,9 @@ import org.hamcrest.CoreMatchers; import java.io.IOException; -import java.util.List; -import java.util.Map; import static org.elasticsearch.index.mapper.AbstractPointGeometryFieldMapper.Names.IGNORE_Z_VALUE; import static org.elasticsearch.index.mapper.AbstractPointGeometryFieldMapper.Names.NULL_VALUE; -import static org.elasticsearch.index.mapper.FieldMapperTestCase.fetchSourceValue; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; @@ -304,35 +296,4 @@ public void testIgnoreZValue() throws IOException { assertThat(ignoreZValue, equalTo(false)); } - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - AbstractGeometryFieldMapper mapper = new PointFieldMapper.Builder("field").build(context); - - Map jsonPoint = Map.of("type", "Point", "coordinates", List.of(42.0, 27.1)); - String wktPoint = "POINT (42.0 27.1)"; - Map otherJsonPoint = Map.of("type", "Point", "coordinates", List.of(30.0, 50.0)); - String otherWktPoint = "POINT (30.0 50.0)"; - - // Test a single point in [x, y] array format. - Object sourceValue = List.of(42.0, 27.1); - assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); - - // Test a single point in "x, y" string format. - sourceValue = "42.0,27.1"; - assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); - - // Test a list of points in [x, y] array format. - sourceValue = List.of(List.of(42.0, 27.1), List.of(30.0, 50.0)); - assertEquals(List.of(jsonPoint, otherJsonPoint), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktPoint, otherWktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); - - // Test a single point in well-known text format. - sourceValue = "POINT (42.0 27.1)"; - assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); - } } diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldTypeTests.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldTypeTests.java new file mode 100644 index 0000000000000..f42e8b1ba9540 --- /dev/null +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldTypeTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.index.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class PointFieldTypeTests extends FieldTypeTestCase { + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + MappedFieldType mapper = new PointFieldMapper.Builder("field").build(context).fieldType(); + + Map jsonPoint = Map.of("type", "Point", "coordinates", List.of(42.0, 27.1)); + String wktPoint = "POINT (42.0 27.1)"; + Map otherJsonPoint = Map.of("type", "Point", "coordinates", List.of(30.0, 50.0)); + String otherWktPoint = "POINT (30.0 50.0)"; + + // Test a single point in [x, y] array format. + Object sourceValue = List.of(42.0, 27.1); + assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a single point in "x, y" string format. + sourceValue = "42.0,27.1"; + assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a list of points in [x, y] array format. + sourceValue = List.of(List.of(42.0, 27.1), List.of(30.0, 50.0)); + assertEquals(List.of(jsonPoint, otherJsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktPoint, otherWktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a single point in well-known text format. + sourceValue = "POINT (42.0 27.1)"; + assertEquals(List.of(jsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + } +} diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java index 47f1d62459a56..d5c1842a627ca 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java @@ -6,20 +6,16 @@ package org.elasticsearch.xpack.spatial.index.mapper; import org.apache.lucene.index.IndexableField; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.AbstractShapeGeometryFieldMapper; -import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.Mapper; @@ -29,11 +25,8 @@ import java.io.IOException; import java.util.Collections; -import java.util.List; -import java.util.Map; import static org.elasticsearch.index.mapper.AbstractPointGeometryFieldMapper.Names.IGNORE_Z_VALUE; -import static org.elasticsearch.index.mapper.FieldMapperTestCase.fetchSourceValue; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; @@ -147,7 +140,6 @@ public void testCoerceParsing() throws IOException { coerce = ((ShapeFieldMapper)fieldMapper).coerce().value(); assertThat(coerce, equalTo(false)); - assertFieldWarnings("tree"); } @@ -224,15 +216,6 @@ public void testIgnoreMalformedParsing() throws IOException { assertThat(ignoreMalformed.value(), equalTo(false)); } - - private void assertFieldWarnings(String... fieldNames) { - String[] warnings = new String[fieldNames.length]; - for (int i = 0; i < fieldNames.length; ++i) { - warnings[i] = "Field parameter [" + fieldNames[i] + "] " - + "is deprecated and will be removed in a future version."; - } - } - public void testShapeMapperMerge() throws Exception { String stage1Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("shape").field("type", "shape") @@ -331,37 +314,4 @@ public String toXContentString(ShapeFieldMapper mapper, boolean includeDefaults) public String toXContentString(ShapeFieldMapper mapper) throws IOException { return toXContentString(mapper, true); } - - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - ShapeFieldMapper mapper = new ShapeFieldMapper.Builder("field").build(context); - - Map jsonLineString = Map.of("type", "LineString", "coordinates", - List.of(List.of(42.0, 27.1), List.of(30.0, 50.0))); - Map jsonPoint = Map.of("type", "Point", "coordinates", List.of(14.3, 15.0)); - String wktLineString = "LINESTRING (42.0 27.1, 30.0 50.0)"; - String wktPoint = "POINT (14.3 15.0)"; - - // Test a single shape in geojson format. - Object sourceValue = jsonLineString; - assertEquals(List.of(jsonLineString), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktLineString), fetchSourceValue(mapper, sourceValue, "wkt")); - - // Test a list of shapes in geojson format. - sourceValue = List.of(jsonLineString, jsonPoint); - assertEquals(List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); - - // Test a single shape in wkt format. - sourceValue = wktLineString; - assertEquals(List.of(jsonLineString), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktLineString), fetchSourceValue(mapper, sourceValue, "wkt")); - - // Test a list of shapes in wkt format. - sourceValue = List.of(wktLineString, wktPoint); - assertEquals(List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null)); - assertEquals(List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); - } } diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldTypeTests.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldTypeTests.java new file mode 100644 index 0000000000000..3eef007c85f30 --- /dev/null +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldTypeTests.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.index.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class ShapeFieldTypeTests extends FieldTypeTestCase { + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + MappedFieldType mapper = new ShapeFieldMapper.Builder("field").build(context).fieldType(); + + Map jsonLineString = Map.of("type", "LineString", "coordinates", + List.of(List.of(42.0, 27.1), List.of(30.0, 50.0))); + Map jsonPoint = Map.of("type", "Point", "coordinates", List.of(14.3, 15.0)); + String wktLineString = "LINESTRING (42.0 27.1, 30.0 50.0)"; + String wktPoint = "POINT (14.3 15.0)"; + + // Test a single shape in geojson format. + Object sourceValue = jsonLineString; + assertEquals(List.of(jsonLineString), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktLineString), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a list of shapes in geojson format. + sourceValue = List.of(jsonLineString, jsonPoint); + assertEquals(List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a single shape in wkt format. + sourceValue = wktLineString; + assertEquals(List.of(jsonLineString), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktLineString), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a list of shapes in wkt format. + sourceValue = List.of(wktLineString, wktPoint); + assertEquals(List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + } +} diff --git a/x-pack/plugin/vectors/src/internalClusterTest/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java b/x-pack/plugin/vectors/src/internalClusterTest/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java index c27f052f5a643..039240ee8f5f6 100644 --- a/x-pack/plugin/vectors/src/internalClusterTest/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java +++ b/x-pack/plugin/vectors/src/internalClusterTest/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; @@ -46,7 +47,7 @@ protected boolean forbidPrivateIndexSettings() { public void testValueFetcherIsNotSupported() { SparseVectorFieldMapper.Builder builder = new SparseVectorFieldMapper.Builder("field"); - SparseVectorFieldMapper fieldMapper = builder.build(new Mapper.BuilderContext(Settings.EMPTY, new ContentPath())); + MappedFieldType fieldMapper = builder.build(new Mapper.BuilderContext(Settings.EMPTY, new ContentPath())).fieldType(); UnsupportedOperationException exc = expectThrows(UnsupportedOperationException.class, () -> fieldMapper.valueFetcher(null, null, null)); assertEquals(SparseVectorFieldMapper.ERROR_MESSAGE_7X, exc.getMessage()); diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index d0087f4c89120..9121d74928c6a 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -117,6 +117,19 @@ public String typeName() { return CONTENT_TYPE; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return new SourceValueFetcher(name(), mapperService, true) { + @Override + protected Object parseSourceValue(Object value) { + return value; + } + }; + } + @Override public DocValueFormat docValueFormat(String format, ZoneId timeZone) { throw new UnsupportedOperationException( @@ -209,19 +222,6 @@ public void parse(ParseContext context) throws IOException { context.doc().addWithKey(fieldType().name(), field); } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { - @Override - protected Object parseSourceValue(Object value) { - return value; - } - }; - } - @Override protected boolean indexedByDefault() { return false; diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java index 0c3f8aba1994f..186b8fe3c3697 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java @@ -99,6 +99,11 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { throw new UnsupportedOperationException(ERROR_MESSAGE_7X); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(ERROR_MESSAGE_7X); + } + @Override public Query existsQuery(QueryShardContext context) { throw new UnsupportedOperationException(ERROR_MESSAGE_7X); @@ -142,11 +147,6 @@ protected void parseCreateField(ParseContext context) { throw new IllegalStateException("parse is implemented directly"); } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException(ERROR_MESSAGE_7X); - } - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index d08d809061ea0..125414b4500b7 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -172,8 +172,10 @@ public Builder ignoreAbove(int ignoreAbove) { @Override public WildcardFieldMapper build(BuilderContext context) { return new WildcardFieldMapper( - name, fieldType, new WildcardFieldType(buildFullName(context), fieldType, meta), ignoreAbove, - multiFieldsBuilder.build(this, context), copyTo, nullValue); + name, fieldType, + new WildcardFieldType(buildFullName(context), fieldType, nullValue, ignoreAbove, meta), + ignoreAbove, + multiFieldsBuilder.build(this, context), copyTo, nullValue); } } @@ -212,10 +214,15 @@ public static final class WildcardFieldType extends MappedFieldType { static Analyzer lowercaseNormalizer = new LowercaseNormalizer(); - private WildcardFieldType(String name, FieldType fieldType, Map meta) { + private final String nullValue; + private final int ignoreAbove; + + private WildcardFieldType(String name, FieldType fieldType, String nullValue, int ignoreAbove, Map meta) { super(name, true, fieldType.stored(), true, new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), meta); setIndexAnalyzer(WILDCARD_ANALYZER); + this.nullValue = nullValue; + this.ignoreAbove = ignoreAbove; } @Override @@ -897,6 +904,24 @@ public IndexFieldData build( }; } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + + return new SourceValueFetcher(name(), mapperService, false, nullValue) { + @Override + protected String parseSourceValue(Object value) { + String keywordValue = value.toString(); + if (keywordValue.length() > ignoreAbove) { + return null; + } + return keywordValue; + } + }; + } + } private int ignoreAbove; @@ -963,24 +988,6 @@ protected void parseCreateField(ParseContext context) throws IOException { parseDoc.addAll(fields); } - @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - - return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValue) { - @Override - protected String parseSourceValue(Object value) { - String keywordValue = value.toString(); - if (keywordValue.length() > ignoreAbove) { - return null; - } - return keywordValue; - } - }; - } - void createFields(String value, Document parseDoc, Listfields) throws IOException { if (value == null || value.length() > ignoreAbove) { return; diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index fbeca28b9af0b..0d57a5257d593 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -37,7 +37,6 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.ByteRunAutomaton; -//import org.apache.lucene.util.automaton.RegExp; import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -68,14 +67,14 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; -import java.util.List; import java.util.function.Supplier; -import static org.elasticsearch.index.mapper.FieldMapperTestCase.fetchSourceValue; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +//import org.apache.lucene.util.automaton.RegExp; + public class WildcardFieldMapperTests extends ESTestCase { static QueryShardContext createMockQueryShardContext(boolean allowExpensiveQueries) { @@ -179,8 +178,8 @@ public void testTooBigQueryField() throws IOException { reader.close(); dir.close(); } - - + + public void testTermAndPrefixQueryIgnoreWildcardSyntax() throws IOException { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(WildcardFieldMapper.WILDCARD_ANALYZER); @@ -196,7 +195,7 @@ public void testTermAndPrefixQueryIgnoreWildcardSyntax() throws IOException { DirectoryReader reader = iw.getReader(); IndexSearcher searcher = newSearcher(reader); iw.close(); - + expectTermMatch(searcher, "f*oo*", 0); expectTermMatch(searcher, "f*oo?", 1); expectTermMatch(searcher, "*oo?", 0); @@ -204,22 +203,22 @@ public void testTermAndPrefixQueryIgnoreWildcardSyntax() throws IOException { expectPrefixMatch(searcher, "f*o", 1); expectPrefixMatch(searcher, "f*oo?", 1); expectPrefixMatch(searcher, "f??o", 0); - + reader.close(); dir.close(); } - + private void expectTermMatch(IndexSearcher searcher, String term,long count) throws IOException { Query q = wildcardFieldType.fieldType().termQuery(term, MOCK_QSC); TopDocs td = searcher.search(q, 10, Sort.RELEVANCE); - assertThat(td.totalHits.value, equalTo(count)); + assertThat(td.totalHits.value, equalTo(count)); } - + private void expectPrefixMatch(IndexSearcher searcher, String term,long count) throws IOException { Query q = wildcardFieldType.fieldType().prefixQuery(term, null, MOCK_QSC); TopDocs td = searcher.search(q, 10, Sort.RELEVANCE); - assertThat(td.totalHits.value, equalTo(count)); - } + assertThat(td.totalHits.value, equalTo(count)); + } public void testSearchResultsVersusKeywordField() throws IOException { @@ -823,28 +822,6 @@ protected String convertToRandomRegex(String randomValue) { return result.toString(); } - public void testFetchSourceValue() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - - WildcardFieldMapper mapper = new WildcardFieldMapper.Builder("field").build(context); - assertEquals(List.of("value"), fetchSourceValue(mapper, "value")); - assertEquals(List.of("42"), fetchSourceValue(mapper, 42L)); - assertEquals(List.of("true"), fetchSourceValue(mapper, true)); - - WildcardFieldMapper ignoreAboveMapper = new WildcardFieldMapper.Builder("field") - .ignoreAbove(4) - .build(context); - assertEquals(List.of(), fetchSourceValue(ignoreAboveMapper, "value")); - assertEquals(List.of("42"), fetchSourceValue(ignoreAboveMapper, 42L)); - assertEquals(List.of("true"), fetchSourceValue(ignoreAboveMapper, true)); - - WildcardFieldMapper nullValueMapper = new WildcardFieldMapper.Builder("field") - .nullValue("NULL") - .build(context); - assertEquals(List.of("NULL"), fetchSourceValue(nullValueMapper, null)); - } - protected MappedFieldType provideMappedFieldType(String name) { if (name.equals(WILDCARD_FIELD_NAME)) { return wildcardFieldType.fieldType(); diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java new file mode 100644 index 0000000000000..63d8e70e65fb2 --- /dev/null +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.wildcard.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; + +import java.io.IOException; +import java.util.List; + +public class WildcardFieldTypeTests extends FieldTypeTestCase { + + public void testFetchSourceValue() throws IOException { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + MappedFieldType mapper = new WildcardFieldMapper.Builder("field").build(context).fieldType(); + assertEquals(List.of("value"), fetchSourceValue(mapper, "value")); + assertEquals(List.of("42"), fetchSourceValue(mapper, 42L)); + assertEquals(List.of("true"), fetchSourceValue(mapper, true)); + + MappedFieldType ignoreAboveMapper = new WildcardFieldMapper.Builder("field") + .ignoreAbove(4) + .build(context) + .fieldType(); + assertEquals(List.of(), fetchSourceValue(ignoreAboveMapper, "value")); + assertEquals(List.of("42"), fetchSourceValue(ignoreAboveMapper, 42L)); + assertEquals(List.of("true"), fetchSourceValue(ignoreAboveMapper, true)); + + MappedFieldType nullValueMapper = new WildcardFieldMapper.Builder("field") + .nullValue("NULL") + .build(context) + .fieldType(); + assertEquals(List.of("NULL"), fetchSourceValue(nullValueMapper, null)); + } +} From d450600079f018d18f130178bad6bc63546daf99 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 28 Sep 2020 18:35:55 +0100 Subject: [PATCH 3/8] tests --- .../index/mapper/AbstractGeometryFieldMapper.java | 7 +++++-- .../index/mapper/AbstractPointGeometryFieldMapper.java | 2 +- .../index/mapper/AbstractShapeGeometryFieldMapper.java | 4 ++-- .../index/mapper/GeoShapeFieldMapper.java | 2 +- .../elasticsearch/index/mapper/KeywordFieldMapper.java | 4 +++- .../index/mapper/LegacyGeoShapeFieldMapper.java | 2 +- .../index/mapper/BooleanFieldTypeTests.java | 2 +- .../index/mapper/RangeFieldTypeTests.java | 2 +- .../xpack/unsignedlong/UnsignedLongFieldMapper.java | 10 ++++++++-- .../xpack/unsignedlong/UnsignedLongFieldTypeTests.java | 3 +-- .../xpack/spatial/index/mapper/ShapeFieldMapper.java | 2 +- 11 files changed, 25 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 358205d9ed3ca..fd0c7faa62498 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -223,9 +223,12 @@ public abstract static class AbstractGeometryFieldType extend protected Indexer geometryIndexer; protected Parser geometryParser; protected QueryProcessor geometryQueryBuilder; + protected final boolean parsesArrayValue; - protected AbstractGeometryFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues, Map meta) { + protected AbstractGeometryFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues, + boolean parsesArrayValue, Map meta) { super(name, indexed, stored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + this.parsesArrayValue = parsesArrayValue; } public void setGeometryQueryBuilder(QueryProcessor geometryQueryBuilder) { @@ -278,7 +281,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc Function valueParser = value -> geometryParser.parseAndFormatObject(value, geoFormat); - return new SourceValueFetcher(name(), mapperService, true) { + return new SourceValueFetcher(name(), mapperService, parsesArrayValue) { @Override protected Object parseSourceValue(Object value) { return valueParser.apply(value); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java index df1773916f132..089429c53b4f0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java @@ -119,7 +119,7 @@ public abstract static class AbstractPointGeometryFieldType extends AbstractGeometryFieldType { protected AbstractPointGeometryFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues, Map meta) { - super(name, indexed, stored, hasDocValues, meta); + super(name, indexed, stored, hasDocValues, true, meta); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java index cc14a6358a308..0c20236230c3c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java @@ -157,8 +157,8 @@ public abstract static class AbstractShapeGeometryFieldType e protected Orientation orientation = Defaults.ORIENTATION.value(); protected AbstractShapeGeometryFieldType(String name, boolean isSearchable, boolean isStored, boolean hasDocValues, - Map meta) { - super(name, isSearchable, isStored, hasDocValues, meta); + boolean parsesArrayValue, Map meta) { + super(name, isSearchable, isStored, hasDocValues, parsesArrayValue, meta); } public Orientation orientation() { return this.orientation; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 798eee3953197..be1e966325ebc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -88,7 +88,7 @@ public GeoShapeFieldMapper build(BuilderContext context) { public static class GeoShapeFieldType extends AbstractShapeGeometryFieldType { public GeoShapeFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues, Map meta) { - super(name, indexed, stored, hasDocValues, meta); + super(name, indexed, stored, hasDocValues, false, meta); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 33234ac68fa8d..55426c34d99a1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -224,7 +224,9 @@ public KeywordFieldType(String name) { public KeywordFieldType(String name, FieldType fieldType) { super(name, fieldType.indexOptions() != IndexOptions.NONE, - false, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + false, false, + new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), + Collections.emptyMap()); this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java index 06280fd14389b..da7a8c0056271 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -323,7 +323,7 @@ public static final class GeoShapeFieldType extends AbstractShapeGeometryFieldTy private TermQueryPrefixTreeStrategy termStrategy; private GeoShapeFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues, Map meta) { - super(name, indexed, stored, hasDocValues, meta); + super(name, indexed, stored, hasDocValues, false, meta); } public GeoShapeFieldType(String name) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java index c4a92fe0d3896..a6b291f90a20f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java @@ -61,7 +61,7 @@ public void testFetchSourceValue() throws IOException { assertEquals(List.of(false), fetchSourceValue(fieldType, "")); MappedFieldType nullFieldType = new BooleanFieldMapper.BooleanFieldType( - "field", true, false, true, false, Collections.emptyMap() + "field", true, false, true, true, Collections.emptyMap() ); assertEquals(List.of(true), fetchSourceValue(nullFieldType, null)); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index ef17a3bc3371f..1e67d7fdb0046 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -260,7 +260,7 @@ public void testDateVsDateRangeBounds() { long lower = randomLongBetween(formatter.parseMillis("2000-01-01T00:00"), formatter.parseMillis("2010-01-01T00:00")); long upper = randomLongBetween(formatter.parseMillis("2011-01-01T00:00"), formatter.parseMillis("2020-01-01T00:00")); - RangeFieldType fieldType = new RangeFieldType("field", formatter); + RangeFieldType fieldType = new RangeFieldType("field", true, false, false, formatter, false, null); String lowerAsString = formatter.formatMillis(lower); String upperAsString = formatter.formatMillis(upper); // also add date math rounding to days occasionally diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index c9b2923c6c618..2de3ba36bd54f 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -129,8 +129,14 @@ public static final class UnsignedLongFieldType extends SimpleMappedFieldType { private final Number nullValueFormatted; - public UnsignedLongFieldType(String name, boolean indexed, boolean isStored, boolean hasDocValues, - Number nullValueFormatted, Map meta) { + public UnsignedLongFieldType( + String name, + boolean indexed, + boolean isStored, + boolean hasDocValues, + Number nullValueFormatted, + Map meta + ) { super(name, indexed, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.nullValueFormatted = nullValueFormatted; } diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java index 364ecf6f49a91..516e3522df8e5 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java @@ -168,8 +168,7 @@ public void testFetchSourceValue() throws IOException { assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(mapper, "18446744073709551615")); assertEquals(List.of(), fetchSourceValue(mapper, "")); - MappedFieldType nullValueMapper = new UnsignedLongFieldMapper.Builder("field", settings) - .nullValue("18446744073709551615") + MappedFieldType nullValueMapper = new UnsignedLongFieldMapper.Builder("field", settings).nullValue("18446744073709551615") .build(context) .fieldType(); assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(nullValueMapper, "")); diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java index d32f42aa0697c..0b05798eb0b82 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java @@ -78,7 +78,7 @@ public Builder newBuilder(String name, Map params) { public static final class ShapeFieldType extends AbstractShapeGeometryFieldType { public ShapeFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues, Map meta) { - super(name, indexed, stored, hasDocValues, meta); + super(name, indexed, stored, hasDocValues, false, meta); } @Override From 1d4ec994bddc44f59d835cd695fe20b7a7724580 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 29 Sep 2020 08:58:59 +0100 Subject: [PATCH 4/8] test jar hell; unsigned long null values --- .../mapper/ConstantKeywordFieldTypeTests.java | 36 ------------------- .../mapper/ConstantKeywordFieldTypeTests.java | 22 ++++++++++++ .../unsignedlong/UnsignedLongFieldMapper.java | 12 +++++-- .../UnsignedLongFieldTypeTests.java | 3 +- 4 files changed, 33 insertions(+), 40 deletions(-) delete mode 100644 x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java diff --git a/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java b/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java deleted file mode 100644 index 5250f10456e46..0000000000000 --- a/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.constantkeyword.mapper; - -import org.elasticsearch.index.mapper.FieldTypeTestCase; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.ValueFetcher; -import org.elasticsearch.search.lookup.SourceLookup; - -import java.util.Collections; -import java.util.List; - -public class ConstantKeywordFieldTypeTests extends FieldTypeTestCase { - - public void testFetchValue() throws Exception { - MappedFieldType fieldType = new ConstantKeywordFieldMapper.ConstantKeywordFieldType("field", null); - ValueFetcher fetcher = fieldType.valueFetcher(null, null, null); - - SourceLookup missingValueLookup = new SourceLookup(); - SourceLookup nullValueLookup = new SourceLookup(); - nullValueLookup.setSource(Collections.singletonMap("field", null)); - - assertTrue(fetcher.fetchValues(missingValueLookup).isEmpty()); - assertTrue(fetcher.fetchValues(nullValueLookup).isEmpty()); - - MappedFieldType valued = new ConstantKeywordFieldMapper.ConstantKeywordFieldType("field", null); - fetcher = valued.valueFetcher(null, null, null); - - assertEquals(List.of("foo"), fetcher.fetchValues(missingValueLookup)); - assertEquals(List.of("foo"), fetcher.fetchValues(nullValueLookup)); - } -} diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java index 93fea8dbfad20..96c55aa33ded8 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java @@ -11,10 +11,14 @@ import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xpack.constantkeyword.mapper.ConstantKeywordFieldMapper.ConstantKeywordFieldType; import java.util.Arrays; import java.util.Collections; +import java.util.List; public class ConstantKeywordFieldTypeTests extends FieldTypeTestCase { @@ -100,4 +104,22 @@ public void testRegexpQuery() { assertEquals(new MatchAllDocsQuery(), ft.regexpQuery("f.o", RegExp.ALL, 0, 10, null, null)); assertEquals(new MatchNoDocsQuery(), ft.regexpQuery("f..o", RegExp.ALL, 0, 10, null, null)); } + + public void testFetchValue() throws Exception { + MappedFieldType fieldType = new ConstantKeywordFieldMapper.ConstantKeywordFieldType("field", null); + ValueFetcher fetcher = fieldType.valueFetcher(null, null, null); + + SourceLookup missingValueLookup = new SourceLookup(); + SourceLookup nullValueLookup = new SourceLookup(); + nullValueLookup.setSource(Collections.singletonMap("field", null)); + + assertTrue(fetcher.fetchValues(missingValueLookup).isEmpty()); + assertTrue(fetcher.fetchValues(nullValueLookup).isEmpty()); + + MappedFieldType valued = new ConstantKeywordFieldMapper.ConstantKeywordFieldType("field", null); + fetcher = valued.valueFetcher(null, null, null); + + assertEquals(List.of("foo"), fetcher.fetchValues(missingValueLookup)); + assertEquals(List.of("foo"), fetcher.fetchValues(nullValueLookup)); + } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 2de3ba36bd54f..fda5dc00e511c 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -107,16 +107,22 @@ protected List> getParameters() { return List.of(indexed, hasDocValues, stored, ignoreMalformed, nullValue, meta); } + Number parsedNullValue() { + if (nullValue.getValue() == null) { + return null; + } + long parsed = parseUnsignedLong(nullValue.getValue()); + return parsed >= 0 ? parsed : BigInteger.valueOf(parsed).and(BIGINTEGER_2_64_MINUS_ONE); + } + @Override public UnsignedLongFieldMapper build(BuilderContext context) { - long parsed = parseUnsignedLong(nullValue); - Number nullValueFormatted = parsed >= 0 ? parsed : BigInteger.valueOf(parsed).and(BIGINTEGER_2_64_MINUS_ONE); UnsignedLongFieldType fieldType = new UnsignedLongFieldType( buildFullName(context), indexed.getValue(), stored.getValue(), hasDocValues.getValue(), - nullValueFormatted, + parsedNullValue(), meta.getValue() ); return new UnsignedLongFieldMapper(name, fieldType, multiFieldsBuilder.build(this, context), copyTo.build(), this); diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java index 516e3522df8e5..364ecf6f49a91 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java @@ -168,7 +168,8 @@ public void testFetchSourceValue() throws IOException { assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(mapper, "18446744073709551615")); assertEquals(List.of(), fetchSourceValue(mapper, "")); - MappedFieldType nullValueMapper = new UnsignedLongFieldMapper.Builder("field", settings).nullValue("18446744073709551615") + MappedFieldType nullValueMapper = new UnsignedLongFieldMapper.Builder("field", settings) + .nullValue("18446744073709551615") .build(context) .fieldType(); assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(nullValueMapper, "")); From 4cfc51ddd1772c18e3100d9988942dd851f2744e Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 29 Sep 2020 09:05:01 +0100 Subject: [PATCH 5/8] spotless --- .../xpack/unsignedlong/UnsignedLongFieldTypeTests.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java index 364ecf6f49a91..516e3522df8e5 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java @@ -168,8 +168,7 @@ public void testFetchSourceValue() throws IOException { assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(mapper, "18446744073709551615")); assertEquals(List.of(), fetchSourceValue(mapper, "")); - MappedFieldType nullValueMapper = new UnsignedLongFieldMapper.Builder("field", settings) - .nullValue("18446744073709551615") + MappedFieldType nullValueMapper = new UnsignedLongFieldMapper.Builder("field", settings).nullValue("18446744073709551615") .build(context) .fieldType(); assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(nullValueMapper, "")); From 3a7494a436e07f31cdcf081c1f5acb4f2c633ccd Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 29 Sep 2020 09:34:43 +0100 Subject: [PATCH 6/8] const keyword tests --- .../constantkeyword/mapper/ConstantKeywordFieldTypeTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java index 96c55aa33ded8..ca8edcbfb30e9 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java @@ -116,7 +116,7 @@ public void testFetchValue() throws Exception { assertTrue(fetcher.fetchValues(missingValueLookup).isEmpty()); assertTrue(fetcher.fetchValues(nullValueLookup).isEmpty()); - MappedFieldType valued = new ConstantKeywordFieldMapper.ConstantKeywordFieldType("field", null); + MappedFieldType valued = new ConstantKeywordFieldMapper.ConstantKeywordFieldType("field", "foo"); fetcher = valued.valueFetcher(null, null, null); assertEquals(List.of("foo"), fetcher.fetchValues(missingValueLookup)); From 477b48bc93e4cbd9c921f29bf52c9113a30a8639 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 29 Sep 2020 17:23:14 +0100 Subject: [PATCH 7/8] feedback --- .../index/mapper/SearchAsYouTypeFieldMapper.java | 10 +--------- .../index/mapper/murmur3/Murmur3FieldMapper.java | 2 +- .../elasticsearch/index/mapper/IndexFieldMapper.java | 2 +- .../elasticsearch/index/mapper/SeqNoFieldMapper.java | 3 +-- .../elasticsearch/index/mapper/TextFieldMapper.java | 4 ++-- .../elasticsearch/index/mapper/TypeFieldMapper.java | 2 +- .../elasticsearch/index/mapper/VersionFieldMapper.java | 3 +-- 7 files changed, 8 insertions(+), 18 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index 78bc37c52d23b..eefa7bdb84640 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -272,15 +272,7 @@ private ShingleFieldType shingleFieldForPositions(int positions) { @Override public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - return new SourceValueFetcher(name(), mapperService, false) { - @Override - protected Object parseSourceValue(Object value) { - return value.toString(); - } - }; + throw new UnsupportedOperationException(); } @Override diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index ffe0f5fca0b04..2220a3489437b 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -159,5 +159,5 @@ protected void parseCreateField(ParseContext context) } } } - + } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index 3c3bf52b0047c..00be3a4b7c2ec 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -74,7 +74,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S @Override public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - return lookup -> Collections.singletonList(mapperService.index().getName()); + throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index 6c43cb2a0d051..61ee5dba24de2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -32,7 +32,6 @@ import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.seqno.SequenceNumbers; -import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; @@ -124,7 +123,7 @@ private long parse(Object value) { @Override public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { - return new DocValueFetcher(DocValueFormat.RAW, lookup.doc().getForField(this)); + throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index ded2ec5dcf850..9db679091080a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -396,7 +396,7 @@ public String typeName() { @Override public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - return parent.valueFetcher(mapperService, searchLookup, format); + throw new UnsupportedOperationException(); } @Override @@ -422,7 +422,7 @@ static final class PrefixFieldType extends StringFieldType { @Override public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - return parentField.valueFetcher(mapperService, searchLookup, format); + throw new UnsupportedOperationException(); } static boolean canMerge(PrefixFieldType first, PrefixFieldType second) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java index 691675692c1e0..29b47aef15346 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java @@ -78,7 +78,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S @Override public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - return lookup -> Collections.singletonList(mapperService.documentMapper().type()); + throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java index ab0d1ea60b35d..563e25b0c2f73 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java @@ -25,7 +25,6 @@ import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; -import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.lookup.SearchLookup; import java.util.Collections; @@ -58,7 +57,7 @@ public Query termQuery(Object value, QueryShardContext context) { @Override public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { - return new DocValueFetcher(DocValueFormat.RAW, lookup.doc().getForField(this)); + throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } } From adc8b9a315fef4cc37fe7fcc3ba4ebe9af56015b Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 30 Sep 2020 09:07:51 +0100 Subject: [PATCH 8/8] compilation --- .../org/elasticsearch/index/mapper/TokenCountFieldMapper.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index e3e7df230337f..e09808a81a7ae 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -78,6 +78,8 @@ public TokenCountFieldMapper build(BuilderContext context) { index.getValue(), store.getValue(), hasDocValues.getValue(), + false, + nullValue.getValue(), meta.getValue()); return new TokenCountFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this); }