From 4a587de9a6d581ebb59eae3f8257000972555968 Mon Sep 17 00:00:00 2001 From: Christos Soulios <1561376+csoulios@users.noreply.github.com> Date: Fri, 18 Sep 2020 09:45:32 +0300 Subject: [PATCH 1/2] Allow metadata fields in the _source (#61590) * Configurable metadata field mappers in the _source * Changes to support metadata fields in _source Added test testDocumentContainsAllowedMetadataField() * Merged DocumentParserTests from master Fixed broken tests * Handle non string values * Allow metadata fields to parse values/objects/arrays/null * Removed MetadataFieldMapper.isAllowedInSource() method Delegated this functionality to MetadataFieldMapper.parse() * Fixed bug that caused tests to break * Cleanup parsing for existing metadata fields * Cleanup parsing for existing metadata fields * Remove doParse() method * Fix broken test * Lookup metadata mapper by name Instead of linear scan --- .../mapper/RankFeatureMetaFieldMapper.java | 11 -- .../RankFeatureMetaFieldMapperTests.java | 3 +- .../index/mapper/size/SizeFieldMapper.java | 14 --- .../index/mapper/DocumentParser.java | 24 +++-- .../index/mapper/FieldNamesFieldMapper.java | 3 +- .../index/mapper/IdFieldMapper.java | 5 - .../index/mapper/IgnoredFieldMapper.java | 14 --- .../index/mapper/IndexFieldMapper.java | 7 -- .../elasticsearch/index/mapper/Mapping.java | 8 ++ .../index/mapper/MetadataFieldMapper.java | 11 +- .../index/mapper/NestedPathFieldMapper.java | 100 ++++++++++++++++++ .../index/mapper/RoutingFieldMapper.java | 12 --- .../index/mapper/SeqNoFieldMapper.java | 10 -- .../index/mapper/SourceFieldMapper.java | 10 -- .../index/mapper/TypeFieldMapper.java | 1 - .../index/mapper/VersionFieldMapper.java | 10 -- .../MetadataIndexTemplateServiceTests.java | 11 -- .../index/mapper/DocumentParserTests.java | 36 ++++++- .../index/mapper/ExternalMetadataMapper.java | 9 -- .../index/mapper/IdFieldMapperTests.java | 3 +- .../mapper/MockMetadataMapperPlugin.java | 98 +++++++++++++++++ .../index/mapper/RoutingFieldMapperTests.java | 2 +- .../DataStreamTimestampFieldMapper.java | 9 -- 23 files changed, 269 insertions(+), 142 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/MockMetadataMapperPlugin.java diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java index 347d018cb72c9..54977637e26ab 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java @@ -65,17 +65,6 @@ private RankFeatureMetaFieldMapper() { super(RankFeatureMetaFieldType.INSTANCE); } - @Override - public void preParse(ParseContext context) {} - - @Override - protected void parseCreateField(ParseContext context) { - throw new AssertionError("Should never be called"); - } - - @Override - public void postParse(ParseContext context) {} - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapperTests.java index f855eb0fe2c8e..332f0dd02740f 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapperTests.java @@ -69,6 +69,7 @@ public void testDocumentParsingFailsOnMetaField() throws Exception { BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field(rfMetaField, 0).endObject()); MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapper.parse(new SourceToParse("test", "_doc", "1", bytes, XContentType.JSON))); - assertTrue(e.getMessage().contains("Field ["+ rfMetaField + "] is a metadata field and cannot be added inside a document.")); + assertTrue( + e.getCause().getMessage().contains("Field ["+ rfMetaField + "] is a metadata field and cannot be added inside a document.")); } } diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index 50ac82f1250af..18ff1c0773c16 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -80,23 +80,9 @@ public boolean enabled() { return this.enabled.value(); } - @Override - public void preParse(ParseContext context) { - } - @Override public void postParse(ParseContext context) throws IOException { // we post parse it so we get the size stored, possibly compressed (source will be preParse) - super.parse(context); - } - - @Override - public void parse(ParseContext context) { - // nothing to do here, we call the parent in postParse - } - - @Override - protected void parseCreateField(ParseContext context) { if (enabled.value() == false) { return; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index e603d9e3917e9..00898667e36ab 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -407,10 +407,7 @@ private static void innerParseObject(ParseContext context, ObjectMapper mapper, if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); paths = splitAndValidatePath(currentFieldName); - if (context.mapperService().isMetadataField(context.path().pathAsText(currentFieldName))) { - throw new MapperParsingException("Field [" + currentFieldName + "] is a metadata field and cannot be added inside" - + " a document. Use the index API request parameters."); - } else if (containsDisabledObjectMapper(mapper, paths)) { + if (containsDisabledObjectMapper(mapper, paths)) { parser.nextToken(); parser.skipChildren(); } @@ -499,7 +496,7 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper, String[] paths) throws IOException { assert currentFieldName != null; - Mapper objectMapper = getMapper(mapper, currentFieldName, paths); + Mapper objectMapper = getMapper(context, mapper, currentFieldName, paths); if (objectMapper != null) { context.path().add(currentFieldName); parseObjectOrField(context, objectMapper); @@ -536,7 +533,7 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper, String[] paths) throws IOException { String arrayFieldName = lastFieldName; - Mapper mapper = getMapper(parentMapper, lastFieldName, paths); + Mapper mapper = getMapper(context, parentMapper, lastFieldName, paths); if (mapper != null) { // There is a concrete mapper for this field already. Need to check if the mapper // expects an array, if so we pass the context straight to the mapper and if not @@ -613,7 +610,7 @@ private static void parseValue(final ParseContext context, ObjectMapper parentMa throw new MapperParsingException("object mapping [" + parentMapper.name() + "] trying to serialize a value with" + " no field associated with it, current value [" + context.parser().textOrNull() + "]"); } - Mapper mapper = getMapper(parentMapper, currentFieldName, paths); + Mapper mapper = getMapper(context, parentMapper, currentFieldName, paths); if (mapper != null) { parseObjectOrField(context, mapper); } else { @@ -630,7 +627,7 @@ private static void parseValue(final ParseContext context, ObjectMapper parentMa private static void parseNullValue(ParseContext context, ObjectMapper parentMapper, String lastFieldName, String[] paths) throws IOException { // we can only handle null values if we have mappings for them - Mapper mapper = getMapper(parentMapper, lastFieldName, paths); + Mapper mapper = getMapper(context, parentMapper, lastFieldName, paths); if (mapper != null) { // TODO: passing null to an object seems bogus? parseObjectOrField(context, mapper); @@ -898,9 +895,16 @@ private static ObjectMapper.Dynamic dynamicOrDefault(ObjectMapper parentMapper, } // looks up a child mapper, but takes into account field names that expand to objects - private static Mapper getMapper(ObjectMapper objectMapper, String fieldName, String[] subfields) { + private static Mapper getMapper(final ParseContext context, ObjectMapper objectMapper, String fieldName, String[] subfields) { + String fieldPath = context.path().pathAsText(fieldName); + // Check if mapper is a metadata mapper first + Mapper mapper = context.docMapper().mapping().getMetadataMapper(fieldPath); + if (mapper != null) { + return mapper; + } + for (int i = 0; i < subfields.length - 1; ++i) { - Mapper mapper = objectMapper.getMapper(subfields[i]); + mapper = objectMapper.getMapper(subfields[i]); if (mapper == null || (mapper instanceof ObjectMapper) == false) { return null; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java index 8a7239cc72b47..4bfb543633611 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java @@ -45,7 +45,6 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(FieldNamesFieldMapper.class); - public static final String NAME = "_field_names"; public static final String CONTENT_TYPE = "_field_names"; @@ -91,6 +90,7 @@ static class Builder extends MetadataFieldMapper.Builder { this.indexVersionCreated = indexVersionCreated; } + @Override protected List> getParameters() { return Collections.singletonList(enabled); } @@ -179,7 +179,6 @@ static Iterable extractFieldNames(final String fullPath) { @Override public Iterator iterator() { return new Iterator() { - int endIndex = nextEndIndex(0); private int nextEndIndex(int index) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index 806538ad1212a..a891bcd32aa42 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -256,11 +256,6 @@ private IdFieldMapper() { @Override public void preParse(ParseContext context) throws IOException { - super.parse(context); - } - - @Override - protected void parseCreateField(ParseContext context) throws IOException { BytesRef id = Uid.encodeId(context.sourceToParse().id()); context.doc().add(new Field(NAME, id, Defaults.FIELD_TYPE)); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java index 766d80fecbada..c1e0568a4db03 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java @@ -82,22 +82,8 @@ private IgnoredFieldMapper() { super(IgnoredFieldType.INSTANCE); } - @Override - public void preParse(ParseContext context) throws IOException { - } - @Override public void postParse(ParseContext context) throws IOException { - super.parse(context); - } - - @Override - public void parse(ParseContext context) throws IOException { - // done in post-parse - } - - @Override - protected void parseCreateField(ParseContext context) throws IOException { for (String field : context.getIgnoredFields()) { context.doc().add(new Field(NAME, field, Defaults.FIELD_TYPE)); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index 20cb3c8153eab..dcfa7e7e1c27c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -27,7 +27,6 @@ import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.SearchLookup; -import java.io.IOException; import java.util.Collections; import java.util.function.Supplier; @@ -73,12 +72,6 @@ public IndexFieldMapper() { super(IndexFieldType.INSTANCE); } - @Override - public void preParse(ParseContext context) throws IOException {} - - @Override - protected void parseCreateField(ParseContext context) throws IOException {} - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java index cbce56d66fa22..b36d240be0b73 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java @@ -48,6 +48,7 @@ public final class Mapping implements ToXContentFragment { final RootObjectMapper root; final MetadataFieldMapper[] metadataMappers; final Map, MetadataFieldMapper> metadataMappersMap; + final Map metadataMappersByName; final Map meta; public Mapping(Version indexCreated, RootObjectMapper rootObjectMapper, @@ -55,8 +56,10 @@ public Mapping(Version indexCreated, RootObjectMapper rootObjectMapper, this.indexCreated = indexCreated; this.metadataMappers = metadataMappers; Map, MetadataFieldMapper> metadataMappersMap = new HashMap<>(); + Map metadataMappersByName = new HashMap<>(); for (MetadataFieldMapper metadataMapper : metadataMappers) { metadataMappersMap.put(metadataMapper.getClass(), metadataMapper); + metadataMappersByName.put(metadataMapper.name(), metadataMapper); } this.root = rootObjectMapper; // keep root mappers sorted for consistent serialization @@ -67,6 +70,7 @@ public int compare(Mapper o1, Mapper o2) { } }); this.metadataMappersMap = unmodifiableMap(metadataMappersMap); + this.metadataMappersByName = unmodifiableMap(metadataMappersByName); this.meta = meta; } @@ -136,6 +140,10 @@ public Mapping merge(Mapping mergeWith, MergeReason reason) { return new Mapping(indexCreated, mergedRoot, mergedMetadataMappers.values().toArray(new MetadataFieldMapper[0]), mergedMeta); } + public MetadataFieldMapper getMetadataMapper(String mapperName) { + return metadataMappersByName.get(mapperName); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { root.toXContent(builder, params, new ToXContent() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 6f59e89195974..41977fc38c691 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -156,10 +156,18 @@ public final XContentBuilder toXContent(XContentBuilder builder, Params params) return builder.endObject(); } + @Override + protected void parseCreateField(ParseContext context) throws IOException { + throw new MapperParsingException("Field [" + name() + "] is a metadata field and cannot be added inside" + + " a document. Use the index API request parameters."); + } + /** * Called before {@link FieldMapper#parse(ParseContext)} on the {@link RootObjectMapper}. */ - public abstract void preParse(ParseContext context) throws IOException; + public void preParse(ParseContext context) throws IOException { + // do nothing + } /** * Called after {@link FieldMapper#parse(ParseContext)} on the {@link RootObjectMapper}. @@ -172,5 +180,4 @@ public void postParse(ParseContext context) throws IOException { public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } - } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java new file mode 100644 index 0000000000000..d9f300a1f7b1f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java @@ -0,0 +1,100 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.query.QueryShardContext; + +import java.util.Collections; + +public class NestedPathFieldMapper extends MetadataFieldMapper { + + public static final String NAME_PRE_V8 = "_type"; + public static final String NAME = "_nested_path"; + + public static String name(Settings settings) { + if (Version.indexCreated(settings).before(Version.V_8_0_0)) { + return NAME_PRE_V8; + } + return NAME; + } + + public static Query filter(Settings settings, String path) { + return new TermQuery(new Term(name(settings), new BytesRef(path))); + } + + public static Field field(Settings settings, String path) { + return new Field(name(settings), path, Defaults.FIELD_TYPE); + } + + public static class Defaults { + + public static final FieldType FIELD_TYPE = new FieldType(); + + static { + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setStored(false); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.freeze(); + } + } + + public static final TypeParser PARSER = new FixedTypeParser(c -> { + final IndexSettings indexSettings = c.mapperService().getIndexSettings(); + return new NestedPathFieldMapper(indexSettings.getSettings()); + }); + + public static final class NestedPathFieldType extends StringFieldType { + + NestedPathFieldType(Settings settings) { + super(NestedPathFieldMapper.name(settings), true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + } + + @Override + public String typeName() { + return NAME; + } + + @Override + public Query existsQuery(QueryShardContext context) { + throw new UnsupportedOperationException("Cannot run exists() query against the nested field path"); + } + } + + private NestedPathFieldMapper(Settings settings) { + super(new NestedPathFieldType(settings)); + } + + @Override + protected String contentType() { + return NAME; + } + +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index f0b98094048b1..c01b9db0087e6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -117,18 +117,6 @@ public boolean required() { @Override public void preParse(ParseContext context) throws IOException { - super.parse(context); - } - - @Override - public void parse(ParseContext context) throws IOException { - // no need ot parse here, we either get the routing in the sourceToParse - // or we don't have routing, if we get it in sourceToParse, we process it in preParse - // which will always be called - } - - @Override - protected void parseCreateField(ParseContext context) throws IOException { String routing = context.sourceToParse().routing(); if (routing != null) { context.doc().add(new Field(fieldType().name(), routing, Defaults.FIELD_TYPE)); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index 4d2317e7e89ae..f26318f7275e8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -183,11 +183,6 @@ public SeqNoFieldMapper() { @Override public void preParse(ParseContext context) throws IOException { - super.parse(context); - } - - @Override - protected void parseCreateField(ParseContext context) throws IOException { // see InternalEngine.innerIndex to see where the real version value is set // also see ParsedDocument.updateSeqID (called by innerIndex) SequenceIDFields seqID = SequenceIDFields.emptySeqID(); @@ -197,11 +192,6 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().add(seqID.primaryTerm); } - @Override - public void parse(ParseContext context) throws IOException { - // fields are added in parseCreateField - } - @Override public void postParse(ParseContext context) throws IOException { // In the case of nested docs, let's fill nested docs with the original diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 34ce34c6b5dce..a1dbd6923d133 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -156,16 +156,6 @@ public boolean isComplete() { @Override public void preParse(ParseContext context) throws IOException { - super.parse(context); - } - - @Override - public void parse(ParseContext context) throws IOException { - // nothing to do here, we will call it in pre parse - } - - @Override - protected void parseCreateField(ParseContext context) throws IOException { BytesReference originalSource = context.sourceToParse().source(); XContentType contentType = context.sourceToParse().getXContentType(); final BytesReference adaptedSource = applyFilters(originalSource, contentType); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java index e213639c3273b..18aadc04e9e89 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java @@ -172,7 +172,6 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, Qu QueryParsers.setRewriteMethod(query, method); return query; } - } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java index 43242bc2b7a1c..bb4a711736f27 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java @@ -68,22 +68,12 @@ private VersionFieldMapper() { @Override public void preParse(ParseContext context) throws IOException { - super.parse(context); - } - - @Override - protected void parseCreateField(ParseContext context) throws IOException { // see InternalEngine.updateVersion to see where the real version value is set final Field version = new NumericDocValuesField(NAME, -1L); context.version(version); context.doc().add(version); } - @Override - public void parse(ParseContext context) throws IOException { - // _version added in preparse - } - @Override public void postParse(ParseContext context) throws IOException { // In the case of nested docs, let's fill nested docs with version=1 so that Lucene doesn't write a Bitset for documents diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 9dd6bda955c15..a279b8716e78d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -46,7 +46,6 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParametrizedFieldMapper; -import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.indices.IndexTemplateMissingException; @@ -1610,16 +1609,6 @@ public ParametrizedFieldMapper.Builder getMergeBuilder() { return new MetadataTimestampFieldBuilder().init(this); } - @Override - public void preParse(ParseContext context) { - - } - - @Override - protected void parseCreateField(ParseContext context) { - - } - @Override protected String contentType() { return "_data_stream_timestamp"; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index e819bc1f86291..98d2c79aaf103 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -31,15 +31,18 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.ParseContext.Document; +import org.elasticsearch.plugins.Plugin; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; +import static java.util.Collections.singletonList; import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.hamcrest.Matchers.containsString; @@ -50,6 +53,11 @@ public class DocumentParserTests extends MapperServiceTestCase { + @Override + protected Collection getPlugins() { + return singletonList(new MockMetadataMapperPlugin()); + } + public void testFieldDisabled() throws Exception { DocumentMapper mapper = createDocumentMapper(mapping(b -> { b.startObject("foo").field("enabled", false).endObject(); @@ -930,12 +938,36 @@ public void testDocumentContainsMetadataField() throws Exception { DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field("_field_names", 0)))); - assertTrue(e.getMessage(), - e.getMessage().contains("Field [_field_names] is a metadata field and cannot be added inside a document.")); + assertTrue(e.getCause().getMessage(), + e.getCause().getMessage().contains("Field [_field_names] is a metadata field and cannot be added inside a document.")); mapper.parse(source(b -> b.field("foo._field_names", 0))); // parses without error } + public void testDocumentContainsAllowedMetadataField() throws Exception { + DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + { + // A metadata field that parses a value fails to parse a null value + MapperParsingException e = expectThrows(MapperParsingException.class, () -> + mapper.parse(source(b -> b.nullField(MockMetadataMapperPlugin.MockMetadataMapper.CONTENT_TYPE)))); + assertTrue(e.getMessage(), e.getMessage().contains("failed to parse field [_mock_metadata]")); + } + { + // A metadata field that parses a value fails to parse an object + MapperParsingException e = expectThrows(MapperParsingException.class, () -> + mapper.parse(source(b -> b.field(MockMetadataMapperPlugin.MockMetadataMapper.CONTENT_TYPE) + .startObject().field("sub-field", "true").endObject()))); + assertTrue(e.getMessage(), e.getMessage().contains("failed to parse field [_mock_metadata]")); + } + { + ParsedDocument doc = mapper.parse(source(b -> + b.field(MockMetadataMapperPlugin.MockMetadataMapper.CONTENT_TYPE, "mock-metadata-field-value") + )); + IndexableField field = doc.rootDoc().getField(MockMetadataMapperPlugin.MockMetadataMapper.CONTENT_TYPE); + assertEquals("mock-metadata-field-value", field.stringValue()); + } + } + public void testSimpleMapper() throws Exception { DocumentMapper docMapper = createDocumentMapper(mapping(b -> { b.startObject("name"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java index 62507f9b38c0e..32919b16179a6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java @@ -37,11 +37,6 @@ protected ExternalMetadataMapper() { super(new BooleanFieldMapper.BooleanFieldType(FIELD_NAME)); } - @Override - protected void parseCreateField(ParseContext context) throws IOException { - // handled in post parse - } - @Override public Iterator iterator() { return Collections.emptyIterator(); @@ -52,10 +47,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public void preParse(ParseContext context) throws IOException { - } - @Override public void postParse(ParseContext context) throws IOException { context.doc().add(new StringField(FIELD_NAME, FIELD_VALUE, Store.YES)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java index 29716293492cd..4c153db690172 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java @@ -59,7 +59,8 @@ public void testIncludeInObjectNotAllowed() throws Exception { .startObject().field("_id", "1").endObject()), XContentType.JSON)); fail("Expected failure to parse metadata field"); } catch (MapperParsingException e) { - assertTrue(e.getMessage(), e.getMessage().contains("Field [_id] is a metadata field and cannot be added inside a document")); + assertTrue(e.getCause().getMessage(), + e.getCause().getMessage().contains("Field [_id] is a metadata field and cannot be added inside a document")); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MockMetadataMapperPlugin.java b/server/src/test/java/org/elasticsearch/index/mapper/MockMetadataMapperPlugin.java new file mode 100644 index 0000000000000..db2ddf8efb5d2 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/MockMetadataMapperPlugin.java @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; + +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * Mapper plugin providing a mock metadata field mapper implementation that supports setting its value + * through the document source. + */ +public class MockMetadataMapperPlugin extends Plugin implements MapperPlugin { + + /** + * A mock metadata field mapper that supports being set from the document source. + */ + public static class MockMetadataMapper extends MetadataFieldMapper { + + static final String CONTENT_TYPE = "_mock_metadata"; + static final String FIELD_NAME = "_mock_metadata"; + + protected MockMetadataMapper() { + super(new KeywordFieldMapper.KeywordFieldType(FIELD_NAME)); + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException { + if (context.parser().currentToken() == XContentParser.Token.VALUE_STRING) { + context.doc().add(new StringField(FIELD_NAME, context.parser().text(), Field.Store.YES)); + } else { + throw new IllegalArgumentException("Field [" + fieldType().name() + "] must be a string."); + } + } + + @Override + public Iterator iterator() { + return Collections.emptyIterator(); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + public static class Builder extends MetadataFieldMapper.Builder { + + protected Builder() { + super(FIELD_NAME); + } + + @Override + protected List> getParameters() { + return Collections.emptyList(); + } + + @Override + public MockMetadataMapper build(BuilderContext context) { + return new MockMetadataMapper(); + } + } + + public static final TypeParser PARSER = new ConfigurableTypeParser( + c -> new MockMetadataMapper(), + c -> new MockMetadataMapper.Builder()) { + }; + } + + @Override + public Map getMetadataMappers() { + return Collections.singletonMap(MockMetadataMapper.CONTENT_TYPE, MockMetadataMapper.PARSER); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java index 3da00e3f787c9..b61dc497cf67e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java @@ -58,7 +58,7 @@ public void testIncludeInObjectNotAllowed() throws Exception { .startObject().field("_routing", "foo").endObject()),XContentType.JSON)); fail("Expected failure to parse metadata field"); } catch (MapperParsingException e) { - assertThat(e.getMessage(), e.getMessage(), + assertThat(e.getCause().getMessage(), e.getCause().getMessage(), containsString("Field [_routing] is a metadata field and cannot be added inside a document")); } } diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java index ffac7bfa0f11c..d2ad6a8df4090 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java @@ -172,15 +172,6 @@ public void doValidate(MappingLookup lookup) { } } - @Override - public void preParse(ParseContext context) throws IOException {} - - @Override - protected void parseCreateField(ParseContext context) throws IOException { - // Meta field doesn't create any fields, so this shouldn't happen. - throw new IllegalStateException(NAME + " field mapper cannot create fields"); - } - @Override public void postParse(ParseContext context) throws IOException { if (enabled == false) { From 07b54ff4b2dfc89b4faec27650388c5b1b321858 Mon Sep 17 00:00:00 2001 From: Christos Soulios Date: Fri, 18 Sep 2020 11:33:05 +0300 Subject: [PATCH 2/2] Fix issues from backporting code --- .../index/mapper/AllFieldMapper.java | 20 -- .../index/mapper/FieldNamesFieldMapper.java | 62 +++--- .../index/mapper/NestedPathFieldMapper.java | 100 --------- .../index/mapper/TypeFieldMapper.java | 12 +- .../index/mapper/DocumentParserTests.java | 195 ++++++++++-------- 5 files changed, 133 insertions(+), 256 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java index ffc1b5bcb1563..c5215b4f45161 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.index.query.QueryShardContext; -import java.io.IOException; import java.util.Collections; import java.util.List; @@ -104,25 +103,6 @@ private AllFieldMapper(Explicit enabled) { this.enabled = enabled; } - @Override - public void preParse(ParseContext context) { - } - - @Override - public void postParse(ParseContext context) throws IOException { - super.parse(context); - } - - @Override - public void parse(ParseContext context) throws IOException { - // we parse in post parse - } - - @Override - protected void parseCreateField(ParseContext context) throws IOException { - // noop mapper - } - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java index 4bfb543633611..fc2d4377f05d1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java @@ -158,22 +158,36 @@ public FieldNamesFieldType fieldType() { return (FieldNamesFieldType) super.fieldType(); } - @Override - public void preParse(ParseContext context) { - } - @Override public void postParse(ParseContext context) throws IOException { if (context.indexSettings().getIndexVersionCreated().before(Version.V_6_1_0)) { - super.parse(context); + if (fieldType().isEnabled() == false) { + return; + } + for (ParseContext.Document document : context) { + final List paths = new ArrayList<>(document.getFields().size()); + String previousPath = ""; // used as a sentinel - field names can't be empty + for (IndexableField field : document.getFields()) { + final String path = field.name(); + if (path.equals(previousPath)) { + // Sometimes mappers create multiple Lucene fields, eg. one for indexing, + // one for doc values and one for storing. Deduplicating is not required + // for correctness but this simple check helps save utf-8 conversions and + // gives Lucene fewer values to deal with. + continue; + } + paths.add(path); + previousPath = path; + } + for (String path : paths) { + for (String fieldName : extractFieldNames(path)) { + document.add(new Field(fieldType().name(), fieldName, Defaults.FIELD_TYPE)); + } + } + } } } - @Override - public void parse(ParseContext context) throws IOException { - // Adding values to the _field_names field is handled by the mappers for each field type - } - static Iterable extractFieldNames(final String fullPath) { return new Iterable() { @Override @@ -210,34 +224,6 @@ public void remove() { }; } - @Override - protected void parseCreateField(ParseContext context) throws IOException { - if (fieldType().isEnabled() == false) { - return; - } - for (ParseContext.Document document : context) { - final List paths = new ArrayList<>(document.getFields().size()); - String previousPath = ""; // used as a sentinel - field names can't be empty - for (IndexableField field : document.getFields()) { - final String path = field.name(); - if (path.equals(previousPath)) { - // Sometimes mappers create multiple Lucene fields, eg. one for indexing, - // one for doc values and one for storing. Deduplicating is not required - // for correctness but this simple check helps save utf-8 conversions and - // gives Lucene fewer values to deal with. - continue; - } - paths.add(path); - previousPath = path; - } - for (String path : paths) { - for (String fieldName : extractFieldNames(path)) { - document.add(new Field(fieldType().name(), fieldName, Defaults.FIELD_TYPE)); - } - } - } - } - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java deleted file mode 100644 index d9f300a1f7b1f..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper; - -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.query.QueryShardContext; - -import java.util.Collections; - -public class NestedPathFieldMapper extends MetadataFieldMapper { - - public static final String NAME_PRE_V8 = "_type"; - public static final String NAME = "_nested_path"; - - public static String name(Settings settings) { - if (Version.indexCreated(settings).before(Version.V_8_0_0)) { - return NAME_PRE_V8; - } - return NAME; - } - - public static Query filter(Settings settings, String path) { - return new TermQuery(new Term(name(settings), new BytesRef(path))); - } - - public static Field field(Settings settings, String path) { - return new Field(name(settings), path, Defaults.FIELD_TYPE); - } - - public static class Defaults { - - public static final FieldType FIELD_TYPE = new FieldType(); - - static { - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - FIELD_TYPE.setTokenized(false); - FIELD_TYPE.setStored(false); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.freeze(); - } - } - - public static final TypeParser PARSER = new FixedTypeParser(c -> { - final IndexSettings indexSettings = c.mapperService().getIndexSettings(); - return new NestedPathFieldMapper(indexSettings.getSettings()); - }); - - public static final class NestedPathFieldType extends StringFieldType { - - NestedPathFieldType(Settings settings) { - super(NestedPathFieldMapper.name(settings), true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); - } - - @Override - public String typeName() { - return NAME; - } - - @Override - public Query existsQuery(QueryShardContext context) { - throw new UnsupportedOperationException("Cannot run exists() query against the nested field path"); - } - } - - private NestedPathFieldMapper(Settings settings) { - super(new NestedPathFieldType(settings)); - } - - @Override - protected String contentType() { - return NAME; - } - -} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java index 18aadc04e9e89..b17dc5db07f88 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java @@ -19,7 +19,7 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.document.Field; + import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexOptions; @@ -262,16 +262,6 @@ private TypeFieldMapper() { @Override public void preParse(ParseContext context) throws IOException { - super.parse(context); - } - - @Override - public void parse(ParseContext context) throws IOException { - // we parse in pre parse - } - - @Override - protected void parseCreateField(ParseContext context) throws IOException { if (fieldType.indexOptions() == IndexOptions.NONE && !fieldType.stored()) { return; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index 98d2c79aaf103..6277f3f4c8107 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -162,10 +162,10 @@ public void testDotsWithExistingNestedMapper() throws Exception { })); MapperParsingException e = expectThrows(MapperParsingException.class, - () -> mapper.parse(source(b -> b.field("field.bar", 123)))); + () -> mapper.parse(source(b -> b.field("field.bar", 123)))); assertEquals( - "Cannot add a value for field [field.bar] since one of the intermediate objects is mapped as a nested object: [field]", - e.getMessage()); + "Cannot add a value for field [field.bar] since one of the intermediate objects is mapped as a nested object: [field]", + e.getMessage()); } public void testUnexpectedFieldMappingType() throws Exception { @@ -175,12 +175,12 @@ public void testUnexpectedFieldMappingType() throws Exception { })); { MapperException exception = expectThrows(MapperException.class, - () -> mapper.parse(source(b -> b.field("foo", true)))); + () -> mapper.parse(source(b -> b.field("foo", true)))); assertThat(exception.getMessage(), containsString("failed to parse field [foo] of type [long] in document with id '1'")); } { MapperException exception = expectThrows(MapperException.class, - () -> mapper.parse(source(b -> b.field("bar", "bar")))); + () -> mapper.parse(source(b -> b.field("bar", "bar")))); assertThat(exception.getMessage(), containsString("failed to parse field [bar] of type [boolean] in document with id '1'")); } } @@ -204,10 +204,10 @@ public void testDotsWithDynamicNestedMapper() throws Exception { })); MapperParsingException e = expectThrows(MapperParsingException.class, - () -> mapper.parse(source(b -> b.field("foo.bar", 42)))); + () -> mapper.parse(source(b -> b.field("foo.bar", 42)))); assertEquals( - "It is forbidden to create dynamic nested objects ([foo]) through `copy_to` or dots in field names", - e.getMessage()); + "It is forbidden to create dynamic nested objects ([foo]) through `copy_to` or dots in field names", + e.getMessage()); } public void testNestedHaveIdAndTypeFields() throws Exception { @@ -358,7 +358,7 @@ ObjectMapper createObjectMapper(MapperService mapperService, String name) { } Mapper.Builder builder = new ObjectMapper.Builder<>(nameParts[nameParts.length - 1]).enabled(true); Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path()); - return (ObjectMapper)builder.build(builderContext); + return (ObjectMapper) builder.build(builderContext); } public void testEmptyMappingUpdate() throws Exception { @@ -382,8 +382,8 @@ public void testSubfieldMappingUpdate() throws Exception { Mapper xMapper = mapping.root().getMapper("x"); assertNotNull(xMapper); assertTrue(xMapper instanceof ObjectMapper); - assertNotNull(((ObjectMapper)xMapper).getMapper("foo")); - assertNull(((ObjectMapper)xMapper).getMapper("subx")); + assertNotNull(((ObjectMapper) xMapper).getMapper("foo")); + assertNull(((ObjectMapper) xMapper).getMapper("subx")); } public void testMultipleSubfieldMappingUpdate() throws Exception { @@ -396,9 +396,9 @@ public void testMultipleSubfieldMappingUpdate() throws Exception { Mapper xMapper = mapping.root().getMapper("x"); assertNotNull(xMapper); assertTrue(xMapper instanceof ObjectMapper); - assertNotNull(((ObjectMapper)xMapper).getMapper("foo")); - assertNotNull(((ObjectMapper)xMapper).getMapper("bar")); - assertNull(((ObjectMapper)xMapper).getMapper("subx")); + assertNotNull(((ObjectMapper) xMapper).getMapper("foo")); + assertNotNull(((ObjectMapper) xMapper).getMapper("bar")); + assertNull(((ObjectMapper) xMapper).getMapper("subx")); } public void testDeepSubfieldMappingUpdate() throws Exception { @@ -409,10 +409,10 @@ public void testDeepSubfieldMappingUpdate() throws Exception { Mapper xMapper = mapping.root().getMapper("x"); assertNotNull(xMapper); assertTrue(xMapper instanceof ObjectMapper); - Mapper subxMapper = ((ObjectMapper)xMapper).getMapper("subx"); + Mapper subxMapper = ((ObjectMapper) xMapper).getMapper("subx"); assertTrue(subxMapper instanceof ObjectMapper); - assertNotNull(((ObjectMapper)subxMapper).getMapper("foo")); - assertNull(((ObjectMapper)subxMapper).getMapper("subsubx")); + assertNotNull(((ObjectMapper) subxMapper).getMapper("foo")); + assertNull(((ObjectMapper) subxMapper).getMapper("subsubx")); } public void testDeepSubfieldAfterSubfieldMappingUpdate() throws Exception { @@ -425,10 +425,10 @@ public void testDeepSubfieldAfterSubfieldMappingUpdate() throws Exception { Mapper xMapper = mapping.root().getMapper("x"); assertNotNull(xMapper); assertTrue(xMapper instanceof ObjectMapper); - assertNotNull(((ObjectMapper)xMapper).getMapper("a")); - Mapper subxMapper = ((ObjectMapper)xMapper).getMapper("subx"); + assertNotNull(((ObjectMapper) xMapper).getMapper("a")); + Mapper subxMapper = ((ObjectMapper) xMapper).getMapper("subx"); assertTrue(subxMapper instanceof ObjectMapper); - assertNotNull(((ObjectMapper)subxMapper).getMapper("b")); + assertNotNull(((ObjectMapper) subxMapper).getMapper("b")); } public void testObjectMappingUpdate() throws Exception { @@ -444,10 +444,10 @@ public void testObjectMappingUpdate() throws Exception { Mapper fooMapper = mapping.root().getMapper("foo"); assertNotNull(fooMapper); assertTrue(fooMapper instanceof ObjectMapper); - assertNotNull(((ObjectMapper)fooMapper).getMapper("field")); - Mapper barMapper = ((ObjectMapper)fooMapper).getMapper("bar"); + assertNotNull(((ObjectMapper) fooMapper).getMapper("field")); + Mapper barMapper = ((ObjectMapper) fooMapper).getMapper("bar"); assertTrue(barMapper instanceof ObjectMapper); - assertNotNull(((ObjectMapper)barMapper).getMapper("baz")); + assertNotNull(((ObjectMapper) barMapper).getMapper("baz")); } public void testDynamicGeoPointArrayWithTemplate() throws Exception { @@ -507,7 +507,8 @@ public void testDynamicLongArrayWithTemplate() throws Exception { } public void testDynamicLongArray() throws Exception { - DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper mapper = createDocumentMapper(mapping(b -> { + })); ParsedDocument doc = mapper.parse(source(b -> b.startArray("foo").value(0).value(1).endArray())); assertEquals(4, doc.rootDoc().getFields("foo").length); } @@ -521,7 +522,7 @@ public void testDynamicFalseLongArray() throws Exception { public void testDynamicStrictLongArray() throws Exception { DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "strict"))); StrictDynamicMappingException exception = expectThrows(StrictDynamicMappingException.class, - () -> mapper.parse(source(b -> b.startArray("foo").value(0).value(1).endArray()))); + () -> mapper.parse(source(b -> b.startArray("foo").value(0).value(1).endArray()))); assertEquals("mapping set to strict, dynamic introduction of [foo] within [_doc] is not allowed", exception.getMessage()); } @@ -587,7 +588,7 @@ public void testDynamicFalseObject() throws Exception { public void testDynamicStrictObject() throws Exception { DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "strict"))); StrictDynamicMappingException exception = expectThrows(StrictDynamicMappingException.class, - () -> mapper.parse(source(b -> b.startObject("foo").field("bar", "baz").endObject()))); + () -> mapper.parse(source(b -> b.startObject("foo").field("bar", "baz").endObject()))); assertEquals("mapping set to strict, dynamic introduction of [foo] within [_doc] is not allowed", exception.getMessage()); } @@ -600,7 +601,7 @@ public void testDynamicFalseValue() throws Exception { public void testDynamicStrictValue() throws Exception { DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "strict"))); StrictDynamicMappingException exception = expectThrows(StrictDynamicMappingException.class, - () -> mapper.parse(source(b -> b.field("bar", "baz")))); + () -> mapper.parse(source(b -> b.field("bar", "baz")))); assertEquals("mapping set to strict, dynamic introduction of [bar] within [_doc] is not allowed", exception.getMessage()); } @@ -613,7 +614,7 @@ public void testDynamicFalseNull() throws Exception { public void testDynamicStrictNull() throws Exception { DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "strict"))); StrictDynamicMappingException exception = expectThrows(StrictDynamicMappingException.class, - () -> mapper.parse(source(b -> b.nullField("bar")))); + () -> mapper.parse(source(b -> b.nullField("bar")))); assertEquals("mapping set to strict, dynamic introduction of [bar] within [_doc] is not allowed", exception.getMessage()); } @@ -677,7 +678,8 @@ public void testDynamicBigDecimal() throws Exception { } public void testDynamicDottedFieldNameLongArray() throws Exception { - DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper mapper = createDocumentMapper(mapping(b -> { + })); ParsedDocument doc = mapper.parse(source(b -> b.startArray("foo.bar.baz").value(0).value(1).endArray())); assertEquals(4, doc.rootDoc().getFields("foo.bar.baz").length); @@ -742,9 +744,9 @@ public void testDynamicDottedFieldNameLongArrayWithExistingParent() throws Excep public void testDynamicDottedFieldNameLongArrayWithExistingParentWrongType() throws Exception { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "long"))); MapperParsingException exception = expectThrows(MapperParsingException.class, - () -> mapper.parse(source(b -> b.startArray("field.bar.baz").value(0).value(1).endArray()))); + () -> mapper.parse(source(b -> b.startArray("field.bar.baz").value(0).value(1).endArray()))); assertEquals("Could not dynamically add mapping for field [field.bar.baz]. " - + "Existing mapping for [field] must be of type object but found [long].", exception.getMessage()); + + "Existing mapping for [field] must be of type object but found [long].", exception.getMessage()); } public void testDynamicFalseDottedFieldNameLongArray() throws Exception { @@ -756,12 +758,13 @@ public void testDynamicFalseDottedFieldNameLongArray() throws Exception { public void testDynamicStrictDottedFieldNameLongArray() throws Exception { DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "strict"))); StrictDynamicMappingException exception = expectThrows(StrictDynamicMappingException.class, - () -> mapper.parse(source(b -> b.startArray("foo.bar.baz").value(0).value(1).endArray()))); + () -> mapper.parse(source(b -> b.startArray("foo.bar.baz").value(0).value(1).endArray()))); assertEquals("mapping set to strict, dynamic introduction of [foo] within [_doc] is not allowed", exception.getMessage()); } public void testDynamicDottedFieldNameLong() throws Exception { - DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper mapper = createDocumentMapper(mapping(b -> { + })); ParsedDocument doc = mapper.parse(source(b -> b.field("foo.bar.baz", 0))); assertEquals(2, doc.rootDoc().getFields("foo.bar.baz").length); Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo"); @@ -824,9 +827,9 @@ public void testDynamicDottedFieldNameLongWithExistingParent() throws Exception public void testDynamicDottedFieldNameLongWithExistingParentWrongType() throws Exception { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "long"))); MapperParsingException exception = expectThrows(MapperParsingException.class, - () -> mapper.parse(source(b -> b.field("field.bar.baz", 0)))); + () -> mapper.parse(source(b -> b.field("field.bar.baz", 0)))); assertEquals("Could not dynamically add mapping for field [field.bar.baz]. " - + "Existing mapping for [field] must be of type object but found [long].", exception.getMessage()); + + "Existing mapping for [field] must be of type object but found [long].", exception.getMessage()); } public void testDynamicFalseDottedFieldNameLong() throws Exception { @@ -838,12 +841,13 @@ public void testDynamicFalseDottedFieldNameLong() throws Exception { public void testDynamicStrictDottedFieldNameLong() throws Exception { DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "strict"))); StrictDynamicMappingException exception = expectThrows(StrictDynamicMappingException.class, - () -> mapper.parse(source(b -> b.field("foo.bar.baz", 0)))); + () -> mapper.parse(source(b -> b.field("foo.bar.baz", 0)))); assertEquals("mapping set to strict, dynamic introduction of [foo] within [_doc] is not allowed", exception.getMessage()); } public void testDynamicDottedFieldNameObject() throws Exception { - DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper mapper = createDocumentMapper(mapping(b -> { + })); ParsedDocument doc = mapper.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject())); assertEquals(2, doc.rootDoc().getFields("foo.bar.baz.a").length); Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo"); @@ -916,9 +920,9 @@ public void testDynamicDottedFieldNameObjectWithExistingParent() throws Exceptio public void testDynamicDottedFieldNameObjectWithExistingParentWrongType() throws Exception { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "long"))); MapperParsingException exception = expectThrows(MapperParsingException.class, - () -> mapper.parse(source(b -> b.startObject("field.bar.baz").field("a", 0).endObject()))); + () -> mapper.parse(source(b -> b.startObject("field.bar.baz").field("a", 0).endObject()))); assertEquals("Could not dynamically add mapping for field [field.bar.baz]. " - + "Existing mapping for [field] must be of type object but found [long].", exception.getMessage()); + + "Existing mapping for [field] must be of type object but found [long].", exception.getMessage()); } public void testDynamicFalseDottedFieldNameObject() throws Exception { @@ -930,12 +934,13 @@ public void testDynamicFalseDottedFieldNameObject() throws Exception { public void testDynamicStrictDottedFieldNameObject() throws Exception { DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "strict"))); StrictDynamicMappingException exception = expectThrows(StrictDynamicMappingException.class, - () -> mapper.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject()))); + () -> mapper.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject()))); assertEquals("mapping set to strict, dynamic introduction of [foo] within [_doc] is not allowed", exception.getMessage()); } public void testDocumentContainsMetadataField() throws Exception { - DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper mapper = createDocumentMapper(mapping(b -> { + })); MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field("_field_names", 0)))); assertTrue(e.getCause().getMessage(), @@ -945,7 +950,8 @@ public void testDocumentContainsMetadataField() throws Exception { } public void testDocumentContainsAllowedMetadataField() throws Exception { - DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper mapper = createDocumentMapper(mapping(b -> { + })); { // A metadata field that parses a value fails to parse a null value MapperParsingException e = expectThrows(MapperParsingException.class, () -> @@ -1002,7 +1008,8 @@ public void testSimpleMapper() throws Exception { public void testParseToJsonAndParse() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/simple/test-mapping.json"); - MapperService mapperService = createMapperService(mapping(b -> {})); + MapperService mapperService = createMapperService(mapping(b -> { + })); merge("person", mapperService, mapping); String builtMapping = mapperService.documentMapper().mappingSource().string(); // reparse it @@ -1047,7 +1054,8 @@ public void testAttributes() throws Exception { } public void testNoDocumentSent() throws Exception { - DocumentMapper docMapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper docMapper = createDocumentMapper(mapping(b -> { + })); BytesReference json = new BytesArray("".getBytes(StandardCharsets.UTF_8)); MapperParsingException e = expectThrows(MapperParsingException.class, () -> docMapper.parse(new SourceToParse("test", "_doc", "1", json, XContentType.JSON))); @@ -1055,7 +1063,8 @@ public void testNoDocumentSent() throws Exception { } public void testNoLevel() throws Exception { - DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> { + })); ParsedDocument doc = defaultMapper.parse(source(b -> { b.field("test1", "value1"); b.field("test2", "value2"); @@ -1070,7 +1079,8 @@ public void testNoLevel() throws Exception { // we no longer have types? public void testTypeLevel() throws Exception { - DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> { + })); ParsedDocument doc = defaultMapper.parse(source(b -> { b.startObject("type"); @@ -1088,7 +1098,8 @@ public void testTypeLevel() throws Exception { } public void testNoLevelWithFieldTypeAsValue() throws Exception { - DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> { + })); ParsedDocument doc = defaultMapper.parse(source(b -> { b.field("type", "value_type"); @@ -1105,7 +1116,8 @@ public void testNoLevelWithFieldTypeAsValue() throws Exception { public void testTypeLevelWithFieldTypeAsValue() throws Exception { - DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> { + })); ParsedDocument doc = defaultMapper.parse(source(b -> { b.startObject("type"); @@ -1125,7 +1137,8 @@ public void testTypeLevelWithFieldTypeAsValue() throws Exception { } public void testNoLevelWithFieldTypeAsObject() throws Exception { - DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> { + })); ParsedDocument doc = defaultMapper.parse(source(b -> { b.startObject("type").field("type_field", "type_value").endObject(); @@ -1141,7 +1154,8 @@ public void testNoLevelWithFieldTypeAsObject() throws Exception { } public void testTypeLevelWithFieldTypeAsObject() throws Exception { - DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> { + })); ParsedDocument doc = defaultMapper.parse(source(b -> { b.startObject("type"); @@ -1161,7 +1175,8 @@ public void testTypeLevelWithFieldTypeAsObject() throws Exception { } public void testNoLevelWithFieldTypeAsValueNotFirst() throws Exception { - DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> { + })); ParsedDocument doc = defaultMapper.parse(source(b -> { b.startObject("type"); @@ -1181,7 +1196,8 @@ public void testNoLevelWithFieldTypeAsValueNotFirst() throws Exception { } public void testTypeLevelWithFieldTypeAsValueNotFirst() throws Exception { - DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> { + })); ParsedDocument doc = defaultMapper.parse(source(b -> { b.startObject("type"); @@ -1201,7 +1217,8 @@ public void testTypeLevelWithFieldTypeAsValueNotFirst() throws Exception { } public void testNoLevelWithFieldTypeAsObjectNotFirst() throws Exception { - DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> { + })); ParsedDocument doc = defaultMapper.parse(source(b -> { b.field("test1", "value1"); @@ -1218,7 +1235,8 @@ public void testNoLevelWithFieldTypeAsObjectNotFirst() throws Exception { } public void testTypeLevelWithFieldTypeAsObjectNotFirst() throws Exception { - DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper defaultMapper = createDocumentMapper(mapping(b -> { + })); ParsedDocument doc = defaultMapper.parse(source(b -> { b.startObject("type"); @@ -1263,7 +1281,8 @@ public void testDynamicDateDetectionEnabledWithNoSpecialCharacters() throws IOEx } public void testDynamicFieldsStartingAndEndingWithDot() throws Exception { - MapperService mapperService = createMapperService(mapping(b -> {})); + MapperService mapperService = createMapperService(mapping(b -> { + })); merge(mapperService, dynamicMapping(mapperService.documentMapper().parse(source(b -> { b.startArray("top."); { @@ -1282,64 +1301,66 @@ public void testDynamicFieldsStartingAndEndingWithDot() throws Exception { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> mapperService.documentMapper().parse(source(b -> { - b.startArray("top."); - { - b.startObject(); + b.startArray("top."); { - b.startArray("foo."); + b.startObject(); { - b.startObject(); + b.startArray("foo."); { - b.startObject("bar."); + b.startObject(); { - b.startObject("aoeu").field("a", 1).field("b", 2).endObject(); + b.startObject("bar."); + { + b.startObject("aoeu").field("a", 1).field("b", 2).endObject(); + } + b.endObject(); } b.endObject(); } - b.endObject(); + b.endArray(); } - b.endArray(); + b.endObject(); } - b.endObject(); - } - b.endArray(); - }))); + b.endArray(); + }))); assertThat(e.getMessage(), - containsString("object field starting or ending with a [.] makes object resolution ambiguous: [top..foo..bar]")); + containsString("object field starting or ending with a [.] makes object resolution ambiguous: [top..foo..bar]")); } public void testDynamicFieldsEmptyName() throws Exception { - DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper mapper = createDocumentMapper(mapping(b -> { + })); IllegalArgumentException emptyFieldNameException = expectThrows(IllegalArgumentException.class, - () -> mapper.parse(source(b -> { - b.startArray("top."); + () -> mapper.parse(source(b -> { + b.startArray("top."); + { + b.startObject(); { - b.startObject(); - { - b.startObject("aoeu").field("a", 1).field(" ", 2).endObject(); - } - b.endObject(); + b.startObject("aoeu").field("a", 1).field(" ", 2).endObject(); } - b.endArray(); - }))); + b.endObject(); + } + b.endArray(); + }))); assertThat(emptyFieldNameException.getMessage(), containsString( - "object field cannot contain only whitespace: ['top.aoeu. ']")); + "object field cannot contain only whitespace: ['top.aoeu. ']")); } public void testBlankFieldNames() throws Exception { - DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + DocumentMapper mapper = createDocumentMapper(mapping(b -> { + })); MapperParsingException err = expectThrows(MapperParsingException.class, () -> - mapper.parse(source(b -> b.field("", "foo")))); + mapper.parse(source(b -> b.field("", "foo")))); assertThat(err.getCause(), notNullValue()); assertThat(err.getCause().getMessage(), containsString("field name cannot be an empty string")); err = expectThrows(MapperParsingException.class, () -> - mapper.parse(source(b -> b.startObject("foo").field("", "bar").endObject()))); + mapper.parse(source(b -> b.startObject("foo").field("", "bar").endObject()))); assertThat(err.getCause(), notNullValue()); assertThat(err.getCause().getMessage(), containsString("field name cannot be an empty string")); } @@ -1361,7 +1382,7 @@ public void testWriteToFieldAlias() throws Exception { assertEquals("Cannot write to a field alias [alias-field].", exception.getCause().getMessage()); } - public void testCopyToFieldAlias() throws Exception { + public void testCopyToFieldAlias() throws Exception { DocumentMapper mapper = createDocumentMapper(mapping(b -> { b.startObject("alias-field"); { @@ -1404,9 +1425,9 @@ public void testDynamicDottedFieldNameWithFieldAlias() throws Exception { public void testTypeless() throws IOException { String mapping = Strings.toString(XContentFactory.jsonBuilder() - .startObject().startObject("type").startObject("properties") - .startObject("foo").field("type", "keyword").endObject() - .endObject().endObject().endObject()); + .startObject().startObject("type").startObject("properties") + .startObject("foo").field("type", "keyword").endObject() + .endObject().endObject().endObject()); DocumentMapper mapper = createDocumentMapper("type", mapping); ParsedDocument doc = mapper.parse(source(b -> b.field("foo", "1234")));