From 855c132d5abc6cebf5790afbea320a2fb41c90f4 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 23 Apr 2018 16:00:11 +0200 Subject: [PATCH 1/3] Add a new `_ignored` meta field. This adds a new `_ignored` meta field which indexes and stores fields that have been ignored at index time because of the `ignore_malformed` option. It makes malformed documents easier to identify by using `exists` or `term(s)` queries on the `_ignored` field. Closes #29494 --- docs/reference/mapping/fields.asciidoc | 10 ++ .../mapping/fields/ignored-field.asciidoc | 45 +++++ .../mapping/params/ignore-malformed.asciidoc | 10 ++ .../test/search/200_ignore_malformed.yml | 92 +++++++++++ .../index/fieldvisitor/FieldsVisitor.java | 5 + .../index/mapper/DateFieldMapper.java | 1 + .../index/mapper/GeoPointFieldMapper.java | 2 + .../index/mapper/GeoShapeFieldMapper.java | 1 + .../index/mapper/IgnoredFieldMapper.java | 154 ++++++++++++++++++ .../index/mapper/IpFieldMapper.java | 1 + .../index/mapper/MapperService.java | 2 +- .../index/mapper/NumberFieldMapper.java | 1 + .../index/mapper/ParseContext.java | 37 +++++ .../elasticsearch/indices/IndicesModule.java | 6 +- .../org/elasticsearch/search/SearchHit.java | 10 +- .../index/mapper/DateFieldMapperTests.java | 1 + .../index/mapper/IgnoredFieldTypeTests.java | 29 ++++ .../index/mapper/IpFieldMapperTests.java | 1 + .../index/mapper/NumberFieldMapperTests.java | 1 + .../indices/IndicesModuleTests.java | 9 +- 20 files changed, 410 insertions(+), 8 deletions(-) create mode 100644 docs/reference/mapping/fields/ignored-field.asciidoc create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java diff --git a/docs/reference/mapping/fields.asciidoc b/docs/reference/mapping/fields.asciidoc index 155e23c9e544a..d3bc90ab8b39b 100644 --- a/docs/reference/mapping/fields.asciidoc +++ b/docs/reference/mapping/fields.asciidoc @@ -40,6 +40,14 @@ can be customised when a mapping type is created. All fields in the document which contain non-null values. +[float] +=== Indexing meta-fields + +<>:: + + All fields in the document that have been ignored at index time because of + <>. + [float] === Routing meta-field @@ -57,6 +65,8 @@ can be customised when a mapping type is created. include::fields/field-names-field.asciidoc[] +include::fields/ignored-field.asciidoc[] + include::fields/id-field.asciidoc[] include::fields/index-field.asciidoc[] diff --git a/docs/reference/mapping/fields/ignored-field.asciidoc b/docs/reference/mapping/fields/ignored-field.asciidoc new file mode 100644 index 0000000000000..d2776ea86b26b --- /dev/null +++ b/docs/reference/mapping/fields/ignored-field.asciidoc @@ -0,0 +1,45 @@ +[[mapping-ignored-field]] +=== `_ignored` field + +added[6.4.0] + +The `_ignored` field indexes and stores the names of every field in a document +that has been ignored because it was malformed and +<> was turned on. + +This field is searchable with <>, +<> and <> +queries, and is returned as part of the search hits. + +For instance the below query matches all documents that have one or more fields +that got ignored: + +[source,js] +-------------------------------------------------- +GET _search +{ + "query": { + "exists": { + "field": "_ignored" + } + } +} +-------------------------------------------------- +// CONSOLE + +Similarly, the below query finds all documents whose `@timestamp` field was +ignored at index time: + +[source,js] +-------------------------------------------------- +GET _search +{ + "query": { + "term": { + "_ignored": "@timestamp" + } + } +} +-------------------------------------------------- +// CONSOLE + diff --git a/docs/reference/mapping/params/ignore-malformed.asciidoc b/docs/reference/mapping/params/ignore-malformed.asciidoc index 9a2fef1a23ebf..be0bdfe4ffaf8 100644 --- a/docs/reference/mapping/params/ignore-malformed.asciidoc +++ b/docs/reference/mapping/params/ignore-malformed.asciidoc @@ -85,3 +85,13 @@ PUT my_index <1> The `number_one` field inherits the index-level setting. <2> The `number_two` field overrides the index-level setting to turn off `ignore_malformed`. + +==== Dealing with malformed fields + +Malformed fields are silently ignored at indexing time when `ignore_malformed` +is turned on. Whenever possible it is recommended to keep the number of +documents that have a malformed field contained, or queries on this field will +become meaningless. Elasticsearch makes it easy to check how many documents +have malformed fields by using `exist` or `term` queries on the special +<> field. + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml new file mode 100644 index 0000000000000..e32b1fa1b9583 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml @@ -0,0 +1,92 @@ +--- +setup: + - skip: + version: " - 6.99.99" # TODO: change to 6.3.99 after backport to 6.4 + reason: _ignored was added in 6.4.0 + + - do: + indices.create: + index: test + body: + mappings: + _doc: + properties: + my_date: + type: date + ignore_malformed: true + store: true + my_ip: + type: ip + ignore_malformed: true + + - do: + index: + index: test + type: _doc + id: 1 + body: { "my_date": "2018-05-11", "my_ip": ":::1" } + + - do: + index: + index: test + type: _doc + id: 2 + body: { "my_date": "bar", "my_ip": "192.168.1.42" } + + - do: + index: + index: test + type: _doc + id: 3 + body: { "my_date": "bar", "my_ip": "quux" } + + - do: + indices.refresh: {} + +--- +"Exists on _ignored": + + - do: + search: + body: { query: { exists: { "field": "_ignored" } } } + + - length: { hits.hits: 3 } + +--- +"Search on _ignored with term": + + - do: + search: + body: { query: { term: { "_ignored": "my_date" } } } + + - length: { hits.hits: 2 } + +--- +"Search on _ignored with terms": + + - do: + search: + body: { query: { terms: { "_ignored": [ "my_date", "my_ip" ] } } } + + - length: { hits.hits: 3 } + +--- +"_ignored is returned by default": + + - do: + search: + body: { query: { ids: { "values": [ "3" ] } } } + + - length: { hits.hits: 1 } + - length: { hits.hits.0._ignored: 2} + +--- +"_ignored is still returned with explicit list of stored fields": + + - do: + search: + stored_fields: [ "my_date" ] + body: { query: { ids: { "values": [ "3" ] } } } + + - length: { hits.hits: 1 } + - is_true: hits.hits.0._ignored diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java index aecbba766f416..374a62dc52b6f 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.RoutingFieldMapper; @@ -69,6 +70,10 @@ public Status needsField(FieldInfo fieldInfo) throws IOException { if (requiredFields.remove(fieldInfo.name)) { return Status.YES; } + // Always load _ignored to be explicit about ignored fields + if (IgnoredFieldMapper.NAME.equals(fieldInfo.name)) { + return Status.YES; + } // All these fields are single-valued so we can stop when the set is // empty return requiredFields.isEmpty() diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 7f1f0b9568209..c8360e468d725 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -446,6 +446,7 @@ protected void parseCreateField(ParseContext context, List field timestamp = fieldType().parse(dateAsString); } catch (IllegalArgumentException e) { if (ignoreMalformed.value()) { + context.addIgnoredField(fieldType.name()); return; } else { throw e; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 0cd200021701e..2ea31f67e2908 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -305,6 +305,7 @@ public Mapper parse(ParseContext context) throws IOException { if (ignoreMalformed.value() == false) { throw e; } + context.addIgnoredField(fieldType.name()); } token = context.parser().nextToken(); } @@ -352,6 +353,7 @@ public Mapper parse(ParseContext context) throws IOException { if (ignoreMalformed.value() == false) { throw e; } + context.addIgnoredField(fieldType.name()); } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 753d91f7be231..c0158f61c3af7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -506,6 +506,7 @@ public Mapper parse(ParseContext context) throws IOException { if (ignoreMalformed.value() == false) { throw new MapperParsingException("failed to parse [" + fieldType().name() + "]", e); } + context.addIgnoredField(fieldType.name()); } return null; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java new file mode 100644 index 0000000000000..69f1e36664ef6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java @@ -0,0 +1,154 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermRangeQuery; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.QueryShardContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * A field mapper that records fields that have been ignored because they were malformed. + */ +public final class IgnoredFieldMapper extends MetadataFieldMapper { + + public static final String NAME = "_ignored"; + + public static final String CONTENT_TYPE = "_ignored"; + + public static class Defaults { + public static final String NAME = IgnoredFieldMapper.NAME; + + public static final MappedFieldType FIELD_TYPE = new IgnoredFieldType(); + + static { + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setStored(true); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); + FIELD_TYPE.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER); + FIELD_TYPE.setName(NAME); + FIELD_TYPE.freeze(); + } + } + + public static class Builder extends MetadataFieldMapper.Builder { + + public Builder(MappedFieldType existing) { + super(Defaults.NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE); + } + + @Override + public IgnoredFieldMapper build(BuilderContext context) { + return new IgnoredFieldMapper(context.indexSettings()); + } + } + + public static class TypeParser implements MetadataFieldMapper.TypeParser { + @Override + public MetadataFieldMapper.Builder parse(String name, Map node, + ParserContext parserContext) throws MapperParsingException { + return new Builder(parserContext.mapperService().fullName(NAME)); + } + + @Override + public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) { + final Settings indexSettings = context.mapperService().getIndexSettings().getSettings(); + return new IgnoredFieldMapper(indexSettings); + } + } + + public static final class IgnoredFieldType extends TermBasedFieldType { + + public IgnoredFieldType() { + } + + protected IgnoredFieldType(IgnoredFieldType ref) { + super(ref); + } + + @Override + public IgnoredFieldType clone() { + return new IgnoredFieldType(this); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public Query existsQuery(QueryShardContext context) { + // This query is not performance sensitive, it only helps assess + // quality of the data, so we may use a slow query. It shouldn't + // be too slow in practice since the number of unique terms in this + // field is bounded by the number of fields in the mappings. + return new TermRangeQuery(name(), null, null, true, true); + } + + } + + private IgnoredFieldMapper(Settings indexSettings) { + super(NAME, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE, indexSettings); + } + + @Override + public void preParse(ParseContext context) throws IOException { + } + + @Override + public void postParse(ParseContext context) throws IOException { + super.parse(context); + } + + @Override + public Mapper parse(ParseContext context) throws IOException { + // done in post-parse + return null; + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + for (String field : context.getIgnoredFields()) { + context.doc().add(new Field(NAME, field, fieldType())); + } + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder; + } + +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index fd5dc080011e6..a8ef46b93060e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -370,6 +370,7 @@ protected void parseCreateField(ParseContext context, List field address = InetAddresses.forString(addressAsString); } catch (IllegalArgumentException e) { if (ignoreMalformed.value()) { + context.addIgnoredField(fieldType.name()); return; } else { throw e; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index c72187b649713..b28ea695f8287 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -112,7 +112,7 @@ public enum MergeReason { //also missing, not sure if on purpose. See IndicesModule#getMetadataMappers private static ObjectHashSet META_FIELDS = ObjectHashSet.from( "_id", "_type", "_routing", "_index", - "_size", "_timestamp", "_ttl" + "_size", "_timestamp", "_ttl", IgnoredFieldMapper.NAME ); private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(MapperService.class)); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 69793ca89b57d..9c327c5294efe 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -990,6 +990,7 @@ protected void parseCreateField(ParseContext context, List field numericValue = fieldType().type.parse(parser, coerce.value()); } catch (IllegalArgumentException e) { if (ignoreMalformed.value()) { + context.addIgnoredField(fieldType.name()); return; } else { throw e; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java index 8c2eda31ca198..b77ffee05caf0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java @@ -29,9 +29,12 @@ import org.elasticsearch.common.xcontent.XContentParser; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Set; public abstract class ParseContext implements Iterable{ @@ -286,6 +289,16 @@ public List getDynamicMappers() { public Iterator iterator() { return in.iterator(); } + + @Override + public void addIgnoredField(String field) { + in.addIgnoredField(field); + } + + @Override + public Collection getIgnoredFields() { + return in.getIgnoredFields(); + } } public static class InternalParseContext extends ParseContext { @@ -319,6 +332,8 @@ public static class InternalParseContext extends ParseContext { private boolean docsReversed = false; + private final Set ignoredFields = new HashSet<>(); + public InternalParseContext(@Nullable Settings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper, SourceToParse source, XContentParser parser) { this.indexSettings = indexSettings; @@ -453,6 +468,17 @@ void postParse() { public Iterator iterator() { return documents.iterator(); } + + + @Override + public void addIgnoredField(String field) { + ignoredFields.add(field); + } + + @Override + public Collection getIgnoredFields() { + return Collections.unmodifiableCollection(ignoredFields); + } } /** @@ -461,6 +487,17 @@ public Iterator iterator() { */ public abstract Iterable nonRootDocuments(); + + /** + * Add the given {@code field} to the set of ignored fields. + */ + public abstract void addIgnoredField(String field); + + /** + * Return the collection of fields that have been ignored so far. + */ + public abstract Collection getIgnoredFields(); + public abstract DocumentMapperParser docMapperParser(); /** diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 92faa0a71fda6..6c786763003c9 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; @@ -130,7 +131,10 @@ private static Map initBuiltInMetadataMa Map builtInMetadataMappers; // Use a LinkedHashMap for metadataMappers because iteration order matters builtInMetadataMappers = new LinkedHashMap<>(); - // ID first so it will be the first stored field to load (so will benefit from "fields: []" early termination + // _ignored first so that we always load it, even if only _id is requested + builtInMetadataMappers.put(IgnoredFieldMapper.NAME, new IgnoredFieldMapper.TypeParser()); + // ID second so it will be the first (if no ignored fields) stored field to load + // (so will benefit from "fields: []" early termination builtInMetadataMappers.put(IdFieldMapper.NAME, new IdFieldMapper.TypeParser()); builtInMetadataMappers.put(RoutingFieldMapper.NAME, new RoutingFieldMapper.TypeParser()); builtInMetadataMappers.put(IndexFieldMapper.NAME, new IndexFieldMapper.TypeParser()); diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index da7a42b22e3e6..bbb88b5fff640 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -43,6 +43,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.shard.ShardId; @@ -444,8 +445,13 @@ public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) t builder.field(Fields._SCORE, score); } for (DocumentField field : metaFields) { - Object value = field.getValue(); - builder.field(field.getName(), value); + // _ignored is the only multi-valued meta field + // TODO: can we avoid having an exception here? + if (field.getName().equals(IgnoredFieldMapper.NAME)) { + builder.field(field.getName(), field.getValues()); + } else { + builder.field(field.getName(), field.getValue()); + } } if (source != null) { XContentHelper.writeRawField(SourceFieldMapper.NAME, source, builder, params); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index 9d334cecb708f..c19965ac5f77a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -192,6 +192,7 @@ public void testIgnoreMalformed() throws Exception { IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(0, fields.length); + assertArrayEquals(new String[] { "field" }, doc.rootDoc().getValues("_ignored")); } public void testChangeFormat() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java new file mode 100644 index 0000000000000..4035383893d81 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +public class IgnoredFieldTypeTests extends FieldTypeTestCase { + + @Override + protected MappedFieldType createDefaultFieldType() { + return new IgnoredFieldMapper.IgnoredFieldType(); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 28a3a2f16f28b..a20c88fe69366 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -194,6 +194,7 @@ public void testIgnoreMalformed() throws Exception { IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(0, fields.length); + assertArrayEquals(new String[] { "field" }, doc.rootDoc().getValues("_ignored")); } public void testNullValue() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 66b90cdca3a3a..9167c0d5a7d97 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -255,6 +255,7 @@ private void doTestIgnoreMalformed(String type) throws IOException { IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(0, fields.length); + assertArrayEquals(new String[] { "field" }, doc.rootDoc().getValues("_ignored")); } public void testRejectNorms() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index 3ebb268a7b8bc..9b88c6ab8f20c 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; @@ -82,9 +83,9 @@ public Map getMetadataMappers() { } }); - private static String[] EXPECTED_METADATA_FIELDS = new String[]{IdFieldMapper.NAME, RoutingFieldMapper.NAME, - IndexFieldMapper.NAME, SourceFieldMapper.NAME, TypeFieldMapper.NAME, VersionFieldMapper.NAME, - SeqNoFieldMapper.NAME, FieldNamesFieldMapper.NAME}; + private static String[] EXPECTED_METADATA_FIELDS = new String[]{IgnoredFieldMapper.NAME, IdFieldMapper.NAME, + RoutingFieldMapper.NAME, IndexFieldMapper.NAME, SourceFieldMapper.NAME, TypeFieldMapper.NAME, + VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, FieldNamesFieldMapper.NAME}; public void testBuiltinMappers() { IndicesModule module = new IndicesModule(Collections.emptyList()); @@ -106,7 +107,7 @@ public void testBuiltinWithPlugins() { greaterThan(noPluginsModule.getMapperRegistry().getMetadataMapperParsers().size())); Map metadataMapperParsers = module.getMapperRegistry().getMetadataMapperParsers(); Iterator iterator = metadataMapperParsers.keySet().iterator(); - assertEquals(IdFieldMapper.NAME, iterator.next()); + assertEquals(IgnoredFieldMapper.NAME, iterator.next()); String last = null; while(iterator.hasNext()) { last = iterator.next(); From 646f7ae23c84bb6599a1b8b28988c636f7f56f58 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 25 Apr 2018 15:30:00 +0200 Subject: [PATCH 2/3] Add comment. --- .../org/elasticsearch/index/fieldvisitor/FieldsVisitor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java index 374a62dc52b6f..0151fc9ec62c6 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java @@ -71,6 +71,8 @@ public Status needsField(FieldInfo fieldInfo) throws IOException { return Status.YES; } // Always load _ignored to be explicit about ignored fields + // This works because _ignored is added as the first metadata mapper, + // so its stored fields always appear first in the list. if (IgnoredFieldMapper.NAME.equals(fieldInfo.name)) { return Status.YES; } From 39fe0745f319d279359b980748bfbe64e6f117a5 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 2 May 2018 10:45:47 +0200 Subject: [PATCH 3/3] iter --- docs/CHANGELOG.asciidoc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index 20a792d0f783e..a880a3c423ed2 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -71,8 +71,12 @@ written to by an older Elasticsearch after writing to it with a newer Elasticsea [[release-notes-6.4.0]] == {es} 6.4.0 -//[float] -//=== New Features +[float] +=== New Features + +The new <> field allows to know which fields +got ignored at index time because of the <> +option. ({pull}30140[#29658]) [float] === Enhancements