From 4d6ff4d1937405edb3319ece541c2adf00b844f4 Mon Sep 17 00:00:00 2001 From: Miroslav Pokorny Date: Mon, 16 Apr 2018 20:16:07 +1000 Subject: [PATCH] TextFieldMapper honour ignore_malformed and handle object/array values. - correct consumes object/array when ignore_malformed=true, so indexing can continue - fixes #12366 --- .../index/mapper/TextFieldMapper.java | 60 +++++++-- .../index/mapper/TextFieldMapperTests.java | 119 ++++++++++++++++++ 2 files changed, 171 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index e2f8eb4e64f63..c0e1d3b34f114 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -33,9 +33,11 @@ import org.apache.lucene.search.NormsFieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import org.elasticsearch.common.Explicit; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -82,6 +84,7 @@ public static class Builder extends FieldMapper.Builder ignoreMalformed(BuilderContext context) { + if (ignoreMalformed != null) { + return new Explicit<>(ignoreMalformed, true); + } + if (context.indexSettings() != null) { + return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false); + } + return NumberFieldMapper.Defaults.IGNORE_MALFORMED; + } + @Override public TextFieldMapper build(BuilderContext context) { if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) { @@ -168,8 +186,15 @@ public TextFieldMapper build(BuilderContext context) { prefixMapper = new PrefixFieldMapper(prefixFieldType, context.indexSettings()); } return new TextFieldMapper( - name, fieldType, defaultFieldType, positionIncrementGap, prefixMapper, - context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); + name, + fieldType, + defaultFieldType, + positionIncrementGap, + prefixMapper, + context.indexSettings(), + multiFieldsBuilder.build(this, context), + ignoreMalformed(context), + copyTo); } } @@ -212,6 +237,9 @@ public Mapper.Builder parse(String fieldName, Map node, ParserCo builder.indexPrefixes(minChars, maxChars); DocumentMapperParser.checkNoRemainingFields(propName, indexPrefix, parserContext.indexVersionCreated()); iterator.remove(); + } else if (propName.equals("ignore_malformed")) { + builder.ignoreMalformed(XContentMapValues.nodeBooleanValue(propNode, propName + ".ignore_malformed")); + iterator.remove(); } } return builder; @@ -468,10 +496,17 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { private int positionIncrementGap; private PrefixFieldMapper prefixFieldMapper; - - protected TextFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, - int positionIncrementGap, PrefixFieldMapper prefixFieldMapper, - Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { + private Explicit ignoreMalformed; + + protected TextFieldMapper(String simpleName, + MappedFieldType fieldType, + MappedFieldType defaultFieldType, + int positionIncrementGap, + PrefixFieldMapper prefixFieldMapper, + Settings indexSettings, + MultiFields multiFields, + Explicit ignoreMalformed, + CopyTo copyTo) { super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); assert fieldType.tokenized(); assert fieldType.hasDocValues() == false; @@ -480,6 +515,7 @@ protected TextFieldMapper(String simpleName, MappedFieldType fieldType, MappedFi } this.positionIncrementGap = positionIncrementGap; this.prefixFieldMapper = prefixFieldMapper; + this.ignoreMalformed = ignoreMalformed; } @Override @@ -493,11 +529,19 @@ public int getPositionIncrementGap() { @Override protected void parseCreateField(ParseContext context, List fields) throws IOException { - final String value; + String value = null; if (context.externalValueSet()) { value = context.externalValue().toString(); } else { - value = context.parser().textOrNull(); + final XContentParser parser = context.parser(); + try { + value = parser.textOrNull(); + } catch (final IllegalStateException failed) { + if (!ignoreMalformed.value()) { + throw failed; + } + parser.skipChildren(); + } } if (value == null) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 459fcb1d37731..e82e77fc913eb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -68,6 +68,13 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase { + private static final boolean FIELD1_IGNORE_MALFORMED = true; + private static final boolean FIELD1_DONT_IGNORE_MALFORMED = !FIELD1_IGNORE_MALFORMED; + + private static final String FIELD1 = "field1"; + private static final String FIELD2 = "field2"; + private static final String VALUE2 = "VALUE2"; + IndexService indexService; DocumentMapperParser parser; @@ -857,4 +864,116 @@ public void testIndexPrefixMapping() throws IOException { assertThat(e.getMessage(), containsString("Cannot set index_prefix on unindexed field [field]")); } } + + public void testString() throws Exception{ + final String value1 = "VALUE1"; + this.parseAndCheck(FIELD1_IGNORE_MALFORMED, + this.sourceWithString(value1), + value1); + } + + private BytesReference sourceWithString(final String value1) throws Exception{ + return BytesReference.bytes(XContentFactory.jsonBuilder() + .startObject() + .field(FIELD1, value1) // IGNORED + .field(FIELD2, VALUE2) + .endObject()); + } + + public void testNumber() throws Exception{ + this.parseAndCheck(FIELD1_IGNORE_MALFORMED, + this.sourceWithNumber(123), + "123"); + } + + private BytesReference sourceWithNumber(final int value) throws Exception{ + return BytesReference.bytes(XContentFactory.jsonBuilder() + .startObject() + .field(FIELD1, value) // IGNORED + .field(FIELD2, VALUE2) + .endObject()); + } + + public void testObject_ignore_malformed() throws Exception{ + this.parseAndCheck(FIELD1_IGNORE_MALFORMED, + this.sourceWithObject(), + null); + } + + public void testObject() throws Exception{ + this.parseFails(this.sourceWithObject()); + } + + private BytesReference sourceWithObject() throws Exception{ + return BytesReference.bytes(XContentFactory.jsonBuilder() + .startObject() + .startObject(FIELD1) // IGNORED + .field("field3", false) + .endObject() + .field(FIELD2, VALUE2) + .endObject()); + } + + public void testArray_ignore_malformed() throws Exception{ + this.parseAndCheck(FIELD1_IGNORE_MALFORMED, + this.sourceWithArray(), + null); + } + + public void testArray() throws Exception{ + this.parseFails(this.sourceWithArray()); + } + + private BytesReference sourceWithArray() throws Exception{ + return BytesReference.bytes(XContentFactory.jsonBuilder() + .startObject() + .startArray(FIELD1) // IGNORED + .startObject() + .endObject() + .endArray() + .field(FIELD2, VALUE2) + .endObject()); + } + + private void parseAndCheck(final boolean field1IgnoreMalformed, + final BytesReference source, + final String expected1) throws IOException{ + final ParseContext.Document document = this.parse(field1IgnoreMalformed, source); + assertEquals(FIELD2, VALUE2, document.get(FIELD2)); + assertEquals(FIELD1, expected1, document.get(FIELD1)); + } + + private void parseFails(final BytesReference source){ + parseFails(FIELD1_DONT_IGNORE_MALFORMED, source); + } + + private void parseFails(final boolean field1IgnoreMalformed, + final BytesReference source) { + MapperParsingException ex = expectThrows(MapperParsingException.class, + () -> parse(field1IgnoreMalformed, source)); + assertEquals("failed to parse [field1]", ex.getMessage()); + } + + private ParseContext.Document parse(final boolean field1IgnoreMalformed, final BytesReference source) throws IOException{ + final String mapping = this.mapping(field1IgnoreMalformed); + final DocumentMapper mapper = this.parser.parse("type", new CompressedXContent(mapping)); + return mapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON)).rootDoc(); + } + + private String mapping(final boolean field1IgnoreMalformed) throws IOException{ + return Strings.toString(XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("properties") + .startObject(FIELD1) + .field("type", "text") + .field("ignore_malformed", field1IgnoreMalformed) + .endObject() + .startObject(FIELD2) + .field("type", "text") + .endObject() + .endObject() + .endObject() + .endObject()); + } }