Skip to content

TextFieldMapper honour ignore_malformed and handle object/array values. #29532

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

TextFieldMapper honour ignore_malformed and handle object/array values. #29532

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 52 additions & 8 deletions server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,11 @@
import org.apache.lucene.search.NormsFieldExistsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.collect.Iterators;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.NamedAnalyzer;
Expand Down Expand Up @@ -82,6 +84,7 @@ public static class Builder extends FieldMapper.Builder<Builder, TextFieldMapper

private int positionIncrementGap = POSITION_INCREMENT_GAP_USE_ANALYZER;
private PrefixFieldType prefixFieldType;
private Boolean ignoreMalformed;

public Builder(String name) {
super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
Expand Down Expand Up @@ -141,6 +144,21 @@ public Builder indexPrefixes(int minChars, int maxChars) {
return this;
}

public Builder ignoreMalformed(boolean ignoreMalformed) {
this.ignoreMalformed = ignoreMalformed;
return builder;
}

protected Explicit<Boolean> ignoreMalformed(BuilderContext context) {
if (ignoreMalformed != null) {
return new Explicit<>(ignoreMalformed, true);
}
if (context.indexSettings() != null) {
return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false);
}
return NumberFieldMapper.Defaults.IGNORE_MALFORMED;
}

@Override
public TextFieldMapper build(BuilderContext context) {
if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) {
Expand Down Expand Up @@ -168,8 +186,15 @@ public TextFieldMapper build(BuilderContext context) {
prefixMapper = new PrefixFieldMapper(prefixFieldType, context.indexSettings());
}
return new TextFieldMapper(
name, fieldType, defaultFieldType, positionIncrementGap, prefixMapper,
context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
name,
fieldType,
defaultFieldType,
positionIncrementGap,
prefixMapper,
context.indexSettings(),
multiFieldsBuilder.build(this, context),
ignoreMalformed(context),
copyTo);
}
}

Expand Down Expand Up @@ -212,6 +237,9 @@ public Mapper.Builder parse(String fieldName, Map<String, Object> node, ParserCo
builder.indexPrefixes(minChars, maxChars);
DocumentMapperParser.checkNoRemainingFields(propName, indexPrefix, parserContext.indexVersionCreated());
iterator.remove();
} else if (propName.equals("ignore_malformed")) {
builder.ignoreMalformed(XContentMapValues.nodeBooleanValue(propNode, propName + ".ignore_malformed"));
iterator.remove();
}
}
return builder;
Expand Down Expand Up @@ -468,10 +496,17 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {

private int positionIncrementGap;
private PrefixFieldMapper prefixFieldMapper;

protected TextFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
int positionIncrementGap, PrefixFieldMapper prefixFieldMapper,
Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
private Explicit<Boolean> ignoreMalformed;

protected TextFieldMapper(String simpleName,
MappedFieldType fieldType,
MappedFieldType defaultFieldType,
int positionIncrementGap,
PrefixFieldMapper prefixFieldMapper,
Settings indexSettings,
MultiFields multiFields,
Explicit<Boolean> ignoreMalformed,
CopyTo copyTo) {
super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
assert fieldType.tokenized();
assert fieldType.hasDocValues() == false;
Expand All @@ -480,6 +515,7 @@ protected TextFieldMapper(String simpleName, MappedFieldType fieldType, MappedFi
}
this.positionIncrementGap = positionIncrementGap;
this.prefixFieldMapper = prefixFieldMapper;
this.ignoreMalformed = ignoreMalformed;
}

@Override
Expand All @@ -493,11 +529,19 @@ public int getPositionIncrementGap() {

@Override
protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
final String value;
String value = null;
if (context.externalValueSet()) {
value = context.externalValue().toString();
} else {
value = context.parser().textOrNull();
final XContentParser parser = context.parser();
try {
value = parser.textOrNull();
} catch (final IllegalStateException failed) {
if (!ignoreMalformed.value()) {
throw failed;
}
parser.skipChildren();
}
}

if (value == null) {
Expand Down
119 changes: 119 additions & 0 deletions server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,13 @@

public class TextFieldMapperTests extends ESSingleNodeTestCase {

private static final boolean FIELD1_IGNORE_MALFORMED = true;
private static final boolean FIELD1_DONT_IGNORE_MALFORMED = !FIELD1_IGNORE_MALFORMED;

private static final String FIELD1 = "field1";
private static final String FIELD2 = "field2";
private static final String VALUE2 = "VALUE2";

IndexService indexService;
DocumentMapperParser parser;

Expand Down Expand Up @@ -857,4 +864,116 @@ public void testIndexPrefixMapping() throws IOException {
assertThat(e.getMessage(), containsString("Cannot set index_prefix on unindexed field [field]"));
}
}

public void testString() throws Exception{
final String value1 = "VALUE1";
this.parseAndCheck(FIELD1_IGNORE_MALFORMED,
this.sourceWithString(value1),
value1);
}

private BytesReference sourceWithString(final String value1) throws Exception{
return BytesReference.bytes(XContentFactory.jsonBuilder()
.startObject()
.field(FIELD1, value1) // IGNORED
.field(FIELD2, VALUE2)
.endObject());
}

public void testNumber() throws Exception{
this.parseAndCheck(FIELD1_IGNORE_MALFORMED,
this.sourceWithNumber(123),
"123");
}

private BytesReference sourceWithNumber(final int value) throws Exception{
return BytesReference.bytes(XContentFactory.jsonBuilder()
.startObject()
.field(FIELD1, value) // IGNORED
.field(FIELD2, VALUE2)
.endObject());
}

public void testObject_ignore_malformed() throws Exception{
this.parseAndCheck(FIELD1_IGNORE_MALFORMED,
this.sourceWithObject(),
null);
}

public void testObject() throws Exception{
this.parseFails(this.sourceWithObject());
}

private BytesReference sourceWithObject() throws Exception{
return BytesReference.bytes(XContentFactory.jsonBuilder()
.startObject()
.startObject(FIELD1) // IGNORED
.field("field3", false)
.endObject()
.field(FIELD2, VALUE2)
.endObject());
}

public void testArray_ignore_malformed() throws Exception{
this.parseAndCheck(FIELD1_IGNORE_MALFORMED,
this.sourceWithArray(),
null);
}

public void testArray() throws Exception{
this.parseFails(this.sourceWithArray());
}

private BytesReference sourceWithArray() throws Exception{
return BytesReference.bytes(XContentFactory.jsonBuilder()
.startObject()
.startArray(FIELD1) // IGNORED
.startObject()
.endObject()
.endArray()
.field(FIELD2, VALUE2)
.endObject());
}

private void parseAndCheck(final boolean field1IgnoreMalformed,
final BytesReference source,
final String expected1) throws IOException{
final ParseContext.Document document = this.parse(field1IgnoreMalformed, source);
assertEquals(FIELD2, VALUE2, document.get(FIELD2));
assertEquals(FIELD1, expected1, document.get(FIELD1));
}

private void parseFails(final BytesReference source){
parseFails(FIELD1_DONT_IGNORE_MALFORMED, source);
}

private void parseFails(final boolean field1IgnoreMalformed,
final BytesReference source) {
MapperParsingException ex = expectThrows(MapperParsingException.class,
() -> parse(field1IgnoreMalformed, source));
assertEquals("failed to parse [field1]", ex.getMessage());
}

private ParseContext.Document parse(final boolean field1IgnoreMalformed, final BytesReference source) throws IOException{
final String mapping = this.mapping(field1IgnoreMalformed);
final DocumentMapper mapper = this.parser.parse("type", new CompressedXContent(mapping));
return mapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON)).rootDoc();
}

private String mapping(final boolean field1IgnoreMalformed) throws IOException{
return Strings.toString(XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject(FIELD1)
.field("type", "text")
.field("ignore_malformed", field1IgnoreMalformed)
.endObject()
.startObject(FIELD2)
.field("type", "text")
.endObject()
.endObject()
.endObject()
.endObject());
}
}