Skip to content

Commit 0293c40

Browse files
nknizenknize
committed
Add ignore_malformed to geo_shape fields
This commit adds ignore_malformed support to geo_shape field types to skip malformed geoJson fields. closes #23747
1 parent 0a50fca commit 0293c40

File tree

4 files changed

+115
-8
lines changed

4 files changed

+115
-8
lines changed

‎core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java‎

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import org.apache.lucene.index.IndexOptions;
2222
import org.apache.lucene.index.IndexableField;
2323
import org.apache.lucene.index.Term;
24-
import org.apache.lucene.search.DocValuesFieldExistsQuery;
2524
import org.apache.lucene.search.Query;
2625
import org.apache.lucene.search.TermQuery;
2726
import org.apache.lucene.spatial.prefix.PrefixTreeStrategy;
@@ -54,6 +53,8 @@
5453
import java.util.Map;
5554
import java.util.Objects;
5655

56+
import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_MALFORMED;
57+
5758
/**
5859
* FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s.
5960
* <p>
@@ -96,6 +97,7 @@ public static class Defaults {
9697
public static final Orientation ORIENTATION = Orientation.RIGHT;
9798
public static final double LEGACY_DISTANCE_ERROR_PCT = 0.025d;
9899
public static final Explicit<Boolean> COERCE = new Explicit<>(false, false);
100+
public static final Explicit<Boolean> IGNORE_MALFORMED = new Explicit<>(false, false);
99101

100102
public static final MappedFieldType FIELD_TYPE = new GeoShapeFieldType();
101103

@@ -115,6 +117,7 @@ public static class Defaults {
115117
public static class Builder extends FieldMapper.Builder<Builder, GeoShapeFieldMapper> {
116118

117119
private Boolean coerce;
120+
private Boolean ignoreMalformed;
118121

119122
public Builder(String name) {
120123
super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
@@ -145,6 +148,21 @@ protected Explicit<Boolean> coerce(BuilderContext context) {
145148
return Defaults.COERCE;
146149
}
147150

151+
public Builder ignoreMalformed(boolean ignoreMalformed) {
152+
this.ignoreMalformed = ignoreMalformed;
153+
return builder;
154+
}
155+
156+
protected Explicit<Boolean> ignoreMalformed(BuilderContext context) {
157+
if (ignoreMalformed != null) {
158+
return new Explicit<>(ignoreMalformed, true);
159+
}
160+
if (context.indexSettings() != null) {
161+
return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false);
162+
}
163+
return Defaults.IGNORE_MALFORMED;
164+
}
165+
148166
@Override
149167
public GeoShapeFieldMapper build(BuilderContext context) {
150168
GeoShapeFieldType geoShapeFieldType = (GeoShapeFieldType)fieldType;
@@ -154,8 +172,8 @@ public GeoShapeFieldMapper build(BuilderContext context) {
154172
}
155173
setupFieldType(context);
156174

157-
return new GeoShapeFieldMapper(name, fieldType, coerce(context), context.indexSettings(), multiFieldsBuilder.build(this,
158-
context), copyTo);
175+
return new GeoShapeFieldMapper(name, fieldType, ignoreMalformed(context), coerce(context), context.indexSettings(),
176+
multiFieldsBuilder.build(this, context), copyTo);
159177
}
160178
}
161179

@@ -186,6 +204,9 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
186204
} else if (Names.STRATEGY.equals(fieldName)) {
187205
builder.fieldType().setStrategyName(fieldNode.toString());
188206
iterator.remove();
207+
} else if (IGNORE_MALFORMED.equals(fieldName)) {
208+
builder.ignoreMalformed(TypeParsers.nodeBooleanValue(fieldName, "ignore_malformed", fieldNode, parserContext));
209+
iterator.remove();
189210
} else if (Names.COERCE.equals(fieldName)) {
190211
builder.coerce(TypeParsers.nodeBooleanValue(fieldName, Names.COERCE, fieldNode, parserContext));
191212
iterator.remove();
@@ -428,11 +449,13 @@ public Query termQuery(Object value, QueryShardContext context) {
428449
}
429450

430451
protected Explicit<Boolean> coerce;
452+
protected Explicit<Boolean> ignoreMalformed;
431453

432-
public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, Explicit<Boolean> coerce, Settings indexSettings,
433-
MultiFields multiFields, CopyTo copyTo) {
454+
public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, Explicit<Boolean> ignoreMalformed,
455+
Explicit<Boolean> coerce, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
434456
super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, multiFields, copyTo);
435457
this.coerce = coerce;
458+
this.ignoreMalformed = ignoreMalformed;
436459
}
437460

438461
@Override
@@ -461,7 +484,9 @@ public Mapper parse(ParseContext context) throws IOException {
461484
context.doc().add(field);
462485
}
463486
} catch (Exception e) {
464-
throw new MapperParsingException("failed to parse [" + fieldType().name() + "]", e);
487+
if (ignoreMalformed.value() == false) {
488+
throw new MapperParsingException("failed to parse [" + fieldType().name() + "]", e);
489+
}
465490
}
466491
return null;
467492
}
@@ -478,6 +503,9 @@ protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
478503
if (gsfm.coerce.explicit()) {
479504
this.coerce = gsfm.coerce;
480505
}
506+
if (gsfm.ignoreMalformed.explicit()) {
507+
this.ignoreMalformed = gsfm.ignoreMalformed;
508+
}
481509
}
482510

483511
@Override
@@ -506,14 +534,21 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
506534
builder.field(Names.STRATEGY_POINTS_ONLY, fieldType().pointsOnly());
507535
}
508536
if (includeDefaults || coerce.explicit()) {
509-
builder.field("coerce", coerce.value());
537+
builder.field(Names.COERCE, coerce.value());
538+
}
539+
if (includeDefaults || ignoreMalformed.explicit()) {
540+
builder.field(IGNORE_MALFORMED, ignoreMalformed.value());
510541
}
511542
}
512543

513544
public Explicit<Boolean> coerce() {
514545
return coerce;
515546
}
516547

548+
public Explicit<Boolean> ignoreMalformed() {
549+
return ignoreMalformed;
550+
}
551+
517552
@Override
518553
protected String contentType() {
519554
return CONTENT_TYPE;

‎core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
2323
import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
2424
import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
25+
import org.elasticsearch.common.Explicit;
2526
import org.elasticsearch.common.compress.CompressedXContent;
2627
import org.elasticsearch.common.geo.GeoUtils;
2728
import org.elasticsearch.common.geo.builders.ShapeBuilder;
@@ -103,7 +104,7 @@ public void testOrientationParsing() throws IOException {
103104
}
104105

105106
/**
106-
* Test that orientation parameter correctly parses
107+
* Test that coerce parameter correctly parses
107108
*/
108109
public void testCoerceParsing() throws IOException {
109110
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
@@ -136,6 +137,41 @@ public void testCoerceParsing() throws IOException {
136137
assertThat(coerce, equalTo(false));
137138
}
138139

140+
/**
141+
* Test that ignore_malformed parameter correctly parses
142+
*/
143+
public void testIgnoreMalformedParsing() throws IOException {
144+
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
145+
.startObject("properties").startObject("location")
146+
.field("type", "geo_shape")
147+
.field("ignore_malformed", "true")
148+
.endObject().endObject()
149+
.endObject().endObject().string();
150+
151+
DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
152+
FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
153+
assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
154+
155+
Explicit<Boolean> ignoreMalformed = ((GeoShapeFieldMapper)fieldMapper).ignoreMalformed();
156+
assertThat(ignoreMalformed.value(), equalTo(true));
157+
158+
// explicit false ignore_malformed test
159+
mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
160+
.startObject("properties").startObject("location")
161+
.field("type", "geo_shape")
162+
.field("ignore_malformed", "false")
163+
.endObject().endObject()
164+
.endObject().endObject().string();
165+
166+
defaultMapper = createIndex("test2").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
167+
fieldMapper = defaultMapper.mappers().getMapper("location");
168+
assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
169+
170+
ignoreMalformed = ((GeoShapeFieldMapper)fieldMapper).ignoreMalformed();
171+
assertThat(ignoreMalformed.explicit(), equalTo(true));
172+
assertThat(ignoreMalformed.value(), equalTo(false));
173+
}
174+
139175
public void testGeohashConfiguration() throws IOException {
140176
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
141177
.startObject("properties").startObject("location")

‎core/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*/
1919
package org.elasticsearch.search.geo;
2020

21+
import org.elasticsearch.action.search.SearchResponse;
2122
import org.elasticsearch.cluster.ClusterState;
2223
import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
2324
import org.elasticsearch.common.geo.builders.ShapeBuilder;
@@ -29,6 +30,7 @@
2930
import org.elasticsearch.indices.IndicesService;
3031
import org.elasticsearch.test.ESIntegTestCase;
3132

33+
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
3234
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
3335
import static org.hamcrest.Matchers.equalTo;
3436
import static org.hamcrest.Matchers.instanceOf;
@@ -88,6 +90,36 @@ public void testOrientationPersistence() throws Exception {
8890
assertThat(orientation, equalTo(ShapeBuilder.Orientation.CCW));
8991
}
9092

93+
/**
94+
* Test that ignore_malformed on GeoShapeFieldMapper does not fail the entire document
95+
*/
96+
public void testIgnoreMalformed() throws Exception {
97+
// create index
98+
assertAcked(client().admin().indices().prepareCreate("test")
99+
.addMapping("geometry", "shape", "type=geo_shape,ignore_malformed=true").get());
100+
ensureGreen();
101+
102+
// test self crossing ccw poly not crossing dateline
103+
String polygonGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Polygon")
104+
.startArray("coordinates")
105+
.startArray()
106+
.startArray().value(176.0).value(15.0).endArray()
107+
.startArray().value(-177.0).value(10.0).endArray()
108+
.startArray().value(-177.0).value(-10.0).endArray()
109+
.startArray().value(176.0).value(-15.0).endArray()
110+
.startArray().value(-177.0).value(15.0).endArray()
111+
.startArray().value(172.0).value(0.0).endArray()
112+
.startArray().value(176.0).value(15.0).endArray()
113+
.endArray()
114+
.endArray()
115+
.endObject().string();
116+
117+
indexRandom(true, client().prepareIndex("test", "geometry", "0").setSource("shape",
118+
polygonGeoJson));
119+
SearchResponse searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).get();
120+
assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
121+
}
122+
91123
private String findNodeName(String index) {
92124
ClusterState state = client().admin().cluster().prepareState().get().getState();
93125
IndexShardRoutingTable shard = state.getRoutingTable().index(index).shard(0);

‎docs/reference/mapping/types/geo-shape.asciidoc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,10 @@ by improving point performance on a `geo_shape` field so that `geo_shape` querie
8686
optimal on a point only field.
8787
| `false`
8888

89+
|`ignore_malformed` |If true, malformed geojson shapes are ignored. If false (default),
90+
malformed geojson shapes throw an exception and reject the whole document.
91+
| `false`
92+
8993

9094
|=======================================================================
9195

0 commit comments

Comments
 (0)