From 15c65172f0a1c6276ad1f3395e88403390c9ca31 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Tue, 6 Nov 2018 16:15:30 -0600 Subject: [PATCH 01/38] [Geo] Expose BKDBackedGeoShapes as new VECTOR strategy This commit exposes lucene's LatLonShape field as a new strategy in GeoShapeFieldMapper. To use the new indexing approach, strategy should be set to "vector" in the geo_shape field mapper. If the tree parameter is set the mapper will throw an IAE. Note the following: When using vector strategy: * geo_shape query does not support querying by POINT, MULTIPOINT, or GEOMETRYCOLLECTION. * LINESTRING and MULTILINESTRING queries do not support WITHIN relation. * CONTAINS relation is not supported. * The tree, precision, tree_levels, distance_error_pct, and points_only parameters will not throw an exception but they have no effect and will be marked as deprecated.. All other features are supported. --- .../percolator/PercolatorFieldMapper.java | 2 +- .../common/geo/ShapeRelation.java | 12 + .../common/geo/SpatialStrategy.java | 5 +- .../common/geo/builders/CircleBuilder.java | 7 +- .../common/geo/builders/EnvelopeBuilder.java | 7 +- .../builders/GeometryCollectionBuilder.java | 24 +- .../geo/builders/LineStringBuilder.java | 32 +- .../geo/builders/MultiLineStringBuilder.java | 26 +- .../geo/builders/MultiPointBuilder.java | 13 +- .../geo/builders/MultiPolygonBuilder.java | 34 ++- .../common/geo/builders/PointBuilder.java | 8 +- .../common/geo/builders/PolygonBuilder.java | 92 +++++- .../common/geo/builders/ShapeBuilder.java | 4 +- .../index/mapper/GeoShapeFieldMapper.java | 284 +++++++++++++----- .../query/GeoBoundingBoxQueryBuilder.java | 2 +- .../index/query/GeoShapeQueryBuilder.java | 99 ++++-- .../elasticsearch/indices/IndicesModule.java | 6 +- .../common/geo/BaseGeoParsingTestCase.java | 2 +- .../common/geo/GeoJsonShapeParserTests.java | 30 +- .../common/geo/GeoWKTShapeParserTests.java | 4 +- .../common/geo/ShapeBuilderTests.java | 74 ++--- .../geo/builders/PolygonBuilderTests.java | 8 +- .../index/mapper/ExternalMapper.java | 10 +- .../ExternalValuesMapperIntegrationIT.java | 6 +- .../mapper/GeoShapeFieldMapperTests.java | 36 ++- .../index/mapper/GeoShapeFieldTypeTests.java | 8 +- .../query/GeoPolygonQueryBuilderTests.java | 2 +- .../query/GeoShapeQueryBuilderTests.java | 28 +- .../index/query/MatchQueryBuilderTests.java | 1 + .../elasticsearch/search/geo/GeoFilterIT.java | 13 +- .../search/geo/GeoShapeIntegrationIT.java | 5 +- .../search/geo/GeoShapeQueryTests.java | 86 +++--- .../test/geo/RandomShapeGenerator.java | 4 +- .../hamcrest/ElasticsearchGeoAssertions.java | 2 +- 34 files changed, 693 insertions(+), 283 deletions(-) diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 6ac073ef90a02..9df921b0ae10f 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -327,7 +327,7 @@ Tuple, Map>> extractTermsAndRanges(IndexRead extractedTerms.add(builder.toBytesRef()); } } - if (info.getPointIndexDimensionCount() == 1) { // not != 0 because range fields are not supported + if (info.getPointDataDimensionCount() == 1) { // not != 0 because range fields are not supported PointValues values = reader.getPointValues(info.name); List encodedPointValues = new ArrayList<>(); encodedPointValues.add(values.getMinPackedValue().clone()); diff --git a/server/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java b/server/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java index e83e18ce43255..2156c858bbc83 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java +++ b/server/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.geo; +import org.apache.lucene.document.LatLonShape.QueryRelation; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -62,6 +63,17 @@ public static ShapeRelation getRelationByName(String name) { return null; } + /** Maps ShapeRelation to Lucene's LatLonShapeRelation */ + public QueryRelation getLuceneRelation() { + switch (this) { + case INTERSECTS: return QueryRelation.INTERSECTS; + case DISJOINT: return QueryRelation.DISJOINT; + case WITHIN: return QueryRelation.WITHIN; + default: + throw new IllegalArgumentException("ShapeRelation [" + CONTAINS + "] not supported"); + } + } + public String getRelationName() { return relationName; } diff --git a/server/src/main/java/org/elasticsearch/common/geo/SpatialStrategy.java b/server/src/main/java/org/elasticsearch/common/geo/SpatialStrategy.java index 0b4f640fd2884..fe0e8b867b5da 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/SpatialStrategy.java +++ b/server/src/main/java/org/elasticsearch/common/geo/SpatialStrategy.java @@ -27,7 +27,8 @@ public enum SpatialStrategy implements Writeable { TERM("term"), - RECURSIVE("recursive"); + RECURSIVE("recursive"), + VECTOR("vector"); private final String strategyName; @@ -50,7 +51,7 @@ public void writeTo(StreamOutput out) throws IOException { public static SpatialStrategy fromString(String strategyName) { for (SpatialStrategy strategy : values()) { - if (strategy.strategyName.equals(strategyName)) { + if (strategy.strategyName.equalsIgnoreCase(strategyName)) { return strategy; } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/CircleBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/CircleBuilder.java index c6a0743980fd3..1befb4261773e 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/CircleBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/CircleBuilder.java @@ -159,10 +159,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public Circle build() { + public Circle buildS4J() { return SPATIAL_CONTEXT.makeCircle(center.x, center.y, 360 * radius / unit.getEarthCircumference()); } + @Override + public Object buildLucene() { + throw new UnsupportedOperationException("CIRCLE geometry not yet supported"); + } + @Override public GeoShapeType type() { return TYPE; diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java index a878a7c6d8618..5f69f4ad44dba 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java @@ -108,10 +108,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public Rectangle build() { + public Rectangle buildS4J() { return SPATIAL_CONTEXT.makeRectangle(topLeft.x, bottomRight.x, bottomRight.y, topLeft.y); } + @Override + public org.apache.lucene.geo.Rectangle buildLucene() { + return new org.apache.lucene.geo.Rectangle(bottomRight.y, topLeft.y, topLeft.x, bottomRight.x); + } + @Override public GeoShapeType type() { return TYPE; diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java index d2ff86ea63ced..ba40e8c250b64 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -168,12 +169,12 @@ public int numDimensions() { } @Override - public Shape build() { + public Shape buildS4J() { List shapes = new ArrayList<>(this.shapes.size()); for (ShapeBuilder shape : this.shapes) { - shapes.add(shape.build()); + shapes.add(shape.buildS4J()); } if (shapes.size() == 1) @@ -183,6 +184,25 @@ public Shape build() { //note: ShapeCollection is probably faster than a Multi* geom. } + @Override + public Object buildLucene() { + List shapes = new ArrayList<>(this.shapes.size()); + + for (ShapeBuilder shape : this.shapes) { + Object o = shape.buildLucene(); + if (o.getClass().isArray()) { + shapes.addAll(Arrays.asList(o)); + } else { + shapes.add(o); + } + } + + if (shapes.size() == 1) { + return shapes.get(0); + } + return shapes.toArray(new Object[shapes.size()]); + } + @Override public int hashCode() { return Objects.hash(shapes); diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java index 035c4566a5763..ec081ffdd2507 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.geo.builders; +import org.apache.lucene.geo.Line; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryFactory; @@ -101,11 +102,11 @@ public int numDimensions() { } @Override - public JtsGeometry build() { + public JtsGeometry buildS4J() { Coordinate[] coordinates = this.coordinates.toArray(new Coordinate[this.coordinates.size()]); Geometry geometry; if(wrapdateline) { - ArrayList strings = decompose(FACTORY, coordinates, new ArrayList()); + ArrayList strings = decomposeS4J(FACTORY, coordinates, new ArrayList()); if(strings.size() == 1) { geometry = strings.get(0); @@ -120,7 +121,23 @@ public JtsGeometry build() { return jtsGeometry(geometry); } - static ArrayList decompose(GeometryFactory factory, Coordinate[] coordinates, ArrayList strings) { + @Override + public Object buildLucene() { + // decompose linestrings crossing dateline into array of Lines + Coordinate[] coordinates = this.coordinates.toArray(new Coordinate[this.coordinates.size()]); + if (wrapdateline) { + ArrayList strings = decomposeLucene(coordinates, new ArrayList<>()); + if (strings.size() == 1) { + return strings.get(0); + } else { + return strings.toArray(new Line[strings.size()]); + } + } + return new Line(Arrays.stream(coordinates).mapToDouble(i->i.y).toArray(), + Arrays.stream(coordinates).mapToDouble(i->i.x).toArray()); + } + + static ArrayList decomposeS4J(GeometryFactory factory, Coordinate[] coordinates, ArrayList strings) { for(Coordinate[] part : decompose(+DATELINE, coordinates)) { for(Coordinate[] line : decompose(-DATELINE, part)) { strings.add(factory.createLineString(line)); @@ -129,6 +146,15 @@ static ArrayList decompose(GeometryFactory factory, Coordinate[] coo return strings; } + static ArrayList decomposeLucene(Coordinate[] coordinates, ArrayList lines) { + for (Coordinate[] part : decompose(+DATELINE, coordinates)) { + for (Coordinate[] line : decompose(-DATELINE, part)) { + lines.add(new Line(Arrays.stream(line).mapToDouble(i->i.y).toArray(), Arrays.stream(line).mapToDouble(i->i.x).toArray())); + } + } + return lines; + } + /** * Decompose a linestring given as array of coordinates at a vertical line. * diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java index 68da45bbf0c68..9902744fc3b2c 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.geo.builders; +import org.apache.lucene.geo.Line; import org.elasticsearch.common.geo.GeoShapeType; import org.elasticsearch.common.geo.parsers.GeoWKTParser; import org.elasticsearch.common.geo.parsers.ShapeParser; @@ -124,12 +125,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public JtsGeometry build() { + public JtsGeometry buildS4J() { final Geometry geometry; if(wrapdateline) { ArrayList parts = new ArrayList<>(); for (LineStringBuilder line : lines) { - LineStringBuilder.decompose(FACTORY, line.coordinates(false), parts); + LineStringBuilder.decomposeS4J(FACTORY, line.coordinates(false), parts); } if(parts.size() == 1) { geometry = parts.get(0); @@ -148,6 +149,27 @@ public JtsGeometry build() { return jtsGeometry(geometry); } + @Override + public Object buildLucene() { + if (wrapdateline) { + ArrayList parts = new ArrayList<>(); + for (LineStringBuilder line : lines) { + LineStringBuilder.decomposeLucene(line.coordinates(false), parts); + } + if (parts.size() == 1) { + return parts.get(0); + } + return parts.toArray(new Line[parts.size()]); + } + Line[] linestrings = new Line[lines.size()]; + for (int i = 0; i < lines.size(); ++i) { + LineStringBuilder lsb = lines.get(i); + linestrings[i] = new Line(lsb.coordinates.stream().mapToDouble(c->c.y).toArray(), + lsb.coordinates.stream().mapToDouble(c->c.x).toArray()); + } + return linestrings; + } + @Override public int hashCode() { return Objects.hash(lines); diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java index c39cc397a34ed..b559bb581179e 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java @@ -61,7 +61,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public XShapeCollection build() { + public XShapeCollection buildS4J() { //Could wrap JtsGeometry but probably slower due to conversions to/from JTS in relate() //MultiPoint geometry = FACTORY.createMultiPoint(points.toArray(new Coordinate[points.size()])); List shapes = new ArrayList<>(coordinates.size()); @@ -73,6 +73,17 @@ public XShapeCollection build() { return multiPoints; } + @Override + public double[][] buildLucene() { + double[][] points = new double[coordinates.size()][]; + Coordinate coord; + for (int i = 0; i < coordinates.size(); ++i) { + coord = coordinates.get(i); + points[i] = new double[] {coord.x, coord.y}; + } + return points; + } + @Override public GeoShapeType type() { return TYPE; diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java index bac74c29dd805..95c2bbc6275a6 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Locale; import java.util.Objects; @@ -162,19 +163,19 @@ public int numDimensions() { } @Override - public Shape build() { + public Shape buildS4J() { List shapes = new ArrayList<>(this.polygons.size()); if(wrapdateline) { for (PolygonBuilder polygon : this.polygons) { for(Coordinate[][] part : polygon.coordinates()) { - shapes.add(jtsGeometry(PolygonBuilder.polygon(FACTORY, part))); + shapes.add(jtsGeometry(PolygonBuilder.polygonS4J(FACTORY, part))); } } } else { for (PolygonBuilder polygon : this.polygons) { - shapes.add(jtsGeometry(polygon.toPolygon(FACTORY))); + shapes.add(jtsGeometry(polygon.toPolygonS4J(FACTORY))); } } if (shapes.size() == 1) @@ -184,6 +185,33 @@ public Shape build() { //note: ShapeCollection is probably faster than a Multi* geom. } + @Override + public Object buildLucene() { + List shapes = new ArrayList<>(this.polygons.size()); + Object poly; + if (wrapdateline) { + for (PolygonBuilder polygon : this.polygons) { + poly = polygon.buildLucene(); + if (poly instanceof org.apache.lucene.geo.Polygon[]) { + shapes.addAll(Arrays.asList((org.apache.lucene.geo.Polygon[])poly)); + } else { + shapes.add((org.apache.lucene.geo.Polygon)poly); + } + } + } else { + for (int i = 0; i < this.polygons.size(); ++i) { + PolygonBuilder pb = this.polygons.get(i); + poly = pb.buildLucene(); + if (poly instanceof org.apache.lucene.geo.Polygon[]) { + shapes.addAll(Arrays.asList((org.apache.lucene.geo.Polygon[])poly)); + } else { + shapes.add((org.apache.lucene.geo.Polygon)poly); + } + } + } + return shapes.stream().toArray(org.apache.lucene.geo.Polygon[]::new); + } + @Override public int hashCode() { return Objects.hash(polygons, orientation); diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/PointBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/PointBuilder.java index f8c18f317b63a..c13eca936e492 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/PointBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/PointBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.geo.builders; +import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoShapeType; import org.elasticsearch.common.geo.parsers.ShapeParser; import org.locationtech.spatial4j.shape.Point; @@ -84,10 +85,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public Point build() { + public Point buildS4J() { return SPATIAL_CONTEXT.makePoint(coordinates.get(0).x, coordinates.get(0).y); } + @Override + public GeoPoint buildLucene() { + return new GeoPoint(coordinates.get(0).y, coordinates.get(0).x); + } + @Override public GeoShapeType type() { return TYPE; diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java index e4d48b8df4566..11f399d06cf90 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java @@ -226,8 +226,19 @@ public Coordinate[][][] coordinates() { } @Override - public JtsGeometry build() { - return jtsGeometry(buildGeometry(FACTORY, wrapdateline)); + public JtsGeometry buildS4J() { + return jtsGeometry(buildS4JGeometry(FACTORY, wrapdateline)); + } + + @Override + public Object buildLucene() { + if (wrapdateline) { + Coordinate[][][] polygons = coordinates(); + return polygons.length == 1 + ? polygonLucene(polygons[0]) + : multipolygonLucene(polygons); + } + return toPolygonLucene(); } protected XContentBuilder coordinatesArray(XContentBuilder builder, Params params) throws IOException { @@ -250,32 +261,46 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - public Geometry buildGeometry(GeometryFactory factory, boolean fixDateline) { + public Geometry buildS4JGeometry(GeometryFactory factory, boolean fixDateline) { if(fixDateline) { Coordinate[][][] polygons = coordinates(); return polygons.length == 1 - ? polygon(factory, polygons[0]) - : multipolygon(factory, polygons); + ? polygonS4J(factory, polygons[0]) + : multipolygonS4J(factory, polygons); } else { - return toPolygon(factory); + return toPolygonS4J(factory); } } - public Polygon toPolygon() { - return toPolygon(FACTORY); + public Polygon toPolygonS4J() { + return toPolygonS4J(FACTORY); } - protected Polygon toPolygon(GeometryFactory factory) { - final LinearRing shell = linearRing(factory, this.shell.coordinates); + protected Polygon toPolygonS4J(GeometryFactory factory) { + final LinearRing shell = linearRingS4J(factory, this.shell.coordinates); final LinearRing[] holes = new LinearRing[this.holes.size()]; Iterator iterator = this.holes.iterator(); for (int i = 0; iterator.hasNext(); i++) { - holes[i] = linearRing(factory, iterator.next().coordinates); + holes[i] = linearRingS4J(factory, iterator.next().coordinates); } return factory.createPolygon(shell, holes); } - protected static LinearRing linearRing(GeometryFactory factory, List coordinates) { + protected Object toPolygonLucene() { + final org.apache.lucene.geo.Polygon[] holes = new org.apache.lucene.geo.Polygon[this.holes.size()]; + for (int i = 0; i < holes.length; ++i) { + holes[i] = linearRing(this.holes.get(i).coordinates); + } + return new org.apache.lucene.geo.Polygon(this.shell.coordinates.stream().mapToDouble(i -> i.y).toArray(), + this.shell.coordinates.stream().mapToDouble(i -> i.x).toArray(), holes); + } + + protected static org.apache.lucene.geo.Polygon linearRing(List coordinates) { + return new org.apache.lucene.geo.Polygon(coordinates.stream().mapToDouble(i -> i.y).toArray(), + coordinates.stream().mapToDouble(i -> i.x).toArray()); + } + + protected static LinearRing linearRingS4J(GeometryFactory factory, List coordinates) { return factory.createLinearRing(coordinates.toArray(new Coordinate[coordinates.size()])); } @@ -293,7 +318,7 @@ public int numDimensions() { return shell.numDimensions(); } - protected static Polygon polygon(GeometryFactory factory, Coordinate[][] polygon) { + protected static Polygon polygonS4J(GeometryFactory factory, Coordinate[][] polygon) { LinearRing shell = factory.createLinearRing(polygon[0]); LinearRing[] holes; @@ -308,6 +333,35 @@ protected static Polygon polygon(GeometryFactory factory, Coordinate[][] polygon return factory.createPolygon(shell, holes); } + protected static org.apache.lucene.geo.Polygon polygonLucene(Coordinate[][] polygon) { + org.apache.lucene.geo.Polygon[] holes; + Coordinate[] shell = polygon[0]; + if (polygon.length > 1) { + holes = new org.apache.lucene.geo.Polygon[polygon.length - 1]; + for (int i = 0; i < holes.length; ++i) { + Coordinate[] coords = polygon[i+1]; + double[] x = new double[coords.length]; + double[] y = new double[coords.length]; + for (int c = 0; c < coords.length; ++c) { + x[c] = coords[c].x; + y[c] = coords[c].y; + } + holes[i] = new org.apache.lucene.geo.Polygon(y, x); + } + } else { + holes = new org.apache.lucene.geo.Polygon[0]; + } + + double[] x = new double[shell.length]; + double[] y = new double[shell.length]; + for (int i = 0; i < shell.length; ++i) { + x[i] = shell[i].x; + y[i] = shell[i].y; + } + + return new org.apache.lucene.geo.Polygon(y, x, holes); + } + /** * Create a Multipolygon from a set of coordinates. Each primary array contains a polygon which * in turn contains an array of linestrings. These line Strings are represented as an array of @@ -318,14 +372,22 @@ protected static Polygon polygon(GeometryFactory factory, Coordinate[][] polygon * @param polygons definition of polygons * @return a new Multipolygon */ - protected static MultiPolygon multipolygon(GeometryFactory factory, Coordinate[][][] polygons) { + protected static MultiPolygon multipolygonS4J(GeometryFactory factory, Coordinate[][][] polygons) { Polygon[] polygonSet = new Polygon[polygons.length]; for (int i = 0; i < polygonSet.length; i++) { - polygonSet[i] = polygon(factory, polygons[i]); + polygonSet[i] = polygonS4J(factory, polygons[i]); } return factory.createMultiPolygon(polygonSet); } + protected static org.apache.lucene.geo.Polygon[] multipolygonLucene(Coordinate[][][] polygons) { + org.apache.lucene.geo.Polygon[] polygonSet = new org.apache.lucene.geo.Polygon[polygons.length]; + for (int i = 0; i < polygonSet.length; ++i) { + polygonSet[i] = polygonLucene(polygons[i]); + } + return polygonSet; + } + /** * This method sets the component id of all edges in a ring to a given id and shifts the * coordinates of this component according to the dateline diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java index e90d7261c9988..7be013f93a8f6 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java @@ -211,7 +211,9 @@ protected JtsGeometry jtsGeometry(Geometry geom) { * the builder looses its validity. So this method should only be called once on a builder * @return new {@link Shape} defined by the builder */ - public abstract T build(); + public abstract T buildS4J(); + + public abstract Object buildLucene(); protected static Coordinate shift(Coordinate coordinate, double dateline) { if (dateline == 0) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index d2d081a63d077..723d0804277ce 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -18,6 +18,11 @@ */ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.LatLonShape; +import org.apache.lucene.geo.Line; +import org.apache.lucene.geo.Polygon; +import org.apache.lucene.geo.Rectangle; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; @@ -30,9 +35,12 @@ import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree; import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.XShapeCollection; import org.elasticsearch.common.geo.builders.ShapeBuilder; @@ -92,30 +100,16 @@ public static class Names { } public static class Defaults { - public static final String TREE = Names.TREE_GEOHASH; - public static final String STRATEGY = SpatialStrategy.RECURSIVE.getStrategyName(); + public static final String TREE = "NONE"; public static final boolean POINTS_ONLY = false; public static final int GEOHASH_LEVELS = GeoUtils.geoHashLevelsForPrecision("50m"); public static final int QUADTREE_LEVELS = GeoUtils.quadTreeLevelsForPrecision("50m"); public static final Orientation ORIENTATION = Orientation.RIGHT; public static final double LEGACY_DISTANCE_ERROR_PCT = 0.025d; + public static final Explicit STRATEGY = new Explicit<>(SpatialStrategy.VECTOR, false); public static final Explicit COERCE = new Explicit<>(false, false); public static final Explicit IGNORE_MALFORMED = new Explicit<>(false, false); public static final Explicit IGNORE_Z_VALUE = new Explicit<>(true, false); - - public static final MappedFieldType FIELD_TYPE = new GeoShapeFieldType(); - - static { - // setting name here is a hack so freeze can be called...instead all these options should be - // moved to the default ctor for GeoShapeFieldType, and defaultFieldType() should be removed from mappers... - FIELD_TYPE.setName("DoesNotExist"); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - FIELD_TYPE.setTokenized(false); - FIELD_TYPE.setStored(false); - FIELD_TYPE.setStoreTermVectors(false); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.freeze(); - } } public static class Builder extends FieldMapper.Builder { @@ -125,7 +119,7 @@ public static class Builder extends FieldMapper.Builder 0 || precisionInMeters >= 0) { + return Math.max(treeLevels, precisionInMeters >= 0 ? (geoHash ? GeoUtils.geoHashLevelsForPrecision(precisionInMeters) + : GeoUtils.quadTreeLevelsForPrecision(precisionInMeters)) : 0); + } + return defaultLevels; + } + @Override public GeoShapeFieldMapper build(BuilderContext context) { GeoShapeFieldType geoShapeFieldType = (GeoShapeFieldType)fieldType; @@ -189,8 +245,8 @@ public GeoShapeFieldMapper build(BuilderContext context) { } setupFieldType(context); - return new GeoShapeFieldMapper(name, fieldType, ignoreMalformed(context), coerce(context), ignoreZValue(context), - context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); + return new GeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), coerce(context), + ignoreZValue(context), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); } } @@ -205,9 +261,15 @@ public Mapper.Builder parse(String name, Map node, ParserContext String fieldName = entry.getKey(); Object fieldNode = entry.getValue(); if (Names.TREE.equals(fieldName)) { + checkPrefixTreeSupport(fieldName); builder.fieldType().setTree(fieldNode.toString()); + // explicitly set strategy + if (builder.fieldType().strategy() == Defaults.STRATEGY.value()) { + builder.fieldType().setStrategy(SpatialStrategy.RECURSIVE); + } iterator.remove(); } else if (Names.TREE_LEVELS.equals(fieldName)) { + checkPrefixTreeSupport(fieldName); builder.fieldType().setTreeLevels(Integer.parseInt(fieldNode.toString())); iterator.remove(); } else if (Names.TREE_PRESISION.equals(fieldName)) { @@ -215,13 +277,19 @@ public Mapper.Builder parse(String name, Map node, ParserContext DistanceUnit.DEFAULT, DistanceUnit.DEFAULT)); iterator.remove(); } else if (Names.DISTANCE_ERROR_PCT.equals(fieldName)) { + checkPrefixTreeSupport(fieldName); builder.fieldType().setDistanceErrorPct(Double.parseDouble(fieldNode.toString())); iterator.remove(); } else if (Names.ORIENTATION.equals(fieldName)) { builder.fieldType().setOrientation(ShapeBuilder.Orientation.fromString(fieldNode.toString())); iterator.remove(); } else if (Names.STRATEGY.equals(fieldName)) { - builder.fieldType().setStrategyName(fieldNode.toString()); + SpatialStrategy strategy = SpatialStrategy.fromString(fieldNode.toString()); + String prefixTree = builder.fieldType().tree(); + if (strategy == SpatialStrategy.VECTOR && prefixTree.equals("NONE") == false) { + throw new ElasticsearchParseException("Strategy [{}] cannot be used with PrefixTree [{}]", strategy, prefixTree); + } + builder.fieldType().setStrategy(strategy); iterator.remove(); } else if (IGNORE_MALFORMED.equals(fieldName)) { builder.ignoreMalformed(XContentMapValues.nodeBooleanValue(fieldNode, name + ".ignore_malformed")); @@ -239,7 +307,7 @@ public Mapper.Builder parse(String name, Map node, ParserContext } } if (pointsOnly != null) { - if (builder.fieldType().strategyName.equals(SpatialStrategy.TERM.getStrategyName()) && pointsOnly == false) { + if (builder.fieldType().strategy == SpatialStrategy.TERM && pointsOnly == false) { throw new IllegalArgumentException("points_only cannot be set to false for term strategy"); } else { builder.fieldType().setPointsOnly(pointsOnly); @@ -247,12 +315,18 @@ public Mapper.Builder parse(String name, Map node, ParserContext } return builder; } + + private void checkPrefixTreeSupport(String fieldName) { + if (ShapesAvailability.JTS_AVAILABLE == false || ShapesAvailability.SPATIAL4J_AVAILABLE == false) { + throw new ElasticsearchParseException("Field parameter [{}] is not supported for [{}] field type", fieldName, CONTENT_TYPE); + } + } } public static final class GeoShapeFieldType extends MappedFieldType { private String tree = Defaults.TREE; - private String strategyName = Defaults.STRATEGY; + private SpatialStrategy strategy = Defaults.STRATEGY.value(); private boolean pointsOnly = Defaults.POINTS_ONLY; private int treeLevels = 0; private double precisionInMeters = -1; @@ -261,16 +335,23 @@ public static final class GeoShapeFieldType extends MappedFieldType { private Orientation orientation = Defaults.ORIENTATION; // these are built when the field type is frozen - private PrefixTreeStrategy defaultStrategy; + private PrefixTreeStrategy defaultPrefixTreeStrategy; private RecursivePrefixTreeStrategy recursiveStrategy; private TermQueryPrefixTreeStrategy termStrategy; - public GeoShapeFieldType() {} + public GeoShapeFieldType() { + setIndexOptions(IndexOptions.DOCS); + setTokenized(false); + setStored(false); + setStoreTermVectors(false); + setOmitNorms(true); + } protected GeoShapeFieldType(GeoShapeFieldType ref) { super(ref); this.tree = ref.tree; - this.strategyName = ref.strategyName; + this.strategy = ref.strategy; +// this.strategyName = ref.strategyName; this.pointsOnly = ref.pointsOnly; this.treeLevels = ref.treeLevels; this.precisionInMeters = ref.precisionInMeters; @@ -292,7 +373,8 @@ public boolean equals(Object o) { precisionInMeters == that.precisionInMeters && defaultDistanceErrorPct == that.defaultDistanceErrorPct && Objects.equals(tree, that.tree) && - Objects.equals(strategyName, that.strategyName) && + Objects.equals(strategy, that.strategy) && +// Objects.equals(strategyName, that.strategyName) && pointsOnly == that.pointsOnly && Objects.equals(distanceErrorPct, that.distanceErrorPct) && orientation == that.orientation; @@ -300,7 +382,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(super.hashCode(), tree, strategyName, pointsOnly, treeLevels, precisionInMeters, distanceErrorPct, + return Objects.hash(super.hashCode(), tree, strategy, pointsOnly, treeLevels, precisionInMeters, distanceErrorPct, defaultDistanceErrorPct, orientation); } @@ -309,40 +391,12 @@ public String typeName() { return CONTENT_TYPE; } - @Override - public void freeze() { - super.freeze(); - // This is a bit hackish: we need to setup the spatial tree and strategies once the field name is set, which - // must be by the time freeze is called. - SpatialPrefixTree prefixTree; - if ("geohash".equals(tree)) { - prefixTree = new GeohashPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, - getLevels(treeLevels, precisionInMeters, Defaults.GEOHASH_LEVELS, true)); - } else if ("legacyquadtree".equals(tree)) { - prefixTree = new QuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, - getLevels(treeLevels, precisionInMeters, Defaults.QUADTREE_LEVELS, false)); - } else if ("quadtree".equals(tree)) { - prefixTree = new PackedQuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, - getLevels(treeLevels, precisionInMeters, Defaults.QUADTREE_LEVELS, false)); - } else { - throw new IllegalArgumentException("Unknown prefix tree type [" + tree + "]"); - } - - recursiveStrategy = new RecursivePrefixTreeStrategy(prefixTree, name()); - recursiveStrategy.setDistErrPct(distanceErrorPct()); - recursiveStrategy.setPruneLeafyBranches(false); - termStrategy = new TermQueryPrefixTreeStrategy(prefixTree, name()); - termStrategy.setDistErrPct(distanceErrorPct()); - defaultStrategy = resolveStrategy(strategyName); - defaultStrategy.setPointsOnly(pointsOnly); - } - @Override public void checkCompatibility(MappedFieldType fieldType, List conflicts) { super.checkCompatibility(fieldType, conflicts); GeoShapeFieldType other = (GeoShapeFieldType)fieldType; // prevent user from changing strategies - if (strategyName().equals(other.strategyName()) == false) { + if (strategy() != other.strategy()) { conflicts.add("mapper [" + name() + "] has different [strategy]"); } @@ -382,14 +436,14 @@ public void setTree(String tree) { this.tree = tree; } - public String strategyName() { - return strategyName; + public SpatialStrategy strategy() { + return strategy; } - public void setStrategyName(String strategyName) { + public void setStrategy(SpatialStrategy strategy) { checkIfFrozen(); - this.strategyName = strategyName; - if (this.strategyName.equals(SpatialStrategy.TERM.getStrategyName())) { + this.strategy = strategy; + if (this.strategy.equals(SpatialStrategy.TERM)) { this.pointsOnly = true; } } @@ -441,15 +495,15 @@ public void setOrientation(Orientation orientation) { this.orientation = orientation; } - public PrefixTreeStrategy defaultStrategy() { - return this.defaultStrategy; + public PrefixTreeStrategy defaultPrefixTreeStrategy() { + return this.defaultPrefixTreeStrategy; } - public PrefixTreeStrategy resolveStrategy(SpatialStrategy strategy) { - return resolveStrategy(strategy.getStrategyName()); + public PrefixTreeStrategy resolvePrefixTreeStrategy(SpatialStrategy strategy) { + return resolvePrefixTreeStrategy(strategy.getStrategyName()); } - public PrefixTreeStrategy resolveStrategy(String strategyName) { + public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) { if (SpatialStrategy.RECURSIVE.getStrategyName().equals(strategyName)) { return recursiveStrategy; } @@ -474,10 +528,11 @@ public Query termQuery(Object value, QueryShardContext context) { protected Explicit ignoreMalformed; protected Explicit ignoreZValue; - public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, Explicit ignoreMalformed, - Explicit coerce, Explicit ignoreZValue, Settings indexSettings, + public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, + Explicit ignoreMalformed, Explicit coerce, + Explicit ignoreZValue, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, multiFields, copyTo); + super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); this.coerce = coerce; this.ignoreMalformed = ignoreMalformed; this.ignoreZValue = ignoreZValue; @@ -487,8 +542,8 @@ public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, Explici public GeoShapeFieldType fieldType() { return (GeoShapeFieldType) super.fieldType(); } - @Override - public void parse(ParseContext context) throws IOException { + + public void parseForInvertedIndexing(ParseContext context) throws IOException { try { Shape shape = context.parseExternalValue(Shape.class); if (shape == null) { @@ -496,7 +551,7 @@ public void parse(ParseContext context) throws IOException { if (shapeBuilder == null) { return; } - shape = shapeBuilder.build(); + shape = shapeBuilder.buildS4J(); } if (fieldType().pointsOnly() == true) { // index configured for pointsOnly @@ -517,20 +572,95 @@ public void parse(ParseContext context) throws IOException { } catch (Exception e) { if (ignoreMalformed.value() == false) { throw new MapperParsingException("failed to parse field [{}] of type [{}]", e, fieldType().name(), - fieldType().typeName()); + fieldType().typeName()); } context.addIgnoredField(fieldType.name()); } } + /** parsing logic for {@link LatLonShape} indexing */ + public void parseForPointsIndexing(ParseContext context) throws IOException { + try { + Object shape = context.parseExternalValue(Object.class); + if (shape == null) { + ShapeBuilder shapeBuilder = ShapeParser.parse(context.parser(), this); + if (shapeBuilder == null) { + return; + } + shape = shapeBuilder.buildLucene(); + } + indexShape(context, shape); + } catch (Exception e) { + if (ignoreMalformed.value() == false) { + throw new MapperParsingException("failed to parse field [{}] of type [{}]", e, fieldType().name(), + fieldType().typeName()); + } + context.addIgnoredField(fieldType().name()); + } + } + + @Override + public void parse(ParseContext context) throws IOException { + if (fieldType().strategy() == SpatialStrategy.VECTOR) { + parseForPointsIndexing(context); + } else { + parseForInvertedIndexing(context); + } + } + private void indexShape(ParseContext context, Shape shape) { - List fields = new ArrayList<>(Arrays.asList(fieldType().defaultStrategy().createIndexableFields(shape))); + List fields = new ArrayList<>(Arrays.asList(fieldType().defaultPrefixTreeStrategy().createIndexableFields(shape))); createFieldNamesField(context, fields); for (IndexableField field : fields) { context.doc().add(field); } } + private void indexShape(ParseContext context, Object luceneShape) { + if (luceneShape instanceof GeoPoint) { + GeoPoint pt = (GeoPoint)luceneShape; + indexFields(context, LatLonShape.createIndexableFields(name(), pt.lat(), pt.lon())); + } else if (luceneShape instanceof Line) { + indexFields(context, LatLonShape.createIndexableFields(name(), (Line)luceneShape)); + } else if (luceneShape instanceof Polygon) { + indexFields(context, LatLonShape.createIndexableFields(name(), (Polygon) luceneShape)); + } else if (luceneShape instanceof double[][]) { + double[][] pts = (double[][])luceneShape; + for (int i = 0; i < pts.length; ++i) { + indexFields(context, LatLonShape.createIndexableFields(name(), pts[i][1], pts[i][0])); + } + } else if (luceneShape instanceof Line[]) { + Line[] lines = (Line[]) luceneShape; + for (int i = 0; i < lines.length; ++i) { + indexFields(context, LatLonShape.createIndexableFields(name(), lines[i])); + } + } else if (luceneShape instanceof Polygon[]) { + Polygon[] polys = (Polygon[]) luceneShape; + for (int i = 0; i < polys.length; ++i) { + indexFields(context, LatLonShape.createIndexableFields(name(), polys[i])); + } + } else if (luceneShape instanceof Rectangle) { + // index rectangle as a polygon + Rectangle r = (Rectangle) luceneShape; + Polygon p = new Polygon(new double[]{r.minLat, r.minLat, r.maxLat, r.maxLat, r.minLat}, + new double[]{r.minLon, r.maxLon, r.maxLon, r.minLon, r.minLon}); + indexFields(context, LatLonShape.createIndexableFields(name(), p)); + } else if (luceneShape instanceof Object[]) { + // recurse to index geometry collection + for (Object o : (Object[])luceneShape) { + indexShape(context, o); + } + } else { + throw new IllegalArgumentException("invalid shape type found [" + luceneShape.getClass().getName() + "] while indexing shape"); + } + } + + private void indexFields(ParseContext context, Field[] fields) { + for (Field f : fields) { + context.doc().add(f); + } + } + @Override protected void parseCreateField(ParseContext context, List fields) throws IOException { } @@ -577,8 +707,8 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, } else if (includeDefaults && fieldType().treeLevels() == 0) { // defaults only make sense if tree levels are not specified builder.field(Names.TREE_PRESISION, DistanceUnit.METERS.toString(50)); } - if (includeDefaults || fieldType().strategyName().equals(Defaults.STRATEGY) == false) { - builder.field(Names.STRATEGY, fieldType().strategyName()); + if (includeDefaults || fieldType().strategy() != Defaults.STRATEGY.value()) { + builder.field(Names.STRATEGY, fieldType().strategy().getStrategyName()); } if (includeDefaults || fieldType().distanceErrorPct() != fieldType().defaultDistanceErrorPct) { builder.field(Names.DISTANCE_ERROR_PCT, fieldType().distanceErrorPct()); @@ -586,7 +716,7 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, if (includeDefaults || fieldType().orientation() != Defaults.ORIENTATION) { builder.field(Names.ORIENTATION, fieldType().orientation()); } - if (fieldType().strategyName().equals(SpatialStrategy.TERM.getStrategyName())) { + if (fieldType().strategy() == SpatialStrategy.TERM) { // For TERMs strategy the defaults for points only change to true if (includeDefaults || fieldType().pointsOnly() != true) { builder.field(Names.STRATEGY_POINTS_ONLY, fieldType().pointsOnly()); diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java index d26973e390ac3..537127bc63507 100644 --- a/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java @@ -524,7 +524,7 @@ public static double[] parseBoundingBox(XContentParser parser) throws IOExceptio throw new ElasticsearchParseException("failed to parse bounding box. Conflicting definition found " + "using well-known text and explicit corners."); } - org.locationtech.spatial4j.shape.Rectangle r = envelope.build(); + org.locationtech.spatial4j.shape.Rectangle r = envelope.buildS4J(); return new double[]{r.getMinY(), r.getMaxY(), r.getMinX(), r.getMaxX()}; } return new double[]{bottom, top, left, right}; diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java index 490f8ee72c45d..7d080f92a6744 100644 --- a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java @@ -19,6 +19,10 @@ package org.elasticsearch.index.query; +import org.apache.lucene.document.LatLonShape; +import org.apache.lucene.geo.Line; +import org.apache.lucene.geo.Polygon; +import org.apache.lucene.geo.Rectangle; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; @@ -36,8 +40,9 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoShapeType; import org.elasticsearch.common.geo.ShapeRelation; -import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.geo.parsers.ShapeParser; @@ -329,9 +334,14 @@ public GeoShapeQueryBuilder relation(ShapeRelation relation) { if (relation == null) { throw new IllegalArgumentException("No Shape Relation defined"); } - if (strategy != null && strategy == SpatialStrategy.TERM && relation != ShapeRelation.INTERSECTS) { - throw new IllegalArgumentException("current strategy [" + strategy.getStrategyName() + "] only supports relation [" + if (strategy != null) { + if (strategy == SpatialStrategy.TERM && relation != ShapeRelation.INTERSECTS) { + throw new IllegalArgumentException("current strategy [" + strategy.getStrategyName() + "] only supports relation [" + ShapeRelation.INTERSECTS.getRelationName() + "] found relation [" + relation.getRelationName() + "]"); + } else if (strategy == SpatialStrategy.VECTOR && relation == ShapeRelation.CONTAINS) { + throw new IllegalArgumentException("current strategy [" + strategy.getStrategyName() + "] does not support relation [" + + ShapeRelation.CONTAINS.getRelationName() + "]"); + } } this.relation = relation; return this; @@ -383,27 +393,71 @@ protected Query doToQuery(QueryShardContext context) { final GeoShapeFieldMapper.GeoShapeFieldType shapeFieldType = (GeoShapeFieldMapper.GeoShapeFieldType) fieldType; - PrefixTreeStrategy strategy = shapeFieldType.defaultStrategy(); - if (this.strategy != null) { - strategy = shapeFieldType.resolveStrategy(this.strategy); + SpatialStrategy spatialStrategy = shapeFieldType.strategy(); + if(this.strategy != null) { + spatialStrategy = this.strategy; } Query query; - if (strategy instanceof RecursivePrefixTreeStrategy && relation == ShapeRelation.DISJOINT) { - // this strategy doesn't support disjoint anymore: but it did - // before, including creating lucene fieldcache (!) - // in this case, execute disjoint as exists && !intersects - BooleanQuery.Builder bool = new BooleanQuery.Builder(); - Query exists = ExistsQueryBuilder.newFilter(context, fieldName); - Query intersects = strategy.makeQuery(getArgs(shapeToQuery, ShapeRelation.INTERSECTS)); - bool.add(exists, BooleanClause.Occur.MUST); - bool.add(intersects, BooleanClause.Occur.MUST_NOT); - query = new ConstantScoreQuery(bool.build()); + if (spatialStrategy != SpatialStrategy.VECTOR) { + PrefixTreeStrategy prefixTreeStrategy = shapeFieldType.resolvePrefixTreeStrategy(spatialStrategy); + if (prefixTreeStrategy instanceof RecursivePrefixTreeStrategy && relation == ShapeRelation.DISJOINT) { + // this strategy doesn't support disjoint anymore: but it did + // before, including creating lucene fieldcache (!) + // in this case, execute disjoint as exists && !intersects + BooleanQuery.Builder bool = new BooleanQuery.Builder(); + Query exists = ExistsQueryBuilder.newFilter(context, fieldName); + Query intersects = prefixTreeStrategy.makeQuery(getArgs(shapeToQuery, ShapeRelation.INTERSECTS)); + bool.add(exists, BooleanClause.Occur.MUST); + bool.add(intersects, BooleanClause.Occur.MUST_NOT); + query = new ConstantScoreQuery(bool.build()); + } else { + query = new ConstantScoreQuery(prefixTreeStrategy.makeQuery(getArgs(shapeToQuery, relation))); + } } else { - query = new ConstantScoreQuery(strategy.makeQuery(getArgs(shapeToQuery, relation))); + query = getVectorQuery(context, shapeFieldType, shapeToQuery); } return query; } + private Query getVectorQuery(QueryShardContext context, GeoShapeFieldMapper.GeoShapeFieldType shapeFieldType, + ShapeBuilder queryShapeBuilder) { + // CONTAINS queries are not yet supported by VECTOR strategy + if (relation == ShapeRelation.CONTAINS) { + throw new QueryShardException(context, + ShapeRelation.CONTAINS + " query relation not supported for Field [" + fieldName + + "] when using Strategy [" + shapeFieldType.strategy() + "]"); + } + + Object queryShape = queryShapeBuilder.buildLucene(); + Query geoQuery; + if (queryShape instanceof Line[]) { + geoQuery = LatLonShape.newLineQuery(fieldName(), relation.getLuceneRelation(), (Line[]) queryShape); + } else if (queryShape instanceof Polygon[]) { + geoQuery = LatLonShape.newPolygonQuery(fieldName(), relation.getLuceneRelation(), (Polygon[]) queryShape); + } else if (queryShape instanceof Line) { + geoQuery = LatLonShape.newLineQuery(fieldName(), relation.getLuceneRelation(), (Line) queryShape); + } else if (queryShape instanceof Polygon) { + geoQuery = LatLonShape.newPolygonQuery(fieldName(), relation.getLuceneRelation(), (Polygon) queryShape); + } else if (queryShape instanceof Rectangle) { + Rectangle r = (Rectangle) queryShape; + geoQuery = LatLonShape.newBoxQuery(fieldName(), relation.getLuceneRelation(), + r.minLat, r.maxLat, r.minLon, r.maxLon); + } else { + // geometry types not yet supported by Lucene's LatLonShape + GeoShapeType geometryType = null; + if (queryShape instanceof double[][]) { + geometryType = GeoShapeType.MULTIPOINT; + } else if (queryShape instanceof double[] || queryShape instanceof GeoPoint) { + geometryType = GeoShapeType.POINT; + } else if (queryShape instanceof Object[]) { + geometryType = GeoShapeType.GEOMETRYCOLLECTION; + } + throw new QueryShardException(context, "Field [" + fieldName + "] does not support " + geometryType + " queries"); + } + // wrap geoQuery as a ConstantScoreQuery + return new ConstantScoreQuery(geoQuery); + } + /** * Fetches the Shape with the given ID in the given type and index. * @@ -414,9 +468,6 @@ protected Query doToQuery(QueryShardContext context) { * Shape itself is located */ private void fetch(Client client, GetRequest getRequest, String path, ActionListener listener) { - if (ShapesAvailability.JTS_AVAILABLE == false) { - throw new IllegalStateException("JTS not available"); - } getRequest.preference("_local"); client.get(getRequest, new ActionListener(){ @@ -472,13 +523,13 @@ public void onFailure(Exception e) { public static SpatialArgs getArgs(ShapeBuilder shape, ShapeRelation relation) { switch (relation) { case DISJOINT: - return new SpatialArgs(SpatialOperation.IsDisjointTo, shape.build()); + return new SpatialArgs(SpatialOperation.IsDisjointTo, shape.buildS4J()); case INTERSECTS: - return new SpatialArgs(SpatialOperation.Intersects, shape.build()); + return new SpatialArgs(SpatialOperation.Intersects, shape.buildS4J()); case WITHIN: - return new SpatialArgs(SpatialOperation.IsWithin, shape.build()); + return new SpatialArgs(SpatialOperation.IsWithin, shape.buildS4J()); case CONTAINS: - return new SpatialArgs(SpatialOperation.Contains, shape.build()); + return new SpatialArgs(SpatialOperation.Contains, shape.buildS4J()); default: throw new IllegalArgumentException("invalid relation [" + relation + "]"); } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index a1038853c0670..94b896c502a5c 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -25,7 +25,6 @@ import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.action.resync.TransportResyncReplicationAction; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; @@ -132,10 +131,7 @@ private Map getMappers(List mapperPlugi mappers.put(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser()); mappers.put(FieldAliasMapper.CONTENT_TYPE, new FieldAliasMapper.TypeParser()); mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser()); - - if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) { - mappers.put(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser()); - } + mappers.put(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser()); for (MapperPlugin mapperPlugin : mapperPlugins) { for (Map.Entry entry : mapperPlugin.getMappers().entrySet()) { diff --git a/server/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java b/server/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java index 420f5c5caefb2..5a8ecae18e21f 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java +++ b/server/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java @@ -59,7 +59,7 @@ protected void assertValidException(XContentBuilder builder, Class expectedEx protected void assertGeometryEquals(Shape expected, XContentBuilder geoJson) throws IOException { try (XContentParser parser = createParser(geoJson)) { parser.nextToken(); - ElasticsearchGeoAssertions.assertEquals(expected, ShapeParser.parse(parser).build()); + ElasticsearchGeoAssertions.assertEquals(expected, ShapeParser.parse(parser).buildS4J()); } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java index 57cb6b6262384..5ee046ea3aa8b 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java @@ -273,7 +273,7 @@ public void testParse3DPolygon() throws IOException { final GeoShapeFieldMapper mapperBuilder = new GeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext); try (XContentParser parser = createParser(polygonGeoJson)) { parser.nextToken(); - ElasticsearchGeoAssertions.assertEquals(jtsGeom(expected), ShapeParser.parse(parser, mapperBuilder).build()); + ElasticsearchGeoAssertions.assertEquals(jtsGeom(expected), ShapeParser.parse(parser, mapperBuilder).buildS4J()); } } @@ -464,7 +464,7 @@ public void testParseOGCPolygonWithoutHoles() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).build(); + Shape shape = ShapeParser.parse(parser).buildS4J(); ElasticsearchGeoAssertions.assertPolygon(shape); } @@ -485,7 +485,7 @@ public void testParseOGCPolygonWithoutHoles() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).build(); + Shape shape = ShapeParser.parse(parser).buildS4J(); ElasticsearchGeoAssertions.assertMultiPolygon(shape); } @@ -506,7 +506,7 @@ public void testParseOGCPolygonWithoutHoles() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).build(); + Shape shape = ShapeParser.parse(parser).buildS4J(); ElasticsearchGeoAssertions.assertPolygon(shape); } @@ -527,7 +527,7 @@ public void testParseOGCPolygonWithoutHoles() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).build(); + Shape shape = ShapeParser.parse(parser).buildS4J(); ElasticsearchGeoAssertions.assertMultiPolygon(shape); } @@ -556,7 +556,7 @@ public void testParseOGCPolygonWithHoles() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).build(); + Shape shape = ShapeParser.parse(parser).buildS4J(); ElasticsearchGeoAssertions.assertPolygon(shape); } @@ -583,7 +583,7 @@ public void testParseOGCPolygonWithHoles() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).build(); + Shape shape = ShapeParser.parse(parser).buildS4J(); ElasticsearchGeoAssertions.assertMultiPolygon(shape); } @@ -610,7 +610,7 @@ public void testParseOGCPolygonWithHoles() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).build(); + Shape shape = ShapeParser.parse(parser).buildS4J(); ElasticsearchGeoAssertions.assertPolygon(shape); } @@ -638,7 +638,7 @@ public void testParseOGCPolygonWithHoles() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).build(); + Shape shape = ShapeParser.parse(parser).buildS4J(); ElasticsearchGeoAssertions.assertMultiPolygon(shape); } @@ -1030,7 +1030,7 @@ public void testParseOrientationOption() throws IOException { try (XContentParser parser = createParser(polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).build(); + Shape shape = ShapeParser.parse(parser).buildS4J(); ElasticsearchGeoAssertions.assertPolygon(shape); } @@ -1060,7 +1060,7 @@ public void testParseOrientationOption() throws IOException { try (XContentParser parser = createParser(polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).build(); + Shape shape = ShapeParser.parse(parser).buildS4J(); ElasticsearchGeoAssertions.assertPolygon(shape); } @@ -1090,7 +1090,7 @@ public void testParseOrientationOption() throws IOException { try (XContentParser parser = createParser(polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).build(); + Shape shape = ShapeParser.parse(parser).buildS4J(); ElasticsearchGeoAssertions.assertPolygon(shape); } @@ -1120,7 +1120,7 @@ public void testParseOrientationOption() throws IOException { try (XContentParser parser = createParser(polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).build(); + Shape shape = ShapeParser.parse(parser).buildS4J(); ElasticsearchGeoAssertions.assertMultiPolygon(shape); } @@ -1150,7 +1150,7 @@ public void testParseOrientationOption() throws IOException { try (XContentParser parser = createParser(polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).build(); + Shape shape = ShapeParser.parse(parser).buildS4J(); ElasticsearchGeoAssertions.assertMultiPolygon(shape); } @@ -1180,7 +1180,7 @@ public void testParseOrientationOption() throws IOException { try (XContentParser parser = createParser(polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).build(); + Shape shape = ShapeParser.parse(parser).buildS4J(); ElasticsearchGeoAssertions.assertMultiPolygon(shape); } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java index 965ca234ddd30..3350dc5b660d8 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java @@ -386,10 +386,10 @@ public void testParseGeometryCollection() throws IOException { // assert empty shape collection GeometryCollectionBuilder builder = new GeometryCollectionBuilder(); Shape[] expected = new Shape[0]; - assertEquals(shapeCollection(expected).isEmpty(), builder.build().isEmpty()); + assertEquals(shapeCollection(expected).isEmpty(), builder.buildS4J().isEmpty()); } else { GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(random()); - assertExpected(gcb.build(), gcb); + assertExpected(gcb.buildS4J(), gcb); } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java b/server/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java index 5dcc811b0c6be..e883e74d21b9c 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java @@ -49,13 +49,13 @@ public class ShapeBuilderTests extends ESTestCase { public void testNewPoint() { - Point point = new PointBuilder().coordinate(-100, 45).build(); + Point point = new PointBuilder().coordinate(-100, 45).buildS4J(); assertEquals(-100D, point.getX(), 0.0d); assertEquals(45D, point.getY(), 0.0d); } public void testNewRectangle() { - Rectangle rectangle = new EnvelopeBuilder(new Coordinate(-45, 30), new Coordinate(45, -30)).build(); + Rectangle rectangle = new EnvelopeBuilder(new Coordinate(-45, 30), new Coordinate(45, -30)).buildS4J(); assertEquals(-45D, rectangle.getMinX(), 0.0d); assertEquals(-30D, rectangle.getMinY(), 0.0d); assertEquals(45D, rectangle.getMaxX(), 0.0d); @@ -68,7 +68,7 @@ public void testNewPolygon() { .coordinate(45, 30) .coordinate(45, -30) .coordinate(-45, -30) - .coordinate(-45, 30)).toPolygon(); + .coordinate(-45, 30)).toPolygonS4J(); LineString exterior = polygon.getExteriorRing(); assertEquals(exterior.getCoordinateN(0), new Coordinate(-45, 30)); @@ -83,7 +83,7 @@ public void testNewPolygon_coordinate() { .coordinate(new Coordinate(45, 30)) .coordinate(new Coordinate(45, -30)) .coordinate(new Coordinate(-45, -30)) - .coordinate(new Coordinate(-45, 30))).toPolygon(); + .coordinate(new Coordinate(-45, 30))).toPolygonS4J(); LineString exterior = polygon.getExteriorRing(); assertEquals(exterior.getCoordinateN(0), new Coordinate(-45, 30)); @@ -96,7 +96,7 @@ public void testNewPolygon_coordinates() { Polygon polygon = new PolygonBuilder(new CoordinatesBuilder() .coordinates(new Coordinate(-45, 30), new Coordinate(45, 30), new Coordinate(45, -30), new Coordinate(-45, -30), new Coordinate(-45, 30)) - ).toPolygon(); + ).toPolygonS4J(); LineString exterior = polygon.getExteriorRing(); assertEquals(exterior.getCoordinateN(0), new Coordinate(-45, 30)); @@ -115,7 +115,7 @@ public void testLineStringBuilder() { .coordinate(-45.0, 50.0) .coordinate(-45.0, -15.0) .coordinate(-110.0, -15.0) - .coordinate(-110.0, 55.0)).build(); + .coordinate(-110.0, 55.0)).buildS4J(); // Building a linestring that needs to be wrapped new LineStringBuilder(new CoordinatesBuilder() @@ -128,7 +128,7 @@ public void testLineStringBuilder() { .coordinate(130.0, -30.0) .coordinate(130.0, 60.0) ) - .build(); + .buildS4J(); // Building a lineString on the dateline new LineStringBuilder(new CoordinatesBuilder() @@ -137,7 +137,7 @@ public void testLineStringBuilder() { .coordinate(-180.0, -40.0) .coordinate(-180.0, -80.0) ) - .build(); + .buildS4J(); // Building a lineString on the dateline new LineStringBuilder(new CoordinatesBuilder() @@ -146,7 +146,7 @@ public void testLineStringBuilder() { .coordinate(180.0, -40.0) .coordinate(180.0, -80.0) ) - .build(); + .buildS4J(); } public void testMultiLineString() { @@ -165,7 +165,7 @@ public void testMultiLineString() { .coordinate(-100.0, 0.0) ) ) - .build(); + .buildS4J(); // LineString that needs to be wrapped new MultiLineStringBuilder() @@ -183,7 +183,7 @@ public void testMultiLineString() { .coordinate(150.0, 0.0) ) ) - .build(); + .buildS4J(); } public void testPolygonSelfIntersection() { @@ -192,31 +192,31 @@ public void testPolygonSelfIntersection() { .coordinate(40.0, 50.0) .coordinate(-40.0, -50.0) .coordinate(40.0, -50.0).close()); - Exception e = expectThrows(InvalidShapeException.class, () -> newPolygon.build()); + Exception e = expectThrows(InvalidShapeException.class, () -> newPolygon.buildS4J()); assertThat(e.getMessage(), containsString("Self-intersection at or near point (0.0")); } public void testGeoCircle() { double earthCircumference = 40075016.69; - Circle circle = new CircleBuilder().center(0, 0).radius("100m").build(); + Circle circle = new CircleBuilder().center(0, 0).radius("100m").buildS4J(); assertEquals((360 * 100) / earthCircumference, circle.getRadius(), 0.00000001); assertEquals(new PointImpl(0, 0, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter()); - circle = new CircleBuilder().center(+180, 0).radius("100m").build(); + circle = new CircleBuilder().center(+180, 0).radius("100m").buildS4J(); assertEquals((360 * 100) / earthCircumference, circle.getRadius(), 0.00000001); assertEquals(new PointImpl(180, 0, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter()); - circle = new CircleBuilder().center(-180, 0).radius("100m").build(); + circle = new CircleBuilder().center(-180, 0).radius("100m").buildS4J(); assertEquals((360 * 100) / earthCircumference, circle.getRadius(), 0.00000001); assertEquals(new PointImpl(-180, 0, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter()); - circle = new CircleBuilder().center(0, 90).radius("100m").build(); + circle = new CircleBuilder().center(0, 90).radius("100m").buildS4J(); assertEquals((360 * 100) / earthCircumference, circle.getRadius(), 0.00000001); assertEquals(new PointImpl(0, 90, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter()); - circle = new CircleBuilder().center(0, -90).radius("100m").build(); + circle = new CircleBuilder().center(0, -90).radius("100m").buildS4J(); assertEquals((360 * 100) / earthCircumference, circle.getRadius(), 0.00000001); assertEquals(new PointImpl(0, -90, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter()); double randomLat = (randomDouble() * 180) - 90; double randomLon = (randomDouble() * 360) - 180; double randomRadius = randomIntBetween(1, (int) earthCircumference / 4); - circle = new CircleBuilder().center(randomLon, randomLat).radius(randomRadius + "m").build(); + circle = new CircleBuilder().center(randomLon, randomLat).radius(randomRadius + "m").buildS4J(); assertEquals((360 * randomRadius) / earthCircumference, circle.getRadius(), 0.00000001); assertEquals(new PointImpl(randomLon, randomLat, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter()); } @@ -229,7 +229,7 @@ public void testPolygonWrapping() { .coordinate(-150.0, -65.0) .close() ) - .build(); + .buildS4J(); assertMultiPolygon(shape); } @@ -242,7 +242,7 @@ public void testLineStringWrapping() { .coordinate(-150.0, -65.0) .close() ) - .build(); + .buildS4J(); assertMultiLineString(shape); } @@ -286,7 +286,7 @@ public void testDatelineOGC() { .coordinate(-179,1) )); - Shape shape = builder.close().build(); + Shape shape = builder.close().buildS4J(); assertMultiPolygon(shape); } @@ -330,7 +330,7 @@ public void testDateline() { .coordinate(-179,1) )); - Shape shape = builder.close().build(); + Shape shape = builder.close().buildS4J(); assertMultiPolygon(shape); } @@ -406,7 +406,7 @@ public void testComplexShapeWithHole() { ) ); - Shape shape = builder.close().build(); + Shape shape = builder.close().buildS4J(); assertPolygon(shape); } @@ -429,7 +429,7 @@ public void testShapeWithHoleAtEdgeEndPoints() { .coordinate(4, 1) )); - Shape shape = builder.close().build(); + Shape shape = builder.close().buildS4J(); assertPolygon(shape); } @@ -441,7 +441,7 @@ public void testShapeWithPointOnDateline() { .coordinate(180, 0) ); - Shape shape = builder.close().build(); + Shape shape = builder.close().buildS4J(); assertPolygon(shape); } @@ -454,7 +454,7 @@ public void testShapeWithEdgeAlongDateline() { .coordinate(180, 0) ); - Shape shape = builder.close().build(); + Shape shape = builder.close().buildS4J(); assertPolygon(shape); // test case 2: test the negative side of the dateline @@ -465,7 +465,7 @@ public void testShapeWithEdgeAlongDateline() { .coordinate(-176, 4) ); - shape = builder.close().build(); + shape = builder.close().buildS4J(); assertPolygon(shape); } @@ -486,7 +486,7 @@ public void testShapeWithBoundaryHoles() { .coordinate(176, -10) .coordinate(176, 10) )); - Shape shape = builder.close().build(); + Shape shape = builder.close().buildS4J(); assertMultiPolygon(shape); // test case 2: test the negative side of the dateline @@ -508,7 +508,7 @@ public void testShapeWithBoundaryHoles() { .coordinate(-176, 10) .close() )); - shape = builder.close().build(); + shape = builder.close().buildS4J(); assertMultiPolygon(shape); } @@ -529,7 +529,7 @@ public void testShapeWithTangentialHole() { .coordinate(-180, 5) .coordinate(-177, 10) )); - Shape shape = builder.close().build(); + Shape shape = builder.close().buildS4J(); assertMultiPolygon(shape); } @@ -550,7 +550,7 @@ public void testShapeWithInvalidTangentialHole() { .coordinate(179, -10) .coordinate(164, 0) )); - Exception e = expectThrows(InvalidShapeException.class, () -> builder.close().build()); + Exception e = expectThrows(InvalidShapeException.class, () -> builder.close().buildS4J()); assertThat(e.getMessage(), containsString("interior cannot share more than one point with the exterior")); } @@ -577,7 +577,7 @@ public void testBoundaryShapeWithTangentialHole() { .coordinate(176, -5) .coordinate(172, 0) )); - Shape shape = builder.close().build(); + Shape shape = builder.close().buildS4J(); assertMultiPolygon(shape); } @@ -598,7 +598,7 @@ public void testBoundaryShapeWithInvalidTangentialHole() { .coordinate(176, -10) .coordinate(-177, 10) )); - Exception e = expectThrows(InvalidShapeException.class, () -> builder.close().build()); + Exception e = expectThrows(InvalidShapeException.class, () -> builder.close().buildS4J()); assertThat(e.getMessage(), containsString("interior cannot share more than one point with the exterior")); } @@ -613,7 +613,7 @@ public void testBoundaryShape() { .coordinate(-180, 90) ); - Shape shape = builder.close().build(); + Shape shape = builder.close().buildS4J(); assertPolygon(shape); } @@ -627,7 +627,7 @@ public void testShapeWithAlternateOrientation() { .coordinate(180, 0) ); - Shape shape = builder.close().build(); + Shape shape = builder.close().buildS4J(); assertPolygon(shape); // cw: geo core will convert to ccw across the dateline @@ -638,7 +638,7 @@ public void testShapeWithAlternateOrientation() { .coordinate(180, 0) ); - shape = builder.close().build(); + shape = builder.close().buildS4J(); assertMultiPolygon(shape); } @@ -651,7 +651,7 @@ public void testInvalidShapeWithConsecutiveDuplicatePoints() { .coordinate(-176, 4) .coordinate(180, 0) ); - Exception e = expectThrows(InvalidShapeException.class, () -> builder.close().build()); + Exception e = expectThrows(InvalidShapeException.class, () -> builder.close().buildS4J()); assertThat(e.getMessage(), containsString("duplicate consecutive coordinates at: (")); } diff --git a/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java b/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java index 83bf2bae6a66f..0d4f142785484 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java @@ -128,7 +128,7 @@ public void testHoleThatIsSouthOfPolygon() { InvalidShapeException e = expectThrows(InvalidShapeException.class, () -> { PolygonBuilder pb = new PolygonBuilder(new CoordinatesBuilder().coordinate(4, 3).coordinate(3, 2).coordinate(3, 3).close()); pb.hole(new LineStringBuilder(new CoordinatesBuilder().coordinate(4, 2).coordinate(3, 1).coordinate(4, 1).close())); - pb.build(); + pb.buildS4J(); }); assertEquals("Hole lies outside shell at or near point (4.0, 1.0, NaN)", e.getMessage()); @@ -138,7 +138,7 @@ public void testHoleThatIsNorthOfPolygon() { InvalidShapeException e = expectThrows(InvalidShapeException.class, () -> { PolygonBuilder pb = new PolygonBuilder(new CoordinatesBuilder().coordinate(3, 2).coordinate(4, 1).coordinate(3, 1).close()); pb.hole(new LineStringBuilder(new CoordinatesBuilder().coordinate(3, 3).coordinate(4, 2).coordinate(4, 3).close())); - pb.build(); + pb.buildS4J(); }); assertEquals("Hole lies outside shell at or near point (4.0, 3.0, NaN)", e.getMessage()); @@ -152,13 +152,13 @@ public void testWidePolygonWithConfusingOrientation() { PolygonBuilder pb = new PolygonBuilder(new CoordinatesBuilder() .coordinate(10, -20).coordinate(100, 0).coordinate(-100, 0).coordinate(20, -45).coordinate(40, -60).close()); - pb.build(); // Should not throw an exception + pb.buildS4J(); // Should not throw an exception } public void testPolygonWithUndefinedOrientationDueToCollinearPoints() { PolygonBuilder pb = new PolygonBuilder(new CoordinatesBuilder() .coordinate(0.0, 0.0).coordinate(1.0, 1.0).coordinate(-1.0, -1.0).close()); - InvalidShapeException e = expectThrows(InvalidShapeException.class, pb::build); + InvalidShapeException e = expectThrows(InvalidShapeException.class, pb::buildS4J); assertEquals("Cannot determine orientation: edges adjacent to (-1.0,-1.0) coincide", e.getMessage()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index 3ffc19f10dee7..d423d6b83b353 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -24,8 +24,8 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.PointBuilder; -import org.locationtech.spatial4j.shape.Point; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; @@ -182,8 +182,12 @@ public void parse(ParseContext context) throws IOException { pointMapper.parse(context.createExternalValueContext(point)); // Let's add a Dummy Shape - Point shape = new PointBuilder(-100, 45).build(); - shapeMapper.parse(context.createExternalValueContext(shape)); + PointBuilder pb = new PointBuilder(-100, 45); + if (shapeMapper.fieldType().strategy() == SpatialStrategy.VECTOR) { + shapeMapper.parse(context.createExternalValueContext(pb.buildLucene())); + } else { + shapeMapper.parse(context.createExternalValueContext(pb.buildS4J())); + } context = context.createExternalValueContext(generatedValue); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java index 48f07b6063c34..d384638920930 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java @@ -21,12 +21,13 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.geo.ShapeRelation; -import org.elasticsearch.common.geo.builders.PointBuilder; +import org.elasticsearch.common.geo.builders.EnvelopeBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.test.ESIntegTestCase; +import org.locationtech.jts.geom.Coordinate; import java.util.Arrays; import java.util.Collection; @@ -118,7 +119,8 @@ public void testExternalValues() throws Exception { assertThat(response.getHits().getTotalHits(), equalTo((long) 1)); response = client().prepareSearch("test-idx") - .setPostFilter(QueryBuilders.geoShapeQuery("field.shape", new PointBuilder(-100, 45)).relation(ShapeRelation.WITHIN)) + .setPostFilter(QueryBuilders.geoShapeQuery("field.shape", + new EnvelopeBuilder(new Coordinate(-101, 46), new Coordinate(-99, 44))).relation(ShapeRelation.WITHIN)) .execute().actionGet(); assertThat(response.getHits().getTotalHits(), equalTo((long) 1)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java index 20e689e9d7e89..a4d816ec728ca 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java @@ -64,11 +64,6 @@ public void testDefaultConfiguration() throws IOException { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.025d)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoShapeFieldMapper.Defaults.GEOHASH_LEVELS)); assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(GeoShapeFieldMapper.Defaults.ORIENTATION)); } @@ -238,7 +233,7 @@ public void testGeohashConfiguration() throws IOException { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); assertThat(strategy.getDistErrPct(), equalTo(0.1)); assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); @@ -262,7 +257,7 @@ public void testQuadtreeConfiguration() throws IOException { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); assertThat(strategy.getDistErrPct(), equalTo(0.5)); assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); @@ -290,7 +285,7 @@ public void testLevelPrecisionConfiguration() throws IOException { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); assertThat(strategy.getDistErrPct(), equalTo(0.5)); assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); @@ -314,7 +309,7 @@ public void testLevelPrecisionConfiguration() throws IOException { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); // distance_error_pct was not specified so we expect the mapper to take the highest precision between "precision" and // "tree_levels" setting distErrPct to 0 to guarantee desired precision @@ -340,7 +335,7 @@ public void testLevelPrecisionConfiguration() throws IOException { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); assertThat(strategy.getDistErrPct(), equalTo(0.5)); assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); @@ -364,7 +359,7 @@ public void testLevelPrecisionConfiguration() throws IOException { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); assertThat(strategy.getDistErrPct(), equalTo(0.5)); assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); @@ -387,7 +382,7 @@ public void testLevelPrecisionConfiguration() throws IOException { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); assertThat(strategy.getDistErrPct(), equalTo(0.5)); assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); @@ -410,7 +405,7 @@ public void testPointsOnlyOption() throws IOException { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); assertThat(strategy.isPointsOnly(), equalTo(true)); @@ -433,7 +428,7 @@ public void testLevelDefaults() throws IOException { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); assertThat(strategy.getDistErrPct(), equalTo(0.5)); assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); @@ -455,7 +450,7 @@ public void testLevelDefaults() throws IOException { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); assertThat(strategy.getDistErrPct(), equalTo(0.5)); assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); @@ -495,7 +490,7 @@ public void testGeoShapeMapperMerge() throws Exception { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); assertThat(strategy, instanceOf(RecursivePrefixTreeStrategy.class)); assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); @@ -505,7 +500,10 @@ public void testGeoShapeMapperMerge() throws Exception { // correct mapping stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("shape").field("type", "geo_shape").field("precision", "1m") + .startObject("properties").startObject("shape").field("type", "geo_shape") + .field("tree", "geohash") + .field("strategy", "recursive") + .field("precision", "1m") .field("tree_levels", 8).field("distance_error_pct", 0.001) .field("orientation", "cw").endObject().endObject().endObject().endObject()); docMapper = mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); @@ -514,7 +512,7 @@ public void testGeoShapeMapperMerge() throws Exception { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); assertThat(strategy, instanceOf(RecursivePrefixTreeStrategy.class)); assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); @@ -622,7 +620,7 @@ public void testPointsOnlyDefaultsWithTermStrategy() throws IOException { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultStrategy(); + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); assertThat(strategy.getDistErrPct(), equalTo(0.0)); assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java index a1c225f8a0657..5a0dab444920e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java @@ -42,7 +42,7 @@ public void modify(MappedFieldType ft) { addModifier(new Modifier("strategy", false) { @Override public void modify(MappedFieldType ft) { - ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setStrategyName("term"); + ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setStrategy(SpatialStrategy.TERM); } }); addModifier(new Modifier("tree_levels", false) { @@ -72,15 +72,15 @@ public void modify(MappedFieldType ft) { } /** - * Test for {@link GeoShapeFieldType#setStrategyName(String)} that checks that {@link GeoShapeFieldType#pointsOnly()} + * Test for {@link GeoShapeFieldType#setStrategy(SpatialStrategy)} that checks that {@link GeoShapeFieldType#pointsOnly()} * gets set as a side effect when using SpatialStrategy.TERM */ public void testSetStrategyName() throws IOException { GeoShapeFieldType fieldType = new GeoShapeFieldMapper.GeoShapeFieldType(); assertFalse(fieldType.pointsOnly()); - fieldType.setStrategyName(SpatialStrategy.RECURSIVE.getStrategyName()); + fieldType.setStrategy(SpatialStrategy.RECURSIVE); assertFalse(fieldType.pointsOnly()); - fieldType.setStrategyName(SpatialStrategy.TERM.getStrategyName()); + fieldType.setStrategy(SpatialStrategy.TERM); assertTrue(fieldType.pointsOnly()); } } diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java index 8a9c98ef4a3fa..83543c24f9bc1 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java @@ -70,7 +70,7 @@ private static List randomPolygon() { while (shapeBuilder == null) { shapeBuilder = RandomShapeGenerator.createShapeWithin(random(), null, ShapeType.POLYGON); } - JtsGeometry shape = (JtsGeometry) shapeBuilder.build(); + JtsGeometry shape = (JtsGeometry) shapeBuilder.buildS4J(); Coordinate[] coordinates = shape.getGeom().getCoordinates(); ArrayList polygonPoints = new ArrayList<>(); for (Coordinate coord : coordinates) { diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java index ca9a21973aa39..0e097658bfee8 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java @@ -66,13 +66,17 @@ protected GeoShapeQueryBuilder doCreateTestQueryBuilder() { return doCreateTestQueryBuilder(randomBoolean()); } private GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { - ShapeType shapeType = ShapeType.randomType(random()); - ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType); + ShapeType shapeType; + ShapeBuilder shape; GeoShapeQueryBuilder builder; clearShapeFields(); if (indexedShape == false) { + shapeType = randomFrom(new ShapeType[] {ShapeType.LINESTRING, ShapeType.MULTILINESTRING, ShapeType.POLYGON}); + shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType); builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); } else { + shapeType = ShapeType.randomType(random()); + shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType); indexedShapeToReturn = shape; indexedShapeId = randomAlphaOfLengthBetween(3, 20); indexedShapeType = randomAlphaOfLengthBetween(3, 20); @@ -94,13 +98,26 @@ private GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { SpatialStrategy strategy = randomFrom(SpatialStrategy.values()); // ShapeType.MULTILINESTRING + SpatialStrategy.TERM can lead to large queries and will slow down tests, so // we try to avoid that combination - while (shapeType == ShapeType.MULTILINESTRING && strategy == SpatialStrategy.TERM) { + // Also, SpatialStrategy.VECTOR does not support POINT or MULTIPOINT Queries + while ((shapeType == ShapeType.MULTILINESTRING && strategy == SpatialStrategy.TERM) + || (strategy == SpatialStrategy.VECTOR && (shapeType == ShapeType.MULTIPOINT || shapeType == ShapeType.POINT))) { strategy = randomFrom(SpatialStrategy.values()); } builder.strategy(strategy); - if (strategy != SpatialStrategy.TERM) { + if (strategy == SpatialStrategy.VECTOR) { + ShapeRelation relation; + if (shapeType != ShapeType.LINESTRING && shapeType != ShapeType.MULTILINESTRING) { + // vector strategy does not yet support WITHIN for linestrings + relation = randomFrom(new ShapeRelation[] {ShapeRelation.INTERSECTS, ShapeRelation.DISJOINT, ShapeRelation.WITHIN}); + } else { + relation = randomFrom(new ShapeRelation[] {ShapeRelation.INTERSECTS, ShapeRelation.DISJOINT}); + } + builder.relation(relation); + } else if (strategy != SpatialStrategy.TERM) { builder.relation(randomFrom(ShapeRelation.values())); } + } else if (shapeType == ShapeType.MULTIPOINT || shapeType == ShapeType.POINT) { + builder.strategy(randomFrom(new SpatialStrategy[] {SpatialStrategy.TERM, SpatialStrategy.RECURSIVE})); } if (randomBoolean()) { @@ -131,7 +148,8 @@ protected GetResponse executeGet(GetRequest getRequest) { } catch (IOException ex) { throw new ElasticsearchException("boom", ex); } - return new GetResponse(new GetResult(indexedShapeIndex, indexedShapeType, indexedShapeId, 0, true, new BytesArray(json), null)); + return new GetResponse(new GetResult(indexedShapeIndex, indexedShapeType, indexedShapeId, + 0, true, new BytesArray(json), null)); } @After diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java index 1067ed62db46e..184ee2759c15e 100644 --- a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java @@ -62,6 +62,7 @@ import static org.hamcrest.Matchers.notNullValue; public class MatchQueryBuilderTests extends AbstractQueryTestCase { + @Override protected MatchQueryBuilder doCreateTestQueryBuilder() { String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, BOOLEAN_FIELD_NAME, INT_FIELD_NAME, diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java b/server/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java index ce6c00f359f77..175c09eb10a3b 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java @@ -123,7 +123,7 @@ public void testShapeBuilders() { .coordinate(-10, 10) .coordinate(10, -10) .close()) - .build(); + .buildS4J(); fail("Self intersection not detected"); } catch (InvalidShapeException e) { } @@ -132,14 +132,14 @@ public void testShapeBuilders() { new PolygonBuilder(new CoordinatesBuilder() .coordinate(-10, -10).coordinate(-10, 10).coordinate(10, 10).coordinate(10, -10).close()) .hole(new LineStringBuilder(new CoordinatesBuilder().coordinate(-5, -5).coordinate(-5, 5).coordinate(5, 5).coordinate(5, -5).close())) - .build(); + .buildS4J(); try { // polygon with overlapping hole new PolygonBuilder(new CoordinatesBuilder() .coordinate(-10, -10).coordinate(-10, 10).coordinate(10, 10).coordinate(10, -10).close()) .hole(new LineStringBuilder(new CoordinatesBuilder() .coordinate(-5, -5).coordinate(-5, 11).coordinate(5, 11).coordinate(5, -5).close())) - .build(); + .buildS4J(); fail("Self intersection not detected"); } catch (InvalidShapeException e) { @@ -151,7 +151,7 @@ public void testShapeBuilders() { .coordinate(-10, -10).coordinate(-10, 10).coordinate(10, 10).coordinate(10, -10).close()) .hole(new LineStringBuilder(new CoordinatesBuilder().coordinate(-5, -5).coordinate(-5, 5).coordinate(5, 5).coordinate(5, -5).close())) .hole(new LineStringBuilder(new CoordinatesBuilder().coordinate(-5, -6).coordinate(5, -6).coordinate(5, -4).coordinate(-5, -4).close())) - .build(); + .buildS4J(); fail("Intersection of holes not detected"); } catch (InvalidShapeException e) { } @@ -167,7 +167,7 @@ public void testShapeBuilders() { .coordinate(10, 20) .coordinate(10, -10) .close()) - .build(); + .buildS4J(); fail("Self intersection not detected"); } catch (InvalidShapeException e) { } @@ -190,7 +190,7 @@ public void testShapeBuilders() { .coordinate(-4, 4) .coordinate(4, 4) .coordinate(4, -4).close())) - .build(); + .buildS4J(); } public void testShapeRelations() throws Exception { @@ -375,6 +375,7 @@ public void testBulk() throws Exception { .endObject() .startObject("location") .field("type", "geo_shape") + .field("ignore_malformed", true) .endObject() .endObject() .endObject() diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java index 83b795621189b..80aa5030d4a49 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java @@ -144,9 +144,8 @@ public void testIndexShapeRouting() throws Exception { String source = "{\n" + " \"shape\" : {\n" + - " \"type\" : \"circle\",\n" + - " \"coordinates\" : [-45.0, 45.0],\n" + - " \"radius\" : \"100m\"\n" + + " \"type\" : \"bbox\",\n" + + " \"coordinates\" : [[-45.0, 45.0], [45.0, -45.0]]\n" + " }\n" + "}"; diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java index d0d05f6a36903..2f560932a349e 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java @@ -19,12 +19,14 @@ package org.elasticsearch.search.geo; +import org.apache.lucene.geo.GeoTestUtil; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.CoordinatesBuilder; import org.elasticsearch.common.geo.builders.EnvelopeBuilder; import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder; @@ -36,6 +38,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.query.GeoShapeQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -63,12 +66,26 @@ import static org.hamcrest.Matchers.nullValue; public class GeoShapeQueryTests extends ESSingleNodeTestCase { + private static final String[] PREFIX_TREES = new String[] { + GeoShapeFieldMapper.Names.TREE_GEOHASH, + GeoShapeFieldMapper.Names.TREE_QUADTREE + }; + + private XContentBuilder createMapping() throws Exception { + XContentBuilder xcb = XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape"); + if (randomBoolean()) { + xcb = xcb.field("tree", randomFrom(PREFIX_TREES)) + .field("strategy", randomFrom(SpatialStrategy.RECURSIVE, SpatialStrategy.TERM)); + } + xcb = xcb.endObject().endObject().endObject().endObject(); + + return xcb; + } + public void testNullShape() throws Exception { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .endObject().endObject() - .endObject().endObject()); + String mapping = Strings.toString(createMapping()); client().admin().indices().prepareCreate("test").addMapping("type1", mapping, XContentType.JSON).execute().actionGet(); ensureGreen(); @@ -79,12 +96,7 @@ public void testNullShape() throws Exception { } public void testIndexPointsFilterRectangle() throws Exception { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .endObject().endObject() - .endObject().endObject()); + String mapping = Strings.toString(createMapping()); client().admin().indices().prepareCreate("test").addMapping("type1", mapping, XContentType.JSON).execute().actionGet(); ensureGreen(); @@ -126,12 +138,12 @@ public void testIndexPointsFilterRectangle() throws Exception { } public void testEdgeCases() throws Exception { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .endObject().endObject() - .endObject().endObject()); + XContentBuilder xcb = XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .endObject().endObject().endObject().endObject(); + String mapping = Strings.toString(xcb); + client().admin().indices().prepareCreate("test").addMapping("type1", mapping, XContentType.JSON).execute().actionGet(); ensureGreen(); @@ -163,12 +175,7 @@ public void testEdgeCases() throws Exception { } public void testIndexedShapeReference() throws Exception { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .endObject().endObject() - .endObject().endObject()); + String mapping = Strings.toString(createMapping()); client().admin().indices().prepareCreate("test").addMapping("type1", mapping, XContentType.JSON).execute().actionGet(); createIndex("shapes"); ensureGreen(); @@ -205,14 +212,7 @@ public void testIndexedShapeReference() throws Exception { } public void testIndexedShapeReferenceSourceDisabled() throws Exception { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() - .startObject("properties") - .startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .endObject() - .endObject() - .endObject(); + XContentBuilder mapping = createMapping(); client().admin().indices().prepareCreate("test").addMapping("type1", mapping).get(); createIndex("shapes", Settings.EMPTY, "shape_type", "_source", "enabled=false"); ensureGreen(); @@ -240,7 +240,7 @@ public void testReusableBuilder() throws IOException { private void assertUnmodified(ShapeBuilder builder) throws IOException { String before = Strings.toString(jsonBuilder().startObject().field("area", builder).endObject()); - builder.build(); + builder.buildS4J(); String after = Strings.toString(jsonBuilder().startObject().field("area", builder).endObject()); assertThat(before, equalTo(after)); } @@ -328,21 +328,31 @@ public void testShapeFetchingPath() throws Exception { public void testShapeFilterWithRandomGeoCollection() throws Exception { // Create a random geometry collection. GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(random()); + org.apache.lucene.geo.Polygon randomPoly = GeoTestUtil.nextPolygon(); + CoordinatesBuilder cb = new CoordinatesBuilder(); + for (int i = 0; i < randomPoly.numPoints(); ++i) { + cb.coordinate(randomPoly.getPolyLon(i), randomPoly.getPolyLat(i)); + } + gcb.shape(new PolygonBuilder(cb)); logger.info("Created Random GeometryCollection containing {} shapes", gcb.numShapes()); - client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree") + if (randomBoolean()) { + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape") .execute().actionGet(); + } else { + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree") + .execute().actionGet(); + } XContentBuilder docSource = gcb.toXContent(jsonBuilder().startObject().field("location"), null).endObject(); client().prepareIndex("test", "type", "1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); - ShapeBuilder filterShape = (gcb.getShapeAt(randomIntBetween(0, gcb.numShapes() - 1))); + ShapeBuilder filterShape = (gcb.getShapeAt(gcb.numShapes() - 1)); - GeoShapeQueryBuilder filter = QueryBuilders.geoShapeQuery("location", filterShape); - filter.relation(ShapeRelation.INTERSECTS); - SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(QueryBuilders.matchAllQuery()) - .setPostFilter(filter).get(); + GeoShapeQueryBuilder geoShapeQueryBuilder = QueryBuilders.geoShapeQuery("location", filterShape); + geoShapeQueryBuilder.relation(ShapeRelation.INTERSECTS); + SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(geoShapeQueryBuilder).get(); assertSearchResponse(result); assertHitCount(result, 1); } diff --git a/server/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java b/server/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java index 7fbfa0670f9c9..a5f3561e95968 100644 --- a/server/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java +++ b/server/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java @@ -204,7 +204,7 @@ private static ShapeBuilder createShape(Random r, Point nearPoint, Rectangle wit numPoints = RandomNumbers.randomIntBetween(r, 5, 25); Coordinate[] coordinates = new Coordinate[numPoints]; for (int i=0; i expectedException) { try { - ShapeParser.parse(parser).build(); + ShapeParser.parse(parser).buildS4J(); Assert.fail("process completed successfully when " + expectedException.getName() + " expected"); } catch (Exception e) { assert(e.getClass().equals(expectedException)): From 31718088568616ae9659b1b53a1a3feb519f4e9c Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Tue, 6 Nov 2018 16:49:32 -0600 Subject: [PATCH 02/38] revert change to PercolatorFieldMapper --- .../org/elasticsearch/percolator/PercolatorFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 9df921b0ae10f..6ac073ef90a02 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -327,7 +327,7 @@ Tuple, Map>> extractTermsAndRanges(IndexRead extractedTerms.add(builder.toBytesRef()); } } - if (info.getPointDataDimensionCount() == 1) { // not != 0 because range fields are not supported + if (info.getPointIndexDimensionCount() == 1) { // not != 0 because range fields are not supported PointValues values = reader.getPointValues(info.name); List encodedPointValues = new ArrayList<>(); encodedPointValues.add(values.getMinPackedValue().clone()); From f3d1736e96d088ecab44ecd53284220e8a559624 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Wed, 7 Nov 2018 13:34:08 -0600 Subject: [PATCH 03/38] fix ExistsQuery for geo_shape vector strategy --- .../index/mapper/GeoShapeFieldMapper.java | 4 +++- .../search/geo/GeoShapeQueryTests.java | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 723d0804277ce..2a30346040439 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -656,7 +656,9 @@ private void indexShape(ParseContext context, Object luceneShape) { } private void indexFields(ParseContext context, Field[] fields) { - for (Field f : fields) { + ArrayList flist = new ArrayList<>(Arrays.asList(fields)); + createFieldNamesField(context, flist); + for (IndexableField f : flist) { context.doc().add(f); } } diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java index 2f560932a349e..1f5a2908d747f 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.index.query.GeoShapeQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -383,6 +384,28 @@ public void testContainsShapeQuery() throws Exception { assertThat(response.getHits().getTotalHits(), greaterThan(0L)); } + public void testExistsQuery() throws Exception { + // Create a random geometry collection. + GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(random()); + logger.info("Created Random GeometryCollection containing {} shapes", gcb.numShapes()); + + if (randomBoolean()) { + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape") + .execute().actionGet(); + } else { + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree") + .execute().actionGet(); + } + + XContentBuilder docSource = gcb.toXContent(jsonBuilder().startObject().field("location"), null).endObject(); + client().prepareIndex("test", "type", "1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); + + ExistsQueryBuilder eqb = QueryBuilders.existsQuery("location"); + SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(eqb).get(); + assertSearchResponse(result); + assertHitCount(result, 1); + } + public void testShapeFilterWithDefinedGeoCollection() throws Exception { createIndex("shapes"); client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree") From ddcbddad8bf8a669f2eea9db253931349f162a24 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Wed, 7 Nov 2018 15:41:36 -0600 Subject: [PATCH 04/38] add deprecation logging for tree, precision, tree_levels, distance_error_pct, and points_only --- .../index/mapper/GeoShapeFieldMapper.java | 12 +++++++++++ .../mapper/GeoShapeFieldMapperTests.java | 20 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 2a30346040439..45c3a6f585f5c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -18,6 +18,8 @@ */ package org.elasticsearch.index.mapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.document.Field; import org.apache.lucene.document.LatLonShape; import org.apache.lucene.geo.Line; @@ -46,6 +48,7 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; import org.elasticsearch.common.geo.parsers.ShapeParser; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -112,6 +115,9 @@ public static class Defaults { public static final Explicit IGNORE_Z_VALUE = new Explicit<>(true, false); } + private static final Logger logger = LogManager.getLogger(GeoShapeFieldMapper.class); + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger); + public static class Builder extends FieldMapper.Builder { private Boolean coerce; @@ -273,6 +279,7 @@ public Mapper.Builder parse(String name, Map node, ParserContext builder.fieldType().setTreeLevels(Integer.parseInt(fieldNode.toString())); iterator.remove(); } else if (Names.TREE_PRESISION.equals(fieldName)) { + checkPrefixTreeSupport(fieldName); builder.fieldType().setPrecisionInMeters(DistanceUnit.parse(fieldNode.toString(), DistanceUnit.DEFAULT, DistanceUnit.DEFAULT)); iterator.remove(); @@ -288,6 +295,8 @@ public Mapper.Builder parse(String name, Map node, ParserContext String prefixTree = builder.fieldType().tree(); if (strategy == SpatialStrategy.VECTOR && prefixTree.equals("NONE") == false) { throw new ElasticsearchParseException("Strategy [{}] cannot be used with PrefixTree [{}]", strategy, prefixTree); + } else if (strategy != SpatialStrategy.VECTOR) { + checkPrefixTreeSupport(fieldName); } builder.fieldType().setStrategy(strategy); iterator.remove(); @@ -302,6 +311,7 @@ public Mapper.Builder parse(String name, Map node, ParserContext name + "." + GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName())); iterator.remove(); } else if (Names.STRATEGY_POINTS_ONLY.equals(fieldName)) { + checkPrefixTreeSupport(fieldName); pointsOnly = XContentMapValues.nodeBooleanValue(fieldNode, name + "." + Names.STRATEGY_POINTS_ONLY); iterator.remove(); } @@ -320,6 +330,8 @@ private void checkPrefixTreeSupport(String fieldName) { if (ShapesAvailability.JTS_AVAILABLE == false || ShapesAvailability.SPATIAL4J_AVAILABLE == false) { throw new ElasticsearchParseException("Field parameter [{}] is not supported for [{}] field type", fieldName, CONTENT_TYPE); } + DEPRECATION_LOGGER.deprecated("Field parameter [{}] is deprecated and will be removed in a future version. " + + "use [" + SpatialStrategy.VECTOR.getStrategyName() + "] indexing strategy instead.", fieldName); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java index a4d816ec728ca..6a84b5118b6ea 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java @@ -238,6 +238,7 @@ public void testGeohashConfiguration() throws IOException { assertThat(strategy.getDistErrPct(), equalTo(0.1)); assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); assertThat(strategy.getGrid().getMaxLevels(), equalTo(4)); + assertFieldWarnings("tree", "tree_levels", "distance_error_pct"); } public void testQuadtreeConfiguration() throws IOException { @@ -263,6 +264,17 @@ public void testQuadtreeConfiguration() throws IOException { assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); assertThat(strategy.getGrid().getMaxLevels(), equalTo(6)); assertThat(strategy.isPointsOnly(), equalTo(true)); + assertFieldWarnings("tree", "tree_levels", "distance_error_pct", "points_only"); + } + + private void assertFieldWarnings(String... fieldNames) { + String[] warnings = new String[fieldNames.length]; + String vectorPostfix = " use [vector] indexing strategy instead."; + for (int i = 0; i < fieldNames.length; ++i) { + warnings[i] = "Field parameter [" + fieldNames[i] + "] " + + "is deprecated and will be removed in a future version." + vectorPostfix; + } + assertWarnings(warnings); } public void testLevelPrecisionConfiguration() throws IOException { @@ -388,6 +400,7 @@ public void testLevelPrecisionConfiguration() throws IOException { assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d)+1)); } + assertFieldWarnings("tree", "tree_levels", "precision", "distance_error_pct"); } public void testPointsOnlyOption() throws IOException { @@ -409,6 +422,7 @@ public void testPointsOnlyOption() throws IOException { assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); assertThat(strategy.isPointsOnly(), equalTo(true)); + assertFieldWarnings("tree", "points_only"); } public void testLevelDefaults() throws IOException { @@ -457,6 +471,7 @@ public void testLevelDefaults() throws IOException { /* 50m is default */ assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(50d))); } + assertFieldWarnings("tree", "distance_error_pct"); } public void testGeoShapeMapperMerge() throws Exception { @@ -519,6 +534,8 @@ public void testGeoShapeMapperMerge() throws Exception { assertThat(strategy.getDistErrPct(), equalTo(0.001)); assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(1d))); assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(ShapeBuilder.Orientation.CW)); + + assertFieldWarnings("tree", "strategy", "precision", "tree_levels", "distance_error_pct"); } public void testEmptyName() throws Exception { @@ -602,6 +619,7 @@ public void testSerializeDefaults() throws Exception { assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); assertTrue(serialized, serialized.contains("\"tree_levels\":5")); } + assertFieldWarnings("tree", "tree_levels", "precision"); } public void testPointsOnlyDefaultsWithTermStrategy() throws IOException { @@ -628,6 +646,7 @@ public void testPointsOnlyDefaultsWithTermStrategy() throws IOException { assertThat(strategy.isPointsOnly(), equalTo(true)); // term strategy changes the default for points_only, check that we handle it correctly assertThat(toXContentString(geoShapeFieldMapper, false), not(containsString("points_only"))); + assertFieldWarnings("tree", "precision", "strategy"); } @@ -648,6 +667,7 @@ public void testPointsOnlyFalseWithTermStrategy() throws Exception { () -> parser.parse("type1", new CompressedXContent(mapping)) ); assertThat(e.getMessage(), containsString("points_only cannot be set to false for term strategy")); + assertFieldWarnings("tree", "precision", "strategy", "points_only"); } public String toXContentString(GeoShapeFieldMapper mapper, boolean includeDefaults) throws IOException { From 551149df27934322a16172f8cca01a92b4a31300 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Wed, 7 Nov 2018 16:18:39 -0600 Subject: [PATCH 05/38] initial update to geoshape docs, including mapping migration updates --- .../mapping/types/geo-shape.asciidoc | 164 +++++++++++------- .../migration/migrate_7_0/mappings.asciidoc | 16 ++ .../query-dsl/geo-shape-query.asciidoc | 5 +- 3 files changed, 119 insertions(+), 66 deletions(-) diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index 2f51465d1109f..91e8fd7c7ea67 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -21,38 +21,47 @@ type. |======================================================================= |Option |Description| Default -|`tree` |Name of the PrefixTree implementation to be used: `geohash` for -GeohashPrefixTree and `quadtree` for QuadPrefixTree. +|`tree |deprecated[6.6, use `vector` strategy] Name of the PrefixTree +implementation to be used: `geohash` for GeohashPrefixTree and `quadtree` +for QuadPrefixTree. | `geohash` -|`precision` |This parameter may be used instead of `tree_levels` to set -an appropriate value for the `tree_levels` parameter. The value -specifies the desired precision and Elasticsearch will calculate the -best tree_levels value to honor this precision. The value should be a -number followed by an optional distance unit. Valid distance units -include: `in`, `inch`, `yd`, `yard`, `mi`, `miles`, `km`, `kilometers`, -`m`,`meters`, `cm`,`centimeters`, `mm`, `millimeters`. +|`precision` |deprecated[6.6, use `vector` strategy] This parameter may +be used instead of `tree_levels` to set an appropriate value for the +`tree_levels` parameter. The value specifies the desired precision and +Elasticsearch will calculate the best tree_levels value to honor this +precision. The value should be a number followed by an optional distance +unit. Valid distance units include: `in`, `inch`, `yd`, `yard`, `mi`, +`miles`, `km`, `kilometers`, `m`,`meters`, `cm`,`centimeters`, `mm`, +`millimeters`. | `50m` -|`tree_levels` |Maximum number of layers to be used by the PrefixTree. -This can be used to control the precision of shape representations and -therefore how many terms are indexed. Defaults to the default value of -the chosen PrefixTree implementation. Since this parameter requires a -certain level of understanding of the underlying implementation, users -may use the `precision` parameter instead. However, Elasticsearch only -uses the tree_levels parameter internally and this is what is returned -via the mapping API even if you use the precision parameter. +|`tree_levels` |deprecated[6.6, use `vector` strategy] Maximum number +of layers to be used by the PrefixTree. This can be used to control the +precision of shape representations andtherefore how many terms are +indexed. Defaults to the default value of the chosen PrefixTree +implementation. Since this parameter requires a certain level of +understanding of the underlying implementation, users may use the +`precision` parameter instead. However, Elasticsearch only uses the +tree_levels parameter internally and this is what is returned via the +mapping API even if you use the precision parameter. | various -|`strategy` |The strategy parameter defines the approach for how to -represent shapes at indexing and search time. It also influences the -capabilities available so it is recommended to let Elasticsearch set -this parameter automatically. There are two strategies available: -`recursive` and `term`. Term strategy supports point types only (the -`points_only` parameter will be automatically set to true) while -Recursive strategy supports all shape types. (IMPORTANT: see -<> for more detailed information) -| `recursive` +|`strategy` |deprecated[6.6, use `vector` strategy] The strategy +parameter defines the approach for how to represent shapes at indexing +and search time. It also influences the capabilities available so it +is recommended to let Elasticsearch set this parameter automatically. +There are three strategies available: `recursive`, `term`, and `vector`. +Vector strategy is the default. This strategy provides near perfect +resolution (down to 1e-6 decimal degree precision) by indexing shapes +as vectory types. It is the preferred method of shape indexing. +Recursive and Term strategies are deprecated and will be removed in a +future version. While they are still available, the Term strategy +supports point types only (the `points_only` parameter will be +automatically set to true) while Recursive strategy supports all +shape types. (IMPORTANT: see <> for more +detailed information about these strategies) +| `vector` |`distance_error_pct` |Used as a hint to the PrefixTree about how precise it should be. Defaults to 0.025 (2.5%) with 0.5 as the maximum @@ -77,13 +86,13 @@ sets vertex order for the coordinate list of a geo_shape field but can be overridden in each individual GeoJSON or WKT document. | `ccw` -|`points_only` |Setting this option to `true` (defaults to `false`) configures -the `geo_shape` field type for point shapes only (NOTE: Multi-Points are not -yet supported). This optimizes index and search performance for the `geohash` and -`quadtree` when it is known that only points will be indexed. At present geo_shape -queries can not be executed on `geo_point` field types. This option bridges the gap -by improving point performance on a `geo_shape` field so that `geo_shape` queries are -optimal on a point only field. +|`points_only` |deprecated[6.6, use `vector` strategy] Setting this option to +`true` (defaults to `false`) configures the `geo_shape` field type for point +shapes only (NOTE: Multi-Points are not yet supported). This optimizes index and +search performance for the `geohash` and `quadtree` when it is known that only points +will be indexed. At present geo_shape queries can not be executed on `geo_point` +field types. This option bridges the gap by improving point performance on a +`geo_shape` field so that `geo_shape` queries are optimal on a point only field. | `false` |`ignore_malformed` |If true, malformed GeoJSON or WKT shapes are ignored. If @@ -104,12 +113,15 @@ and reject the whole document. [float] ==== Prefix trees -To efficiently represent shapes in the index, Shapes are converted into -a series of hashes representing grid squares (commonly referred to as "rasters") -using implementations of a PrefixTree. The tree notion comes from the fact that -the PrefixTree uses multiple grid layers, each with an increasing level of -precision to represent the Earth. This can be thought of as increasing the level -of detail of a map or image at higher zoom levels. +deprecated[6.6, use `vector` strategy] To efficiently represent shapes in +an inverted index, Shapes are converted into a series of hashes representing +grid squares (commonly referred to as "rasters") using implementations of a +PrefixTree. The tree notion comes from the fact that the PrefixTree uses multiple +grid layers, each with an increasing level of precision to represent the Earth. +This can be thought of as increasing the level of detail of a map or image at higher +zoom levels. Since this approach causes precision issues with indexed shape, it has +been deprecated in favor of a vector approach that indexes the shapes as a triangular +mesh. Multiple PrefixTree implementations are provided: @@ -131,9 +143,9 @@ number of levels for the quad trees in Elasticsearch is 29; the default is 21. [[spatial-strategy]] [float] ===== Spatial strategies -The PrefixTree implementations rely on a SpatialStrategy for decomposing -the provided Shape(s) into approximated grid squares. Each strategy answers -the following: +The indexing implementation selected relies on a SpatialStrategy for choosing +how to decompose the shapes (either as grid squares or a tessellated triangular +mesh). Each strategy answers the following: * What type of Shapes can be indexed? * What types of Query Operations and Shapes can be used? @@ -146,25 +158,49 @@ are provided: |======================================================================= |Strategy |Supported Shapes |Supported Queries |Multiple Shapes -|`recursive` |<> |`INTERSECTS`, `DISJOINT`, `WITHIN`, `CONTAINS` |Yes -|`term` |<> |`INTERSECTS` |Yes +|`vector` | <> | `INTERSECTS`, `DISJOINT`, `WITHIN` |Yes +|`recursive` deprecated[6.6, use `vector` strategy] |<> |`INTERSECTS`, `DISJOINT`, `WITHIN`, `CONTAINS` |Yes +|`term` deprecated[6.6, use `vector` strategy] |<> |`INTERSECTS` |Yes |======================================================================= [float] ===== Accuracy -Geo_shape does not provide 100% accuracy and depending on how it is configured -it may return some false positives for `INTERSECTS`, `WITHIN` and `CONTAINS` -queries, and some false negatives for `DISJOINT` queries. To mitigate this, it -is important to select an appropriate value for the tree_levels parameter and -to adjust expectations accordingly. For example, a point may be near the border -of a particular grid cell and may thus not match a query that only matches the -cell right next to it -- even though the shape is very close to the point. +`Recursive` and `Term` strategies do not provide 100% accuracy and depending on +how they are configured it may return some false positives for `INTERSECTS`, +`WITHIN` and `CONTAINS` queries, and some false negatives for `DISJOINT` queries. +To mitigate this, it is important to select an appropriate value for the tree_levels +parameter and to adjust expectations accordingly. For example, a point may be near +the border of a particular grid cell and may thus not match a query that only matches +the cell right next to it -- even though the shape is very close to the point. [float] ===== Example +[source,js] +-------------------------------------------------- +PUT /example +{ + "mappings": { + "doc": { + "properties": { + "location": { + "type": "geo_shape", + "tree": "quadtree" + } + } + } + } +} +-------------------------------------------------- +// CONSOLE +// TESTSETUP + +This mapping definition maps the location field to the geo_shape +type using the default vector implementation. It provides +approximately 1e-6 decimal degree precision. + [source,js] -------------------------------------------------- PUT /example @@ -183,24 +219,23 @@ PUT /example } -------------------------------------------------- // CONSOLE -// TESTSETUP This mapping maps the location field to the geo_shape type using the quad_tree implementation and a precision of 100m. Elasticsearch translates this into a tree_levels setting of 20. [float] -===== Performance considerations - -Elasticsearch uses the paths in the prefix tree as terms in the index -and in queries. The higher the level is (and thus the precision), the -more terms are generated. Of course, calculating the terms, keeping them in -memory, and storing them on disk all have a price. Especially with higher -tree levels, indices can become extremely large even with a modest -amount of data. Additionally, the size of the features also matters. -Big, complex polygons can take up a lot of space at higher tree levels. -Which setting is right depends on the use case. Generally one trades off -accuracy against index size and query performance. +===== Performance considerations with Prefix Trees + +With prefix trees, Elasticsearch uses the paths in the tree as terms in +the inverted index and in queries. The higher the level (and thus the +precision), the more terms are generated. Of course, calculating the +terms, keeping them in memory, and storing them on disk all have a price. +Especially with higher tree levels, indices can become extremely large +even with a modest amount of data. Additionally, the size of the features +also matters. Big, complex polygons can take up a lot of space at higher +tree levels. Which setting is right depends on the use case. Generally one +trades off accuracy against index size and query performance. The defaults in Elasticsearch for both implementations are a compromise between index size and a reasonable level of precision of 50m at the @@ -598,7 +633,8 @@ POST /example/doc ===== Circle Elasticsearch supports a `circle` type, which consists of a center -point with a radius: +point with a radius. Note that circles can only be indexed when using +the `recursive` Prefix Tree strategy only. [source,js] -------------------------------------------------- diff --git a/docs/reference/migration/migrate_7_0/mappings.asciidoc b/docs/reference/migration/migrate_7_0/mappings.asciidoc index 5ee1615796c98..3cda7fc84d844 100644 --- a/docs/reference/migration/migrate_7_0/mappings.asciidoc +++ b/docs/reference/migration/migrate_7_0/mappings.asciidoc @@ -52,3 +52,19 @@ as a better alternative. An error will now be thrown when unknown configuration options are provided to similarities. Such unknown parameters were ignored before. + +[float] +==== `geo_shape` defaults to `vector` strategy + +`geo_shape` types now default to using the `vector` indexing strategy. This indexes +shapes as a triangular mesh instead of decomposing them into individual grid cells. +To index using legacy prefix trees `recursive` or `term` strategy must be explicitly +defined. Note that these strategies are now deprecated and will be removed in a future +version. + +[float] +==== deprecated `geo_shape` parameters + +The following type parameters are deprecated for the `geo_shape` field type: `tree`, +`precision`, `tree_levels`, `distance_error_pct`, `points_only`, and `strategy`. They +will be removed in a future version. \ No newline at end of file diff --git a/docs/reference/query-dsl/geo-shape-query.asciidoc b/docs/reference/query-dsl/geo-shape-query.asciidoc index 4e00a2f49b475..f796881d520c6 100644 --- a/docs/reference/query-dsl/geo-shape-query.asciidoc +++ b/docs/reference/query-dsl/geo-shape-query.asciidoc @@ -7,7 +7,7 @@ Requires the <>. The `geo_shape` query uses the same grid square representation as the `geo_shape` mapping to find documents that have a shape that intersects -with the query shape. It will also use the same PrefixTree configuration +with the query shape. It will also use the same Prefix Tree configuration as defined for the field mapping. The query supports two ways of defining the query shape, either by @@ -157,7 +157,8 @@ has nothing in common with the query geometry. * `WITHIN` - Return all documents whose `geo_shape` field is within the query geometry. * `CONTAINS` - Return all documents whose `geo_shape` field -contains the query geometry. +contains the query geometry. Note: this is only supported using the +`recursive` Prefix Tree Strategy deprecated[6.6] [float] ==== Ignore Unmapped From c81747d7c96ba773967c224fa2ad8691e44723a3 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Wed, 7 Nov 2018 17:03:17 -0600 Subject: [PATCH 06/38] initial support for GeoCollection queries --- .../index/query/GeoShapeQueryBuilder.java | 19 ++++++++-- .../search/geo/GeoShapeQueryTests.java | 36 ++++++++++++++++++- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java index 7d080f92a6744..4a4dbe38ffd3e 100644 --- a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java @@ -428,7 +428,11 @@ private Query getVectorQuery(QueryShardContext context, GeoShapeFieldMapper.GeoS + "] when using Strategy [" + shapeFieldType.strategy() + "]"); } - Object queryShape = queryShapeBuilder.buildLucene(); + // wrap geoQuery as a ConstantScoreQuery + return new ConstantScoreQuery(getVectorQuery(context, queryShapeBuilder.buildLucene())); + } + + private Query getVectorQuery(QueryShardContext context, Object queryShape) { Query geoQuery; if (queryShape instanceof Line[]) { geoQuery = LatLonShape.newLineQuery(fieldName(), relation.getLuceneRelation(), (Line[]) queryShape); @@ -442,6 +446,8 @@ private Query getVectorQuery(QueryShardContext context, GeoShapeFieldMapper.GeoS Rectangle r = (Rectangle) queryShape; geoQuery = LatLonShape.newBoxQuery(fieldName(), relation.getLuceneRelation(), r.minLat, r.maxLat, r.minLon, r.maxLon); + } else if (queryShape instanceof Object[]) { + geoQuery = getGeometryCollectionQuery(context, (Object[]) queryShape); } else { // geometry types not yet supported by Lucene's LatLonShape GeoShapeType geometryType = null; @@ -454,8 +460,15 @@ private Query getVectorQuery(QueryShardContext context, GeoShapeFieldMapper.GeoS } throw new QueryShardException(context, "Field [" + fieldName + "] does not support " + geometryType + " queries"); } - // wrap geoQuery as a ConstantScoreQuery - return new ConstantScoreQuery(geoQuery); + return geoQuery; + } + + private Query getGeometryCollectionQuery(QueryShardContext context, Object... shapes) { + BooleanQuery.Builder bqb = new BooleanQuery.Builder(); + for (Object shape : shapes) { + bqb.add(getVectorQuery(context, shape), BooleanClause.Occur.SHOULD); + } + return bqb.build(); } /** diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java index 1f5a2908d747f..577dca3b9eb48 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java @@ -326,7 +326,7 @@ public void testShapeFetchingPath() throws Exception { assertHitCount(result, 1); } - public void testShapeFilterWithRandomGeoCollection() throws Exception { + public void testQueryRandomGeoCollection() throws Exception { // Create a random geometry collection. GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(random()); org.apache.lucene.geo.Polygon randomPoly = GeoTestUtil.nextPolygon(); @@ -358,6 +358,40 @@ public void testShapeFilterWithRandomGeoCollection() throws Exception { assertHitCount(result, 1); } + public void testRandomGeoCollectionQuery() throws Exception { + // Create a random geometry collection to index. + GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(random()); + org.apache.lucene.geo.Polygon randomPoly = GeoTestUtil.nextPolygon(); + CoordinatesBuilder cb = new CoordinatesBuilder(); + for (int i = 0; i < randomPoly.numPoints(); ++i) { + cb.coordinate(randomPoly.getPolyLon(i), randomPoly.getPolyLat(i)); + } + gcb.shape(new PolygonBuilder(cb)); + + logger.info("Created Random GeometryCollection containing {} shapes", gcb.numShapes()); + + if (randomBoolean()) { + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape") + .execute().actionGet(); + } else { + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree") + .execute().actionGet(); + } + + XContentBuilder docSource = gcb.toXContent(jsonBuilder().startObject().field("location"), null).endObject(); + client().prepareIndex("test", "type", "1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); + + // Create a random geometry collection to query + GeometryCollectionBuilder queryCollection = RandomShapeGenerator.createGeometryCollection(random()); + queryCollection.shape(new PolygonBuilder(cb)); + + GeoShapeQueryBuilder geoShapeQueryBuilder = QueryBuilders.geoShapeQuery("location", queryCollection); + geoShapeQueryBuilder.relation(ShapeRelation.INTERSECTS); + SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(geoShapeQueryBuilder).get(); + assertSearchResponse(result); + assertTrue(result.getHits().totalHits > 0); + } + public void testContainsShapeQuery() throws Exception { // Create a random geometry collection. Rectangle mbr = xRandomRectangle(random(), xRandomPoint(random()), true); From 8306a1a2d7b5471fbbc2288478d5cca20b945eb2 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Wed, 7 Nov 2018 17:15:22 -0600 Subject: [PATCH 07/38] fix docs and javadoc errors --- .../mapping/types/geo-shape.asciidoc | 45 +++++++++++++++++++ .../test/geo/RandomShapeGenerator.java | 2 +- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index 91e8fd7c7ea67..568748c295214 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -219,6 +219,9 @@ PUT /example } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] + This mapping maps the location field to the geo_shape type using the quad_tree implementation and a precision of 100m. Elasticsearch translates @@ -303,6 +306,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of a point in WKT: @@ -314,6 +319,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] [float] ===== http://geojson.org/geojson-spec.html#id3[LineString] @@ -334,6 +341,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of a LineString in WKT: @@ -345,6 +354,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The above `linestring` would draw a straight line starting at the White House to the US Capitol Building. @@ -369,6 +380,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of a Polygon in WKT: @@ -380,6 +393,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The first array represents the outer boundary of the polygon, the other arrays represent the interior shapes ("holes"). The following is a GeoJSON example @@ -399,6 +414,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of a Polygon with a hole in WKT: @@ -410,6 +427,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] *IMPORTANT NOTE:* WKT does not enforce a specific order for vertices thus ambiguous polygons around the dateline and poles are possible. @@ -445,6 +464,8 @@ POST /example/doc -------------------------------------------------- // CONSOLE // TEST[catch:/mapper_parsing_exception/] +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] An `orientation` parameter can be defined when setting the geo_shape mapping (see <>). This will define vertex order for the coordinate list on the mapped geo_shape field. It can also be overridden on each document. The following is an example for @@ -464,6 +485,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] [float] ===== http://www.geojson.org/geojson-spec.html#id5[MultiPoint] @@ -483,6 +506,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of a list of WKT points: @@ -494,6 +519,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] [float] ===== http://www.geojson.org/geojson-spec.html#id6[MultiLineString] @@ -515,6 +542,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of a list of WKT linestrings: @@ -526,6 +555,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] [float] ===== http://www.geojson.org/geojson-spec.html#id7[MultiPolygon] @@ -547,6 +578,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of a list of WKT polygons (second polygon contains a hole): @@ -558,6 +591,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] [float] ===== http://geojson.org/geojson-spec.html#geometrycollection[Geometry Collection] @@ -584,6 +619,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of a collection of WKT geometry objects: @@ -595,6 +632,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] [float] @@ -615,6 +654,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of an envelope using the WKT BBOX format: @@ -628,6 +669,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] [float] ===== Circle @@ -648,6 +691,8 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] +// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] Note: The inner `radius` field is required. If not specified, then the units of the `radius` will default to `METERS`. diff --git a/server/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java b/server/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java index a5f3561e95968..52e8249dd609a 100644 --- a/server/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java +++ b/server/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java @@ -154,7 +154,7 @@ private static ShapeBuilder createShape(Random r, Point nearPoint, Rectangle wit /** * Creates a random shape useful for randomized testing, NOTE: exercise caution when using this to build random GeometryCollections * as creating a large random number of random shapes can result in massive resource consumption - * see: {@link GeoShapeQueryTests#testShapeFilterWithRandomGeoCollection} + * see: {@link GeoShapeQueryTests#testQueryRandomGeoCollection()} * * The following options are included * @param nearPoint Create a shape near a provided point From cdf12863ff9875cd440ac6c52a33eef957af6f89 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Thu, 8 Nov 2018 10:59:06 -0600 Subject: [PATCH 08/38] clean up geocollection queries --- .../index/query/GeoShapeQueryBuilder.java | 19 ++++++--------- .../search/geo/GeoShapeQueryTests.java | 24 +++++++++++++++++-- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java index 4a4dbe38ffd3e..db665a8bfba55 100644 --- a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java @@ -446,24 +446,19 @@ private Query getVectorQuery(QueryShardContext context, Object queryShape) { Rectangle r = (Rectangle) queryShape; geoQuery = LatLonShape.newBoxQuery(fieldName(), relation.getLuceneRelation(), r.minLat, r.maxLat, r.minLon, r.maxLon); + } else if (queryShape instanceof double[][]) { + throw new QueryShardException(context, "Field [" + fieldName + "] does not support " + GeoShapeType.MULTIPOINT + " queries"); + } else if (queryShape instanceof double[] || queryShape instanceof GeoPoint) { + throw new QueryShardException(context, "Field [" + fieldName + "] does not support " + GeoShapeType.POINT + " queries"); } else if (queryShape instanceof Object[]) { - geoQuery = getGeometryCollectionQuery(context, (Object[]) queryShape); + geoQuery = createGeometryCollectionQuery(context, (Object[]) queryShape); } else { - // geometry types not yet supported by Lucene's LatLonShape - GeoShapeType geometryType = null; - if (queryShape instanceof double[][]) { - geometryType = GeoShapeType.MULTIPOINT; - } else if (queryShape instanceof double[] || queryShape instanceof GeoPoint) { - geometryType = GeoShapeType.POINT; - } else if (queryShape instanceof Object[]) { - geometryType = GeoShapeType.GEOMETRYCOLLECTION; - } - throw new QueryShardException(context, "Field [" + fieldName + "] does not support " + geometryType + " queries"); + throw new QueryShardException(context, "Field [" + fieldName + "] found and unknown shape"); } return geoQuery; } - private Query getGeometryCollectionQuery(QueryShardContext context, Object... shapes) { + private Query createGeometryCollectionQuery(QueryShardContext context, Object... shapes) { BooleanQuery.Builder bqb = new BooleanQuery.Builder(); for (Object shape : shapes) { bqb.add(getVectorQuery(context, shape), BooleanClause.Occur.SHOULD); diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java index 577dca3b9eb48..bac89589a872d 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java @@ -19,7 +19,9 @@ package org.elasticsearch.search.geo; +import com.carrotsearch.randomizedtesting.generators.RandomNumbers; import org.apache.lucene.geo.GeoTestUtil; +import org.apache.lucene.geo.Polygon; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchResponse; @@ -31,6 +33,8 @@ import org.elasticsearch.common.geo.builders.EnvelopeBuilder; import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder; import org.elasticsearch.common.geo.builders.LineStringBuilder; +import org.elasticsearch.common.geo.builders.MultiPointBuilder; +import org.elasticsearch.common.geo.builders.PointBuilder; import org.elasticsearch.common.geo.builders.PolygonBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.settings.Settings; @@ -49,6 +53,7 @@ import org.locationtech.spatial4j.shape.Rectangle; import java.io.IOException; +import java.util.ArrayList; import java.util.Locale; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; @@ -359,8 +364,23 @@ public void testQueryRandomGeoCollection() throws Exception { } public void testRandomGeoCollectionQuery() throws Exception { + boolean usePrefixTrees = randomBoolean(); // Create a random geometry collection to index. - GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(random()); + GeometryCollectionBuilder gcb; + if (usePrefixTrees) { + gcb = RandomShapeGenerator.createGeometryCollection(random()); + } else { + // vector strategy does not yet support point or multipoint queries + gcb = new GeometryCollectionBuilder(); + int numShapes = RandomNumbers.randomIntBetween(random(), 1, 4); + for (int i = 0; i < numShapes; ++i) { + ShapeBuilder shape; + do { + shape = RandomShapeGenerator.createShape(random()); + } while (shape instanceof MultiPointBuilder || shape instanceof PointBuilder); + gcb.shape(shape); + } + } org.apache.lucene.geo.Polygon randomPoly = GeoTestUtil.nextPolygon(); CoordinatesBuilder cb = new CoordinatesBuilder(); for (int i = 0; i < randomPoly.numPoints(); ++i) { @@ -370,7 +390,7 @@ public void testRandomGeoCollectionQuery() throws Exception { logger.info("Created Random GeometryCollection containing {} shapes", gcb.numShapes()); - if (randomBoolean()) { + if (usePrefixTrees == false) { client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape") .execute().actionGet(); } else { From 5cc71a8efc012c21294aca032fc72fd7d618a088 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Thu, 8 Nov 2018 10:59:50 -0600 Subject: [PATCH 09/38] set deprecated mapping tests to NOTCONSOLE --- docs/reference/mapping/types/geo-shape.asciidoc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index 568748c295214..f2ee627b26028 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -218,10 +218,7 @@ PUT /example } } -------------------------------------------------- -// CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] - +// NOTCONSOLE This mapping maps the location field to the geo_shape type using the quad_tree implementation and a precision of 100m. Elasticsearch translates From 83e34b527e857034a716f362941c4abed50a6a5c Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Thu, 8 Nov 2018 18:28:36 -0600 Subject: [PATCH 10/38] fix geo-shape mapper asciidoc mapping and test warnings --- .../mapping/types/geo-shape.asciidoc | 49 ++----------------- 1 file changed, 3 insertions(+), 46 deletions(-) diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index f2ee627b26028..3e515043660fe 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -186,8 +186,7 @@ PUT /example "doc": { "properties": { "location": { - "type": "geo_shape", - "tree": "quadtree" + "type": "geo_shape" } } } @@ -218,9 +217,9 @@ PUT /example } } -------------------------------------------------- -// NOTCONSOLE +// TEST[skip:deprecated mapping] -This mapping maps the location field to the geo_shape type using the +This mapping maps the location field to the geo_shape type using the deprecated quad_tree implementation and a precision of 100m. Elasticsearch translates this into a tree_levels setting of 20. @@ -303,8 +302,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of a point in WKT: @@ -316,8 +313,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] [float] ===== http://geojson.org/geojson-spec.html#id3[LineString] @@ -338,8 +333,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of a LineString in WKT: @@ -351,8 +344,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The above `linestring` would draw a straight line starting at the White House to the US Capitol Building. @@ -377,8 +368,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of a Polygon in WKT: @@ -390,8 +379,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The first array represents the outer boundary of the polygon, the other arrays represent the interior shapes ("holes"). The following is a GeoJSON example @@ -411,8 +398,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of a Polygon with a hole in WKT: @@ -424,8 +409,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] *IMPORTANT NOTE:* WKT does not enforce a specific order for vertices thus ambiguous polygons around the dateline and poles are possible. @@ -461,8 +444,6 @@ POST /example/doc -------------------------------------------------- // CONSOLE // TEST[catch:/mapper_parsing_exception/] -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] An `orientation` parameter can be defined when setting the geo_shape mapping (see <>). This will define vertex order for the coordinate list on the mapped geo_shape field. It can also be overridden on each document. The following is an example for @@ -482,8 +463,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] [float] ===== http://www.geojson.org/geojson-spec.html#id5[MultiPoint] @@ -503,8 +482,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of a list of WKT points: @@ -516,8 +493,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] [float] ===== http://www.geojson.org/geojson-spec.html#id6[MultiLineString] @@ -539,8 +514,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of a list of WKT linestrings: @@ -552,8 +525,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] [float] ===== http://www.geojson.org/geojson-spec.html#id7[MultiPolygon] @@ -575,8 +546,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of a list of WKT polygons (second polygon contains a hole): @@ -588,8 +557,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] [float] ===== http://geojson.org/geojson-spec.html#geometrycollection[Geometry Collection] @@ -616,8 +583,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of a collection of WKT geometry objects: @@ -629,8 +594,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] [float] @@ -651,8 +614,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] The following is an example of an envelope using the WKT BBOX format: @@ -666,8 +627,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] [float] ===== Circle @@ -688,8 +647,6 @@ POST /example/doc } -------------------------------------------------- // CONSOLE -// TEST[warning:Field parameter [tree] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] -// TEST[warning:Field parameter [strategy] is deprecated and will be removed in a future version. use [vector] indexing strategy instead.] Note: The inner `radius` field is required. If not specified, then the units of the `radius` will default to `METERS`. From c4bc5f69e7051add1218e22d0d64d748e6972f78 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Mon, 12 Nov 2018 14:24:05 -0600 Subject: [PATCH 11/38] add support for point queries using LatLonShapeBoundingBoxQuery --- .../index/query/GeoShapeQueryBuilder.java | 9 +++++- .../search/geo/GeoShapeQueryTests.java | 28 +++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java index db665a8bfba55..acb1148508c9c 100644 --- a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java @@ -447,9 +447,16 @@ private Query getVectorQuery(QueryShardContext context, Object queryShape) { geoQuery = LatLonShape.newBoxQuery(fieldName(), relation.getLuceneRelation(), r.minLat, r.maxLat, r.minLon, r.maxLon); } else if (queryShape instanceof double[][]) { + // note: we decompose point queries into a bounding box query with min values == max values + // to do this for multipoint we would have to create a BooleanQuery for each point + // this is *way* too costly. So we do not allow multipoint queries throw new QueryShardException(context, "Field [" + fieldName + "] does not support " + GeoShapeType.MULTIPOINT + " queries"); } else if (queryShape instanceof double[] || queryShape instanceof GeoPoint) { - throw new QueryShardException(context, "Field [" + fieldName + "] does not support " + GeoShapeType.POINT + " queries"); + // for now just create a single bounding box query with min values == max values + double[] pt = queryShape instanceof GeoPoint + ? new double[] {((GeoPoint)queryShape).lon(), ((GeoPoint)queryShape).lat()} + : (double[])queryShape; + return LatLonShape.newBoxQuery(fieldName, relation.getLuceneRelation(), pt[1], pt[1], pt[0], pt[0]); } else if (queryShape instanceof Object[]) { geoQuery = createGeometryCollectionQuery(context, (Object[]) queryShape); } else { diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java index bac89589a872d..edf7930b9bf3b 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java @@ -370,14 +370,14 @@ public void testRandomGeoCollectionQuery() throws Exception { if (usePrefixTrees) { gcb = RandomShapeGenerator.createGeometryCollection(random()); } else { - // vector strategy does not yet support point or multipoint queries + // vector strategy does not yet support multipoint queries gcb = new GeometryCollectionBuilder(); int numShapes = RandomNumbers.randomIntBetween(random(), 1, 4); for (int i = 0; i < numShapes; ++i) { ShapeBuilder shape; do { shape = RandomShapeGenerator.createShape(random()); - } while (shape instanceof MultiPointBuilder || shape instanceof PointBuilder); + } while (shape instanceof MultiPointBuilder); gcb.shape(shape); } } @@ -412,6 +412,30 @@ public void testRandomGeoCollectionQuery() throws Exception { assertTrue(result.getHits().totalHits > 0); } + /** tests querying a random geometry collection with a point */ + public void testPointQuery() throws Exception { + // Create a random geometry collection to index. + GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(random()); + double[] pt = new double[] {GeoTestUtil.nextLongitude(), GeoTestUtil.nextLatitude()}; + PointBuilder pb = new PointBuilder(pt[0], pt[1]); + gcb.shape(pb); + if (randomBoolean()) { + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape") + .execute().actionGet(); + } else { + client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree") + .execute().actionGet(); + } + XContentBuilder docSource = gcb.toXContent(jsonBuilder().startObject().field("location"), null).endObject(); + client().prepareIndex("test", "type", "1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get(); + + GeoShapeQueryBuilder geoShapeQueryBuilder = QueryBuilders.geoShapeQuery("location", pb); + geoShapeQueryBuilder.relation(ShapeRelation.INTERSECTS); + SearchResponse result = client().prepareSearch("test").setTypes("type").setQuery(geoShapeQueryBuilder).get(); + assertSearchResponse(result); + assertHitCount(result, 1); + } + public void testContainsShapeQuery() throws Exception { // Create a random geometry collection. Rectangle mbr = xRandomRectangle(random(), xRandomPoint(random()), true); From 2ae341c03bd645f77b0e7718ba0881fd8a9f352a Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Tue, 13 Nov 2018 10:28:58 -0600 Subject: [PATCH 12/38] update GeoShapeQueryBuilderTests to include POINT queries for VECTOR strategy. Other comment cleanups --- .../elasticsearch/index/mapper/GeoShapeFieldMapper.java | 2 -- .../index/query/GeoShapeQueryBuilderTests.java | 8 ++++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 45c3a6f585f5c..1b7f1bdb7d3d9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -363,7 +363,6 @@ protected GeoShapeFieldType(GeoShapeFieldType ref) { super(ref); this.tree = ref.tree; this.strategy = ref.strategy; -// this.strategyName = ref.strategyName; this.pointsOnly = ref.pointsOnly; this.treeLevels = ref.treeLevels; this.precisionInMeters = ref.precisionInMeters; @@ -386,7 +385,6 @@ public boolean equals(Object o) { defaultDistanceErrorPct == that.defaultDistanceErrorPct && Objects.equals(tree, that.tree) && Objects.equals(strategy, that.strategy) && -// Objects.equals(strategyName, that.strategyName) && pointsOnly == that.pointsOnly && Objects.equals(distanceErrorPct, that.distanceErrorPct) && orientation == that.orientation; diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java index 0e097658bfee8..03a484ceddcb2 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java @@ -71,7 +71,7 @@ private GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { GeoShapeQueryBuilder builder; clearShapeFields(); if (indexedShape == false) { - shapeType = randomFrom(new ShapeType[] {ShapeType.LINESTRING, ShapeType.MULTILINESTRING, ShapeType.POLYGON}); + shapeType = randomFrom(new ShapeType[] {ShapeType.POINT, ShapeType.LINESTRING, ShapeType.MULTILINESTRING, ShapeType.POLYGON}); shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType); builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); } else { @@ -98,9 +98,9 @@ private GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { SpatialStrategy strategy = randomFrom(SpatialStrategy.values()); // ShapeType.MULTILINESTRING + SpatialStrategy.TERM can lead to large queries and will slow down tests, so // we try to avoid that combination - // Also, SpatialStrategy.VECTOR does not support POINT or MULTIPOINT Queries + // Also, SpatialStrategy.VECTOR does not support MULTIPOINT Queries while ((shapeType == ShapeType.MULTILINESTRING && strategy == SpatialStrategy.TERM) - || (strategy == SpatialStrategy.VECTOR && (shapeType == ShapeType.MULTIPOINT || shapeType == ShapeType.POINT))) { + || (strategy == SpatialStrategy.VECTOR && (shapeType == ShapeType.MULTIPOINT))) { strategy = randomFrom(SpatialStrategy.values()); } builder.strategy(strategy); @@ -116,7 +116,7 @@ private GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { } else if (strategy != SpatialStrategy.TERM) { builder.relation(randomFrom(ShapeRelation.values())); } - } else if (shapeType == ShapeType.MULTIPOINT || shapeType == ShapeType.POINT) { + } else if (shapeType == ShapeType.MULTIPOINT) { builder.strategy(randomFrom(new SpatialStrategy[] {SpatialStrategy.TERM, SpatialStrategy.RECURSIVE})); } From 6bb412245d2c370e54e46ff6314cc9ba64d2768f Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Tue, 13 Nov 2018 16:48:33 -0600 Subject: [PATCH 13/38] add lucene geometry build testing to ShapeBuilder tests --- .../elasticsearch/common/geo/GeoUtils.java | 17 +- .../geo/builders/LineStringBuilder.java | 10 +- .../common/geo/builders/PolygonBuilder.java | 20 +- .../common/geo/BaseGeoParsingTestCase.java | 8 +- .../common/geo/GeoJsonShapeParserTests.java | 338 ++++++++++++----- .../common/geo/GeoWKTShapeParserTests.java | 148 ++++++-- .../common/geo/ShapeBuilderTests.java | 352 +++++++++++++----- .../query/GeoShapeQueryBuilderTests.java | 3 +- .../hamcrest/ElasticsearchGeoAssertions.java | 56 ++- 9 files changed, 696 insertions(+), 256 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java index 2f3443639cdb7..aff463105f4c4 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java @@ -233,7 +233,10 @@ public static int geoHashLevelsForPrecision(String distance) { * @return The normalized longitude. */ public static double normalizeLon(double lon) { - return centeredModulus(lon, 360); + if (lon > 180d || lon < -180d) { + return centeredModulus(lon, 360); + } + return lon; } /** @@ -250,11 +253,13 @@ public static double normalizeLon(double lon) { * @see #normalizePoint(GeoPoint) */ public static double normalizeLat(double lat) { - lat = centeredModulus(lat, 360); - if (lat < -90) { - lat = -180 - lat; - } else if (lat > 90) { - lat = 180 - lat; + if (lat > 90d || lat < -90d) { + lat = centeredModulus(lat, 360); + if (lat < -90) { + lat = -180 - lat; + } else if (lat > 90) { + lat = 180 - lat; + } } return lat; } diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java index ec081ffdd2507..dd50a0736c7f9 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java @@ -36,6 +36,9 @@ import java.util.Arrays; import java.util.List; +import static org.elasticsearch.common.geo.GeoUtils.normalizeLat; +import static org.elasticsearch.common.geo.GeoUtils.normalizeLon; + public class LineStringBuilder extends ShapeBuilder { public static final GeoShapeType TYPE = GeoShapeType.LINESTRING; @@ -133,8 +136,8 @@ public Object buildLucene() { return strings.toArray(new Line[strings.size()]); } } - return new Line(Arrays.stream(coordinates).mapToDouble(i->i.y).toArray(), - Arrays.stream(coordinates).mapToDouble(i->i.x).toArray()); + return new Line(Arrays.stream(coordinates).mapToDouble(i->normalizeLat(i.y)).toArray(), + Arrays.stream(coordinates).mapToDouble(i->normalizeLon(i.x)).toArray()); } static ArrayList decomposeS4J(GeometryFactory factory, Coordinate[] coordinates, ArrayList strings) { @@ -149,7 +152,8 @@ static ArrayList decomposeS4J(GeometryFactory factory, Coordinate[] static ArrayList decomposeLucene(Coordinate[] coordinates, ArrayList lines) { for (Coordinate[] part : decompose(+DATELINE, coordinates)) { for (Coordinate[] line : decompose(-DATELINE, part)) { - lines.add(new Line(Arrays.stream(line).mapToDouble(i->i.y).toArray(), Arrays.stream(line).mapToDouble(i->i.x).toArray())); + lines.add(new Line(Arrays.stream(line).mapToDouble(i->normalizeLat(i.y)).toArray(), + Arrays.stream(line).mapToDouble(i->normalizeLon(i.x)).toArray())); } } return lines; diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java index 11f399d06cf90..8f7876d2ba9f2 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java @@ -46,6 +46,8 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; +import static org.elasticsearch.common.geo.GeoUtils.normalizeLat; +import static org.elasticsearch.common.geo.GeoUtils.normalizeLon; import static org.apache.lucene.geo.GeoUtils.orient; /** @@ -286,18 +288,18 @@ protected Polygon toPolygonS4J(GeometryFactory factory) { return factory.createPolygon(shell, holes); } - protected Object toPolygonLucene() { + public Object toPolygonLucene() { final org.apache.lucene.geo.Polygon[] holes = new org.apache.lucene.geo.Polygon[this.holes.size()]; for (int i = 0; i < holes.length; ++i) { holes[i] = linearRing(this.holes.get(i).coordinates); } - return new org.apache.lucene.geo.Polygon(this.shell.coordinates.stream().mapToDouble(i -> i.y).toArray(), - this.shell.coordinates.stream().mapToDouble(i -> i.x).toArray(), holes); + return new org.apache.lucene.geo.Polygon(this.shell.coordinates.stream().mapToDouble(i -> normalizeLat(i.y)).toArray(), + this.shell.coordinates.stream().mapToDouble(i -> normalizeLon(i.x)).toArray(), holes); } protected static org.apache.lucene.geo.Polygon linearRing(List coordinates) { - return new org.apache.lucene.geo.Polygon(coordinates.stream().mapToDouble(i -> i.y).toArray(), - coordinates.stream().mapToDouble(i -> i.x).toArray()); + return new org.apache.lucene.geo.Polygon(coordinates.stream().mapToDouble(i -> normalizeLat(i.y)).toArray(), + coordinates.stream().mapToDouble(i -> normalizeLon(i.x)).toArray()); } protected static LinearRing linearRingS4J(GeometryFactory factory, List coordinates) { @@ -343,8 +345,8 @@ protected static org.apache.lucene.geo.Polygon polygonLucene(Coordinate[][] poly double[] x = new double[coords.length]; double[] y = new double[coords.length]; for (int c = 0; c < coords.length; ++c) { - x[c] = coords[c].x; - y[c] = coords[c].y; + x[c] = normalizeLon(coords[c].x); + y[c] = normalizeLat(coords[c].y); } holes[i] = new org.apache.lucene.geo.Polygon(y, x); } @@ -355,8 +357,8 @@ protected static org.apache.lucene.geo.Polygon polygonLucene(Coordinate[][] poly double[] x = new double[shell.length]; double[] y = new double[shell.length]; for (int i = 0; i < shell.length; ++i) { - x[i] = shell[i].x; - y[i] = shell[i].y; + x[i] = normalizeLon(shell[i].x); + y[i] = normalizeLat(shell[i].y); } return new org.apache.lucene.geo.Polygon(y, x, holes); diff --git a/server/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java b/server/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java index 5a8ecae18e21f..104aa74c15871 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java +++ b/server/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java @@ -56,10 +56,14 @@ protected void assertValidException(XContentBuilder builder, Class expectedEx } } - protected void assertGeometryEquals(Shape expected, XContentBuilder geoJson) throws IOException { + protected void assertGeometryEquals(Object expected, XContentBuilder geoJson, boolean useJTS) throws IOException { try (XContentParser parser = createParser(geoJson)) { parser.nextToken(); - ElasticsearchGeoAssertions.assertEquals(expected, ShapeParser.parse(parser).buildS4J()); + if (useJTS) { + ElasticsearchGeoAssertions.assertEquals(expected, ShapeParser.parse(parser).buildS4J()); + } else { + ElasticsearchGeoAssertions.assertEquals(expected, ShapeParser.parse(parser).buildLucene()); + } } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java index 5ee046ea3aa8b..cb799501fb9e7 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.geo; +import org.apache.lucene.geo.Line; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -49,6 +50,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import static org.elasticsearch.common.geo.builders.ShapeBuilder.SPATIAL_CONTEXT; @@ -66,9 +68,13 @@ public void testParsePoint() throws IOException { .field("type", "Point") .startArray("coordinates").value(100.0).value(0.0).endArray() .endObject(); - - Point expected = GEOMETRY_FACTORY.createPoint(new Coordinate(100.0, 0.0)); - assertGeometryEquals(new JtsPoint(expected, SPATIAL_CONTEXT), pointGeoJson); + boolean useJTS = randomBoolean(); + if (useJTS) { + Point expected = GEOMETRY_FACTORY.createPoint(new Coordinate(100.0, 0.0)); + assertGeometryEquals(new JtsPoint(expected, SPATIAL_CONTEXT), pointGeoJson, useJTS); + } else { + assertGeometryEquals(new GeoPoint(0d, 100d), pointGeoJson, useJTS); + } } @Override @@ -86,9 +92,19 @@ public void testParseLineString() throws IOException { lineCoordinates.add(new Coordinate(100, 0)); lineCoordinates.add(new Coordinate(101, 1)); - LineString expected = GEOMETRY_FACTORY.createLineString( - lineCoordinates.toArray(new Coordinate[lineCoordinates.size()])); - assertGeometryEquals(jtsGeom(expected), lineGeoJson); + try (XContentParser parser = createParser(lineGeoJson)) { + parser.nextToken(); + + boolean useJTS = randomBoolean(); + if (useJTS) { + Shape shape = ShapeParser.parse(parser).buildS4J(); + ElasticsearchGeoAssertions.assertLineString(shape, useJTS); + } else { + ElasticsearchGeoAssertions.assertLineString(ShapeParser.parse(parser).buildLucene(), useJTS); + } + } + + } @Override @@ -118,7 +134,15 @@ public void testParseMultiLineString() throws IOException { new Coordinate(103, 3), }), }); - assertGeometryEquals(jtsGeom(expected), multilinesGeoJson); + boolean useJTS = randomBoolean(); + if (useJTS) { + assertGeometryEquals(jtsGeom(expected), multilinesGeoJson, useJTS); + } else { + assertGeometryEquals(new Line[] { + new Line(new double[] {0d, 1d}, new double[] {100d, 101d}), + new Line(new double[] {2d, 3d}, new double[] {102d, 103d})}, + multilinesGeoJson, useJTS); + } } public void testParseCircle() throws IOException { @@ -130,7 +154,7 @@ public void testParseCircle() throws IOException { .endObject(); Circle expected = SPATIAL_CONTEXT.makeCircle(100.0, 0.0, 360 * 100 / GeoUtils.EARTH_EQUATOR); - assertGeometryEquals(expected, multilinesGeoJson); + assertGeometryEquals(expected, multilinesGeoJson, true); } public void testParseMultiDimensionShapes() throws IOException { @@ -171,9 +195,14 @@ public void testParseEnvelope() throws IOException { .startArray().value(50).value(-30).endArray() .endArray() .endObject(); - - Rectangle expected = SPATIAL_CONTEXT.makeRectangle(-50, 50, -30, 30); - assertGeometryEquals(expected, multilinesGeoJson); + boolean useJTS = randomBoolean(); + if (useJTS) { + Rectangle expected = SPATIAL_CONTEXT.makeRectangle(-50, 50, -30, 30); + assertGeometryEquals(expected, multilinesGeoJson, useJTS); + } else { + assertGeometryEquals(new org.apache.lucene.geo.Rectangle(-30, 30, -50, 50), + multilinesGeoJson, useJTS); + } // test #2: envelope that spans dateline multilinesGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "envelope") @@ -183,8 +212,13 @@ public void testParseEnvelope() throws IOException { .endArray() .endObject(); - expected = SPATIAL_CONTEXT.makeRectangle(50, -50, -30, 30); - assertGeometryEquals(expected, multilinesGeoJson); + if (useJTS) { + Rectangle expected = SPATIAL_CONTEXT.makeRectangle(50, -50, -30, 30); + assertGeometryEquals(expected, multilinesGeoJson, useJTS); + } else { + assertGeometryEquals(new org.apache.lucene.geo.Rectangle(-30, 30, 50, -50), + multilinesGeoJson, useJTS); + } // test #3: "envelope" (actually a triangle) with invalid number of coordinates (TopRight, BottomLeft, BottomRight) multilinesGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "envelope") @@ -234,10 +268,18 @@ public void testParsePolygon() throws IOException { shellCoordinates.add(new Coordinate(101, 1)); shellCoordinates.add(new Coordinate(100, 1)); shellCoordinates.add(new Coordinate(100, 0)); - - LinearRing shell = GEOMETRY_FACTORY.createLinearRing(shellCoordinates.toArray(new Coordinate[shellCoordinates.size()])); - Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, null); - assertGeometryEquals(jtsGeom(expected), polygonGeoJson); + Coordinate[] coordinates = shellCoordinates.toArray(new Coordinate[shellCoordinates.size()]); + boolean useJTS = randomBoolean(); + if (useJTS) { + LinearRing shell = GEOMETRY_FACTORY.createLinearRing(coordinates); + Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, null); + assertGeometryEquals(jtsGeom(expected), polygonGeoJson, useJTS); + } else { + org.apache.lucene.geo.Polygon p = new org.apache.lucene.geo.Polygon( + Arrays.stream(coordinates).mapToDouble(i->i.y).toArray(), + Arrays.stream(coordinates).mapToDouble(i->i.x).toArray()); + assertGeometryEquals(p, polygonGeoJson, useJTS); + } } public void testParse3DPolygon() throws IOException { @@ -464,9 +506,13 @@ public void testParseOGCPolygonWithoutHoles() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).buildS4J(); - - ElasticsearchGeoAssertions.assertPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + Shape shape = ShapeParser.parse(parser).buildS4J(); + ElasticsearchGeoAssertions.assertPolygon(shape, useJTS); + } else { + ElasticsearchGeoAssertions.assertPolygon(ShapeParser.parse(parser).buildLucene(), useJTS); + } } // test 2: ccw poly crossing dateline @@ -485,9 +531,13 @@ public void testParseOGCPolygonWithoutHoles() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).buildS4J(); - - ElasticsearchGeoAssertions.assertMultiPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + Shape shape = ShapeParser.parse(parser).buildS4J(); + ElasticsearchGeoAssertions.assertMultiPolygon(shape, useJTS); + } else { + ElasticsearchGeoAssertions.assertMultiPolygon(ShapeParser.parse(parser).buildLucene(), useJTS); + } } // test 3: cw poly not crossing dateline @@ -506,9 +556,13 @@ public void testParseOGCPolygonWithoutHoles() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).buildS4J(); - - ElasticsearchGeoAssertions.assertPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + Shape shape = ShapeParser.parse(parser).buildS4J(); + ElasticsearchGeoAssertions.assertPolygon(shape, useJTS); + } else { + ElasticsearchGeoAssertions.assertPolygon(ShapeParser.parse(parser).buildLucene(), useJTS); + } } // test 4: cw poly crossing dateline @@ -527,9 +581,13 @@ public void testParseOGCPolygonWithoutHoles() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).buildS4J(); - - ElasticsearchGeoAssertions.assertMultiPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + Shape shape = ShapeParser.parse(parser).buildS4J(); + ElasticsearchGeoAssertions.assertMultiPolygon(shape, useJTS); + } else { + ElasticsearchGeoAssertions.assertMultiPolygon(ShapeParser.parse(parser).buildLucene(), useJTS); + } } } @@ -556,9 +614,13 @@ public void testParseOGCPolygonWithHoles() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).buildS4J(); - - ElasticsearchGeoAssertions.assertPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + Shape shape = ShapeParser.parse(parser).buildS4J(); + ElasticsearchGeoAssertions.assertPolygon(shape, useJTS); + } else { + ElasticsearchGeoAssertions.assertPolygon(ShapeParser.parse(parser).buildLucene(), useJTS); + } } // test 2: ccw poly crossing dateline @@ -583,9 +645,13 @@ public void testParseOGCPolygonWithHoles() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).buildS4J(); - - ElasticsearchGeoAssertions.assertMultiPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + Shape shape = ShapeParser.parse(parser).buildS4J(); + ElasticsearchGeoAssertions.assertMultiPolygon(shape, useJTS); + } else { + ElasticsearchGeoAssertions.assertMultiPolygon(ShapeParser.parse(parser).buildLucene(), useJTS); + } } // test 3: cw poly not crossing dateline @@ -610,9 +676,13 @@ public void testParseOGCPolygonWithHoles() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).buildS4J(); - - ElasticsearchGeoAssertions.assertPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + Shape shape = ShapeParser.parse(parser).buildS4J(); + ElasticsearchGeoAssertions.assertPolygon(shape, useJTS); + } else { + ElasticsearchGeoAssertions.assertPolygon(ShapeParser.parse(parser).buildLucene(), useJTS); + } } @@ -638,9 +708,13 @@ public void testParseOGCPolygonWithHoles() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).buildS4J(); - - ElasticsearchGeoAssertions.assertMultiPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + Shape shape = ShapeParser.parse(parser).buildS4J(); + ElasticsearchGeoAssertions.assertMultiPolygon(shape, useJTS); + } else { + ElasticsearchGeoAssertions.assertMultiPolygon(ShapeParser.parse(parser).buildLucene(), useJTS); + } } } @@ -784,13 +858,24 @@ public void testParsePolygonWithHole() throws IOException { holeCoordinates.add(new Coordinate(100.2, 0.8)); holeCoordinates.add(new Coordinate(100.2, 0.2)); - LinearRing shell = GEOMETRY_FACTORY.createLinearRing( + boolean useJTS = randomBoolean(); + if (useJTS) { + LinearRing shell = GEOMETRY_FACTORY.createLinearRing( shellCoordinates.toArray(new Coordinate[shellCoordinates.size()])); - LinearRing[] holes = new LinearRing[1]; - holes[0] = GEOMETRY_FACTORY.createLinearRing( + LinearRing[] holes = new LinearRing[1]; + holes[0] = GEOMETRY_FACTORY.createLinearRing( holeCoordinates.toArray(new Coordinate[holeCoordinates.size()])); - Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, holes); - assertGeometryEquals(jtsGeom(expected), polygonGeoJson); + Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, holes); + assertGeometryEquals(jtsGeom(expected), polygonGeoJson, useJTS); + } else { + org.apache.lucene.geo.Polygon hole = + new org.apache.lucene.geo.Polygon( + new double[] {0.8d, 0.2d, 0.2d, 0.8d, 0.8d}, new double[] {100.8d, 100.8d, 100.2d, 100.2d, 100.8d}); + org.apache.lucene.geo.Polygon p = + new org.apache.lucene.geo.Polygon( + new double[] {0d, 0d, 1d, 1d, 0d}, new double[] {100d, 101d, 101d, 100d, 100d}, hole); + assertGeometryEquals(p, polygonGeoJson, useJTS); + } } public void testParseSelfCrossingPolygon() throws IOException { @@ -826,11 +911,17 @@ public void testParseMultiPoint() throws IOException { .startArray().value(101.0).value(1.0).endArray() .endArray() .endObject(); - - ShapeCollection expected = shapeCollection( + boolean useJTS = randomBoolean(); + if (useJTS) { + ShapeCollection expected = shapeCollection( SPATIAL_CONTEXT.makePoint(100, 0), SPATIAL_CONTEXT.makePoint(101, 1.0)); - assertGeometryEquals(expected, multiPointGeoJson); + assertGeometryEquals(expected, multiPointGeoJson, useJTS); + } else { + assertGeometryEquals(new double[][]{ + new double[] {100d, 0d}, + new double[] {101d, 1d}}, multiPointGeoJson, useJTS); + } } @Override @@ -894,13 +985,27 @@ public void testParseMultiPolygon() throws IOException { shellCoordinates.add(new Coordinate(102, 2)); shellCoordinates.add(new Coordinate(102, 3)); - - shell = GEOMETRY_FACTORY.createLinearRing(shellCoordinates.toArray(new Coordinate[shellCoordinates.size()])); - Polygon withoutHoles = GEOMETRY_FACTORY.createPolygon(shell, null); - - Shape expected = shapeCollection(withoutHoles, withHoles); - - assertGeometryEquals(expected, multiPolygonGeoJson); + boolean useJTS = randomBoolean(); + if (useJTS) { + shell = GEOMETRY_FACTORY.createLinearRing(shellCoordinates.toArray(new Coordinate[shellCoordinates.size()])); + Polygon withoutHoles = GEOMETRY_FACTORY.createPolygon(shell, null); + + Shape expected = shapeCollection(withoutHoles, withHoles); + + assertGeometryEquals(expected, multiPolygonGeoJson, useJTS); + } else { + org.apache.lucene.geo.Polygon hole = + new org.apache.lucene.geo.Polygon( + new double[] {0.8d, 0.2d, 0.2d, 0.8d, 0.8d}, new double[] {100.8d, 100.8d, 100.2d, 100.2d, 100.8d}); + + org.apache.lucene.geo.Polygon[] polygons = new org.apache.lucene.geo.Polygon[] { + new org.apache.lucene.geo.Polygon( + new double[] {2d, 3d, 3d, 2d, 2d}, new double[] {103d, 103d, 102d, 102d, 103d}), + new org.apache.lucene.geo.Polygon( + new double[] {0d, 1d, 1d, 0d, 0d}, new double[] {101d, 101d, 100d, 100d, 101d}, hole) + }; + assertGeometryEquals(polygons, multiPolygonGeoJson, useJTS); + } // test #2: multipolygon; one polygon with one hole // this test converting the multipolygon from a ShapeCollection type @@ -942,12 +1047,24 @@ public void testParseMultiPolygon() throws IOException { holeCoordinates.add(new Coordinate(100.8, 0.8)); holeCoordinates.add(new Coordinate(100.2, 0.8)); - shell = GEOMETRY_FACTORY.createLinearRing(shellCoordinates.toArray(new Coordinate[shellCoordinates.size()])); - holes = new LinearRing[1]; - holes[0] = GEOMETRY_FACTORY.createLinearRing(holeCoordinates.toArray(new Coordinate[holeCoordinates.size()])); - withHoles = GEOMETRY_FACTORY.createPolygon(shell, holes); - - assertGeometryEquals(jtsGeom(withHoles), multiPolygonGeoJson); + if (useJTS) { + shell = GEOMETRY_FACTORY.createLinearRing(shellCoordinates.toArray(new Coordinate[shellCoordinates.size()])); + holes = new LinearRing[1]; + holes[0] = GEOMETRY_FACTORY.createLinearRing(holeCoordinates.toArray(new Coordinate[holeCoordinates.size()])); + withHoles = GEOMETRY_FACTORY.createPolygon(shell, holes); + + assertGeometryEquals(jtsGeom(withHoles), multiPolygonGeoJson, useJTS); + } else { + org.apache.lucene.geo.Polygon hole = + new org.apache.lucene.geo.Polygon( + new double[] {0.8d, 0.2d, 0.2d, 0.8d, 0.8d}, new double[] {100.8d, 100.8d, 100.2d, 100.2d, 100.8d}); + + org.apache.lucene.geo.Polygon[] polygons = new org.apache.lucene.geo.Polygon[] { + new org.apache.lucene.geo.Polygon( + new double[] {0d, 0d, 1d, 1d, 0d}, new double[] {100d, 101d, 101d, 100d, 100d}, hole) + }; + assertGeometryEquals(polygons, multiPolygonGeoJson, useJTS); + } } @Override @@ -969,18 +1086,26 @@ public void testParseGeometryCollection() throws IOException { .endObject() .endArray() .endObject(); + boolean useJTS = randomBoolean(); - Shape[] expected = new Shape[2]; - LineString expectedLineString = GEOMETRY_FACTORY.createLineString(new Coordinate[]{ + if (useJTS) { + Shape[] expected = new Shape[2]; + LineString expectedLineString = GEOMETRY_FACTORY.createLineString(new Coordinate[]{ new Coordinate(100, 0), new Coordinate(101, 1), - }); - expected[0] = jtsGeom(expectedLineString); - Point expectedPoint = GEOMETRY_FACTORY.createPoint(new Coordinate(102.0, 2.0)); - expected[1] = new JtsPoint(expectedPoint, SPATIAL_CONTEXT); - - //equals returns true only if geometries are in the same order - assertGeometryEquals(shapeCollection(expected), geometryCollectionGeoJson); + }); + expected[0] = jtsGeom(expectedLineString); + Point expectedPoint = GEOMETRY_FACTORY.createPoint(new Coordinate(102.0, 2.0)); + expected[1] = new JtsPoint(expectedPoint, SPATIAL_CONTEXT); + + //equals returns true only if geometries are in the same order + assertGeometryEquals(shapeCollection(expected), geometryCollectionGeoJson, useJTS); + } else { + Object[] expected = new Object[] { + new Line(new double[] {0d, 1d}, new double[] {100d, 101d}), + new GeoPoint(2d, 102d)}; + assertGeometryEquals(expected, geometryCollectionGeoJson, useJTS); + } } public void testThatParserExtractsCorrectTypeAndCoordinatesFromArbitraryJson() throws IOException { @@ -999,9 +1124,14 @@ public void testThatParserExtractsCorrectTypeAndCoordinatesFromArbitraryJson() t .startObject("nested").startArray("coordinates").value(200.0).value(0.0).endArray().endObject() .startObject("lala").field("type", "NotAPoint").endObject() .endObject(); - - Point expected = GEOMETRY_FACTORY.createPoint(new Coordinate(100.0, 0.0)); - assertGeometryEquals(new JtsPoint(expected, SPATIAL_CONTEXT), pointGeoJson); + boolean useJTS = randomBoolean(); + if (useJTS) { + Point expected = GEOMETRY_FACTORY.createPoint(new Coordinate(100.0, 0.0)); + assertGeometryEquals(new JtsPoint(expected, SPATIAL_CONTEXT), pointGeoJson, useJTS); + } else { + double[] expected = new double[] {100d, 0d}; + assertGeometryEquals(expected, pointGeoJson, useJTS); + } } public void testParseOrientationOption() throws IOException { @@ -1030,9 +1160,13 @@ public void testParseOrientationOption() throws IOException { try (XContentParser parser = createParser(polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).buildS4J(); - - ElasticsearchGeoAssertions.assertPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + Shape shape = ShapeParser.parse(parser).buildS4J(); + ElasticsearchGeoAssertions.assertPolygon(shape, useJTS); + } else { + ElasticsearchGeoAssertions.assertPolygon(ShapeParser.parse(parser).buildLucene(), useJTS); + } } // test 2: valid ccw (right handed system) poly not crossing dateline (with 'ccw' field) @@ -1060,9 +1194,13 @@ public void testParseOrientationOption() throws IOException { try (XContentParser parser = createParser(polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).buildS4J(); - - ElasticsearchGeoAssertions.assertPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + Shape shape = ShapeParser.parse(parser).buildS4J(); + ElasticsearchGeoAssertions.assertPolygon(shape, useJTS); + } else { + ElasticsearchGeoAssertions.assertPolygon(ShapeParser.parse(parser).buildLucene(), useJTS); + } } // test 3: valid ccw (right handed system) poly not crossing dateline (with 'counterclockwise' field) @@ -1090,9 +1228,13 @@ public void testParseOrientationOption() throws IOException { try (XContentParser parser = createParser(polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).buildS4J(); - - ElasticsearchGeoAssertions.assertPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + Shape shape = ShapeParser.parse(parser).buildS4J(); + ElasticsearchGeoAssertions.assertPolygon(shape, useJTS); + } else { + ElasticsearchGeoAssertions.assertPolygon(ShapeParser.parse(parser).buildLucene(), useJTS); + } } // test 4: valid cw (left handed system) poly crossing dateline (with 'left' field) @@ -1120,9 +1262,13 @@ public void testParseOrientationOption() throws IOException { try (XContentParser parser = createParser(polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).buildS4J(); - - ElasticsearchGeoAssertions.assertMultiPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + Shape shape = ShapeParser.parse(parser).buildS4J(); + ElasticsearchGeoAssertions.assertMultiPolygon(shape, useJTS); + } else { + ElasticsearchGeoAssertions.assertMultiPolygon(ShapeParser.parse(parser).buildLucene(), useJTS); + } } // test 5: valid cw multipoly (left handed system) poly crossing dateline (with 'cw' field) @@ -1150,9 +1296,13 @@ public void testParseOrientationOption() throws IOException { try (XContentParser parser = createParser(polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).buildS4J(); - - ElasticsearchGeoAssertions.assertMultiPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + Shape shape = ShapeParser.parse(parser).buildS4J(); + ElasticsearchGeoAssertions.assertMultiPolygon(shape, useJTS); + } else { + ElasticsearchGeoAssertions.assertMultiPolygon(ShapeParser.parse(parser).buildLucene(), useJTS); + } } // test 6: valid cw multipoly (left handed system) poly crossing dateline (with 'clockwise' field) @@ -1180,9 +1330,13 @@ public void testParseOrientationOption() throws IOException { try (XContentParser parser = createParser(polygonGeoJson)) { parser.nextToken(); - Shape shape = ShapeParser.parse(parser).buildS4J(); - - ElasticsearchGeoAssertions.assertMultiPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + Shape shape = ShapeParser.parse(parser).buildS4J(); + ElasticsearchGeoAssertions.assertMultiPolygon(shape, useJTS); + } else { + ElasticsearchGeoAssertions.assertMultiPolygon(ShapeParser.parse(parser).buildLucene(), useJTS); + } } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java index 3350dc5b660d8..be33ffe26a9dc 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.geo; import org.apache.lucene.geo.GeoTestUtil; +import org.apache.lucene.geo.Line; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; @@ -58,6 +59,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import static org.elasticsearch.common.geo.builders.ShapeBuilder.SPATIAL_CONTEXT; @@ -84,9 +86,9 @@ private static XContentBuilder toWKTContent(ShapeBuilder builder, boolean return XContentFactory.jsonBuilder().value(wkt); } - private void assertExpected(Shape expected, ShapeBuilder builder) throws IOException { + private void assertExpected(Object expected, ShapeBuilder builder, boolean useJTS) throws IOException { XContentBuilder xContentBuilder = toWKTContent(builder, false); - assertGeometryEquals(expected, xContentBuilder); + assertGeometryEquals(expected, xContentBuilder, useJTS); } private void assertMalformed(ShapeBuilder builder) throws IOException { @@ -99,7 +101,12 @@ public void testParsePoint() throws IOException { GeoPoint p = RandomShapeGenerator.randomPoint(random()); Coordinate c = new Coordinate(p.lon(), p.lat()); Point expected = GEOMETRY_FACTORY.createPoint(c); - assertExpected(new JtsPoint(expected, SPATIAL_CONTEXT), new PointBuilder().coordinate(c)); + boolean useJTS = randomBoolean(); + if (useJTS) { + assertExpected(new JtsPoint(expected, SPATIAL_CONTEXT), new PointBuilder().coordinate(c), useJTS); + } else { + assertExpected(new GeoPoint(p.lat(), p.lon()), new PointBuilder().coordinate(c), useJTS); + } assertMalformed(new PointBuilder().coordinate(c)); } @@ -107,15 +114,27 @@ public void testParsePoint() throws IOException { public void testParseMultiPoint() throws IOException { int numPoints = randomIntBetween(2, 100); List coordinates = new ArrayList<>(numPoints); - Shape[] shapes = new Shape[numPoints]; - GeoPoint p = new GeoPoint(); + boolean useJTS = randomBoolean(); for (int i = 0; i < numPoints; ++i) { - p.reset(GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude()); - coordinates.add(new Coordinate(p.lon(), p.lat())); - shapes[i] = SPATIAL_CONTEXT.makePoint(p.lon(), p.lat()); + coordinates.add(new Coordinate(GeoTestUtil.nextLongitude(), GeoTestUtil.nextLatitude())); + } + if (useJTS) { + Shape[] shapes = new Shape[numPoints]; + for (int i = 0; i < numPoints; ++i) { + Coordinate c = coordinates.get(i); + shapes[i] = SPATIAL_CONTEXT.makePoint(c.x, c.y); + } + ShapeCollection expected = shapeCollection(shapes); + assertExpected(expected, new MultiPointBuilder(coordinates), useJTS); + } else { + double[][] shapes = new double[numPoints][2]; + for (int i = 0; i < numPoints; ++i) { + Coordinate c = coordinates.get(i); + shapes[i][0] = c.x; + shapes[i][1] = c.y; + } + assertExpected(shapes, new MultiPointBuilder(coordinates), useJTS); } - ShapeCollection expected = shapeCollection(shapes); - assertExpected(expected, new MultiPointBuilder(coordinates)); assertMalformed(new MultiPointBuilder(coordinates)); } @@ -133,8 +152,19 @@ private List randomLineStringCoords() { @Override public void testParseLineString() throws IOException { List coordinates = randomLineStringCoords(); - LineString expected = GEOMETRY_FACTORY.createLineString(coordinates.toArray(new Coordinate[coordinates.size()])); - assertExpected(jtsGeom(expected), new LineStringBuilder(coordinates)); + boolean useJTS = randomBoolean(); + if (useJTS) { + LineString expected = GEOMETRY_FACTORY.createLineString(coordinates.toArray(new Coordinate[coordinates.size()])); + assertExpected(jtsGeom(expected), new LineStringBuilder(coordinates), useJTS); + } else { + double[] lats = new double[coordinates.size()]; + double[] lons = new double[lats.length]; + for (int i = 0; i < lats.length; ++i) { + lats[i] = coordinates.get(i).y; + lons[i] = coordinates.get(i).x; + } + assertExpected(new Line(lats, lons), new LineStringBuilder(coordinates), useJTS); + } } @Override @@ -148,9 +178,20 @@ public void testParseMultiLineString() throws IOException { lineStrings.add(GEOMETRY_FACTORY.createLineString(coords)); builder.linestring(new LineStringBuilder(lsc)); } - MultiLineString expected = GEOMETRY_FACTORY.createMultiLineString( - lineStrings.toArray(new LineString[lineStrings.size()])); - assertExpected(jtsGeom(expected), builder); + boolean useJTS = randomBoolean(); + if (useJTS) { + MultiLineString expected = GEOMETRY_FACTORY.createMultiLineString( + lineStrings.toArray(new LineString[lineStrings.size()])); + assertExpected(jtsGeom(expected), builder, useJTS); + } else { + Line[] lines = new Line[lineStrings.size()]; + for (int j = 0; j < lineStrings.size(); ++j) { + Coordinate[] c = lineStrings.get(j).getCoordinates(); + lines[j] = new Line(Arrays.stream(c).mapToDouble(i->i.y).toArray(), + Arrays.stream(c).mapToDouble(i->i.x).toArray()); + } + assertExpected(lines, builder, useJTS); + } assertMalformed(builder); } @@ -159,9 +200,16 @@ public void testParsePolygon() throws IOException { PolygonBuilder builder = PolygonBuilder.class.cast( RandomShapeGenerator.createShape(random(), RandomShapeGenerator.ShapeType.POLYGON)); Coordinate[] coords = builder.coordinates()[0][0]; - LinearRing shell = GEOMETRY_FACTORY.createLinearRing(coords); - Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, null); - assertExpected(jtsGeom(expected), builder); + + boolean useJTS = randomBoolean(); + if (useJTS) { + LinearRing shell = GEOMETRY_FACTORY.createLinearRing(coords); + Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, null); + assertExpected(jtsGeom(expected), builder, useJTS); + } else { + assertExpected(new org.apache.lucene.geo.Polygon(Arrays.stream(coords).mapToDouble(i->i.y).toArray(), + Arrays.stream(coords).mapToDouble(i->i.x).toArray()), builder, useJTS); + } assertMalformed(builder); } @@ -180,8 +228,20 @@ public void testParseMultiPolygon() throws IOException { shell = GEOMETRY_FACTORY.createLinearRing(coordinates); shapes[i] = GEOMETRY_FACTORY.createPolygon(shell, null); } - Shape expected = shapeCollection(shapes); - assertExpected(expected, builder); + + boolean useJTS = randomBoolean(); + if (useJTS) { + Shape expected = shapeCollection(shapes); + assertExpected(expected, builder, useJTS); + } else { + org.apache.lucene.geo.Polygon[] polygons = new org.apache.lucene.geo.Polygon[shapes.length]; + for (int j = 0; j < polygons.length; ++j) { + Coordinate[] c = shapes[j].getCoordinates(); + polygons[j] = new org.apache.lucene.geo.Polygon(Arrays.stream(c).mapToDouble(i->i.y).toArray(), + Arrays.stream(c).mapToDouble(i->i.x).toArray()); + } + assertExpected(polygons, builder, useJTS); + } assertMalformed(builder); } @@ -204,14 +264,24 @@ public void testParsePolygonWithHole() throws IOException { PolygonBuilder polygonWithHole = new PolygonBuilder(new CoordinatesBuilder().coordinates(shellCoordinates)); polygonWithHole.hole(new LineStringBuilder(holeCoordinates)); - LinearRing shell = GEOMETRY_FACTORY.createLinearRing( - shellCoordinates.toArray(new Coordinate[shellCoordinates.size()])); - LinearRing[] holes = new LinearRing[1]; - holes[0] = GEOMETRY_FACTORY.createLinearRing( - holeCoordinates.toArray(new Coordinate[holeCoordinates.size()])); - Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, holes); - - assertExpected(jtsGeom(expected), polygonWithHole); + boolean useJTS = randomBoolean(); + if (useJTS) { + LinearRing shell = GEOMETRY_FACTORY.createLinearRing( + shellCoordinates.toArray(new Coordinate[shellCoordinates.size()])); + LinearRing[] holes = new LinearRing[1]; + holes[0] = GEOMETRY_FACTORY.createLinearRing( + holeCoordinates.toArray(new Coordinate[holeCoordinates.size()])); + Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, holes); + assertExpected(jtsGeom(expected), polygonWithHole, useJTS); + } else { + org.apache.lucene.geo.Polygon hole = + new org.apache.lucene.geo.Polygon( + new double[] {0.8d, 0.8d, 0.2d, 0.2d, 0.8d}, new double[] {100.2d, 100.8d, 100.8d, 100.2d, 100.2d}); + org.apache.lucene.geo.Polygon p = + new org.apache.lucene.geo.Polygon( + new double[] {0d, 1d, 1d, 0d, 0d}, new double[] {101d, 101d, 100d, 100d, 101d}, hole); + assertExpected(p, polygonWithHole, useJTS); + } assertMalformed(polygonWithHole); } @@ -370,8 +440,13 @@ public void testMalformedWKT() throws IOException { public void testParseEnvelope() throws IOException { org.apache.lucene.geo.Rectangle r = GeoTestUtil.nextBox(); EnvelopeBuilder builder = new EnvelopeBuilder(new Coordinate(r.minLon, r.maxLat), new Coordinate(r.maxLon, r.minLat)); - Rectangle expected = SPATIAL_CONTEXT.makeRectangle(r.minLon, r.maxLon, r.minLat, r.maxLat); - assertExpected(expected, builder); + boolean useJTS = randomBoolean(); + if (useJTS) { + Rectangle expected = SPATIAL_CONTEXT.makeRectangle(r.minLon, r.maxLon, r.minLat, r.maxLat); + assertExpected(expected, builder, useJTS); + } else { + assertExpected(r, builder, useJTS); + } assertMalformed(builder); } @@ -386,10 +461,19 @@ public void testParseGeometryCollection() throws IOException { // assert empty shape collection GeometryCollectionBuilder builder = new GeometryCollectionBuilder(); Shape[] expected = new Shape[0]; - assertEquals(shapeCollection(expected).isEmpty(), builder.buildS4J().isEmpty()); + if (randomBoolean()) { + assertEquals(shapeCollection(expected).isEmpty(), builder.buildS4J().isEmpty()); + } else { + assertEquals(shapeCollection(expected).isEmpty(), ((Object[])builder.buildLucene()).length == 0); + } } else { GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(random()); - assertExpected(gcb.buildS4J(), gcb); + boolean useJTS = randomBoolean(); + if (useJTS) { + assertExpected(gcb.buildS4J(), gcb, useJTS); + } else { + assertExpected(gcb.buildLucene(), gcb, useJTS); + } } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java b/server/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java index e883e74d21b9c..27c79b2217c8b 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java @@ -49,65 +49,124 @@ public class ShapeBuilderTests extends ESTestCase { public void testNewPoint() { - Point point = new PointBuilder().coordinate(-100, 45).buildS4J(); - assertEquals(-100D, point.getX(), 0.0d); - assertEquals(45D, point.getY(), 0.0d); + PointBuilder pb = new PointBuilder().coordinate(-100, 45); + boolean useJTS = randomBoolean(); + if (useJTS) { + Point point = pb.buildS4J(); + assertEquals(-100D, point.getX(), 0.0d); + assertEquals(45D, point.getY(), 0.0d); + } else { + GeoPoint point = pb.buildLucene(); + assertEquals(-100D, point.getLon(), 0.0d); + assertEquals(45D, point.getLat(), 0.0d); + } } public void testNewRectangle() { - Rectangle rectangle = new EnvelopeBuilder(new Coordinate(-45, 30), new Coordinate(45, -30)).buildS4J(); - assertEquals(-45D, rectangle.getMinX(), 0.0d); - assertEquals(-30D, rectangle.getMinY(), 0.0d); - assertEquals(45D, rectangle.getMaxX(), 0.0d); - assertEquals(30D, rectangle.getMaxY(), 0.0d); + EnvelopeBuilder eb = new EnvelopeBuilder(new Coordinate(-45, 30), new Coordinate(45, -30)); + boolean useJTS = randomBoolean(); + if (useJTS) { + Rectangle rectangle = eb.buildS4J(); + assertEquals(-45D, rectangle.getMinX(), 0.0d); + assertEquals(-30D, rectangle.getMinY(), 0.0d); + assertEquals(45D, rectangle.getMaxX(), 0.0d); + assertEquals(30D, rectangle.getMaxY(), 0.0d); + } else { + org.apache.lucene.geo.Rectangle rectangle = eb.buildLucene(); + assertEquals(-45D, rectangle.minLon, 0.0d); + assertEquals(-30D, rectangle.minLat, 0.0d); + assertEquals(45D, rectangle.maxLon, 0.0d); + assertEquals(30D, rectangle.maxLat, 0.0d); + } } public void testNewPolygon() { - Polygon polygon = new PolygonBuilder(new CoordinatesBuilder() + PolygonBuilder pb = new PolygonBuilder(new CoordinatesBuilder() .coordinate(-45, 30) .coordinate(45, 30) .coordinate(45, -30) .coordinate(-45, -30) - .coordinate(-45, 30)).toPolygonS4J(); - - LineString exterior = polygon.getExteriorRing(); - assertEquals(exterior.getCoordinateN(0), new Coordinate(-45, 30)); - assertEquals(exterior.getCoordinateN(1), new Coordinate(45, 30)); - assertEquals(exterior.getCoordinateN(2), new Coordinate(45, -30)); - assertEquals(exterior.getCoordinateN(3), new Coordinate(-45, -30)); + .coordinate(-45, 30)); + boolean useJTS = randomBoolean(); + if (useJTS) { + Polygon poly = pb.toPolygonS4J(); + LineString exterior = poly.getExteriorRing(); + assertEquals(exterior.getCoordinateN(0), new Coordinate(-45, 30)); + assertEquals(exterior.getCoordinateN(1), new Coordinate(45, 30)); + assertEquals(exterior.getCoordinateN(2), new Coordinate(45, -30)); + assertEquals(exterior.getCoordinateN(3), new Coordinate(-45, -30)); + } else { + org.apache.lucene.geo.Polygon poly = (org.apache.lucene.geo.Polygon)(pb.toPolygonLucene()); + assertEquals(poly.getPolyLat(0), 30, 0d); + assertEquals(poly.getPolyLon(0), -45, 0d); + assertEquals(poly.getPolyLat(1), 30, 0d); + assertEquals(poly.getPolyLon(1), 45, 0d); + assertEquals(poly.getPolyLat(2), -30, 0d); + assertEquals(poly.getPolyLon(2), 45, 0d); + assertEquals(poly.getPolyLat(3), -30, 0d); + assertEquals(poly.getPolyLon(3), -45, 0d); + } } public void testNewPolygon_coordinate() { - Polygon polygon = new PolygonBuilder(new CoordinatesBuilder() + PolygonBuilder pb = new PolygonBuilder(new CoordinatesBuilder() .coordinate(new Coordinate(-45, 30)) .coordinate(new Coordinate(45, 30)) .coordinate(new Coordinate(45, -30)) .coordinate(new Coordinate(-45, -30)) - .coordinate(new Coordinate(-45, 30))).toPolygonS4J(); - - LineString exterior = polygon.getExteriorRing(); - assertEquals(exterior.getCoordinateN(0), new Coordinate(-45, 30)); - assertEquals(exterior.getCoordinateN(1), new Coordinate(45, 30)); - assertEquals(exterior.getCoordinateN(2), new Coordinate(45, -30)); - assertEquals(exterior.getCoordinateN(3), new Coordinate(-45, -30)); + .coordinate(new Coordinate(-45, 30))); + + boolean useJTS = randomBoolean(); + if (useJTS) { + Polygon poly = pb.toPolygonS4J(); + LineString exterior = poly.getExteriorRing(); + assertEquals(exterior.getCoordinateN(0), new Coordinate(-45, 30)); + assertEquals(exterior.getCoordinateN(1), new Coordinate(45, 30)); + assertEquals(exterior.getCoordinateN(2), new Coordinate(45, -30)); + assertEquals(exterior.getCoordinateN(3), new Coordinate(-45, -30)); + } else { + org.apache.lucene.geo.Polygon poly = (org.apache.lucene.geo.Polygon)(pb.toPolygonLucene()); + assertEquals(poly.getPolyLat(0), 30, 0d); + assertEquals(poly.getPolyLon(0), -45, 0d); + assertEquals(poly.getPolyLat(1), 30, 0d); + assertEquals(poly.getPolyLon(1), 45, 0d); + assertEquals(poly.getPolyLat(2), -30, 0d); + assertEquals(poly.getPolyLon(2), 45, 0d); + assertEquals(poly.getPolyLat(3), -30, 0d); + assertEquals(poly.getPolyLon(3), -45, 0d); + } } public void testNewPolygon_coordinates() { - Polygon polygon = new PolygonBuilder(new CoordinatesBuilder() + PolygonBuilder pb = new PolygonBuilder(new CoordinatesBuilder() .coordinates(new Coordinate(-45, 30), new Coordinate(45, 30), new Coordinate(45, -30), new Coordinate(-45, -30), new Coordinate(-45, 30)) - ).toPolygonS4J(); + ); - LineString exterior = polygon.getExteriorRing(); - assertEquals(exterior.getCoordinateN(0), new Coordinate(-45, 30)); - assertEquals(exterior.getCoordinateN(1), new Coordinate(45, 30)); - assertEquals(exterior.getCoordinateN(2), new Coordinate(45, -30)); - assertEquals(exterior.getCoordinateN(3), new Coordinate(-45, -30)); + boolean useJTS = randomBoolean(); + if (useJTS) { + Polygon poly = pb.toPolygonS4J(); + LineString exterior = poly.getExteriorRing(); + assertEquals(exterior.getCoordinateN(0), new Coordinate(-45, 30)); + assertEquals(exterior.getCoordinateN(1), new Coordinate(45, 30)); + assertEquals(exterior.getCoordinateN(2), new Coordinate(45, -30)); + assertEquals(exterior.getCoordinateN(3), new Coordinate(-45, -30)); + } else { + org.apache.lucene.geo.Polygon poly = (org.apache.lucene.geo.Polygon)(pb.toPolygonLucene()); + assertEquals(poly.getPolyLat(0), 30, 0d); + assertEquals(poly.getPolyLon(0), -45, 0d); + assertEquals(poly.getPolyLat(1), 30, 0d); + assertEquals(poly.getPolyLon(1), 45, 0d); + assertEquals(poly.getPolyLat(2), -30, 0d); + assertEquals(poly.getPolyLon(2), 45, 0d); + assertEquals(poly.getPolyLat(3), -30, 0d); + assertEquals(poly.getPolyLon(3), -45, 0d); + } } public void testLineStringBuilder() { // Building a simple LineString - new LineStringBuilder(new CoordinatesBuilder() + LineStringBuilder lsb = new LineStringBuilder(new CoordinatesBuilder() .coordinate(-130.0, 55.0) .coordinate(-130.0, -40.0) .coordinate(-15.0, -40.0) @@ -115,10 +174,15 @@ public void testLineStringBuilder() { .coordinate(-45.0, 50.0) .coordinate(-45.0, -15.0) .coordinate(-110.0, -15.0) - .coordinate(-110.0, 55.0)).buildS4J(); + .coordinate(-110.0, 55.0)); + if (randomBoolean()) { + lsb.buildS4J(); + } else { + lsb.buildLucene(); + } // Building a linestring that needs to be wrapped - new LineStringBuilder(new CoordinatesBuilder() + lsb = new LineStringBuilder(new CoordinatesBuilder() .coordinate(100.0, 50.0) .coordinate(110.0, -40.0) .coordinate(240.0, -40.0) @@ -126,31 +190,40 @@ public void testLineStringBuilder() { .coordinate(200.0, 60.0) .coordinate(200.0, -30.0) .coordinate(130.0, -30.0) - .coordinate(130.0, 60.0) - ) - .buildS4J(); + .coordinate(130.0, 60.0)); + if (randomBoolean()) { + lsb.buildS4J(); + } else { + lsb.buildLucene(); + } // Building a lineString on the dateline - new LineStringBuilder(new CoordinatesBuilder() + lsb = new LineStringBuilder(new CoordinatesBuilder() .coordinate(-180.0, 80.0) .coordinate(-180.0, 40.0) .coordinate(-180.0, -40.0) - .coordinate(-180.0, -80.0) - ) - .buildS4J(); + .coordinate(-180.0, -80.0)); + if (randomBoolean()) { + lsb.buildS4J(); + } else { + lsb.buildLucene(); + } // Building a lineString on the dateline - new LineStringBuilder(new CoordinatesBuilder() + lsb = new LineStringBuilder(new CoordinatesBuilder() .coordinate(180.0, 80.0) .coordinate(180.0, 40.0) .coordinate(180.0, -40.0) - .coordinate(180.0, -80.0) - ) - .buildS4J(); + .coordinate(180.0, -80.0)); + if (randomBoolean()) { + lsb.buildS4J(); + } else { + lsb.buildLucene(); + } } public void testMultiLineString() { - new MultiLineStringBuilder() + MultiLineStringBuilder mlsb = new MultiLineStringBuilder() .linestring(new LineStringBuilder(new CoordinatesBuilder() .coordinate(-100.0, 50.0) .coordinate(50.0, 50.0) @@ -164,8 +237,12 @@ public void testMultiLineString() { .coordinate(50.0, 0.0) .coordinate(-100.0, 0.0) ) - ) - .buildS4J(); + ); + if (randomBoolean()) { + mlsb.buildS4J(); + } else { + mlsb.buildLucene(); + } // LineString that needs to be wrapped new MultiLineStringBuilder() @@ -182,8 +259,12 @@ public void testMultiLineString() { .coordinate(200.0, 0.0) .coordinate(150.0, 0.0) ) - ) - .buildS4J(); + ); + if (randomBoolean()) { + mlsb.buildS4J(); + } else { + mlsb.buildLucene(); + } } public void testPolygonSelfIntersection() { @@ -196,6 +277,7 @@ public void testPolygonSelfIntersection() { assertThat(e.getMessage(), containsString("Self-intersection at or near point (0.0")); } + /** note: only supported by S4J at the moment */ public void testGeoCircle() { double earthCircumference = 40075016.69; Circle circle = new CircleBuilder().center(0, 0).radius("100m").buildS4J(); @@ -222,28 +304,34 @@ public void testGeoCircle() { } public void testPolygonWrapping() { - Shape shape = new PolygonBuilder(new CoordinatesBuilder() + PolygonBuilder pb = new PolygonBuilder(new CoordinatesBuilder() .coordinate(-150.0, 65.0) .coordinate(-250.0, 65.0) .coordinate(-250.0, -65.0) .coordinate(-150.0, -65.0) - .close() - ) - .buildS4J(); - - assertMultiPolygon(shape); + .close()); + + boolean useJTS = randomBoolean(); + if (useJTS) { + assertMultiPolygon(pb.buildS4J(), useJTS); + } else { + assertMultiPolygon(pb.buildLucene(), useJTS); + } } public void testLineStringWrapping() { - Shape shape = new LineStringBuilder(new CoordinatesBuilder() + LineStringBuilder lsb = new LineStringBuilder(new CoordinatesBuilder() .coordinate(-150.0, 65.0) .coordinate(-250.0, 65.0) .coordinate(-250.0, -65.0) .coordinate(-150.0, -65.0) - .close() - ) - .buildS4J(); - assertMultiLineString(shape); + .close()); + boolean useJTS = randomBoolean(); + if (useJTS) { + assertMultiLineString(lsb.buildS4J(), useJTS); + } else { + assertMultiLineString(lsb.buildLucene(), useJTS); + } } public void testDatelineOGC() { @@ -285,9 +373,12 @@ public void testDatelineOGC() { .coordinate(-177,1) .coordinate(-179,1) )); - - Shape shape = builder.close().buildS4J(); - assertMultiPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + assertMultiPolygon(builder.close().buildS4J(), useJTS); + } else { + assertMultiPolygon(builder.close().buildLucene(), useJTS); + } } public void testDateline() { @@ -329,9 +420,12 @@ public void testDateline() { .coordinate(-179,2) .coordinate(-179,1) )); - - Shape shape = builder.close().buildS4J(); - assertMultiPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + assertMultiPolygon(builder.close().buildS4J(), useJTS); + } else { + assertMultiPolygon(builder.close().buildLucene(), useJTS); + } } public void testComplexShapeWithHole() { @@ -405,9 +499,12 @@ public void testComplexShapeWithHole() { .coordinate(-85.0000002,37.1317672) ) ); - - Shape shape = builder.close().buildS4J(); - assertPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + assertPolygon(builder.close().buildS4J(), useJTS); + } else { + assertPolygon(builder.close().buildLucene(), useJTS); + } } public void testShapeWithHoleAtEdgeEndPoints() { @@ -428,9 +525,12 @@ public void testShapeWithHoleAtEdgeEndPoints() { .coordinate(-4, 1) .coordinate(4, 1) )); - - Shape shape = builder.close().buildS4J(); - assertPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + assertPolygon(builder.close().buildS4J(), useJTS); + } else { + assertPolygon(builder.close().buildLucene(), useJTS); + } } public void testShapeWithPointOnDateline() { @@ -440,9 +540,12 @@ public void testShapeWithPointOnDateline() { .coordinate(176, -4) .coordinate(180, 0) ); - - Shape shape = builder.close().buildS4J(); - assertPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + assertPolygon(builder.close().buildS4J(), useJTS); + } else { + assertPolygon(builder.close().buildLucene(), useJTS); + } } public void testShapeWithEdgeAlongDateline() { @@ -454,8 +557,12 @@ public void testShapeWithEdgeAlongDateline() { .coordinate(180, 0) ); - Shape shape = builder.close().buildS4J(); - assertPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + assertPolygon(builder.close().buildS4J(), useJTS); + } else { + assertPolygon(builder.close().buildLucene(), useJTS); + } // test case 2: test the negative side of the dateline builder = new PolygonBuilder(new CoordinatesBuilder() @@ -464,9 +571,11 @@ public void testShapeWithEdgeAlongDateline() { .coordinate(-180, -4) .coordinate(-176, 4) ); - - shape = builder.close().buildS4J(); - assertPolygon(shape); + if (useJTS) { + assertPolygon(builder.close().buildS4J(), useJTS); + } else { + assertPolygon(builder.close().buildLucene(), useJTS); + } } public void testShapeWithBoundaryHoles() { @@ -486,8 +595,12 @@ public void testShapeWithBoundaryHoles() { .coordinate(176, -10) .coordinate(176, 10) )); - Shape shape = builder.close().buildS4J(); - assertMultiPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + assertMultiPolygon(builder.close().buildS4J(), useJTS); + } else { + assertMultiPolygon(builder.close().buildLucene(), useJTS); + } // test case 2: test the negative side of the dateline builder = new PolygonBuilder( @@ -508,8 +621,11 @@ public void testShapeWithBoundaryHoles() { .coordinate(-176, 10) .close() )); - shape = builder.close().buildS4J(); - assertMultiPolygon(shape); + if (useJTS) { + assertMultiPolygon(builder.close().buildS4J(), useJTS); + } else { + assertMultiPolygon(builder.close().buildLucene(), useJTS); + } } public void testShapeWithTangentialHole() { @@ -529,8 +645,12 @@ public void testShapeWithTangentialHole() { .coordinate(-180, 5) .coordinate(-177, 10) )); - Shape shape = builder.close().buildS4J(); - assertMultiPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + assertMultiPolygon(builder.close().buildS4J(), useJTS); + } else { + assertMultiPolygon(builder.close().buildLucene(), useJTS); + } } public void testShapeWithInvalidTangentialHole() { @@ -550,7 +670,13 @@ public void testShapeWithInvalidTangentialHole() { .coordinate(179, -10) .coordinate(164, 0) )); - Exception e = expectThrows(InvalidShapeException.class, () -> builder.close().buildS4J()); + Exception e; + boolean useJTS = randomBoolean(); + if (useJTS) { + e = expectThrows(InvalidShapeException.class, () -> builder.close().buildS4J()); + } else { + e = expectThrows(InvalidShapeException.class, () -> builder.close().buildLucene()); + } assertThat(e.getMessage(), containsString("interior cannot share more than one point with the exterior")); } @@ -577,8 +703,12 @@ public void testBoundaryShapeWithTangentialHole() { .coordinate(176, -5) .coordinate(172, 0) )); - Shape shape = builder.close().buildS4J(); - assertMultiPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + assertMultiPolygon(builder.close().buildS4J(), useJTS); + } else { + assertMultiPolygon(builder.close().buildLucene(), useJTS); + } } public void testBoundaryShapeWithInvalidTangentialHole() { @@ -598,7 +728,13 @@ public void testBoundaryShapeWithInvalidTangentialHole() { .coordinate(176, -10) .coordinate(-177, 10) )); - Exception e = expectThrows(InvalidShapeException.class, () -> builder.close().buildS4J()); + Exception e; + boolean useJTS = randomBoolean(); + if (useJTS) { + e = expectThrows(InvalidShapeException.class, () -> builder.close().buildS4J()); + } else { + e = expectThrows(InvalidShapeException.class, () -> builder.close().buildLucene()); + } assertThat(e.getMessage(), containsString("interior cannot share more than one point with the exterior")); } @@ -613,9 +749,12 @@ public void testBoundaryShape() { .coordinate(-180, 90) ); - Shape shape = builder.close().buildS4J(); - - assertPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + assertPolygon(builder.close().buildS4J(), useJTS); + } else { + assertPolygon(builder.close().buildLucene(), useJTS); + } } public void testShapeWithAlternateOrientation() { @@ -627,8 +766,12 @@ public void testShapeWithAlternateOrientation() { .coordinate(180, 0) ); - Shape shape = builder.close().buildS4J(); - assertPolygon(shape); + boolean useJTS = randomBoolean(); + if (useJTS) { + assertPolygon(builder.close().buildS4J(), useJTS); + } else { + assertPolygon(builder.close().buildLucene(), useJTS); + } // cw: geo core will convert to ccw across the dateline builder = new PolygonBuilder(new CoordinatesBuilder() @@ -637,10 +780,11 @@ public void testShapeWithAlternateOrientation() { .coordinate(176, 4) .coordinate(180, 0) ); - - shape = builder.close().buildS4J(); - - assertMultiPolygon(shape); + if (useJTS) { + assertMultiPolygon(builder.close().buildS4J(), useJTS); + } else { + assertMultiPolygon(builder.close().buildLucene(), useJTS); + } } public void testInvalidShapeWithConsecutiveDuplicatePoints() { @@ -651,7 +795,13 @@ public void testInvalidShapeWithConsecutiveDuplicatePoints() { .coordinate(-176, 4) .coordinate(180, 0) ); - Exception e = expectThrows(InvalidShapeException.class, () -> builder.close().buildS4J()); + Exception e; + boolean useJTS = randomBoolean(); + if (useJTS) { + e = expectThrows(InvalidShapeException.class, () -> builder.close().buildS4J()); + } else { + e = expectThrows(InvalidShapeException.class, () -> builder.close().buildLucene()); + } assertThat(e.getMessage(), containsString("duplicate consecutive coordinates at: (")); } diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java index 03a484ceddcb2..b1cbb31c1f884 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java @@ -100,11 +100,12 @@ private GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { // we try to avoid that combination // Also, SpatialStrategy.VECTOR does not support MULTIPOINT Queries while ((shapeType == ShapeType.MULTILINESTRING && strategy == SpatialStrategy.TERM) - || (strategy == SpatialStrategy.VECTOR && (shapeType == ShapeType.MULTIPOINT))) { + || (strategy == SpatialStrategy.VECTOR && shapeType == ShapeType.MULTIPOINT)) { strategy = randomFrom(SpatialStrategy.values()); } builder.strategy(strategy); if (strategy == SpatialStrategy.VECTOR) { + // vector strategy does not yet support CONTAINS queries ShapeRelation relation; if (shapeType != ShapeType.LINESTRING && shapeType != ShapeType.MULTILINESTRING) { // vector strategy does not yet support WITHIN for linestrings diff --git a/server/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchGeoAssertions.java b/server/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchGeoAssertions.java index 74a9fb4522cff..6d156951c6e7d 100644 --- a/server/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchGeoAssertions.java +++ b/server/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchGeoAssertions.java @@ -198,7 +198,7 @@ public static void assertEquals(ShapeCollection s1, ShapeCollection s2) { } } - public static void assertEquals(Shape s1, Shape s2) { + public static void assertEquals(Object s1, Object s2) { if(s1 instanceof JtsGeometry && s2 instanceof JtsGeometry) { assertEquals((JtsGeometry) s1, (JtsGeometry) s2); } else if(s1 instanceof JtsPoint && s2 instanceof JtsPoint) { @@ -211,6 +211,17 @@ public static void assertEquals(Shape s1, Shape s2) { Assert.assertEquals(s1, s2); } else if (s1 instanceof RectangleImpl && s2 instanceof RectangleImpl) { Assert.assertEquals(s1, s2); + } else if (s1 instanceof org.apache.lucene.geo.Line[] && s2 instanceof org.apache.lucene.geo.Line[]) { + Assert.assertArrayEquals((org.apache.lucene.geo.Line[])s1, (org.apache.lucene.geo.Line[])s2); + } else if (s1 instanceof org.apache.lucene.geo.Polygon[] && s2 instanceof org.apache.lucene.geo.Polygon[]) { + Assert.assertArrayEquals((org.apache.lucene.geo.Polygon[]) s1, (org.apache.lucene.geo.Polygon[]) s2); + } else if ((s1 instanceof org.apache.lucene.geo.Line && s2 instanceof org.apache.lucene.geo.Line) + || (s1 instanceof org.apache.lucene.geo.Polygon && s2 instanceof org.apache.lucene.geo.Polygon) + || (s1 instanceof org.apache.lucene.geo.Rectangle && s2 instanceof org.apache.lucene.geo.Rectangle) + || (s1 instanceof GeoPoint && s2 instanceof GeoPoint)) { + Assert.assertEquals(s1, s2); + } else if (s1 instanceof Object[] && s2 instanceof Object[]) { + Assert.assertArrayEquals((Object[])s1, (Object[])s2); } else { //We want to know the type of the shape because we test shape equality in a special way... //... in particular we test that one ring is equivalent to another ring even if the points are rotated or reversed. @@ -219,25 +230,50 @@ public static void assertEquals(Shape s1, Shape s2) { } } - private static Geometry unwrap(Shape shape) { + @Deprecated + private static Geometry unwrapJTS(Object shape) { assertThat(shape, instanceOf(JtsGeometry.class)); return ((JtsGeometry)shape).getGeom(); } - public static void assertMultiPolygon(Shape shape) { - assert(unwrap(shape) instanceof MultiPolygon): "expected MultiPolygon but found " + unwrap(shape).getClass().getName(); + public static void assertMultiPolygon(Object shape, boolean useJTS) { + if (useJTS) { + assert(unwrapJTS(shape) instanceof MultiPolygon) + : "expected MultiPolygon but found " + unwrapJTS(shape).getClass().getName(); + } else { + assert(shape instanceof org.apache.lucene.geo.Polygon[]) + : "expected Polygon[] but found " + shape.getClass().getName(); + } } - public static void assertPolygon(Shape shape) { - assert(unwrap(shape) instanceof Polygon): "expected Polygon but found " + unwrap(shape).getClass().getName(); + public static void assertPolygon(Object shape, boolean useJTS) { + if (useJTS) { + assert (unwrapJTS(shape) instanceof Polygon) : "expected Polygon but found " + + unwrapJTS(shape).getClass().getName(); + } else { + assert(shape instanceof org.apache.lucene.geo.Polygon) + : "expected Polygon but found " + shape.getClass().getName(); + } } - public static void assertLineString(Shape shape) { - assert(unwrap(shape) instanceof LineString): "expected LineString but found " + unwrap(shape).getClass().getName(); + public static void assertLineString(Object shape, boolean useJTS) { + if (useJTS) { + assert (unwrapJTS(shape) instanceof LineString) : "expected LineString but found " + + unwrapJTS(shape).getClass().getName(); + } else { + assert(shape instanceof org.apache.lucene.geo.Line) + : "expected Line but found " + shape.getClass().getName(); + } } - public static void assertMultiLineString(Shape shape) { - assert(unwrap(shape) instanceof MultiLineString): "expected MultiLineString but found " + unwrap(shape).getClass().getName(); + public static void assertMultiLineString(Object shape, boolean useJTS) { + if (useJTS) { + assert (unwrapJTS(shape) instanceof MultiLineString) : "expected MultiLineString but found " + + unwrapJTS(shape).getClass().getName(); + } else { + assert(shape instanceof org.apache.lucene.geo.Line[]) + : "expected Line[] but found " + shape.getClass().getName(); + } } public static void assertDistance(String geohash1, String geohash2, Matcher match) { From 067e84faa5d45ec72eb04250e4dc6183c6b427dd Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Tue, 13 Nov 2018 16:49:33 -0600 Subject: [PATCH 14/38] remove deprecated prefix tree mapping from geo-shape.asciidoc --- .../mapping/types/geo-shape.asciidoc | 23 ------------------- .../common/geo/ShapeBuilderTests.java | 1 - .../search/geo/GeoShapeQueryTests.java | 2 -- .../hamcrest/ElasticsearchGeoAssertions.java | 1 - 4 files changed, 27 deletions(-) diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index 3e515043660fe..a4d62a365186f 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -200,29 +200,6 @@ This mapping definition maps the location field to the geo_shape type using the default vector implementation. It provides approximately 1e-6 decimal degree precision. -[source,js] --------------------------------------------------- -PUT /example -{ - "mappings": { - "doc": { - "properties": { - "location": { - "type": "geo_shape", - "tree": "quadtree", - "precision": "100m" - } - } - } - } -} --------------------------------------------------- -// TEST[skip:deprecated mapping] - -This mapping maps the location field to the geo_shape type using the deprecated -quad_tree implementation and a precision of 100m. Elasticsearch translates -this into a tree_levels setting of 20. - [float] ===== Performance considerations with Prefix Trees diff --git a/server/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java b/server/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java index 27c79b2217c8b..01951caef5f68 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java @@ -36,7 +36,6 @@ import org.locationtech.spatial4j.shape.Circle; import org.locationtech.spatial4j.shape.Point; import org.locationtech.spatial4j.shape.Rectangle; -import org.locationtech.spatial4j.shape.Shape; import org.locationtech.spatial4j.shape.impl.PointImpl; import static org.elasticsearch.test.hamcrest.ElasticsearchGeoAssertions.assertMultiLineString; diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java index edf7930b9bf3b..53f9260bcb88f 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java @@ -21,7 +21,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomNumbers; import org.apache.lucene.geo.GeoTestUtil; -import org.apache.lucene.geo.Polygon; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchResponse; @@ -53,7 +52,6 @@ import org.locationtech.spatial4j.shape.Rectangle; import java.io.IOException; -import java.util.ArrayList; import java.util.Locale; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; diff --git a/server/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchGeoAssertions.java b/server/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchGeoAssertions.java index 6d156951c6e7d..6b6f7be8cba5d 100644 --- a/server/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchGeoAssertions.java +++ b/server/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchGeoAssertions.java @@ -33,7 +33,6 @@ import org.locationtech.jts.geom.MultiPoint; import org.locationtech.jts.geom.MultiPolygon; import org.locationtech.jts.geom.Polygon; -import org.locationtech.spatial4j.shape.Shape; import org.locationtech.spatial4j.shape.ShapeCollection; import org.locationtech.spatial4j.shape.impl.GeoCircle; import org.locationtech.spatial4j.shape.impl.RectangleImpl; From b38908404cb81c9460ae634c23edc38693e45195 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Mon, 19 Nov 2018 11:08:41 -0600 Subject: [PATCH 15/38] refactor GeoShapeFieldMapper into LegacyGeoShapeFieldMapper and GeoShapeFieldMapper Both classes derive from BaseGeoShapeFieldMapper that provides shared parameters: coerce, ignoreMalformed, ignore_z_value, orientation. --- .../common/geo/ShapeRelation.java | 2 +- .../common/geo/SpatialStrategy.java | 5 +- .../common/geo/parsers/GeoJsonParser.java | 24 +- .../common/geo/parsers/GeoWKTParser.java | 13 +- .../common/geo/parsers/ShapeParser.java | 4 +- .../index/mapper/BaseGeoShapeFieldMapper.java | 342 +++++++++ .../index/mapper/GeoShapeFieldMapper.java | 633 +--------------- .../mapper/LegacyGeoShapeFieldMapper.java | 593 +++++++++++++++ .../index/query/GeoShapeQueryBuilder.java | 43 +- .../elasticsearch/indices/IndicesModule.java | 4 +- .../common/geo/GeoJsonShapeParserTests.java | 32 +- .../common/geo/GeoWKTShapeParserTests.java | 17 +- .../index/mapper/ExternalMapper.java | 15 +- .../mapper/GeoShapeFieldMapperTests.java | 462 +----------- .../index/mapper/GeoShapeFieldTypeTests.java | 52 +- .../LegacyGeoShapeFieldMapperTests.java | 714 ++++++++++++++++++ .../mapper/LegacyGeoShapeFieldTypeTests.java | 86 +++ .../query/GeoShapeQueryBuilderTests.java | 98 +-- .../query/LegacyGeoShapeFieldQueryTests.java | 94 +++ .../search/geo/GeoShapeIntegrationIT.java | 20 +- .../search/geo/GeoShapeQueryTests.java | 6 +- .../geo/LegacyGeoShapeIntegrationIT.java | 170 +++++ .../test/AbstractBuilderTestCase.java | 4 +- 23 files changed, 2202 insertions(+), 1231 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/query/LegacyGeoShapeFieldQueryTests.java create mode 100644 server/src/test/java/org/elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java b/server/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java index 2156c858bbc83..e2e177c8f0fd2 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java +++ b/server/src/main/java/org/elasticsearch/common/geo/ShapeRelation.java @@ -70,7 +70,7 @@ public QueryRelation getLuceneRelation() { case DISJOINT: return QueryRelation.DISJOINT; case WITHIN: return QueryRelation.WITHIN; default: - throw new IllegalArgumentException("ShapeRelation [" + CONTAINS + "] not supported"); + throw new IllegalArgumentException("ShapeRelation [" + this + "] not supported"); } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/SpatialStrategy.java b/server/src/main/java/org/elasticsearch/common/geo/SpatialStrategy.java index fe0e8b867b5da..0b4f640fd2884 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/SpatialStrategy.java +++ b/server/src/main/java/org/elasticsearch/common/geo/SpatialStrategy.java @@ -27,8 +27,7 @@ public enum SpatialStrategy implements Writeable { TERM("term"), - RECURSIVE("recursive"), - VECTOR("vector"); + RECURSIVE("recursive"); private final String strategyName; @@ -51,7 +50,7 @@ public void writeTo(StreamOutput out) throws IOException { public static SpatialStrategy fromString(String strategyName) { for (SpatialStrategy strategy : values()) { - if (strategy.strategyName.equalsIgnoreCase(strategyName)) { + if (strategy.strategyName.equals(strategyName)) { return strategy; } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java index 45ce2b610ca0c..7d7e4b0e1f118 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java +++ b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java @@ -25,9 +25,10 @@ import org.elasticsearch.common.geo.builders.CircleBuilder; import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper; import org.locationtech.jts.geom.Coordinate; import java.io.IOException; @@ -40,17 +41,22 @@ * complies with geojson specification: https://tools.ietf.org/html/rfc7946 */ abstract class GeoJsonParser { - protected static ShapeBuilder parse(XContentParser parser, GeoShapeFieldMapper shapeMapper) + protected static ShapeBuilder parse(XContentParser parser, BaseGeoShapeFieldMapper shapeMapper) throws IOException { GeoShapeType shapeType = null; DistanceUnit.Distance radius = null; CoordinateNode coordinateNode = null; GeometryCollectionBuilder geometryCollections = null; - ShapeBuilder.Orientation requestedOrientation = - (shapeMapper == null) ? ShapeBuilder.Orientation.RIGHT : shapeMapper.fieldType().orientation(); - Explicit coerce = (shapeMapper == null) ? GeoShapeFieldMapper.Defaults.COERCE : shapeMapper.coerce(); - Explicit ignoreZValue = (shapeMapper == null) ? GeoShapeFieldMapper.Defaults.IGNORE_Z_VALUE : shapeMapper.ignoreZValue(); + Orientation orientation = (shapeMapper == null) + ? BaseGeoShapeFieldMapper.Defaults.ORIENTATION.value() + : shapeMapper.orientation().value(); + Explicit coerce = (shapeMapper == null) + ? BaseGeoShapeFieldMapper.Defaults.COERCE + : shapeMapper.coerce(); + Explicit ignoreZValue = (shapeMapper == null) + ? BaseGeoShapeFieldMapper.Defaults.IGNORE_Z_VALUE + : shapeMapper.ignoreZValue(); String malformedException = null; @@ -101,7 +107,7 @@ protected static ShapeBuilder parse(XContentParser parser, GeoShapeFieldMapper s malformedException = "cannot have [" + ShapeParser.FIELD_ORIENTATION + "] with type set to [" + shapeType + "]"; } parser.nextToken(); - requestedOrientation = ShapeBuilder.Orientation.fromString(parser.text()); + orientation = ShapeBuilder.Orientation.fromString(parser.text()); } else { parser.nextToken(); parser.skipChildren(); @@ -134,7 +140,7 @@ protected static ShapeBuilder parse(XContentParser parser, GeoShapeFieldMapper s return geometryCollections; } - return shapeType.getBuilder(coordinateNode, radius, requestedOrientation, coerce.value()); + return shapeType.getBuilder(coordinateNode, radius, orientation, coerce.value()); } /** @@ -208,7 +214,7 @@ private static Coordinate parseCoordinate(XContentParser parser, boolean ignoreZ * @return Geometry[] geometries of the GeometryCollection * @throws IOException Thrown if an error occurs while reading from the XContentParser */ - static GeometryCollectionBuilder parseGeometries(XContentParser parser, GeoShapeFieldMapper mapper) throws + static GeometryCollectionBuilder parseGeometries(XContentParser parser, BaseGeoShapeFieldMapper mapper) throws IOException { if (parser.currentToken() != XContentParser.Token.START_ARRAY) { throw new ElasticsearchParseException("geometries must be an array of geojson objects"); diff --git a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java index e1d990f0cff25..bf26980c92651 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java +++ b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java @@ -34,7 +34,7 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper; import org.locationtech.jts.geom.Coordinate; import java.io.IOException; @@ -63,7 +63,7 @@ public class GeoWKTParser { // no instance private GeoWKTParser() {} - public static ShapeBuilder parse(XContentParser parser, final GeoShapeFieldMapper shapeMapper) + public static ShapeBuilder parse(XContentParser parser, final BaseGeoShapeFieldMapper shapeMapper) throws IOException, ElasticsearchParseException { return parseExpectedType(parser, null, shapeMapper); } @@ -75,12 +75,12 @@ public static ShapeBuilder parseExpectedType(XContentParser parser, final GeoSha /** throws an exception if the parsed geometry type does not match the expected shape type */ public static ShapeBuilder parseExpectedType(XContentParser parser, final GeoShapeType shapeType, - final GeoShapeFieldMapper shapeMapper) + final BaseGeoShapeFieldMapper shapeMapper) throws IOException, ElasticsearchParseException { try (StringReader reader = new StringReader(parser.text())) { - Explicit ignoreZValue = (shapeMapper == null) ? GeoShapeFieldMapper.Defaults.IGNORE_Z_VALUE : + Explicit ignoreZValue = (shapeMapper == null) ? BaseGeoShapeFieldMapper.Defaults.IGNORE_Z_VALUE : shapeMapper.ignoreZValue(); - Explicit coerce = (shapeMapper == null) ? GeoShapeFieldMapper.Defaults.COERCE : shapeMapper.coerce(); + Explicit coerce = (shapeMapper == null) ? BaseGeoShapeFieldMapper.Defaults.COERCE : shapeMapper.coerce(); // setup the tokenizer; configured to read words w/o numbers StreamTokenizer tokenizer = new StreamTokenizer(reader); tokenizer.resetSyntax(); @@ -257,7 +257,8 @@ private static PolygonBuilder parsePolygon(StreamTokenizer stream, final boolean if (nextEmptyOrOpen(stream).equals(EMPTY)) { return null; } - PolygonBuilder builder = new PolygonBuilder(parseLinearRing(stream, ignoreZValue, coerce), ShapeBuilder.Orientation.RIGHT); + PolygonBuilder builder = new PolygonBuilder(parseLinearRing(stream, ignoreZValue, coerce), + BaseGeoShapeFieldMapper.Defaults.ORIENTATION.value()); while (nextCloserOrComma(stream).equals(COMMA)) { builder.hole(parseLinearRing(stream, ignoreZValue, coerce)); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java b/server/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java index 79582c3365bdb..21d1bd9f25564 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java +++ b/server/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java @@ -23,7 +23,7 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper; import java.io.IOException; @@ -46,7 +46,7 @@ public interface ShapeParser { * if the parsers current token has been null * @throws IOException if the input could not be read */ - static ShapeBuilder parse(XContentParser parser, GeoShapeFieldMapper shapeMapper) throws IOException { + static ShapeBuilder parse(XContentParser parser, BaseGeoShapeFieldMapper shapeMapper) throws IOException { if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { return null; } if (parser.currentToken() == XContentParser.Token.START_OBJECT) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java new file mode 100644 index 0000000000000..57d3aead021c1 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java @@ -0,0 +1,342 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.elasticsearch.Version; +import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper.DeprecatedParameters; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.QueryShardException; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_MALFORMED; + +/** + * Base class for {@link GeoShapeFieldMapper} and {@link LegacyGeoShapeFieldMapper} + */ +public abstract class BaseGeoShapeFieldMapper extends FieldMapper { + public static final String CONTENT_TYPE = "geo_shape"; + + public static class Names { + public static final ParseField ORIENTATION = new ParseField("orientation"); + public static final ParseField COERCE = new ParseField("coerce"); + } + + public static class Defaults { + public static final Explicit ORIENTATION = new Explicit<>(Orientation.RIGHT, false); + public static final Explicit COERCE = new Explicit<>(false, false); + public static final Explicit IGNORE_MALFORMED = new Explicit<>(false, false); + public static final Explicit IGNORE_Z_VALUE = new Explicit<>(true, false); + } + + public abstract static class Builder + extends FieldMapper.Builder { + protected Boolean coerce; + protected Boolean ignoreMalformed; + protected Boolean ignoreZValue; + protected Orientation orientation; + + /** default builder - used for external mapper*/ + public Builder(String name, MappedFieldType fieldType, MappedFieldType defaultFieldType) { + super(name, fieldType, defaultFieldType); + } + + public Builder(String name, MappedFieldType fieldType, MappedFieldType defaultFieldType, + boolean coerce, boolean ignoreMalformed, Orientation orientation, boolean ignoreZ) { + super(name, fieldType, defaultFieldType); + this.coerce = coerce; + this.ignoreMalformed = ignoreMalformed; + this.orientation = orientation; + this.ignoreZValue = ignoreZ; + } + + public Builder coerce(boolean coerce) { + this.coerce = coerce; + return this; + } + + protected Explicit coerce(BuilderContext context) { + if (coerce != null) { + return new Explicit<>(coerce, true); + } + if (context.indexSettings() != null) { + return new Explicit<>(COERCE_SETTING.get(context.indexSettings()), false); + } + return Defaults.COERCE; + } + + public Builder orientation(Orientation orientation) { + this.orientation = orientation; + return this; + } + + protected Explicit orientation() { + if (orientation != null) { + return new Explicit<>(orientation, true); + } + return Defaults.ORIENTATION; + } + + @Override + protected boolean defaultDocValues(Version indexCreated) { + return false; + } + + public Builder ignoreMalformed(boolean ignoreMalformed) { + this.ignoreMalformed = ignoreMalformed; + return this; + } + + protected Explicit ignoreMalformed(BuilderContext context) { + if (ignoreMalformed != null) { + return new Explicit<>(ignoreMalformed, true); + } + if (context.indexSettings() != null) { + return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false); + } + return Defaults.IGNORE_MALFORMED; + } + + protected Explicit ignoreZValue() { + if (ignoreZValue != null) { + return new Explicit<>(ignoreZValue, true); + } + return Defaults.IGNORE_Z_VALUE; + } + + public Builder ignoreZValue(final boolean ignoreZValue) { + this.ignoreZValue = ignoreZValue; + return this; + } + + @Override + protected void setupFieldType(BuilderContext context) { + super.setupFieldType(context); + + // field mapper handles this at build time + // but prefix tree strategies require a name, so throw a similar exception + if (name().isEmpty()) { + throw new IllegalArgumentException("name cannot be empty string"); + } + + BaseGeoShapeFieldType ft = (BaseGeoShapeFieldType)fieldType(); + ft.setOrientation(orientation); + } + } + + public static class TypeParser implements Mapper.TypeParser { + + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + boolean coerce = Defaults.COERCE.value(); + boolean ignoreZ = Defaults.IGNORE_Z_VALUE.value(); + boolean ignoreMalformed = Defaults.IGNORE_MALFORMED.value(); + Orientation orientation = Defaults.ORIENTATION.value(); + DeprecatedParameters deprecatedParameters = new DeprecatedParameters(); + boolean parsedDeprecatedParams = false; + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String fieldName = entry.getKey(); + Object fieldNode = entry.getValue(); + if (DeprecatedParameters.parse(name, fieldName, fieldNode, deprecatedParameters)) { + parsedDeprecatedParams = true; + iterator.remove(); + } else if (Names.ORIENTATION.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { + orientation = ShapeBuilder.Orientation.fromString(fieldNode.toString()); + iterator.remove(); + } else if (IGNORE_MALFORMED.equals(fieldName)) { + ignoreMalformed = XContentMapValues.nodeBooleanValue(fieldNode, name + ".ignore_malformed"); + iterator.remove(); + } else if (Names.COERCE.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { + coerce = XContentMapValues.nodeBooleanValue(fieldNode, name + "." + Names.COERCE.getPreferredName()); + iterator.remove(); + } else if (GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName().equals(fieldName)) { + ignoreZ = XContentMapValues.nodeBooleanValue(fieldNode, + name + "." + GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName()); + iterator.remove(); + } + } + return getBuilder(name, coerce, ignoreMalformed, orientation, ignoreZ, parsedDeprecatedParams ? deprecatedParameters : null); + } + + private Builder getBuilder(String name, boolean coerce, boolean ignoreMalformed, Orientation orientation, + boolean ignoreZ, DeprecatedParameters deprecatedParameters) { + if (deprecatedParameters != null) { + return getLegacyBuilder(name, coerce, ignoreMalformed, orientation, ignoreZ, deprecatedParameters); + } + return new GeoShapeFieldMapper.Builder(name, coerce, ignoreMalformed, orientation, ignoreZ); + } + + private Builder getLegacyBuilder(String name, boolean coerce, boolean ignoreMalformed, Orientation orientation, + boolean ignoreZ, DeprecatedParameters deprecatedParameters) { + deprecatedParameters.setup(); + return new LegacyGeoShapeFieldMapper.Builder(name, coerce, ignoreMalformed, orientation, ignoreZ, deprecatedParameters); + } + } + + public abstract static class BaseGeoShapeFieldType extends MappedFieldType { + protected Orientation orientation = Defaults.ORIENTATION.value(); + + protected BaseGeoShapeFieldType() { + setIndexOptions(IndexOptions.DOCS); + setTokenized(false); + setStored(false); + setStoreTermVectors(false); + setOmitNorms(true); + } + + protected BaseGeoShapeFieldType(BaseGeoShapeFieldType ref) { + super(ref); + this.orientation = ref.orientation; + } + + @Override + public boolean equals(Object o) { + if (!super.equals(o)) return false; + BaseGeoShapeFieldType that = (BaseGeoShapeFieldType) o; + return orientation == that.orientation; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), orientation); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public void checkCompatibility(MappedFieldType fieldType, List conflicts) { + super.checkCompatibility(fieldType, conflicts); + } + + public Orientation orientation() { return this.orientation; } + + public void setOrientation(Orientation orientation) { + checkIfFrozen(); + this.orientation = orientation; + } + + @Override + public Query existsQuery(QueryShardContext context) { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead"); + } + } + + protected Explicit coerce; + protected Explicit ignoreMalformed; + protected Explicit ignoreZValue; + protected Explicit orientation; + + protected BaseGeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, + Explicit ignoreMalformed, Explicit coerce, Explicit orientation, + Explicit ignoreZValue, Settings indexSettings, + MultiFields multiFields, CopyTo copyTo) { + super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); + this.coerce = coerce; + this.ignoreMalformed = ignoreMalformed; + this.ignoreZValue = ignoreZValue; + this.orientation = orientation; + } + + @Override + protected void doMerge(Mapper mergeWith) { + super.doMerge(mergeWith); + BaseGeoShapeFieldMapper gsfm = (BaseGeoShapeFieldMapper)mergeWith; + if (gsfm.coerce.explicit()) { + this.coerce = gsfm.coerce; + } + if (gsfm.ignoreMalformed.explicit()) { + this.ignoreMalformed = gsfm.ignoreMalformed; + } + if (gsfm.ignoreZValue.explicit()) { + this.ignoreZValue = gsfm.ignoreZValue; + } + if (gsfm.orientation.explicit()) { + this.orientation = gsfm.orientation; + } + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + builder.field("type", contentType()); + + if (includeDefaults || orientation.explicit()) { + builder.field(Names.ORIENTATION.getPreferredName(), orientation.value()); + } + if (includeDefaults || coerce.explicit()) { + builder.field(Names.COERCE.getPreferredName(), coerce.value()); + } + if (includeDefaults || ignoreMalformed.explicit()) { + builder.field(IGNORE_MALFORMED, ignoreMalformed.value()); + } + if (includeDefaults || ignoreZValue.explicit()) { + builder.field(GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName(), ignoreZValue.value()); + } + } + + public Explicit coerce() { + return coerce; + } + + public Explicit ignoreMalformed() { + return ignoreMalformed; + } + + public Explicit ignoreZValue() { + return ignoreZValue; + } + + public Explicit orientation() { + return orientation; + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 1b7f1bdb7d3d9..1ba9aa2f59cc2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -18,59 +18,24 @@ */ package org.elasticsearch.index.mapper; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.lucene.document.Field; import org.apache.lucene.document.LatLonShape; import org.apache.lucene.geo.Line; import org.apache.lucene.geo.Polygon; import org.apache.lucene.geo.Rectangle; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; -import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree; -import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; -import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.geo.GeoUtils; -import org.elasticsearch.common.geo.ShapesAvailability; -import org.elasticsearch.common.geo.SpatialStrategy; -import org.elasticsearch.common.geo.XShapeCollection; import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; import org.elasticsearch.common.geo.parsers.ShapeParser; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.DistanceUnit; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.query.QueryShardException; -import org.locationtech.spatial4j.shape.Point; -import org.locationtech.spatial4j.shape.Shape; -import org.locationtech.spatial4j.shape.jts.JtsGeometry; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_MALFORMED; /** - * FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s. + * FieldMapper for indexing {@link org.apache.lucene.document.LatLonShape}s. *

* Currently Shapes can only be indexed and can only be queried using * {@link org.elasticsearch.index.query.GeoShapeQueryBuilder}, consequently @@ -84,468 +49,52 @@ * [ [100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0] ] * ] * } + *

+ * or: + *

+ * "field" : "POLYGON ((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0)) */ -public class GeoShapeFieldMapper extends FieldMapper { - - public static final String CONTENT_TYPE = "geo_shape"; - - public static class Names { - public static final String TREE = "tree"; - public static final String TREE_GEOHASH = "geohash"; - public static final String TREE_QUADTREE = "quadtree"; - public static final String TREE_LEVELS = "tree_levels"; - public static final String TREE_PRESISION = "precision"; - public static final String DISTANCE_ERROR_PCT = "distance_error_pct"; - public static final String ORIENTATION = "orientation"; - public static final String STRATEGY = "strategy"; - public static final String STRATEGY_POINTS_ONLY = "points_only"; - public static final String COERCE = "coerce"; - } - - public static class Defaults { - public static final String TREE = "NONE"; - public static final boolean POINTS_ONLY = false; - public static final int GEOHASH_LEVELS = GeoUtils.geoHashLevelsForPrecision("50m"); - public static final int QUADTREE_LEVELS = GeoUtils.quadTreeLevelsForPrecision("50m"); - public static final Orientation ORIENTATION = Orientation.RIGHT; - public static final double LEGACY_DISTANCE_ERROR_PCT = 0.025d; - public static final Explicit STRATEGY = new Explicit<>(SpatialStrategy.VECTOR, false); - public static final Explicit COERCE = new Explicit<>(false, false); - public static final Explicit IGNORE_MALFORMED = new Explicit<>(false, false); - public static final Explicit IGNORE_Z_VALUE = new Explicit<>(true, false); - } - - private static final Logger logger = LogManager.getLogger(GeoShapeFieldMapper.class); - private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger); - - public static class Builder extends FieldMapper.Builder { - - private Boolean coerce; - private Boolean ignoreMalformed; - private Boolean ignoreZValue; +public class GeoShapeFieldMapper extends BaseGeoShapeFieldMapper { + public static class Builder extends BaseGeoShapeFieldMapper.Builder { public Builder(String name) { - super(name, new GeoShapeFieldType(), new GeoShapeFieldType()); - } - - @Override - public GeoShapeFieldType fieldType() { - return (GeoShapeFieldType)fieldType; + super (name, new GeoShapeFieldType(), new GeoShapeFieldType()); } - public Builder coerce(boolean coerce) { - this.coerce = coerce; - return this; - } - - @Override - protected boolean defaultDocValues(Version indexCreated) { - return false; - } - - protected Explicit coerce(BuilderContext context) { - if (coerce != null) { - return new Explicit<>(coerce, true); - } - if (context.indexSettings() != null) { - return new Explicit<>(COERCE_SETTING.get(context.indexSettings()), false); - } - return Defaults.COERCE; - } - - public Builder ignoreMalformed(boolean ignoreMalformed) { - this.ignoreMalformed = ignoreMalformed; - return this; - } - - protected Explicit ignoreMalformed(BuilderContext context) { - if (ignoreMalformed != null) { - return new Explicit<>(ignoreMalformed, true); - } - if (context.indexSettings() != null) { - return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false); - } - return Defaults.IGNORE_MALFORMED; - } - - protected Explicit ignoreZValue(BuilderContext context) { - if (ignoreZValue != null) { - return new Explicit<>(ignoreZValue, true); - } - return Defaults.IGNORE_Z_VALUE; - } - - public Builder ignoreZValue(final boolean ignoreZValue) { - this.ignoreZValue = ignoreZValue; - return this; - } - - private void setupPrefixTrees() { - SpatialPrefixTree prefixTree; - String tree = fieldType().tree().equals("NONE") ? "quadtree" : fieldType().tree(); - int treeLevels = fieldType().treeLevels(); - double precisionInMeters = fieldType().precisionInMeters(); - if ("geohash".equals(tree)) { - prefixTree = new GeohashPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, - getLevels(treeLevels, precisionInMeters, Defaults.GEOHASH_LEVELS, true)); - } else if ("legacyquadtree".equals(tree)) { - prefixTree = new QuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, - getLevels(treeLevels, precisionInMeters, Defaults.QUADTREE_LEVELS, false)); - } else if ("quadtree".equals(tree)) { - prefixTree = new PackedQuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, - getLevels(treeLevels, precisionInMeters, Defaults.QUADTREE_LEVELS, false)); - } else { - throw new IllegalArgumentException("Unknown prefix tree type [" + tree + "]"); - } - - RecursivePrefixTreeStrategy rpts = new RecursivePrefixTreeStrategy(prefixTree, name()); - rpts.setDistErrPct(fieldType().distanceErrorPct()); - rpts.setPruneLeafyBranches(false); - fieldType().recursiveStrategy = rpts; - - TermQueryPrefixTreeStrategy termStrategy = new TermQueryPrefixTreeStrategy(prefixTree, name()); - termStrategy.setDistErrPct(fieldType().distanceErrorPct()); - fieldType().termStrategy = termStrategy; - } - - @Override - protected void setupFieldType(BuilderContext context) { - super.setupFieldType(context); - - // field mapper handles this at build time - // but prefix tree strategies require a name, so throw a similar exception - if (name().isEmpty()) { - throw new IllegalArgumentException("name cannot be empty string"); - } - - // throw an exception if spatial strategy is set to vector and a "tree" was explicitly set - if (fieldType().strategy() == SpatialStrategy.VECTOR && fieldType().tree().equals("NONE") == false) { - throw new IllegalArgumentException("prefix trees cannot be used when index strategy is set to [" - + SpatialStrategy.VECTOR + "]"); - } - - // setup prefix trees regardless of strategy (this is used for the QueryBuilder) - setupPrefixTrees(); - - // set the default PrefixTree Strategy if the spatial strategy is not set to "VECTOR" - if (fieldType().strategy() != SpatialStrategy.VECTOR) { - fieldType().defaultPrefixTreeStrategy = fieldType().resolvePrefixTreeStrategy(fieldType().strategy()); - fieldType().defaultPrefixTreeStrategy.setPointsOnly(fieldType().pointsOnly()); - } - } - - private static int getLevels(int treeLevels, double precisionInMeters, int defaultLevels, boolean geoHash) { - if (treeLevels > 0 || precisionInMeters >= 0) { - return Math.max(treeLevels, precisionInMeters >= 0 ? (geoHash ? GeoUtils.geoHashLevelsForPrecision(precisionInMeters) - : GeoUtils.quadTreeLevelsForPrecision(precisionInMeters)) : 0); - } - return defaultLevels; + public Builder(String name, boolean coerce, boolean ignoreMalformed, ShapeBuilder.Orientation orientation, + boolean ignoreZ) { + super(name, new GeoShapeFieldType(), new GeoShapeFieldType(), coerce, ignoreMalformed, orientation, ignoreZ); } @Override public GeoShapeFieldMapper build(BuilderContext context) { - GeoShapeFieldType geoShapeFieldType = (GeoShapeFieldType)fieldType; - - if (geoShapeFieldType.treeLevels() == 0 && geoShapeFieldType.precisionInMeters() < 0) { - geoShapeFieldType.setDefaultDistanceErrorPct(Defaults.LEGACY_DISTANCE_ERROR_PCT); - } setupFieldType(context); - return new GeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), coerce(context), - ignoreZValue(context), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); - } - } - - public static class TypeParser implements Mapper.TypeParser { - - @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - Builder builder = new Builder(name); - Boolean pointsOnly = null; - for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - String fieldName = entry.getKey(); - Object fieldNode = entry.getValue(); - if (Names.TREE.equals(fieldName)) { - checkPrefixTreeSupport(fieldName); - builder.fieldType().setTree(fieldNode.toString()); - // explicitly set strategy - if (builder.fieldType().strategy() == Defaults.STRATEGY.value()) { - builder.fieldType().setStrategy(SpatialStrategy.RECURSIVE); - } - iterator.remove(); - } else if (Names.TREE_LEVELS.equals(fieldName)) { - checkPrefixTreeSupport(fieldName); - builder.fieldType().setTreeLevels(Integer.parseInt(fieldNode.toString())); - iterator.remove(); - } else if (Names.TREE_PRESISION.equals(fieldName)) { - checkPrefixTreeSupport(fieldName); - builder.fieldType().setPrecisionInMeters(DistanceUnit.parse(fieldNode.toString(), - DistanceUnit.DEFAULT, DistanceUnit.DEFAULT)); - iterator.remove(); - } else if (Names.DISTANCE_ERROR_PCT.equals(fieldName)) { - checkPrefixTreeSupport(fieldName); - builder.fieldType().setDistanceErrorPct(Double.parseDouble(fieldNode.toString())); - iterator.remove(); - } else if (Names.ORIENTATION.equals(fieldName)) { - builder.fieldType().setOrientation(ShapeBuilder.Orientation.fromString(fieldNode.toString())); - iterator.remove(); - } else if (Names.STRATEGY.equals(fieldName)) { - SpatialStrategy strategy = SpatialStrategy.fromString(fieldNode.toString()); - String prefixTree = builder.fieldType().tree(); - if (strategy == SpatialStrategy.VECTOR && prefixTree.equals("NONE") == false) { - throw new ElasticsearchParseException("Strategy [{}] cannot be used with PrefixTree [{}]", strategy, prefixTree); - } else if (strategy != SpatialStrategy.VECTOR) { - checkPrefixTreeSupport(fieldName); - } - builder.fieldType().setStrategy(strategy); - iterator.remove(); - } else if (IGNORE_MALFORMED.equals(fieldName)) { - builder.ignoreMalformed(XContentMapValues.nodeBooleanValue(fieldNode, name + ".ignore_malformed")); - iterator.remove(); - } else if (Names.COERCE.equals(fieldName)) { - builder.coerce(XContentMapValues.nodeBooleanValue(fieldNode, name + "." + Names.COERCE)); - iterator.remove(); - } else if (GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName().equals(fieldName)) { - builder.ignoreZValue(XContentMapValues.nodeBooleanValue(fieldNode, - name + "." + GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName())); - iterator.remove(); - } else if (Names.STRATEGY_POINTS_ONLY.equals(fieldName)) { - checkPrefixTreeSupport(fieldName); - pointsOnly = XContentMapValues.nodeBooleanValue(fieldNode, name + "." + Names.STRATEGY_POINTS_ONLY); - iterator.remove(); - } - } - if (pointsOnly != null) { - if (builder.fieldType().strategy == SpatialStrategy.TERM && pointsOnly == false) { - throw new IllegalArgumentException("points_only cannot be set to false for term strategy"); - } else { - builder.fieldType().setPointsOnly(pointsOnly); - } - } - return builder; - } - - private void checkPrefixTreeSupport(String fieldName) { - if (ShapesAvailability.JTS_AVAILABLE == false || ShapesAvailability.SPATIAL4J_AVAILABLE == false) { - throw new ElasticsearchParseException("Field parameter [{}] is not supported for [{}] field type", fieldName, CONTENT_TYPE); - } - DEPRECATION_LOGGER.deprecated("Field parameter [{}] is deprecated and will be removed in a future version. " - + "use [" + SpatialStrategy.VECTOR.getStrategyName() + "] indexing strategy instead.", fieldName); + orientation(), ignoreZValue(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); } } - public static final class GeoShapeFieldType extends MappedFieldType { - - private String tree = Defaults.TREE; - private SpatialStrategy strategy = Defaults.STRATEGY.value(); - private boolean pointsOnly = Defaults.POINTS_ONLY; - private int treeLevels = 0; - private double precisionInMeters = -1; - private Double distanceErrorPct; - private double defaultDistanceErrorPct = 0.0; - private Orientation orientation = Defaults.ORIENTATION; - - // these are built when the field type is frozen - private PrefixTreeStrategy defaultPrefixTreeStrategy; - private RecursivePrefixTreeStrategy recursiveStrategy; - private TermQueryPrefixTreeStrategy termStrategy; - + public static final class GeoShapeFieldType extends BaseGeoShapeFieldType { public GeoShapeFieldType() { - setIndexOptions(IndexOptions.DOCS); - setTokenized(false); - setStored(false); - setStoreTermVectors(false); - setOmitNorms(true); + super(); } protected GeoShapeFieldType(GeoShapeFieldType ref) { super(ref); - this.tree = ref.tree; - this.strategy = ref.strategy; - this.pointsOnly = ref.pointsOnly; - this.treeLevels = ref.treeLevels; - this.precisionInMeters = ref.precisionInMeters; - this.distanceErrorPct = ref.distanceErrorPct; - this.defaultDistanceErrorPct = ref.defaultDistanceErrorPct; - this.orientation = ref.orientation; } @Override public GeoShapeFieldType clone() { return new GeoShapeFieldType(this); } - - @Override - public boolean equals(Object o) { - if (!super.equals(o)) return false; - GeoShapeFieldType that = (GeoShapeFieldType) o; - return treeLevels == that.treeLevels && - precisionInMeters == that.precisionInMeters && - defaultDistanceErrorPct == that.defaultDistanceErrorPct && - Objects.equals(tree, that.tree) && - Objects.equals(strategy, that.strategy) && - pointsOnly == that.pointsOnly && - Objects.equals(distanceErrorPct, that.distanceErrorPct) && - orientation == that.orientation; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), tree, strategy, pointsOnly, treeLevels, precisionInMeters, distanceErrorPct, - defaultDistanceErrorPct, orientation); - } - - @Override - public String typeName() { - return CONTENT_TYPE; - } - - @Override - public void checkCompatibility(MappedFieldType fieldType, List conflicts) { - super.checkCompatibility(fieldType, conflicts); - GeoShapeFieldType other = (GeoShapeFieldType)fieldType; - // prevent user from changing strategies - if (strategy() != other.strategy()) { - conflicts.add("mapper [" + name() + "] has different [strategy]"); - } - - // prevent user from changing trees (changes encoding) - if (tree().equals(other.tree()) == false) { - conflicts.add("mapper [" + name() + "] has different [tree]"); - } - - if ((pointsOnly() != other.pointsOnly())) { - conflicts.add("mapper [" + name() + "] has different points_only"); - } - - // TODO we should allow this, but at the moment levels is used to build bookkeeping variables - // in lucene's SpatialPrefixTree implementations, need a patch to correct that first - if (treeLevels() != other.treeLevels()) { - conflicts.add("mapper [" + name() + "] has different [tree_levels]"); - } - if (precisionInMeters() != other.precisionInMeters()) { - conflicts.add("mapper [" + name() + "] has different [precision]"); - } - } - - private static int getLevels(int treeLevels, double precisionInMeters, int defaultLevels, boolean geoHash) { - if (treeLevels > 0 || precisionInMeters >= 0) { - return Math.max(treeLevels, precisionInMeters >= 0 ? (geoHash ? GeoUtils.geoHashLevelsForPrecision(precisionInMeters) - : GeoUtils.quadTreeLevelsForPrecision(precisionInMeters)) : 0); - } - return defaultLevels; - } - - public String tree() { - return tree; - } - - public void setTree(String tree) { - checkIfFrozen(); - this.tree = tree; - } - - public SpatialStrategy strategy() { - return strategy; - } - - public void setStrategy(SpatialStrategy strategy) { - checkIfFrozen(); - this.strategy = strategy; - if (this.strategy.equals(SpatialStrategy.TERM)) { - this.pointsOnly = true; - } - } - - public boolean pointsOnly() { - return pointsOnly; - } - - public void setPointsOnly(boolean pointsOnly) { - checkIfFrozen(); - this.pointsOnly = pointsOnly; - } - public int treeLevels() { - return treeLevels; - } - - public void setTreeLevels(int treeLevels) { - checkIfFrozen(); - this.treeLevels = treeLevels; - } - - public double precisionInMeters() { - return precisionInMeters; - } - - public void setPrecisionInMeters(double precisionInMeters) { - checkIfFrozen(); - this.precisionInMeters = precisionInMeters; - } - - public double distanceErrorPct() { - return distanceErrorPct == null ? defaultDistanceErrorPct : distanceErrorPct; - } - - public void setDistanceErrorPct(double distanceErrorPct) { - checkIfFrozen(); - this.distanceErrorPct = distanceErrorPct; - } - - public void setDefaultDistanceErrorPct(double defaultDistanceErrorPct) { - checkIfFrozen(); - this.defaultDistanceErrorPct = defaultDistanceErrorPct; - } - - public Orientation orientation() { return this.orientation; } - - public void setOrientation(Orientation orientation) { - checkIfFrozen(); - this.orientation = orientation; - } - - public PrefixTreeStrategy defaultPrefixTreeStrategy() { - return this.defaultPrefixTreeStrategy; - } - - public PrefixTreeStrategy resolvePrefixTreeStrategy(SpatialStrategy strategy) { - return resolvePrefixTreeStrategy(strategy.getStrategyName()); - } - - public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) { - if (SpatialStrategy.RECURSIVE.getStrategyName().equals(strategyName)) { - return recursiveStrategy; - } - if (SpatialStrategy.TERM.getStrategyName().equals(strategyName)) { - return termStrategy; - } - throw new IllegalArgumentException("Unknown prefix tree strategy [" + strategyName + "]"); - } - - @Override - public Query existsQuery(QueryShardContext context) { - return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); - } - - @Override - public Query termQuery(Object value, QueryShardContext context) { - throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead"); - } } - protected Explicit coerce; - protected Explicit ignoreMalformed; - protected Explicit ignoreZValue; - public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, - Explicit ignoreMalformed, Explicit coerce, + Explicit ignoreMalformed, Explicit coerce, Explicit orientation, Explicit ignoreZValue, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); - this.coerce = coerce; - this.ignoreMalformed = ignoreMalformed; - this.ignoreZValue = ignoreZValue; + super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, orientation, ignoreZValue, indexSettings, + multiFields, copyTo); } @Override @@ -553,43 +102,9 @@ public GeoShapeFieldType fieldType() { return (GeoShapeFieldType) super.fieldType(); } - public void parseForInvertedIndexing(ParseContext context) throws IOException { - try { - Shape shape = context.parseExternalValue(Shape.class); - if (shape == null) { - ShapeBuilder shapeBuilder = ShapeParser.parse(context.parser(), this); - if (shapeBuilder == null) { - return; - } - shape = shapeBuilder.buildS4J(); - } - if (fieldType().pointsOnly() == true) { - // index configured for pointsOnly - if (shape instanceof XShapeCollection && XShapeCollection.class.cast(shape).pointsOnly()) { - // MULTIPOINT data: index each point separately - List shapes = ((XShapeCollection) shape).getShapes(); - for (Shape s : shapes) { - indexShape(context, s); - } - return; - } else if (shape instanceof Point == false) { - throw new MapperParsingException("[{" + fieldType().name() + "}] is configured for points only but a " - + ((shape instanceof JtsGeometry) ? ((JtsGeometry)shape).getGeom().getGeometryType() : shape.getClass()) - + " was found"); - } - } - indexShape(context, shape); - } catch (Exception e) { - if (ignoreMalformed.value() == false) { - throw new MapperParsingException("failed to parse field [{}] of type [{}]", e, fieldType().name(), - fieldType().typeName()); - } - context.addIgnoredField(fieldType.name()); - } - } - /** parsing logic for {@link LatLonShape} indexing */ - public void parseForPointsIndexing(ParseContext context) throws IOException { + @Override + public void parse(ParseContext context) throws IOException { try { Object shape = context.parseExternalValue(Object.class); if (shape == null) { @@ -609,23 +124,6 @@ public void parseForPointsIndexing(ParseContext context) throws IOException { } } - @Override - public void parse(ParseContext context) throws IOException { - if (fieldType().strategy() == SpatialStrategy.VECTOR) { - parseForPointsIndexing(context); - } else { - parseForInvertedIndexing(context); - } - } - - private void indexShape(ParseContext context, Shape shape) { - List fields = new ArrayList<>(Arrays.asList(fieldType().defaultPrefixTreeStrategy().createIndexableFields(shape))); - createFieldNamesField(context, fields); - for (IndexableField field : fields) { - context.doc().add(field); - } - } - private void indexShape(ParseContext context, Object luceneShape) { if (luceneShape instanceof GeoPoint) { GeoPoint pt = (GeoPoint)luceneShape; @@ -672,97 +170,4 @@ private void indexFields(ParseContext context, Field[] fields) { context.doc().add(f); } } - - @Override - protected void parseCreateField(ParseContext context, List fields) throws IOException { - } - - @Override - protected void doMerge(Mapper mergeWith) { - super.doMerge(mergeWith); - - GeoShapeFieldMapper gsfm = (GeoShapeFieldMapper)mergeWith; - if (gsfm.coerce.explicit()) { - this.coerce = gsfm.coerce; - } - if (gsfm.ignoreMalformed.explicit()) { - this.ignoreMalformed = gsfm.ignoreMalformed; - } - if (gsfm.ignoreZValue.explicit()) { - this.ignoreZValue = gsfm.ignoreZValue; - } - } - - @Override - protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { - builder.field("type", contentType()); - - if (includeDefaults || fieldType().tree().equals(Defaults.TREE) == false) { - builder.field(Names.TREE, fieldType().tree()); - } - - if (fieldType().treeLevels() != 0) { - builder.field(Names.TREE_LEVELS, fieldType().treeLevels()); - } else if(includeDefaults && fieldType().precisionInMeters() == -1) { // defaults only make sense if precision is not specified - if ("geohash".equals(fieldType().tree())) { - builder.field(Names.TREE_LEVELS, Defaults.GEOHASH_LEVELS); - } else if ("legacyquadtree".equals(fieldType().tree())) { - builder.field(Names.TREE_LEVELS, Defaults.QUADTREE_LEVELS); - } else if ("quadtree".equals(fieldType().tree())) { - builder.field(Names.TREE_LEVELS, Defaults.QUADTREE_LEVELS); - } else { - throw new IllegalArgumentException("Unknown prefix tree type [" + fieldType().tree() + "]"); - } - } - if (fieldType().precisionInMeters() != -1) { - builder.field(Names.TREE_PRESISION, DistanceUnit.METERS.toString(fieldType().precisionInMeters())); - } else if (includeDefaults && fieldType().treeLevels() == 0) { // defaults only make sense if tree levels are not specified - builder.field(Names.TREE_PRESISION, DistanceUnit.METERS.toString(50)); - } - if (includeDefaults || fieldType().strategy() != Defaults.STRATEGY.value()) { - builder.field(Names.STRATEGY, fieldType().strategy().getStrategyName()); - } - if (includeDefaults || fieldType().distanceErrorPct() != fieldType().defaultDistanceErrorPct) { - builder.field(Names.DISTANCE_ERROR_PCT, fieldType().distanceErrorPct()); - } - if (includeDefaults || fieldType().orientation() != Defaults.ORIENTATION) { - builder.field(Names.ORIENTATION, fieldType().orientation()); - } - if (fieldType().strategy() == SpatialStrategy.TERM) { - // For TERMs strategy the defaults for points only change to true - if (includeDefaults || fieldType().pointsOnly() != true) { - builder.field(Names.STRATEGY_POINTS_ONLY, fieldType().pointsOnly()); - } - } else { - if (includeDefaults || fieldType().pointsOnly() != GeoShapeFieldMapper.Defaults.POINTS_ONLY) { - builder.field(Names.STRATEGY_POINTS_ONLY, fieldType().pointsOnly()); - } - } - if (includeDefaults || coerce.explicit()) { - builder.field(Names.COERCE, coerce.value()); - } - if (includeDefaults || ignoreMalformed.explicit()) { - builder.field(IGNORE_MALFORMED, ignoreMalformed.value()); - } - if (includeDefaults || ignoreZValue.explicit()) { - builder.field(GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName(), ignoreZValue.value()); - } - } - - public Explicit coerce() { - return coerce; - } - - public Explicit ignoreMalformed() { - return ignoreMalformed; - } - - public Explicit ignoreZValue() { - return ignoreZValue; - } - - @Override - protected String contentType() { - return CONTENT_TYPE; - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java new file mode 100644 index 0000000000000..ee870a88aa8dd --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -0,0 +1,593 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; +import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree; +import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; +import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.geo.ShapesAvailability; +import org.elasticsearch.common.geo.SpatialStrategy; +import org.elasticsearch.common.geo.XShapeCollection; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; +import org.elasticsearch.common.geo.parsers.ShapeParser; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.DistanceUnit; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.jts.JtsGeometry; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s. + *

+ * Currently Shapes can only be indexed and can only be queried using + * {@link org.elasticsearch.index.query.GeoShapeQueryBuilder}, consequently + * a lot of behavior in this Mapper is disabled. + *

+ * Format supported: + *

+ * "field" : { + * "type" : "polygon", + * "coordinates" : [ + * [ [100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0] ] + * ] + * } + *

+ * or: + *

+ * "field" : "POLYGON ((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0)) + * + * @deprecated use {@link GeoShapeFieldMapper} + */ +@Deprecated +public class LegacyGeoShapeFieldMapper extends BaseGeoShapeFieldMapper { + + public static final String CONTENT_TYPE = "geo_shape"; + + @Deprecated + public static class DeprecatedParameters { + public static class Names { + public static final ParseField STRATEGY = new ParseField("strategy"); + public static final ParseField TREE = new ParseField("tree"); + public static final ParseField TREE_LEVELS = new ParseField("tree_levels"); + public static final ParseField PRECISION = new ParseField("precision"); + public static final ParseField DISTANCE_ERROR_PCT = new ParseField("distance_error_pct"); + public static final ParseField POINTS_ONLY = new ParseField("points_only"); + } + + public static class PrefixTrees { + public static final String LEGACY_QUADTREE = "legacyquadtree"; + public static final String QUADTREE = "quadtree"; + public static final String GEOHASH = "geohash"; + } + + public static class Defaults { + public static final SpatialStrategy STRATEGY = SpatialStrategy.RECURSIVE; + public static final String TREE = "quadtree"; + public static final String PRECISION = "50m"; + public static final int QUADTREE_LEVELS = GeoUtils.quadTreeLevelsForPrecision(PRECISION); + public static final int GEOHASH_TREE_LEVELS = GeoUtils.geoHashLevelsForPrecision(PRECISION); + public static final boolean POINTS_ONLY = false; + public static final double DISTANCE_ERROR_PCT = 0.025d; + } + + public SpatialStrategy strategy = null; + public String tree = null; + public int treeLevels = Integer.MIN_VALUE; + public String precision = null; + public Boolean pointsOnly = null; + public double distanceErrorPct = Double.NaN; + + public void setSpatialStrategy(SpatialStrategy strategy) { + this.strategy = strategy; + } + + public void setTree(String prefixTree) { + this.tree = prefixTree; + } + + public void setTreeLevels(int treeLevels) { + this.treeLevels = treeLevels; + } + + public void setPrecision(String precision) { + this.precision = precision; + } + + public void setPointsOnly(boolean pointsOnly) { + if (this.strategy == SpatialStrategy.TERM && pointsOnly == false) { + throw new ElasticsearchParseException("points_only cannot be set to false for term strategy"); + } + this.pointsOnly = pointsOnly; + } + + public void setDistanceErrorPct(double distanceErrorPct) { + this.distanceErrorPct = distanceErrorPct; + } + + public void setup() { + if (strategy == null) { + strategy = Defaults.STRATEGY; + } + if (tree == null) { + tree = Defaults.TREE; + } + if (Double.isNaN(distanceErrorPct)) { + if (precision != null || treeLevels != Integer.MIN_VALUE) { + distanceErrorPct = 0d; + } else { + distanceErrorPct = Defaults.DISTANCE_ERROR_PCT; + } + } + if (treeLevels == Integer.MIN_VALUE && precision == null) { + // set default precision if treeLevels is not explicitly set + precision = Defaults.PRECISION; + } + if (treeLevels == Integer.MIN_VALUE) { + if (precision.equals(Defaults.PRECISION)) { + treeLevels = tree.equals(Defaults.TREE) + ? Defaults.QUADTREE_LEVELS + : Defaults.GEOHASH_TREE_LEVELS; + } else { + treeLevels = tree == Defaults.TREE + ? GeoUtils.quadTreeLevelsForPrecision(precision) + : GeoUtils.geoHashLevelsForPrecision(precision); + } + } + if (pointsOnly == null) { + if (strategy == SpatialStrategy.TERM) { + pointsOnly = true; + } else { + pointsOnly = Defaults.POINTS_ONLY; + } + } + } + + public static boolean parse(String name, String fieldName, Object fieldNode, DeprecatedParameters deprecatedParameters) { + if (Names.STRATEGY.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { + checkPrefixTreeSupport(fieldName); + deprecatedParameters.setSpatialStrategy(SpatialStrategy.fromString(fieldNode.toString())); + } else if (Names.TREE.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { + checkPrefixTreeSupport(fieldName); + deprecatedParameters.setTree(fieldNode.toString()); + } else if (Names.TREE_LEVELS.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { + checkPrefixTreeSupport(fieldName); + deprecatedParameters.setTreeLevels(Integer.parseInt(fieldNode.toString())); + } else if (Names.PRECISION.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { + checkPrefixTreeSupport(fieldName); + deprecatedParameters.setPrecision(fieldNode.toString()); + } else if (Names.DISTANCE_ERROR_PCT.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { + checkPrefixTreeSupport(fieldName); + deprecatedParameters.setDistanceErrorPct(Double.parseDouble(fieldNode.toString())); + } else if (Names.POINTS_ONLY.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { + checkPrefixTreeSupport(fieldName); + deprecatedParameters.setPointsOnly( + XContentMapValues.nodeBooleanValue(fieldNode, name + "." + DeprecatedParameters.Names.POINTS_ONLY)); + } else { + return false; + } + return true; + } + + private static void checkPrefixTreeSupport(String fieldName) { + if (ShapesAvailability.JTS_AVAILABLE == false || ShapesAvailability.SPATIAL4J_AVAILABLE == false) { + throw new ElasticsearchParseException("Field parameter [{}] is not supported for [{}] field type", + fieldName, CONTENT_TYPE); + } + DEPRECATION_LOGGER.deprecated("Field parameter [{}] is deprecated and will be removed in a future version.", + fieldName); + } + } + + private static final Logger logger = LogManager.getLogger(LegacyGeoShapeFieldMapper.class); + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger); + + public static class Builder extends BaseGeoShapeFieldMapper.Builder { + + DeprecatedParameters deprecatedParameters; + + public Builder(String name) { + super(name, new GeoShapeFieldType(), new GeoShapeFieldType()); + } + + public Builder(String name, boolean coerce, boolean ignoreMalformed, Orientation orientation, + boolean ignoreZ, DeprecatedParameters deprecatedParameters) { + super(name, new GeoShapeFieldType(), new GeoShapeFieldType(), coerce, ignoreMalformed, orientation, ignoreZ); + this.deprecatedParameters = deprecatedParameters; + } + + @Override + public GeoShapeFieldType fieldType() { + return (GeoShapeFieldType)fieldType; + } + + private void setupFieldTypeDeprecatedParameters() { + GeoShapeFieldType ft = fieldType(); + ft.setStrategy(deprecatedParameters.strategy); + ft.setTree(deprecatedParameters.tree); + ft.setTreeLevels(deprecatedParameters.treeLevels); + if (deprecatedParameters.precision != null) { + // precision is only set iff: a. treeLevel is not explicitly set, b. its explicitly set + ft.setPrecisionInMeters(DistanceUnit.parse(deprecatedParameters.precision, + DistanceUnit.DEFAULT, DistanceUnit.DEFAULT)); + } + ft.setDistanceErrorPct(deprecatedParameters.distanceErrorPct); + ft.setPointsOnly(deprecatedParameters.pointsOnly); + } + + private void setupPrefixTrees() { + GeoShapeFieldType ft = fieldType(); + SpatialPrefixTree prefixTree; + if (ft.tree().equals(DeprecatedParameters.PrefixTrees.GEOHASH)) { + prefixTree = new GeohashPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, + getLevels(ft.treeLevels(), ft.precisionInMeters(), DeprecatedParameters.Defaults.GEOHASH_TREE_LEVELS, true)); + } else if (ft.tree().equals(DeprecatedParameters.PrefixTrees.LEGACY_QUADTREE)) { + prefixTree = new QuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, + getLevels(ft.treeLevels(), ft.precisionInMeters(), DeprecatedParameters.Defaults.QUADTREE_LEVELS, false)); + } else if (ft.tree().equals(DeprecatedParameters.PrefixTrees.QUADTREE)) { + prefixTree = new PackedQuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, + getLevels(ft.treeLevels(), ft.precisionInMeters(), DeprecatedParameters.Defaults.QUADTREE_LEVELS, false)); + } else { + throw new IllegalArgumentException("Unknown prefix tree type [" + ft.tree() + "]"); + } + + // setup prefix trees regardless of strategy (this is used for the QueryBuilder) + // recursive: + RecursivePrefixTreeStrategy rpts = new RecursivePrefixTreeStrategy(prefixTree, name()); + rpts.setDistErrPct(ft.distanceErrorPct()); + rpts.setPruneLeafyBranches(false); + ft.recursiveStrategy = rpts; + + // term: + TermQueryPrefixTreeStrategy termStrategy = new TermQueryPrefixTreeStrategy(prefixTree, name()); + termStrategy.setDistErrPct(ft.distanceErrorPct()); + ft.termStrategy = termStrategy; + + // set default (based on strategy): + ft.defaultPrefixTreeStrategy = ft.resolvePrefixTreeStrategy(ft.strategy()); + ft.defaultPrefixTreeStrategy.setPointsOnly(ft.pointsOnly()); + } + + @Override + protected void setupFieldType(BuilderContext context) { + super.setupFieldType(context); + + // field mapper handles this at build time + // but prefix tree strategies require a name, so throw a similar exception + if (name().isEmpty()) { + throw new IllegalArgumentException("name cannot be empty string"); + } + + // setup the deprecated parameters and the prefix tree configuration + setupFieldTypeDeprecatedParameters(); + setupPrefixTrees(); + } + + private static int getLevels(int treeLevels, double precisionInMeters, int defaultLevels, boolean geoHash) { + if (treeLevels > 0 || precisionInMeters >= 0) { + return Math.max(treeLevels, precisionInMeters >= 0 ? (geoHash ? GeoUtils.geoHashLevelsForPrecision(precisionInMeters) + : GeoUtils.quadTreeLevelsForPrecision(precisionInMeters)) : 0); + } + return defaultLevels; + } + + @Override + public LegacyGeoShapeFieldMapper build(BuilderContext context) { + setupFieldType(context); + + return new LegacyGeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), + coerce(context), orientation(), ignoreZValue(), context.indexSettings(), + multiFieldsBuilder.build(this, context), copyTo); + } + } + + public static final class GeoShapeFieldType extends BaseGeoShapeFieldType { + + private String tree = DeprecatedParameters.Defaults.TREE; + private SpatialStrategy strategy = DeprecatedParameters.Defaults.STRATEGY; + private boolean pointsOnly = DeprecatedParameters.Defaults.POINTS_ONLY; + private int treeLevels = 0; + private double precisionInMeters = -1; + private Double distanceErrorPct; + private double defaultDistanceErrorPct = 0.0; + + // these are built when the field type is frozen + private PrefixTreeStrategy defaultPrefixTreeStrategy; + private RecursivePrefixTreeStrategy recursiveStrategy; + private TermQueryPrefixTreeStrategy termStrategy; + + public GeoShapeFieldType() { + setIndexOptions(IndexOptions.DOCS); + setTokenized(false); + setStored(false); + setStoreTermVectors(false); + setOmitNorms(true); + } + + protected GeoShapeFieldType(GeoShapeFieldType ref) { + super(ref); + this.tree = ref.tree; + this.strategy = ref.strategy; + this.pointsOnly = ref.pointsOnly; + this.treeLevels = ref.treeLevels; + this.precisionInMeters = ref.precisionInMeters; + this.distanceErrorPct = ref.distanceErrorPct; + this.defaultDistanceErrorPct = ref.defaultDistanceErrorPct; + } + + @Override + public GeoShapeFieldType clone() { + return new GeoShapeFieldType(this); + } + + @Override + public boolean equals(Object o) { + if (!super.equals(o)) return false; + GeoShapeFieldType that = (GeoShapeFieldType) o; + return treeLevels == that.treeLevels && + precisionInMeters == that.precisionInMeters && + defaultDistanceErrorPct == that.defaultDistanceErrorPct && + Objects.equals(tree, that.tree) && + Objects.equals(strategy, that.strategy) && + pointsOnly == that.pointsOnly && + Objects.equals(distanceErrorPct, that.distanceErrorPct); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), tree, strategy, pointsOnly, treeLevels, precisionInMeters, distanceErrorPct, + defaultDistanceErrorPct); + } + + @Override + public void checkCompatibility(MappedFieldType fieldType, List conflicts) { + super.checkCompatibility(fieldType, conflicts); + GeoShapeFieldType other = (GeoShapeFieldType)fieldType; + // prevent user from changing strategies + if (strategy() != other.strategy()) { + conflicts.add("mapper [" + name() + "] has different [strategy]"); + } + + // prevent user from changing trees (changes encoding) + if (tree().equals(other.tree()) == false) { + conflicts.add("mapper [" + name() + "] has different [tree]"); + } + + if ((pointsOnly() != other.pointsOnly())) { + conflicts.add("mapper [" + name() + "] has different points_only"); + } + + // TODO we should allow this, but at the moment levels is used to build bookkeeping variables + // in lucene's SpatialPrefixTree implementations, need a patch to correct that first + if (treeLevels() != other.treeLevels()) { + conflicts.add("mapper [" + name() + "] has different [tree_levels]"); + } + if (precisionInMeters() != other.precisionInMeters()) { + conflicts.add("mapper [" + name() + "] has different [precision]"); + } + } + + public String tree() { + return tree; + } + + public void setTree(String tree) { + checkIfFrozen(); + this.tree = tree; + } + + public SpatialStrategy strategy() { + return strategy; + } + + public void setStrategy(SpatialStrategy strategy) { + checkIfFrozen(); + this.strategy = strategy; + if (this.strategy.equals(SpatialStrategy.TERM)) { + this.pointsOnly = true; + } + } + + public boolean pointsOnly() { + return pointsOnly; + } + + public void setPointsOnly(boolean pointsOnly) { + checkIfFrozen(); + this.pointsOnly = pointsOnly; + } + public int treeLevels() { + return treeLevels; + } + + public void setTreeLevels(int treeLevels) { + checkIfFrozen(); + this.treeLevels = treeLevels; + } + + public double precisionInMeters() { + return precisionInMeters; + } + + public void setPrecisionInMeters(double precisionInMeters) { + checkIfFrozen(); + this.precisionInMeters = precisionInMeters; + } + + public double distanceErrorPct() { + return distanceErrorPct == null ? defaultDistanceErrorPct : distanceErrorPct; + } + + public void setDistanceErrorPct(double distanceErrorPct) { + checkIfFrozen(); + this.distanceErrorPct = distanceErrorPct; + } + + public void setDefaultDistanceErrorPct(double defaultDistanceErrorPct) { + checkIfFrozen(); + this.defaultDistanceErrorPct = defaultDistanceErrorPct; + } + + public PrefixTreeStrategy defaultPrefixTreeStrategy() { + return this.defaultPrefixTreeStrategy; + } + + public PrefixTreeStrategy resolvePrefixTreeStrategy(SpatialStrategy strategy) { + return resolvePrefixTreeStrategy(strategy.getStrategyName()); + } + + public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) { + if (SpatialStrategy.RECURSIVE.getStrategyName().equals(strategyName)) { + return recursiveStrategy; + } + if (SpatialStrategy.TERM.getStrategyName().equals(strategyName)) { + return termStrategy; + } + throw new IllegalArgumentException("Unknown prefix tree strategy [" + strategyName + "]"); + } + } + + public LegacyGeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, + Explicit ignoreMalformed, Explicit coerce, Explicit orientation, + Explicit ignoreZValue, Settings indexSettings, + MultiFields multiFields, CopyTo copyTo) { + super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, orientation, ignoreZValue, indexSettings, + multiFields, copyTo); + } + + @Override + public GeoShapeFieldType fieldType() { + return (GeoShapeFieldType) super.fieldType(); + } + + @Override + public void parse(ParseContext context) throws IOException { + try { + Shape shape = context.parseExternalValue(Shape.class); + if (shape == null) { + ShapeBuilder shapeBuilder = ShapeParser.parse(context.parser(), this); + if (shapeBuilder == null) { + return; + } + shape = shapeBuilder.buildS4J(); + } + if (fieldType().pointsOnly() == true) { + // index configured for pointsOnly + if (shape instanceof XShapeCollection && XShapeCollection.class.cast(shape).pointsOnly()) { + // MULTIPOINT data: index each point separately + List shapes = ((XShapeCollection) shape).getShapes(); + for (Shape s : shapes) { + indexShape(context, s); + } + return; + } else if (shape instanceof Point == false) { + throw new MapperParsingException("[{" + fieldType().name() + "}] is configured for points only but a " + + ((shape instanceof JtsGeometry) ? ((JtsGeometry)shape).getGeom().getGeometryType() : shape.getClass()) + + " was found"); + } + } + indexShape(context, shape); + } catch (Exception e) { + if (ignoreMalformed.value() == false) { + throw new MapperParsingException("failed to parse field [{}] of type [{}]", e, fieldType().name(), + fieldType().typeName()); + } + context.addIgnoredField(fieldType.name()); + } + } + + private void indexShape(ParseContext context, Shape shape) { + List fields = new ArrayList<>(Arrays.asList(fieldType().defaultPrefixTreeStrategy().createIndexableFields(shape))); + createFieldNamesField(context, fields); + for (IndexableField field : fields) { + context.doc().add(field); + } + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + super.doXContentBody(builder, includeDefaults, params); + + if (includeDefaults || fieldType().tree().equals(DeprecatedParameters.Defaults.TREE) == false) { + builder.field(DeprecatedParameters.Names.TREE.getPreferredName(), fieldType().tree()); + } + + if (fieldType().treeLevels() != 0) { + builder.field(DeprecatedParameters.Names.TREE_LEVELS.getPreferredName(), fieldType().treeLevels()); + } else if(includeDefaults && fieldType().precisionInMeters() == -1) { // defaults only make sense if precision is not specified + if (DeprecatedParameters.PrefixTrees.GEOHASH.equals(fieldType().tree())) { + builder.field(DeprecatedParameters.Names.TREE_LEVELS.getPreferredName(), + DeprecatedParameters.Defaults.GEOHASH_TREE_LEVELS); + } else if (DeprecatedParameters.PrefixTrees.LEGACY_QUADTREE.equals(fieldType().tree())) { + builder.field(DeprecatedParameters.Names.TREE_LEVELS.getPreferredName(), + DeprecatedParameters.Defaults.QUADTREE_LEVELS); + } else if (DeprecatedParameters.PrefixTrees.QUADTREE.equals(fieldType().tree())) { + builder.field(DeprecatedParameters.Names.TREE_LEVELS.getPreferredName(), + DeprecatedParameters.Defaults.QUADTREE_LEVELS); + } else { + throw new IllegalArgumentException("Unknown prefix tree type [" + fieldType().tree() + "]"); + } + } + if (fieldType().precisionInMeters() != -1) { + builder.field(DeprecatedParameters.Names.PRECISION.getPreferredName(), + DistanceUnit.METERS.toString(fieldType().precisionInMeters())); + } else if (includeDefaults && fieldType().treeLevels() == 0) { // defaults only make sense if tree levels are not specified + builder.field(DeprecatedParameters.Names.PRECISION.getPreferredName(), + DistanceUnit.METERS.toString(50)); + } + if (includeDefaults || fieldType().strategy() != DeprecatedParameters.Defaults.STRATEGY) { + builder.field(DeprecatedParameters.Names.STRATEGY.getPreferredName(), fieldType().strategy().getStrategyName()); + } + if (includeDefaults || fieldType().distanceErrorPct() != fieldType().defaultDistanceErrorPct) { + builder.field(DeprecatedParameters.Names.DISTANCE_ERROR_PCT.getPreferredName(), fieldType().distanceErrorPct()); + } + if (fieldType().strategy() == SpatialStrategy.TERM) { + // For TERMs strategy the defaults for points only change to true + if (includeDefaults || fieldType().pointsOnly() != true) { + builder.field(DeprecatedParameters.Names.POINTS_ONLY.getPreferredName(), fieldType().pointsOnly()); + } + } else { + if (includeDefaults || fieldType().pointsOnly() != DeprecatedParameters.Defaults.POINTS_ONLY) { + builder.field(DeprecatedParameters.Names.POINTS_ONLY.getPreferredName(), fieldType().pointsOnly()); + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java index acb1148508c9c..e16f178b78c22 100644 --- a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java @@ -53,7 +53,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper; +import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import java.io.IOException; @@ -334,14 +335,9 @@ public GeoShapeQueryBuilder relation(ShapeRelation relation) { if (relation == null) { throw new IllegalArgumentException("No Shape Relation defined"); } - if (strategy != null) { - if (strategy == SpatialStrategy.TERM && relation != ShapeRelation.INTERSECTS) { - throw new IllegalArgumentException("current strategy [" + strategy.getStrategyName() + "] only supports relation [" - + ShapeRelation.INTERSECTS.getRelationName() + "] found relation [" + relation.getRelationName() + "]"); - } else if (strategy == SpatialStrategy.VECTOR && relation == ShapeRelation.CONTAINS) { - throw new IllegalArgumentException("current strategy [" + strategy.getStrategyName() + "] does not support relation [" - + ShapeRelation.CONTAINS.getRelationName() + "]"); - } + if (SpatialStrategy.TERM.equals(strategy) && relation != ShapeRelation.INTERSECTS) { + throw new IllegalArgumentException("current strategy [" + strategy.getStrategyName() + "] only supports relation [" + + ShapeRelation.INTERSECTS.getRelationName() + "] found relation [" + relation.getRelationName() + "]"); } this.relation = relation; return this; @@ -386,19 +382,19 @@ protected Query doToQuery(QueryShardContext context) { } else { throw new QueryShardException(context, "failed to find geo_shape field [" + fieldName + "]"); } - } else if (fieldType.typeName().equals(GeoShapeFieldMapper.CONTENT_TYPE) == false) { + } else if (fieldType.typeName().equals(BaseGeoShapeFieldMapper.CONTENT_TYPE) == false) { throw new QueryShardException(context, "Field [" + fieldName + "] is not of type [geo_shape] but of type [" + fieldType.typeName() + "]"); } - final GeoShapeFieldMapper.GeoShapeFieldType shapeFieldType = (GeoShapeFieldMapper.GeoShapeFieldType) fieldType; - - SpatialStrategy spatialStrategy = shapeFieldType.strategy(); - if(this.strategy != null) { - spatialStrategy = this.strategy; - } + final BaseGeoShapeFieldMapper.BaseGeoShapeFieldType ft = (BaseGeoShapeFieldMapper.BaseGeoShapeFieldType) fieldType; Query query; - if (spatialStrategy != SpatialStrategy.VECTOR) { + if (strategy != null || ft instanceof LegacyGeoShapeFieldMapper.GeoShapeFieldType) { + LegacyGeoShapeFieldMapper.GeoShapeFieldType shapeFieldType = (LegacyGeoShapeFieldMapper.GeoShapeFieldType) ft; + SpatialStrategy spatialStrategy = shapeFieldType.strategy(); + if (this.strategy != null) { + spatialStrategy = this.strategy; + } PrefixTreeStrategy prefixTreeStrategy = shapeFieldType.resolvePrefixTreeStrategy(spatialStrategy); if (prefixTreeStrategy instanceof RecursivePrefixTreeStrategy && relation == ShapeRelation.DISJOINT) { // this strategy doesn't support disjoint anymore: but it did @@ -414,25 +410,24 @@ protected Query doToQuery(QueryShardContext context) { query = new ConstantScoreQuery(prefixTreeStrategy.makeQuery(getArgs(shapeToQuery, relation))); } } else { - query = getVectorQuery(context, shapeFieldType, shapeToQuery); + query = new ConstantScoreQuery(getVectorQuery(context, shapeToQuery)); } return query; } - private Query getVectorQuery(QueryShardContext context, GeoShapeFieldMapper.GeoShapeFieldType shapeFieldType, - ShapeBuilder queryShapeBuilder) { + private Query getVectorQuery(QueryShardContext context, ShapeBuilder queryShapeBuilder) { // CONTAINS queries are not yet supported by VECTOR strategy if (relation == ShapeRelation.CONTAINS) { throw new QueryShardException(context, ShapeRelation.CONTAINS + " query relation not supported for Field [" + fieldName - + "] when using Strategy [" + shapeFieldType.strategy() + "]"); + + "] Use prefix tree strategy [" + SpatialStrategy.RECURSIVE + "] or [" + SpatialStrategy.TERM + "] instead."); } // wrap geoQuery as a ConstantScoreQuery - return new ConstantScoreQuery(getVectorQuery(context, queryShapeBuilder.buildLucene())); + return getVectorQueryFromShape(context, queryShapeBuilder.buildLucene()); } - private Query getVectorQuery(QueryShardContext context, Object queryShape) { + private Query getVectorQueryFromShape(QueryShardContext context, Object queryShape) { Query geoQuery; if (queryShape instanceof Line[]) { geoQuery = LatLonShape.newLineQuery(fieldName(), relation.getLuceneRelation(), (Line[]) queryShape); @@ -468,7 +463,7 @@ private Query getVectorQuery(QueryShardContext context, Object queryShape) { private Query createGeometryCollectionQuery(QueryShardContext context, Object... shapes) { BooleanQuery.Builder bqb = new BooleanQuery.Builder(); for (Object shape : shapes) { - bqb.add(getVectorQuery(context, shape), BooleanClause.Occur.SHOULD); + bqb.add(getVectorQueryFromShape(context, shape), BooleanClause.Occur.SHOULD); } return bqb.build(); } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 94b896c502a5c..24b5d7f427ca2 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; +import org.elasticsearch.index.mapper.BaseGeoShapeFieldMapper; import org.elasticsearch.index.mapper.BinaryFieldMapper; import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.CompletionFieldMapper; @@ -38,7 +39,6 @@ import org.elasticsearch.index.mapper.FieldAliasMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.GeoPointFieldMapper; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; @@ -131,7 +131,7 @@ private Map getMappers(List mapperPlugi mappers.put(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser()); mappers.put(FieldAliasMapper.CONTENT_TYPE, new FieldAliasMapper.TypeParser()); mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser()); - mappers.put(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser()); + mappers.put(BaseGeoShapeFieldMapper.CONTENT_TYPE, new BaseGeoShapeFieldMapper.TypeParser()); for (MapperPlugin mapperPlugin : mapperPlugins) { for (Map.Entry entry : mapperPlugin.getMappers().entrySet()) { diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java index cb799501fb9e7..5a9d79374bca4 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java @@ -32,8 +32,9 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.hamcrest.ElasticsearchGeoAssertions; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.LineString; @@ -303,19 +304,32 @@ public void testParse3DPolygon() throws IOException { shellCoordinates.add(new Coordinate(101, 1, 10)); shellCoordinates.add(new Coordinate(100, 1, 10)); shellCoordinates.add(new Coordinate(100, 0, 10)); + Coordinate[] coordinates = shellCoordinates.toArray(new Coordinate[shellCoordinates.size()]); + Version randomVersion = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.CURRENT); Settings indexSettings = Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_6_3_0) + .put(IndexMetaData.SETTING_VERSION_CREATED, randomVersion) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build(); - LinearRing shell = GEOMETRY_FACTORY.createLinearRing(shellCoordinates.toArray(new Coordinate[shellCoordinates.size()])); - Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, null); - Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final GeoShapeFieldMapper mapperBuilder = new GeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext); - try (XContentParser parser = createParser(polygonGeoJson)) { - parser.nextToken(); - ElasticsearchGeoAssertions.assertEquals(jtsGeom(expected), ShapeParser.parse(parser, mapperBuilder).buildS4J()); + + boolean useJTS = randomVersion.before(Version.V_6_6_0); + + if (useJTS) { + LinearRing shell = GEOMETRY_FACTORY.createLinearRing(shellCoordinates.toArray(new Coordinate[shellCoordinates.size()])); + Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, null); + Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); + final LegacyGeoShapeFieldMapper mapperBuilder = + (LegacyGeoShapeFieldMapper) (new LegacyGeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext)); + try (XContentParser parser = createParser(polygonGeoJson)) { + parser.nextToken(); + ElasticsearchGeoAssertions.assertEquals(jtsGeom(expected), ShapeParser.parse(parser, mapperBuilder).buildS4J()); + } + } else { + org.apache.lucene.geo.Polygon p = new org.apache.lucene.geo.Polygon( + Arrays.stream(coordinates).mapToDouble(i->i.y).toArray(), + Arrays.stream(coordinates).mapToDouble(i->i.x).toArray()); + assertGeometryEquals(p, polygonGeoJson, useJTS); } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java index be33ffe26a9dc..acdc9b1c6e5ea 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java @@ -42,7 +42,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.test.geo.RandomShapeGenerator; import org.locationtech.jts.geom.Coordinate; @@ -315,7 +315,8 @@ public void testParseMixedDimensionPolyWithHole() throws IOException { .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build(); Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final GeoShapeFieldMapper mapperBuilder = new GeoShapeFieldMapper.Builder("test").ignoreZValue(false).build(mockBuilderContext); + final LegacyGeoShapeFieldMapper mapperBuilder = + (LegacyGeoShapeFieldMapper)(new LegacyGeoShapeFieldMapper.Builder("test").ignoreZValue(false).build(mockBuilderContext)); // test store z disabled ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, @@ -353,7 +354,8 @@ public void testParseMixedDimensionPolyWithHoleStoredZ() throws IOException { .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build(); Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final GeoShapeFieldMapper mapperBuilder = new GeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext); + final LegacyGeoShapeFieldMapper mapperBuilder = + (LegacyGeoShapeFieldMapper)(new LegacyGeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext)); // test store z disabled ElasticsearchException e = expectThrows(ElasticsearchException.class, @@ -382,7 +384,8 @@ public void testParsePolyWithStoredZ() throws IOException { .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build(); Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final GeoShapeFieldMapper mapperBuilder = new GeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext); + final LegacyGeoShapeFieldMapper mapperBuilder = + (LegacyGeoShapeFieldMapper)(new LegacyGeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext)); ShapeBuilder shapeBuilder = ShapeParser.parse(parser, mapperBuilder); assertEquals(shapeBuilder.numDimensions(), 3); @@ -402,12 +405,14 @@ public void testParseOpenPolygon() throws IOException { .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build(); Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final GeoShapeFieldMapper defaultMapperBuilder = new GeoShapeFieldMapper.Builder("test").coerce(false).build(mockBuilderContext); + final LegacyGeoShapeFieldMapper defaultMapperBuilder = + (LegacyGeoShapeFieldMapper)(new LegacyGeoShapeFieldMapper.Builder("test").coerce(false).build(mockBuilderContext)); ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class, () -> ShapeParser.parse(parser, defaultMapperBuilder)); assertEquals("invalid LinearRing found (coordinates are not closed)", exception.getMessage()); - final GeoShapeFieldMapper coercingMapperBuilder = new GeoShapeFieldMapper.Builder("test").coerce(true).build(mockBuilderContext); + final LegacyGeoShapeFieldMapper coercingMapperBuilder = + (LegacyGeoShapeFieldMapper)(new LegacyGeoShapeFieldMapper.Builder("test").coerce(true).build(mockBuilderContext)); ShapeBuilder shapeBuilder = ShapeParser.parse(parser, coercingMapperBuilder); assertNotNull(shapeBuilder); assertEquals("polygon ((100.0 5.0, 100.0 10.0, 90.0 10.0, 90.0 5.0, 100.0 5.0))", shapeBuilder.toWKT()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index d423d6b83b353..28638125309bd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -24,7 +24,7 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.elasticsearch.common.geo.SpatialStrategy; +import org.elasticsearch.Version; import org.elasticsearch.common.geo.builders.PointBuilder; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.geo.GeoPoint; @@ -63,6 +63,7 @@ public static class Builder extends FieldMapper.Builder private BooleanFieldMapper.Builder boolBuilder = new BooleanFieldMapper.Builder(Names.FIELD_BOOL); private GeoPointFieldMapper.Builder latLonPointBuilder = new GeoPointFieldMapper.Builder(Names.FIELD_POINT); private GeoShapeFieldMapper.Builder shapeBuilder = new GeoShapeFieldMapper.Builder(Names.FIELD_SHAPE); + private LegacyGeoShapeFieldMapper.Builder legacyShapeBuilder = new LegacyGeoShapeFieldMapper.Builder(Names.FIELD_SHAPE); private Mapper.Builder stringBuilder; private String generatedValue; private String mapperName; @@ -86,7 +87,9 @@ public ExternalMapper build(BuilderContext context) { BinaryFieldMapper binMapper = binBuilder.build(context); BooleanFieldMapper boolMapper = boolBuilder.build(context); GeoPointFieldMapper pointMapper = latLonPointBuilder.build(context); - GeoShapeFieldMapper shapeMapper = shapeBuilder.build(context); + BaseGeoShapeFieldMapper shapeMapper = (context.indexCreatedVersion().before(Version.V_6_0_0)) + ? legacyShapeBuilder.build(context) + : shapeBuilder.build(context); FieldMapper stringMapper = (FieldMapper)stringBuilder.build(context); context.path().remove(); @@ -150,13 +153,13 @@ public Query existsQuery(QueryShardContext context) { private BinaryFieldMapper binMapper; private BooleanFieldMapper boolMapper; private GeoPointFieldMapper pointMapper; - private GeoShapeFieldMapper shapeMapper; + private BaseGeoShapeFieldMapper shapeMapper; private FieldMapper stringMapper; public ExternalMapper(String simpleName, MappedFieldType fieldType, String generatedValue, String mapperName, BinaryFieldMapper binMapper, BooleanFieldMapper boolMapper, GeoPointFieldMapper pointMapper, - GeoShapeFieldMapper shapeMapper, FieldMapper stringMapper, Settings indexSettings, + BaseGeoShapeFieldMapper shapeMapper, FieldMapper stringMapper, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { super(simpleName, fieldType, new ExternalFieldType(), indexSettings, multiFields, copyTo); this.generatedValue = generatedValue; @@ -183,7 +186,7 @@ public void parse(ParseContext context) throws IOException { // Let's add a Dummy Shape PointBuilder pb = new PointBuilder(-100, 45); - if (shapeMapper.fieldType().strategy() == SpatialStrategy.VECTOR) { + if (shapeMapper instanceof GeoShapeFieldMapper) { shapeMapper.parse(context.createExternalValueContext(pb.buildLucene())); } else { shapeMapper.parse(context.createExternalValueContext(pb.buildS4J())); @@ -214,7 +217,7 @@ public FieldMapper updateFieldType(Map fullNameToFieldT BinaryFieldMapper binMapperUpdate = (BinaryFieldMapper) binMapper.updateFieldType(fullNameToFieldType); BooleanFieldMapper boolMapperUpdate = (BooleanFieldMapper) boolMapper.updateFieldType(fullNameToFieldType); GeoPointFieldMapper pointMapperUpdate = (GeoPointFieldMapper) pointMapper.updateFieldType(fullNameToFieldType); - GeoShapeFieldMapper shapeMapperUpdate = (GeoShapeFieldMapper) shapeMapper.updateFieldType(fullNameToFieldType); + BaseGeoShapeFieldMapper shapeMapperUpdate = (BaseGeoShapeFieldMapper) shapeMapper.updateFieldType(fullNameToFieldType); TextFieldMapper stringMapperUpdate = (TextFieldMapper) stringMapper.updateFieldType(fullNameToFieldType); if (update == this && multiFieldsUpdate == multiFields diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java index 6a84b5118b6ea..a5e2d7c31afe2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java @@ -18,14 +18,9 @@ */ package org.elasticsearch.index.mapper; -import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; -import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -42,7 +37,6 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.not; public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { @@ -53,10 +47,10 @@ protected Collection> getPlugins() { public void testDefaultConfiguration() throws IOException { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .endObject().endObject() + .endObject().endObject()); DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() .parse("type1", new CompressedXContent(mapping)); @@ -64,7 +58,8 @@ public void testDefaultConfiguration() throws IOException { assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(GeoShapeFieldMapper.Defaults.ORIENTATION)); + assertThat(geoShapeFieldMapper.fieldType().orientation(), + equalTo(GeoShapeFieldMapper.Defaults.ORIENTATION.value())); } /** @@ -72,11 +67,11 @@ public void testDefaultConfiguration() throws IOException { */ public void testOrientationParsing() throws IOException { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("orientation", "left") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("orientation", "left") + .endObject().endObject() + .endObject().endObject()); DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() .parse("type1", new CompressedXContent(mapping)); @@ -90,11 +85,11 @@ public void testOrientationParsing() throws IOException { // explicit right orientation test mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("orientation", "right") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("orientation", "right") + .endObject().endObject() + .endObject().endObject()); defaultMapper = createIndex("test2").mapperService().documentMapperParser() .parse("type1", new CompressedXContent(mapping)); @@ -112,11 +107,11 @@ public void testOrientationParsing() throws IOException { */ public void testCoerceParsing() throws IOException { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("coerce", "true") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("coerce", "true") + .endObject().endObject() + .endObject().endObject()); DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() .parse("type1", new CompressedXContent(mapping)); @@ -128,11 +123,11 @@ public void testCoerceParsing() throws IOException { // explicit false coerce test mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("coerce", "false") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("coerce", "false") + .endObject().endObject() + .endObject().endObject()); defaultMapper = createIndex("test2").mapperService().documentMapperParser() .parse("type1", new CompressedXContent(mapping)); @@ -141,6 +136,7 @@ public void testCoerceParsing() throws IOException { coerce = ((GeoShapeFieldMapper)fieldMapper).coerce().value(); assertThat(coerce, equalTo(false)); + assertFieldWarnings("tree"); } @@ -217,325 +213,46 @@ public void testIgnoreMalformedParsing() throws IOException { assertThat(ignoreMalformed.value(), equalTo(false)); } - public void testGeohashConfiguration() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .field("tree_levels", "4") - .field("distance_error_pct", "0.1") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.1)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(4)); - assertFieldWarnings("tree", "tree_levels", "distance_error_pct"); - } - - public void testQuadtreeConfiguration() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("tree_levels", "6") - .field("distance_error_pct", "0.5") - .field("points_only", true) - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(6)); - assertThat(strategy.isPointsOnly(), equalTo(true)); - assertFieldWarnings("tree", "tree_levels", "distance_error_pct", "points_only"); - } private void assertFieldWarnings(String... fieldNames) { String[] warnings = new String[fieldNames.length]; - String vectorPostfix = " use [vector] indexing strategy instead."; for (int i = 0; i < fieldNames.length; ++i) { warnings[i] = "Field parameter [" + fieldNames[i] + "] " - + "is deprecated and will be removed in a future version." + vectorPostfix; + + "is deprecated and will be removed in a future version."; } - assertWarnings(warnings); - } - - public void testLevelPrecisionConfiguration() throws IOException { - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("tree_levels", "6") - .field("precision", "70m") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - // 70m is more precise so it wins - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d))); - } - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("tree_levels", "26") - .field("precision", "70m") - .endObject().endObject() - .endObject().endObject()); - - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - // distance_error_pct was not specified so we expect the mapper to take the highest precision between "precision" and - // "tree_levels" setting distErrPct to 0 to guarantee desired precision - assertThat(strategy.getDistErrPct(), equalTo(0.0)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - // 70m is less precise so it loses - assertThat(strategy.getGrid().getMaxLevels(), equalTo(26)); - } - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .field("tree_levels", "6") - .field("precision", "70m") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - // 70m is more precise so it wins - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(70d))); - } - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .field("tree_levels", GeoUtils.geoHashLevelsForPrecision(70d)+1) - .field("precision", "70m") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(70d)+1)); - } - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("tree_levels", GeoUtils.quadTreeLevelsForPrecision(70d)+1) - .field("precision", "70m") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d)+1)); - } - assertFieldWarnings("tree", "tree_levels", "precision", "distance_error_pct"); - } - - public void testPointsOnlyOption() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .field("points_only", true) - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.isPointsOnly(), equalTo(true)); - assertFieldWarnings("tree", "points_only"); - } - - public void testLevelDefaults() throws IOException { - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - /* 50m is default */ - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(50d))); - } - - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .field("distance_error_pct", "0.5") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.5)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - /* 50m is default */ - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(50d))); - } - assertFieldWarnings("tree", "distance_error_pct"); } public void testGeoShapeMapperMerge() throws Exception { String stage1Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("shape").field("type", "geo_shape").field("tree", "geohash") - .field("strategy", "recursive") - .field("precision", "1m").field("tree_levels", 8).field("distance_error_pct", 0.01) - .field("orientation", "ccw") - .endObject().endObject().endObject().endObject()); + .startObject("shape").field("type", "geo_shape") + .field("orientation", "ccw") + .endObject().endObject().endObject().endObject()); MapperService mapperService = createIndex("test").mapperService(); DocumentMapper docMapper = mapperService.merge("type", new CompressedXContent(stage1Mapping), MapperService.MergeReason.MAPPING_UPDATE); String stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("shape").field("type", "geo_shape") - .field("tree", "quadtree") - .field("strategy", "term").field("precision", "1km") - .field("tree_levels", 26).field("distance_error_pct", 26) - .field("orientation", "cw").endObject().endObject().endObject().endObject()); - try { - mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); - fail(); - } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("mapper [shape] has different [strategy]")); - assertThat(e.getMessage(), containsString("mapper [shape] has different [tree]")); - assertThat(e.getMessage(), containsString("mapper [shape] has different [tree_levels]")); - assertThat(e.getMessage(), containsString("mapper [shape] has different [precision]")); - } + .startObject("properties").startObject("shape").field("type", "geo_shape") + .field("orientation", "cw").endObject().endObject().endObject().endObject()); + mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); // verify nothing changed Mapper fieldMapper = docMapper.mappers().getMapper("shape"); assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy, instanceOf(RecursivePrefixTreeStrategy.class)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.getDistErrPct(), equalTo(0.01)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(1d))); assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(ShapeBuilder.Orientation.CCW)); - // correct mapping + // change mapping; orientation stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("shape").field("type", "geo_shape") - .field("tree", "geohash") - .field("strategy", "recursive") - .field("precision", "1m") - .field("tree_levels", 8).field("distance_error_pct", 0.001) - .field("orientation", "cw").endObject().endObject().endObject().endObject()); + .startObject("properties").startObject("shape").field("type", "geo_shape") + .field("orientation", "cw").endObject().endObject().endObject().endObject()); docMapper = mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); fieldMapper = docMapper.mappers().getMapper("shape"); assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy, instanceOf(RecursivePrefixTreeStrategy.class)); - assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); - assertThat(strategy.getDistErrPct(), equalTo(0.001)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(1d))); assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(ShapeBuilder.Orientation.CW)); - - assertFieldWarnings("tree", "strategy", "precision", "tree_levels", "distance_error_pct"); } public void testEmptyName() throws Exception { @@ -559,115 +276,12 @@ public void testSerializeDefaults() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") .field("type", "geo_shape") - .field("tree", "quadtree") .endObject().endObject() .endObject().endObject()); DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertTrue(serialized, serialized.contains("\"precision\":\"50.0m\"")); - assertTrue(serialized, serialized.contains("\"tree_levels\":21")); + assertTrue(serialized, serialized.contains("\"orientation\":\"" + BaseGeoShapeFieldMapper.Defaults.ORIENTATION.value() + "\"")); } - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "geohash") - .endObject().endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertTrue(serialized, serialized.contains("\"precision\":\"50.0m\"")); - assertTrue(serialized, serialized.contains("\"tree_levels\":9")); - } - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("tree_levels", "6") - .endObject().endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertFalse(serialized, serialized.contains("\"precision\":")); - assertTrue(serialized, serialized.contains("\"tree_levels\":6")); - } - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("precision", "6") - .endObject().endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); - assertFalse(serialized, serialized.contains("\"tree_levels\":")); - } - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("precision", "6m") - .field("tree_levels", "5") - .endObject().endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); - assertTrue(serialized, serialized.contains("\"tree_levels\":5")); - } - assertFieldWarnings("tree", "tree_levels", "precision"); - } - - public void testPointsOnlyDefaultsWithTermStrategy() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("precision", "10m") - .field("strategy", "term") - .endObject().endObject() - .endObject().endObject()); - - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); - Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); - assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); - - GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; - PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); - - assertThat(strategy.getDistErrPct(), equalTo(0.0)); - assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); - assertThat(strategy.getGrid().getMaxLevels(), equalTo(23)); - assertThat(strategy.isPointsOnly(), equalTo(true)); - // term strategy changes the default for points_only, check that we handle it correctly - assertThat(toXContentString(geoShapeFieldMapper, false), not(containsString("points_only"))); - assertFieldWarnings("tree", "precision", "strategy"); - } - - - public void testPointsOnlyFalseWithTermStrategy() throws Exception { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("tree", "quadtree") - .field("precision", "10m") - .field("strategy", "term") - .field("points_only", false) - .endObject().endObject() - .endObject().endObject()); - - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> parser.parse("type1", new CompressedXContent(mapping)) - ); - assertThat(e.getMessage(), containsString("points_only cannot be set to false for term strategy")); - assertFieldWarnings("tree", "precision", "strategy", "points_only"); } public String toXContentString(GeoShapeFieldMapper mapper, boolean includeDefaults) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java index 5a0dab444920e..c10ec5facf806 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java @@ -18,69 +18,23 @@ */ package org.elasticsearch.index.mapper; -import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.index.mapper.GeoShapeFieldMapper.GeoShapeFieldType; import org.junit.Before; -import java.io.IOException; - public class GeoShapeFieldTypeTests extends FieldTypeTestCase { @Override protected MappedFieldType createDefaultFieldType() { - return new GeoShapeFieldMapper.GeoShapeFieldType(); + return new GeoShapeFieldType(); } @Before public void setupProperties() { - addModifier(new Modifier("tree", false) { - @Override - public void modify(MappedFieldType ft) { - ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setTree("quadtree"); - } - }); - addModifier(new Modifier("strategy", false) { - @Override - public void modify(MappedFieldType ft) { - ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setStrategy(SpatialStrategy.TERM); - } - }); - addModifier(new Modifier("tree_levels", false) { + addModifier(new FieldTypeTestCase.Modifier("orientation", true) { @Override public void modify(MappedFieldType ft) { - ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setTreeLevels(10); + ((GeoShapeFieldType)ft).setOrientation(ShapeBuilder.Orientation.LEFT); } }); - addModifier(new Modifier("precision", false) { - @Override - public void modify(MappedFieldType ft) { - ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setPrecisionInMeters(20); - } - }); - addModifier(new Modifier("distance_error_pct", true) { - @Override - public void modify(MappedFieldType ft) { - ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setDefaultDistanceErrorPct(0.5); - } - }); - addModifier(new Modifier("orientation", true) { - @Override - public void modify(MappedFieldType ft) { - ((GeoShapeFieldMapper.GeoShapeFieldType)ft).setOrientation(ShapeBuilder.Orientation.LEFT); - } - }); - } - - /** - * Test for {@link GeoShapeFieldType#setStrategy(SpatialStrategy)} that checks that {@link GeoShapeFieldType#pointsOnly()} - * gets set as a side effect when using SpatialStrategy.TERM - */ - public void testSetStrategyName() throws IOException { - GeoShapeFieldType fieldType = new GeoShapeFieldMapper.GeoShapeFieldType(); - assertFalse(fieldType.pointsOnly()); - fieldType.setStrategy(SpatialStrategy.RECURSIVE); - assertFalse(fieldType.pointsOnly()); - fieldType.setStrategy(SpatialStrategy.TERM); - assertTrue(fieldType.pointsOnly()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java new file mode 100644 index 0000000000000..11d8c72531db2 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java @@ -0,0 +1,714 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper; + +import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; +import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_Z_VALUE; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.not; + +public class LegacyGeoShapeFieldMapperTests extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return pluginList(InternalSettingsPlugin.class); + } + + public void testDefaultConfiguration() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("strategy", "recursive") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + assertThat(geoShapeFieldMapper.fieldType().tree(), + equalTo(LegacyGeoShapeFieldMapper.DeprecatedParameters.Defaults.TREE)); + assertThat(geoShapeFieldMapper.fieldType().treeLevels(), + equalTo(LegacyGeoShapeFieldMapper.DeprecatedParameters.Defaults.QUADTREE_LEVELS)); + assertThat(geoShapeFieldMapper.fieldType().pointsOnly(), + equalTo(LegacyGeoShapeFieldMapper.DeprecatedParameters.Defaults.POINTS_ONLY)); + assertThat(geoShapeFieldMapper.fieldType().distanceErrorPct(), + equalTo(LegacyGeoShapeFieldMapper.DeprecatedParameters.Defaults.DISTANCE_ERROR_PCT)); + assertThat(geoShapeFieldMapper.fieldType().orientation(), + equalTo(LegacyGeoShapeFieldMapper.Defaults.ORIENTATION.value())); + assertFieldWarnings("strategy"); + } + + /** + * Test that orientation parameter correctly parses + */ + public void testOrientationParsing() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("orientation", "left") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + ShapeBuilder.Orientation orientation = ((LegacyGeoShapeFieldMapper)fieldMapper).fieldType().orientation(); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.CLOCKWISE)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.LEFT)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.CW)); + + // explicit right orientation test + mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("orientation", "right") + .endObject().endObject() + .endObject().endObject()); + + defaultMapper = createIndex("test2").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + orientation = ((LegacyGeoShapeFieldMapper)fieldMapper).fieldType().orientation(); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.COUNTER_CLOCKWISE)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.RIGHT)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.CCW)); + assertFieldWarnings("tree"); + } + + /** + * Test that coerce parameter correctly parses + */ + public void testCoerceParsing() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("coerce", "true") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + boolean coerce = ((LegacyGeoShapeFieldMapper)fieldMapper).coerce().value(); + assertThat(coerce, equalTo(true)); + + // explicit false coerce test + mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("coerce", "false") + .endObject().endObject() + .endObject().endObject()); + + defaultMapper = createIndex("test2").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + coerce = ((LegacyGeoShapeFieldMapper)fieldMapper).coerce().value(); + assertThat(coerce, equalTo(false)); + assertFieldWarnings("tree"); + } + + + /** + * Test that accept_z_value parameter correctly parses + */ + public void testIgnoreZValue() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("strategy", "recursive") + .field(IGNORE_Z_VALUE.getPreferredName(), "true") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + boolean ignoreZValue = ((LegacyGeoShapeFieldMapper)fieldMapper).ignoreZValue().value(); + assertThat(ignoreZValue, equalTo(true)); + + // explicit false accept_z_value test + mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field(IGNORE_Z_VALUE.getPreferredName(), "false") + .endObject().endObject() + .endObject().endObject()); + + defaultMapper = createIndex("test2").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + ignoreZValue = ((LegacyGeoShapeFieldMapper)fieldMapper).ignoreZValue().value(); + assertThat(ignoreZValue, equalTo(false)); + assertFieldWarnings("strategy", "tree"); + } + + /** + * Test that ignore_malformed parameter correctly parses + */ + public void testIgnoreMalformedParsing() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("ignore_malformed", "true") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + Explicit ignoreMalformed = ((LegacyGeoShapeFieldMapper)fieldMapper).ignoreMalformed(); + assertThat(ignoreMalformed.value(), equalTo(true)); + + // explicit false ignore_malformed test + mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("ignore_malformed", "false") + .endObject().endObject() + .endObject().endObject()); + + defaultMapper = createIndex("test2").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + ignoreMalformed = ((LegacyGeoShapeFieldMapper)fieldMapper).ignoreMalformed(); + assertThat(ignoreMalformed.explicit(), equalTo(true)); + assertThat(ignoreMalformed.value(), equalTo(false)); + assertFieldWarnings("tree"); + } + + public void testGeohashConfiguration() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .field("tree_levels", "4") + .field("distance_error_pct", "0.1") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.1)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(4)); + assertFieldWarnings("tree", "tree_levels", "distance_error_pct"); + } + + public void testQuadtreeConfiguration() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("tree_levels", "6") + .field("distance_error_pct", "0.5") + .field("points_only", true) + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(6)); + assertThat(strategy.isPointsOnly(), equalTo(true)); + assertFieldWarnings("tree", "tree_levels", "distance_error_pct", "points_only"); + } + + private void assertFieldWarnings(String... fieldNames) { + String[] warnings = new String[fieldNames.length]; + for (int i = 0; i < fieldNames.length; ++i) { + warnings[i] = "Field parameter [" + fieldNames[i] + "] " + + "is deprecated and will be removed in a future version."; + } + assertWarnings(warnings); + } + + public void testLevelPrecisionConfiguration() throws IOException { + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("tree_levels", "6") + .field("precision", "70m") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + // 70m is more precise so it wins + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d))); + } + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("tree_levels", "26") + .field("precision", "70m") + .endObject().endObject() + .endObject().endObject()); + + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + // distance_error_pct was not specified so we expect the mapper to take the highest precision between "precision" and + // "tree_levels" setting distErrPct to 0 to guarantee desired precision + assertThat(strategy.getDistErrPct(), equalTo(0.0)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + // 70m is less precise so it loses + assertThat(strategy.getGrid().getMaxLevels(), equalTo(26)); + } + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .field("tree_levels", "6") + .field("precision", "70m") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + // 70m is more precise so it wins + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(70d))); + } + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .field("tree_levels", GeoUtils.geoHashLevelsForPrecision(70d)+1) + .field("precision", "70m") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(70d)+1)); + } + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("tree_levels", GeoUtils.quadTreeLevelsForPrecision(70d)+1) + .field("precision", "70m") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(70d)+1)); + } + assertFieldWarnings("tree", "tree_levels", "precision", "distance_error_pct"); + } + + public void testPointsOnlyOption() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .field("points_only", true) + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + assertThat(strategy.isPointsOnly(), equalTo(true)); + assertFieldWarnings("tree", "points_only"); + } + + public void testLevelDefaults() throws IOException { + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + /* 50m is default */ + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.quadTreeLevelsForPrecision(50d))); + } + + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .field("distance_error_pct", "0.5") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.5)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + /* 50m is default */ + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(50d))); + } + assertFieldWarnings("tree", "distance_error_pct"); + } + + public void testGeoShapeMapperMerge() throws Exception { + String stage1Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("shape").field("type", "geo_shape").field("tree", "geohash") + .field("strategy", "recursive") + .field("precision", "1m").field("tree_levels", 8).field("distance_error_pct", 0.01) + .field("orientation", "ccw") + .endObject().endObject().endObject().endObject()); + MapperService mapperService = createIndex("test").mapperService(); + DocumentMapper docMapper = mapperService.merge("type", new CompressedXContent(stage1Mapping), + MapperService.MergeReason.MAPPING_UPDATE); + String stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("shape").field("type", "geo_shape") + .field("tree", "quadtree") + .field("strategy", "term").field("precision", "1km") + .field("tree_levels", 26).field("distance_error_pct", 26) + .field("orientation", "cw").endObject().endObject().endObject().endObject()); + try { + mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("mapper [shape] has different [strategy]")); + assertThat(e.getMessage(), containsString("mapper [shape] has different [tree]")); + assertThat(e.getMessage(), containsString("mapper [shape] has different [tree_levels]")); + assertThat(e.getMessage(), containsString("mapper [shape] has different [precision]")); + } + + // verify nothing changed + Mapper fieldMapper = docMapper.mappers().getMapper("shape"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy, instanceOf(RecursivePrefixTreeStrategy.class)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + assertThat(strategy.getDistErrPct(), equalTo(0.01)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(1d))); + assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(ShapeBuilder.Orientation.CCW)); + + // correct mapping + stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("shape").field("type", "geo_shape") + .field("tree", "geohash") + .field("strategy", "recursive") + .field("precision", "1m") + .field("tree_levels", 8).field("distance_error_pct", 0.001) + .field("orientation", "cw").endObject().endObject().endObject().endObject()); + docMapper = mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); + + fieldMapper = docMapper.mappers().getMapper("shape"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy, instanceOf(RecursivePrefixTreeStrategy.class)); + assertThat(strategy.getGrid(), instanceOf(GeohashPrefixTree.class)); + assertThat(strategy.getDistErrPct(), equalTo(0.001)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(GeoUtils.geoHashLevelsForPrecision(1d))); + assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(ShapeBuilder.Orientation.CW)); + + assertFieldWarnings("tree", "strategy", "precision", "tree_levels", "distance_error_pct"); + } + + public void testEmptyName() throws Exception { + // after 5.x + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("") + .field("type", "geo_shape") + .field("tree", "quadtree") + .endObject().endObject() + .endObject().endObject()); + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> parser.parse("type1", new CompressedXContent(mapping)) + ); + assertThat(e.getMessage(), containsString("name cannot be empty string")); + assertFieldWarnings("tree"); + } + + public void testSerializeDefaults() throws Exception { + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .endObject().endObject() + .endObject().endObject()); + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"precision\":\"50.0m\"")); + assertTrue(serialized, serialized.contains("\"tree_levels\":21")); + } + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "geohash") + .endObject().endObject() + .endObject().endObject()); + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"precision\":\"50.0m\"")); + assertTrue(serialized, serialized.contains("\"tree_levels\":9")); + } + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("tree_levels", "6") + .endObject().endObject() + .endObject().endObject()); + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); + assertFalse(serialized, serialized.contains("\"precision\":")); + assertTrue(serialized, serialized.contains("\"tree_levels\":6")); + } + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("precision", "6") + .endObject().endObject() + .endObject().endObject()); + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); + assertTrue(serialized, serialized.contains("\"tree_levels\":10")); + } + { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("precision", "6m") + .field("tree_levels", "5") + .endObject().endObject() + .endObject().endObject()); + DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); + assertTrue(serialized, serialized.contains("\"tree_levels\":5")); + } + assertFieldWarnings("tree", "tree_levels", "precision"); + } + + public void testPointsOnlyDefaultsWithTermStrategy() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("precision", "10m") + .field("strategy", "term") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + LegacyGeoShapeFieldMapper geoShapeFieldMapper = (LegacyGeoShapeFieldMapper) fieldMapper; + PrefixTreeStrategy strategy = geoShapeFieldMapper.fieldType().defaultPrefixTreeStrategy(); + + assertThat(strategy.getDistErrPct(), equalTo(0.0)); + assertThat(strategy.getGrid(), instanceOf(QuadPrefixTree.class)); + assertThat(strategy.getGrid().getMaxLevels(), equalTo(23)); + assertThat(strategy.isPointsOnly(), equalTo(true)); + // term strategy changes the default for points_only, check that we handle it correctly + assertThat(toXContentString(geoShapeFieldMapper, false), not(containsString("points_only"))); + assertFieldWarnings("tree", "precision", "strategy"); + } + + + public void testPointsOnlyFalseWithTermStrategy() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("precision", "10m") + .field("strategy", "term") + .field("points_only", false) + .endObject().endObject() + .endObject().endObject()); + + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, + () -> parser.parse("type1", new CompressedXContent(mapping)) + ); + assertThat(e.getMessage(), containsString("points_only cannot be set to false for term strategy")); + assertFieldWarnings("tree", "precision", "strategy", "points_only"); + } + + public String toXContentString(LegacyGeoShapeFieldMapper mapper, boolean includeDefaults) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + ToXContent.Params params; + if (includeDefaults) { + params = new ToXContent.MapParams(Collections.singletonMap("include_defaults", "true")); + } else { + params = ToXContent.EMPTY_PARAMS; + } + mapper.doXContentBody(builder, includeDefaults, params); + return Strings.toString(builder.endObject()); + } + + public String toXContentString(LegacyGeoShapeFieldMapper mapper) throws IOException { + return toXContentString(mapper, true); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java new file mode 100644 index 0000000000000..7a8795aba051f --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.geo.SpatialStrategy; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper.GeoShapeFieldType; +import org.junit.Before; + +import java.io.IOException; + +public class LegacyGeoShapeFieldTypeTests extends FieldTypeTestCase { + @Override + protected MappedFieldType createDefaultFieldType() { + return new GeoShapeFieldType(); + } + + @Before + public void setupProperties() { + addModifier(new Modifier("tree", false) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldType)ft).setTree("quadtree"); + } + }); + addModifier(new Modifier("strategy", false) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldType)ft).setStrategy(SpatialStrategy.TERM); + } + }); + addModifier(new Modifier("tree_levels", false) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldType)ft).setTreeLevels(10); + } + }); + addModifier(new Modifier("precision", false) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldType)ft).setPrecisionInMeters(20); + } + }); + addModifier(new Modifier("distance_error_pct", true) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldType)ft).setDefaultDistanceErrorPct(0.5); + } + }); + addModifier(new Modifier("orientation", true) { + @Override + public void modify(MappedFieldType ft) { + ((GeoShapeFieldType)ft).setOrientation(ShapeBuilder.Orientation.LEFT); + } + }); + } + + /** + * Test for {@link LegacyGeoShapeFieldMapper.GeoShapeFieldType#setStrategy(SpatialStrategy)} that checks + * that {@link LegacyGeoShapeFieldMapper.GeoShapeFieldType#pointsOnly()} gets set as a side effect when using SpatialStrategy.TERM + */ + public void testSetStrategyName() throws IOException { + GeoShapeFieldType fieldType = new GeoShapeFieldType(); + assertFalse(fieldType.pointsOnly()); + fieldType.setStrategy(SpatialStrategy.RECURSIVE); + assertFalse(fieldType.pointsOnly()); + fieldType.setStrategy(SpatialStrategy.TERM); + assertTrue(fieldType.pointsOnly()); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java index b1cbb31c1f884..f19729336e6d3 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.elasticsearch.index.query; import org.apache.lucene.search.BooleanQuery; @@ -29,7 +28,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.geo.ShapeRelation; -import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.EnvelopeBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -54,33 +52,41 @@ public class GeoShapeQueryBuilderTests extends AbstractQueryTestCase { - private static String indexedShapeId; - private static String indexedShapeType; - private static String indexedShapePath; - private static String indexedShapeIndex; - private static String indexedShapeRouting; - private static ShapeBuilder indexedShapeToReturn; + protected static String indexedShapeId; + protected static String indexedShapeType; + protected static String indexedShapePath; + protected static String indexedShapeIndex; + protected static String indexedShapeRouting; + protected static ShapeBuilder indexedShapeToReturn; + + @Override + protected boolean enableWarningsCheck() { + return false; + } + + protected String fieldName() { + return GEO_SHAPE_FIELD_NAME; + } @Override protected GeoShapeQueryBuilder doCreateTestQueryBuilder() { return doCreateTestQueryBuilder(randomBoolean()); } - private GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { - ShapeType shapeType; - ShapeBuilder shape; + + protected GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { + // LatLonShape does not support MultiPoint queries + RandomShapeGenerator.ShapeType shapeType = + randomFrom(ShapeType.POINT, ShapeType.LINESTRING, ShapeType.MULTILINESTRING, ShapeType.POLYGON); + ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType); GeoShapeQueryBuilder builder; clearShapeFields(); if (indexedShape == false) { - shapeType = randomFrom(new ShapeType[] {ShapeType.POINT, ShapeType.LINESTRING, ShapeType.MULTILINESTRING, ShapeType.POLYGON}); - shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType); - builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); + builder = new GeoShapeQueryBuilder(fieldName(), shape); } else { - shapeType = ShapeType.randomType(random()); - shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType); indexedShapeToReturn = shape; indexedShapeId = randomAlphaOfLengthBetween(3, 20); indexedShapeType = randomAlphaOfLengthBetween(3, 20); - builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, indexedShapeId, indexedShapeType); + builder = new GeoShapeQueryBuilder(fieldName(), indexedShapeId, indexedShapeType); if (randomBoolean()) { indexedShapeIndex = randomAlphaOfLengthBetween(3, 20); builder.indexedShapeIndex(indexedShapeIndex); @@ -95,30 +101,12 @@ private GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { } } if (randomBoolean()) { - SpatialStrategy strategy = randomFrom(SpatialStrategy.values()); - // ShapeType.MULTILINESTRING + SpatialStrategy.TERM can lead to large queries and will slow down tests, so - // we try to avoid that combination - // Also, SpatialStrategy.VECTOR does not support MULTIPOINT Queries - while ((shapeType == ShapeType.MULTILINESTRING && strategy == SpatialStrategy.TERM) - || (strategy == SpatialStrategy.VECTOR && shapeType == ShapeType.MULTIPOINT)) { - strategy = randomFrom(SpatialStrategy.values()); - } - builder.strategy(strategy); - if (strategy == SpatialStrategy.VECTOR) { - // vector strategy does not yet support CONTAINS queries - ShapeRelation relation; - if (shapeType != ShapeType.LINESTRING && shapeType != ShapeType.MULTILINESTRING) { - // vector strategy does not yet support WITHIN for linestrings - relation = randomFrom(new ShapeRelation[] {ShapeRelation.INTERSECTS, ShapeRelation.DISJOINT, ShapeRelation.WITHIN}); - } else { - relation = randomFrom(new ShapeRelation[] {ShapeRelation.INTERSECTS, ShapeRelation.DISJOINT}); - } - builder.relation(relation); - } else if (strategy != SpatialStrategy.TERM) { - builder.relation(randomFrom(ShapeRelation.values())); + if (shapeType == ShapeType.LINESTRING || shapeType == ShapeType.MULTILINESTRING) { + builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS)); + } else { + // LatLonShape does not support CONTAINS: + builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.WITHIN)); } - } else if (shapeType == ShapeType.MULTIPOINT) { - builder.strategy(randomFrom(new SpatialStrategy[] {SpatialStrategy.TERM, SpatialStrategy.RECURSIVE})); } if (randomBoolean()) { @@ -149,8 +137,7 @@ protected GetResponse executeGet(GetRequest getRequest) { } catch (IOException ex) { throw new ElasticsearchException("boom", ex); } - return new GetResponse(new GetResult(indexedShapeIndex, indexedShapeType, indexedShapeId, - 0, true, new BytesArray(json), null)); + return new GetResponse(new GetResult(indexedShapeIndex, indexedShapeType, indexedShapeId, 0, true, new BytesArray(json), null)); } @After @@ -179,41 +166,28 @@ public void testNoFieldName() throws Exception { } public void testNoShape() throws IOException { - expectThrows(IllegalArgumentException.class, () -> new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, null)); + expectThrows(IllegalArgumentException.class, () -> new GeoShapeQueryBuilder(fieldName(), null)); } public void testNoIndexedShape() throws IOException { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, null, "type")); + () -> new GeoShapeQueryBuilder(fieldName(), null, "type")); assertEquals("either shapeBytes or indexedShapeId and indexedShapeType are required", e.getMessage()); } public void testNoIndexedShapeType() throws IOException { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, "id", null)); + () -> new GeoShapeQueryBuilder(fieldName(), "id", null)); assertEquals("indexedShapeType is required if indexedShapeId is specified", e.getMessage()); } public void testNoRelation() throws IOException { ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null); - GeoShapeQueryBuilder builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); + GeoShapeQueryBuilder builder = new GeoShapeQueryBuilder(fieldName(), shape); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.relation(null)); assertEquals("No Shape Relation defined", e.getMessage()); } - public void testInvalidRelation() throws IOException { - ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null); - GeoShapeQueryBuilder builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); - builder.strategy(SpatialStrategy.TERM); - expectThrows(IllegalArgumentException.class, () -> builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN))); - GeoShapeQueryBuilder builder2 = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); - builder2.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN)); - expectThrows(IllegalArgumentException.class, () -> builder2.strategy(SpatialStrategy.TERM)); - GeoShapeQueryBuilder builder3 = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); - builder3.strategy(SpatialStrategy.TERM); - expectThrows(IllegalArgumentException.class, () -> builder3.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN))); - } - // see #3878 public void testThatXContentSerializationInsideOfArrayWorks() throws Exception { EnvelopeBuilder envelopeBuilder = new EnvelopeBuilder(new Coordinate(0, 0), new Coordinate(10, 10)); @@ -223,7 +197,7 @@ public void testThatXContentSerializationInsideOfArrayWorks() throws Exception { public void testFromJson() throws IOException { String json = - "{\n" + + "{\n" + " \"geo_shape\" : {\n" + " \"location\" : {\n" + " \"shape\" : {\n" + @@ -248,7 +222,7 @@ public void testMustRewrite() throws IOException { UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, () -> query.toQuery(createShardContext())); assertEquals("query must be rewritten first", e.getMessage()); QueryBuilder rewrite = rewriteAndFetch(query, createShardContext()); - GeoShapeQueryBuilder geoShapeQueryBuilder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, indexedShapeToReturn); + GeoShapeQueryBuilder geoShapeQueryBuilder = new GeoShapeQueryBuilder(fieldName(), indexedShapeToReturn); geoShapeQueryBuilder.strategy(query.strategy()); geoShapeQueryBuilder.relation(query.relation()); assertEquals(geoShapeQueryBuilder, rewrite); @@ -262,7 +236,7 @@ public void testMultipleRewrite() throws IOException { builder = rewriteAndFetch(builder, createShardContext()); - GeoShapeQueryBuilder expectedShape = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, indexedShapeToReturn); + GeoShapeQueryBuilder expectedShape = new GeoShapeQueryBuilder(fieldName(), indexedShapeToReturn); expectedShape.strategy(shape.strategy()); expectedShape.relation(shape.relation()); QueryBuilder expected = new BoolQueryBuilder() diff --git a/server/src/test/java/org/elasticsearch/index/query/LegacyGeoShapeFieldQueryTests.java b/server/src/test/java/org/elasticsearch/index/query/LegacyGeoShapeFieldQueryTests.java new file mode 100644 index 0000000000000..3cf6f2031810a --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/query/LegacyGeoShapeFieldQueryTests.java @@ -0,0 +1,94 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.query; + +import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.geo.SpatialStrategy; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.test.geo.RandomShapeGenerator; +import org.elasticsearch.test.geo.RandomShapeGenerator.ShapeType; + +import java.io.IOException; + +public class LegacyGeoShapeFieldQueryTests extends GeoShapeQueryBuilderTests { + + @Override + protected String fieldName() { + return LEGACY_GEO_SHAPE_FIELD_NAME; + } + + @Override + protected GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { + ShapeType shapeType = ShapeType.randomType(random()); + ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType); + GeoShapeQueryBuilder builder; + clearShapeFields(); + if (indexedShape == false) { + builder = new GeoShapeQueryBuilder(fieldName(), shape); + } else { + indexedShapeToReturn = shape; + indexedShapeId = randomAlphaOfLengthBetween(3, 20); + indexedShapeType = randomAlphaOfLengthBetween(3, 20); + builder = new GeoShapeQueryBuilder(fieldName(), indexedShapeId, indexedShapeType); + if (randomBoolean()) { + indexedShapeIndex = randomAlphaOfLengthBetween(3, 20); + builder.indexedShapeIndex(indexedShapeIndex); + } + if (randomBoolean()) { + indexedShapePath = randomAlphaOfLengthBetween(3, 20); + builder.indexedShapePath(indexedShapePath); + } + if (randomBoolean()) { + indexedShapeRouting = randomAlphaOfLengthBetween(3, 20); + builder.indexedShapeRouting(indexedShapeRouting); + } + } + if (randomBoolean()) { + SpatialStrategy strategy = randomFrom(SpatialStrategy.values()); + // ShapeType.MULTILINESTRING + SpatialStrategy.TERM can lead to large queries and will slow down tests, so + // we try to avoid that combination + while (shapeType == ShapeType.MULTILINESTRING && strategy == SpatialStrategy.TERM) { + strategy = randomFrom(SpatialStrategy.values()); + } + builder.strategy(strategy); + if (strategy != SpatialStrategy.TERM) { + builder.relation(randomFrom(ShapeRelation.values())); + } + } + + if (randomBoolean()) { + builder.ignoreUnmapped(randomBoolean()); + } + return builder; + } + + public void testInvalidRelation() throws IOException { + ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null); + GeoShapeQueryBuilder builder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); + builder.strategy(SpatialStrategy.TERM); + expectThrows(IllegalArgumentException.class, () -> builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN))); + GeoShapeQueryBuilder builder2 = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); + builder2.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN)); + expectThrows(IllegalArgumentException.class, () -> builder2.strategy(SpatialStrategy.TERM)); + GeoShapeQueryBuilder builder3 = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, shape); + builder3.strategy(SpatialStrategy.TERM); + expectThrows(IllegalArgumentException.class, () -> builder3.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.WITHIN))); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java index 80aa5030d4a49..6181f8fc8b6c6 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java @@ -45,21 +45,21 @@ public class GeoShapeIntegrationIT extends ESIntegTestCase { public void testOrientationPersistence() throws Exception { String idxName = "orientation"; String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("shape") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("orientation", "left") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("orientation", "left") + .endObject().endObject() + .endObject().endObject()); // create index assertAcked(prepareCreate(idxName).addMapping("shape", mapping, XContentType.JSON)); mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("shape") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .field("orientation", "right") - .endObject().endObject() - .endObject().endObject()); + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("orientation", "right") + .endObject().endObject() + .endObject().endObject()); assertAcked(prepareCreate(idxName+"2").addMapping("shape", mapping, XContentType.JSON)); ensureGreen(idxName, idxName+"2"); diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java index 53f9260bcb88f..26866ec034054 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java @@ -41,7 +41,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.index.query.GeoShapeQueryBuilder; @@ -71,8 +71,8 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase { private static final String[] PREFIX_TREES = new String[] { - GeoShapeFieldMapper.Names.TREE_GEOHASH, - GeoShapeFieldMapper.Names.TREE_QUADTREE + LegacyGeoShapeFieldMapper.DeprecatedParameters.PrefixTrees.GEOHASH, + LegacyGeoShapeFieldMapper.DeprecatedParameters.PrefixTrees.QUADTREE }; private XContentBuilder createMapping() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java b/server/src/test/java/org/elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java new file mode 100644 index 0000000000000..f705bd59b7450 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java @@ -0,0 +1,170 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.geo; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.test.ESIntegTestCase; + +import static org.elasticsearch.index.query.QueryBuilders.geoShapeQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class LegacyGeoShapeIntegrationIT extends ESIntegTestCase { + + /** + * Test that orientation parameter correctly persists across cluster restart + */ + public void testOrientationPersistence() throws Exception { + String idxName = "orientation"; + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("shape") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("orientation", "left") + .endObject().endObject() + .endObject().endObject()); + + // create index + assertAcked(prepareCreate(idxName).addMapping("shape", mapping, XContentType.JSON)); + + mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("shape") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("orientation", "right") + .endObject().endObject() + .endObject().endObject()); + + assertAcked(prepareCreate(idxName+"2").addMapping("shape", mapping, XContentType.JSON)); + ensureGreen(idxName, idxName+"2"); + + internalCluster().fullRestart(); + ensureGreen(idxName, idxName+"2"); + + // left orientation test + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, findNodeName(idxName)); + IndexService indexService = indicesService.indexService(resolveIndex(idxName)); + MappedFieldType fieldType = indexService.mapperService().fullName("location"); + assertThat(fieldType, instanceOf(LegacyGeoShapeFieldMapper.GeoShapeFieldType.class)); + + LegacyGeoShapeFieldMapper.GeoShapeFieldType gsfm = (LegacyGeoShapeFieldMapper.GeoShapeFieldType)fieldType; + ShapeBuilder.Orientation orientation = gsfm.orientation(); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.CLOCKWISE)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.LEFT)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.CW)); + + // right orientation test + indicesService = internalCluster().getInstance(IndicesService.class, findNodeName(idxName+"2")); + indexService = indicesService.indexService(resolveIndex((idxName+"2"))); + fieldType = indexService.mapperService().fullName("location"); + assertThat(fieldType, instanceOf(LegacyGeoShapeFieldMapper.GeoShapeFieldType.class)); + + gsfm = (LegacyGeoShapeFieldMapper.GeoShapeFieldType)fieldType; + orientation = gsfm.orientation(); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.COUNTER_CLOCKWISE)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.RIGHT)); + assertThat(orientation, equalTo(ShapeBuilder.Orientation.CCW)); + } + + /** + * Test that ignore_malformed on GeoShapeFieldMapper does not fail the entire document + */ + public void testIgnoreMalformed() throws Exception { + // create index + assertAcked(client().admin().indices().prepareCreate("test") + .addMapping("geometry", "shape", "type=geo_shape,tree=quadtree,ignore_malformed=true").get()); + ensureGreen(); + + // test self crossing ccw poly not crossing dateline + String polygonGeoJson = Strings.toString(XContentFactory.jsonBuilder().startObject().field("type", "Polygon") + .startArray("coordinates") + .startArray() + .startArray().value(176.0).value(15.0).endArray() + .startArray().value(-177.0).value(10.0).endArray() + .startArray().value(-177.0).value(-10.0).endArray() + .startArray().value(176.0).value(-15.0).endArray() + .startArray().value(-177.0).value(15.0).endArray() + .startArray().value(172.0).value(0.0).endArray() + .startArray().value(176.0).value(15.0).endArray() + .endArray() + .endArray() + .endObject()); + + indexRandom(true, client().prepareIndex("test", "geometry", "0").setSource("shape", + polygonGeoJson)); + SearchResponse searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).get(); + assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); + } + + /** + * Test that the indexed shape routing can be provided if it is required + */ + public void testIndexShapeRouting() throws Exception { + String mapping = "{\n" + + " \"_routing\": {\n" + + " \"required\": true\n" + + " },\n" + + " \"properties\": {\n" + + " \"shape\": {\n" + + " \"type\": \"geo_shape\",\n" + + " \"tree\" : \"quadtree\"\n" + + " }\n" + + " }\n" + + " }"; + + + // create index + assertAcked(client().admin().indices().prepareCreate("test").addMapping("doc", mapping, XContentType.JSON).get()); + ensureGreen(); + + String source = "{\n" + + " \"shape\" : {\n" + + " \"type\" : \"bbox\",\n" + + " \"coordinates\" : [[-45.0, 45.0], [45.0, -45.0]]\n" + + " }\n" + + "}"; + + indexRandom(true, client().prepareIndex("test", "doc", "0").setSource(source, XContentType.JSON).setRouting("ABC")); + + SearchResponse searchResponse = client().prepareSearch("test").setQuery( + geoShapeQuery("shape", "0", "doc").indexedShapeIndex("test").indexedShapeRouting("ABC") + ).get(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); + } + + private String findNodeName(String index) { + ClusterState state = client().admin().cluster().prepareState().get().getState(); + IndexShardRoutingTable shard = state.getRoutingTable().index(index).shard(0); + String nodeId = shard.assignedShards().get(0).currentNodeId(); + return state.getNodes().get(nodeId).getName(); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 5eef0a249b687..fc1c3ea6a6454 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -113,6 +113,7 @@ public abstract class AbstractBuilderTestCase extends ESTestCase { protected static final String GEO_POINT_FIELD_NAME = "mapped_geo_point"; protected static final String GEO_POINT_ALIAS_FIELD_NAME = "mapped_geo_point_alias"; protected static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; + protected static final String LEGACY_GEO_SHAPE_FIELD_NAME = "mapped_legacy_geo_shape"; protected static final String[] MAPPED_FIELD_NAMES = new String[]{STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, INT_FIELD_NAME, INT_RANGE_FIELD_NAME, DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, DATE_RANGE_FIELD_NAME, OBJECT_FIELD_NAME, GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, @@ -396,7 +397,8 @@ public void onRemoval(ShardId shardId, Accountable accountable) { OBJECT_FIELD_NAME, "type=object", GEO_POINT_FIELD_NAME, "type=geo_point", GEO_POINT_ALIAS_FIELD_NAME, "type=alias,path=" + GEO_POINT_FIELD_NAME, - GEO_SHAPE_FIELD_NAME, "type=geo_shape" + GEO_SHAPE_FIELD_NAME, "type=geo_shape", + LEGACY_GEO_SHAPE_FIELD_NAME, "type=geo_shape,tree=quadtree" ))), MapperService.MergeReason.MAPPING_UPDATE); // also add mappings for two inner field in the object field mapperService.merge("_doc", new CompressedXContent("{\"properties\":{\"" + OBJECT_FIELD_NAME + "\":{\"type\":\"object\"," From c2b5bf5ec302afee54d11d013cacfeaafcf72d54 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Mon, 19 Nov 2018 11:30:42 -0600 Subject: [PATCH 16/38] update docs to remove vector strategy --- .../mapping/types/geo-shape.asciidoc | 95 +++++++++++-------- .../migration/migrate_7_0/mappings.asciidoc | 12 +-- 2 files changed, 61 insertions(+), 46 deletions(-) diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index a4d62a365186f..f16c41bb18396 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -21,12 +21,12 @@ type. |======================================================================= |Option |Description| Default -|`tree |deprecated[6.6, use `vector` strategy] Name of the PrefixTree +|`tree |deprecated[6.6, PrefixTrees no longer used] Name of the PrefixTree implementation to be used: `geohash` for GeohashPrefixTree and `quadtree` for QuadPrefixTree. | `geohash` -|`precision` |deprecated[6.6, use `vector` strategy] This parameter may +|`precision` |deprecated[6.6, PrefixTrees no longer used] This parameter may be used instead of `tree_levels` to set an appropriate value for the `tree_levels` parameter. The value specifies the desired precision and Elasticsearch will calculate the best tree_levels value to honor this @@ -36,7 +36,7 @@ unit. Valid distance units include: `in`, `inch`, `yd`, `yard`, `mi`, `millimeters`. | `50m` -|`tree_levels` |deprecated[6.6, use `vector` strategy] Maximum number +|`tree_levels` |deprecated[6.6, PrefixTrees no longer used] Maximum number of layers to be used by the PrefixTree. This can be used to control the precision of shape representations andtherefore how many terms are indexed. Defaults to the default value of the chosen PrefixTree @@ -47,31 +47,29 @@ tree_levels parameter internally and this is what is returned via the mapping API even if you use the precision parameter. | various -|`strategy` |deprecated[6.6, use `vector` strategy] The strategy +|`strategy` |deprecated[6.6, PrefixTrees no longer used] The strategy parameter defines the approach for how to represent shapes at indexing and search time. It also influences the capabilities available so it is recommended to let Elasticsearch set this parameter automatically. -There are three strategies available: `recursive`, `term`, and `vector`. -Vector strategy is the default. This strategy provides near perfect -resolution (down to 1e-6 decimal degree precision) by indexing shapes -as vectory types. It is the preferred method of shape indexing. +There are three strategies available: `recursive`, and `term`. Recursive and Term strategies are deprecated and will be removed in a future version. While they are still available, the Term strategy supports point types only (the `points_only` parameter will be automatically set to true) while Recursive strategy supports all shape types. (IMPORTANT: see <> for more detailed information about these strategies) -| `vector` - -|`distance_error_pct` |Used as a hint to the PrefixTree about how -precise it should be. Defaults to 0.025 (2.5%) with 0.5 as the maximum -supported value. PERFORMANCE NOTE: This value will default to 0 if a `precision` or -`tree_level` definition is explicitly defined. This guarantees spatial precision -at the level defined in the mapping. This can lead to significant memory usage -for high resolution shapes with low error (e.g., large shapes at 1m with < 0.001 error). -To improve indexing performance (at the cost of query accuracy) explicitly define -`tree_level` or `precision` along with a reasonable `distance_error_pct`, noting -that large shapes will have greater false positives. +| `recursive` + +|`distance_error_pct` |deprecated[6.6, PrefixTrees no longer used] Used as a +hint to the PrefixTree about how precise it should be. Defaults to 0.025 (2.5%) +with 0.5 as the maximum supported value. PERFORMANCE NOTE: This value will +default to 0 if a `precision` or `tree_level` definition is explicitly defined. +This guarantees spatial precision at the level defined in the mapping. This can +lead to significant memory usage for high resolution shapes with low error +(e.g., large shapes at 1m with < 0.001 error). To improve indexing performance +(at the cost of query accuracy) explicitly define `tree_level` or `precision` +along with a reasonable `distance_error_pct`, noting that large shapes will have +greater false positives. | `0.025` |`orientation` |Optionally define how to interpret vertex order for @@ -86,7 +84,7 @@ sets vertex order for the coordinate list of a geo_shape field but can be overridden in each individual GeoJSON or WKT document. | `ccw` -|`points_only` |deprecated[6.6, use `vector` strategy] Setting this option to +|`points_only` |deprecated[6.6, PrefixTrees no longer used] Setting this option to `true` (defaults to `false`) configures the `geo_shape` field type for point shapes only (NOTE: Multi-Points are not yet supported). This optimizes index and search performance for the `geohash` and `quadtree` when it is known that only points @@ -109,19 +107,33 @@ and reject the whole document. |======================================================================= + +[[indexing-approach]] +[float] +==== Indexing approach +GeoShape types are indexed by decomposing the shape into a triangular mesh and +indexing each triangle as a 7 dimension point in a BKD tree. This provides +near perfect spatial resolution (down to 1e-6 decimal degree precision) since all +spatial relations are computed using an encoded vector representation of the +original shape instead of a raster-grid representation as used by the +<> indexing approach. While this is the default indexing technique +prefix trees can still be used by setting the `tree` or `strategy` parameters +according to the appropriate <>. Note that these parameters +are now deprecated and will be removed in a future version. + [[prefix-trees]] [float] ==== Prefix trees -deprecated[6.6, use `vector` strategy] To efficiently represent shapes in +deprecated[6.6, PrefixTrees no longer used] To efficiently represent shapes in an inverted index, Shapes are converted into a series of hashes representing grid squares (commonly referred to as "rasters") using implementations of a PrefixTree. The tree notion comes from the fact that the PrefixTree uses multiple grid layers, each with an increasing level of precision to represent the Earth. This can be thought of as increasing the level of detail of a map or image at higher zoom levels. Since this approach causes precision issues with indexed shape, it has -been deprecated in favor of a vector approach that indexes the shapes as a triangular -mesh. +been deprecated in favor of a vector indexing approach that indexes the shapes as a +triangular mesh (see <>). Multiple PrefixTree implementations are provided: @@ -143,9 +155,10 @@ number of levels for the quad trees in Elasticsearch is 29; the default is 21. [[spatial-strategy]] [float] ===== Spatial strategies -The indexing implementation selected relies on a SpatialStrategy for choosing -how to decompose the shapes (either as grid squares or a tessellated triangular -mesh). Each strategy answers the following: +deprecated[6.6, PrefixTrees no longer used] The indexing implementation +selected relies on a SpatialStrategy for choosing how to decompose the shapes +(either as grid squares or a tessellated triangular mesh). Each strategy +answers the following: * What type of Shapes can be indexed? * What types of Query Operations and Shapes can be used? @@ -158,9 +171,8 @@ are provided: |======================================================================= |Strategy |Supported Shapes |Supported Queries |Multiple Shapes -|`vector` | <> | `INTERSECTS`, `DISJOINT`, `WITHIN` |Yes -|`recursive` deprecated[6.6, use `vector` strategy] |<> |`INTERSECTS`, `DISJOINT`, `WITHIN`, `CONTAINS` |Yes -|`term` deprecated[6.6, use `vector` strategy] |<> |`INTERSECTS` |Yes +|`recursive` |<> |`INTERSECTS`, `DISJOINT`, `WITHIN`, `CONTAINS` |Yes +|`term` |<> |`INTERSECTS` |Yes |======================================================================= @@ -203,15 +215,16 @@ approximately 1e-6 decimal degree precision. [float] ===== Performance considerations with Prefix Trees -With prefix trees, Elasticsearch uses the paths in the tree as terms in -the inverted index and in queries. The higher the level (and thus the -precision), the more terms are generated. Of course, calculating the -terms, keeping them in memory, and storing them on disk all have a price. -Especially with higher tree levels, indices can become extremely large -even with a modest amount of data. Additionally, the size of the features -also matters. Big, complex polygons can take up a lot of space at higher -tree levels. Which setting is right depends on the use case. Generally one -trades off accuracy against index size and query performance. +deprecated[6.6, PrefixTrees no longer used] With prefix trees, +Elasticsearch uses the paths in the tree as terms in the inverted index +and in queries. The higher the level (and thus the precision), the more +terms are generated. Of course, calculating the terms, keeping them in +memory, and storing them on disk all have a price. Especially with higher +tree levels, indices can become extremely large even with a modest amount +of data. Additionally, the size of the features also matters. Big, complex +polygons can take up a lot of space at higher tree levels. Which setting +is right depends on the use case. Generally one trades off accuracy against +index size and query performance. The defaults in Elasticsearch for both implementations are a compromise between index size and a reasonable level of precision of 50m at the @@ -609,8 +622,10 @@ POST /example/doc ===== Circle Elasticsearch supports a `circle` type, which consists of a center -point with a radius. Note that circles can only be indexed when using -the `recursive` Prefix Tree strategy only. +point with a radius. Note that this circle representation can only +be indexed when using the `recursive` Prefix Tree strategy. For +the default <> circles should be approximated using +a `POLYGON`. [source,js] -------------------------------------------------- diff --git a/docs/reference/migration/migrate_7_0/mappings.asciidoc b/docs/reference/migration/migrate_7_0/mappings.asciidoc index 3cda7fc84d844..f08ea3ab89c1d 100644 --- a/docs/reference/migration/migrate_7_0/mappings.asciidoc +++ b/docs/reference/migration/migrate_7_0/mappings.asciidoc @@ -54,13 +54,13 @@ An error will now be thrown when unknown configuration options are provided to similarities. Such unknown parameters were ignored before. [float] -==== `geo_shape` defaults to `vector` strategy +==== deprecated `geo_shape` Prefix Tree indexing -`geo_shape` types now default to using the `vector` indexing strategy. This indexes -shapes as a triangular mesh instead of decomposing them into individual grid cells. -To index using legacy prefix trees `recursive` or `term` strategy must be explicitly -defined. Note that these strategies are now deprecated and will be removed in a future -version. +`geo_shape` types now default to using a vector indexing approach based on Lucene's new +`LatLonShape` field type. This indexes shapes as a triangular mesh instead of decomposing +them into individual grid cells. To index using legacy prefix trees `recursive` or `term` +strategy must be explicitly defined. Note that these strategies are now deprecated and will +be removed in a future version. [float] ==== deprecated `geo_shape` parameters From 24bff63ddd731024659a30bf1b12d32b685bd455 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Wed, 21 Nov 2018 14:19:17 -0600 Subject: [PATCH 17/38] fix GeometryCollectionBuilder#buildLucene to return the object created by the shape builder --- .../common/geo/builders/GeometryCollectionBuilder.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java index b6e94c012c603..fdf7073bd7454 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java @@ -197,9 +197,6 @@ public Object buildLucene() { } } - if (shapes.size() == 1) { - return shapes.get(0); - } return shapes.toArray(new Object[shapes.size()]); } From fea0972fd5a74005394d31bde96ae48539d4fbed Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Wed, 21 Nov 2018 14:38:16 -0600 Subject: [PATCH 18/38] fix LineLength failure in GeoJsonShapeParserTests --- .../org/elasticsearch/common/geo/GeoJsonShapeParserTests.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java index f3e55805af588..fd2b90df8f6a6 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java @@ -296,7 +296,9 @@ public void testParse3DPolygon() throws IOException { LinearRing shell = GEOMETRY_FACTORY.createLinearRing(shellCoordinates.toArray(new Coordinate[shellCoordinates.size()])); Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, null); Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final LegacyGeoShapeFieldMapper mapperBuilder = (LegacyGeoShapeFieldMapper) (new LegacyGeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext)); try (XContentParser parser = createParser(polygonGeoJson)) { + final LegacyGeoShapeFieldMapper mapperBuilder = + (LegacyGeoShapeFieldMapper) (new LegacyGeoShapeFieldMapper.Builder("test").ignoreZValue(true).build(mockBuilderContext)); + try (XContentParser parser = createParser(polygonGeoJson)) { parser.nextToken(); ElasticsearchGeoAssertions.assertEquals(jtsGeom(expected), ShapeParser.parse(parser, mapperBuilder).buildS4J()); } From f323b4318cecb493fbc8aca1083e430b74936c3d Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Mon, 10 Dec 2018 11:50:25 -0600 Subject: [PATCH 19/38] ShapeMapper refactor changes from PR feedback --- .../mapping/types/geo-shape.asciidoc | 20 +++++++++++-------- .../index/mapper/GeoShapeFieldMapper.java | 2 +- .../index/query/GeoShapeQueryBuilder.java | 13 +++++++++--- .../common/geo/GeoJsonShapeParserTests.java | 1 - .../common/geo/GeoWKTShapeParserTests.java | 7 ++++--- .../index/mapper/ExternalMapper.java | 2 +- 6 files changed, 28 insertions(+), 17 deletions(-) diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index f16c41bb18396..b758f2f72c2fb 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -23,8 +23,9 @@ type. |`tree |deprecated[6.6, PrefixTrees no longer used] Name of the PrefixTree implementation to be used: `geohash` for GeohashPrefixTree and `quadtree` -for QuadPrefixTree. -| `geohash` +for QuadPrefixTree. Note: This parameter is only relevant for `term` and +`recursive` strategies. +| `quadtree` |`precision` |deprecated[6.6, PrefixTrees no longer used] This parameter may be used instead of `tree_levels` to set an appropriate value for the @@ -33,7 +34,8 @@ Elasticsearch will calculate the best tree_levels value to honor this precision. The value should be a number followed by an optional distance unit. Valid distance units include: `in`, `inch`, `yd`, `yard`, `mi`, `miles`, `km`, `kilometers`, `m`,`meters`, `cm`,`centimeters`, `mm`, -`millimeters`. +`millimeters`. Note: This parameter is only relevant for `term` and +`recursive` strategies. | `50m` |`tree_levels` |deprecated[6.6, PrefixTrees no longer used] Maximum number @@ -44,7 +46,8 @@ implementation. Since this parameter requires a certain level of understanding of the underlying implementation, users may use the `precision` parameter instead. However, Elasticsearch only uses the tree_levels parameter internally and this is what is returned via the -mapping API even if you use the precision parameter. +mapping API even if you use the precision parameter. Note: This parameter +is only relevant for `term` and `recursive` strategies. | various |`strategy` |deprecated[6.6, PrefixTrees no longer used] The strategy @@ -69,7 +72,8 @@ lead to significant memory usage for high resolution shapes with low error (e.g., large shapes at 1m with < 0.001 error). To improve indexing performance (at the cost of query accuracy) explicitly define `tree_level` or `precision` along with a reasonable `distance_error_pct`, noting that large shapes will have -greater false positives. +greater false positives. Note: This parameter is only relevant for `term` and +`recursive` strategies. | `0.025` |`orientation` |Optionally define how to interpret vertex order for @@ -108,12 +112,12 @@ and reject the whole document. |======================================================================= -[[indexing-approach]] +[[geoshape-indexing-approach]] [float] ==== Indexing approach GeoShape types are indexed by decomposing the shape into a triangular mesh and indexing each triangle as a 7 dimension point in a BKD tree. This provides -near perfect spatial resolution (down to 1e-6 decimal degree precision) since all +near perfect spatial resolution (down to 1e-7 decimal degree precision) since all spatial relations are computed using an encoded vector representation of the original shape instead of a raster-grid representation as used by the <> indexing approach. While this is the default indexing technique @@ -210,7 +214,7 @@ PUT /example This mapping definition maps the location field to the geo_shape type using the default vector implementation. It provides -approximately 1e-6 decimal degree precision. +approximately 1e-7 decimal degree precision. [float] ===== Performance considerations with Prefix Trees diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 1ba9aa2f59cc2..02fb24a0de146 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -159,7 +159,7 @@ private void indexShape(ParseContext context, Object luceneShape) { indexShape(context, o); } } else { - throw new IllegalArgumentException("invalid shape type found [" + luceneShape.getClass().getName() + "] while indexing shape"); + throw new IllegalArgumentException("invalid shape type found [" + luceneShape.toString() + "] while indexing shape"); } } diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java index e16f178b78c22..3cec5bfc5f16a 100644 --- a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java @@ -448,9 +448,16 @@ private Query getVectorQueryFromShape(QueryShardContext context, Object querySha throw new QueryShardException(context, "Field [" + fieldName + "] does not support " + GeoShapeType.MULTIPOINT + " queries"); } else if (queryShape instanceof double[] || queryShape instanceof GeoPoint) { // for now just create a single bounding box query with min values == max values - double[] pt = queryShape instanceof GeoPoint - ? new double[] {((GeoPoint)queryShape).lon(), ((GeoPoint)queryShape).lat()} - : (double[])queryShape; + double[] pt; + if (queryShape instanceof GeoPoint) { + pt = new double[] {((GeoPoint)queryShape).lon(), ((GeoPoint)queryShape).lat()}; + } else { + pt = (double[])queryShape; + if (pt.length != 2) { + throw new QueryShardException(context, "Expected double array of length 2. " + + "But found length " + pt.length + " for field [" + fieldName + "]"); + } + } return LatLonShape.newBoxQuery(fieldName, relation.getLuceneRelation(), pt[1], pt[1], pt[0], pt[0]); } else if (queryShape instanceof Object[]) { geoQuery = createGeometryCollectionQuery(context, (Object[]) queryShape); diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java index 0bdfc31861f80..2acabee8797f4 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java @@ -1077,7 +1077,6 @@ public void testParseGeometryCollection() throws IOException { .endObject() .endArray() .endObject(); - boolean useJTS = randomBoolean(); ArrayList shellCoordinates1 = new ArrayList<>(); shellCoordinates1.add(new Coordinate(180.0, -12.142857142857142)); diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java index 4a0a946a5304f..94c96e00d9236 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java @@ -42,6 +42,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.test.geo.RandomShapeGenerator; @@ -278,14 +279,14 @@ public void testParseMixedDimensionPolyWithHole() throws IOException { parser.nextToken(); Settings indexSettings = Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_6_3_0) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_7_0_0) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build(); Mapper.BuilderContext mockBuilderContext = new Mapper.BuilderContext(indexSettings, new ContentPath()); - final LegacyGeoShapeFieldMapper mapperBuilder = - (LegacyGeoShapeFieldMapper)(new LegacyGeoShapeFieldMapper.Builder("test").ignoreZValue(false).build(mockBuilderContext)); + final GeoShapeFieldMapper mapperBuilder = + (GeoShapeFieldMapper) (new GeoShapeFieldMapper.Builder("test").ignoreZValue(false).build(mockBuilderContext)); // test store z disabled ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index 28638125309bd..20c49c00935e3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -87,7 +87,7 @@ public ExternalMapper build(BuilderContext context) { BinaryFieldMapper binMapper = binBuilder.build(context); BooleanFieldMapper boolMapper = boolBuilder.build(context); GeoPointFieldMapper pointMapper = latLonPointBuilder.build(context); - BaseGeoShapeFieldMapper shapeMapper = (context.indexCreatedVersion().before(Version.V_6_0_0)) + BaseGeoShapeFieldMapper shapeMapper = (context.indexCreatedVersion().before(Version.V_6_6_0)) ? legacyShapeBuilder.build(context) : shapeBuilder.build(context); FieldMapper stringMapper = (FieldMapper)stringBuilder.build(context); From fb389e04367fe884f1a4c1aac02420522504285b Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Mon, 10 Dec 2018 11:58:54 -0600 Subject: [PATCH 20/38] fix typo in geo-shape.asciidoc --- docs/reference/mapping/types/geo-shape.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index b758f2f72c2fb..cf023142948fd 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -54,7 +54,7 @@ is only relevant for `term` and `recursive` strategies. parameter defines the approach for how to represent shapes at indexing and search time. It also influences the capabilities available so it is recommended to let Elasticsearch set this parameter automatically. -There are three strategies available: `recursive`, and `term`. +There are two strategies available: `recursive`, and `term`. Recursive and Term strategies are deprecated and will be removed in a future version. While they are still available, the Term strategy supports point types only (the `points_only` parameter will be From 653f6fc2c5d4b23eaf3329165f5a6bc66db5b29e Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Mon, 10 Dec 2018 12:23:18 -0600 Subject: [PATCH 21/38] ignore circle test in docs --- docs/reference/mapping/types/geo-shape.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index cf023142948fd..f7c9bcb3cedee 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -643,6 +643,7 @@ POST /example/doc } -------------------------------------------------- // CONSOLE +// TEST[skip:not supported in default] Note: The inner `radius` field is required. If not specified, then the units of the `radius` will default to `METERS`. From b129a3b0b2ef5f3565daed14cc648aae613f4eff Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Mon, 10 Dec 2018 12:35:35 -0600 Subject: [PATCH 22/38] update indexing-approach ref to geoshape-indexing-approach --- docs/reference/mapping/types/geo-shape.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index f7c9bcb3cedee..5c8a15b5f7d7f 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -137,7 +137,7 @@ grid layers, each with an increasing level of precision to represent the Earth. This can be thought of as increasing the level of detail of a map or image at higher zoom levels. Since this approach causes precision issues with indexed shape, it has been deprecated in favor of a vector indexing approach that indexes the shapes as a -triangular mesh (see <>). +triangular mesh (see <>). Multiple PrefixTree implementations are provided: From 819b0b844b9ac2a2b8af61b08fa095c0486a529d Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Mon, 10 Dec 2018 13:01:52 -0600 Subject: [PATCH 23/38] add warnings check for LegacyGeoShapeFieldMapper to AbstractBuilderTestCase --- .../elasticsearch/test/AbstractBuilderTestCase.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index fc1c3ea6a6454..4b6d83e985c9b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -218,12 +218,25 @@ public void beforeTest() throws Exception { AbstractBuilderTestCase.this, false); return null; }); + assertDeprecatedGeoWarnings(); } serviceHolder.clientInvocationHandler.delegate = this; serviceHolderWithNoType.clientInvocationHandler.delegate = this; } + private void assertDeprecatedGeoWarnings() { + String prefix = "Field parameter ["; + String postfix = "] is deprecated and will be removed in a future version."; + String[] deprecationWarnings = new String[] { + prefix + "tree" + postfix, + prefix + "tree_levels" + postfix, + prefix + "precision" + postfix, + prefix + "distance_error_pct" + postfix + }; + assertWarnings(deprecationWarnings); + } + protected static SearchContext getSearchContext(QueryShardContext context) { TestSearchContext testSearchContext = new TestSearchContext(context) { @Override From 95c04c99b065ca87a81e8572d5ace2503b69a9fd Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Mon, 10 Dec 2018 13:27:36 -0600 Subject: [PATCH 24/38] fix deprecatedParameters setup --- .../elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java | 1 - .../index/mapper/LegacyGeoShapeFieldMapper.java | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java index 57d3aead021c1..c85cd66ad6354 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java @@ -202,7 +202,6 @@ private Builder getBuilder(String name, boolean coerce, boolean ignoreMalformed, private Builder getLegacyBuilder(String name, boolean coerce, boolean ignoreMalformed, Orientation orientation, boolean ignoreZ, DeprecatedParameters deprecatedParameters) { - deprecatedParameters.setup(); return new LegacyGeoShapeFieldMapper.Builder(name, coerce, ignoreMalformed, orientation, ignoreZ, deprecatedParameters); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java index ee870a88aa8dd..838013148b8f4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -143,7 +143,7 @@ public void setDistanceErrorPct(double distanceErrorPct) { this.distanceErrorPct = distanceErrorPct; } - public void setup() { + protected void setup() { if (strategy == null) { strategy = Defaults.STRATEGY; } @@ -226,12 +226,15 @@ public static class Builder extends BaseGeoShapeFieldMapper.Builder Date: Mon, 10 Dec 2018 13:33:34 -0600 Subject: [PATCH 25/38] update indexing approach --- docs/reference/mapping/types/geo-shape.asciidoc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index 5c8a15b5f7d7f..4e2705864d355 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -120,10 +120,12 @@ indexing each triangle as a 7 dimension point in a BKD tree. This provides near perfect spatial resolution (down to 1e-7 decimal degree precision) since all spatial relations are computed using an encoded vector representation of the original shape instead of a raster-grid representation as used by the -<> indexing approach. While this is the default indexing technique -prefix trees can still be used by setting the `tree` or `strategy` parameters -according to the appropriate <>. Note that these parameters -are now deprecated and will be removed in a future version. +<> indexing approach. Performance of the tessellator primarily +depends on the number of vertices that define the polygon/multi-polyogn. While +this is the default indexing technique prefix trees can still be used by setting +the `tree` or `strategy` parameters according to the appropriate +<>. Note that these parameters are now deprecated +and will be removed in a future version. [[prefix-trees]] [float] From 0b07a4fb96386fa17552e79dc31a9165471070a3 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Mon, 10 Dec 2018 14:19:27 -0600 Subject: [PATCH 26/38] fixing unexpected warnings failures --- .../index/mapper/LegacyGeoShapeFieldTypeTests.java | 2 +- .../index/query/QueryStringQueryBuilderTests.java | 5 +++++ .../org/elasticsearch/test/AbstractBuilderTestCase.java | 6 ++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java index 7a8795aba051f..2fcbed82e33b4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java @@ -36,7 +36,7 @@ public void setupProperties() { addModifier(new Modifier("tree", false) { @Override public void modify(MappedFieldType ft) { - ((GeoShapeFieldType)ft).setTree("quadtree"); + ((GeoShapeFieldType)ft).setTree("geohash"); } }); addModifier(new Modifier("strategy", false) { diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index 70f504516ec8a..dac24dff45b3b 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -1048,6 +1048,11 @@ public void testDisabledFieldNamesField() throws Exception { "_field_names", "enabled=true"))), MapperService.MergeReason.MAPPING_UPDATE); } + assertWarnings(new String[] { + "Field parameter [tree_levels] is deprecated and will be removed in a future version.", + "Field parameter [precision] is deprecated and will be removed in a future version.", + "Field parameter [distance_error_pct] is deprecated and will be removed in a future version." + }); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 4b6d83e985c9b..9b2376edaad93 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -218,14 +218,16 @@ public void beforeTest() throws Exception { AbstractBuilderTestCase.this, false); return null; }); - assertDeprecatedGeoWarnings(); + if (enableWarningsCheck() == true) { + assertDeprecatedGeoWarnings(); + } } serviceHolder.clientInvocationHandler.delegate = this; serviceHolderWithNoType.clientInvocationHandler.delegate = this; } - private void assertDeprecatedGeoWarnings() { + protected void assertDeprecatedGeoWarnings() { String prefix = "Field parameter ["; String postfix = "] is deprecated and will be removed in a future version."; String[] deprecationWarnings = new String[] { From 5f7061c67ea79da899356229484615fe8dd22820 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Wed, 12 Dec 2018 15:50:27 -0600 Subject: [PATCH 27/38] move orientation back to field type --- .../common/geo/parsers/GeoJsonParser.java | 4 ++-- .../index/mapper/BaseGeoShapeFieldMapper.java | 19 +++++++------------ .../index/mapper/GeoShapeFieldMapper.java | 6 +++--- .../mapper/LegacyGeoShapeFieldMapper.java | 2 +- .../index/query/GeoShapeQueryBuilder.java | 3 +-- 5 files changed, 14 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java index b7f6cc2cc62fc..b008786ed9211 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java +++ b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java @@ -51,7 +51,7 @@ protected static ShapeBuilder parse(XContentParser parser, BaseGeoShapeFieldMapp Orientation orientation = (shapeMapper == null) ? BaseGeoShapeFieldMapper.Defaults.ORIENTATION.value() - : shapeMapper.orientation().value(); + : shapeMapper.orientation(); Explicit coerce = (shapeMapper == null) ? BaseGeoShapeFieldMapper.Defaults.COERCE : shapeMapper.coerce(); @@ -108,7 +108,7 @@ protected static ShapeBuilder parse(XContentParser parser, BaseGeoShapeFieldMapp malformedException = "cannot have [" + ShapeParser.FIELD_ORIENTATION + "] with type set to [" + shapeType + "]"; } subParser.nextToken(); - orientation = ShapeBuilder.Orientation.fromString(parser.text()); + orientation = ShapeBuilder.Orientation.fromString(subParser.text()); } else { subParser.nextToken(); subParser.skipChildren(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java index c85cd66ad6354..3f1e49e525e81 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java @@ -153,7 +153,7 @@ protected void setupFieldType(BuilderContext context) { } BaseGeoShapeFieldType ft = (BaseGeoShapeFieldType)fieldType(); - ft.setOrientation(orientation); + ft.setOrientation(orientation().value()); } } @@ -265,17 +265,15 @@ public Query termQuery(Object value, QueryShardContext context) { protected Explicit coerce; protected Explicit ignoreMalformed; protected Explicit ignoreZValue; - protected Explicit orientation; protected BaseGeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, - Explicit ignoreMalformed, Explicit coerce, Explicit orientation, + Explicit ignoreMalformed, Explicit coerce, Explicit ignoreZValue, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); this.coerce = coerce; this.ignoreMalformed = ignoreMalformed; this.ignoreZValue = ignoreZValue; - this.orientation = orientation; } @Override @@ -291,9 +289,6 @@ protected void doMerge(Mapper mergeWith) { if (gsfm.ignoreZValue.explicit()) { this.ignoreZValue = gsfm.ignoreZValue; } - if (gsfm.orientation.explicit()) { - this.orientation = gsfm.orientation; - } } @Override @@ -303,9 +298,9 @@ protected void parseCreateField(ParseContext context, List field @Override protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { builder.field("type", contentType()); - - if (includeDefaults || orientation.explicit()) { - builder.field(Names.ORIENTATION.getPreferredName(), orientation.value()); + BaseGeoShapeFieldType ft = (BaseGeoShapeFieldType)fieldType(); + if (includeDefaults || ft.orientation() != Defaults.ORIENTATION.value()) { + builder.field(Names.ORIENTATION.getPreferredName(), ft.orientation()); } if (includeDefaults || coerce.explicit()) { builder.field(Names.COERCE.getPreferredName(), coerce.value()); @@ -330,8 +325,8 @@ public Explicit ignoreZValue() { return ignoreZValue; } - public Explicit orientation() { - return orientation; + public Orientation orientation() { + return ((BaseGeoShapeFieldType)fieldType).orientation(); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 02fb24a0de146..00484192cfab9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -70,7 +70,7 @@ public Builder(String name, boolean coerce, boolean ignoreMalformed, ShapeBuilde public GeoShapeFieldMapper build(BuilderContext context) { setupFieldType(context); return new GeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), coerce(context), - orientation(), ignoreZValue(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); + ignoreZValue(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); } } @@ -90,10 +90,10 @@ public GeoShapeFieldType clone() { } public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, - Explicit ignoreMalformed, Explicit coerce, Explicit orientation, + Explicit ignoreMalformed, Explicit coerce, Explicit ignoreZValue, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, orientation, ignoreZValue, indexSettings, + super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings, multiFields, copyTo); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java index 838013148b8f4..c64a5eb652097 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -492,7 +492,7 @@ public LegacyGeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, M Explicit ignoreMalformed, Explicit coerce, Explicit orientation, Explicit ignoreZValue, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, orientation, ignoreZValue, indexSettings, + super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings, multiFields, copyTo); } diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java index 3cec5bfc5f16a..6ee0f3f10ddcc 100644 --- a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java @@ -419,8 +419,7 @@ private Query getVectorQuery(QueryShardContext context, ShapeBuilder queryShapeB // CONTAINS queries are not yet supported by VECTOR strategy if (relation == ShapeRelation.CONTAINS) { throw new QueryShardException(context, - ShapeRelation.CONTAINS + " query relation not supported for Field [" + fieldName - + "] Use prefix tree strategy [" + SpatialStrategy.RECURSIVE + "] or [" + SpatialStrategy.TERM + "] instead."); + ShapeRelation.CONTAINS + " query relation not supported for Field [" + fieldName + "]"); } // wrap geoQuery as a ConstantScoreQuery From 16ad89fc9475a4f22605032bf82aaa714fde2070 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Sun, 16 Dec 2018 21:16:08 -0600 Subject: [PATCH 28/38] remove if in LegacyGeoShapeFieldMapper#doXContent. Fix GeoShapeFieldMapper to work with double array as a point --- .../elasticsearch/index/mapper/GeoShapeFieldMapper.java | 7 +++++-- .../index/mapper/LegacyGeoShapeFieldMapper.java | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 00484192cfab9..65ee2e428faa3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -126,8 +126,11 @@ public void parse(ParseContext context) throws IOException { private void indexShape(ParseContext context, Object luceneShape) { if (luceneShape instanceof GeoPoint) { - GeoPoint pt = (GeoPoint)luceneShape; + GeoPoint pt = (GeoPoint) luceneShape; indexFields(context, LatLonShape.createIndexableFields(name(), pt.lat(), pt.lon())); + } else if (luceneShape instanceof double[]) { + double[] pt = (double[]) luceneShape; + indexFields(context, LatLonShape.createIndexableFields(name(), pt[1], pt[0])); } else if (luceneShape instanceof Line) { indexFields(context, LatLonShape.createIndexableFields(name(), (Line)luceneShape)); } else if (luceneShape instanceof Polygon) { @@ -159,7 +162,7 @@ private void indexShape(ParseContext context, Object luceneShape) { indexShape(context, o); } } else { - throw new IllegalArgumentException("invalid shape type found [" + luceneShape.toString() + "] while indexing shape"); + throw new IllegalArgumentException("invalid shape type found [" + luceneShape.getClass() + "] while indexing shape"); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java index c64a5eb652097..a7fb43d67bd7c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -576,9 +576,9 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, builder.field(DeprecatedParameters.Names.PRECISION.getPreferredName(), DistanceUnit.METERS.toString(50)); } - if (includeDefaults || fieldType().strategy() != DeprecatedParameters.Defaults.STRATEGY) { - builder.field(DeprecatedParameters.Names.STRATEGY.getPreferredName(), fieldType().strategy().getStrategyName()); - } + + builder.field(DeprecatedParameters.Names.STRATEGY.getPreferredName(), fieldType().strategy().getStrategyName()); + if (includeDefaults || fieldType().distanceErrorPct() != fieldType().defaultDistanceErrorPct) { builder.field(DeprecatedParameters.Names.DISTANCE_ERROR_PCT.getPreferredName(), fieldType().distanceErrorPct()); } From cbe14bf89bf2b8e9e6f899f0b4f6da57c3c5c3da Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Sun, 16 Dec 2018 21:44:36 -0600 Subject: [PATCH 29/38] fix indexing-approach link in circle section of geoshape docs --- docs/reference/mapping/types/geo-shape.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index 4e2705864d355..8efb184afa6ba 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -630,7 +630,7 @@ POST /example/doc Elasticsearch supports a `circle` type, which consists of a center point with a radius. Note that this circle representation can only be indexed when using the `recursive` Prefix Tree strategy. For -the default <> circles should be approximated using +the default <> circles should be approximated using a `POLYGON`. [source,js] From d5049d5dc15259d9baa30359dc71a82cb08a5c23 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Sun, 16 Dec 2018 22:04:07 -0600 Subject: [PATCH 30/38] add strategy to deprecation warnings check --- .../java/org/elasticsearch/test/AbstractBuilderTestCase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 9b2376edaad93..daf29e46b0519 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -234,6 +234,7 @@ protected void assertDeprecatedGeoWarnings() { prefix + "tree" + postfix, prefix + "tree_levels" + postfix, prefix + "precision" + postfix, + prefix + "strategy" + postfix, prefix + "distance_error_pct" + postfix }; assertWarnings(deprecationWarnings); From 04898109acb15afbc6f5a50bf06dc61b83bfdabf Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Sun, 16 Dec 2018 22:44:14 -0600 Subject: [PATCH 31/38] fix test failures --- .../index/mapper/LegacyGeoShapeFieldMapper.java | 6 +++--- .../index/query/QueryStringQueryBuilderTests.java | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java index a7fb43d67bd7c..b68e48305b24b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -274,13 +274,13 @@ private void setupPrefixTrees() { // setup prefix trees regardless of strategy (this is used for the QueryBuilder) // recursive: - RecursivePrefixTreeStrategy rpts = new RecursivePrefixTreeStrategy(prefixTree, name()); + RecursivePrefixTreeStrategy rpts = new RecursivePrefixTreeStrategy(prefixTree, ft.name()); rpts.setDistErrPct(ft.distanceErrorPct()); rpts.setPruneLeafyBranches(false); ft.recursiveStrategy = rpts; // term: - TermQueryPrefixTreeStrategy termStrategy = new TermQueryPrefixTreeStrategy(prefixTree, name()); + TermQueryPrefixTreeStrategy termStrategy = new TermQueryPrefixTreeStrategy(prefixTree, ft.name()); termStrategy.setDistErrPct(ft.distanceErrorPct()); ft.termStrategy = termStrategy; @@ -295,7 +295,7 @@ protected void setupFieldType(BuilderContext context) { // field mapper handles this at build time // but prefix tree strategies require a name, so throw a similar exception - if (name().isEmpty()) { + if (fieldType().name().isEmpty()) { throw new IllegalArgumentException("name cannot be empty string"); } diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index dac24dff45b3b..716e71976454b 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -1051,6 +1051,7 @@ public void testDisabledFieldNamesField() throws Exception { assertWarnings(new String[] { "Field parameter [tree_levels] is deprecated and will be removed in a future version.", "Field parameter [precision] is deprecated and will be removed in a future version.", + "Field parameter [strategy] is deprecated and will be removed in a future version." "Field parameter [distance_error_pct] is deprecated and will be removed in a future version." }); } From 2d677cff927d5022440ef22ca675e79bf3f1bb74 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Sun, 16 Dec 2018 23:13:06 -0600 Subject: [PATCH 32/38] fix typo in QueryStringQueryBuilderTests --- .../elasticsearch/index/query/QueryStringQueryBuilderTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index 716e71976454b..1c34057457a4c 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -1051,7 +1051,7 @@ public void testDisabledFieldNamesField() throws Exception { assertWarnings(new String[] { "Field parameter [tree_levels] is deprecated and will be removed in a future version.", "Field parameter [precision] is deprecated and will be removed in a future version.", - "Field parameter [strategy] is deprecated and will be removed in a future version." + "Field parameter [strategy] is deprecated and will be removed in a future version.", "Field parameter [distance_error_pct] is deprecated and will be removed in a future version." }); } From 5949365ab4ca618ec2447c843f546530189866d9 Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Sun, 16 Dec 2018 23:37:41 -0600 Subject: [PATCH 33/38] fix total hits to totalHits().value --- .../elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java b/server/src/test/java/org/elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java index f705bd59b7450..574bdd46bba5b 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java +++ b/server/src/test/java/org/elasticsearch/search/geo/LegacyGeoShapeIntegrationIT.java @@ -121,7 +121,7 @@ public void testIgnoreMalformed() throws Exception { indexRandom(true, client().prepareIndex("test", "geometry", "0").setSource("shape", polygonGeoJson)); SearchResponse searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).get(); - assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); + assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L)); } /** @@ -158,7 +158,7 @@ public void testIndexShapeRouting() throws Exception { geoShapeQuery("shape", "0", "doc").indexedShapeIndex("test").indexedShapeRouting("ABC") ).get(); - assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); + assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L)); } private String findNodeName(String index) { From a09ad6f80b7c8194bb5506708bf4918a59fbf0cf Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Mon, 17 Dec 2018 20:54:00 -0600 Subject: [PATCH 34/38] fix version number --- .../java/org/elasticsearch/index/mapper/ExternalMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index 20c49c00935e3..2fb4cfeb81dbf 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -87,7 +87,7 @@ public ExternalMapper build(BuilderContext context) { BinaryFieldMapper binMapper = binBuilder.build(context); BooleanFieldMapper boolMapper = boolBuilder.build(context); GeoPointFieldMapper pointMapper = latLonPointBuilder.build(context); - BaseGeoShapeFieldMapper shapeMapper = (context.indexCreatedVersion().before(Version.V_6_6_0)) + BaseGeoShapeFieldMapper shapeMapper = (context.indexCreatedVersion().before(Version.V_7_0_0)) ? legacyShapeBuilder.build(context) : shapeBuilder.build(context); FieldMapper stringMapper = (FieldMapper)stringBuilder.build(context); From efa093706acb9df8cc12ac028e858ce026bfb8ae Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Mon, 17 Dec 2018 22:26:14 -0600 Subject: [PATCH 35/38] add version check to BaseGeoShapeFieldMapper --- .../index/mapper/BaseGeoShapeFieldMapper.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java index 3f1e49e525e81..74979073fcc18 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java @@ -189,12 +189,13 @@ public Mapper.Builder parse(String name, Map node, ParserContext iterator.remove(); } } - return getBuilder(name, coerce, ignoreMalformed, orientation, ignoreZ, parsedDeprecatedParams ? deprecatedParameters : null); + return getBuilder(parserContext, name, coerce, ignoreMalformed, orientation, ignoreZ, + parsedDeprecatedParams ? deprecatedParameters : null); } - private Builder getBuilder(String name, boolean coerce, boolean ignoreMalformed, Orientation orientation, + private Builder getBuilder(ParserContext parserContext, String name, boolean coerce, boolean ignoreMalformed, Orientation orientation, boolean ignoreZ, DeprecatedParameters deprecatedParameters) { - if (deprecatedParameters != null) { + if (parserContext.indexVersionCreated().before(Version.V_7_0_0) && deprecatedParameters != null) { return getLegacyBuilder(name, coerce, ignoreMalformed, orientation, ignoreZ, deprecatedParameters); } return new GeoShapeFieldMapper.Builder(name, coerce, ignoreMalformed, orientation, ignoreZ); From 32b04f973297832740dfd3dc1b85b234b6c8096d Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Mon, 17 Dec 2018 22:31:18 -0600 Subject: [PATCH 36/38] fix line length! --- .../elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java index 74979073fcc18..479661845fe73 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java @@ -193,8 +193,8 @@ public Mapper.Builder parse(String name, Map node, ParserContext parsedDeprecatedParams ? deprecatedParameters : null); } - private Builder getBuilder(ParserContext parserContext, String name, boolean coerce, boolean ignoreMalformed, Orientation orientation, - boolean ignoreZ, DeprecatedParameters deprecatedParameters) { + private Builder getBuilder(ParserContext parserContext, String name, boolean coerce, boolean ignoreMalformed, + Orientation orientation, boolean ignoreZ, DeprecatedParameters deprecatedParameters) { if (parserContext.indexVersionCreated().before(Version.V_7_0_0) && deprecatedParameters != null) { return getLegacyBuilder(name, coerce, ignoreMalformed, orientation, ignoreZ, deprecatedParameters); } From 0ba3bfa1233730a5748f2c9d6276c5fe96b6f0bf Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Mon, 17 Dec 2018 22:49:46 -0600 Subject: [PATCH 37/38] revert version check in BaseGeoShapeFieldMapper --- .../index/mapper/BaseGeoShapeFieldMapper.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java index 479661845fe73..27074b3854298 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java @@ -189,13 +189,12 @@ public Mapper.Builder parse(String name, Map node, ParserContext iterator.remove(); } } - return getBuilder(parserContext, name, coerce, ignoreMalformed, orientation, ignoreZ, - parsedDeprecatedParams ? deprecatedParameters : null); + return getBuilder(name, coerce, ignoreMalformed, orientation, ignoreZ, parsedDeprecatedParams ? deprecatedParameters : null); } - private Builder getBuilder(ParserContext parserContext, String name, boolean coerce, boolean ignoreMalformed, + private Builder getBuilder(String name, boolean coerce, boolean ignoreMalformed, Orientation orientation, boolean ignoreZ, DeprecatedParameters deprecatedParameters) { - if (parserContext.indexVersionCreated().before(Version.V_7_0_0) && deprecatedParameters != null) { + if (deprecatedParameters != null) { return getLegacyBuilder(name, coerce, ignoreMalformed, orientation, ignoreZ, deprecatedParameters); } return new GeoShapeFieldMapper.Builder(name, coerce, ignoreMalformed, orientation, ignoreZ); From 17f3e8db10e4aa566a19d64c7e2a2ae5f65e9cff Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 18 Dec 2018 14:15:57 +0100 Subject: [PATCH 38/38] Fix serialization of mappings of legacy shapes. --- .../index/mapper/BaseGeoShapeFieldMapper.java | 42 ++++++---- .../index/mapper/GeoShapeFieldMapper.java | 5 -- .../mapper/LegacyGeoShapeFieldMapper.java | 83 +++++++------------ .../LegacyGeoShapeFieldMapperTests.java | 6 +- .../query/GeoShapeQueryBuilderTests.java | 19 +++-- .../query/LegacyGeoShapeFieldQueryTests.java | 16 +++- .../query/QueryStringQueryBuilderTests.java | 6 -- .../test/AbstractBuilderTestCase.java | 20 +---- 8 files changed, 90 insertions(+), 107 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java index 27074b3854298..ea30f1c5c2312 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java @@ -161,10 +161,10 @@ public static class TypeParser implements Mapper.TypeParser { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - boolean coerce = Defaults.COERCE.value(); - boolean ignoreZ = Defaults.IGNORE_Z_VALUE.value(); - boolean ignoreMalformed = Defaults.IGNORE_MALFORMED.value(); - Orientation orientation = Defaults.ORIENTATION.value(); + Boolean coerce = null; + Boolean ignoreZ = null; + Boolean ignoreMalformed = null; + Orientation orientation = null; DeprecatedParameters deprecatedParameters = new DeprecatedParameters(); boolean parsedDeprecatedParams = false; for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { @@ -189,20 +189,32 @@ public Mapper.Builder parse(String name, Map node, ParserContext iterator.remove(); } } - return getBuilder(name, coerce, ignoreMalformed, orientation, ignoreZ, parsedDeprecatedParams ? deprecatedParameters : null); - } + final Builder builder; + if (parsedDeprecatedParams || parserContext.indexVersionCreated().before(Version.V_7_0_0)) { + // Legacy index-based shape + builder = new LegacyGeoShapeFieldMapper.Builder(name, deprecatedParameters); + } else { + // BKD-based shape + builder = new GeoShapeFieldMapper.Builder(name); + } + + if (coerce != null) { + builder.coerce(coerce); + } - private Builder getBuilder(String name, boolean coerce, boolean ignoreMalformed, - Orientation orientation, boolean ignoreZ, DeprecatedParameters deprecatedParameters) { - if (deprecatedParameters != null) { - return getLegacyBuilder(name, coerce, ignoreMalformed, orientation, ignoreZ, deprecatedParameters); + if (ignoreZ != null) { + builder.ignoreZValue(ignoreZ); + } + + if (ignoreMalformed != null) { + builder.ignoreMalformed(ignoreMalformed); + } + + if (orientation != null) { + builder.orientation(orientation); } - return new GeoShapeFieldMapper.Builder(name, coerce, ignoreMalformed, orientation, ignoreZ); - } - private Builder getLegacyBuilder(String name, boolean coerce, boolean ignoreMalformed, Orientation orientation, - boolean ignoreZ, DeprecatedParameters deprecatedParameters) { - return new LegacyGeoShapeFieldMapper.Builder(name, coerce, ignoreMalformed, orientation, ignoreZ, deprecatedParameters); + return builder; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 65ee2e428faa3..441cb8ac3e12e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -61,11 +61,6 @@ public Builder(String name) { super (name, new GeoShapeFieldType(), new GeoShapeFieldType()); } - public Builder(String name, boolean coerce, boolean ignoreMalformed, ShapeBuilder.Orientation orientation, - boolean ignoreZ) { - super(name, new GeoShapeFieldType(), new GeoShapeFieldType(), coerce, ignoreMalformed, orientation, ignoreZ); - } - @Override public GeoShapeFieldMapper build(BuilderContext context) { setupFieldType(context); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java index b68e48305b24b..c0b931225b8e6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -30,6 +30,7 @@ import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.geo.GeoUtils; @@ -111,10 +112,10 @@ public static class Defaults { public SpatialStrategy strategy = null; public String tree = null; - public int treeLevels = Integer.MIN_VALUE; + public Integer treeLevels = null; public String precision = null; public Boolean pointsOnly = null; - public double distanceErrorPct = Double.NaN; + public Double distanceErrorPct = null; public void setSpatialStrategy(SpatialStrategy strategy) { this.strategy = strategy; @@ -143,44 +144,6 @@ public void setDistanceErrorPct(double distanceErrorPct) { this.distanceErrorPct = distanceErrorPct; } - protected void setup() { - if (strategy == null) { - strategy = Defaults.STRATEGY; - } - if (tree == null) { - tree = Defaults.TREE; - } - if (Double.isNaN(distanceErrorPct)) { - if (precision != null || treeLevels != Integer.MIN_VALUE) { - distanceErrorPct = 0d; - } else { - distanceErrorPct = Defaults.DISTANCE_ERROR_PCT; - } - } - if (treeLevels == Integer.MIN_VALUE && precision == null) { - // set default precision if treeLevels is not explicitly set - precision = Defaults.PRECISION; - } - if (treeLevels == Integer.MIN_VALUE) { - if (precision.equals(Defaults.PRECISION)) { - treeLevels = tree.equals(Defaults.TREE) - ? Defaults.QUADTREE_LEVELS - : Defaults.GEOHASH_TREE_LEVELS; - } else { - treeLevels = tree == Defaults.TREE - ? GeoUtils.quadTreeLevelsForPrecision(precision) - : GeoUtils.geoHashLevelsForPrecision(precision); - } - } - if (pointsOnly == null) { - if (strategy == SpatialStrategy.TERM) { - pointsOnly = true; - } else { - pointsOnly = Defaults.POINTS_ONLY; - } - } - } - public static boolean parse(String name, String fieldName, Object fieldNode, DeprecatedParameters deprecatedParameters) { if (Names.STRATEGY.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { checkPrefixTreeSupport(fieldName); @@ -225,16 +188,12 @@ public static class Builder extends BaseGeoShapeFieldMapper.Builder indexedShapeToReturn; - @Override - protected boolean enableWarningsCheck() { - return false; - } - protected String fieldName() { return GEO_SHAPE_FIELD_NAME; } + @Override + protected Settings createTestIndexSettings() { + // force the new shape impl + Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_6_0, Version.CURRENT); + return Settings.builder() + .put(super.createTestIndexSettings()) + .put(IndexMetaData.SETTING_VERSION_CREATED, version) + .build(); + } + @Override protected GeoShapeQueryBuilder doCreateTestQueryBuilder() { return doCreateTestQueryBuilder(randomBoolean()); diff --git a/server/src/test/java/org/elasticsearch/index/query/LegacyGeoShapeFieldQueryTests.java b/server/src/test/java/org/elasticsearch/index/query/LegacyGeoShapeFieldQueryTests.java index 3cf6f2031810a..f549d17977dc1 100644 --- a/server/src/test/java/org/elasticsearch/index/query/LegacyGeoShapeFieldQueryTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/LegacyGeoShapeFieldQueryTests.java @@ -19,9 +19,13 @@ package org.elasticsearch.index.query; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.geo.RandomShapeGenerator; import org.elasticsearch.test.geo.RandomShapeGenerator.ShapeType; @@ -31,7 +35,17 @@ public class LegacyGeoShapeFieldQueryTests extends GeoShapeQueryBuilderTests { @Override protected String fieldName() { - return LEGACY_GEO_SHAPE_FIELD_NAME; + return GEO_SHAPE_FIELD_NAME; + } + + @Override + protected Settings createTestIndexSettings() { + // force the legacy shape impl + Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.V_6_5_0); + return Settings.builder() + .put(super.createTestIndexSettings()) + .put(IndexMetaData.SETTING_VERSION_CREATED, version) + .build(); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index 1c34057457a4c..70f504516ec8a 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -1048,12 +1048,6 @@ public void testDisabledFieldNamesField() throws Exception { "_field_names", "enabled=true"))), MapperService.MergeReason.MAPPING_UPDATE); } - assertWarnings(new String[] { - "Field parameter [tree_levels] is deprecated and will be removed in a future version.", - "Field parameter [precision] is deprecated and will be removed in a future version.", - "Field parameter [strategy] is deprecated and will be removed in a future version.", - "Field parameter [distance_error_pct] is deprecated and will be removed in a future version." - }); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index daf29e46b0519..5eef0a249b687 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -113,7 +113,6 @@ public abstract class AbstractBuilderTestCase extends ESTestCase { protected static final String GEO_POINT_FIELD_NAME = "mapped_geo_point"; protected static final String GEO_POINT_ALIAS_FIELD_NAME = "mapped_geo_point_alias"; protected static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; - protected static final String LEGACY_GEO_SHAPE_FIELD_NAME = "mapped_legacy_geo_shape"; protected static final String[] MAPPED_FIELD_NAMES = new String[]{STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, INT_FIELD_NAME, INT_RANGE_FIELD_NAME, DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, DATE_RANGE_FIELD_NAME, OBJECT_FIELD_NAME, GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, @@ -218,28 +217,12 @@ public void beforeTest() throws Exception { AbstractBuilderTestCase.this, false); return null; }); - if (enableWarningsCheck() == true) { - assertDeprecatedGeoWarnings(); - } } serviceHolder.clientInvocationHandler.delegate = this; serviceHolderWithNoType.clientInvocationHandler.delegate = this; } - protected void assertDeprecatedGeoWarnings() { - String prefix = "Field parameter ["; - String postfix = "] is deprecated and will be removed in a future version."; - String[] deprecationWarnings = new String[] { - prefix + "tree" + postfix, - prefix + "tree_levels" + postfix, - prefix + "precision" + postfix, - prefix + "strategy" + postfix, - prefix + "distance_error_pct" + postfix - }; - assertWarnings(deprecationWarnings); - } - protected static SearchContext getSearchContext(QueryShardContext context) { TestSearchContext testSearchContext = new TestSearchContext(context) { @Override @@ -413,8 +396,7 @@ public void onRemoval(ShardId shardId, Accountable accountable) { OBJECT_FIELD_NAME, "type=object", GEO_POINT_FIELD_NAME, "type=geo_point", GEO_POINT_ALIAS_FIELD_NAME, "type=alias,path=" + GEO_POINT_FIELD_NAME, - GEO_SHAPE_FIELD_NAME, "type=geo_shape", - LEGACY_GEO_SHAPE_FIELD_NAME, "type=geo_shape,tree=quadtree" + GEO_SHAPE_FIELD_NAME, "type=geo_shape" ))), MapperService.MergeReason.MAPPING_UPDATE); // also add mappings for two inner field in the object field mapperService.merge("_doc", new CompressedXContent("{\"properties\":{\"" + OBJECT_FIELD_NAME + "\":{\"type\":\"object\","