From 0efa2314b2966ab8a39cea7d4978ff6e3ee9f3d5 Mon Sep 17 00:00:00 2001 From: iverase Date: Wed, 2 Mar 2022 09:54:12 +0100 Subject: [PATCH 01/10] Integrate GeoHexGridAggregation with vector tiles API This commit adds a new optional parameter on the vector tiles API called `grid_agg` with two possible values, geotile (default) and geohex. This will allow to build the aggs layer using different grid aggregations, for example we can have a grid aggregation that is built using hexagons. Note the GeoHexGridAggregation only support data indexed using geo_point field type so this limitation applies here too. --- .../search/search-vector-tile-api.asciidoc | 43 +++- .../test/vector-tile/10_basic.yml | 24 ++ .../xpack/vectortile/VectorTileRestIT.java | 22 +- .../vectortile/feature/FeatureFactory.java | 30 +++ .../vectortile/rest/GridAggregation.java | 208 ++++++++++++++++++ .../xpack/vectortile/rest/GridType.java | 67 ++++++ .../vectortile/rest/RestVectorTileAction.java | 68 +++--- .../vectortile/rest/VectorTileRequest.java | 41 ++-- .../FeatureFactoriesConsistencyTests.java | 14 +- .../vectortile/rest/GridAggregationTests.java | 53 +++++ .../rest/VectorTileRequestTests.java | 11 +- 11 files changed, 503 insertions(+), 78 deletions(-) create mode 100644 x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java create mode 100644 x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridType.java create mode 100644 x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/GridAggregationTests.java diff --git a/docs/reference/search/search-vector-tile-api.asciidoc b/docs/reference/search/search-vector-tile-api.asciidoc index cee2a908cb839..4ac56cefb154c 100644 --- a/docs/reference/search/search-vector-tile-api.asciidoc +++ b/docs/reference/search/search-vector-tile-api.asciidoc @@ -200,19 +200,52 @@ larger than the vector tile. square with equal sides. Defaults to `4096`. // end::extent-param[] +// tag::grid-agg[] +`grid_agg`:: +(Optional, string) Determines the GeoGridAggregation used to build the grid. +Accepts: + +`geotile` (Default)::: +Grid is constructed using a GeoTileGridAggregation. + +`geohex`::: +Grid is constructed using a GeoHexGridAggregation. Note that GeoHexGridAggregation +only supports running against geo_point field type. +// end::grid-agg[] + // tag::grid-precision[] `grid_precision`:: -(Optional, integer) Additional zoom levels available through the `aggs` layer. -For example, if `` is `7` and `grid_precision` is `8`, you can zoom in up to -level 15. Accepts `0`-`8`. Defaults to `8`. If `0`, results don't include the +(Optional, integer) It defines how many bins will be created on the grid +layer at each zoom level which depends on the `grid_agg`. +Accepts `0`-`8`. Defaults to `8`. If `0`, results don't include the `aggs` layer. + -This value determines the grid size of the `geotile_grid` as follows: +In case of a geotile aggregation, the grid precision represents the additional zoom levels +available through the `aggs` layer. For example, if `` is `7` +and `grid_precision` is `8`, then getile aggregation will use precision 15. ++ +The precision is computed by as follows: ++ +` + grid_precision` ++ +With a maximum value of 29. This value determines the +number of cells of the `geotile_grid` as follows: + `(2^grid_precision) x (2^grid_precision)` + For example, a value of `8` divides the tile into a grid of 256 x 256 cells. The `aggs` layer only contains features for cells with matching data. ++ +In case of a geohex aggregation, the grid precision translated to the +https://h3geo.org/docs/core-library/restable[precision of the hexagonal cells]. ++ +The precision for the hex grid is computed as follows: ++ +`( + gridPrecision - 1) / 2` ++ +where the minimum precision for a hexagonal bin is 2 and the maximum precision is 15. +Note that hexagonal cells do not align perfectly on the vector tile so some cells might +intersect more than one vector tile. // end::grid-precision[] // tag::grid-type[] @@ -222,7 +255,7 @@ layer. In the `aggs` layer, each feature represents a `geotile_grid` cell. Accepts: `grid` (Default)::: -Each feature is a `Polygon` of the cell's bounding box. +Each feature is a `Polygon` of the cell's geometry that depends on the selected `grid_agg`. `point`::: Each feature is a `Point` that's the centroid of the cell. diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/vector-tile/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/vector-tile/10_basic.yml index 48498141412f9..0efcbc47c3bf6 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/vector-tile/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/vector-tile/10_basic.yml @@ -88,6 +88,30 @@ setup: body: grid_type: point +--- +"grid agg geotile": + - do: + search_mvt: + index: locations + field: location + x: 0 + y: 0 + zoom: 0 + body: + grid_agg: geotile + +--- +"grid agg geohex": + - do: + search_mvt: + index: locations + field: location + x: 0 + y: 0 + zoom: 0 + body: + grid_agg: geohex + --- "grid type grid": - do: diff --git a/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java b/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java index 70b2f3dc73bd2..7f9702dcd8bda 100644 --- a/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java +++ b/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java @@ -317,10 +317,18 @@ public void testGridPrecision() throws Exception { } } - public void testGridType() throws Exception { + public void testGeoTileGrid() throws Exception { + doGridAggType(randomBoolean() ? "" : ", \"grid_agg\": \"geotile\""); + } + + public void testGeoHexGrid() throws Exception { + doGridAggType(", \"grid_agg\": \"geohex\""); + } + + private void doGridAggType(String gridAgg) throws Exception { { final Request mvtRequest = new Request(getHttpMethod(), INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); - mvtRequest.setJsonEntity("{\"size\" : 100, \"grid_type\": \"point\" }"); + mvtRequest.setJsonEntity("{\"size\" : 100" + gridAgg + ",\"grid_type\": \"point\" }"); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 2); @@ -330,7 +338,7 @@ public void testGridType() throws Exception { } { final Request mvtRequest = new Request(getHttpMethod(), INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); - mvtRequest.setJsonEntity("{\"size\" : 100, \"grid_type\": \"grid\" }"); + mvtRequest.setJsonEntity("{\"size\" : 100" + gridAgg + ", \"grid_type\": \"grid\" }"); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 2); @@ -340,7 +348,7 @@ public void testGridType() throws Exception { } { final Request mvtRequest = new Request(getHttpMethod(), INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); - mvtRequest.setJsonEntity("{\"size\" : 100, \"grid_type\": \"centroid\" }"); + mvtRequest.setJsonEntity("{\"size\" : 100" + gridAgg + ", \"grid_type\": \"centroid\" }"); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 2); @@ -354,6 +362,12 @@ public void testGridType() throws Exception { final ResponseException ex = expectThrows(ResponseException.class, () -> execute(mvtRequest)); assertThat(ex.getResponse().getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_BAD_REQUEST)); } + { + final Request mvtRequest = new Request(getHttpMethod(), INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"grid_agg\": \"invalid_agg\" }"); + final ResponseException ex = expectThrows(ResponseException.class, () -> execute(mvtRequest)); + assertThat(ex.getResponse().getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_BAD_REQUEST)); + } } public void testInvalidAggName() { diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java index 963ae1aee6d29..c0a6ddea8e1c4 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java @@ -13,6 +13,8 @@ import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter; import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.SimpleFeatureFactory; import org.elasticsearch.common.geo.SphericalMercatorUtils; import org.elasticsearch.geometry.Circle; import org.elasticsearch.geometry.Geometry; @@ -37,6 +39,7 @@ import org.locationtech.jts.geom.TopologyException; import org.locationtech.jts.simplify.TopologyPreservingSimplifier; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -56,6 +59,8 @@ public class FeatureFactory { private final CoordinateSequenceFilter sequenceFilter; // pixel precision of the tile in the mercator projection. private final double pixelPrecision; + // optimization for points and rectangles + private final SimpleFeatureFactory simpleFeatureFactory; // size of the buffer in pixels for the clip envelope. we choose a values that makes sure // we have values outside the tile for polygon crossing the tile so the outline of the // tile is not part of the final result. @@ -73,8 +78,33 @@ public FeatureFactory(int z, int x, int y, int extent) { this.builder = new JTSGeometryBuilder(geomFactory); this.clipTile = geomFactory.toGeometry(clipEnvelope); this.sequenceFilter = new MvtCoordinateSequenceFilter(tileEnvelope, extent); + this.simpleFeatureFactory = new SimpleFeatureFactory(z, x, y, extent); } + /** + * Returns a {@code byte[]} containing the mvt representation of the provided point + */ + public byte[] point(double lon, double lat) throws IOException { + return simpleFeatureFactory.point(lon, lat); + } + + /** + * Returns a {@code byte[]} containing the mvt representation of the provided rectangle + */ + public byte[] box(double minLon, double maxLon, double minLat, double maxLat) throws IOException { + return simpleFeatureFactory.box(minLon, maxLon, minLat, maxLat); + } + + /** + * Returns a {@code byte[]} containing the mvt representation of the provided points + */ + public byte[] points(List multiPoint) { + return simpleFeatureFactory.points(multiPoint); + } + + /** + * Returns a List {@code byte[]} containing the mvt representation of the provided geometry + */ public List getFeatures(Geometry geometry) { // Get geometry in spherical mercator final org.locationtech.jts.geom.Geometry jtsGeometry = geometry.visit(builder); diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java new file mode 100644 index 0000000000000..83103f29e27c8 --- /dev/null +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java @@ -0,0 +1,208 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectortile.rest; + +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.geo.GeometryNormalizer; +import org.elasticsearch.common.geo.Orientation; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.h3.CellBoundary; +import org.elasticsearch.h3.H3; +import org.elasticsearch.h3.LatLng; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoHexGridAggregationBuilder; +import org.elasticsearch.xpack.vectortile.feature.FeatureFactory; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +/** + * Enum containing the basic operations for different GeoGridAggregations. + */ +enum GridAggregation { + GEOTILE { + @Override + public GeoGridAggregationBuilder newAgg(String aggName) { + return new GeoTileGridAggregationBuilder(aggName); + } + + @Override + public Rectangle bufferTile(Rectangle tile, int z, int gridPrecision) { + // No buffering needed as GeoTile bins aligns with the tile + return tile; + } + + @Override + public int gridPrecisionToAggPrecision(int z, int gridPrecision) { + return Math.min(GeoTileUtils.MAX_ZOOM, z + gridPrecision); + } + + @Override + public byte[] toGrid(String bucketKey, FeatureFactory featureFactory) throws IOException { + final Rectangle r = toRectangle(bucketKey); + return featureFactory.box(r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); + } + + @Override + public Rectangle toRectangle(String bucketKey) { + return GeoTileUtils.toBoundingBox(bucketKey); + } + }, + GEOHEX { + + // Because hex bins do not fit perfectly on a tile, we need to buffer our queries in order to collect + // all points inside the bin. We never have aggregations for levels 0 and 1, values for level 2 have + // been computed manually and approximated. The amount that the buffer decreases by level has been + // approximated to 2.5 (brute force computation suggest ~2.6 in the first 9 levels so 2.5 should be safe). + private static final double[] LAT_BUFFER_SIZE = new double[16]; + private static final double[] LON_BUFFER_SIZE = new double[16]; + static { + LAT_BUFFER_SIZE[0] = LAT_BUFFER_SIZE[1] = Double.NaN; + LON_BUFFER_SIZE[0] = LON_BUFFER_SIZE[1] = Double.NaN; + LAT_BUFFER_SIZE[2] = 3.7; + LON_BUFFER_SIZE[2] = 51.2; + for (int i = 3; i < LON_BUFFER_SIZE.length; i++) { + LAT_BUFFER_SIZE[i] = LAT_BUFFER_SIZE[i - 1] / 2.5; + LON_BUFFER_SIZE[i] = LON_BUFFER_SIZE[i - 1] / 2.5; + } + } + + @Override + public GeoGridAggregationBuilder newAgg(String aggName) { + return new GeoHexGridAggregationBuilder(aggName); + } + + @Override + public Rectangle bufferTile(Rectangle tile, int z, int gridPrecision) { + if (z == 0 || gridPrecision == 0) { + // no need to buffer at level 0 as we are looking to all data. + return tile; + } + final int aggPrecision = gridPrecisionToAggPrecision(z, gridPrecision); + return new Rectangle( + GeoUtils.normalizeLon(tile.getMinX() - LON_BUFFER_SIZE[aggPrecision]), + GeoUtils.normalizeLon(tile.getMaxX() + LON_BUFFER_SIZE[aggPrecision]), + Math.min(GeoTileUtils.LATITUDE_MASK, tile.getMaxY() + LAT_BUFFER_SIZE[aggPrecision]), + Math.max(-GeoTileUtils.LATITUDE_MASK, tile.getMinY() - LAT_BUFFER_SIZE[aggPrecision]) + ); + } + + @Override + public int gridPrecisionToAggPrecision(int z, int gridPrecision) { + // The minimum resolution we allow is 2 to avoid having hexagons containing the pole. + // this is the table of Hex precision: + // precision: 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | + // ------------------------------------------- + // zoom 0: 2 | 2 | 2 | 2 | 2 | 2 | 3 | 3 | + // zoom 1: 2 | 2 | 2 | 2 | 2 | 3 | 3 | 4 | + // zoom 2: 2 | 2 | 2 | 2 | 3 | 3 | 4 | 4 | + // zoom 3: 2 | 2 | 2 | 3 | 3 | 4 | 4 | 5 | + // zoom 4: 2 | 2 | 3 | 3 | 4 | 4 | 5 | 5 | + // zoom 5: 2 | 3 | 3 | 4 | 4 | 5 | 5 | 6 | + // zoom 6: 3 | 3 | 4 | 4 | 5 | 5 | 6 | 6 | + // zoom 7: 3 | 4 | 4 | 5 | 5 | 6 | 6 | 7 | + // zoom 8: 4 | 4 | 5 | 5 | 6 | 6 | 7 | 7 | + // zoom 9: 4 | 5 | 5 | 6 | 6 | 7 | 7 | 8 | + // zoom 10: 5 | 5 | 6 | 6 | 7 | 7 | 8 | 8 | + // zoom 11: 5 | 6 | 6 | 7 | 7 | 8 | 8 | 9 | + // zoom 12: 6 | 6 | 7 | 7 | 8 | 8 | 9 | 9 | + // zoom 13: 6 | 7 | 7 | 8 | 8 | 9 | 9 | 10 | + // zoom 14: 7 | 7 | 8 | 8 | 9 | 9 | 10 | 10 | + // zoom 15: 7 | 8 | 8 | 9 | 9 | 10 | 10 | 11 | + // zoom 16: 8 | 8 | 9 | 9 | 10 | 10 | 11 | 11 | + // zoom 17: 8 | 9 | 9 | 10 | 10 | 11 | 11 | 12 | + // zoom 18: 9 | 9 | 10 | 10 | 11 | 11 | 12 | 12 | + // zoom 19: 9 | 10 | 10 | 11 | 11 | 12 | 12 | 13 | + // zoom 20: 10 | 10 | 11 | 11 | 12 | 12 | 13 | 13 | + // zoom 21: 10 | 11 | 11 | 12 | 12 | 13 | 13 | 14 | + // zoom 22: 11 | 11 | 12 | 12 | 13 | 13 | 14 | 14 | + // zoom 23: 11 | 12 | 12 | 13 | 13 | 14 | 14 | 15 | + // zoom 24: 12 | 12 | 13 | 13 | 14 | 14 | 15 | 15 | + // zoom 25: 12 | 13 | 13 | 14 | 14 | 15 | 15 | 15 | + // zoom 26: 13 | 13 | 14 | 14 | 15 | 15 | 15 | 15 | + // zoom 27: 13 | 14 | 14 | 15 | 15 | 15 | 15 | 15 | + // zoom 28: 14 | 14 | 15 | 15 | 15 | 15 | 15 | 15 | + // zoom 29: 14 | 15 | 15 | 15 | 15 | 15 | 15 | 15 | + return Math.min(H3.MAX_H3_RES, Math.max(2, (z + gridPrecision - 1) / 2)); + } + + @Override + public byte[] toGrid(String bucketKey, FeatureFactory featureFactory) { + final CellBoundary boundary = H3.h3ToGeoBoundary(bucketKey); + final double[] lats = new double[boundary.numPoints() + 1]; + final double[] lons = new double[boundary.numPoints() + 1]; + for (int i = 0; i < boundary.numPoints(); i++) { + final LatLng latLng = boundary.getLatLon(i); + lats[i] = latLng.getLatDeg(); + lons[i] = latLng.getLonDeg(); + } + lats[boundary.numPoints()] = boundary.getLatLon(0).getLatDeg(); + lons[boundary.numPoints()] = boundary.getLatLon(0).getLonDeg(); + final Polygon polygon = new Polygon(new LinearRing(lons, lats)); + final List x = featureFactory.getFeatures(GeometryNormalizer.apply(Orientation.CCW, polygon)); + return x.size() > 0 ? x.get(0) : null; + } + + @Override + public Rectangle toRectangle(String bucketKey) { + final CellBoundary boundary = H3.h3ToGeoBoundary(bucketKey); + double minLat = Double.POSITIVE_INFINITY; + double minLon = Double.POSITIVE_INFINITY; + double maxLat = Double.NEGATIVE_INFINITY; + double maxLon = Double.NEGATIVE_INFINITY; + for (int i = 0; i < boundary.numPoints(); i++) { + final LatLng latLng = boundary.getLatLon(i); + minLat = Math.min(minLat, latLng.getLatDeg()); + minLon = Math.min(minLon, latLng.getLonDeg()); + maxLat = Math.max(maxLat, latLng.getLatDeg()); + maxLon = Math.max(maxLon, latLng.getLonDeg()); + } + return new Rectangle(minLon, maxLon, maxLat, minLat); + } + }; + + /** + * New {@link GeoGridAggregationBuilder} instance. + */ + public abstract GeoGridAggregationBuilder newAgg(String aggName); + + /** + * Buffer the query bounding box so the bins of an aggregation see + * all data that is inside them. + */ + public abstract Rectangle bufferTile(Rectangle tile, int z, int gridPrecision); + + /** + * Transform the provided grid precision at the given zoom to the + * agg precision. + */ + public abstract int gridPrecisionToAggPrecision(int z, int gridPrecision); + + /** + * transforms the geometry of a given bin into the vector tile feature. + */ + public abstract byte[] toGrid(String bucketKey, FeatureFactory featureFactory) throws IOException; + + /** + * Returns the bounding box of the bin. + */ + public abstract Rectangle toRectangle(String bucketKey); + + public static GridAggregation fromString(String type) { + return switch (type.toLowerCase(Locale.ROOT)) { + case "geotile" -> GEOTILE; + case "geohex" -> GEOHEX; + default -> throw new IllegalArgumentException("Invalid agg type [" + type + "]"); + }; + } +} diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridType.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridType.java new file mode 100644 index 0000000000000..f1ac053085de0 --- /dev/null +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridType.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectortile.rest; + +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket; +import org.elasticsearch.search.aggregations.metrics.InternalGeoCentroid; +import org.elasticsearch.xpack.vectortile.feature.FeatureFactory; + +import java.io.IOException; +import java.util.Locale; + +/** + * Enum containing the basic geometry types for serializing {@link InternalGeoGridBucket} + */ +enum GridType { + + GRID { + @Override + public byte[] toFeature(GridAggregation gridAggregation, InternalGeoGridBucket bucket, String key, FeatureFactory featureFactory) + throws IOException { + return gridAggregation.toGrid(key, featureFactory); + } + }, + POINT { + @Override + public byte[] toFeature(GridAggregation gridAggregation, InternalGeoGridBucket bucket, String key, FeatureFactory featureFactory) + throws IOException { + final GeoPoint point = (GeoPoint) bucket.getKey(); + return featureFactory.point(point.lon(), point.lat()); + } + }, + CENTROID { + @Override + public byte[] toFeature(GridAggregation gridAggregation, InternalGeoGridBucket bucket, String key, FeatureFactory featureFactory) + throws IOException { + final Rectangle r = gridAggregation.toRectangle(key); + final InternalGeoCentroid centroid = bucket.getAggregations().get(RestVectorTileAction.CENTROID_AGG_NAME); + final double featureLon = Math.min(Math.max(centroid.centroid().lon(), r.getMinLon()), r.getMaxLon()); + final double featureLat = Math.min(Math.max(centroid.centroid().lat(), r.getMinLat()), r.getMaxLat()); + return featureFactory.point(featureLon, featureLat); + } + }; + + /** Builds the corresponding vector tile feature for the provided bucket */ + public abstract byte[] toFeature( + GridAggregation gridAggregation, + InternalGeoGridBucket bucket, + String key, + FeatureFactory featureFactory + ) throws IOException; + + public static GridType fromString(String type) { + return switch (type.toLowerCase(Locale.ROOT)) { + case "grid" -> GRID; + case "point" -> POINT; + case "centroid" -> CENTROID; + default -> throw new IllegalArgumentException("Invalid grid type [" + type + "]"); + }; + } +} diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java index fac2d19e531a5..75da5997749a3 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.geo.SimpleFeatureFactory; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStream; import org.elasticsearch.geometry.Rectangle; @@ -34,20 +33,19 @@ import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.MultiBucketConsumerService; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGrid; import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket; -import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.InternalGeoBounds; -import org.elasticsearch.search.aggregations.metrics.InternalGeoCentroid; import org.elasticsearch.search.aggregations.pipeline.StatsBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder.MetricsAggregationBuilder; import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.profile.SearchProfileResults; import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.xpack.vectortile.feature.FeatureFactory; import java.io.IOException; import java.util.List; @@ -79,7 +77,7 @@ public class RestVectorTileAction extends BaseRestHandler { // prefox for internal aggregations. User aggregations cannot start with this prefix private static final String INTERNAL_AGG_PREFIX = "_mvt_"; // internal centroid aggregation name - private static final String CENTROID_AGG_NAME = INTERNAL_AGG_PREFIX + "centroid"; + static final String CENTROID_AGG_NAME = INTERNAL_AGG_PREFIX + "centroid"; public RestVectorTileAction() {} @@ -112,18 +110,18 @@ public RestResponse buildResponse(SearchResponse searchResponse) throws Exceptio tileBuilder.addLayers(buildHitsLayer(hits, request)); } ensureOpen(); - final SimpleFeatureFactory geomBuilder = new SimpleFeatureFactory( + final FeatureFactory featureFactory = new FeatureFactory( request.getZ(), request.getX(), request.getY(), request.getExtent() ); - final InternalGeoTileGrid grid = searchResponse.getAggregations() != null + final InternalGeoGrid grid = searchResponse.getAggregations() != null ? searchResponse.getAggregations().get(GRID_FIELD) : null; // TODO: should we expose the total number of buckets on InternalGeoTileGrid? if (grid != null && grid.getBuckets().size() > 0) { - tileBuilder.addLayers(buildAggsLayer(grid, request, geomBuilder)); + tileBuilder.addLayers(buildAggsLayer(grid, request, featureFactory)); } ensureOpen(); final InternalGeoBounds bounds = searchResponse.getAggregations() != null @@ -162,7 +160,7 @@ public RestResponse buildResponse(SearchResponse searchResponse) throws Exceptio searchResponse.getShardFailures(), searchResponse.getClusters() ); - tileBuilder.addLayers(buildMetaLayer(meta, bounds, request, geomBuilder)); + tileBuilder.addLayers(buildMetaLayer(meta, bounds, request, featureFactory)); ensureOpen(); tileBuilder.build().writeTo(bytesOut); return new BytesRestResponse(RestStatus.OK, MIME_TYPE, bytesOut.bytes()); @@ -187,7 +185,9 @@ private static SearchRequestBuilder searchRequestBuilder(RestCancellableNodeClie searchRequestBuilder.addFetchField(field); } searchRequestBuilder.setRuntimeMappings(request.getRuntimeMappings()); - QueryBuilder qBuilder = QueryBuilders.geoShapeQuery(request.getField(), request.getBoundingBox()); + // For Hex aggregation we might need to buffer the bounding box + final Rectangle boxFilter = request.getGridAgg().bufferTile(request.getBoundingBox(), request.getZ(), request.getGridPrecision()); + QueryBuilder qBuilder = QueryBuilders.geoShapeQuery(request.getField(), boxFilter); if (request.getQueryBuilder() != null) { final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); boolQueryBuilder.filter(request.getQueryBuilder()); @@ -201,14 +201,16 @@ private static SearchRequestBuilder searchRequestBuilder(RestCancellableNodeClie new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) ); - final int extent = 1 << request.getGridPrecision(); - final GeoGridAggregationBuilder tileAggBuilder = new GeoTileGridAggregationBuilder(GRID_FIELD).field(request.getField()) - .precision(Math.min(GeoTileUtils.MAX_ZOOM, request.getZ() + request.getGridPrecision())) + final GeoGridAggregationBuilder tileAggBuilder = request.getGridAgg() + .newAgg(GRID_FIELD) + .field(request.getField()) + .precision(request.getGridAgg().gridPrecisionToAggPrecision(request.getZ(), request.getGridPrecision())) .setGeoBoundingBox(boundingBox) - .size(extent * extent); + .size(MultiBucketConsumerService.DEFAULT_MAX_BUCKETS); + searchRequestBuilder.addAggregation(tileAggBuilder); searchRequestBuilder.addAggregation(new StatsBucketPipelineAggregationBuilder(COUNT_TAG, GRID_FIELD + "." + COUNT_TAG)); - if (request.getGridType() == VectorTileRequest.GRID_TYPE.CENTROID) { + if (request.getGridType() == GridType.CENTROID) { tileAggBuilder.subAggregation(new GeoCentroidAggregationBuilder(CENTROID_AGG_NAME).field(request.getField())); } final List> aggregations = request.getAggBuilder(); @@ -282,9 +284,9 @@ private static VectorTile.Tile.Layer.Builder buildHitsLayer(SearchHit[] hits, Ve } private static VectorTile.Tile.Layer.Builder buildAggsLayer( - InternalGeoTileGrid grid, + InternalGeoGrid grid, VectorTileRequest request, - SimpleFeatureFactory geomBuilder + FeatureFactory featureFactory ) throws IOException { final VectorTile.Tile.Layer.Builder aggLayerBuilder = VectorTileUtils.createLayerBuilder(AGGS_LAYER, request.getExtent()); final MvtLayerProps layerProps = new MvtLayerProps(); @@ -293,23 +295,13 @@ private static VectorTile.Tile.Layer.Builder buildAggsLayer( featureBuilder.clear(); final String bucketKey = bucket.getKeyAsString(); // Add geometry - switch (request.getGridType()) { - case GRID -> { - final Rectangle r = GeoTileUtils.toBoundingBox(bucketKey); - featureBuilder.mergeFrom(geomBuilder.box(r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat())); - } - case POINT -> { - final GeoPoint point = (GeoPoint) bucket.getKey(); - featureBuilder.mergeFrom(geomBuilder.point(point.lon(), point.lat())); - } - case CENTROID -> { - final Rectangle r = GeoTileUtils.toBoundingBox(bucketKey); - final InternalGeoCentroid centroid = bucket.getAggregations().get(CENTROID_AGG_NAME); - final double featureLon = Math.min(Math.max(centroid.centroid().lon(), r.getMinLon()), r.getMaxLon()); - final double featureLat = Math.min(Math.max(centroid.centroid().lat(), r.getMinLat()), r.getMaxLat()); - featureBuilder.mergeFrom(geomBuilder.point(featureLon, featureLat)); - } - default -> throw new IllegalArgumentException("unsupported grid type + [" + request.getGridType() + "]"); + final byte[] feature = request.getGridType().toFeature(request.getGridAgg(), bucket, bucketKey, featureFactory); + if (feature != null) { + featureBuilder.mergeFrom(feature); + } else { + // It can only happen in GeoHexAggregation because hex bins are not aligned with the tiles. + assert request.getGridAgg() == GridAggregation.GEOHEX; + continue; } // Add bucket key as key value pair VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, KEY_TAG, bucketKey); @@ -330,7 +322,7 @@ private static VectorTile.Tile.Layer.Builder buildMetaLayer( SearchResponse response, InternalGeoBounds bounds, VectorTileRequest request, - SimpleFeatureFactory geomBuilder + FeatureFactory featureFactory ) throws IOException { final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder(META_LAYER, request.getExtent()); final MvtLayerProps layerProps = new MvtLayerProps(); @@ -338,10 +330,10 @@ private static VectorTile.Tile.Layer.Builder buildMetaLayer( if (bounds != null && bounds.topLeft() != null) { final GeoPoint topLeft = bounds.topLeft(); final GeoPoint bottomRight = bounds.bottomRight(); - featureBuilder.mergeFrom(geomBuilder.box(topLeft.lon(), bottomRight.lon(), bottomRight.lat(), topLeft.lat())); + featureBuilder.mergeFrom(featureFactory.box(topLeft.lon(), bottomRight.lon(), bottomRight.lat(), topLeft.lat())); } else { final Rectangle tile = request.getBoundingBox(); - featureBuilder.mergeFrom(geomBuilder.box(tile.getMinLon(), tile.getMaxLon(), tile.getMinLat(), tile.getMaxLat())); + featureBuilder.mergeFrom(featureFactory.box(tile.getMinLon(), tile.getMaxLon(), tile.getMinLat(), tile.getMaxLat())); } VectorTileUtils.addToXContentToFeature(featureBuilder, layerProps, response); metaLayerBuilder.addFeatures(featureBuilder); diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequest.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequest.java index 6b2bf91cc67e8..8fae1d8005d5d 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequest.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequest.java @@ -32,7 +32,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Locale; import java.util.Map; import static java.util.Collections.emptyList; @@ -52,34 +51,21 @@ class VectorTileRequest { protected static final String X_PARAM = "x"; protected static final String Y_PARAM = "y"; - protected static final ParseField GRID_PRECISION_FIELD = new ParseField("grid_precision"); + protected static final ParseField GRID_AGG_FIELD = new ParseField("grid_agg"); protected static final ParseField GRID_TYPE_FIELD = new ParseField("grid_type"); + protected static final ParseField GRID_PRECISION_FIELD = new ParseField("grid_precision"); protected static final ParseField EXTENT_FIELD = new ParseField("extent"); protected static final ParseField EXACT_BOUNDS_FIELD = new ParseField("exact_bounds"); - protected enum GRID_TYPE { - GRID, - POINT, - CENTROID; - - private static GRID_TYPE fromString(String type) { - return switch (type.toLowerCase(Locale.ROOT)) { - case "grid" -> GRID; - case "point" -> POINT; - case "centroid" -> CENTROID; - default -> throw new IllegalArgumentException("Invalid grid type [" + type + "]"); - }; - } - } - protected static class Defaults { public static final int SIZE = 10000; public static final List FETCH = emptyList(); public static final Map RUNTIME_MAPPINGS = emptyMap(); public static final QueryBuilder QUERY = null; public static final List> AGGS = emptyList(); + public static final GridAggregation GRID_AGG = GridAggregation.GEOTILE; public static final int GRID_PRECISION = 8; - public static final GRID_TYPE GRID_TYPE = VectorTileRequest.GRID_TYPE.GRID; + public static final GridType GRID_TYPE = GridType.GRID; public static final int EXTENT = 4096; public static final boolean EXACT_BOUNDS = false; public static final int TRACK_TOTAL_HITS_UP_TO = DEFAULT_TRACK_TOTAL_HITS_UP_TO; @@ -122,6 +108,7 @@ protected static class Defaults { ObjectParser.ValueType.OBJECT_ARRAY ); // Specific for vector tiles + PARSER.declareString(VectorTileRequest::setGridAgg, GRID_AGG_FIELD); PARSER.declareInt(VectorTileRequest::setGridPrecision, GRID_PRECISION_FIELD); PARSER.declareString(VectorTileRequest::setGridType, GRID_TYPE_FIELD); PARSER.declareInt(VectorTileRequest::setExtent, EXTENT_FIELD); @@ -161,6 +148,9 @@ static VectorTileRequest parseRestRequest(RestRequest restRequest) throws IOExce if (restRequest.hasParam(EXTENT_FIELD.getPreferredName())) { request.setExtent(restRequest.paramAsInt(EXTENT_FIELD.getPreferredName(), Defaults.EXTENT)); } + if (restRequest.hasParam(GRID_AGG_FIELD.getPreferredName())) { + request.setGridAgg(restRequest.param(GRID_AGG_FIELD.getPreferredName(), Defaults.GRID_AGG.name())); + } if (restRequest.hasParam(GRID_TYPE_FIELD.getPreferredName())) { request.setGridType(restRequest.param(GRID_TYPE_FIELD.getPreferredName(), Defaults.GRID_TYPE.name())); } @@ -200,7 +190,8 @@ static VectorTileRequest parseRestRequest(RestRequest restRequest) throws IOExce private QueryBuilder queryBuilder = Defaults.QUERY; private Map runtimeMappings = Defaults.RUNTIME_MAPPINGS; private int gridPrecision = Defaults.GRID_PRECISION; - private GRID_TYPE gridType = Defaults.GRID_TYPE; + private GridAggregation gridAgg = Defaults.GRID_AGG; + private GridType gridType = Defaults.GRID_TYPE; private int size = Defaults.SIZE; private int extent = Defaults.EXTENT; private List> aggs = Defaults.AGGS; @@ -301,12 +292,20 @@ private void setGridPrecision(int gridPrecision) { this.gridPrecision = gridPrecision; } - public GRID_TYPE getGridType() { + public GridAggregation getGridAgg() { + return gridAgg; + } + + private void setGridAgg(String gridAgg) { + this.gridAgg = GridAggregation.fromString(gridAgg); + } + + public GridType getGridType() { return gridType; } private void setGridType(String gridType) { - this.gridType = GRID_TYPE.fromString(gridType); + this.gridType = GridType.fromString(gridType); } public int getSize() { diff --git a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoriesConsistencyTests.java b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoriesConsistencyTests.java index 8080067f71aa1..806aa48c97604 100644 --- a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoriesConsistencyTests.java +++ b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoriesConsistencyTests.java @@ -9,7 +9,6 @@ import org.apache.lucene.tests.geo.GeoTestUtil; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.geo.SimpleFeatureFactory; import org.elasticsearch.geometry.MultiPoint; import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Rectangle; @@ -28,21 +27,20 @@ public void testPoint() throws IOException { int y = randomIntBetween(0, (1 << z) - 1); int extent = randomIntBetween(1 << 8, 1 << 14); Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z); - SimpleFeatureFactory builder = new SimpleFeatureFactory(z, x, y, extent); FeatureFactory factory = new FeatureFactory(z, x, y, extent); List points = new ArrayList<>(); List geoPoints = new ArrayList<>(); for (int i = 0; i < 10; i++) { double lat = randomValueOtherThanMany((l) -> rectangle.getMinY() > l || rectangle.getMaxY() < l, GeoTestUtil::nextLatitude); double lon = randomValueOtherThanMany((l) -> rectangle.getMinX() > l || rectangle.getMaxX() < l, GeoTestUtil::nextLongitude); - byte[] b1 = builder.point(lon, lat); + byte[] b1 = factory.point(lon, lat); Point point = new Point(lon, lat); byte[] b2 = factory.getFeatures(point).get(0); assertArrayEquals(b1, b2); points.add(point); geoPoints.add(new GeoPoint(lat, lon)); } - byte[] b1 = builder.points(geoPoints); + byte[] b1 = factory.points(geoPoints); byte[] b2 = factory.getFeatures(new MultiPoint(points)).get(0); assertArrayEquals(b1, b2); } @@ -52,11 +50,10 @@ public void testRectangle() throws IOException { int x = randomIntBetween(0, (1 << z) - 1); int y = randomIntBetween(0, (1 << z) - 1); int extent = randomIntBetween(1 << 8, 1 << 14); - SimpleFeatureFactory builder = new SimpleFeatureFactory(z, x, y, extent); FeatureFactory factory = new FeatureFactory(z, x, y, extent); Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); for (int i = 0; i < extent; i++) { - byte[] b1 = builder.box(r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); + byte[] b1 = factory.box(r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); byte[] b2 = factory.getFeatures(r).get(0); assertArrayEquals(extent + "", b1, b2); } @@ -67,19 +64,18 @@ public void testDegeneratedRectangle() throws IOException { int x = randomIntBetween(1, (1 << z) - 1); int y = randomIntBetween(1, (1 << z) - 1); int extent = randomIntBetween(1 << 8, 1 << 14); - SimpleFeatureFactory builder = new SimpleFeatureFactory(z, x, y, extent); FeatureFactory factory = new FeatureFactory(z, x, y, extent); { Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); // box is a point - byte[] b1 = builder.box(r.getMaxLon(), r.getMaxLon(), r.getMaxLat(), r.getMaxLat()); + byte[] b1 = factory.box(r.getMaxLon(), r.getMaxLon(), r.getMaxLat(), r.getMaxLat()); byte[] b2 = factory.getFeatures(new Rectangle(r.getMaxLon(), r.getMaxLon(), r.getMaxLat(), r.getMaxLat())).get(0); assertArrayEquals(extent + "", b1, b2); } { Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); // box is a line - byte[] b1 = builder.box(r.getMinLon(), r.getMinLon(), r.getMinLat(), r.getMaxLat()); + byte[] b1 = factory.box(r.getMinLon(), r.getMinLon(), r.getMinLat(), r.getMaxLat()); byte[] b2 = factory.getFeatures(new Rectangle(r.getMinLon(), r.getMinLon(), r.getMaxLat(), r.getMinLat())).get(0); assertArrayEquals(extent + "", b1, b2); } diff --git a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/GridAggregationTests.java b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/GridAggregationTests.java new file mode 100644 index 0000000000000..ed9f910818645 --- /dev/null +++ b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/GridAggregationTests.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectortile.rest; + +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.h3.CellBoundary; +import org.elasticsearch.h3.H3; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.test.ESTestCase; + +public class GridAggregationTests extends ESTestCase { + + /** Make sure that buffering covers all points it should be covering */ + public void testGeoHexBufferTile() { + final Point point = randomValueOtherThanMany( + p -> Math.abs(p.getLat()) > GeoTileUtils.LATITUDE_MASK, + GeometryTestUtils::randomPoint + ); + final int z = randomIntBetween(0, GeoTileUtils.MAX_ZOOM); + final int x = GeoTileUtils.getXTile(point.getLon(), 1 << z); + final int y = GeoTileUtils.getYTile(point.getLat(), 1 << z); + // current tile + final Rectangle tile = GeoTileUtils.toBoundingBox(x, y, z); + for (int i = 1; i <= 8; i++) { + final int geoHexPrecision = GridAggregation.GEOHEX.gridPrecisionToAggPrecision(z, i); + // buffered tile + final Rectangle bufferedTile = GridAggregation.GEOHEX.bufferTile(tile, z, i); + // Hex bin of the original point + final long l = H3.geoToH3(point.getLat(), point.getLon(), geoHexPrecision); + final CellBoundary boundary = H3.h3ToGeoBoundary(l); + // Check that all points of the hex bin are inside our buffered tile + for (int j = 0; j < boundary.numPoints(); j++) { + final double lat = boundary.getLatLon(j).getLatDeg(); + if (Math.abs(lat) <= GeoTileUtils.LATITUDE_MASK) { // We only consider points inside the mercator projection + assertTrue(bufferedTile.getMinLat() <= lat && bufferedTile.getMaxLat() >= lat); + final double lon = boundary.getLatLon(j).getLonDeg(); + if (bufferedTile.getMinLon() < bufferedTile.getMaxLon()) { + assertTrue(bufferedTile.getMinLon() <= lon && bufferedTile.getMaxLon() >= lon); + } else { + assertTrue(bufferedTile.getMinLon() <= lon || bufferedTile.getMaxLon() >= lon); + } + } + } + } + } +} diff --git a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java index cd63fe150a3e0..f8c468757a64c 100644 --- a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java +++ b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java @@ -52,6 +52,7 @@ public void testDefaults() throws IOException { assertThat(vectorTileRequest.getExtent(), Matchers.equalTo(VectorTileRequest.Defaults.EXTENT)); assertThat(vectorTileRequest.getAggBuilder(), Matchers.equalTo(VectorTileRequest.Defaults.AGGS)); assertThat(vectorTileRequest.getFieldAndFormats(), Matchers.equalTo(VectorTileRequest.Defaults.FETCH)); + assertThat(vectorTileRequest.getGridAgg(), Matchers.equalTo(VectorTileRequest.Defaults.GRID_AGG)); assertThat(vectorTileRequest.getGridType(), Matchers.equalTo(VectorTileRequest.Defaults.GRID_TYPE)); assertThat(vectorTileRequest.getGridPrecision(), Matchers.equalTo(VectorTileRequest.Defaults.GRID_PRECISION)); assertThat(vectorTileRequest.getExactBounds(), Matchers.equalTo(VectorTileRequest.Defaults.EXACT_BOUNDS)); @@ -111,8 +112,16 @@ public void testFieldFetch() throws IOException { ); } + public void testFieldGridAgg() throws IOException { + final GridAggregation grid_agg = RandomPicks.randomFrom(random(), GridAggregation.values()); + assertRestRequest( + (builder) -> { builder.field(VectorTileRequest.GRID_AGG_FIELD.getPreferredName(), grid_agg.name()); }, + (vectorTileRequest) -> { assertThat(vectorTileRequest.getGridAgg(), Matchers.equalTo(grid_agg)); } + ); + } + public void testFieldGridType() throws IOException { - final VectorTileRequest.GRID_TYPE grid_type = RandomPicks.randomFrom(random(), VectorTileRequest.GRID_TYPE.values()); + final GridType grid_type = RandomPicks.randomFrom(random(), GridType.values()); assertRestRequest( (builder) -> { builder.field(VectorTileRequest.GRID_TYPE_FIELD.getPreferredName(), grid_type.name()); }, (vectorTileRequest) -> { assertThat(vectorTileRequest.getGridType(), Matchers.equalTo(grid_type)); } From d1ac4d4f785cf8d9153136fde753be0c0a0eb485 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Wed, 2 Mar 2022 09:57:11 +0100 Subject: [PATCH 02/10] Update docs/changelog/84553.yaml --- docs/changelog/84553.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/84553.yaml diff --git a/docs/changelog/84553.yaml b/docs/changelog/84553.yaml new file mode 100644 index 0000000000000..83811967ad151 --- /dev/null +++ b/docs/changelog/84553.yaml @@ -0,0 +1,5 @@ +pr: 84553 +summary: Integrate `GeoHexGridAggregation` with vector tiles API +area: Geo +type: enhancement +issues: [] From 6cea458335a8606219ce618c53802ee599129750 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Wed, 2 Mar 2022 10:06:26 -0500 Subject: [PATCH 03/10] Doc edits --- docs/changelog/84553.yaml | 2 +- .../search/search-vector-tile-api.asciidoc | 129 +++++++++++------- 2 files changed, 77 insertions(+), 54 deletions(-) diff --git a/docs/changelog/84553.yaml b/docs/changelog/84553.yaml index 83811967ad151..f020bead011ac 100644 --- a/docs/changelog/84553.yaml +++ b/docs/changelog/84553.yaml @@ -1,5 +1,5 @@ pr: 84553 -summary: Integrate `GeoHexGridAggregation` with vector tiles API +summary: Add `geohex_grid` aggregation to vector tiles API area: Geo type: enhancement issues: [] diff --git a/docs/reference/search/search-vector-tile-api.asciidoc b/docs/reference/search/search-vector-tile-api.asciidoc index 4ac56cefb154c..f0de872c2fecd 100644 --- a/docs/reference/search/search-vector-tile-api.asciidoc +++ b/docs/reference/search/search-vector-tile-api.asciidoc @@ -91,17 +91,19 @@ Internally, {es} translates a vector tile search API request into a * A <> query on the ``. The query uses the `//` tile as a bounding box. -* A <> -aggregation on the ``. The aggregation uses the `//` tile as -a bounding box. +* A <> or +<> aggregation +on the ``. The `grid_agg` parameter determines the aggregation type. The +aggregation uses the `//` tile as a bounding box. * Optionally, a <> aggregation on the ``. The search only includes this aggregation if the `exact_bounds` parameter is `true`. -For example, {es} may translate a vector tile search API request with an -`exact_bounds` argument of `true` into the following search: +For example, {es} may translate a vector tile search API request with a +`grid_agg` argument of `geotile` and an `exact_bounds` argument of `true` +into the following search: [source,console] ---- @@ -159,14 +161,13 @@ Protobufs (PBF)]. By default, the tile contains three layers: * A `hits` layer containing a feature for each `` value matching the `geo_bounding_box` query. -* An `aggs` layer containing a feature for each cell of the `geotile_grid`. You -can use these cells as tiles for lower zoom levels. The layer only contains -features for cells with matching data. +* An `aggs` layer containing a feature for each cell of the `geotile_grid` or +`geohex_grid`. The layer only contains features for cells with matching data. * A `meta` layer containing: ** A feature containing a bounding box. By default, this is the bounding box of the tile. -** Value ranges for any sub-aggregations on the `geotile_grid`. +** Value ranges for any sub-aggregations on the `geotile_grid` or `geohex_grid`. ** Metadata for the search. The API only returns features that can display at its zoom level. For example, @@ -174,6 +175,7 @@ if a polygon feature has no area at its zoom level, the API omits it. The API returns errors as UTF-8 encoded JSON. +[role="child_attributes"] [[search-vector-tile-api-query-params]] ==== {api-query-parms-title} @@ -202,68 +204,87 @@ square with equal sides. Defaults to `4096`. // tag::grid-agg[] `grid_agg`:: -(Optional, string) Determines the GeoGridAggregation used to build the grid. -Accepts: - -`geotile` (Default)::: -Grid is constructed using a GeoTileGridAggregation. - -`geohex`::: -Grid is constructed using a GeoHexGridAggregation. Note that GeoHexGridAggregation -only supports running against geo_point field type. +(Optional, string) Aggregation used to create a grid for the ``. ++ +.Valid values for `grid_agg` +[%collapsible%open] +==== +`geotile` (Default):: +<> +aggregation. + +`geohex`:: +<> aggregation. +If you specify this value, the `` must be a <> +field. +==== // end::grid-agg[] // tag::grid-precision[] `grid_precision`:: -(Optional, integer) It defines how many bins will be created on the grid -layer at each zoom level which depends on the `grid_agg`. -Accepts `0`-`8`. Defaults to `8`. If `0`, results don't include the -`aggs` layer. -+ -In case of a geotile aggregation, the grid precision represents the additional zoom levels -available through the `aggs` layer. For example, if `` is `7` -and `grid_precision` is `8`, then getile aggregation will use precision 15. -+ -The precision is computed by as follows: +(Optional, integer) Precision level for cells in the `grid_agg`. Accepts +`0`-`8`. Defaults to `8`. If `0`, results don't include the `aggs` layer. + +.Grid precision for `geotile` +[%collapsible%open] +==== +For a `grid_agg` of `geotile`, you can use cells in the `aggs` layer as tiles +for lower zoom levels. `grid_precision` represents the additional zoom levels +available through these cells. The final precision is computed by as +follows: + ` + grid_precision` -+ -With a maximum value of 29. This value determines the -number of cells of the `geotile_grid` as follows: -+ + +For example, if `` is `7` and `grid_precision` is `8`, then the +`geotile_grid` aggregation will use a precision of `15`. The maximum final +precision is `29`. + +The `grid_precision` also determines the number of cells for the grid as +follows: + `(2^grid_precision) x (2^grid_precision)` -+ + For example, a value of `8` divides the tile into a grid of 256 x 256 cells. The `aggs` layer only contains features for cells with matching data. +==== + -In case of a geohex aggregation, the grid precision translated to the +.Grid precision for `geohex` +[%collapsible%open] +==== +For a `grid_agg` of `geohex`, `grid_precision` is used to calculate the https://h3geo.org/docs/core-library/restable[precision of the hexagonal cells]. -+ -The precision for the hex grid is computed as follows: -+ -`( + gridPrecision - 1) / 2` -+ -where the minimum precision for a hexagonal bin is 2 and the maximum precision is 15. -Note that hexagonal cells do not align perfectly on the vector tile so some cells might -intersect more than one vector tile. +This is computed as follows: + +`( + grid_precision - 1) / 2` + +The minimum final precision is `2`. The maximum final precision is `15`. + +Hexagonal cells don't align perfectly on a vector tile. Some cells may intersect +more than one vector tile. +==== // end::grid-precision[] // tag::grid-type[] `grid_type`:: (Optional, string) Determines the geometry type for features in the `aggs` -layer. In the `aggs` layer, each feature represents a `geotile_grid` cell. -Accepts: - -`grid` (Default)::: -Each feature is a `Polygon` of the cell's geometry that depends on the selected `grid_agg`. +layer. In the `aggs` layer, each feature represents a cell in the grid. ++ +.Valid values for `grid_type` +[%collapsible%open] +==== +`grid` (Default):: +Each feature is a `Polygon` of the cell's geometry. For a `grid_agg` of +`geotile`, the feature is the cell's bounding box. For a `grid_agg` of +`geohex`, the feature is the hexagonal cell's boundaries. -`point`::: +`point`:: Each feature is a `Point` that's the centroid of the cell. -`centroid`::: +`centroid`:: Each feature is a `Point` that's the centroid of the data within the cell. For complex geometries, the actual centroid may be outside the cell. In these cases, the feature is set to the closest point to the centroid inside the cell. +==== // end::grid-type[] // tag::size[] @@ -288,7 +309,7 @@ If `false`, the response does not include the total number of hits matching the `aggs`:: (Optional, <>) -<> for the `geotile_grid`. Supports the following +<> for the `grid_agg`. Supports the following aggregation types: + * <> @@ -326,6 +347,8 @@ You can specify fields in the array as a string or object. include::search.asciidoc[tag=fields-param-props] ==== +include::search-vector-tile-api.asciidoc[tag=grid-agg] + include::search-vector-tile-api.asciidoc[tag=grid-precision] include::search-vector-tile-api.asciidoc[tag=grid-type] @@ -430,7 +453,7 @@ Field value. Only returned for fields in the `fields` parameter. ==== `aggs`:: -(object) Layer containing results for the `geotile_grid` aggregation and its +(object) Layer containing results for the `grid_agg` aggregation and its sub-aggregations. + .Properties of `aggs` @@ -441,8 +464,7 @@ include::search-vector-tile-api.asciidoc[tag=extent] include::search-vector-tile-api.asciidoc[tag=version] `features`:: -(array of objects) Array of features. Contains a feature for each cell of the -`geotile_grid`. +(array of objects) Array of features. Contains a feature for each cell of the grid. + .Properties of `features` objects [%collapsible%open] @@ -615,6 +637,7 @@ the `13/4207/2692` vector tile. ---- GET museums/_mvt/location/13/4207/2692 { + "grid_agg": "geotile", "grid_precision": 2, "fields": [ "name", From d2b5f5e554f2a0d98f60b2b372be718020f0ec06 Mon Sep 17 00:00:00 2001 From: iverase Date: Mon, 14 Mar 2022 09:52:46 +0100 Subject: [PATCH 04/10] Update how H3 resolutions are zoom levels are mapped --- .../search/search-vector-tile-api.asciidoc | 47 ++++++-- .../vectortile/rest/GridAggregation.java | 105 ++++++++++++------ 2 files changed, 109 insertions(+), 43 deletions(-) diff --git a/docs/reference/search/search-vector-tile-api.asciidoc b/docs/reference/search/search-vector-tile-api.asciidoc index f0de872c2fecd..f27dd20b02a6c 100644 --- a/docs/reference/search/search-vector-tile-api.asciidoc +++ b/docs/reference/search/search-vector-tile-api.asciidoc @@ -253,14 +253,47 @@ For example, a value of `8` divides the tile into a grid of 256 x 256 cells. The ==== For a `grid_agg` of `geohex`, `grid_precision` is used to calculate the https://h3geo.org/docs/core-library/restable[precision of the hexagonal cells]. -This is computed as follows: -`( + grid_precision - 1) / 2` - -The minimum final precision is `2`. The maximum final precision is `15`. - -Hexagonal cells don't align perfectly on a vector tile. Some cells may intersect -more than one vector tile. +Hexagonal cells don't align perfectly on a vector tile, some cells may intersect +more than one vector tile. In order to compute the resolution, we compute the +average density of hexagonal bins and use the resolution that is closer to the +`geotile` density. + +The mapping between zoom levels and H3 resolutions is giving in the following table. + +[cols="<,<,<,<,<",options="header",] +|======================================================================= +|zoom level | unique tile bins| H3 resolution| unique hex bins | ratio +|1 |4 |0 |122 |30.5 +|2 |16 |0 |122 |7.625 +|3 |64 |1 |842 |13.15625 +|4 |256 |1 |842 |3.2890625 +|5 |1024 |2 |5882 |5.744140625 +|6 |4096 |2 |5882 |1.436035156 +|7 |16384 |3 |41162 |2.512329102 +|8 |65536 |3 |41162 |0.6280822754 +|9 |262144 |4 |288122 |1.099098206 +|10 |1048576 |4 |288122 |0.2747745514 +|11 |4194304 |5 |2016842 |0.4808526039 +|12 |16777216 |6 |14117882 |0.8414913416 +|13 |67108864 |6 |14117882 |0.2103728354 +|14 |268435456 |7 |98825162 |0.3681524172 +|15 |1073741824 |8 |691776122 |0.644266719 +|16 |4294967296 |8 |691776122 |0.1610666797 +|17 |17179869184 |9 |4842432842 |0.2818666889 +|18 |68719476736 |10 |33897029882 |0.4932667053 +|19 |274877906944 |11 |237279209162 |0.8632167343 +|20 |1099511627776 |11 |237279209162 |0.2158041836 +|21 |4398046511104 |12 |1660954464122 |0.3776573213 +|22 |17592186044416 |13 |11626681248842 |0.6609003122 +|23 |70368744177664 |13 |11626681248842 |0.165225078 +|24 |281474976710656 |14 |81386768741882 |0.2891438866 +|25 |1125899906842620 |15 |569707381193162 |0.5060018015 +|26 |4503599627370500 |15 |569707381193162 |0.1265004504 +|27 |18014398509482000 |15 |569707381193162 |0.03162511259 +|28 |72057594037927900 |15 |569707381193162 |0.007906278149 +|29 |288230376151712000 |15 |569707381193162 |0.001976569537 +|======================================================================= ==== // end::grid-precision[] diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java index 83103f29e27c8..f7397dcb8be07 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java @@ -61,7 +61,7 @@ public Rectangle toRectangle(String bucketKey) { GEOHEX { // Because hex bins do not fit perfectly on a tile, we need to buffer our queries in order to collect - // all points inside the bin. We never have aggregations for levels 0 and 1, values for level 2 have + // all points inside the bin. For levels 0 and 1 we will consider all data, values for level 2 have // been computed manually and approximated. The amount that the buffer decreases by level has been // approximated to 2.5 (brute force computation suggest ~2.6 in the first 9 levels so 2.5 should be safe). private static final double[] LAT_BUFFER_SIZE = new double[16]; @@ -76,6 +76,69 @@ public Rectangle toRectangle(String bucketKey) { LON_BUFFER_SIZE[i] = LON_BUFFER_SIZE[i - 1] / 2.5; } } + // Mapping between a vector tile zoom and a H3 resolution. The mapping tries to keep the density of hexes similar + // to the density of tile bins but trying not to be bigger. + // Level unique tiles H3 resolution unique hexes ratio + // 1 4 0 122 30.5 + // 2 16 0 122 7.625 + // 3 64 1 842 13.15625 + // 4 256 1 842 3.2890625 + // 5 1024 2 5882 5.744140625 + // 6 4096 2 5882 1.436035156 + // 7 16384 3 41162 2.512329102 + // 8 65536 3 41162 0.6280822754 + // 9 262144 4 288122 1.099098206 + // 10 1048576 4 288122 0.2747745514 + // 11 4194304 5 2016842 0.4808526039 + // 12 16777216 6 14117882 0.8414913416 + // 13 67108864 6 14117882 0.2103728354 + // 14 268435456 7 98825162 0.3681524172 + // 15 1073741824 8 691776122 0.644266719 + // 16 4294967296 8 691776122 0.1610666797 + // 17 17179869184 9 4842432842 0.2818666889 + // 18 68719476736 10 33897029882 0.4932667053 + // 19 274877906944 11 237279209162 0.8632167343 + // 20 1099511627776 11 237279209162 0.2158041836 + // 21 4398046511104 12 1660954464122 0.3776573213 + // 22 17592186044416 13 11626681248842 0.6609003122 + // 23 70368744177664 13 11626681248842 0.165225078 + // 24 281474976710656 14 81386768741882 0.2891438866 + // 25 1125899906842620 15 569707381193162 0.5060018015 + // 26 4503599627370500 15 569707381193162 0.1265004504 + // 27 18014398509482000 15 569707381193162 0.03162511259 + // 28 72057594037927900 15 569707381193162 0.007906278149 + // 29 288230376151712000 15 569707381193162 0.001976569537 + private static final int[] ZOOM2RESOLUTION = new int[] { + 0, + 0, + 0, + 1, + 1, + 2, + 2, + 3, + 3, + 4, + 4, + 5, + 6, + 6, + 7, + 8, + 8, + 9, + 10, + 11, + 11, + 12, + 13, + 13, + 14, + 15, + 15, + 15, + 15, + 15 }; @Override public GeoGridAggregationBuilder newAgg(String aggName) { @@ -89,6 +152,10 @@ public Rectangle bufferTile(Rectangle tile, int z, int gridPrecision) { return tile; } final int aggPrecision = gridPrecisionToAggPrecision(z, gridPrecision); + if (aggPrecision < 2) { + // we need to consider all data + return new Rectangle(-180, 180, GeoTileUtils.LATITUDE_MASK, -GeoTileUtils.LATITUDE_MASK); + } return new Rectangle( GeoUtils.normalizeLon(tile.getMinX() - LON_BUFFER_SIZE[aggPrecision]), GeoUtils.normalizeLon(tile.getMaxX() + LON_BUFFER_SIZE[aggPrecision]), @@ -99,41 +166,7 @@ public Rectangle bufferTile(Rectangle tile, int z, int gridPrecision) { @Override public int gridPrecisionToAggPrecision(int z, int gridPrecision) { - // The minimum resolution we allow is 2 to avoid having hexagons containing the pole. - // this is the table of Hex precision: - // precision: 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | - // ------------------------------------------- - // zoom 0: 2 | 2 | 2 | 2 | 2 | 2 | 3 | 3 | - // zoom 1: 2 | 2 | 2 | 2 | 2 | 3 | 3 | 4 | - // zoom 2: 2 | 2 | 2 | 2 | 3 | 3 | 4 | 4 | - // zoom 3: 2 | 2 | 2 | 3 | 3 | 4 | 4 | 5 | - // zoom 4: 2 | 2 | 3 | 3 | 4 | 4 | 5 | 5 | - // zoom 5: 2 | 3 | 3 | 4 | 4 | 5 | 5 | 6 | - // zoom 6: 3 | 3 | 4 | 4 | 5 | 5 | 6 | 6 | - // zoom 7: 3 | 4 | 4 | 5 | 5 | 6 | 6 | 7 | - // zoom 8: 4 | 4 | 5 | 5 | 6 | 6 | 7 | 7 | - // zoom 9: 4 | 5 | 5 | 6 | 6 | 7 | 7 | 8 | - // zoom 10: 5 | 5 | 6 | 6 | 7 | 7 | 8 | 8 | - // zoom 11: 5 | 6 | 6 | 7 | 7 | 8 | 8 | 9 | - // zoom 12: 6 | 6 | 7 | 7 | 8 | 8 | 9 | 9 | - // zoom 13: 6 | 7 | 7 | 8 | 8 | 9 | 9 | 10 | - // zoom 14: 7 | 7 | 8 | 8 | 9 | 9 | 10 | 10 | - // zoom 15: 7 | 8 | 8 | 9 | 9 | 10 | 10 | 11 | - // zoom 16: 8 | 8 | 9 | 9 | 10 | 10 | 11 | 11 | - // zoom 17: 8 | 9 | 9 | 10 | 10 | 11 | 11 | 12 | - // zoom 18: 9 | 9 | 10 | 10 | 11 | 11 | 12 | 12 | - // zoom 19: 9 | 10 | 10 | 11 | 11 | 12 | 12 | 13 | - // zoom 20: 10 | 10 | 11 | 11 | 12 | 12 | 13 | 13 | - // zoom 21: 10 | 11 | 11 | 12 | 12 | 13 | 13 | 14 | - // zoom 22: 11 | 11 | 12 | 12 | 13 | 13 | 14 | 14 | - // zoom 23: 11 | 12 | 12 | 13 | 13 | 14 | 14 | 15 | - // zoom 24: 12 | 12 | 13 | 13 | 14 | 14 | 15 | 15 | - // zoom 25: 12 | 13 | 13 | 14 | 14 | 15 | 15 | 15 | - // zoom 26: 13 | 13 | 14 | 14 | 15 | 15 | 15 | 15 | - // zoom 27: 13 | 14 | 14 | 15 | 15 | 15 | 15 | 15 | - // zoom 28: 14 | 14 | 15 | 15 | 15 | 15 | 15 | 15 | - // zoom 29: 14 | 15 | 15 | 15 | 15 | 15 | 15 | 15 | - return Math.min(H3.MAX_H3_RES, Math.max(2, (z + gridPrecision - 1) / 2)); + return ZOOM2RESOLUTION[GEOTILE.gridPrecisionToAggPrecision(z, gridPrecision)]; } @Override From f82de835e2aa9acc9ec2a6a1d3dffc8517e9833b Mon Sep 17 00:00:00 2001 From: iverase Date: Tue, 15 Mar 2022 10:55:54 +0100 Subject: [PATCH 05/10] more docs --- .../search/search-vector-tile-api.asciidoc | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/reference/search/search-vector-tile-api.asciidoc b/docs/reference/search/search-vector-tile-api.asciidoc index f27dd20b02a6c..43bbe4ef8a354 100644 --- a/docs/reference/search/search-vector-tile-api.asciidoc +++ b/docs/reference/search/search-vector-tile-api.asciidoc @@ -252,14 +252,18 @@ For example, a value of `8` divides the tile into a grid of 256 x 256 cells. The [%collapsible%open] ==== For a `grid_agg` of `geohex`, `grid_precision` is used to calculate the -https://h3geo.org/docs/core-library/restable[precision of the hexagonal cells]. +https://h3geo.org/docs/core-library/restable[resolution of the hexagonal cells]. -Hexagonal cells don't align perfectly on a vector tile, some cells may intersect -more than one vector tile. In order to compute the resolution, we compute the -average density of hexagonal bins and use the resolution that is closer to the -`geotile` density. +Hexagonal cells don't align perfectly on a vector tile and some cells may intersect +more than one vector tile. In order to compute the H3 resolution for the precision +` + grid_precision`, we compare the average density of hexagonal bins at each +resolution with the average density of tile bins and each zoom level. We use the H3 +resolution that is closer to the`geotile` density. -The mapping between zoom levels and H3 resolutions is giving in the following table. +The mapping between ` + grid_precision` and H3 resolutions is giving in the following table. +For example if is 3 and grid_precision is 3, hexagonal cells are created with H3 +resolution 2 (zoom level 6 row). If is 3 and grid_precision is 4, hexagonal cells are created +with H3 resolution 3 (zoom level 7 row)." [cols="<,<,<,<,<",options="header",] |======================================================================= @@ -293,8 +297,7 @@ The mapping between zoom levels and H3 resolutions is giving in the following ta |27 |18014398509482000 |15 |569707381193162 |0.03162511259 |28 |72057594037927900 |15 |569707381193162 |0.007906278149 |29 |288230376151712000 |15 |569707381193162 |0.001976569537 -|======================================================================= -==== +|=========================================================================== // end::grid-precision[] // tag::grid-type[] From c990950ad178002a9c01d3d1fd5c86b7a22e750f Mon Sep 17 00:00:00 2001 From: iverase Date: Tue, 15 Mar 2022 11:06:22 +0100 Subject: [PATCH 06/10] more docs --- docs/reference/search/search-vector-tile-api.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/search/search-vector-tile-api.asciidoc b/docs/reference/search/search-vector-tile-api.asciidoc index 43bbe4ef8a354..7e0a67d329f38 100644 --- a/docs/reference/search/search-vector-tile-api.asciidoc +++ b/docs/reference/search/search-vector-tile-api.asciidoc @@ -297,7 +297,7 @@ with H3 resolution 3 (zoom level 7 row)." |27 |18014398509482000 |15 |569707381193162 |0.03162511259 |28 |72057594037927900 |15 |569707381193162 |0.007906278149 |29 |288230376151712000 |15 |569707381193162 |0.001976569537 -|=========================================================================== +|======================================================================= // end::grid-precision[] // tag::grid-type[] From 901b5c19d23984fe013c451bb45a11834b05ee36 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Tue, 15 Mar 2022 10:59:09 -0400 Subject: [PATCH 07/10] Fix docs rendering. Reorder. --- .../search/search-vector-tile-api.asciidoc | 75 ++++++++++--------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/docs/reference/search/search-vector-tile-api.asciidoc b/docs/reference/search/search-vector-tile-api.asciidoc index 7e0a67d329f38..f62d4e39c64b2 100644 --- a/docs/reference/search/search-vector-tile-api.asciidoc +++ b/docs/reference/search/search-vector-tile-api.asciidoc @@ -251,53 +251,60 @@ For example, a value of `8` divides the tile into a grid of 256 x 256 cells. The .Grid precision for `geohex` [%collapsible%open] ==== -For a `grid_agg` of `geohex`, `grid_precision` is used to calculate the -https://h3geo.org/docs/core-library/restable[resolution of the hexagonal cells]. +For a `grid_agg` of `geohex`, {es} uses `` and `grid_precision` to +calculate a final precision as follows: -Hexagonal cells don't align perfectly on a vector tile and some cells may intersect -more than one vector tile. In order to compute the H3 resolution for the precision -` + grid_precision`, we compare the average density of hexagonal bins at each -resolution with the average density of tile bins and each zoom level. We use the H3 -resolution that is closer to the`geotile` density. +` + grid_precision` + +This precision determines the https://h3geo.org/docs/core-library/restable[H3 +resolution of the hexagonal cells] produced by the `geohex` aggregation. The +following table maps the H3 resolution for each precision. -The mapping between ` + grid_precision` and H3 resolutions is giving in the following table. -For example if is 3 and grid_precision is 3, hexagonal cells are created with H3 -resolution 2 (zoom level 6 row). If is 3 and grid_precision is 4, hexagonal cells are created -with H3 resolution 3 (zoom level 7 row)." +For example, if `` is `3` and `grid_precision` is `3`, the precision is +`6`. At a precision of `6`, hexagonal cells have an H3 resolution of `2`. If +`` is `3` and `grid_precision` is `4`, the precision is `7`. At a +precision of `7`, hexagonal cells have an H3 resolution of `3`. [cols="<,<,<,<,<",options="header",] -|======================================================================= -|zoom level | unique tile bins| H3 resolution| unique hex bins | ratio +|==== +|Precision | Unique tile bins| H3 resolution| Unique hex bins | Ratio |1 |4 |0 |122 |30.5 |2 |16 |0 |122 |7.625 |3 |64 |1 |842 |13.15625 -|4 |256 |1 |842 |3.2890625 -|5 |1024 |2 |5882 |5.744140625 -|6 |4096 |2 |5882 |1.436035156 -|7 |16384 |3 |41162 |2.512329102 -|8 |65536 |3 |41162 |0.6280822754 -|9 |262144 |4 |288122 |1.099098206 -|10 |1048576 |4 |288122 |0.2747745514 -|11 |4194304 |5 |2016842 |0.4808526039 -|12 |16777216 |6 |14117882 |0.8414913416 -|13 |67108864 |6 |14117882 |0.2103728354 +|4 |256 |1 |842 |3.2890625 +|5 |1024 |2 |5882 |5.744140625 +|6 |4096 |2 |5882 |1.436035156 +|7 |16384 |3 |41162 |2.512329102 +|8 |65536 |3 |41162 |0.6280822754 +|9 |262144 |4 |288122 |1.099098206 +|10 |1048576 |4 |288122 |0.2747745514 +|11 |4194304 |5 |2016842 |0.4808526039 +|12 |16777216 |6 |14117882 |0.8414913416 +|13 |67108864 |6 |14117882 |0.2103728354 |14 |268435456 |7 |98825162 |0.3681524172 -|15 |1073741824 |8 |691776122 |0.644266719 -|16 |4294967296 |8 |691776122 |0.1610666797 -|17 |17179869184 |9 |4842432842 |0.2818666889 -|18 |68719476736 |10 |33897029882 |0.4932667053 -|19 |274877906944 |11 |237279209162 |0.8632167343 -|20 |1099511627776 |11 |237279209162 |0.2158041836 -|21 |4398046511104 |12 |1660954464122 |0.3776573213 +|15 |1073741824 |8 |691776122 |0.644266719 +|16 |4294967296 |8 |691776122 |0.1610666797 +|17 |17179869184 |9 |4842432842 |0.2818666889 +|18 |68719476736 |10 |33897029882 |0.4932667053 +|19 |274877906944 |11 |237279209162 |0.8632167343 +|20 |1099511627776 |11 |237279209162 |0.2158041836 +|21 |4398046511104 |12 |1660954464122 |0.3776573213 |22 |17592186044416 |13 |11626681248842 |0.6609003122 |23 |70368744177664 |13 |11626681248842 |0.165225078 -|24 |281474976710656 |14 |81386768741882 |0.2891438866 -|25 |1125899906842620 |15 |569707381193162 |0.5060018015 -|26 |4503599627370500 |15 |569707381193162 |0.1265004504 +|24 |281474976710656 |14 |81386768741882 |0.2891438866 +|25 |1125899906842620 |15 |569707381193162 |0.5060018015 +|26 |4503599627370500 |15 |569707381193162 |0.1265004504 |27 |18014398509482000 |15 |569707381193162 |0.03162511259 |28 |72057594037927900 |15 |569707381193162 |0.007906278149 |29 |288230376151712000 |15 |569707381193162 |0.001976569537 -|======================================================================= +|==== + +Hexagonal cells don't align perfectly on a vector tile. Some cells may intersect +more than one vector tile. To compute the H3 resolution for each precision, {es} +compares the average density of hexagonal bins at each resolution with the +average density of tile bins at each zoom level. {es} uses the H3 resolution +that is closest to the corresponding `geotile` density. +==== // end::grid-precision[] // tag::grid-type[] From b71d5319067ecdb5638178c947955f072418e78e Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Tue, 15 Mar 2022 17:37:42 +0100 Subject: [PATCH 08/10] Update x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java Co-authored-by: Craig Taverner --- .../elasticsearch/xpack/vectortile/rest/GridAggregation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java index f7397dcb8be07..26cdfb9d2ca6c 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java @@ -180,7 +180,7 @@ public byte[] toGrid(String bucketKey, FeatureFactory featureFactory) { lons[i] = latLng.getLonDeg(); } lats[boundary.numPoints()] = boundary.getLatLon(0).getLatDeg(); - lons[boundary.numPoints()] = boundary.getLatLon(0).getLonDeg(); + lons[boundary.numPoints()] = lons[0]; final Polygon polygon = new Polygon(new LinearRing(lons, lats)); final List x = featureFactory.getFeatures(GeometryNormalizer.apply(Orientation.CCW, polygon)); return x.size() > 0 ? x.get(0) : null; From e7ea94d131471992e1376184448bb3094fdf7981 Mon Sep 17 00:00:00 2001 From: iverase Date: Tue, 15 Mar 2022 17:38:48 +0100 Subject: [PATCH 09/10] iter --- .../elasticsearch/xpack/vectortile/rest/GridAggregation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java index 26cdfb9d2ca6c..45a756976ea36 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java @@ -179,7 +179,7 @@ public byte[] toGrid(String bucketKey, FeatureFactory featureFactory) { lats[i] = latLng.getLatDeg(); lons[i] = latLng.getLonDeg(); } - lats[boundary.numPoints()] = boundary.getLatLon(0).getLatDeg(); + lats[boundary.numPoints()] = lats[0]; lons[boundary.numPoints()] = lons[0]; final Polygon polygon = new Polygon(new LinearRing(lons, lats)); final List x = featureFactory.getFeatures(GeometryNormalizer.apply(Orientation.CCW, polygon)); From c060cb23d32146dcb037df81eb08484faa20414e Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Tue, 15 Mar 2022 17:39:55 +0100 Subject: [PATCH 10/10] Update x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridType.java Co-authored-by: Craig Taverner --- .../java/org/elasticsearch/xpack/vectortile/rest/GridType.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridType.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridType.java index f1ac053085de0..ac8a4831153ab 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridType.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridType.java @@ -48,7 +48,7 @@ public byte[] toFeature(GridAggregation gridAggregation, InternalGeoGridBucket b } }; - /** Builds the corresponding vector tile feature for the provided bucket */ + /** Builds the corresponding vector tile feature for the provided bucket */ public abstract byte[] toFeature( GridAggregation gridAggregation, InternalGeoGridBucket bucket,