Skip to content

Commit f3cde06

Browse files
nyuriknyurik
authored
geotile_grid implementation (#37842)
Implements `geotile_grid` aggregation This patch refactors previous implementation #30240 This code uses the same base classes as `geohash_grid` agg, but uses a different hashing algorithm to allow zoom consistency. Each grid bucket is aligned to Web Mercator tiles.
1 parent 6c1e9fa commit f3cde06

File tree

22 files changed

+1310
-3
lines changed

22 files changed

+1310
-3
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,8 @@
9595
import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters;
9696
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder;
9797
import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid;
98+
import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoTileGrid;
99+
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
98100
import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
99101
import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal;
100102
import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
@@ -1760,6 +1762,7 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
17601762
map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c));
17611763
map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c));
17621764
map.put(GeoHashGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c));
1765+
map.put(GeoTileGridAggregationBuilder.NAME, (p, c) -> ParsedGeoTileGrid.fromXContent(p, (String) c));
17631766
map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c));
17641767
map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c));
17651768
map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c));

‎docs/reference/aggregations/bucket/geotilegrid-aggregation.asciidoc‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
[[search-aggregations-bucket-geotilegrid-aggregation]]
2+
=== GeoTile Grid Aggregation
3+
4+
A multi-bucket aggregation that works on `geo_point` fields and groups points into
5+
buckets that represent cells in a grid. The resulting grid can be sparse and only
6+
contains cells that have matching data. Each cell corresponds to a
7+
https://en.wikipedia.org/wiki/Tiled_web_map[map tile] as used by many online map
8+
sites. Each cell is labeled using a "{zoom}/{x}/{y}" format, where zoom is equal
9+
to the user-specified precision.
10+
11+
* High precision keys have a larger range for x and y, and represent tiles that
12+
cover only a small area.
13+
* Low precision keys have a smaller range for x and y, and represent tiles that
14+
each cover a large area.
15+
16+
See https://wiki.openstreetmap.org/wiki/Zoom_levels[Zoom level documentation]
17+
on how precision (zoom) correlates to size on the ground. Precision for this
18+
aggregation can be between 0 and 29, inclusive.
19+
20+
WARNING: The highest-precision geotile of length 29 produces cells that cover
21+
less than a 10cm by 10cm of land and so high-precision requests can be very
22+
costly in terms of RAM and result sizes. Please see the example below on how
23+
to first filter the aggregation to a smaller geographic area before requesting
24+
high-levels of detail.
25+
26+
The specified field must be of type `geo_point` (which can only be set
27+
explicitly in the mappings) and it can also hold an array of `geo_point`
28+
fields, in which case all points will be taken into account during aggregation.
29+
30+
31+
==== Simple low-precision request
32+
33+
[source,js]
34+
--------------------------------------------------
35+
PUT /museums
36+
{
37+
"mappings": {
38+
"properties": {
39+
"location": {
40+
"type": "geo_point"
41+
}
42+
}
43+
}
44+
}
45+
46+
POST /museums/_bulk?refresh
47+
{"index":{"_id":1}}
48+
{"location": "52.374081,4.912350", "name": "NEMO Science Museum"}
49+
{"index":{"_id":2}}
50+
{"location": "52.369219,4.901618", "name": "Museum Het Rembrandthuis"}
51+
{"index":{"_id":3}}
52+
{"location": "52.371667,4.914722", "name": "Nederlands Scheepvaartmuseum"}
53+
{"index":{"_id":4}}
54+
{"location": "51.222900,4.405200", "name": "Letterenhuis"}
55+
{"index":{"_id":5}}
56+
{"location": "48.861111,2.336389", "name": "Musée du Louvre"}
57+
{"index":{"_id":6}}
58+
{"location": "48.860000,2.327000", "name": "Musée d'Orsay"}
59+
60+
POST /museums/_search?size=0
61+
{
62+
"aggregations" : {
63+
"large-grid" : {
64+
"geotile_grid" : {
65+
"field" : "location",
66+
"precision" : 8
67+
}
68+
}
69+
}
70+
}
71+
--------------------------------------------------
72+
// CONSOLE
73+
74+
Response:
75+
76+
[source,js]
77+
--------------------------------------------------
78+
{
79+
...
80+
"aggregations": {
81+
"large-grid": {
82+
"buckets": [
83+
{
84+
"key" : "8/131/84",
85+
"doc_count" : 3
86+
},
87+
{
88+
"key" : "8/129/88",
89+
"doc_count" : 2
90+
},
91+
{
92+
"key" : "8/131/85",
93+
"doc_count" : 1
94+
}
95+
]
96+
}
97+
}
98+
}
99+
--------------------------------------------------
100+
// TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/]
101+
102+
==== High-precision requests
103+
104+
When requesting detailed buckets (typically for displaying a "zoomed in" map)
105+
a filter like <<query-dsl-geo-bounding-box-query,geo_bounding_box>> should be
106+
applied to narrow the subject area otherwise potentially millions of buckets
107+
will be created and returned.
108+
109+
[source,js]
110+
--------------------------------------------------
111+
POST /museums/_search?size=0
112+
{
113+
"aggregations" : {
114+
"zoomed-in" : {
115+
"filter" : {
116+
"geo_bounding_box" : {
117+
"location" : {
118+
"top_left" : "52.4, 4.9",
119+
"bottom_right" : "52.3, 5.0"
120+
}
121+
}
122+
},
123+
"aggregations":{
124+
"zoom1":{
125+
"geotile_grid" : {
126+
"field": "location",
127+
"precision": 22
128+
}
129+
}
130+
}
131+
}
132+
}
133+
}
134+
--------------------------------------------------
135+
// CONSOLE
136+
// TEST[continued]
137+
138+
[source,js]
139+
--------------------------------------------------
140+
{
141+
...
142+
"aggregations" : {
143+
"zoomed-in" : {
144+
"doc_count" : 3,
145+
"zoom1" : {
146+
"buckets" : [
147+
{
148+
"key" : "22/2154412/1378379",
149+
"doc_count" : 1
150+
},
151+
{
152+
"key" : "22/2154385/1378332",
153+
"doc_count" : 1
154+
},
155+
{
156+
"key" : "22/2154259/1378425",
157+
"doc_count" : 1
158+
}
159+
]
160+
}
161+
}
162+
}
163+
}
164+
--------------------------------------------------
165+
// TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/]
166+
167+
168+
==== Options
169+
170+
[horizontal]
171+
field:: Mandatory. The name of the field indexed with GeoPoints.
172+
173+
precision:: Optional. The integer zoom of the key used to define
174+
cells/buckets in the results. Defaults to 7.
175+
Values outside of [0,29] will be rejected.
176+
177+
size:: Optional. The maximum number of geohash buckets to return
178+
(defaults to 10,000). When results are trimmed, buckets are
179+
prioritised based on the volumes of documents they contain.
180+
181+
shard_size:: Optional. To allow for more accurate counting of the top cells
182+
returned in the final result the aggregation defaults to
183+
returning `max(10,(size x number-of-shards))` buckets from each
184+
shard. If this heuristic is undesirable, the number considered
185+
from each shard can be over-ridden using this parameter.

‎rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/290_geotile_grid.yml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
setup:
2+
- skip:
3+
version: " - 6.99.99"
4+
reason: "added in 7.0.0"
5+
- do:
6+
indices.create:
7+
include_type_name: false
8+
index: test_1
9+
body:
10+
settings:
11+
number_of_replicas: 0
12+
mappings:
13+
properties:
14+
location:
15+
type: geo_point
16+
17+
---
18+
"Basic test":
19+
- do:
20+
bulk:
21+
refresh: true
22+
body:
23+
- index:
24+
_index: test_1
25+
_id: 1
26+
- location: "52.374081,4.912350"
27+
- index:
28+
_index: test_1
29+
_id: 2
30+
- location: "52.369219,4.901618"
31+
- index:
32+
_index: test_1
33+
_id: 3
34+
- location: "52.371667,4.914722"
35+
- index:
36+
_index: test_1
37+
_id: 4
38+
- location: "51.222900,4.405200"
39+
- index:
40+
_index: test_1
41+
_id: 5
42+
- location: "48.861111,2.336389"
43+
- index:
44+
_index: test_1
45+
_id: 6
46+
- location: "48.860000,2.327000"
47+
48+
- do:
49+
search:
50+
rest_total_hits_as_int: true
51+
body:
52+
aggregations:
53+
grid:
54+
geotile_grid:
55+
field: location
56+
precision: 8
57+
58+
59+
- match: { hits.total: 6 }
60+
- match: { aggregations.grid.buckets.0.key: "8/131/84" }
61+
- match: { aggregations.grid.buckets.0.doc_count: 3 }
62+
- match: { aggregations.grid.buckets.1.key: "8/129/88" }
63+
- match: { aggregations.grid.buckets.1.doc_count: 2 }
64+
- match: { aggregations.grid.buckets.2.key: "8/131/85" }
65+
- match: { aggregations.grid.buckets.2.doc_count: 1 }

‎server/src/main/java/org/elasticsearch/search/SearchModule.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,8 @@
110110
import org.elasticsearch.search.aggregations.bucket.filter.InternalFilters;
111111
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder;
112112
import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid;
113+
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
114+
import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid;
113115
import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
114116
import org.elasticsearch.search.aggregations.bucket.global.InternalGlobal;
115117
import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
@@ -422,6 +424,8 @@ private void registerAggregations(List<SearchPlugin> plugins) {
422424
GeoDistanceAggregationBuilder::parse).addResultReader(InternalGeoDistance::new));
423425
registerAggregation(new AggregationSpec(GeoHashGridAggregationBuilder.NAME, GeoHashGridAggregationBuilder::new,
424426
GeoHashGridAggregationBuilder::parse).addResultReader(InternalGeoHashGrid::new));
427+
registerAggregation(new AggregationSpec(GeoTileGridAggregationBuilder.NAME, GeoTileGridAggregationBuilder::new,
428+
GeoTileGridAggregationBuilder::parse).addResultReader(InternalGeoTileGrid::new));
425429
registerAggregation(new AggregationSpec(NestedAggregationBuilder.NAME, NestedAggregationBuilder::new,
426430
NestedAggregationBuilder::parse).addResultReader(InternalNested::new));
427431
registerAggregation(new AggregationSpec(ReverseNestedAggregationBuilder.NAME, ReverseNestedAggregationBuilder::new,

‎server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator.KeyedFilter;
3131
import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid;
3232
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder;
33+
import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid;
34+
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
3335
import org.elasticsearch.search.aggregations.bucket.global.Global;
3436
import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
3537
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
@@ -250,6 +252,13 @@ public static GeoHashGridAggregationBuilder geohashGrid(String name) {
250252
return new GeoHashGridAggregationBuilder(name);
251253
}
252254

255+
/**
256+
* Create a new {@link InternalGeoTileGrid} aggregation with the given name.
257+
*/
258+
public static GeoTileGridAggregationBuilder geotileGrid(String name) {
259+
return new GeoTileGridAggregationBuilder(name);
260+
}
261+
253262
/**
254263
* Create a new {@link SignificantTerms} aggregation with the given name.
255264
*/

‎server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.search.aggregations.bucket.geogrid;
21+
22+
import org.elasticsearch.common.io.stream.StreamInput;
23+
import org.elasticsearch.common.xcontent.ObjectParser;
24+
import org.elasticsearch.common.xcontent.XContentParser;
25+
import org.elasticsearch.search.aggregations.AggregationBuilder;
26+
import org.elasticsearch.search.aggregations.AggregatorFactories;
27+
import org.elasticsearch.search.aggregations.AggregatorFactory;
28+
import org.elasticsearch.search.aggregations.support.ValuesSource;
29+
import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
30+
import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
31+
import org.elasticsearch.search.internal.SearchContext;
32+
33+
import java.io.IOException;
34+
import java.util.Map;
35+
36+
public class GeoTileGridAggregationBuilder extends GeoGridAggregationBuilder {
37+
public static final String NAME = "geotile_grid";
38+
private static final int DEFAULT_PRECISION = 7;
39+
private static final int DEFAULT_MAX_NUM_CELLS = 10000;
40+
41+
private static final ObjectParser<GeoGridAggregationBuilder, Void> PARSER = createParser(NAME, GeoTileUtils::parsePrecision);
42+
43+
public GeoTileGridAggregationBuilder(String name) {
44+
super(name);
45+
precision(DEFAULT_PRECISION);
46+
size(DEFAULT_MAX_NUM_CELLS);
47+
shardSize = -1;
48+
}
49+
50+
public GeoTileGridAggregationBuilder(StreamInput in) throws IOException {
51+
super(in);
52+
}
53+
54+
@Override
55+
public GeoGridAggregationBuilder precision(int precision) {
56+
this.precision = GeoTileUtils.checkPrecisionRange(precision);
57+
return this;
58+
}
59+
60+
@Override
61+
protected ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, ?> createFactory(
62+
String name, ValuesSourceConfig<ValuesSource.GeoPoint> config, int precision, int requiredSize, int shardSize,
63+
SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
64+
Map<String, Object> metaData
65+
) throws IOException {
66+
return new GeoTileGridAggregatorFactory(name, config, precision, requiredSize, shardSize, context, parent,
67+
subFactoriesBuilder, metaData);
68+
}
69+
70+
private GeoTileGridAggregationBuilder(GeoTileGridAggregationBuilder clone, AggregatorFactories.Builder factoriesBuilder,
71+
Map<String, Object> metaData) {
72+
super(clone, factoriesBuilder, metaData);
73+
}
74+
75+
@Override
76+
protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map<String, Object> metaData) {
77+
return new GeoTileGridAggregationBuilder(this, factoriesBuilder, metaData);
78+
}
79+
80+
public static GeoGridAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
81+
return PARSER.parse(parser, new GeoTileGridAggregationBuilder(aggregationName), null);
82+
}
83+
84+
@Override
85+
public String getType() {
86+
return NAME;
87+
}
88+
}

0 commit comments

Comments
 (0)