From 8848778d7635c71960a226db407612b1bc2f0777 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 25 Jan 2019 00:02:01 -0500 Subject: [PATCH 01/28] Refactored GeoHashGrid unit tests This change allows other grid aggregations to reuse the same tests. The change mostly just moves code to the base classes, trying to keep changes to a bare minimum. --- .../geogrid/GeoGridAggregatorTestCase.java | 141 +++++++++++++++ .../bucket/geogrid/GeoGridTestCase.java | 167 ++++++++++++++++++ .../geogrid/GeoHashGridAggregatorTests.java | 111 ++---------- .../bucket/geogrid/GeoHashGridTests.java | 112 +----------- 4 files changed, 328 insertions(+), 203 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java new file mode 100644 index 0000000000000..5965574bef6e8 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java @@ -0,0 +1,141 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.apache.lucene.document.LatLonDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; + +public abstract class GeoGridAggregatorTestCase extends AggregatorTestCase { + + private static final String FIELD_NAME = "location"; + + /** + * Generate a random precision according to the rules of the given aggregation. + */ + protected abstract int randomPrecision(); + + /** + * Convert geo point into a hash string (bucket string ID) + */ + protected abstract String hashAsString(double lng, double lat, int precision); + + /** + * Create a new named {@link GeoGridAggregationBuilder}-derived builder + */ + protected abstract GeoGridAggregationBuilder createBuilder(String name); + + public void testNoDocs() throws IOException { + testCase(new MatchAllDocsQuery(), FIELD_NAME, randomPrecision(), iw -> { + // Intentionally not writing any docs + }, geoGrid -> { + assertEquals(0, geoGrid.getBuckets().size()); + }); + } + + public void testFieldMissing() throws IOException { + testCase(new MatchAllDocsQuery(), "wrong_field", randomPrecision(), iw -> { + iw.addDocument(Collections.singleton(new LatLonDocValuesField(FIELD_NAME, 10D, 10D))); + }, geoGrid -> { + assertEquals(0, geoGrid.getBuckets().size()); + }); + } + + public void testWithSeveralDocs() throws IOException { + int precision = randomPrecision(); + int numPoints = randomIntBetween(8, 128); + Map expectedCountPerGeoHash = new HashMap<>(); + testCase(new MatchAllDocsQuery(), FIELD_NAME, precision, iw -> { + List points = new ArrayList<>(); + Set distinctHashesPerDoc = new HashSet<>(); + for (int pointId = 0; pointId < numPoints; pointId++) { + double lat = (180d * randomDouble()) - 90d; + double lng = (360d * randomDouble()) - 180d; + + points.add(new LatLonDocValuesField(FIELD_NAME, lat, lng)); + String hash = hashAsString(lng, lat, precision); + if (distinctHashesPerDoc.contains(hash) == false) { + expectedCountPerGeoHash.put(hash, expectedCountPerGeoHash.getOrDefault(hash, 0) + 1); + } + distinctHashesPerDoc.add(hash); + if (usually()) { + iw.addDocument(points); + points.clear(); + distinctHashesPerDoc.clear(); + } + } + if (points.size() != 0) { + iw.addDocument(points); + } + }, geoHashGrid -> { + assertEquals(expectedCountPerGeoHash.size(), geoHashGrid.getBuckets().size()); + for (GeoGrid.Bucket bucket : geoHashGrid.getBuckets()) { + assertEquals((long) expectedCountPerGeoHash.get(bucket.getKeyAsString()), bucket.getDocCount()); + } + assertTrue(AggregationInspectionHelper.hasValue(geoHashGrid)); + }); + } + + private void testCase(Query query, String field, int precision, CheckedConsumer buildIndex, + Consumer> verify) throws IOException { + Directory directory = newDirectory(); + RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); + buildIndex.accept(indexWriter); + indexWriter.close(); + + IndexReader indexReader = DirectoryReader.open(directory); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + + GeoGridAggregationBuilder aggregationBuilder = createBuilder("_name").field(field); + aggregationBuilder.precision(precision); + MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType(); + fieldType.setHasDocValues(true); + fieldType.setName(FIELD_NAME); + + Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator.preCollection(); + indexSearcher.search(query, aggregator); + aggregator.postCollection(); + verify.accept((InternalGeoGrid) aggregator.buildAggregation(0L)); + + indexReader.close(); + directory.close(); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java new file mode 100644 index 0000000000000..45c348d19fc14 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java @@ -0,0 +1,167 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.apache.lucene.index.IndexWriter; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.test.InternalMultiBucketAggregationTestCase; +import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public abstract class GeoGridTestCase> + extends InternalMultiBucketAggregationTestCase { + + /** + * Instantiate a {@link InternalGeoGrid}-derived class using the same parameters as constructor. + */ + protected abstract T createInternalGeoGrid(String name, int size, List buckets, + List pipelineAggregators, Map metaData); + + /** + * Instantiate a {@link InternalGeoGridBucket}-derived class using the same parameters as constructor. + */ + protected abstract B createInternalGeoHashGridBucket(Long key, long docCount, InternalAggregations aggregations); + + /** + * Encode longitude and latitude with a given precision as a long hash. + */ + protected abstract long longEncode(double lng, double lat, int precision); + + /** + * Generate a random precision according to the rules of the given aggregation. + */ + protected abstract int randomPrecision(); + + @Override + protected int minNumberOfBuckets() { + return 1; + } + + @Override + protected int maxNumberOfBuckets() { + return 3; + } + + @Override + protected T createTestInstance(String name, + List pipelineAggregators, + Map metaData, + InternalAggregations aggregations) { + final int precision = randomPrecision(); + int size = randomNumberOfBuckets(); + List buckets = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + double latitude = randomDoubleBetween(-90.0, 90.0, false); + double longitude = randomDoubleBetween(-180.0, 180.0, false); + + long hashAsLong = longEncode(longitude, latitude, precision); + buckets.add(createInternalGeoHashGridBucket(hashAsLong, randomInt(IndexWriter.MAX_DOCS), aggregations)); + } + return createInternalGeoGrid(name, size, buckets, pipelineAggregators, metaData); + } + + @Override + protected void assertReduced(T reduced, List inputs) { + Map> map = new HashMap<>(); + for (T input : inputs) { + for (GeoGrid.Bucket bucketBase : input.getBuckets()) { + B bucket = (B) bucketBase; + List buckets = map.get(bucket.hashAsLong); + if (buckets == null) { + map.put(bucket.hashAsLong, buckets = new ArrayList<>()); + } + buckets.add(bucket); + } + } + List expectedBuckets = new ArrayList<>(); + for (Map.Entry> entry : map.entrySet()) { + long docCount = 0; + for (B bucket : entry.getValue()) { + docCount += bucket.docCount; + } + expectedBuckets.add(createInternalGeoHashGridBucket(entry.getKey(), docCount, InternalAggregations.EMPTY)); + } + expectedBuckets.sort((first, second) -> { + int cmp = Long.compare(second.docCount, first.docCount); + if (cmp == 0) { + return second.compareTo(first); + } + return cmp; + }); + int requestedSize = inputs.get(0).getRequiredSize(); + expectedBuckets = expectedBuckets.subList(0, Math.min(requestedSize, expectedBuckets.size())); + assertEquals(expectedBuckets.size(), reduced.getBuckets().size()); + for (int i = 0; i < reduced.getBuckets().size(); i++) { + GeoGrid.Bucket expected = expectedBuckets.get(i); + GeoGrid.Bucket actual = reduced.getBuckets().get(i); + assertEquals(expected.getDocCount(), actual.getDocCount()); + assertEquals(expected.getKey(), actual.getKey()); + } + } + + @Override + protected Class implementationClass() { + return ParsedGeoGrid.class; + } + + @Override + protected T mutateInstance(T instance) { + String name = instance.getName(); + int size = instance.getRequiredSize(); + List buckets = instance.getBuckets(); + List pipelineAggregators = instance.pipelineAggregators(); + Map metaData = instance.getMetaData(); + switch (between(0, 3)) { + case 0: + name += randomAlphaOfLength(5); + break; + case 1: + buckets = new ArrayList<>(buckets); + buckets.add( + createInternalGeoHashGridBucket(randomNonNegativeLong(), randomInt(IndexWriter.MAX_DOCS), InternalAggregations.EMPTY)); + break; + case 2: + size = size + between(1, 10); + break; + case 3: + if (metaData == null) { + metaData = new HashMap<>(1); + } else { + metaData = new HashMap<>(instance.getMetaData()); + } + metaData.put(randomAlphaOfLength(15), randomInt()); + break; + default: + throw new AssertionError("Illegal randomisation branch"); + } + return createInternalGeoGrid(name, size, buckets, pipelineAggregators, metaData); + } + + public void testCreateFromBuckets() { + InternalGeoGrid original = createTestInstance(); + assertThat(original, equalTo(original.create(original.buckets))); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java index ce4a065ef4c77..d01a1d3b6d098 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java @@ -16,114 +16,25 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.search.aggregations.bucket.geogrid; - -import org.apache.lucene.document.LatLonDocValuesField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.store.Directory; -import org.elasticsearch.common.CheckedConsumer; -import org.elasticsearch.index.mapper.GeoPointFieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.search.aggregations.Aggregator; -import org.elasticsearch.search.aggregations.AggregatorTestCase; -import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Consumer; +package org.elasticsearch.search.aggregations.bucket.geogrid; import static org.elasticsearch.common.geo.GeoHashUtils.stringEncode; -public class GeoHashGridAggregatorTests extends AggregatorTestCase { - - private static final String FIELD_NAME = "location"; - - public void testNoDocs() throws IOException { - testCase(new MatchAllDocsQuery(), FIELD_NAME, 1, iw -> { - // Intentionally not writing any docs - }, geoHashGrid -> { - assertEquals(0, geoHashGrid.getBuckets().size()); - assertFalse(AggregationInspectionHelper.hasValue(geoHashGrid)); - }); - } +public class GeoHashGridAggregatorTests extends GeoGridAggregatorTestCase { - public void testFieldMissing() throws IOException { - testCase(new MatchAllDocsQuery(), "wrong_field", 1, iw -> { - iw.addDocument(Collections.singleton(new LatLonDocValuesField(FIELD_NAME, 10D, 10D))); - }, geoHashGrid -> { - assertEquals(0, geoHashGrid.getBuckets().size()); - assertFalse(AggregationInspectionHelper.hasValue(geoHashGrid)); - }); + @Override + protected int randomPrecision() { + return randomIntBetween(1, 12); } - public void testWithSeveralDocs() throws IOException { - int precision = randomIntBetween(1, 12); - int numPoints = randomIntBetween(8, 128); - Map expectedCountPerGeoHash = new HashMap<>(); - testCase(new MatchAllDocsQuery(), FIELD_NAME, precision, iw -> { - List points = new ArrayList<>(); - Set distinctHashesPerDoc = new HashSet<>(); - for (int pointId = 0; pointId < numPoints; pointId++) { - double lat = (180d * randomDouble()) - 90d; - double lng = (360d * randomDouble()) - 180d; - points.add(new LatLonDocValuesField(FIELD_NAME, lat, lng)); - String hash = stringEncode(lng, lat, precision); - if (distinctHashesPerDoc.contains(hash) == false) { - expectedCountPerGeoHash.put(hash, expectedCountPerGeoHash.getOrDefault(hash, 0) + 1); - } - distinctHashesPerDoc.add(hash); - if (usually()) { - iw.addDocument(points); - points.clear(); - distinctHashesPerDoc.clear(); - } - } - if (points.size() != 0) { - iw.addDocument(points); - } - }, geoHashGrid -> { - assertEquals(expectedCountPerGeoHash.size(), geoHashGrid.getBuckets().size()); - for (GeoGrid.Bucket bucket : geoHashGrid.getBuckets()) { - assertEquals((long) expectedCountPerGeoHash.get(bucket.getKeyAsString()), bucket.getDocCount()); - } - assertTrue(AggregationInspectionHelper.hasValue(geoHashGrid)); - }); + @Override + protected String hashAsString(double lng, double lat, int precision) { + return stringEncode(lng, lat, precision); } - private void testCase(Query query, String field, int precision, CheckedConsumer buildIndex, - Consumer verify) throws IOException { - Directory directory = newDirectory(); - RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); - buildIndex.accept(indexWriter); - indexWriter.close(); - - IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); - - GeoGridAggregationBuilder aggregationBuilder = new GeoHashGridAggregationBuilder("_name").field(field); - aggregationBuilder.precision(precision); - MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType(); - fieldType.setHasDocValues(true); - fieldType.setName(FIELD_NAME); - - Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); - aggregator.preCollection(); - indexSearcher.search(query, aggregator); - aggregator.postCollection(); - verify.accept((InternalGeoHashGrid) aggregator.buildAggregation(0L)); - - indexReader.close(); - directory.close(); + @Override + protected GeoGridAggregationBuilder createBuilder(String name) { + return new GeoHashGridAggregationBuilder(name); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java index 02c8016556220..e07f4cdf3b74c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java @@ -18,47 +18,19 @@ */ package org.elasticsearch.search.aggregations.bucket.geogrid; -import org.apache.lucene.index.IndexWriter; import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.aggregations.InternalAggregations; -import org.elasticsearch.test.InternalMultiBucketAggregationTestCase; -import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.hamcrest.Matchers.equalTo; - -public class GeoHashGridTests extends InternalMultiBucketAggregationTestCase { - - @Override - protected int minNumberOfBuckets() { - return 1; - } +public class GeoHashGridTests extends GeoGridTestCase { @Override - protected int maxNumberOfBuckets() { - return 3; - } - - @Override - protected InternalGeoHashGrid createTestInstance(String name, - List pipelineAggregators, - Map metaData, - InternalAggregations aggregations) { - int size = randomNumberOfBuckets(); - List buckets = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - double latitude = randomDoubleBetween(-90.0, 90.0, false); - double longitude = randomDoubleBetween(-180.0, 180.0, false); - - long geoHashAsLong = GeoHashUtils.longEncode(longitude, latitude, 4); - buckets.add(new InternalGeoHashGridBucket(geoHashAsLong, randomInt(IndexWriter.MAX_DOCS), aggregations)); - } + protected InternalGeoHashGrid createInternalGeoGrid(String name, int size, List buckets, + List pipelineAggregators, Map metaData) { return new InternalGeoHashGrid(name, size, buckets, pipelineAggregators, metaData); } @@ -68,83 +40,17 @@ protected Writeable.Reader instanceReader() { } @Override - protected void assertReduced(InternalGeoHashGrid reduced, List inputs) { - Map> map = new HashMap<>(); - for (InternalGeoHashGrid input : inputs) { - for (InternalGeoGridBucket bucket : input.getBuckets()) { - List buckets = map.get(bucket.hashAsLong); - if (buckets == null) { - map.put(bucket.hashAsLong, buckets = new ArrayList<>()); - } - buckets.add(bucket); - } - } - List expectedBuckets = new ArrayList<>(); - for (Map.Entry> entry : map.entrySet()) { - long docCount = 0; - for (InternalGeoGridBucket bucket : entry.getValue()) { - docCount += bucket.docCount; - } - expectedBuckets.add(new InternalGeoHashGridBucket(entry.getKey(), docCount, InternalAggregations.EMPTY)); - } - expectedBuckets.sort((first, second) -> { - int cmp = Long.compare(second.docCount, first.docCount); - if (cmp == 0) { - return second.compareTo(first); - } - return cmp; - }); - int requestedSize = inputs.get(0).getRequiredSize(); - expectedBuckets = expectedBuckets.subList(0, Math.min(requestedSize, expectedBuckets.size())); - assertEquals(expectedBuckets.size(), reduced.getBuckets().size()); - for (int i = 0; i < reduced.getBuckets().size(); i++) { - GeoGrid.Bucket expected = expectedBuckets.get(i); - GeoGrid.Bucket actual = reduced.getBuckets().get(i); - assertEquals(expected.getDocCount(), actual.getDocCount()); - assertEquals(expected.getKey(), actual.getKey()); - } + protected InternalGeoHashGridBucket createInternalGeoHashGridBucket(Long key, long docCount, InternalAggregations aggregations) { + return new InternalGeoHashGridBucket(key, docCount, aggregations); } @Override - protected Class implementationClass() { - return ParsedGeoHashGrid.class; + protected long longEncode(double lng, double lat, int precision) { + return GeoHashUtils.longEncode(lng, lat, precision); } @Override - protected InternalGeoHashGrid mutateInstance(InternalGeoHashGrid instance) { - String name = instance.getName(); - int size = instance.getRequiredSize(); - List buckets = instance.getBuckets(); - List pipelineAggregators = instance.pipelineAggregators(); - Map metaData = instance.getMetaData(); - switch (between(0, 3)) { - case 0: - name += randomAlphaOfLength(5); - break; - case 1: - buckets = new ArrayList<>(buckets); - buckets.add( - new InternalGeoHashGridBucket(randomNonNegativeLong(), randomInt(IndexWriter.MAX_DOCS), InternalAggregations.EMPTY)); - break; - case 2: - size = size + between(1, 10); - break; - case 3: - if (metaData == null) { - metaData = new HashMap<>(1); - } else { - metaData = new HashMap<>(instance.getMetaData()); - } - metaData.put(randomAlphaOfLength(15), randomInt()); - break; - default: - throw new AssertionError("Illegal randomisation branch"); - } - return new InternalGeoHashGrid(name, size, buckets, pipelineAggregators, metaData); - } - - public void testCreateFromBuckets() { - InternalGeoHashGrid original = createTestInstance(); - assertThat(original, equalTo(original.create(original.buckets))); + protected int randomPrecision() { + return randomIntBetween(1, 12); } } From c927e15f93a7c1674ddf622c436d3009f65278b3 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 24 Jan 2019 12:42:58 -0500 Subject: [PATCH 02/28] Quadkey implementation Implements support for the quadkey geo aggregation. --- .../common/geo/QuadkeyUtils.java | 187 ++++++++++++++++++ .../elasticsearch/search/SearchModule.java | 4 + .../aggregations/AggregationBuilders.java | 9 + .../bucket/geogrid/InternalGeoHashGrid.java | 4 +- .../bucket/geogrid/InternalQuadkeyGrid.java | 69 +++++++ .../geogrid/InternalQuadkeyGridBucket.java | 56 ++++++ .../bucket/geogrid/ParsedQuadkeyGrid.java | 42 ++++ .../geogrid/ParsedQuadkeyGridBucket.java | 43 ++++ .../QuadkeyGridAggregationBuilder.java | 89 +++++++++ .../bucket/geogrid/QuadkeyGridAggregator.java | 57 ++++++ .../geogrid/QuadkeyGridAggregatorFactory.java | 78 ++++++++ 11 files changed, 635 insertions(+), 3 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGrid.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGridBucket.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGrid.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGridBucket.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregationBuilder.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregator.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregatorFactory.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java b/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java new file mode 100644 index 0000000000000..431c4c85fe9b3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java @@ -0,0 +1,187 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.util.BitUtil; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.support.XContentMapValues; + +import java.io.IOException; + +import static org.elasticsearch.common.geo.GeoUtils.normalizeLat; +import static org.elasticsearch.common.geo.GeoUtils.normalizeLon; + +/** + * Implements quad key hashing, same as used by map tiles. + * The string key is formatted as "zoom/x/y" + * The hash value (long) contains all three of those values. + */ +public class QuadkeyUtils { + + /** + * Largest number of tiles (precision) to use. + * This value cannot be more than (64-5)/2 = 29, because 5 bits are used for zoom level itself + * If zoom is not stored inside hash, it would be possible to use up to 32. + * Another consideration is that index optimizes lat/lng storage, loosing some precision. + * E.g. hash lng=140.74779717298918D lat=45.61884022447444D == "18/233561/93659", but shown as "18/233561/93658" + */ + public static final int MAX_ZOOM = 29; + + /** + * Bit position of the zoom value within hash. Must be >= 2*MAX_ZOOM + * Keeping it at a constant place allows MAX_ZOOM to be increased + * without breaking serialization binary compatibility + * (still, the newer version should not use higher MAX_ZOOM in the mixed cases) + */ + private static final int ZOOM_SHIFT = 29 * 2; + + /** + * Mask of all the bits used by the quadkey in a hash + */ + private static final long QUADKEY_MASK = (1L << ZOOM_SHIFT) - 1; + + /** + * Parse quadkey hash as zoom, x, y integers. + */ + private static int[] parseHash(final long hash) { + final int zoom = checkPrecisionRange((int) (hash >>> ZOOM_SHIFT)); + final int tiles = 1 << zoom; + + // decode the quadkey bits as interleaved xTile and yTile + long val = hash & QUADKEY_MASK; + int xTile = (int) BitUtil.deinterleave(val); + int yTile = (int) BitUtil.deinterleave(val >>> 1); + if (xTile < 0 || yTile < 0 || xTile >= tiles || yTile >= tiles) { + throw new IllegalArgumentException("hash-tile"); + } + + return new int[]{zoom, xTile, yTile}; + } + + /** + * Parse a precision that can be expressed as an integer or a distance measure like "1km", "10m". + * + * The precision is expressed as a zoom level between 0 and MAX_ZOOM. + * + * @param parser {@link XContentParser} to parse the value from + * @return int representing precision + */ + public static int parsePrecision(XContentParser parser) throws IOException, ElasticsearchParseException { + XContentParser.Token token = parser.currentToken(); + if (token.equals(XContentParser.Token.VALUE_NUMBER)) { + return XContentMapValues.nodeIntegerValue(parser.intValue()); + } else { + String precision = parser.text(); + try { + // we want to treat simple integer strings as precision levels, not distances + return XContentMapValues.nodeIntegerValue(precision); + } catch (NumberFormatException e) { + // try to parse as a distance value + final int parsedPrecision = GeoUtils.quadTreeLevelsForPrecision(precision); + try { + return checkPrecisionRange(parsedPrecision); + } catch (IllegalArgumentException e2) { + // this happens when distance too small, so precision > max. + // We'd like to see the original string + throw new IllegalArgumentException("precision too high [" + precision + "]", e2); + } + } + } + } + + public static int checkPrecisionRange(int precision) { + if (precision < 0 || precision > MAX_ZOOM) { + throw new IllegalArgumentException("Invalid quadkey aggregation precision of " + + precision + ". Must be between 0 and " + MAX_ZOOM + "."); + } + return precision; + } + + /** + * Encode lon/lat to the quadkey based long format. + * The resulting hash contains interleaved tile X and Y coordinates. + * The precision itself is also encoded as a few high bits. + */ + public static long longEncode(double longitude, double latitude, int precision) { + // Adapted from https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Java + + // How many tiles in X and in Y + final int tiles = 1 << checkPrecisionRange(precision); + final double lon = normalizeLon(longitude); + final double lat = normalizeLat(latitude); + + int xTile = (int) Math.floor((lon + 180) / 360 * tiles); + int yTile = (int) Math.floor( + (1 - Math.log( + Math.tan(Math.toRadians(lat)) + 1 / Math.cos(Math.toRadians(lat)) + ) / Math.PI) / 2 * tiles); + if (xTile < 0) { + xTile = 0; + } + if (xTile >= tiles) { + xTile = tiles - 1; + } + if (yTile < 0) { + yTile = 0; + } + if (yTile >= tiles) { + yTile = tiles - 1; + } + + // Zoom value is placed in front of all the bits used for the quadkey + // e.g. if max zoom is 26, the largest index would use 52 bits (51st..0th), + // leaving 12 bits unused for zoom. See MAX_ZOOM comment above. + return BitUtil.interleave(xTile, yTile) | ((long) precision << ZOOM_SHIFT); + } + + /** + * Encode to a quadkey string from the quadkey based long format + */ + public static String stringEncode(long hash) { + int[] res = parseHash(hash); + return "" + res[0] + "/" + res[1] + "/" + res[2]; + } + + public static GeoPoint hashToGeoPoint(String hashAsString) { + Throwable cause = null; + try { + final String[] parts = hashAsString.split("/", 4); + if (parts.length == 3) { + final int zoom = Integer.parseInt(parts[0]); + final int xTile = Integer.parseInt(parts[1]); + final int yTile = Integer.parseInt(parts[2]); + + final int maxTiles = 1 << checkPrecisionRange(zoom); + if (xTile >= 0 && xTile < maxTiles && yTile >= 0 && yTile < maxTiles) { + final double tiles = Math.pow(2.0, zoom); + final double n = Math.PI - (2.0 * Math.PI * (yTile + 0.5)) / tiles; + final double lat = Math.toDegrees(Math.atan(Math.sinh(n))); + final double lon = ((xTile + 0.5) / tiles * 360.0) - 180; + return new GeoPoint(lat, lon); + } + } + } catch (IllegalArgumentException e) { + // This will also handle NumberFormatException + cause = e; + } + throw new IllegalArgumentException("Invalid quadkey hash string of " + + hashAsString + ". Must be three integers in a form \"zoom/x/y\".", cause); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 3d93effecc545..361fb47bf0b67 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -110,6 +110,8 @@ import org.elasticsearch.search.aggregations.bucket.filter.InternalFilters; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid.QuadkeyGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalQuadkeyGrid; import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.InternalGlobal; import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -422,6 +424,8 @@ private void registerAggregations(List plugins) { GeoDistanceAggregationBuilder::parse).addResultReader(InternalGeoDistance::new)); registerAggregation(new AggregationSpec(GeoHashGridAggregationBuilder.NAME, GeoHashGridAggregationBuilder::new, GeoHashGridAggregationBuilder::parse).addResultReader(InternalGeoHashGrid::new)); + registerAggregation(new AggregationSpec(QuadkeyGridAggregationBuilder.NAME, QuadkeyGridAggregationBuilder::new, + QuadkeyGridAggregationBuilder::parse).addResultReader(InternalQuadkeyGrid::new)); registerAggregation(new AggregationSpec(NestedAggregationBuilder.NAME, NestedAggregationBuilder::new, NestedAggregationBuilder::parse).addResultReader(InternalNested::new)); registerAggregation(new AggregationSpec(ReverseNestedAggregationBuilder.NAME, ReverseNestedAggregationBuilder::new, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java index fd56172325230..586a5496391cc 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java @@ -30,6 +30,8 @@ import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator.KeyedFilter; import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalQuadkeyGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid.QuadkeyGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; @@ -250,6 +252,13 @@ public static GeoHashGridAggregationBuilder geohashGrid(String name) { return new GeoHashGridAggregationBuilder(name); } + /** + * Create a new {@link InternalQuadkeyGrid} aggregation with the given name. + */ + public static QuadkeyGridAggregationBuilder quadkeyGrid(String name) { + return new QuadkeyGridAggregationBuilder(name); + } + /** * Create a new {@link SignificantTerms} aggregation with the given name. */ diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java index 0c28788666249..7c874781d0c22 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java @@ -33,8 +33,6 @@ */ public class InternalGeoHashGrid extends InternalGeoGrid { - private static final String NAME = "geohash_grid"; - InternalGeoHashGrid(String name, int requiredSize, List buckets, List pipelineAggregators, Map metaData) { super(name, requiredSize, buckets, pipelineAggregators, metaData); @@ -66,6 +64,6 @@ Reader getBucketReader() { @Override public String getWriteableName() { - return NAME; + return GeoHashGridAggregationBuilder.NAME; } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGrid.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGrid.java new file mode 100644 index 0000000000000..8821139289c91 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGrid.java @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Represents a grid of cells where each cell's location is determined by a geohash. + * All geohashes in a grid are of the same precision and held internally as a single long + * for efficiency's sake. + */ +public class InternalQuadkeyGrid extends InternalGeoGrid { + + InternalQuadkeyGrid(String name, int requiredSize, List buckets, + List pipelineAggregators, Map metaData) { + super(name, requiredSize, buckets, pipelineAggregators, metaData); + } + + public InternalQuadkeyGrid(StreamInput in) throws IOException { + super(in); + } + + @Override + public InternalGeoGrid create(List buckets) { + return new InternalQuadkeyGrid(name, requiredSize, buckets, pipelineAggregators(), metaData); + } + + @Override + public InternalGeoGridBucket createBucket(InternalAggregations aggregations, InternalGeoGridBucket prototype) { + return new InternalQuadkeyGridBucket(prototype.hashAsLong, prototype.docCount, aggregations); + } + + @Override + InternalGeoGrid create(String name, int requiredSize, List buckets, List list, Map metaData) { + return new InternalQuadkeyGrid(name, requiredSize, buckets, list, metaData); + } + + @Override + Reader getBucketReader() { + return InternalQuadkeyGridBucket::new; + } + + @Override + public String getWriteableName() { + return QuadkeyGridAggregationBuilder.NAME; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGridBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGridBucket.java new file mode 100644 index 0000000000000..d7d9b64e7659a --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGridBucket.java @@ -0,0 +1,56 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.QuadkeyUtils; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.search.aggregations.InternalAggregations; + +import java.io.IOException; + +public class InternalQuadkeyGridBucket extends InternalGeoGridBucket { + InternalQuadkeyGridBucket(long geohashAsLong, long docCount, InternalAggregations aggregations) { + super(geohashAsLong, docCount, aggregations); + } + + /** + * Read from a stream. + */ + public InternalQuadkeyGridBucket(StreamInput in) throws IOException { + super(in); + } + + @Override + InternalQuadkeyGridBucket buildBucket(InternalGeoGridBucket bucket, long geoHashAsLong, long docCount, + InternalAggregations aggregations) { + return new InternalQuadkeyGridBucket(geoHashAsLong, docCount, aggregations); + } + + @Override + public String getKeyAsString() { + return QuadkeyUtils.stringEncode(geohashAsLong); + } + + @Override + public GeoPoint getKey() { + return GeoPoint.fromGeohash(geohashAsLong); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGrid.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGrid.java new file mode 100644 index 0000000000000..d9798cb3c59ee --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGrid.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; + +public class ParsedQuadkeyGrid extends ParsedGeoGrid { + + private static ObjectParser PARSER = createParser(ParsedQuadkeyGrid::new, + ParsedQuadkeyGridBucket::fromXContent, ParsedQuadkeyGridBucket::fromXContent); + + public static ParsedGeoGrid fromXContent(XContentParser parser, String name) throws IOException { + ParsedGeoGrid aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } + + @Override + public String getType() { + return QuadkeyGridAggregationBuilder.NAME; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGridBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGridBucket.java new file mode 100644 index 0000000000000..859160cc5ba38 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGridBucket.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.QuadkeyUtils; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; + +public class ParsedQuadkeyGridBucket extends ParsedGeoGridBucket { + + @Override + public GeoPoint getKey() { + return QuadkeyUtils.hashToGeoPoint(geohashAsString); + } + + @Override + public String getKeyAsString() { + return geohashAsString; + } + + static ParsedQuadkeyGridBucket fromXContent(XContentParser parser) throws IOException { + return parseXContent(parser, false, ParsedQuadkeyGridBucket::new, (p, bucket) -> bucket.geohashAsString = p.textOrNull()); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregationBuilder.java new file mode 100644 index 0000000000000..896cc0d31eebe --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregationBuilder.java @@ -0,0 +1,89 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.QuadkeyUtils; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Map; + +public class QuadkeyGridAggregationBuilder extends GeoGridAggregationBuilder { + public static final String NAME = "quadkey_grid"; + public static final int DEFAULT_PRECISION = 5; + public static final int DEFAULT_MAX_NUM_CELLS = 10000; + + private static final ObjectParser PARSER = createParser(NAME, QuadkeyUtils::parsePrecision); + + public QuadkeyGridAggregationBuilder(String name) { + super(name); + precision(DEFAULT_PRECISION); + size(DEFAULT_MAX_NUM_CELLS); + shardSize = -1; + } + + public QuadkeyGridAggregationBuilder(StreamInput in) throws IOException { + super(in); + } + + @Override + public GeoGridAggregationBuilder precision(int precision) { + this.precision = QuadkeyUtils.checkPrecisionRange(precision); + return this; + } + + @Override + protected ValuesSourceAggregatorFactory createFactory( + String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, + SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, + Map metaData + ) throws IOException { + return new QuadkeyGridAggregatorFactory(name, config, precision, requiredSize, shardSize, context, parent, + subFactoriesBuilder, metaData); + } + + private QuadkeyGridAggregationBuilder(QuadkeyGridAggregationBuilder clone, AggregatorFactories.Builder factoriesBuilder, + Map metaData) { + super(clone, factoriesBuilder, metaData); + } + + @Override + protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map metaData) { + return new QuadkeyGridAggregationBuilder(this, factoriesBuilder, metaData); + } + + public static GeoGridAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException { + return PARSER.parse(parser, new QuadkeyGridAggregationBuilder(aggregationName), null); + } + + @Override + public String getType() { + return NAME; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregator.java new file mode 100644 index 0000000000000..193b8d6507459 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregator.java @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Aggregates data expressed as Quadkey longs (for efficiency's sake) but formats results as Quadkey strings. + */ +public class QuadkeyGridAggregator extends GeoGridAggregator { + + QuadkeyGridAggregator(String name, AggregatorFactories factories, CellIdSource valuesSource, + int requiredSize, int shardSize, SearchContext aggregationContext, Aggregator parent, + List pipelineAggregators, Map metaData) throws IOException { + super(name, factories, valuesSource, requiredSize, shardSize, aggregationContext, parent, pipelineAggregators, metaData); + } + + @Override + InternalQuadkeyGrid buildAggregation(String name, int requiredSize, List buckets, + List pipelineAggregators, Map metaData) { + return new InternalQuadkeyGrid(name, requiredSize, buckets, pipelineAggregators, metaData); + } + + @Override + public InternalQuadkeyGrid buildEmptyAggregation() { + return new InternalQuadkeyGrid(name, requiredSize, Collections.emptyList(), pipelineAggregators(), metaData()); + } + + InternalGeoGridBucket newEmptyBucket() { + return new InternalQuadkeyGridBucket(0, 0, null); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregatorFactory.java new file mode 100644 index 0000000000000..a75a2dc03c305 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregatorFactory.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.QuadkeyUtils; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.NonCollectingAggregator; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSource.GeoPoint; +import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class QuadkeyGridAggregatorFactory extends ValuesSourceAggregatorFactory { + + private final int precision; + private final int requiredSize; + private final int shardSize; + + QuadkeyGridAggregatorFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, + int shardSize, SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, + Map metaData) throws IOException { + super(name, config, context, parent, subFactoriesBuilder, metaData); + this.precision = precision; + this.requiredSize = requiredSize; + this.shardSize = shardSize; + } + + @Override + protected Aggregator createUnmapped(Aggregator parent, List pipelineAggregators, Map metaData) + throws IOException { + final InternalAggregation aggregation = new InternalQuadkeyGrid(name, requiredSize, + Collections.emptyList(), pipelineAggregators, metaData); + return new NonCollectingAggregator(name, context, parent, pipelineAggregators, metaData) { + @Override + public InternalAggregation buildEmptyAggregation() { + return aggregation; + } + }; + } + + @Override + protected Aggregator doCreateInternal(final ValuesSource.GeoPoint valuesSource, Aggregator parent, boolean collectsFromSingleBucket, + List pipelineAggregators, Map metaData) throws IOException { + if (collectsFromSingleBucket == false) { + return asMultiBucketAggregator(this, context, parent); + } + CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, QuadkeyUtils::longEncode); + return new QuadkeyGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, context, parent, + pipelineAggregators, metaData); + } +} From 859040383f92a94d252ed8eaf6c0e446847631e8 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 25 Jan 2019 00:45:01 -0500 Subject: [PATCH 03/28] fix rebased artifacts --- .../bucket/geogrid/InternalQuadkeyGridBucket.java | 8 ++++---- .../bucket/geogrid/ParsedQuadkeyGridBucket.java | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGridBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGridBucket.java index d7d9b64e7659a..f52a0941476c8 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGridBucket.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGridBucket.java @@ -27,8 +27,8 @@ import java.io.IOException; public class InternalQuadkeyGridBucket extends InternalGeoGridBucket { - InternalQuadkeyGridBucket(long geohashAsLong, long docCount, InternalAggregations aggregations) { - super(geohashAsLong, docCount, aggregations); + InternalQuadkeyGridBucket(long hashAsLong, long docCount, InternalAggregations aggregations) { + super(hashAsLong, docCount, aggregations); } /** @@ -46,11 +46,11 @@ InternalQuadkeyGridBucket buildBucket(InternalGeoGridBucket bucket, long geoHash @Override public String getKeyAsString() { - return QuadkeyUtils.stringEncode(geohashAsLong); + return QuadkeyUtils.stringEncode(hashAsLong); } @Override public GeoPoint getKey() { - return GeoPoint.fromGeohash(geohashAsLong); + return GeoPoint.fromGeohash(hashAsLong); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGridBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGridBucket.java index 859160cc5ba38..db7e65fbfc708 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGridBucket.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGridBucket.java @@ -29,15 +29,15 @@ public class ParsedQuadkeyGridBucket extends ParsedGeoGridBucket { @Override public GeoPoint getKey() { - return QuadkeyUtils.hashToGeoPoint(geohashAsString); + return QuadkeyUtils.hashToGeoPoint(hashAsString); } @Override public String getKeyAsString() { - return geohashAsString; + return hashAsString; } static ParsedQuadkeyGridBucket fromXContent(XContentParser parser) throws IOException { - return parseXContent(parser, false, ParsedQuadkeyGridBucket::new, (p, bucket) -> bucket.geohashAsString = p.textOrNull()); + return parseXContent(parser, false, ParsedQuadkeyGridBucket::new, (p, bucket) -> bucket.hashAsString = p.textOrNull()); } } From 8d63835191ee839c892466101cdac182033e398b Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 25 Jan 2019 01:00:10 -0500 Subject: [PATCH 04/28] quadkey unit tests --- .../bucket/geogrid/GeoGridTestCase.java | 8 +- .../bucket/geogrid/GeoHashGridTests.java | 2 +- .../geogrid/QuadkeyGridAggregatorTests.java | 42 +++++++ .../geogrid/QuadkeyGridParserTests.java | 118 ++++++++++++++++++ .../bucket/geogrid/QuadkeyGridTests.java | 58 +++++++++ 5 files changed, 223 insertions(+), 5 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregatorTests.java create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridParserTests.java create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridTests.java diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java index 45c348d19fc14..ad831eee4a617 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java @@ -43,7 +43,7 @@ protected abstract T createInternalGeoGrid(String name, int size, List inputs) { for (B bucket : entry.getValue()) { docCount += bucket.docCount; } - expectedBuckets.add(createInternalGeoHashGridBucket(entry.getKey(), docCount, InternalAggregations.EMPTY)); + expectedBuckets.add(createInternalGeoGridBucket(entry.getKey(), docCount, InternalAggregations.EMPTY)); } expectedBuckets.sort((first, second) -> { int cmp = Long.compare(second.docCount, first.docCount); @@ -141,7 +141,7 @@ protected T mutateInstance(T instance) { case 1: buckets = new ArrayList<>(buckets); buckets.add( - createInternalGeoHashGridBucket(randomNonNegativeLong(), randomInt(IndexWriter.MAX_DOCS), InternalAggregations.EMPTY)); + createInternalGeoGridBucket(randomNonNegativeLong(), randomInt(IndexWriter.MAX_DOCS), InternalAggregations.EMPTY)); break; case 2: size = size + between(1, 10); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java index e07f4cdf3b74c..c48308e6e1724 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java @@ -40,7 +40,7 @@ protected Writeable.Reader instanceReader() { } @Override - protected InternalGeoHashGridBucket createInternalGeoHashGridBucket(Long key, long docCount, InternalAggregations aggregations) { + protected InternalGeoHashGridBucket createInternalGeoGridBucket(Long key, long docCount, InternalAggregations aggregations) { return new InternalGeoHashGridBucket(key, docCount, aggregations); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregatorTests.java new file mode 100644 index 0000000000000..ecaac269411c5 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregatorTests.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import static org.elasticsearch.common.geo.QuadkeyUtils.MAX_ZOOM; +import static org.elasticsearch.common.geo.QuadkeyUtils.longEncode; +import static org.elasticsearch.common.geo.QuadkeyUtils.stringEncode; + +public class QuadkeyGridAggregatorTests extends GeoGridAggregatorTestCase { + + @Override + protected int randomPrecision() { + return randomIntBetween(0, MAX_ZOOM); + } + + @Override + protected String hashAsString(double lng, double lat, int precision) { + return stringEncode(longEncode(lng, lat, precision)); + } + + @Override + protected GeoGridAggregationBuilder createBuilder(String name) { + return new QuadkeyGridAggregationBuilder(name); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridParserTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridParserTests.java new file mode 100644 index 0000000000000..0b41578bc9b59 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridParserTests.java @@ -0,0 +1,118 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.unit.DistanceUnit; +import org.elasticsearch.common.xcontent.XContentParseException; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +public class QuadkeyGridParserTests extends ESTestCase { + public void testParseValidFromInts() throws Exception { + int precision = randomIntBetween(1, 12); + XContentParser stParser = createParser(JsonXContent.jsonXContent, + "{\"field\":\"my_loc\", \"precision\":" + precision + ", \"size\": 500, \"shard_size\": 550}"); + XContentParser.Token token = stParser.nextToken(); + assertSame(XContentParser.Token.START_OBJECT, token); + // can create a factory + assertNotNull(QuadkeyGridAggregationBuilder.parse("quadkey_grid", stParser)); + } + + public void testParseValidFromStrings() throws Exception { + int precision = randomIntBetween(1, 12); + XContentParser stParser = createParser(JsonXContent.jsonXContent, + "{\"field\":\"my_loc\", \"precision\":\"" + precision + "\", \"size\": \"500\", \"shard_size\": \"550\"}"); + XContentParser.Token token = stParser.nextToken(); + assertSame(XContentParser.Token.START_OBJECT, token); + // can create a factory + assertNotNull(QuadkeyGridAggregationBuilder.parse("quadkey_grid", stParser)); + } + + public void testParseDistanceUnitPrecision() throws Exception { + double distance = randomDoubleBetween(10.0, 100.00, true); + DistanceUnit unit = randomFrom(DistanceUnit.values()); + if (unit.equals(DistanceUnit.MILLIMETERS)) { + distance = 5600 + randomDouble(); // 5.6cm is approx. smallest distance represented by precision 12 + } + String distanceString = distance + unit.toString(); + XContentParser stParser = createParser(JsonXContent.jsonXContent, + "{\"field\":\"my_loc\", \"precision\": \"" + distanceString + "\", \"size\": \"500\", \"shard_size\": \"550\"}"); + XContentParser.Token token = stParser.nextToken(); + assertSame(XContentParser.Token.START_OBJECT, token); + // can create a factory + GeoGridAggregationBuilder builder = QuadkeyGridAggregationBuilder.parse("quadkey_grid", stParser); + assertNotNull(builder); + assertThat(builder.precision(), greaterThanOrEqualTo(0)); + assertThat(builder.precision(), lessThanOrEqualTo(12)); + } + + public void testParseInvalidUnitPrecision() throws Exception { + XContentParser stParser = createParser(JsonXContent.jsonXContent, + "{\"field\":\"my_loc\", \"precision\": \"10kg\", \"size\": \"500\", \"shard_size\": \"550\"}"); + XContentParser.Token token = stParser.nextToken(); + assertSame(XContentParser.Token.START_OBJECT, token); + XContentParseException ex = expectThrows(XContentParseException.class, + () -> QuadkeyGridAggregationBuilder.parse("quadkey_grid", stParser)); + assertThat(ex.getMessage(), containsString("[quadkey_grid] failed to parse field [precision]")); + assertThat(ex.getCause(), instanceOf(NumberFormatException.class)); + assertEquals("For input string: \"10kg\"", ex.getCause().getMessage()); + } + + public void testParseDistanceUnitPrecisionTooSmall() throws Exception { + XContentParser stParser = createParser(JsonXContent.jsonXContent, + "{\"field\":\"my_loc\", \"precision\": \"1cm\", \"size\": \"500\", \"shard_size\": \"550\"}"); + XContentParser.Token token = stParser.nextToken(); + assertSame(XContentParser.Token.START_OBJECT, token); + XContentParseException ex = expectThrows(XContentParseException.class, + () -> QuadkeyGridAggregationBuilder.parse("quadkey_grid", stParser)); + assertThat(ex.getMessage(), containsString("[quadkey_grid] failed to parse field [precision]")); + assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class)); + assertEquals("precision too high [1cm]", ex.getCause().getMessage()); + } + + public void testParseErrorOnBooleanPrecision() throws Exception { + XContentParser stParser = createParser(JsonXContent.jsonXContent, "{\"field\":\"my_loc\", \"precision\":false}"); + XContentParser.Token token = stParser.nextToken(); + assertSame(XContentParser.Token.START_OBJECT, token); + XContentParseException e = expectThrows(XContentParseException.class, + () -> QuadkeyGridAggregationBuilder.parse("quadkey_grid", stParser)); + assertThat(ExceptionsHelper.detailedMessage(e), + containsString("[quadkey_grid] precision doesn't support values of type: VALUE_BOOLEAN")); + } + + public void testParseErrorOnPrecisionOutOfRange() throws Exception { + XContentParser stParser = createParser(JsonXContent.jsonXContent, "{\"field\":\"my_loc\", \"precision\":\"30\"}"); + XContentParser.Token token = stParser.nextToken(); + assertSame(XContentParser.Token.START_OBJECT, token); + try { + QuadkeyGridAggregationBuilder.parse("quadkey_grid", stParser); + fail(); + } catch (XContentParseException ex) { + assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class)); + assertEquals("Invalid quadkey aggregation precision of 13. Must be between 0 and 29.", ex.getCause().getMessage()); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridTests.java new file mode 100644 index 0000000000000..487b0b1e17245 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridTests.java @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.QuadkeyUtils; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; + +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.common.geo.QuadkeyUtils.MAX_ZOOM; + +public class QuadkeyGridTests extends GeoGridTestCase { + + @Override + protected InternalQuadkeyGrid createInternalGeoGrid(String name, int size, List buckets, + List pipelineAggregators, Map metaData) { + return new InternalQuadkeyGrid(name, size, buckets, pipelineAggregators, metaData); + } + + @Override + protected Writeable.Reader instanceReader() { + return InternalQuadkeyGrid::new; + } + + @Override + protected InternalQuadkeyGridBucket createInternalGeoGridBucket(Long key, long docCount, InternalAggregations aggregations) { + return new InternalQuadkeyGridBucket(key, docCount, aggregations); + } + + @Override + protected long longEncode(double lng, double lat, int precision) { + return QuadkeyUtils.longEncode(lng, lat, precision); + } + + @Override + protected int randomPrecision() { + return randomIntBetween(0, MAX_ZOOM); + } +} From 820130305f9a71c24066346d9bbf36f84453a336 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 25 Jan 2019 01:02:49 -0500 Subject: [PATCH 05/28] rename createInternalGeoHashGridBucket to createInternalGeoGridBucket --- .../aggregations/bucket/geogrid/GeoGridTestCase.java | 8 ++++---- .../aggregations/bucket/geogrid/GeoHashGridTests.java | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java index 45c348d19fc14..ad831eee4a617 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java @@ -43,7 +43,7 @@ protected abstract T createInternalGeoGrid(String name, int size, List inputs) { for (B bucket : entry.getValue()) { docCount += bucket.docCount; } - expectedBuckets.add(createInternalGeoHashGridBucket(entry.getKey(), docCount, InternalAggregations.EMPTY)); + expectedBuckets.add(createInternalGeoGridBucket(entry.getKey(), docCount, InternalAggregations.EMPTY)); } expectedBuckets.sort((first, second) -> { int cmp = Long.compare(second.docCount, first.docCount); @@ -141,7 +141,7 @@ protected T mutateInstance(T instance) { case 1: buckets = new ArrayList<>(buckets); buckets.add( - createInternalGeoHashGridBucket(randomNonNegativeLong(), randomInt(IndexWriter.MAX_DOCS), InternalAggregations.EMPTY)); + createInternalGeoGridBucket(randomNonNegativeLong(), randomInt(IndexWriter.MAX_DOCS), InternalAggregations.EMPTY)); break; case 2: size = size + between(1, 10); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java index e07f4cdf3b74c..c48308e6e1724 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java @@ -40,7 +40,7 @@ protected Writeable.Reader instanceReader() { } @Override - protected InternalGeoHashGridBucket createInternalGeoHashGridBucket(Long key, long docCount, InternalAggregations aggregations) { + protected InternalGeoHashGridBucket createInternalGeoGridBucket(Long key, long docCount, InternalAggregations aggregations) { return new InternalGeoHashGridBucket(key, docCount, aggregations); } From eebcb6617b5be2ae718a105e97ba9c30f8f313cc Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 25 Jan 2019 13:19:14 -0500 Subject: [PATCH 06/28] add lat/lng adjustment, declarations --- .../org/elasticsearch/client/RestHighLevelClient.java | 3 +++ .../bucket/geogrid/GeoGridAggregatorTestCase.java | 8 ++++++++ .../elasticsearch/test/InternalAggregationTestCase.java | 3 +++ 3 files changed, 14 insertions(+) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 5ef0e0110c12d..1debc3d4d697f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -95,6 +95,8 @@ import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedQuadkeyGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid.QuadkeyGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal; import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -1759,6 +1761,7 @@ static List getDefaultNamedXContents() { map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)); map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); map.put(GeoHashGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c)); + map.put(QuadkeyGridAggregationBuilder.NAME, (p, c) -> ParsedQuadkeyGrid.fromXContent(p, (String) c)); map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c)); map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c)); map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java index 5965574bef6e8..047903bc86100 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.apache.lucene.document.LatLonDocValuesField; +import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; @@ -89,6 +90,13 @@ public void testWithSeveralDocs() throws IOException { double lat = (180d * randomDouble()) - 90d; double lng = (360d * randomDouble()) - 180d; + // Precision-adjust longitude/latitude to avoid wrong bucket placement + // Internally, lat/lng get converted to 32 bit integers, loosing some precision. + // This does not affect geohashing because geohash uses the same algorithm, + // but it does affect other bucketing algos, thus we need to do the same steps here. + lng = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lng)); + lat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); + points.add(new LatLonDocValuesField(FIELD_NAME, lat, lng)); String hash = hashAsString(lng, lat, precision); if (distinctHashesPerDoc.contains(hash) == false) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java index fd560af806066..7b7a75c9a0333 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java @@ -51,6 +51,8 @@ import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedQuadkeyGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid.QuadkeyGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal; import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -212,6 +214,7 @@ public abstract class InternalAggregationTestCase map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)); map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); map.put(GeoHashGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c)); + map.put(QuadkeyGridAggregationBuilder.NAME, (p, c) -> ParsedQuadkeyGrid.fromXContent(p, (String) c)); map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c)); map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c)); map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c)); From acdb8047d7240cd8838c0ab250f3b1c592390ac0 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 25 Jan 2019 13:44:30 -0500 Subject: [PATCH 07/28] fix quadkey tests --- .../common/geo/QuadkeyUtils.java | 30 +++++++++++-------- .../geogrid/InternalQuadkeyGridBucket.java | 6 ++-- .../geogrid/QuadkeyGridParserTests.java | 11 +++---- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java b/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java index 431c4c85fe9b3..de303c203ce80 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java +++ b/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java @@ -159,23 +159,17 @@ public static String stringEncode(long hash) { return "" + res[0] + "/" + res[1] + "/" + res[2]; } + public static GeoPoint hashToGeoPoint(long hash) { + int[] res = parseHash(hash); + return zxyToGeoPoint(res[0], res[1], res[2]); + } + public static GeoPoint hashToGeoPoint(String hashAsString) { Throwable cause = null; try { final String[] parts = hashAsString.split("/", 4); if (parts.length == 3) { - final int zoom = Integer.parseInt(parts[0]); - final int xTile = Integer.parseInt(parts[1]); - final int yTile = Integer.parseInt(parts[2]); - - final int maxTiles = 1 << checkPrecisionRange(zoom); - if (xTile >= 0 && xTile < maxTiles && yTile >= 0 && yTile < maxTiles) { - final double tiles = Math.pow(2.0, zoom); - final double n = Math.PI - (2.0 * Math.PI * (yTile + 0.5)) / tiles; - final double lat = Math.toDegrees(Math.atan(Math.sinh(n))); - final double lon = ((xTile + 0.5) / tiles * 360.0) - 180; - return new GeoPoint(lat, lon); - } + return zxyToGeoPoint(Integer.parseInt(parts[0]), Integer.parseInt(parts[1]), Integer.parseInt(parts[2])); } } catch (IllegalArgumentException e) { // This will also handle NumberFormatException @@ -184,4 +178,16 @@ public static GeoPoint hashToGeoPoint(String hashAsString) { throw new IllegalArgumentException("Invalid quadkey hash string of " + hashAsString + ". Must be three integers in a form \"zoom/x/y\".", cause); } + + private static GeoPoint zxyToGeoPoint(int zoom, int xTile, int yTile) { + final int maxTiles = 1 << checkPrecisionRange(zoom); + if (xTile >= 0 && xTile < maxTiles && yTile >= 0 && yTile < maxTiles) { + final double tiles = Math.pow(2.0, zoom); + final double n = Math.PI - (2.0 * Math.PI * (yTile + 0.5)) / tiles; + final double lat = Math.toDegrees(Math.atan(Math.sinh(n))); + final double lon = ((xTile + 0.5) / tiles * 360.0) - 180; + return new GeoPoint(lat, lon); + } + throw new IllegalArgumentException(String.format("Invalid quadkey z/x/y values of %s/%s/%s", zoom, xTile, yTile)); + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGridBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGridBucket.java index f52a0941476c8..1d28d1d6b8b52 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGridBucket.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGridBucket.java @@ -39,9 +39,9 @@ public InternalQuadkeyGridBucket(StreamInput in) throws IOException { } @Override - InternalQuadkeyGridBucket buildBucket(InternalGeoGridBucket bucket, long geoHashAsLong, long docCount, + InternalQuadkeyGridBucket buildBucket(InternalGeoGridBucket bucket, long hashAsLong, long docCount, InternalAggregations aggregations) { - return new InternalQuadkeyGridBucket(geoHashAsLong, docCount, aggregations); + return new InternalQuadkeyGridBucket(hashAsLong, docCount, aggregations); } @Override @@ -51,6 +51,6 @@ public String getKeyAsString() { @Override public GeoPoint getKey() { - return GeoPoint.fromGeohash(hashAsLong); + return QuadkeyUtils.hashToGeoPoint(hashAsLong); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridParserTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridParserTests.java index 0b41578bc9b59..70be9cf7041ca 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridParserTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridParserTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.geo.QuadkeyUtils; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentParseException; import org.elasticsearch.common.xcontent.XContentParser; @@ -32,7 +33,7 @@ public class QuadkeyGridParserTests extends ESTestCase { public void testParseValidFromInts() throws Exception { - int precision = randomIntBetween(1, 12); + int precision = randomIntBetween(0, QuadkeyUtils.MAX_ZOOM); XContentParser stParser = createParser(JsonXContent.jsonXContent, "{\"field\":\"my_loc\", \"precision\":" + precision + ", \"size\": 500, \"shard_size\": 550}"); XContentParser.Token token = stParser.nextToken(); @@ -42,7 +43,7 @@ public void testParseValidFromInts() throws Exception { } public void testParseValidFromStrings() throws Exception { - int precision = randomIntBetween(1, 12); + int precision = randomIntBetween(0, QuadkeyUtils.MAX_ZOOM); XContentParser stParser = createParser(JsonXContent.jsonXContent, "{\"field\":\"my_loc\", \"precision\":\"" + precision + "\", \"size\": \"500\", \"shard_size\": \"550\"}"); XContentParser.Token token = stParser.nextToken(); @@ -55,7 +56,7 @@ public void testParseDistanceUnitPrecision() throws Exception { double distance = randomDoubleBetween(10.0, 100.00, true); DistanceUnit unit = randomFrom(DistanceUnit.values()); if (unit.equals(DistanceUnit.MILLIMETERS)) { - distance = 5600 + randomDouble(); // 5.6cm is approx. smallest distance represented by precision 12 + distance = 5600 + randomDouble(); // 5.6cm is approx. smallest distance } String distanceString = distance + unit.toString(); XContentParser stParser = createParser(JsonXContent.jsonXContent, @@ -66,7 +67,7 @@ public void testParseDistanceUnitPrecision() throws Exception { GeoGridAggregationBuilder builder = QuadkeyGridAggregationBuilder.parse("quadkey_grid", stParser); assertNotNull(builder); assertThat(builder.precision(), greaterThanOrEqualTo(0)); - assertThat(builder.precision(), lessThanOrEqualTo(12)); + assertThat(builder.precision(), lessThanOrEqualTo(29)); } public void testParseInvalidUnitPrecision() throws Exception { @@ -112,7 +113,7 @@ public void testParseErrorOnPrecisionOutOfRange() throws Exception { fail(); } catch (XContentParseException ex) { assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class)); - assertEquals("Invalid quadkey aggregation precision of 13. Must be between 0 and 29.", ex.getCause().getMessage()); + assertEquals("Invalid quadkey aggregation precision of 30. Must be between 0 and 29.", ex.getCause().getMessage()); } } } From c5fee3025f9301d7fd028549e26fc09fd65aa88b Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 25 Jan 2019 15:30:01 -0500 Subject: [PATCH 08/28] QuadkeyUtils tests --- .../common/geo/QuadkeyUtils.java | 4 +- .../common/geo/QuadkeyUtilsTests.java | 83 +++++++++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/common/geo/QuadkeyUtilsTests.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java b/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java index de303c203ce80..2c12960cdfd23 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java +++ b/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java @@ -108,7 +108,7 @@ public static int parsePrecision(XContentParser parser) throws IOException, Elas public static int checkPrecisionRange(int precision) { if (precision < 0 || precision > MAX_ZOOM) { - throw new IllegalArgumentException("Invalid quadkey aggregation precision of " + + throw new IllegalArgumentException("Invalid quadkey precision of " + precision + ". Must be between 0 and " + MAX_ZOOM + "."); } return precision; @@ -120,7 +120,7 @@ public static int checkPrecisionRange(int precision) { * The precision itself is also encoded as a few high bits. */ public static long longEncode(double longitude, double latitude, int precision) { - // Adapted from https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Java + // Mathematics for this code was adapted from https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Java // How many tiles in X and in Y final int tiles = 1 << checkPrecisionRange(precision); diff --git a/server/src/test/java/org/elasticsearch/common/geo/QuadkeyUtilsTests.java b/server/src/test/java/org/elasticsearch/common/geo/QuadkeyUtilsTests.java new file mode 100644 index 0000000000000..00fea08cb8ef6 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/QuadkeyUtilsTests.java @@ -0,0 +1,83 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.geo; + +import org.elasticsearch.test.ESTestCase; + +import static org.elasticsearch.common.geo.QuadkeyUtils.checkPrecisionRange; +import static org.elasticsearch.common.geo.QuadkeyUtils.hashToGeoPoint; +import static org.elasticsearch.common.geo.QuadkeyUtils.longEncode; +import static org.elasticsearch.common.geo.QuadkeyUtils.stringEncode; +import static org.hamcrest.Matchers.containsString; + +public class QuadkeyUtilsTests extends ESTestCase { + + /** + * Precision validation should throw an error if its outside of the valid range. + */ + public void testCheckPrecisionRange() { + for (int i = 0; i <= 29; i++) { + assertEquals(i, checkPrecisionRange(i)); + } + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> checkPrecisionRange(-1)); + assertThat(ex.getMessage(), containsString("Invalid quadkey precision of -1. Must be between 0 and 29.")); + ex = expectThrows(IllegalArgumentException.class, () -> checkPrecisionRange(30)); + assertThat(ex.getMessage(), containsString("Invalid quadkey precision of 30. Must be between 0 and 29.")); + } + + /** + * A few hardcoded lat/lng/zoom hashing expectations + */ + public void testLongEncode() { + assertEquals(0, longEncode(0, 0, 0)); + assertEquals(0x3c00000012e4cc66L, longEncode(30, 70, 15)); + assertEquals(0x7555555555440450L, longEncode(179.999, 89.999, 29)); + assertEquals(0x76aaaaaaaabbfbafL, longEncode(-179.999, -89.999, 29)); + assertEquals(0x0800000000000006L, longEncode(1, 1, 2)); + assertEquals(0x0c00000000000005L, longEncode(-20, 100, 3)); + assertEquals(0x70c0c9a5dc692fdcL, longEncode(13, -15, 28)); + assertEquals(0x4c00000fcdc7cde5L, longEncode(-12, 15, 19)); + } + + /** + * Ensure that for all points at all supported precision levels that the long encoding of a quadkey + * is compatible with its String based counterpart + */ + public void testQuadkeyAsLongRoutines() { + for (double lat=-90; lat<=90; lat++) { + for (double lng=-180; lng<=180; lng++) { + for(int p=0; p<=29; p++) { + long hash = longEncode(lng, lat, p); + if (p > 0) { + assertNotEquals(0, hash); + } + + // GeoPoint would be in the center of the bucket, thus must produce the same hash + GeoPoint point = hashToGeoPoint(hash); + long hashAsLong2 = longEncode(point.lon(), point.lat(), p); + assertEquals(hash, hashAsLong2); + + // Same point should be generated from the string key + assertEquals(point, hashToGeoPoint(stringEncode(hash))); + } + } + } + } +} From 3eeadb1604f6364d780083309f2d59b5c25a19da Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 25 Jan 2019 16:26:13 -0500 Subject: [PATCH 09/28] Added quadkey docs, changed dflt to 8 --- .../bucket/quadkeygrid-aggregation.asciidoc | 182 ++++++++++++++++++ .../QuadkeyGridAggregationBuilder.java | 2 +- 2 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 docs/reference/aggregations/bucket/quadkeygrid-aggregation.asciidoc diff --git a/docs/reference/aggregations/bucket/quadkeygrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/quadkeygrid-aggregation.asciidoc new file mode 100644 index 0000000000000..1648703b815cb --- /dev/null +++ b/docs/reference/aggregations/bucket/quadkeygrid-aggregation.asciidoc @@ -0,0 +1,182 @@ +[[search-aggregations-bucket-quadkeygrid-aggregation]] +=== Quadkey Grid Aggregation + +A multi-bucket aggregation that works on `geo_point` fields and groups points into buckets that represent cells in a grid. +The resulting grid can be sparse and only contains cells that have matching data. +Each cell corresponds to a https://en.wikipedia.org/wiki/Tiled_web_map[map tile] as used by many online map sites. +Each cell is labeled using a "{zoom}/{x}/{y}" format, where zoom is equal to the user-specified precision. + +* High precision quadkeys have a larger range for x and y, and represent tiles that cover only a small area. +* Low precision quadkeys have a smaller range for x and y, and represent tiles that each cover a large area. + +Quadkey used in this aggregation can have a choice of precision between 0 and 29. + +WARNING: The highest-precision quadkey of length 29 produces cells that cover less than a square metre of land and +so high-precision requests can be very costly in terms of RAM and result sizes. +Please see the example below on how to first filter the aggregation to a smaller geographic area before requesting +high-levels of detail. + +The specified field must be of type `geo_point` (which can only be set explicitly in the mappings) and it can also hold +an array of `geo_point` fields, in which case all points will be taken into account during aggregation. + + +==== Simple low-precision request + +[source,js] +-------------------------------------------------- +PUT /museums +{ + "mappings": { + "properties": { + "location": { + "type": "geo_point" + } + } + } +} + +POST /museums/_bulk?refresh +{"index":{"_id":1}} +{"location": "52.374081,4.912350", "name": "NEMO Science Museum"} +{"index":{"_id":2}} +{"location": "52.369219,4.901618", "name": "Museum Het Rembrandthuis"} +{"index":{"_id":3}} +{"location": "52.371667,4.914722", "name": "Nederlands Scheepvaartmuseum"} +{"index":{"_id":4}} +{"location": "51.222900,4.405200", "name": "Letterenhuis"} +{"index":{"_id":5}} +{"location": "48.861111,2.336389", "name": "Musée du Louvre"} +{"index":{"_id":6}} +{"location": "48.860000,2.327000", "name": "Musée d'Orsay"} + +POST /museums/_search?size=0 +{ + "aggregations" : { + "large-grid" : { + "quadkey_grid" : { + "field" : "location", + "precision" : 8 + } + } + } +} +-------------------------------------------------- +// CONSOLE + +Response: + +[source,js] +-------------------------------------------------- +{ + ... + "aggregations": { + "large-grid": { + "buckets": [ + { + "key" : "8/131/84", + "doc_count" : 3 + }, + { + "key" : "8/129/88", + "doc_count" : 2 + }, + { + "key" : "8/131/85", + "doc_count" : 1 + } + ] + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/] + +==== High-precision requests + +When requesting detailed buckets (typically for displaying a "zoomed in" map) a filter like <> should be applied to narrow the subject area otherwise potentially millions of buckets will be created and returned. + +[source,js] +-------------------------------------------------- +POST /museums/_search?size=0 +{ + "aggregations" : { + "zoomed-in" : { + "filter" : { + "geo_bounding_box" : { + "location" : { + "top_left" : "52.4, 4.9", + "bottom_right" : "52.3, 5.0" + } + } + }, + "aggregations":{ + "zoom1":{ + "quadkey_grid" : { + "field": "location", + "precision": 22 + } + } + } + } + } +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +[source,js] +-------------------------------------------------- +{ + ... + "aggregations" : { + "zoomed-in" : { + "doc_count" : 3, + "zoom1" : { + "buckets" : [ + { + "key" : "20/538603/344594", + "doc_count" : 1 + }, + { + "key" : "20/538596/344583", + "doc_count" : 1 + }, + { + "key" : "20/538564/344606", + "doc_count" : 1 + } + ] + } + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/] + + +==== Options + +[horizontal] +field:: Mandatory. The name of the field indexed with GeoPoints. + +precision:: Optional. The string zoom of the key used to define + cells/buckets in the results. Defaults to 7. + The precision can either be defined in terms of the integer + precision levels mentioned above. Values outside of [0,29] will + be rejected. + Alternatively, the precision level can be approximated from a + distance measure like "1km", "10m". The precision level is + calculate such that cells will not exceed the specified + size (diagonal) of the required precision. When this would lead + to precision levels higher than the supported 29 levels, + the value is rejected. + +size:: Optional. The maximum number of geohash buckets to return + (defaults to 10,000). When results are trimmed, buckets are + prioritised based on the volumes of documents they contain. + +shard_size:: Optional. To allow for more accurate counting of the top cells + returned in the final result the aggregation defaults to + returning `max(10,(size x number-of-shards))` buckets from each + shard. If this heuristic is undesirable, the number considered + from each shard can be over-ridden using this parameter. diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregationBuilder.java index 896cc0d31eebe..10a1c2d997b59 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregationBuilder.java @@ -36,7 +36,7 @@ public class QuadkeyGridAggregationBuilder extends GeoGridAggregationBuilder { public static final String NAME = "quadkey_grid"; - public static final int DEFAULT_PRECISION = 5; + public static final int DEFAULT_PRECISION = 7; public static final int DEFAULT_MAX_NUM_CELLS = 10000; private static final ObjectParser PARSER = createParser(NAME, QuadkeyUtils::parsePrecision); From 288178df1d2127035370416a40ac4d0494f54210 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 25 Jan 2019 17:16:28 -0500 Subject: [PATCH 10/28] string.format fix --- .../main/java/org/elasticsearch/common/geo/QuadkeyUtils.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java b/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java index 2c12960cdfd23..1afe6bd3f51a0 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java +++ b/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import java.io.IOException; +import java.util.Locale; import static org.elasticsearch.common.geo.GeoUtils.normalizeLat; import static org.elasticsearch.common.geo.GeoUtils.normalizeLon; @@ -188,6 +189,7 @@ private static GeoPoint zxyToGeoPoint(int zoom, int xTile, int yTile) { final double lon = ((xTile + 0.5) / tiles * 360.0) - 180; return new GeoPoint(lat, lon); } - throw new IllegalArgumentException(String.format("Invalid quadkey z/x/y values of %s/%s/%s", zoom, xTile, yTile)); + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid quadkey z/x/y values of %s/%s/%s", zoom, xTile, yTile)); } } From c1bc488b5ecc2ed98c354a330968c5f5c0ea7b8e Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 25 Jan 2019 17:33:07 -0500 Subject: [PATCH 11/28] fix doc example --- .../aggregations/bucket/quadkeygrid-aggregation.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/aggregations/bucket/quadkeygrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/quadkeygrid-aggregation.asciidoc index 1648703b815cb..0fa253a7a285a 100644 --- a/docs/reference/aggregations/bucket/quadkeygrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/quadkeygrid-aggregation.asciidoc @@ -134,15 +134,15 @@ POST /museums/_search?size=0 "zoom1" : { "buckets" : [ { - "key" : "20/538603/344594", + "key" : "22/2154412/1378379", "doc_count" : 1 }, { - "key" : "20/538596/344583", + "key" : "22/2154385/1378332", "doc_count" : 1 }, { - "key" : "20/538564/344606", + "key" : "22/2154259/1378425", "doc_count" : 1 } ] From caf478ecec4e1590c3c2d308b02b4538a9fa9f07 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 25 Jan 2019 18:50:14 -0500 Subject: [PATCH 12/28] fix unittest --- .../aggregations/bucket/geogrid/QuadkeyGridParserTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridParserTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridParserTests.java index 70be9cf7041ca..932df19fa4151 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridParserTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridParserTests.java @@ -113,7 +113,7 @@ public void testParseErrorOnPrecisionOutOfRange() throws Exception { fail(); } catch (XContentParseException ex) { assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class)); - assertEquals("Invalid quadkey aggregation precision of 30. Must be between 0 and 29.", ex.getCause().getMessage()); + assertEquals("Invalid quadkey precision of 30. Must be between 0 and 29.", ex.getCause().getMessage()); } } } From 5ac5c8238c4b3de534ba613b3f7f463af66bce1b Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 25 Jan 2019 19:02:57 -0500 Subject: [PATCH 13/28] renamed to geotile_grid --- .../client/RestHighLevelClient.java | 6 ++-- .../bucket/quadkeygrid-aggregation.asciidoc | 16 +++++----- .../{QuadkeyUtils.java => GeoTileUtils.java} | 24 +++++++-------- .../elasticsearch/search/SearchModule.java | 8 ++--- .../aggregations/AggregationBuilders.java | 10 +++---- ...ava => GeoTileGridAggregationBuilder.java} | 22 +++++++------- ...egator.java => GeoTileGridAggregator.java} | 16 +++++----- ...java => GeoTileGridAggregatorFactory.java} | 16 +++++----- ...dkeyGrid.java => InternalGeoTileGrid.java} | 16 +++++----- ...et.java => InternalGeoTileGridBucket.java} | 16 +++++----- ...uadkeyGrid.java => ParsedGeoTileGrid.java} | 8 ++--- ...cket.java => ParsedGeoTileGridBucket.java} | 10 +++---- ...UtilsTests.java => GeoTileUtilsTests.java} | 18 +++++------ ...s.java => GeoTileGridAggregatorTests.java} | 10 +++---- ...Tests.java => GeoTileGridParserTests.java} | 30 +++++++++---------- ...eyGridTests.java => GeoTileGridTests.java} | 20 ++++++------- .../test/InternalAggregationTestCase.java | 6 ++-- 17 files changed, 126 insertions(+), 126 deletions(-) rename server/src/main/java/org/elasticsearch/common/geo/{QuadkeyUtils.java => GeoTileUtils.java} (91%) rename server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/{QuadkeyGridAggregationBuilder.java => GeoTileGridAggregationBuilder.java} (82%) rename server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/{QuadkeyGridAggregator.java => GeoTileGridAggregator.java} (77%) rename server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/{QuadkeyGridAggregatorFactory.java => GeoTileGridAggregatorFactory.java} (82%) rename server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/{InternalQuadkeyGrid.java => InternalGeoTileGrid.java} (80%) rename server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/{InternalQuadkeyGridBucket.java => InternalGeoTileGridBucket.java} (74%) rename server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/{ParsedQuadkeyGrid.java => ParsedGeoTileGrid.java} (87%) rename server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/{ParsedQuadkeyGridBucket.java => ParsedGeoTileGridBucket.java} (80%) rename server/src/test/java/org/elasticsearch/common/geo/{QuadkeyUtilsTests.java => GeoTileUtilsTests.java} (82%) rename server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/{QuadkeyGridAggregatorTests.java => GeoTileGridAggregatorTests.java} (77%) rename server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/{QuadkeyGridParserTests.java => GeoTileGridParserTests.java} (82%) rename server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/{QuadkeyGridTests.java => GeoTileGridTests.java} (72%) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 1debc3d4d697f..4d5478e9434f0 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -95,8 +95,8 @@ import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid; -import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedQuadkeyGrid; -import org.elasticsearch.search.aggregations.bucket.geogrid.QuadkeyGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoTileGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal; import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -1761,7 +1761,7 @@ static List getDefaultNamedXContents() { map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)); map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); map.put(GeoHashGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c)); - map.put(QuadkeyGridAggregationBuilder.NAME, (p, c) -> ParsedQuadkeyGrid.fromXContent(p, (String) c)); + map.put(GeoTileGridAggregationBuilder.NAME, (p, c) -> ParsedGeoTileGrid.fromXContent(p, (String) c)); map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c)); map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c)); map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c)); diff --git a/docs/reference/aggregations/bucket/quadkeygrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/quadkeygrid-aggregation.asciidoc index 0fa253a7a285a..c565f75dfea90 100644 --- a/docs/reference/aggregations/bucket/quadkeygrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/quadkeygrid-aggregation.asciidoc @@ -1,17 +1,17 @@ -[[search-aggregations-bucket-quadkeygrid-aggregation]] -=== Quadkey Grid Aggregation +[[search-aggregations-bucket-geotilegrid-aggregation]] +=== GeoTile Grid Aggregation A multi-bucket aggregation that works on `geo_point` fields and groups points into buckets that represent cells in a grid. The resulting grid can be sparse and only contains cells that have matching data. Each cell corresponds to a https://en.wikipedia.org/wiki/Tiled_web_map[map tile] as used by many online map sites. Each cell is labeled using a "{zoom}/{x}/{y}" format, where zoom is equal to the user-specified precision. -* High precision quadkeys have a larger range for x and y, and represent tiles that cover only a small area. -* Low precision quadkeys have a smaller range for x and y, and represent tiles that each cover a large area. +* High precision keys have a larger range for x and y, and represent tiles that cover only a small area. +* Low precision keys have a smaller range for x and y, and represent tiles that each cover a large area. -Quadkey used in this aggregation can have a choice of precision between 0 and 29. +geotile used in this aggregation can have a choice of precision between 0 and 29. -WARNING: The highest-precision quadkey of length 29 produces cells that cover less than a square metre of land and +WARNING: The highest-precision geotile of length 29 produces cells that cover less than a square metre of land and so high-precision requests can be very costly in terms of RAM and result sizes. Please see the example below on how to first filter the aggregation to a smaller geographic area before requesting high-levels of detail. @@ -53,7 +53,7 @@ POST /museums/_search?size=0 { "aggregations" : { "large-grid" : { - "quadkey_grid" : { + "geotile_grid" : { "field" : "location", "precision" : 8 } @@ -111,7 +111,7 @@ POST /museums/_search?size=0 }, "aggregations":{ "zoom1":{ - "quadkey_grid" : { + "geotile_grid" : { "field": "location", "precision": 22 } diff --git a/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java b/server/src/main/java/org/elasticsearch/common/geo/GeoTileUtils.java similarity index 91% rename from server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java rename to server/src/main/java/org/elasticsearch/common/geo/GeoTileUtils.java index 1afe6bd3f51a0..a764a64ea9a48 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/QuadkeyUtils.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoTileUtils.java @@ -34,7 +34,7 @@ * The string key is formatted as "zoom/x/y" * The hash value (long) contains all three of those values. */ -public class QuadkeyUtils { +public class GeoTileUtils { /** * Largest number of tiles (precision) to use. @@ -54,19 +54,19 @@ public class QuadkeyUtils { private static final int ZOOM_SHIFT = 29 * 2; /** - * Mask of all the bits used by the quadkey in a hash + * Mask of all the bits used by the geotile in a hash */ - private static final long QUADKEY_MASK = (1L << ZOOM_SHIFT) - 1; + private static final long GEOTILE_MASK = (1L << ZOOM_SHIFT) - 1; /** - * Parse quadkey hash as zoom, x, y integers. + * Parse geotile hash as zoom, x, y integers. */ private static int[] parseHash(final long hash) { final int zoom = checkPrecisionRange((int) (hash >>> ZOOM_SHIFT)); final int tiles = 1 << zoom; - // decode the quadkey bits as interleaved xTile and yTile - long val = hash & QUADKEY_MASK; + // decode the geotile bits as interleaved xTile and yTile + long val = hash & GEOTILE_MASK; int xTile = (int) BitUtil.deinterleave(val); int yTile = (int) BitUtil.deinterleave(val >>> 1); if (xTile < 0 || yTile < 0 || xTile >= tiles || yTile >= tiles) { @@ -109,14 +109,14 @@ public static int parsePrecision(XContentParser parser) throws IOException, Elas public static int checkPrecisionRange(int precision) { if (precision < 0 || precision > MAX_ZOOM) { - throw new IllegalArgumentException("Invalid quadkey precision of " + + throw new IllegalArgumentException("Invalid geotile_grid precision of " + precision + ". Must be between 0 and " + MAX_ZOOM + "."); } return precision; } /** - * Encode lon/lat to the quadkey based long format. + * Encode lon/lat to the geotile based long format. * The resulting hash contains interleaved tile X and Y coordinates. * The precision itself is also encoded as a few high bits. */ @@ -146,14 +146,14 @@ public static long longEncode(double longitude, double latitude, int precision) yTile = tiles - 1; } - // Zoom value is placed in front of all the bits used for the quadkey + // Zoom value is placed in front of all the bits used for the geotile // e.g. if max zoom is 26, the largest index would use 52 bits (51st..0th), // leaving 12 bits unused for zoom. See MAX_ZOOM comment above. return BitUtil.interleave(xTile, yTile) | ((long) precision << ZOOM_SHIFT); } /** - * Encode to a quadkey string from the quadkey based long format + * Encode to a geotile string from the geotile based long format */ public static String stringEncode(long hash) { int[] res = parseHash(hash); @@ -176,7 +176,7 @@ public static GeoPoint hashToGeoPoint(String hashAsString) { // This will also handle NumberFormatException cause = e; } - throw new IllegalArgumentException("Invalid quadkey hash string of " + + throw new IllegalArgumentException("Invalid geotile_grid hash string of " + hashAsString + ". Must be three integers in a form \"zoom/x/y\".", cause); } @@ -190,6 +190,6 @@ private static GeoPoint zxyToGeoPoint(int zoom, int xTile, int yTile) { return new GeoPoint(lat, lon); } throw new IllegalArgumentException( - String.format(Locale.ROOT, "Invalid quadkey z/x/y values of %s/%s/%s", zoom, xTile, yTile)); + String.format(Locale.ROOT, "Invalid geotile_grid z/x/y values of %s/%s/%s", zoom, xTile, yTile)); } } diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 361fb47bf0b67..81c6273ec1a36 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -110,8 +110,8 @@ import org.elasticsearch.search.aggregations.bucket.filter.InternalFilters; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid; -import org.elasticsearch.search.aggregations.bucket.geogrid.QuadkeyGridAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.geogrid.InternalQuadkeyGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.InternalGlobal; import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -424,8 +424,8 @@ private void registerAggregations(List plugins) { GeoDistanceAggregationBuilder::parse).addResultReader(InternalGeoDistance::new)); registerAggregation(new AggregationSpec(GeoHashGridAggregationBuilder.NAME, GeoHashGridAggregationBuilder::new, GeoHashGridAggregationBuilder::parse).addResultReader(InternalGeoHashGrid::new)); - registerAggregation(new AggregationSpec(QuadkeyGridAggregationBuilder.NAME, QuadkeyGridAggregationBuilder::new, - QuadkeyGridAggregationBuilder::parse).addResultReader(InternalQuadkeyGrid::new)); + registerAggregation(new AggregationSpec(GeoTileGridAggregationBuilder.NAME, GeoTileGridAggregationBuilder::new, + GeoTileGridAggregationBuilder::parse).addResultReader(InternalGeoTileGrid::new)); registerAggregation(new AggregationSpec(NestedAggregationBuilder.NAME, NestedAggregationBuilder::new, NestedAggregationBuilder::parse).addResultReader(InternalNested::new)); registerAggregation(new AggregationSpec(ReverseNestedAggregationBuilder.NAME, ReverseNestedAggregationBuilder::new, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java index 586a5496391cc..fee15ccf20e8f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java @@ -30,8 +30,8 @@ import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator.KeyedFilter; import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.geogrid.InternalQuadkeyGrid; -import org.elasticsearch.search.aggregations.bucket.geogrid.QuadkeyGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; @@ -253,10 +253,10 @@ public static GeoHashGridAggregationBuilder geohashGrid(String name) { } /** - * Create a new {@link InternalQuadkeyGrid} aggregation with the given name. + * Create a new {@link InternalGeoTileGrid} aggregation with the given name. */ - public static QuadkeyGridAggregationBuilder quadkeyGrid(String name) { - return new QuadkeyGridAggregationBuilder(name); + public static GeoTileGridAggregationBuilder geoTileGrid(String name) { + return new GeoTileGridAggregationBuilder(name); } /** diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java similarity index 82% rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregationBuilder.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java index 10a1c2d997b59..70acd8fa742e9 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java @@ -19,7 +19,7 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; -import org.elasticsearch.common.geo.QuadkeyUtils; +import org.elasticsearch.common.geo.GeoTileUtils; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; @@ -34,27 +34,27 @@ import java.io.IOException; import java.util.Map; -public class QuadkeyGridAggregationBuilder extends GeoGridAggregationBuilder { - public static final String NAME = "quadkey_grid"; +public class GeoTileGridAggregationBuilder extends GeoGridAggregationBuilder { + public static final String NAME = "geotile_grid"; public static final int DEFAULT_PRECISION = 7; public static final int DEFAULT_MAX_NUM_CELLS = 10000; - private static final ObjectParser PARSER = createParser(NAME, QuadkeyUtils::parsePrecision); + private static final ObjectParser PARSER = createParser(NAME, GeoTileUtils::parsePrecision); - public QuadkeyGridAggregationBuilder(String name) { + public GeoTileGridAggregationBuilder(String name) { super(name); precision(DEFAULT_PRECISION); size(DEFAULT_MAX_NUM_CELLS); shardSize = -1; } - public QuadkeyGridAggregationBuilder(StreamInput in) throws IOException { + public GeoTileGridAggregationBuilder(StreamInput in) throws IOException { super(in); } @Override public GeoGridAggregationBuilder precision(int precision) { - this.precision = QuadkeyUtils.checkPrecisionRange(precision); + this.precision = GeoTileUtils.checkPrecisionRange(precision); return this; } @@ -64,22 +64,22 @@ public GeoGridAggregationBuilder precision(int precision) { SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData ) throws IOException { - return new QuadkeyGridAggregatorFactory(name, config, precision, requiredSize, shardSize, context, parent, + return new GeoTileGridAggregatorFactory(name, config, precision, requiredSize, shardSize, context, parent, subFactoriesBuilder, metaData); } - private QuadkeyGridAggregationBuilder(QuadkeyGridAggregationBuilder clone, AggregatorFactories.Builder factoriesBuilder, + private GeoTileGridAggregationBuilder(GeoTileGridAggregationBuilder clone, AggregatorFactories.Builder factoriesBuilder, Map metaData) { super(clone, factoriesBuilder, metaData); } @Override protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map metaData) { - return new QuadkeyGridAggregationBuilder(this, factoriesBuilder, metaData); + return new GeoTileGridAggregationBuilder(this, factoriesBuilder, metaData); } public static GeoGridAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException { - return PARSER.parse(parser, new QuadkeyGridAggregationBuilder(aggregationName), null); + return PARSER.parse(parser, new GeoTileGridAggregationBuilder(aggregationName), null); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java similarity index 77% rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregator.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java index 193b8d6507459..d2ff5ed82513c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java @@ -30,28 +30,28 @@ import java.util.Map; /** - * Aggregates data expressed as Quadkey longs (for efficiency's sake) but formats results as Quadkey strings. + * Aggregates data expressed as geotile longs (for efficiency's sake) but formats results as geotile strings. */ -public class QuadkeyGridAggregator extends GeoGridAggregator { +public class GeoTileGridAggregator extends GeoGridAggregator { - QuadkeyGridAggregator(String name, AggregatorFactories factories, CellIdSource valuesSource, + GeoTileGridAggregator(String name, AggregatorFactories factories, CellIdSource valuesSource, int requiredSize, int shardSize, SearchContext aggregationContext, Aggregator parent, List pipelineAggregators, Map metaData) throws IOException { super(name, factories, valuesSource, requiredSize, shardSize, aggregationContext, parent, pipelineAggregators, metaData); } @Override - InternalQuadkeyGrid buildAggregation(String name, int requiredSize, List buckets, + InternalGeoTileGrid buildAggregation(String name, int requiredSize, List buckets, List pipelineAggregators, Map metaData) { - return new InternalQuadkeyGrid(name, requiredSize, buckets, pipelineAggregators, metaData); + return new InternalGeoTileGrid(name, requiredSize, buckets, pipelineAggregators, metaData); } @Override - public InternalQuadkeyGrid buildEmptyAggregation() { - return new InternalQuadkeyGrid(name, requiredSize, Collections.emptyList(), pipelineAggregators(), metaData()); + public InternalGeoTileGrid buildEmptyAggregation() { + return new InternalGeoTileGrid(name, requiredSize, Collections.emptyList(), pipelineAggregators(), metaData()); } InternalGeoGridBucket newEmptyBucket() { - return new InternalQuadkeyGridBucket(0, 0, null); + return new InternalGeoTileGridBucket(0, 0, null); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java similarity index 82% rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregatorFactory.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java index a75a2dc03c305..5aae1f15cc7ee 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java @@ -19,7 +19,7 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; -import org.elasticsearch.common.geo.QuadkeyUtils; +import org.elasticsearch.common.geo.GeoTileUtils; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; @@ -37,15 +37,15 @@ import java.util.List; import java.util.Map; -public class QuadkeyGridAggregatorFactory extends ValuesSourceAggregatorFactory { +public class GeoTileGridAggregatorFactory extends ValuesSourceAggregatorFactory { private final int precision; private final int requiredSize; private final int shardSize; - QuadkeyGridAggregatorFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, - int shardSize, SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, - Map metaData) throws IOException { + GeoTileGridAggregatorFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, + int shardSize, SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, + Map metaData) throws IOException { super(name, config, context, parent, subFactoriesBuilder, metaData); this.precision = precision; this.requiredSize = requiredSize; @@ -55,7 +55,7 @@ public class QuadkeyGridAggregatorFactory extends ValuesSourceAggregatorFactory< @Override protected Aggregator createUnmapped(Aggregator parent, List pipelineAggregators, Map metaData) throws IOException { - final InternalAggregation aggregation = new InternalQuadkeyGrid(name, requiredSize, + final InternalAggregation aggregation = new InternalGeoTileGrid(name, requiredSize, Collections.emptyList(), pipelineAggregators, metaData); return new NonCollectingAggregator(name, context, parent, pipelineAggregators, metaData) { @Override @@ -71,8 +71,8 @@ protected Aggregator doCreateInternal(final ValuesSource.GeoPoint valuesSource, if (collectsFromSingleBucket == false) { return asMultiBucketAggregator(this, context, parent); } - CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, QuadkeyUtils::longEncode); - return new QuadkeyGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, context, parent, + CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, GeoTileUtils::longEncode); + return new GeoTileGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, context, parent, pipelineAggregators, metaData); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGrid.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java similarity index 80% rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGrid.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java index 8821139289c91..8a842b66dcfca 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGrid.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java @@ -31,39 +31,39 @@ * All geohashes in a grid are of the same precision and held internally as a single long * for efficiency's sake. */ -public class InternalQuadkeyGrid extends InternalGeoGrid { +public class InternalGeoTileGrid extends InternalGeoGrid { - InternalQuadkeyGrid(String name, int requiredSize, List buckets, + InternalGeoTileGrid(String name, int requiredSize, List buckets, List pipelineAggregators, Map metaData) { super(name, requiredSize, buckets, pipelineAggregators, metaData); } - public InternalQuadkeyGrid(StreamInput in) throws IOException { + public InternalGeoTileGrid(StreamInput in) throws IOException { super(in); } @Override public InternalGeoGrid create(List buckets) { - return new InternalQuadkeyGrid(name, requiredSize, buckets, pipelineAggregators(), metaData); + return new InternalGeoTileGrid(name, requiredSize, buckets, pipelineAggregators(), metaData); } @Override public InternalGeoGridBucket createBucket(InternalAggregations aggregations, InternalGeoGridBucket prototype) { - return new InternalQuadkeyGridBucket(prototype.hashAsLong, prototype.docCount, aggregations); + return new InternalGeoTileGridBucket(prototype.hashAsLong, prototype.docCount, aggregations); } @Override InternalGeoGrid create(String name, int requiredSize, List buckets, List list, Map metaData) { - return new InternalQuadkeyGrid(name, requiredSize, buckets, list, metaData); + return new InternalGeoTileGrid(name, requiredSize, buckets, list, metaData); } @Override Reader getBucketReader() { - return InternalQuadkeyGridBucket::new; + return InternalGeoTileGridBucket::new; } @Override public String getWriteableName() { - return QuadkeyGridAggregationBuilder.NAME; + return GeoTileGridAggregationBuilder.NAME; } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGridBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java similarity index 74% rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGridBucket.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java index 1d28d1d6b8b52..9cecf52fb0639 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalQuadkeyGridBucket.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java @@ -19,38 +19,38 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; -import org.elasticsearch.common.geo.QuadkeyUtils; +import org.elasticsearch.common.geo.GeoTileUtils; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.search.aggregations.InternalAggregations; import java.io.IOException; -public class InternalQuadkeyGridBucket extends InternalGeoGridBucket { - InternalQuadkeyGridBucket(long hashAsLong, long docCount, InternalAggregations aggregations) { +public class InternalGeoTileGridBucket extends InternalGeoGridBucket { + InternalGeoTileGridBucket(long hashAsLong, long docCount, InternalAggregations aggregations) { super(hashAsLong, docCount, aggregations); } /** * Read from a stream. */ - public InternalQuadkeyGridBucket(StreamInput in) throws IOException { + public InternalGeoTileGridBucket(StreamInput in) throws IOException { super(in); } @Override - InternalQuadkeyGridBucket buildBucket(InternalGeoGridBucket bucket, long hashAsLong, long docCount, + InternalGeoTileGridBucket buildBucket(InternalGeoGridBucket bucket, long hashAsLong, long docCount, InternalAggregations aggregations) { - return new InternalQuadkeyGridBucket(hashAsLong, docCount, aggregations); + return new InternalGeoTileGridBucket(hashAsLong, docCount, aggregations); } @Override public String getKeyAsString() { - return QuadkeyUtils.stringEncode(hashAsLong); + return GeoTileUtils.stringEncode(hashAsLong); } @Override public GeoPoint getKey() { - return QuadkeyUtils.hashToGeoPoint(hashAsLong); + return GeoTileUtils.hashToGeoPoint(hashAsLong); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGrid.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java similarity index 87% rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGrid.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java index d9798cb3c59ee..e88c7ad305433 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGrid.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java @@ -24,10 +24,10 @@ import java.io.IOException; -public class ParsedQuadkeyGrid extends ParsedGeoGrid { +public class ParsedGeoTileGrid extends ParsedGeoGrid { - private static ObjectParser PARSER = createParser(ParsedQuadkeyGrid::new, - ParsedQuadkeyGridBucket::fromXContent, ParsedQuadkeyGridBucket::fromXContent); + private static ObjectParser PARSER = createParser(ParsedGeoTileGrid::new, + ParsedGeoTileGridBucket::fromXContent, ParsedGeoTileGridBucket::fromXContent); public static ParsedGeoGrid fromXContent(XContentParser parser, String name) throws IOException { ParsedGeoGrid aggregation = PARSER.parse(parser, null); @@ -37,6 +37,6 @@ public static ParsedGeoGrid fromXContent(XContentParser parser, String name) thr @Override public String getType() { - return QuadkeyGridAggregationBuilder.NAME; + return GeoTileGridAggregationBuilder.NAME; } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGridBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java similarity index 80% rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGridBucket.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java index db7e65fbfc708..4b3080c18562b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedQuadkeyGridBucket.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java @@ -20,16 +20,16 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.geo.QuadkeyUtils; +import org.elasticsearch.common.geo.GeoTileUtils; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; -public class ParsedQuadkeyGridBucket extends ParsedGeoGridBucket { +public class ParsedGeoTileGridBucket extends ParsedGeoGridBucket { @Override public GeoPoint getKey() { - return QuadkeyUtils.hashToGeoPoint(hashAsString); + return GeoTileUtils.hashToGeoPoint(hashAsString); } @Override @@ -37,7 +37,7 @@ public String getKeyAsString() { return hashAsString; } - static ParsedQuadkeyGridBucket fromXContent(XContentParser parser) throws IOException { - return parseXContent(parser, false, ParsedQuadkeyGridBucket::new, (p, bucket) -> bucket.hashAsString = p.textOrNull()); + static ParsedGeoTileGridBucket fromXContent(XContentParser parser) throws IOException { + return parseXContent(parser, false, ParsedGeoTileGridBucket::new, (p, bucket) -> bucket.hashAsString = p.textOrNull()); } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/QuadkeyUtilsTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoTileUtilsTests.java similarity index 82% rename from server/src/test/java/org/elasticsearch/common/geo/QuadkeyUtilsTests.java rename to server/src/test/java/org/elasticsearch/common/geo/GeoTileUtilsTests.java index 00fea08cb8ef6..3db84e8403595 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/QuadkeyUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoTileUtilsTests.java @@ -21,13 +21,13 @@ import org.elasticsearch.test.ESTestCase; -import static org.elasticsearch.common.geo.QuadkeyUtils.checkPrecisionRange; -import static org.elasticsearch.common.geo.QuadkeyUtils.hashToGeoPoint; -import static org.elasticsearch.common.geo.QuadkeyUtils.longEncode; -import static org.elasticsearch.common.geo.QuadkeyUtils.stringEncode; +import static org.elasticsearch.common.geo.GeoTileUtils.checkPrecisionRange; +import static org.elasticsearch.common.geo.GeoTileUtils.hashToGeoPoint; +import static org.elasticsearch.common.geo.GeoTileUtils.longEncode; +import static org.elasticsearch.common.geo.GeoTileUtils.stringEncode; import static org.hamcrest.Matchers.containsString; -public class QuadkeyUtilsTests extends ESTestCase { +public class GeoTileUtilsTests extends ESTestCase { /** * Precision validation should throw an error if its outside of the valid range. @@ -37,9 +37,9 @@ public void testCheckPrecisionRange() { assertEquals(i, checkPrecisionRange(i)); } IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> checkPrecisionRange(-1)); - assertThat(ex.getMessage(), containsString("Invalid quadkey precision of -1. Must be between 0 and 29.")); + assertThat(ex.getMessage(), containsString("Invalid geotile_grid precision of -1. Must be between 0 and 29.")); ex = expectThrows(IllegalArgumentException.class, () -> checkPrecisionRange(30)); - assertThat(ex.getMessage(), containsString("Invalid quadkey precision of 30. Must be between 0 and 29.")); + assertThat(ex.getMessage(), containsString("Invalid geotile_grid precision of 30. Must be between 0 and 29.")); } /** @@ -57,10 +57,10 @@ public void testLongEncode() { } /** - * Ensure that for all points at all supported precision levels that the long encoding of a quadkey + * Ensure that for all points at all supported precision levels that the long encoding of a geotile * is compatible with its String based counterpart */ - public void testQuadkeyAsLongRoutines() { + public void testGeoTileAsLongRoutines() { for (double lat=-90; lat<=90; lat++) { for (double lng=-180; lng<=180; lng++) { for(int p=0; p<=29; p++) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java similarity index 77% rename from server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregatorTests.java rename to server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java index ecaac269411c5..e72ad337424ec 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java @@ -19,11 +19,11 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; -import static org.elasticsearch.common.geo.QuadkeyUtils.MAX_ZOOM; -import static org.elasticsearch.common.geo.QuadkeyUtils.longEncode; -import static org.elasticsearch.common.geo.QuadkeyUtils.stringEncode; +import static org.elasticsearch.common.geo.GeoTileUtils.MAX_ZOOM; +import static org.elasticsearch.common.geo.GeoTileUtils.longEncode; +import static org.elasticsearch.common.geo.GeoTileUtils.stringEncode; -public class QuadkeyGridAggregatorTests extends GeoGridAggregatorTestCase { +public class GeoTileGridAggregatorTests extends GeoGridAggregatorTestCase { @Override protected int randomPrecision() { @@ -37,6 +37,6 @@ protected String hashAsString(double lng, double lat, int precision) { @Override protected GeoGridAggregationBuilder createBuilder(String name) { - return new QuadkeyGridAggregationBuilder(name); + return new GeoTileGridAggregationBuilder(name); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridParserTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java similarity index 82% rename from server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridParserTests.java rename to server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java index 932df19fa4151..39a2f559f7fff 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridParserTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java @@ -19,7 +19,7 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.common.geo.QuadkeyUtils; +import org.elasticsearch.common.geo.GeoTileUtils; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentParseException; import org.elasticsearch.common.xcontent.XContentParser; @@ -31,25 +31,25 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThanOrEqualTo; -public class QuadkeyGridParserTests extends ESTestCase { +public class GeoTileGridParserTests extends ESTestCase { public void testParseValidFromInts() throws Exception { - int precision = randomIntBetween(0, QuadkeyUtils.MAX_ZOOM); + int precision = randomIntBetween(0, GeoTileUtils.MAX_ZOOM); XContentParser stParser = createParser(JsonXContent.jsonXContent, "{\"field\":\"my_loc\", \"precision\":" + precision + ", \"size\": 500, \"shard_size\": 550}"); XContentParser.Token token = stParser.nextToken(); assertSame(XContentParser.Token.START_OBJECT, token); // can create a factory - assertNotNull(QuadkeyGridAggregationBuilder.parse("quadkey_grid", stParser)); + assertNotNull(GeoTileGridAggregationBuilder.parse("geotile_grid", stParser)); } public void testParseValidFromStrings() throws Exception { - int precision = randomIntBetween(0, QuadkeyUtils.MAX_ZOOM); + int precision = randomIntBetween(0, GeoTileUtils.MAX_ZOOM); XContentParser stParser = createParser(JsonXContent.jsonXContent, "{\"field\":\"my_loc\", \"precision\":\"" + precision + "\", \"size\": \"500\", \"shard_size\": \"550\"}"); XContentParser.Token token = stParser.nextToken(); assertSame(XContentParser.Token.START_OBJECT, token); // can create a factory - assertNotNull(QuadkeyGridAggregationBuilder.parse("quadkey_grid", stParser)); + assertNotNull(GeoTileGridAggregationBuilder.parse("geotile_grid", stParser)); } public void testParseDistanceUnitPrecision() throws Exception { @@ -64,7 +64,7 @@ public void testParseDistanceUnitPrecision() throws Exception { XContentParser.Token token = stParser.nextToken(); assertSame(XContentParser.Token.START_OBJECT, token); // can create a factory - GeoGridAggregationBuilder builder = QuadkeyGridAggregationBuilder.parse("quadkey_grid", stParser); + GeoGridAggregationBuilder builder = GeoTileGridAggregationBuilder.parse("geotile_grid", stParser); assertNotNull(builder); assertThat(builder.precision(), greaterThanOrEqualTo(0)); assertThat(builder.precision(), lessThanOrEqualTo(29)); @@ -76,8 +76,8 @@ public void testParseInvalidUnitPrecision() throws Exception { XContentParser.Token token = stParser.nextToken(); assertSame(XContentParser.Token.START_OBJECT, token); XContentParseException ex = expectThrows(XContentParseException.class, - () -> QuadkeyGridAggregationBuilder.parse("quadkey_grid", stParser)); - assertThat(ex.getMessage(), containsString("[quadkey_grid] failed to parse field [precision]")); + () -> GeoTileGridAggregationBuilder.parse("geotile_grid", stParser)); + assertThat(ex.getMessage(), containsString("[geotile_grid] failed to parse field [precision]")); assertThat(ex.getCause(), instanceOf(NumberFormatException.class)); assertEquals("For input string: \"10kg\"", ex.getCause().getMessage()); } @@ -88,8 +88,8 @@ public void testParseDistanceUnitPrecisionTooSmall() throws Exception { XContentParser.Token token = stParser.nextToken(); assertSame(XContentParser.Token.START_OBJECT, token); XContentParseException ex = expectThrows(XContentParseException.class, - () -> QuadkeyGridAggregationBuilder.parse("quadkey_grid", stParser)); - assertThat(ex.getMessage(), containsString("[quadkey_grid] failed to parse field [precision]")); + () -> GeoTileGridAggregationBuilder.parse("geotile_grid", stParser)); + assertThat(ex.getMessage(), containsString("[geotile_grid] failed to parse field [precision]")); assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class)); assertEquals("precision too high [1cm]", ex.getCause().getMessage()); } @@ -99,9 +99,9 @@ public void testParseErrorOnBooleanPrecision() throws Exception { XContentParser.Token token = stParser.nextToken(); assertSame(XContentParser.Token.START_OBJECT, token); XContentParseException e = expectThrows(XContentParseException.class, - () -> QuadkeyGridAggregationBuilder.parse("quadkey_grid", stParser)); + () -> GeoTileGridAggregationBuilder.parse("geotile_grid", stParser)); assertThat(ExceptionsHelper.detailedMessage(e), - containsString("[quadkey_grid] precision doesn't support values of type: VALUE_BOOLEAN")); + containsString("[geotile_grid] precision doesn't support values of type: VALUE_BOOLEAN")); } public void testParseErrorOnPrecisionOutOfRange() throws Exception { @@ -109,11 +109,11 @@ public void testParseErrorOnPrecisionOutOfRange() throws Exception { XContentParser.Token token = stParser.nextToken(); assertSame(XContentParser.Token.START_OBJECT, token); try { - QuadkeyGridAggregationBuilder.parse("quadkey_grid", stParser); + GeoTileGridAggregationBuilder.parse("geotile_grid", stParser); fail(); } catch (XContentParseException ex) { assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class)); - assertEquals("Invalid quadkey precision of 30. Must be between 0 and 29.", ex.getCause().getMessage()); + assertEquals("Invalid geotile_grid precision of 30. Must be between 0 and 29.", ex.getCause().getMessage()); } } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java similarity index 72% rename from server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridTests.java rename to server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java index 487b0b1e17245..900e9f312d83a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/QuadkeyGridTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java @@ -18,7 +18,7 @@ */ package org.elasticsearch.search.aggregations.bucket.geogrid; -import org.elasticsearch.common.geo.QuadkeyUtils; +import org.elasticsearch.common.geo.GeoTileUtils; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; @@ -26,29 +26,29 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.common.geo.QuadkeyUtils.MAX_ZOOM; +import static org.elasticsearch.common.geo.GeoTileUtils.MAX_ZOOM; -public class QuadkeyGridTests extends GeoGridTestCase { +public class GeoTileGridTests extends GeoGridTestCase { @Override - protected InternalQuadkeyGrid createInternalGeoGrid(String name, int size, List buckets, + protected InternalGeoTileGrid createInternalGeoGrid(String name, int size, List buckets, List pipelineAggregators, Map metaData) { - return new InternalQuadkeyGrid(name, size, buckets, pipelineAggregators, metaData); + return new InternalGeoTileGrid(name, size, buckets, pipelineAggregators, metaData); } @Override - protected Writeable.Reader instanceReader() { - return InternalQuadkeyGrid::new; + protected Writeable.Reader instanceReader() { + return InternalGeoTileGrid::new; } @Override - protected InternalQuadkeyGridBucket createInternalGeoGridBucket(Long key, long docCount, InternalAggregations aggregations) { - return new InternalQuadkeyGridBucket(key, docCount, aggregations); + protected InternalGeoTileGridBucket createInternalGeoGridBucket(Long key, long docCount, InternalAggregations aggregations) { + return new InternalGeoTileGridBucket(key, docCount, aggregations); } @Override protected long longEncode(double lng, double lat, int precision) { - return QuadkeyUtils.longEncode(lng, lat, precision); + return GeoTileUtils.longEncode(lng, lat, precision); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java index 7b7a75c9a0333..f9d72e38044fb 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java @@ -51,8 +51,8 @@ import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid; -import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedQuadkeyGrid; -import org.elasticsearch.search.aggregations.bucket.geogrid.QuadkeyGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoTileGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal; import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -214,7 +214,7 @@ public abstract class InternalAggregationTestCase map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)); map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); map.put(GeoHashGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c)); - map.put(QuadkeyGridAggregationBuilder.NAME, (p, c) -> ParsedQuadkeyGrid.fromXContent(p, (String) c)); + map.put(GeoTileGridAggregationBuilder.NAME, (p, c) -> ParsedGeoTileGrid.fromXContent(p, (String) c)); map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c)); map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c)); map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c)); From 01b2a3ff853867d21487f0995afd638abba70649 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 25 Jan 2019 19:13:09 -0500 Subject: [PATCH 14/28] style fix --- .../bucket/geogrid/GeoTileGridAggregatorFactory.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java index 5aae1f15cc7ee..b58fb196186e2 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java @@ -44,8 +44,9 @@ public class GeoTileGridAggregatorFactory extends ValuesSourceAggregatorFactory< private final int shardSize; GeoTileGridAggregatorFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, - int shardSize, SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, - Map metaData) throws IOException { + int shardSize, SearchContext context, AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, Map metaData + ) throws IOException { super(name, config, context, parent, subFactoriesBuilder, metaData); this.precision = precision; this.requiredSize = requiredSize; From c5975297350367d7608daded990b11cc4e56282d Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Mon, 28 Jan 2019 13:17:00 -0500 Subject: [PATCH 15/28] Add GeoTileGridTests to AggregationsTests --- .../elasticsearch/search/aggregations/AggregationsTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java index 874623132f36a..012171ec25a0b 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.InternalFilterTests; import org.elasticsearch.search.aggregations.bucket.filter.InternalFiltersTests; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridTests; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridTests; import org.elasticsearch.search.aggregations.bucket.global.InternalGlobalTests; import org.elasticsearch.search.aggregations.bucket.histogram.InternalAutoDateHistogramTests; import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogramTests; @@ -140,6 +141,7 @@ private static List> getAggsTests() { aggsTests.add(new InternalFilterTests()); aggsTests.add(new InternalSamplerTests()); aggsTests.add(new GeoHashGridTests()); + aggsTests.add(new GeoTileGridTests()); aggsTests.add(new InternalRangeTests()); aggsTests.add(new InternalDateRangeTests()); aggsTests.add(new InternalGeoDistanceTests()); From 778452da824c4c2ffe552cbbe15ccb71fe3362c7 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Mon, 28 Jan 2019 13:48:00 -0500 Subject: [PATCH 16/28] Add geotile_grid test to ShardReduceIT --- .../search/aggregations/AggregationBuilders.java | 2 +- .../aggregations/bucket/ShardReduceIT.java | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java index fee15ccf20e8f..d78e42ba89603 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java @@ -255,7 +255,7 @@ public static GeoHashGridAggregationBuilder geohashGrid(String name) { /** * Create a new {@link InternalGeoTileGrid} aggregation with the given name. */ - public static GeoTileGridAggregationBuilder geoTileGrid(String name) { + public static GeoTileGridAggregationBuilder geotileGrid(String name) { return new GeoTileGridAggregationBuilder(name); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java index 664edba7db0d8..8cb42e352156b 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java @@ -39,6 +39,7 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.dateRange; import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; import static org.elasticsearch.search.aggregations.AggregationBuilders.geohashGrid; +import static org.elasticsearch.search.aggregations.AggregationBuilders.geotileGrid; import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.ipRange; @@ -306,5 +307,20 @@ public void testGeoHashGrid() throws Exception { assertThat(histo.getBuckets().size(), equalTo(4)); } + public void testGeoTileGrid() throws Exception { + SearchResponse response = client().prepareSearch("idx") + .setQuery(QueryBuilders.matchAllQuery()) + .addAggregation(geotileGrid("grid").field("location") + .subAggregation(dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY) + .minDocCount(0))) + .get(); + + assertSearchResponse(response); + + GeoGrid grid = response.getAggregations().get("grid"); + Histogram histo = grid.getBuckets().iterator().next().getAggregations().get("histo"); + assertThat(histo.getBuckets().size(), equalTo(4)); + } + } From 9d3246838dad8a33739d39cd1a802286a0c3690f Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 29 Jan 2019 15:42:59 -0800 Subject: [PATCH 17/28] add basic REST test --- .../search.aggregation/290_geotile_grid.yml | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/290_geotile_grid.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/290_geotile_grid.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/290_geotile_grid.yml new file mode 100644 index 0000000000000..efed932bccbeb --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/290_geotile_grid.yml @@ -0,0 +1,71 @@ +setup: + - skip: + version: " - 6.99.99" + reason: "added in 7.0.0" + - do: + indices.create: + index: test_1 + body: + settings: + number_of_replicas: 0 + mappings: + _doc: + properties: + location: + type: geo_point + +--- +"Basic test": + - do: + bulk: + refresh: true + body: + - index: + _index: test_1 + _type: _doc + _id: 1 + - location: "52.374081,4.912350" + - index: + _index: test_1 + _type: _doc + _id: 2 + - location: "52.369219,4.901618" + - index: + _index: test_1 + _type: _doc + _id: 3 + - location: "52.371667,4.914722" + - index: + _index: test_1 + _type: _doc + _id: 4 + - location: "51.222900,4.405200" + - index: + _index: test_1 + _type: _doc + _id: 5 + - location: "48.861111,2.336389" + - index: + _index: test_1 + _type: _doc + _id: 6 + - location: "48.860000,2.327000" + + - do: + search: + rest_total_hits_as_int: true + body: + aggregations: + grid: + geotile_grid: + field: location + precision: 8 + + + - match: { hits.total: 6 } + - match: { aggregations.grid.buckets.0.key: "8/131/84" } + - match: { aggregations.grid.buckets.0.doc_count: 3 } + - match: { aggregations.grid.buckets.1.key: "8/129/88" } + - match: { aggregations.grid.buckets.1.doc_count: 2 } + - match: { aggregations.grid.buckets.2.key: "8/131/85" } + - match: { aggregations.grid.buckets.2.doc_count: 1 } From 11d57dfdcc4cfaacf6b6c1ddb2be3970d295edff Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 29 Jan 2019 15:52:36 -0800 Subject: [PATCH 18/28] restrict randomPrecision in GeoTileGridTests --- .../search/aggregations/bucket/geogrid/GeoTileGridTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java index 900e9f312d83a..3240ba299af0f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java @@ -53,6 +53,7 @@ protected long longEncode(double lng, double lat, int precision) { @Override protected int randomPrecision() { - return randomIntBetween(0, MAX_ZOOM); + // precision values below 8 can lead to parsing errors + return randomIntBetween(8, MAX_ZOOM); } } From 475e9458ca2e605e021656c8556755835b6b745b Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 29 Jan 2019 17:43:40 -0800 Subject: [PATCH 19/28] remove type usage from rest test --- .../test/search.aggregation/290_geotile_grid.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/290_geotile_grid.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/290_geotile_grid.yml index efed932bccbeb..2db498a0cacf0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/290_geotile_grid.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/290_geotile_grid.yml @@ -4,15 +4,15 @@ setup: reason: "added in 7.0.0" - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - _doc: - properties: - location: - type: geo_point + properties: + location: + type: geo_point --- "Basic test": @@ -22,32 +22,26 @@ setup: body: - index: _index: test_1 - _type: _doc _id: 1 - location: "52.374081,4.912350" - index: _index: test_1 - _type: _doc _id: 2 - location: "52.369219,4.901618" - index: _index: test_1 - _type: _doc _id: 3 - location: "52.371667,4.914722" - index: _index: test_1 - _type: _doc _id: 4 - location: "51.222900,4.405200" - index: _index: test_1 - _type: _doc _id: 5 - location: "48.861111,2.336389" - index: _index: test_1 - _type: _doc _id: 6 - location: "48.860000,2.327000" From ef0acd6b41e50276d765c7e89212bc90196cc351 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Tue, 29 Jan 2019 21:59:20 -0500 Subject: [PATCH 20/28] Remove support for non-integer precision --- ...iidoc => geotilegrid-aggregation.asciidoc} | 17 +++---- .../common/geo/GeoTileUtils.java | 32 +++++-------- .../geogrid/GeoTileGridParserTests.java | 45 ------------------- 3 files changed, 15 insertions(+), 79 deletions(-) rename docs/reference/aggregations/bucket/{quadkeygrid-aggregation.asciidoc => geotilegrid-aggregation.asciidoc} (87%) diff --git a/docs/reference/aggregations/bucket/quadkeygrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geotilegrid-aggregation.asciidoc similarity index 87% rename from docs/reference/aggregations/bucket/quadkeygrid-aggregation.asciidoc rename to docs/reference/aggregations/bucket/geotilegrid-aggregation.asciidoc index c565f75dfea90..130a2edd4b4d0 100644 --- a/docs/reference/aggregations/bucket/quadkeygrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geotilegrid-aggregation.asciidoc @@ -9,9 +9,10 @@ Each cell is labeled using a "{zoom}/{x}/{y}" format, where zoom is equal to the * High precision keys have a larger range for x and y, and represent tiles that cover only a small area. * Low precision keys have a smaller range for x and y, and represent tiles that each cover a large area. -geotile used in this aggregation can have a choice of precision between 0 and 29. +See https://wiki.openstreetmap.org/wiki/Zoom_levels[Zoom level documentation] on how precision (zoom) correlates +to size on the ground. Precision for this aggregation can be between 0 and 29, inclusive. -WARNING: The highest-precision geotile of length 29 produces cells that cover less than a square metre of land and +WARNING: The highest-precision geotile of length 29 produces cells that cover less than a 10cm by 10cm of land and so high-precision requests can be very costly in terms of RAM and result sizes. Please see the example below on how to first filter the aggregation to a smaller geographic area before requesting high-levels of detail. @@ -159,17 +160,9 @@ POST /museums/_search?size=0 [horizontal] field:: Mandatory. The name of the field indexed with GeoPoints. -precision:: Optional. The string zoom of the key used to define +precision:: Optional. The integer zoom of the key used to define cells/buckets in the results. Defaults to 7. - The precision can either be defined in terms of the integer - precision levels mentioned above. Values outside of [0,29] will - be rejected. - Alternatively, the precision level can be approximated from a - distance measure like "1km", "10m". The precision level is - calculate such that cells will not exceed the specified - size (diagonal) of the required precision. When this would lead - to precision levels higher than the supported 29 levels, - the value is rejected. + Values outside of [0,29] will be rejected. size:: Optional. The maximum number of geohash buckets to return (defaults to 10,000). When results are trimmed, buckets are diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/common/geo/GeoTileUtils.java index a764a64ea9a48..25494765e21a7 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoTileUtils.java @@ -20,6 +20,7 @@ import org.apache.lucene.util.BitUtil; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; @@ -77,36 +78,23 @@ private static int[] parseHash(final long hash) { } /** - * Parse a precision that can be expressed as an integer or a distance measure like "1km", "10m". + * Parse an integer precision (zoom level). The {@link ValueType#INT} allows it to be a number or a string. * - * The precision is expressed as a zoom level between 0 and MAX_ZOOM. + * The precision is expressed as a zoom level between 0 and {@link #MAX_ZOOM} (inclusive). * * @param parser {@link XContentParser} to parse the value from * @return int representing precision */ public static int parsePrecision(XContentParser parser) throws IOException, ElasticsearchParseException { - XContentParser.Token token = parser.currentToken(); - if (token.equals(XContentParser.Token.VALUE_NUMBER)) { - return XContentMapValues.nodeIntegerValue(parser.intValue()); - } else { - String precision = parser.text(); - try { - // we want to treat simple integer strings as precision levels, not distances - return XContentMapValues.nodeIntegerValue(precision); - } catch (NumberFormatException e) { - // try to parse as a distance value - final int parsedPrecision = GeoUtils.quadTreeLevelsForPrecision(precision); - try { - return checkPrecisionRange(parsedPrecision); - } catch (IllegalArgumentException e2) { - // this happens when distance too small, so precision > max. - // We'd like to see the original string - throw new IllegalArgumentException("precision too high [" + precision + "]", e2); - } - } - } + final Object node = parser.currentToken().equals(XContentParser.Token.VALUE_NUMBER) + ? Integer.valueOf(parser.intValue()) + : parser.text(); + return XContentMapValues.nodeIntegerValue(node); } + /** + * Assert the precision value is within the allowed range, and return it if ok, or throw. + */ public static int checkPrecisionRange(int precision) { if (precision < 0 || precision > MAX_ZOOM) { throw new IllegalArgumentException("Invalid geotile_grid precision of " + diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java index 39a2f559f7fff..2bd66073acda2 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java @@ -20,16 +20,13 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.geo.GeoTileUtils; -import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentParseException; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.ESTestCase; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.lessThanOrEqualTo; public class GeoTileGridParserTests extends ESTestCase { public void testParseValidFromInts() throws Exception { @@ -52,48 +49,6 @@ public void testParseValidFromStrings() throws Exception { assertNotNull(GeoTileGridAggregationBuilder.parse("geotile_grid", stParser)); } - public void testParseDistanceUnitPrecision() throws Exception { - double distance = randomDoubleBetween(10.0, 100.00, true); - DistanceUnit unit = randomFrom(DistanceUnit.values()); - if (unit.equals(DistanceUnit.MILLIMETERS)) { - distance = 5600 + randomDouble(); // 5.6cm is approx. smallest distance - } - String distanceString = distance + unit.toString(); - XContentParser stParser = createParser(JsonXContent.jsonXContent, - "{\"field\":\"my_loc\", \"precision\": \"" + distanceString + "\", \"size\": \"500\", \"shard_size\": \"550\"}"); - XContentParser.Token token = stParser.nextToken(); - assertSame(XContentParser.Token.START_OBJECT, token); - // can create a factory - GeoGridAggregationBuilder builder = GeoTileGridAggregationBuilder.parse("geotile_grid", stParser); - assertNotNull(builder); - assertThat(builder.precision(), greaterThanOrEqualTo(0)); - assertThat(builder.precision(), lessThanOrEqualTo(29)); - } - - public void testParseInvalidUnitPrecision() throws Exception { - XContentParser stParser = createParser(JsonXContent.jsonXContent, - "{\"field\":\"my_loc\", \"precision\": \"10kg\", \"size\": \"500\", \"shard_size\": \"550\"}"); - XContentParser.Token token = stParser.nextToken(); - assertSame(XContentParser.Token.START_OBJECT, token); - XContentParseException ex = expectThrows(XContentParseException.class, - () -> GeoTileGridAggregationBuilder.parse("geotile_grid", stParser)); - assertThat(ex.getMessage(), containsString("[geotile_grid] failed to parse field [precision]")); - assertThat(ex.getCause(), instanceOf(NumberFormatException.class)); - assertEquals("For input string: \"10kg\"", ex.getCause().getMessage()); - } - - public void testParseDistanceUnitPrecisionTooSmall() throws Exception { - XContentParser stParser = createParser(JsonXContent.jsonXContent, - "{\"field\":\"my_loc\", \"precision\": \"1cm\", \"size\": \"500\", \"shard_size\": \"550\"}"); - XContentParser.Token token = stParser.nextToken(); - assertSame(XContentParser.Token.START_OBJECT, token); - XContentParseException ex = expectThrows(XContentParseException.class, - () -> GeoTileGridAggregationBuilder.parse("geotile_grid", stParser)); - assertThat(ex.getMessage(), containsString("[geotile_grid] failed to parse field [precision]")); - assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class)); - assertEquals("precision too high [1cm]", ex.getCause().getMessage()); - } - public void testParseErrorOnBooleanPrecision() throws Exception { XContentParser stParser = createParser(JsonXContent.jsonXContent, "{\"field\":\"my_loc\", \"precision\":false}"); XContentParser.Token token = stParser.nextToken(); From 01c24231763c9d47efd559aefb42052d0080f991 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 30 Jan 2019 13:47:33 -0500 Subject: [PATCH 21/28] reformat asciidoc --- .../bucket/geotilegrid-aggregation.asciidoc | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/docs/reference/aggregations/bucket/geotilegrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geotilegrid-aggregation.asciidoc index 130a2edd4b4d0..ac173ec2b002f 100644 --- a/docs/reference/aggregations/bucket/geotilegrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geotilegrid-aggregation.asciidoc @@ -1,24 +1,31 @@ [[search-aggregations-bucket-geotilegrid-aggregation]] === GeoTile Grid Aggregation -A multi-bucket aggregation that works on `geo_point` fields and groups points into buckets that represent cells in a grid. -The resulting grid can be sparse and only contains cells that have matching data. -Each cell corresponds to a https://en.wikipedia.org/wiki/Tiled_web_map[map tile] as used by many online map sites. -Each cell is labeled using a "{zoom}/{x}/{y}" format, where zoom is equal to the user-specified precision. - -* High precision keys have a larger range for x and y, and represent tiles that cover only a small area. -* Low precision keys have a smaller range for x and y, and represent tiles that each cover a large area. - -See https://wiki.openstreetmap.org/wiki/Zoom_levels[Zoom level documentation] on how precision (zoom) correlates -to size on the ground. Precision for this aggregation can be between 0 and 29, inclusive. - -WARNING: The highest-precision geotile of length 29 produces cells that cover less than a 10cm by 10cm of land and -so high-precision requests can be very costly in terms of RAM and result sizes. -Please see the example below on how to first filter the aggregation to a smaller geographic area before requesting +A multi-bucket aggregation that works on `geo_point` fields and groups points into +buckets that represent cells in a grid. The resulting grid can be sparse and only +contains cells that have matching data. Each cell corresponds to a +https://en.wikipedia.org/wiki/Tiled_web_map[map tile] as used by many online map +sites. Each cell is labeled using a "{zoom}/{x}/{y}" format, where zoom is equal +to the user-specified precision. + +* High precision keys have a larger range for x and y, and represent tiles that +cover only a small area. +* Low precision keys have a smaller range for x and y, and represent tiles that +each cover a large area. + +See https://wiki.openstreetmap.org/wiki/Zoom_levels[Zoom level documentation] +on how precision (zoom) correlates to size on the ground. Precision for this +aggregation can be between 0 and 29, inclusive. + +WARNING: The highest-precision geotile of length 29 produces cells that cover +less than a 10cm by 10cm of land and so high-precision requests can be very +costly in terms of RAM and result sizes. Please see the example below on how +to first filter the aggregation to a smaller geographic area before requesting high-levels of detail. -The specified field must be of type `geo_point` (which can only be set explicitly in the mappings) and it can also hold -an array of `geo_point` fields, in which case all points will be taken into account during aggregation. +The specified field must be of type `geo_point` (which can only be set +explicitly in the mappings) and it can also hold an array of `geo_point` +fields, in which case all points will be taken into account during aggregation. ==== Simple low-precision request @@ -94,7 +101,10 @@ Response: ==== High-precision requests -When requesting detailed buckets (typically for displaying a "zoomed in" map) a filter like <> should be applied to narrow the subject area otherwise potentially millions of buckets will be created and returned. +When requesting detailed buckets (typically for displaying a "zoomed in" map) +a filter like <> should be +applied to narrow the subject area otherwise potentially millions of buckets +will be created and returned. [source,js] -------------------------------------------------- From be1132ffa532d838ef16bedf756a9238f3849d2c Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 30 Jan 2019 19:15:40 -0500 Subject: [PATCH 22/28] optimize geotile hashing algorithm --- .../GeoTileGridAggregationBuilder.java | 1 - .../geogrid/GeoTileGridAggregatorFactory.java | 1 - .../bucket/geogrid}/GeoTileUtils.java | 96 +++++------ .../geogrid/InternalGeoTileGridBucket.java | 1 - .../geogrid/ParsedGeoTileGridBucket.java | 1 - .../common/geo/GeoTileUtilsTests.java | 83 ---------- .../geogrid/GeoTileGridAggregatorTests.java | 8 +- .../geogrid/GeoTileGridParserTests.java | 1 - .../bucket/geogrid/GeoTileGridTests.java | 5 +- .../bucket/geogrid/GeoTileUtilsTests.java | 149 ++++++++++++++++++ 10 files changed, 194 insertions(+), 152 deletions(-) rename server/src/main/java/org/elasticsearch/{common/geo => search/aggregations/bucket/geogrid}/GeoTileUtils.java (67%) delete mode 100644 server/src/test/java/org/elasticsearch/common/geo/GeoTileUtilsTests.java create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java index 70acd8fa742e9..df75063e6838e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; -import org.elasticsearch.common.geo.GeoTileUtils; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java index b58fb196186e2..87077a89d6c23 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; -import org.elasticsearch.common.geo.GeoTileUtils; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java similarity index 67% rename from server/src/main/java/org/elasticsearch/common/geo/GeoTileUtils.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index 25494765e21a7..31ee1f2e892e7 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -16,16 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.common.geo; +package org.elasticsearch.search.aggregations.bucket.geogrid; -import org.apache.lucene.util.BitUtil; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; import java.io.IOException; -import java.util.Locale; import static org.elasticsearch.common.geo.GeoUtils.normalizeLat; import static org.elasticsearch.common.geo.GeoUtils.normalizeLon; @@ -35,47 +34,25 @@ * The string key is formatted as "zoom/x/y" * The hash value (long) contains all three of those values. */ -public class GeoTileUtils { +class GeoTileUtils { /** * Largest number of tiles (precision) to use. - * This value cannot be more than (64-5)/2 = 29, because 5 bits are used for zoom level itself + * This value cannot be more than (64-5)/2 = 29, because 5 bits are used for zoom level itself (0-31) * If zoom is not stored inside hash, it would be possible to use up to 32. * Another consideration is that index optimizes lat/lng storage, loosing some precision. * E.g. hash lng=140.74779717298918D lat=45.61884022447444D == "18/233561/93659", but shown as "18/233561/93658" */ - public static final int MAX_ZOOM = 29; + static final int MAX_ZOOM = 29; /** * Bit position of the zoom value within hash. Must be >= 2*MAX_ZOOM * Keeping it at a constant place allows MAX_ZOOM to be increased * without breaking serialization binary compatibility - * (still, the newer version should not use higher MAX_ZOOM in the mixed cases) */ - private static final int ZOOM_SHIFT = 29 * 2; + private static final int ZOOM_SHIFT = MAX_ZOOM * 2; - /** - * Mask of all the bits used by the geotile in a hash - */ - private static final long GEOTILE_MASK = (1L << ZOOM_SHIFT) - 1; - - /** - * Parse geotile hash as zoom, x, y integers. - */ - private static int[] parseHash(final long hash) { - final int zoom = checkPrecisionRange((int) (hash >>> ZOOM_SHIFT)); - final int tiles = 1 << zoom; - - // decode the geotile bits as interleaved xTile and yTile - long val = hash & GEOTILE_MASK; - int xTile = (int) BitUtil.deinterleave(val); - int yTile = (int) BitUtil.deinterleave(val >>> 1); - if (xTile < 0 || yTile < 0 || xTile >= tiles || yTile >= tiles) { - throw new IllegalArgumentException("hash-tile"); - } - - return new int[]{zoom, xTile, yTile}; - } + private static final long VALUE_MASK = (1L << MAX_ZOOM) - 1; /** * Parse an integer precision (zoom level). The {@link ValueType#INT} allows it to be a number or a string. @@ -85,7 +62,7 @@ private static int[] parseHash(final long hash) { * @param parser {@link XContentParser} to parse the value from * @return int representing precision */ - public static int parsePrecision(XContentParser parser) throws IOException, ElasticsearchParseException { + static int parsePrecision(XContentParser parser) throws IOException, ElasticsearchParseException { final Object node = parser.currentToken().equals(XContentParser.Token.VALUE_NUMBER) ? Integer.valueOf(parser.intValue()) : parser.text(); @@ -95,7 +72,7 @@ public static int parsePrecision(XContentParser parser) throws IOException, Elas /** * Assert the precision value is within the allowed range, and return it if ok, or throw. */ - public static int checkPrecisionRange(int precision) { + static int checkPrecisionRange(int precision) { if (precision < 0 || precision > MAX_ZOOM) { throw new IllegalArgumentException("Invalid geotile_grid precision of " + precision + ". Must be between 0 and " + MAX_ZOOM + "."); @@ -108,19 +85,16 @@ public static int checkPrecisionRange(int precision) { * The resulting hash contains interleaved tile X and Y coordinates. * The precision itself is also encoded as a few high bits. */ - public static long longEncode(double longitude, double latitude, int precision) { + static long longEncode(double longitude, double latitude, int precision) { // Mathematics for this code was adapted from https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Java // How many tiles in X and in Y final int tiles = 1 << checkPrecisionRange(precision); final double lon = normalizeLon(longitude); - final double lat = normalizeLat(latitude); + final double lat_rad = Math.toRadians(normalizeLat(latitude)); int xTile = (int) Math.floor((lon + 180) / 360 * tiles); - int yTile = (int) Math.floor( - (1 - Math.log( - Math.tan(Math.toRadians(lat)) + 1 / Math.cos(Math.toRadians(lat)) - ) / Math.PI) / 2 * tiles); + int yTile = (int) Math.floor((1 - Math.log(Math.tan(lat_rad) + 1 / Math.cos(lat_rad)) / Math.PI) / 2 * tiles); if (xTile < 0) { xTile = 0; } @@ -135,25 +109,36 @@ public static long longEncode(double longitude, double latitude, int precision) } // Zoom value is placed in front of all the bits used for the geotile - // e.g. if max zoom is 26, the largest index would use 52 bits (51st..0th), - // leaving 12 bits unused for zoom. See MAX_ZOOM comment above. - return BitUtil.interleave(xTile, yTile) | ((long) precision << ZOOM_SHIFT); + // e.g. when max zoom is 29, the largest index would use 58 bits (57th..0th), + // leaving 5 bits unused for zoom. See MAX_ZOOM comment above. + return ((long) precision << ZOOM_SHIFT) | ((long) xTile << MAX_ZOOM) | ((long) yTile); + } + + /** + * Parse geotile hash as zoom, x, y integers. + */ + private static int[] parseHash(long hash) { + final int zoom = (int) (hash >>> ZOOM_SHIFT); + int xTile = (int) ((hash >>> MAX_ZOOM) & VALUE_MASK); + int yTile = (int) (hash & VALUE_MASK); + return new int[]{zoom, xTile, yTile}; } /** * Encode to a geotile string from the geotile based long format */ - public static String stringEncode(long hash) { + static String stringEncode(long hash) { int[] res = parseHash(hash); + validateZXY(res[0], res[1], res[2]); return "" + res[0] + "/" + res[1] + "/" + res[2]; } - public static GeoPoint hashToGeoPoint(long hash) { + static GeoPoint hashToGeoPoint(long hash) { int[] res = parseHash(hash); return zxyToGeoPoint(res[0], res[1], res[2]); } - public static GeoPoint hashToGeoPoint(String hashAsString) { + static GeoPoint hashToGeoPoint(String hashAsString) { Throwable cause = null; try { final String[] parts = hashAsString.split("/", 4); @@ -168,16 +153,19 @@ public static GeoPoint hashToGeoPoint(String hashAsString) { hashAsString + ". Must be three integers in a form \"zoom/x/y\".", cause); } - private static GeoPoint zxyToGeoPoint(int zoom, int xTile, int yTile) { - final int maxTiles = 1 << checkPrecisionRange(zoom); - if (xTile >= 0 && xTile < maxTiles && yTile >= 0 && yTile < maxTiles) { - final double tiles = Math.pow(2.0, zoom); - final double n = Math.PI - (2.0 * Math.PI * (yTile + 0.5)) / tiles; - final double lat = Math.toDegrees(Math.atan(Math.sinh(n))); - final double lon = ((xTile + 0.5) / tiles * 360.0) - 180; - return new GeoPoint(lat, lon); + private static int validateZXY(int zoom, int xTile, int yTile) { + final int tiles = 1 << checkPrecisionRange(zoom); + if (xTile < 0 || yTile < 0 || xTile >= tiles || yTile >= tiles) { + throw new IllegalArgumentException("hash-tile"); } - throw new IllegalArgumentException( - String.format(Locale.ROOT, "Invalid geotile_grid z/x/y values of %s/%s/%s", zoom, xTile, yTile)); + return tiles; + } + + private static GeoPoint zxyToGeoPoint(int zoom, int xTile, int yTile) { + final int tiles = validateZXY(zoom, xTile, yTile); + final double n = Math.PI - (2.0 * Math.PI * (yTile + 0.5)) / tiles; + final double lat = Math.toDegrees(Math.atan(Math.sinh(n))); + final double lon = ((xTile + 0.5) / tiles * 360.0) - 180; + return new GeoPoint(lat, lon); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java index 9cecf52fb0639..fb9afbaaca4f8 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; -import org.elasticsearch.common.geo.GeoTileUtils; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.search.aggregations.InternalAggregations; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java index 4b3080c18562b..1324ad49f0165 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java @@ -20,7 +20,6 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.geo.GeoTileUtils; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoTileUtilsTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoTileUtilsTests.java deleted file mode 100644 index 3db84e8403595..0000000000000 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoTileUtilsTests.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.geo; - -import org.elasticsearch.test.ESTestCase; - -import static org.elasticsearch.common.geo.GeoTileUtils.checkPrecisionRange; -import static org.elasticsearch.common.geo.GeoTileUtils.hashToGeoPoint; -import static org.elasticsearch.common.geo.GeoTileUtils.longEncode; -import static org.elasticsearch.common.geo.GeoTileUtils.stringEncode; -import static org.hamcrest.Matchers.containsString; - -public class GeoTileUtilsTests extends ESTestCase { - - /** - * Precision validation should throw an error if its outside of the valid range. - */ - public void testCheckPrecisionRange() { - for (int i = 0; i <= 29; i++) { - assertEquals(i, checkPrecisionRange(i)); - } - IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> checkPrecisionRange(-1)); - assertThat(ex.getMessage(), containsString("Invalid geotile_grid precision of -1. Must be between 0 and 29.")); - ex = expectThrows(IllegalArgumentException.class, () -> checkPrecisionRange(30)); - assertThat(ex.getMessage(), containsString("Invalid geotile_grid precision of 30. Must be between 0 and 29.")); - } - - /** - * A few hardcoded lat/lng/zoom hashing expectations - */ - public void testLongEncode() { - assertEquals(0, longEncode(0, 0, 0)); - assertEquals(0x3c00000012e4cc66L, longEncode(30, 70, 15)); - assertEquals(0x7555555555440450L, longEncode(179.999, 89.999, 29)); - assertEquals(0x76aaaaaaaabbfbafL, longEncode(-179.999, -89.999, 29)); - assertEquals(0x0800000000000006L, longEncode(1, 1, 2)); - assertEquals(0x0c00000000000005L, longEncode(-20, 100, 3)); - assertEquals(0x70c0c9a5dc692fdcL, longEncode(13, -15, 28)); - assertEquals(0x4c00000fcdc7cde5L, longEncode(-12, 15, 19)); - } - - /** - * Ensure that for all points at all supported precision levels that the long encoding of a geotile - * is compatible with its String based counterpart - */ - public void testGeoTileAsLongRoutines() { - for (double lat=-90; lat<=90; lat++) { - for (double lng=-180; lng<=180; lng++) { - for(int p=0; p<=29; p++) { - long hash = longEncode(lng, lat, p); - if (p > 0) { - assertNotEquals(0, hash); - } - - // GeoPoint would be in the center of the bucket, thus must produce the same hash - GeoPoint point = hashToGeoPoint(hash); - long hashAsLong2 = longEncode(point.lon(), point.lat(), p); - assertEquals(hash, hashAsLong2); - - // Same point should be generated from the string key - assertEquals(point, hashToGeoPoint(stringEncode(hash))); - } - } - } - } -} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java index e72ad337424ec..4ce6d1582d84c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java @@ -19,20 +19,16 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; -import static org.elasticsearch.common.geo.GeoTileUtils.MAX_ZOOM; -import static org.elasticsearch.common.geo.GeoTileUtils.longEncode; -import static org.elasticsearch.common.geo.GeoTileUtils.stringEncode; - public class GeoTileGridAggregatorTests extends GeoGridAggregatorTestCase { @Override protected int randomPrecision() { - return randomIntBetween(0, MAX_ZOOM); + return randomIntBetween(0, GeoTileUtils.MAX_ZOOM); } @Override protected String hashAsString(double lng, double lat, int precision) { - return stringEncode(longEncode(lng, lat, precision)); + return GeoTileUtils.stringEncode(GeoTileUtils.longEncode(lng, lat, precision)); } @Override diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java index 2bd66073acda2..d3a9992af5305 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.common.geo.GeoTileUtils; import org.elasticsearch.common.xcontent.XContentParseException; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java index 3240ba299af0f..0a8aa8df56eec 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.search.aggregations.bucket.geogrid; -import org.elasticsearch.common.geo.GeoTileUtils; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; @@ -26,8 +25,6 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.common.geo.GeoTileUtils.MAX_ZOOM; - public class GeoTileGridTests extends GeoGridTestCase { @Override @@ -54,6 +51,6 @@ protected long longEncode(double lng, double lat, int precision) { @Override protected int randomPrecision() { // precision values below 8 can lead to parsing errors - return randomIntBetween(8, MAX_ZOOM); + return randomIntBetween(8, GeoTileUtils.MAX_ZOOM); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java new file mode 100644 index 0000000000000..b8ad8868ead26 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java @@ -0,0 +1,149 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.test.ESTestCase; + +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.MAX_ZOOM; +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.checkPrecisionRange; +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.hashToGeoPoint; +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode; +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.stringEncode; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.containsString; + +public class GeoTileUtilsTests extends ESTestCase { + + private static final double GEOTILE_TOLERANCE = 1E-5D; + + /** + * Precision validation should throw an error if its outside of the valid range. + */ + public void testCheckPrecisionRange() { + for (int i = 0; i <= 29; i++) { + assertEquals(i, checkPrecisionRange(i)); + } + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> checkPrecisionRange(-1)); + assertThat(ex.getMessage(), containsString("Invalid geotile_grid precision of -1. Must be between 0 and 29.")); + ex = expectThrows(IllegalArgumentException.class, () -> checkPrecisionRange(30)); + assertThat(ex.getMessage(), containsString("Invalid geotile_grid precision of 30. Must be between 0 and 29.")); + } + + /** + * A few hardcoded lat/lng/zoom hashing expectations + */ + public void testLongEncode() { + assertEquals(0, longEncode(0, 0, 0)); + assertEquals(0x3C00095540001CA5L, longEncode(30, 70, 15)); + assertEquals(0x77FFFF4580000000L, longEncode(179.999, 89.999, 29)); + assertEquals(0x740000BA7FFFFFFFL, longEncode(-179.999, -89.999, 29)); + assertEquals(0x0800000040000001L, longEncode(1, 1, 2)); + assertEquals(0x0C00000060000000L, longEncode(-20, 100, 3)); + assertEquals(0x71127D27C8ACA67AL, longEncode(13, -15, 28)); + assertEquals(0x4C0077776003A9ACL, longEncode(-12, 15, 19)); + + expectThrows(IllegalArgumentException.class, () -> longEncode(0, 0, -1)); + expectThrows(IllegalArgumentException.class, () -> longEncode(-1, 0, MAX_ZOOM + 1)); + } + + private void assertGeoPointEquals(GeoPoint gp, final double longitude, final double latitude) { + assertThat(gp.lon(), closeTo(longitude, GEOTILE_TOLERANCE)); + assertThat(gp.lat(), closeTo(latitude, GEOTILE_TOLERANCE)); + } + + public void testHashToGeoPoint() { + assertGeoPointEquals(hashToGeoPoint("0/0/0"), 0.0, 0.0); + assertGeoPointEquals(hashToGeoPoint("1/0/0"), -90.0, 66.51326044311186); + assertGeoPointEquals(hashToGeoPoint("1/1/0"), 90.0, 66.51326044311186); + assertGeoPointEquals(hashToGeoPoint("1/0/1"), -90.0, -66.51326044311186); + assertGeoPointEquals(hashToGeoPoint("1/1/1"), 90.0, -66.51326044311186); + assertGeoPointEquals(hashToGeoPoint("29/536870000/10"), 179.99938879162073, 85.05112817241982); + assertGeoPointEquals(hashToGeoPoint("29/10/536870000"), -179.99999295920134, -85.0510760525731); + + expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("0/-1/-1")); + expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("0/-1/1")); + expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("0/1/-1")); + expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("-1/0/0")); + expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("" + (MAX_ZOOM + 1) + "/0/0")); + + for (int z = 0; z <= MAX_ZOOM; z++) { + final int zoom = z; + expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("" + zoom + "/0")); + expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("" + zoom + "/0/0/0")); + final int max_index = (int) Math.pow(2, zoom); + expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("" + zoom + "/0/" + max_index)); + expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("" + zoom + "/" + max_index + "/0")); + } + } + + /** + * Make sure that hash produces the expected key, and that the key could be converted to hash via a GeoPoint + */ + private void assertStrCodec(long hash, String key, int zoom) { + assertEquals(key, stringEncode(hash)); + final GeoPoint gp = hashToGeoPoint(key); + assertEquals(hash, longEncode(gp.lon(), gp.lat(), zoom)); + } + + /** + * A few hardcoded lat/lng/zoom hashing expectations + */ + public void testStringEncode() { + assertStrCodec(0x0000000000000000L, "0/0/0", 0); + assertStrCodec(0x3C00095540001CA5L, "15/19114/7333", 15); + assertStrCodec(0x77FFFF4580000000L, "29/536869420/0", 29); + assertStrCodec(0x740000BA7FFFFFFFL, "29/1491/536870911", 29); + assertStrCodec(0x0800000040000001L, "2/2/1", 2); + assertStrCodec(0x0C00000060000000L, "3/3/0", 3); + assertStrCodec(0x71127D27C8ACA67AL, "28/143911230/145532538", 28); + assertStrCodec(0x4C0077776003A9ACL, "19/244667/240044", 19); + + expectThrows(IllegalArgumentException.class, () -> stringEncode(-1L)); + expectThrows(IllegalArgumentException.class, () -> stringEncode(0x7800000000000000L)); // z=30 + expectThrows(IllegalArgumentException.class, () -> stringEncode(0x0000000000000001L)); // z=0,x=0,y=1 + expectThrows(IllegalArgumentException.class, () -> stringEncode(0x0000000020000000L)); // z=0,x=1,y=0 + } + + /** + * Ensure that for all points at all supported precision levels that the long encoding of a geotile + * is compatible with its String based counterpart + */ + public void testGeoTileAsLongRoutines() { + for (double lat = -90; lat <= 90; lat++) { + for (double lng = -180; lng <= 180; lng++) { + for (int p = 0; p <= 29; p++) { + long hash = longEncode(lng, lat, p); + if (p > 0) { + assertNotEquals(0, hash); + } + + // GeoPoint would be in the center of the bucket, thus must produce the same hash + GeoPoint point = hashToGeoPoint(hash); + long hashAsLong2 = longEncode(point.lon(), point.lat(), p); + assertEquals(hash, hashAsLong2); + + // Same point should be generated from the string key + assertEquals(point, hashToGeoPoint(stringEncode(hash))); + } + } + } + } +} From b6cd525e420d8dbe582ac93cd369b591759cf0be Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 30 Jan 2019 23:00:20 -0500 Subject: [PATCH 23/28] stringEncode test extra by Tal --- .../bucket/geogrid/GeoTileUtilsTests.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java index b8ad8868ead26..5c4571963bbc4 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java @@ -82,15 +82,15 @@ public void testHashToGeoPoint() { expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("0/-1/1")); expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("0/1/-1")); expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("-1/0/0")); - expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("" + (MAX_ZOOM + 1) + "/0/0")); + expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint((MAX_ZOOM + 1) + "/0/0")); for (int z = 0; z <= MAX_ZOOM; z++) { final int zoom = z; - expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("" + zoom + "/0")); - expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("" + zoom + "/0/0/0")); + expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint(zoom + "/0")); + expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint(zoom + "/0/0/0")); final int max_index = (int) Math.pow(2, zoom); - expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("" + zoom + "/0/" + max_index)); - expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("" + zoom + "/" + max_index + "/0")); + expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint(zoom + "/0/" + max_index)); + expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint(zoom + "/" + max_index + "/0")); } } @@ -120,6 +120,18 @@ public void testStringEncode() { expectThrows(IllegalArgumentException.class, () -> stringEncode(0x7800000000000000L)); // z=30 expectThrows(IllegalArgumentException.class, () -> stringEncode(0x0000000000000001L)); // z=0,x=0,y=1 expectThrows(IllegalArgumentException.class, () -> stringEncode(0x0000000020000000L)); // z=0,x=1,y=0 + + for (int zoom = 0; zoom < 5; zoom++) { + int maxTile = 1 << zoom; + for (int x = 0; x < maxTile; x++) { + for (int y = 0; y < maxTile; y++) { + String expectedTileIndex = zoom + "/" + x + "/" + y; + GeoPoint point = hashToGeoPoint(expectedTileIndex); + String actualTileIndex = stringEncode(longEncode(point.lon(), point.lat(), zoom)); + assertEquals(expectedTileIndex, actualTileIndex); + } + } + } } /** From d040f415b86af4afb4f3461573fc2c57935634c6 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 31 Jan 2019 00:30:29 -0500 Subject: [PATCH 24/28] Docs, optimize lat/lng encoding --- .../bucket/geogrid/GeoTileUtils.java | 46 +++++++++++++------ .../geogrid/ParsedGeoTileGridBucket.java | 2 +- .../bucket/geogrid/GeoTileUtilsTests.java | 46 +++++++++++-------- 3 files changed, 59 insertions(+), 35 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index 31ee1f2e892e7..c22405f749cd8 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -30,9 +30,12 @@ import static org.elasticsearch.common.geo.GeoUtils.normalizeLon; /** - * Implements quad key hashing, same as used by map tiles. + * Implements geotile key hashing, same as used by many map tile implementations. * The string key is formatted as "zoom/x/y" - * The hash value (long) contains all three of those values. + * The hash value (long) contains all three of those values compacted into a single 64bit value: + * bits 58..63 -- zoom (0..29) + * bits 29..57 -- X tile index (0..2^zoom) + * bits 0..28 -- Y tile index (0..2^zoom) */ class GeoTileUtils { @@ -40,19 +43,21 @@ class GeoTileUtils { * Largest number of tiles (precision) to use. * This value cannot be more than (64-5)/2 = 29, because 5 bits are used for zoom level itself (0-31) * If zoom is not stored inside hash, it would be possible to use up to 32. + * Note that changing this value will make serialization binary-incompatible between versions. * Another consideration is that index optimizes lat/lng storage, loosing some precision. * E.g. hash lng=140.74779717298918D lat=45.61884022447444D == "18/233561/93659", but shown as "18/233561/93658" */ static final int MAX_ZOOM = 29; /** - * Bit position of the zoom value within hash. Must be >= 2*MAX_ZOOM - * Keeping it at a constant place allows MAX_ZOOM to be increased - * without breaking serialization binary compatibility + * Bit position of the zoom value within hash - zoom is stored in the most significant 6 bits of a long number. */ private static final int ZOOM_SHIFT = MAX_ZOOM * 2; - private static final long VALUE_MASK = (1L << MAX_ZOOM) - 1; + /** + * Bit mask to extract just the lowest 29 bits of a long + */ + private static final long X_Y_VALUE_MASK = (1L << MAX_ZOOM) - 1; /** * Parse an integer precision (zoom level). The {@link ValueType#INT} allows it to be a number or a string. @@ -88,13 +93,14 @@ static int checkPrecisionRange(int precision) { static long longEncode(double longitude, double latitude, int precision) { // Mathematics for this code was adapted from https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Java - // How many tiles in X and in Y + // Number of tiles for the current zoom level along the X and Y axis final int tiles = 1 << checkPrecisionRange(precision); - final double lon = normalizeLon(longitude); - final double lat_rad = Math.toRadians(normalizeLat(latitude)); - int xTile = (int) Math.floor((lon + 180) / 360 * tiles); - int yTile = (int) Math.floor((1 - Math.log(Math.tan(lat_rad) + 1 / Math.cos(lat_rad)) / Math.PI) / 2 * tiles); + int xTile = (int) Math.floor((normalizeLon(longitude) + 180) / 360 * tiles); + + double latSin = Math.sin(Math.toRadians(normalizeLat(latitude))); + int yTile = (int) Math.floor((0.5 - (Math.log((1 + latSin) / (1 - latSin)) / (4 * Math.PI))) * tiles); + if (xTile < 0) { xTile = 0; } @@ -119,8 +125,8 @@ static long longEncode(double longitude, double latitude, int precision) { */ private static int[] parseHash(long hash) { final int zoom = (int) (hash >>> ZOOM_SHIFT); - int xTile = (int) ((hash >>> MAX_ZOOM) & VALUE_MASK); - int yTile = (int) (hash & VALUE_MASK); + int xTile = (int) ((hash >>> MAX_ZOOM) & X_Y_VALUE_MASK); + int yTile = (int) (hash & X_Y_VALUE_MASK); return new int[]{zoom, xTile, yTile}; } @@ -133,12 +139,18 @@ static String stringEncode(long hash) { return "" + res[0] + "/" + res[1] + "/" + res[2]; } + /** + * Decode long hash as a GeoPoint (center of the tile) + */ static GeoPoint hashToGeoPoint(long hash) { int[] res = parseHash(hash); return zxyToGeoPoint(res[0], res[1], res[2]); } - static GeoPoint hashToGeoPoint(String hashAsString) { + /** + * Decode a string bucket key in "zoom/x/y" format to a GeoPoint (center of the tile) + */ + static GeoPoint keyToGeoPoint(String hashAsString) { Throwable cause = null; try { final String[] parts = hashAsString.split("/", 4); @@ -153,6 +165,9 @@ static GeoPoint hashToGeoPoint(String hashAsString) { hashAsString + ". Must be three integers in a form \"zoom/x/y\".", cause); } + /** + * Validates Zoom, X, and Y values, and returns the total number of allowed tiles along the x/y axis. + */ private static int validateZXY(int zoom, int xTile, int yTile) { final int tiles = 1 << checkPrecisionRange(zoom); if (xTile < 0 || yTile < 0 || xTile >= tiles || yTile >= tiles) { @@ -161,6 +176,9 @@ private static int validateZXY(int zoom, int xTile, int yTile) { return tiles; } + /** + * Converts zoom/x/y integers into a GeoPoint. + */ private static GeoPoint zxyToGeoPoint(int zoom, int xTile, int yTile) { final int tiles = validateZXY(zoom, xTile, yTile); final double n = Math.PI - (2.0 * Math.PI * (yTile + 0.5)) / tiles; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java index 1324ad49f0165..5e82d1ffc71c4 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java @@ -28,7 +28,7 @@ public class ParsedGeoTileGridBucket extends ParsedGeoGridBucket { @Override public GeoPoint getKey() { - return GeoTileUtils.hashToGeoPoint(hashAsString); + return GeoTileUtils.keyToGeoPoint(hashAsString); } @Override diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java index 5c4571963bbc4..10b84e68eed16 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java @@ -25,6 +25,7 @@ import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.MAX_ZOOM; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.checkPrecisionRange; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.hashToGeoPoint; +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.keyToGeoPoint; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.stringEncode; import static org.hamcrest.Matchers.closeTo; @@ -59,6 +60,11 @@ public void testLongEncode() { assertEquals(0x0C00000060000000L, longEncode(-20, 100, 3)); assertEquals(0x71127D27C8ACA67AL, longEncode(13, -15, 28)); assertEquals(0x4C0077776003A9ACL, longEncode(-12, 15, 19)); + assertEquals(0x140000024000000EL, longEncode(-328.231870,16.064082, 5)); + assertEquals(0x6436F96B60000000L, longEncode(-590.769588,89.549167, 25)); + assertEquals(0x6411BD6BA0A98359L, longEncode(999.787079,51.830093, 25)); + assertEquals(0x751BD6BBCA983596L, longEncode(999.787079,51.830093, 29)); + assertEquals(0x77CF880A20000000L, longEncode(-557.039740,-632.103969, 29)); expectThrows(IllegalArgumentException.class, () -> longEncode(0, 0, -1)); expectThrows(IllegalArgumentException.class, () -> longEncode(-1, 0, MAX_ZOOM + 1)); @@ -70,27 +76,27 @@ private void assertGeoPointEquals(GeoPoint gp, final double longitude, final dou } public void testHashToGeoPoint() { - assertGeoPointEquals(hashToGeoPoint("0/0/0"), 0.0, 0.0); - assertGeoPointEquals(hashToGeoPoint("1/0/0"), -90.0, 66.51326044311186); - assertGeoPointEquals(hashToGeoPoint("1/1/0"), 90.0, 66.51326044311186); - assertGeoPointEquals(hashToGeoPoint("1/0/1"), -90.0, -66.51326044311186); - assertGeoPointEquals(hashToGeoPoint("1/1/1"), 90.0, -66.51326044311186); - assertGeoPointEquals(hashToGeoPoint("29/536870000/10"), 179.99938879162073, 85.05112817241982); - assertGeoPointEquals(hashToGeoPoint("29/10/536870000"), -179.99999295920134, -85.0510760525731); - - expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("0/-1/-1")); - expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("0/-1/1")); - expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("0/1/-1")); - expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint("-1/0/0")); - expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint((MAX_ZOOM + 1) + "/0/0")); + assertGeoPointEquals(keyToGeoPoint("0/0/0"), 0.0, 0.0); + assertGeoPointEquals(keyToGeoPoint("1/0/0"), -90.0, 66.51326044311186); + assertGeoPointEquals(keyToGeoPoint("1/1/0"), 90.0, 66.51326044311186); + assertGeoPointEquals(keyToGeoPoint("1/0/1"), -90.0, -66.51326044311186); + assertGeoPointEquals(keyToGeoPoint("1/1/1"), 90.0, -66.51326044311186); + assertGeoPointEquals(keyToGeoPoint("29/536870000/10"), 179.99938879162073, 85.05112817241982); + assertGeoPointEquals(keyToGeoPoint("29/10/536870000"), -179.99999295920134, -85.0510760525731); + + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("0/-1/-1")); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("0/-1/1")); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("0/1/-1")); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("-1/0/0")); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint((MAX_ZOOM + 1) + "/0/0")); for (int z = 0; z <= MAX_ZOOM; z++) { final int zoom = z; - expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint(zoom + "/0")); - expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint(zoom + "/0/0/0")); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint(zoom + "/0")); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint(zoom + "/0/0/0")); final int max_index = (int) Math.pow(2, zoom); - expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint(zoom + "/0/" + max_index)); - expectThrows(IllegalArgumentException.class, () -> hashToGeoPoint(zoom + "/" + max_index + "/0")); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint(zoom + "/0/" + max_index)); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint(zoom + "/" + max_index + "/0")); } } @@ -99,7 +105,7 @@ public void testHashToGeoPoint() { */ private void assertStrCodec(long hash, String key, int zoom) { assertEquals(key, stringEncode(hash)); - final GeoPoint gp = hashToGeoPoint(key); + final GeoPoint gp = keyToGeoPoint(key); assertEquals(hash, longEncode(gp.lon(), gp.lat(), zoom)); } @@ -126,7 +132,7 @@ public void testStringEncode() { for (int x = 0; x < maxTile; x++) { for (int y = 0; y < maxTile; y++) { String expectedTileIndex = zoom + "/" + x + "/" + y; - GeoPoint point = hashToGeoPoint(expectedTileIndex); + GeoPoint point = keyToGeoPoint(expectedTileIndex); String actualTileIndex = stringEncode(longEncode(point.lon(), point.lat(), zoom)); assertEquals(expectedTileIndex, actualTileIndex); } @@ -153,7 +159,7 @@ public void testGeoTileAsLongRoutines() { assertEquals(hash, hashAsLong2); // Same point should be generated from the string key - assertEquals(point, hashToGeoPoint(stringEncode(hash))); + assertEquals(point, keyToGeoPoint(stringEncode(hash))); } } } From 5369cbb7ba0df8e288275452356cd42d43d10fdf Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 31 Jan 2019 12:32:49 -0500 Subject: [PATCH 25/28] address review comments --- .../bucket/geogrid/GeoTileUtils.java | 8 ++++++-- .../bucket/geogrid/GeoTileUtilsTests.java | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index c22405f749cd8..ab9d10d8a8290 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import java.io.IOException; +import java.util.Locale; import static org.elasticsearch.common.geo.GeoUtils.normalizeLat; import static org.elasticsearch.common.geo.GeoUtils.normalizeLon; @@ -37,7 +38,9 @@ * bits 29..57 -- X tile index (0..2^zoom) * bits 0..28 -- Y tile index (0..2^zoom) */ -class GeoTileUtils { +final class GeoTileUtils { + + private GeoTileUtils() {} /** * Largest number of tiles (precision) to use. @@ -171,7 +174,8 @@ static GeoPoint keyToGeoPoint(String hashAsString) { private static int validateZXY(int zoom, int xTile, int yTile) { final int tiles = 1 << checkPrecisionRange(zoom); if (xTile < 0 || yTile < 0 || xTile >= tiles || yTile >= tiles) { - throw new IllegalArgumentException("hash-tile"); + throw new IllegalArgumentException(String.format( + Locale.ROOT, "Zoom/X/Y combination is not valid: %d/%d/%d", zoom, xTile, yTile)); } return tiles; } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java index 10b84e68eed16..8f21e177a0684 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java @@ -52,7 +52,7 @@ public void testCheckPrecisionRange() { * A few hardcoded lat/lng/zoom hashing expectations */ public void testLongEncode() { - assertEquals(0, longEncode(0, 0, 0)); + assertEquals(0x0000000000000000L, longEncode(0, 0, 0)); assertEquals(0x3C00095540001CA5L, longEncode(30, 70, 15)); assertEquals(0x77FFFF4580000000L, longEncode(179.999, 89.999, 29)); assertEquals(0x740000BA7FFFFFFFL, longEncode(-179.999, -89.999, 29)); @@ -65,6 +65,12 @@ public void testLongEncode() { assertEquals(0x6411BD6BA0A98359L, longEncode(999.787079,51.830093, 25)); assertEquals(0x751BD6BBCA983596L, longEncode(999.787079,51.830093, 29)); assertEquals(0x77CF880A20000000L, longEncode(-557.039740,-632.103969, 29)); + assertEquals(0x7624FA4FA0000000L, longEncode(13,88, 29)); + assertEquals(0x7624FA4FBFFFFFFFL, longEncode(13,-88, 29)); + assertEquals(0x0400000020000000L, longEncode(13,89, 1)); + assertEquals(0x0400000020000001L, longEncode(13,-89, 1)); + assertEquals(0x0400000020000000L, longEncode(13,95, 1)); + assertEquals(0x0400000020000001L, longEncode(13,-95, 1)); expectThrows(IllegalArgumentException.class, () -> longEncode(0, 0, -1)); expectThrows(IllegalArgumentException.class, () -> longEncode(-1, 0, MAX_ZOOM + 1)); @@ -121,6 +127,15 @@ public void testStringEncode() { assertStrCodec(0x0C00000060000000L, "3/3/0", 3); assertStrCodec(0x71127D27C8ACA67AL, "28/143911230/145532538", 28); assertStrCodec(0x4C0077776003A9ACL, "19/244667/240044", 19); + assertStrCodec(0x140000024000000EL, "5/18/14", 5); + assertStrCodec(0x6436F96B60000000L, "25/28822363/0", 25); + assertStrCodec(0x6411BD6BA0A98359L, "25/9300829/11109209", 25); + assertStrCodec(0x751BD6BBCA983596L, "29/148813278/177747350", 29); + assertStrCodec(0x77CF880A20000000L, "29/511459409/0", 29); + assertStrCodec(0x7624FA4FA0000000L, "29/287822461/0", 29); + assertStrCodec(0x7624FA4FBFFFFFFFL, "29/287822461/536870911", 29); + assertStrCodec(0x0400000020000000L, "1/1/0", 1); + assertStrCodec(0x0400000020000001L, "1/1/1", 1); expectThrows(IllegalArgumentException.class, () -> stringEncode(-1L)); expectThrows(IllegalArgumentException.class, () -> stringEncode(0x7800000000000000L)); // z=30 From 8c0125c0e66c834f8eb75998671171fa60da039c Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 31 Jan 2019 13:10:39 -0500 Subject: [PATCH 26/28] extra test for polar coords (thx Tal) --- .../bucket/geogrid/GeoTileUtils.java | 2 ++ .../bucket/geogrid/GeoTileUtilsTests.java | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index ab9d10d8a8290..011da2f0c6b9f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -104,6 +104,8 @@ static long longEncode(double longitude, double latitude, int precision) { double latSin = Math.sin(Math.toRadians(normalizeLat(latitude))); int yTile = (int) Math.floor((0.5 - (Math.log((1 + latSin) / (1 - latSin)) / (4 * Math.PI))) * tiles); + // Edge values may generate invalid values, and need to be clipped. + // For example, polar regions (above/below lat 85.05112878) get normalized. if (xTile < 0) { xTile = 0; } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java index 8f21e177a0684..5af6629e02a0c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java @@ -179,4 +179,23 @@ public void testGeoTileAsLongRoutines() { } } } + + /** + * Make sure the polar regions are handled properly. + * Mercator projection does not show anything above 85 or below -85, + * so ensure they are clipped correctly. + */ + public void testSingularityAtPoles() { + double minLat = -85.05112878; + double maxLat = 85.05112878; + double lon = randomIntBetween(-180, 180); + double lat = randomBoolean() + ? randomDoubleBetween(-90, minLat, true) + : randomDoubleBetween(maxLat, 90, true); + double clippedLat = Math.min(Math.max(lat, minLat), maxLat); + int zoom = randomIntBetween(0, MAX_ZOOM); + String tileIndex = stringEncode(longEncode(lon, lat, zoom)); + String clippedTileIndex = stringEncode(longEncode(lon, clippedLat, zoom)); + assertEquals(tileIndex, clippedTileIndex); + } } From 680a5070b7442f83b97bfaca0af4b478c2520b14 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 31 Jan 2019 15:08:28 -0500 Subject: [PATCH 27/28] address review comments --- .../GeoTileGridAggregationBuilder.java | 4 +-- .../bucket/geogrid/GeoTileUtils.java | 32 +++++++++---------- .../geogrid/ParsedGeoTileGridBucket.java | 2 +- .../bucket/geogrid/GeoTileUtilsTests.java | 16 +++++++--- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java index df75063e6838e..33efeeb5d38b6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java @@ -35,8 +35,8 @@ public class GeoTileGridAggregationBuilder extends GeoGridAggregationBuilder { public static final String NAME = "geotile_grid"; - public static final int DEFAULT_PRECISION = 7; - public static final int DEFAULT_MAX_NUM_CELLS = 10000; + private static final int DEFAULT_PRECISION = 7; + private static final int DEFAULT_MAX_NUM_CELLS = 10000; private static final ObjectParser PARSER = createParser(NAME, GeoTileUtils::parsePrecision); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index 011da2f0c6b9f..d85cf6b1a56ce 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -97,12 +97,12 @@ static long longEncode(double longitude, double latitude, int precision) { // Mathematics for this code was adapted from https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Java // Number of tiles for the current zoom level along the X and Y axis - final int tiles = 1 << checkPrecisionRange(precision); + final long tiles = 1 << checkPrecisionRange(precision); - int xTile = (int) Math.floor((normalizeLon(longitude) + 180) / 360 * tiles); + long xTile = (long) Math.floor((normalizeLon(longitude) + 180) / 360 * tiles); double latSin = Math.sin(Math.toRadians(normalizeLat(latitude))); - int yTile = (int) Math.floor((0.5 - (Math.log((1 + latSin) / (1 - latSin)) / (4 * Math.PI))) * tiles); + long yTile = (long) Math.floor((0.5 - (Math.log((1 + latSin) / (1 - latSin)) / (4 * Math.PI))) * tiles); // Edge values may generate invalid values, and need to be clipped. // For example, polar regions (above/below lat 85.05112878) get normalized. @@ -122,7 +122,7 @@ static long longEncode(double longitude, double latitude, int precision) { // Zoom value is placed in front of all the bits used for the geotile // e.g. when max zoom is 29, the largest index would use 58 bits (57th..0th), // leaving 5 bits unused for zoom. See MAX_ZOOM comment above. - return ((long) precision << ZOOM_SHIFT) | ((long) xTile << MAX_ZOOM) | ((long) yTile); + return ((long) precision << ZOOM_SHIFT) | (xTile << MAX_ZOOM) | yTile; } /** @@ -130,8 +130,8 @@ static long longEncode(double longitude, double latitude, int precision) { */ private static int[] parseHash(long hash) { final int zoom = (int) (hash >>> ZOOM_SHIFT); - int xTile = (int) ((hash >>> MAX_ZOOM) & X_Y_VALUE_MASK); - int yTile = (int) (hash & X_Y_VALUE_MASK); + final int xTile = (int) ((hash >>> MAX_ZOOM) & X_Y_VALUE_MASK); + final int yTile = (int) (hash & X_Y_VALUE_MASK); return new int[]{zoom, xTile, yTile}; } @@ -156,18 +156,18 @@ static GeoPoint hashToGeoPoint(long hash) { * Decode a string bucket key in "zoom/x/y" format to a GeoPoint (center of the tile) */ static GeoPoint keyToGeoPoint(String hashAsString) { - Throwable cause = null; + final String[] parts = hashAsString.split("/", 4); + if (parts.length != 3) { + throw new IllegalArgumentException("Invalid geotile_grid hash string of " + + hashAsString + ". Must be three integers in a form \"zoom/x/y\"."); + } + try { - final String[] parts = hashAsString.split("/", 4); - if (parts.length == 3) { - return zxyToGeoPoint(Integer.parseInt(parts[0]), Integer.parseInt(parts[1]), Integer.parseInt(parts[2])); - } - } catch (IllegalArgumentException e) { - // This will also handle NumberFormatException - cause = e; + return zxyToGeoPoint(Integer.parseInt(parts[0]), Integer.parseInt(parts[1]), Integer.parseInt(parts[2])); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid geotile_grid hash string of " + + hashAsString + ". Must be three integers in a form \"zoom/x/y\".", e); } - throw new IllegalArgumentException("Invalid geotile_grid hash string of " + - hashAsString + ". Must be three integers in a form \"zoom/x/y\".", cause); } /** diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java index 5e82d1ffc71c4..d2d18b40e76d1 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java @@ -37,6 +37,6 @@ public String getKeyAsString() { } static ParsedGeoTileGridBucket fromXContent(XContentParser parser) throws IOException { - return parseXContent(parser, false, ParsedGeoTileGridBucket::new, (p, bucket) -> bucket.hashAsString = p.textOrNull()); + return parseXContent(parser, false, ParsedGeoTileGridBucket::new, (p, bucket) -> bucket.hashAsString = p.text()); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java index 5af6629e02a0c..e2881fd9b9145 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java @@ -90,16 +90,24 @@ public void testHashToGeoPoint() { assertGeoPointEquals(keyToGeoPoint("29/536870000/10"), 179.99938879162073, 85.05112817241982); assertGeoPointEquals(keyToGeoPoint("29/10/536870000"), -179.99999295920134, -85.0510760525731); + //noinspection ConstantConditions + expectThrows(NullPointerException.class, () -> keyToGeoPoint(null)); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("")); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("a")); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("0")); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("0/0")); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("0/0/0/0")); expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("0/-1/-1")); - expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("0/-1/1")); - expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("0/1/-1")); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("0/-1/0")); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("0/0/-1")); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("a/0/0")); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("0/a/0")); + expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("0/0/a")); expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint("-1/0/0")); expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint((MAX_ZOOM + 1) + "/0/0")); for (int z = 0; z <= MAX_ZOOM; z++) { final int zoom = z; - expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint(zoom + "/0")); - expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint(zoom + "/0/0/0")); final int max_index = (int) Math.pow(2, zoom); expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint(zoom + "/0/" + max_index)); expectThrows(IllegalArgumentException.class, () -> keyToGeoPoint(zoom + "/" + max_index + "/0")); From 880db7277d05c80ed855ce4db35cc4532bef4701 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 31 Jan 2019 15:25:29 -0500 Subject: [PATCH 28/28] test builder precision --- .../bucket/geogrid/GeoTileGridAggregatorTests.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java index 4ce6d1582d84c..6544344543e34 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java @@ -35,4 +35,16 @@ protected String hashAsString(double lng, double lat, int precision) { protected GeoGridAggregationBuilder createBuilder(String name) { return new GeoTileGridAggregationBuilder(name); } + + public void testPrecision() { + final GeoGridAggregationBuilder builder = createBuilder("_name"); + + expectThrows(IllegalArgumentException.class, () -> builder.precision(-1)); + expectThrows(IllegalArgumentException.class, () -> builder.precision(30)); + + int precision = randomIntBetween(0, 29); + builder.precision(precision); + assertEquals(precision, builder.precision()); + } + }