From a089dc9dcd769191f8a69d7922960bc565dbbf29 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 29 May 2017 21:26:19 +0200 Subject: [PATCH] Added more unit test coverage for terms aggregation and removed terms agg integration tests that were replaced by unit tests. --- .../search/aggregations/BucketOrder.java | 6 + .../bucket/terms/DoubleTermsAggregator.java | 4 +- .../bucket/terms/StringTermsAggregator.java | 8 +- .../bucket/terms/TermsAggregatorFactory.java | 6 +- .../aggregations/bucket/DoubleTermsIT.java | 323 +------ .../search/aggregations/bucket/IpTermsIT.java | 30 +- .../aggregations/bucket/LongTermsIT.java | 337 +------- .../aggregations/bucket/StringTermsIT.java | 492 +---------- .../bucket/terms/TermsAggregatorTests.java | 810 +++++++++++++++++- .../aggregations/AggregatorTestCase.java | 29 + 10 files changed, 815 insertions(+), 1230 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/BucketOrder.java b/core/src/main/java/org/elasticsearch/search/aggregations/BucketOrder.java index 482a90f08526c..d1e57e12321db 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/BucketOrder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/BucketOrder.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.search.aggregations; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ToXContentObject; @@ -124,4 +125,9 @@ public static BucketOrder compound(BucketOrder... orders) { public void writeTo(StreamOutput out) throws IOException { InternalOrder.Streams.writeOrder(this, out); } + + @Override + public String toString() { + return Strings.toString(this); + } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsAggregator.java index 0ae42abd9a4d3..098f523e514d1 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsAggregator.java @@ -25,9 +25,9 @@ import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.internal.SearchContext; @@ -39,7 +39,7 @@ public class DoubleTermsAggregator extends LongTermsAggregator { - public DoubleTermsAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, DocValueFormat format, + DoubleTermsAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, DocValueFormat format, BucketOrder order, BucketCountThresholds bucketCountThresholds, SearchContext aggregationContext, Aggregator parent, SubAggCollectionMode collectionMode, boolean showTermDocCountError, IncludeExclude.LongFilter longFilter, List pipelineAggregators, Map metaData) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java index 6161f7912a8ad..064850c441385 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java @@ -27,14 +27,14 @@ import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalOrder; import org.elasticsearch.search.aggregations.LeafBucketCollector; import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; import org.elasticsearch.search.aggregations.bucket.terms.support.BucketPriorityQueue; import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import org.elasticsearch.search.aggregations.BucketOrder; -import org.elasticsearch.search.aggregations.InternalOrder; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.internal.SearchContext; @@ -153,7 +153,7 @@ public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOE final StringTerms.Bucket[] list = new StringTerms.Bucket[ordered.size()]; long survivingBucketOrds[] = new long[ordered.size()]; for (int i = ordered.size() - 1; i >= 0; --i) { - final StringTerms.Bucket bucket = (StringTerms.Bucket) ordered.pop(); + final StringTerms.Bucket bucket = ordered.pop(); survivingBucketOrds[i] = bucket.bucketOrd; list[i] = bucket; otherDocCount -= bucket.docCount; @@ -163,7 +163,7 @@ public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOE // Now build the aggs for (int i = 0; i < list.length; i++) { - final StringTerms.Bucket bucket = (StringTerms.Bucket)list[i]; + final StringTerms.Bucket bucket = list[i]; bucket.termBytes = BytesRef.deepCopyOf(bucket.termBytes); bucket.aggregations = bucketAggregations(bucket.bucketOrd); bucket.docCountError = 0; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java index ca2974a10595f..627e1808e4200 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java @@ -27,14 +27,14 @@ import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalOrder; import org.elasticsearch.search.aggregations.NonCollectingAggregator; import org.elasticsearch.search.aggregations.bucket.BucketUtils; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds; import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import org.elasticsearch.search.aggregations.BucketOrder; -import org.elasticsearch.search.aggregations.InternalOrder; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; @@ -53,7 +53,7 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory config, BucketOrder order, IncludeExclude includeExclude, diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java index 2363c21c7d112..e55b2c9e6486f 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java @@ -30,9 +30,8 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; -import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.bucket.filter.Filter; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; @@ -41,9 +40,7 @@ import org.elasticsearch.search.aggregations.metrics.stats.Stats; import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats; import org.elasticsearch.search.aggregations.metrics.sum.Sum; -import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.test.ESIntegTestCase; -import org.hamcrest.Matchers; import java.io.IOException; import java.util.ArrayList; @@ -61,12 +58,10 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats; import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; -import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.max; import static org.elasticsearch.search.aggregations.AggregationBuilders.stats; import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; @@ -75,7 +70,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; @ESIntegTestCase.SuiteScopeTestCase @@ -287,85 +281,6 @@ public void testSizeIsZero() { assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [terms]")); } - public void testSingleValueField() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values()))) - .execute().actionGet(); - - assertSearchResponse(response); - - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - for (int i = 0; i < 5; i++) { - Terms.Bucket bucket = terms.getBucketByKey("" + (double) i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + (double)i)); - assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - } - } - - public void testSingleValueFieldWithMaxSize() throws Exception { - SearchResponse response = client().prepareSearch("high_card_idx") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .size(20) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .order(BucketOrder.key(true))) // we need to sort by terms cause we're checking the first 20 values - .execute().actionGet(); - - assertSearchResponse(response); - - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(20)); - - for (int i = 0; i < 20; i++) { - Terms.Bucket bucket = terms.getBucketByKey("" + (double) i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + (double) i)); - assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - } - } - - public void testSingleValueFieldWithFiltering() throws Exception { - double includes[] = { 1, 2, 3, 98.2 }; - double excludes[] = { 2, 4, 99 }; - double empty[] = {}; - testIncludeExcludeResults(includes, empty, new double[] { 1, 2, 3 }); - testIncludeExcludeResults(includes, excludes, new double[] { 1, 3 }); - testIncludeExcludeResults(empty, excludes, new double[] { 0, 1, 3 }); - } - - private void testIncludeExcludeResults(double[] includes, double[] excludes, double[] expecteds) { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .includeExclude(new IncludeExclude(includes, excludes)) - .collectMode(randomFrom(SubAggCollectionMode.values()))) - .execute().actionGet(); - assertSearchResponse(response); - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(expecteds.length)); - - for (int i = 0; i < expecteds.length; i++) { - Terms.Bucket bucket = terms.getBucketByKey("" + expecteds[i]); - assertThat(bucket, notNullValue()); - assertThat(bucket.getDocCount(), equalTo(1L)); - } - } - public void testSingleValueFieldWithPartitionedFiltering() throws Exception { runTestFieldWithPartitionedFiltering(SINGLE_VALUED_FIELD_NAME); } @@ -403,120 +318,6 @@ private void runTestFieldWithPartitionedFiltering(String field) throws Exception assertEquals(expectedCardinality, foundTerms.size()); } - public void testSingleValueFieldOrderedByTermAsc() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .order(BucketOrder.key(true))) - .execute().actionGet(); - - assertSearchResponse(response); - - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - int i = 0; - for (Terms.Bucket bucket : terms.getBuckets()) { - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + (double)i)); - assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - i++; - } - } - - public void testSingleValueFieldOrderedByTermDesc() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .order(BucketOrder.key(false))) - .execute().actionGet(); - - assertSearchResponse(response); - - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - int i = 4; - for (Terms.Bucket bucket : terms.getBuckets()) { - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + (double) i)); - assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - i--; - } - } - - public void testSingleValueFieldOrderedByTieBreaker() throws Exception { - SearchResponse response = client().prepareSearch("idx").setTypes("type") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .order(BucketOrder.aggregation("max_constant", randomBoolean())) - .subAggregation(max("max_constant").field("constant"))) - .execute().actionGet(); - - assertSearchResponse(response); - - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - int i = 0; - for (Terms.Bucket bucket : terms.getBuckets()) { - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + (double)i)); - assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - i++; - } - } - - public void testSingleValuedFieldWithSubAggregation() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .subAggregation(sum("sum").field(MULTI_VALUED_FIELD_NAME))) - .execute().actionGet(); - - assertSearchResponse(response); - - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - assertThat(((InternalAggregation)terms).getProperty("_bucket_count"), equalTo(5)); - Object[] propertiesKeys = (Object[]) ((InternalAggregation)terms).getProperty("_key"); - Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)terms).getProperty("_count"); - Object[] propertiesCounts = (Object[]) ((InternalAggregation)terms).getProperty("sum.value"); - - for (int i = 0; i < 5; i++) { - Terms.Bucket bucket = terms.getBucketByKey("" + (double) i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + (double) i)); - assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - Sum sum = bucket.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat((long) sum.getValue(), equalTo(i+i+1L)); - assertThat((double) propertiesKeys[i], equalTo((double) i)); - assertThat((long) propertiesDocCounts[i], equalTo(1L)); - assertThat((double) propertiesCounts[i], equalTo((double) i + i + 1L)); - } - } - public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(terms("terms") @@ -542,34 +343,6 @@ public void testSingleValuedFieldWithValueScript() throws Exception { } } - public void testMultiValuedField() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(terms("terms") - .field(MULTI_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values()))) - .execute().actionGet(); - - assertSearchResponse(response); - - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(6)); - - for (int i = 0; i < 6; i++) { - Terms.Bucket bucket = terms.getBucketByKey("" + (double) i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + (double) i)); - assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i)); - if (i == 0 || i == 5) { - assertThat(bucket.getDocCount(), equalTo(1L)); - } else { - assertThat(bucket.getDocCount(), equalTo(2L)); - } - } - } - public void testMultiValuedFieldWithValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(terms("terms") @@ -696,23 +469,6 @@ public void testScriptMultiValued() throws Exception { } } - public void testUnmapped() throws Exception { - SearchResponse response = client().prepareSearch("idx_unmapped") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .size(randomIntBetween(1, 5)) - .collectMode(randomFrom(SubAggCollectionMode.values()))) - .execute().actionGet(); - - assertSearchResponse(response); - - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(0)); - } - public void testPartiallyUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx_unmapped", "idx") .addAggregation(terms("terms") @@ -763,53 +519,6 @@ public void testPartiallyUnmappedWithFormat() throws Exception { } } - public void testEmptyAggregation() throws Exception { - SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") - .setQuery(matchAllQuery()) - .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(1L).minDocCount(0) - .subAggregation(terms("terms").field(SINGLE_VALUED_FIELD_NAME))) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L)); - Histogram histo = searchResponse.getAggregations().get("histo"); - assertThat(histo, Matchers.notNullValue()); - Histogram.Bucket bucket = histo.getBuckets().get(1); - assertThat(bucket, Matchers.notNullValue()); - - Terms terms = bucket.getAggregations().get("terms"); - assertThat(terms, Matchers.notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().isEmpty(), is(true)); - } - - public void testSingleValuedFieldOrderedBySingleValueSubAggregationAsc() throws Exception { - boolean asc = true; - SearchResponse response = client() - .prepareSearch("idx") - .addAggregation( - terms("terms").field(SINGLE_VALUED_FIELD_NAME).collectMode(randomFrom(SubAggCollectionMode.values())) - .order(BucketOrder.aggregation("avg_i", asc)).subAggregation(avg("avg_i").field(SINGLE_VALUED_FIELD_NAME))) - .execute().actionGet(); - - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - for (int i = 0; i < 5; i++) { - Terms.Bucket bucket = terms.getBucketByKey("" + (double) i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + (double)i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - Avg avg = bucket.getAggregations().get("avg_i"); - assertThat(avg, notNullValue()); - assertThat(avg.getValue(), equalTo((double) i)); - } - } - public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscWithSubTermsAgg() throws Exception { boolean asc = true; SearchResponse response = client() @@ -1019,36 +728,6 @@ public void testSingleValuedFieldOrderedByMultiValuedSubAggregationWithoutMetric } } - public void testSingleValuedFieldOrderedBySingleValueSubAggregationDesc() throws Exception { - boolean asc = false; - SearchResponse response = client() - .prepareSearch("idx") - .addAggregation( - terms("terms").field(SINGLE_VALUED_FIELD_NAME).collectMode(randomFrom(SubAggCollectionMode.values())) - .order(BucketOrder.aggregation("avg_i", asc)).subAggregation(avg("avg_i").field(SINGLE_VALUED_FIELD_NAME))) - .execute().actionGet(); - - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - for (int i = 4; i >= 0; i--) { - - Terms.Bucket bucket = terms.getBucketByKey("" + (double) i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + (double)i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - - Avg avg = bucket.getAggregations().get("avg_i"); - assertThat(avg, notNullValue()); - assertThat(avg.getValue(), equalTo((double) i)); - } - } - public void testSingleValuedFieldOrderedByMultiValueSubAggregationAsc() throws Exception { boolean asc = true; SearchResponse response = client() diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java index 3a69812df1fe9..402d7d2648b7e 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java @@ -18,9 +18,6 @@ */ package org.elasticsearch.search.aggregations.bucket; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; - import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.plugins.Plugin; @@ -35,6 +32,9 @@ import java.util.Map; import java.util.function.Function; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; + public class IpTermsIT extends AbstractTermsTestCase { @Override @@ -62,30 +62,6 @@ protected Map, Object>> pluginScripts() { } } - public void testBasics() throws Exception { - assertAcked(prepareCreate("index").addMapping("type", "ip", "type=ip")); - indexRandom(true, - client().prepareIndex("index", "type", "1").setSource("ip", "192.168.1.7"), - client().prepareIndex("index", "type", "2").setSource("ip", "192.168.1.7"), - client().prepareIndex("index", "type", "3").setSource("ip", "2001:db8::2:1")); - - SearchResponse response = client().prepareSearch("index").addAggregation( - AggregationBuilders.terms("my_terms").field("ip").executionHint(randomExecutionHint())).get(); - assertSearchResponse(response); - Terms terms = response.getAggregations().get("my_terms"); - assertEquals(2, terms.getBuckets().size()); - - Terms.Bucket bucket1 = terms.getBuckets().get(0); - assertEquals(2, bucket1.getDocCount()); - assertEquals("192.168.1.7", bucket1.getKey()); - assertEquals("192.168.1.7", bucket1.getKeyAsString()); - - Terms.Bucket bucket2 = terms.getBuckets().get(1); - assertEquals(1, bucket2.getDocCount()); - assertEquals("2001:db8::2:1", bucket2.getKey()); - assertEquals("2001:db8::2:1", bucket2.getKeyAsString()); - } - public void testScriptValue() throws Exception { assertAcked(prepareCreate("index").addMapping("type", "ip", "type=ip")); indexRandom(true, diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java index 565cdaaa87e0a..db66a46f31217 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java @@ -29,9 +29,8 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; -import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.bucket.filter.Filter; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; @@ -40,9 +39,7 @@ import org.elasticsearch.search.aggregations.metrics.stats.Stats; import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats; import org.elasticsearch.search.aggregations.metrics.sum.Sum; -import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.test.ESIntegTestCase; -import org.hamcrest.Matchers; import java.io.IOException; import java.util.ArrayList; @@ -59,11 +56,9 @@ import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats; import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; -import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.max; import static org.elasticsearch.search.aggregations.AggregationBuilders.stats; import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; @@ -72,7 +67,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; @ESIntegTestCase.SuiteScopeTestCase @@ -280,74 +274,6 @@ public void testSizeIsZero() { assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [terms]")); } - public void testSingleValueField() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values()))) - .execute().actionGet(); - - assertSearchResponse(response); - - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - for (int i = 0; i < 5; i++) { - Terms.Bucket bucket = terms.getBucketByKey("" + i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + i)); - assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - } - } - - public void testSingleValueFieldWithFiltering() throws Exception { - long includes[] = { 1, 2, 3, 98 }; - long excludes[] = { -1, 2, 4 }; - long empty[] = {}; - testIncludeExcludeResults(1, includes, empty, new long[] { 1, 2, 3 }, new long[0]); - testIncludeExcludeResults(1, includes, excludes, new long[] { 1, 3 }, new long[0]); - testIncludeExcludeResults(1, empty, excludes, new long[] { 0, 1, 3 }, new long[0]); - - testIncludeExcludeResults(0, includes, empty, new long[] { 1, 2, 3}, new long[] { 98 }); - testIncludeExcludeResults(0, includes, excludes, new long[] { 1, 3 }, new long[] { 98 }); - testIncludeExcludeResults(0, empty, excludes, new long[] { 0, 1, 3 }, new long[] {5, 6, 7, 8, 9, 10, 11}); - } - - private void testIncludeExcludeResults(int minDocCount, long[] includes, long[] excludes, - long[] expectedWithCounts, long[] expectedZeroCounts) { - SearchResponse response = client().prepareSearch("idx", "high_card_idx") - .setQuery(QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery("_index", "high_card_idx"))) - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .includeExclude(new IncludeExclude(includes, excludes)) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .minDocCount(minDocCount)) - .execute().actionGet(); - assertSearchResponse(response); - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(expectedWithCounts.length + expectedZeroCounts.length)); - - for (int i = 0; i < expectedWithCounts.length; i++) { - Terms.Bucket bucket = terms.getBucketByKey("" + expectedWithCounts[i]); - assertThat(bucket, notNullValue()); - assertThat(bucket.getDocCount(), equalTo(1L)); - } - - for (int i = 0; i < expectedZeroCounts.length; i++) { - Terms.Bucket bucket = terms.getBucketByKey("" + expectedZeroCounts[i]); - assertThat(bucket, notNullValue()); - assertThat(bucket.getDocCount(), equalTo(0L)); - } - } - - - public void testSingleValueFieldWithPartitionedFiltering() throws Exception { runTestFieldWithPartitionedFiltering(SINGLE_VALUED_FIELD_NAME); } @@ -387,142 +313,6 @@ private void runTestFieldWithPartitionedFiltering(String field) throws Exception assertEquals(expectedCardinality, foundTerms.size()); } - - public void testSingleValueFieldWithMaxSize() throws Exception { - SearchResponse response = client().prepareSearch("high_card_idx") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .size(20) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .order(BucketOrder.key(true))) // we need to sort by terms cause we're checking the first 20 values - .execute().actionGet(); - - assertSearchResponse(response); - - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(20)); - - for (int i = 0; i < 20; i++) { - Terms.Bucket bucket = terms.getBucketByKey("" + i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + i)); - assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - } - } - - public void testSingleValueFieldOrderedByTermAsc() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .order(BucketOrder.key(true))) - .execute().actionGet(); - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - int i = 0; - for (Terms.Bucket bucket : terms.getBuckets()) { - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + i)); - assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - i++; - } - } - - public void testSingleValueFieldOrderedByTermDesc() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .order(BucketOrder.key(false))) - .execute().actionGet(); - - assertSearchResponse(response); - - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - int i = 4; - for (Terms.Bucket bucket : terms.getBuckets()) { - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + i)); - assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - i--; - } - } - - public void testSingleValueFieldOrderedByTieBreaker() throws Exception { - SearchResponse response = client().prepareSearch("idx").setTypes("type") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .order(BucketOrder.aggregation("max_constant", randomBoolean())) - .subAggregation(max("max_constant").field("constant"))) - .execute().actionGet(); - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - int i = 0; - for (Terms.Bucket bucket : terms.getBuckets()) { - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + i)); - assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - i++; - } - } - - public void testSingleValuedFieldWithSubAggregation() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .subAggregation(sum("sum").field(MULTI_VALUED_FIELD_NAME))) - .execute().actionGet(); - - assertSearchResponse(response); - - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - Object[] propertiesKeys = (Object[]) ((InternalAggregation)terms).getProperty("_key"); - Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)terms).getProperty("_count"); - Object[] propertiesCounts = (Object[]) ((InternalAggregation)terms).getProperty("sum.value"); - - for (int i = 0; i < 5; i++) { - Terms.Bucket bucket = terms.getBucketByKey("" + i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + i)); - assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - Sum sum = bucket.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - assertThat((long) sum.getValue(), equalTo(i+i+1L)); - assertThat((long) propertiesKeys[i], equalTo((long) i)); - assertThat((long) propertiesDocCounts[i], equalTo(1L)); - assertThat((double) propertiesCounts[i], equalTo((double) i + i + 1L)); - } - } - public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(terms("terms") @@ -548,34 +338,6 @@ public void testSingleValuedFieldWithValueScript() throws Exception { } } - public void testMultiValuedField() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(terms("terms") - .field(MULTI_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values()))) - .execute().actionGet(); - - assertSearchResponse(response); - - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(6)); - - for (int i = 0; i < 6; i++) { - Terms.Bucket bucket = terms.getBucketByKey("" + i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + i)); - assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i)); - if (i == 0 || i == 5) { - assertThat(bucket.getDocCount(), equalTo(1L)); - } else { - assertThat(bucket.getDocCount(), equalTo(2L)); - } - } - } - public void testMultiValuedFieldWithValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx") .addAggregation(terms("terms") @@ -704,23 +466,6 @@ public void testScriptMultiValued() throws Exception { } } - public void testUnmapped() throws Exception { - SearchResponse response = client().prepareSearch("idx_unmapped") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .size(randomIntBetween(1, 5)) - .collectMode(randomFrom(SubAggCollectionMode.values()))) - .execute().actionGet(); - - assertSearchResponse(response); - - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(0)); - } - public void testPartiallyUnmapped() throws Exception { SearchResponse response = client().prepareSearch("idx_unmapped", "idx") .addAggregation(terms("terms") @@ -771,54 +516,6 @@ public void testPartiallyUnmappedWithFormat() throws Exception { } } - public void testEmptyAggregation() throws Exception { - SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") - .setQuery(matchAllQuery()) - .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(1L).minDocCount(0) - .subAggregation(terms("terms").field(SINGLE_VALUED_FIELD_NAME))) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L)); - Histogram histo = searchResponse.getAggregations().get("histo"); - assertThat(histo, Matchers.notNullValue()); - Histogram.Bucket bucket = histo.getBuckets().get(1); - assertThat(bucket, Matchers.notNullValue()); - - Terms terms = bucket.getAggregations().get("terms"); - assertThat(terms, Matchers.notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().isEmpty(), is(true)); - } - - public void testSingleValuedFieldOrderedBySingleValueSubAggregationAsc() throws Exception { - boolean asc = true; - SearchResponse response = client().prepareSearch("idx") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .order(BucketOrder.aggregation("avg_i", asc)) - .subAggregation(avg("avg_i").field(SINGLE_VALUED_FIELD_NAME)) - ).execute().actionGet(); - - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - for (int i = 0; i < 5; i++) { - Terms.Bucket bucket = terms.getBucketByKey("" + i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - Avg avg = bucket.getAggregations().get("avg_i"); - assertThat(avg, notNullValue()); - assertThat(avg.getValue(), equalTo((double) i)); - } - } - public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscWithTermsSubAgg() throws Exception { boolean asc = true; SearchResponse response = client().prepareSearch("idx") @@ -1032,38 +729,6 @@ public void testSingleValuedFieldOrderedByMultiValuedSubAggregationWithoutMetric } } - public void testSingleValuedFieldOrderedBySingleValueSubAggregationDesc() throws Exception { - boolean asc = false; - SearchResponse response = client().prepareSearch("idx") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .order(BucketOrder.aggregation("avg_i", asc)) - .subAggregation(avg("avg_i").field(SINGLE_VALUED_FIELD_NAME)) - ).execute().actionGet(); - - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - for (int i = 4; i >= 0; i--) { - - Terms.Bucket bucket = terms.getBucketByKey("" + i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("" + i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - - Avg avg = bucket.getAggregations().get("avg_i"); - assertThat(avg, notNullValue()); - assertThat(avg.getValue(), equalTo((double) i)); - } - - } - public void testSingleValuedFieldOrderedByMultiValueSubAggregationAsc() throws Exception { boolean asc = true; SearchResponse response = client().prepareSearch("idx") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java index 0c93ff2f6bbd5..b2575c3dc572b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.search.aggregations.bucket; -import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; @@ -34,25 +33,19 @@ import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; -import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.bucket.filter.Filter; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.ExecutionMode; import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; import org.elasticsearch.search.aggregations.metrics.avg.Avg; -import org.elasticsearch.search.aggregations.metrics.max.Max; import org.elasticsearch.search.aggregations.metrics.stats.Stats; import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats; import org.elasticsearch.search.aggregations.metrics.sum.Sum; -import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount; -import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; import java.io.IOException; -import java.text.NumberFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -61,20 +54,15 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; -import static org.elasticsearch.search.aggregations.AggregationBuilders.count; import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats; import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; -import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; -import static org.elasticsearch.search.aggregations.AggregationBuilders.max; import static org.elasticsearch.search.aggregations.AggregationBuilders.stats; import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; @@ -82,9 +70,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; -import static org.hamcrest.core.IsNull.nullValue; @ESIntegTestCase.SuiteScopeTestCase public class StringTermsIT extends AbstractTermsTestCase { @@ -255,217 +241,6 @@ public void testSizeIsZero() { assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [terms]")); } - public void testSingleValueField() throws Exception { - SearchResponse response = client() - .prepareSearch("idx") - .setTypes("type") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values()))).execute().actionGet(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - Object[] propertiesKeys = (Object[]) ((InternalAggregation)terms).getProperty("_key"); - Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)terms).getProperty("_count"); - - for (int i = 0; i < 5; i++) { - Terms.Bucket bucket = terms.getBucketByKey("val" + i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("val" + i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - assertThat((String) propertiesKeys[i], equalTo("val" + i)); - assertThat((long) propertiesDocCounts[i], equalTo(1L)); - } - } - - public void testSingleValueFieldWithGlobalOrdinals() throws Exception { - ExecutionMode[] executionModes = new ExecutionMode[] { null, ExecutionMode.GLOBAL_ORDINALS, ExecutionMode.GLOBAL_ORDINALS_HASH, - ExecutionMode.GLOBAL_ORDINALS_LOW_CARDINALITY }; - for (ExecutionMode executionMode : executionModes) { - logger.info("Execution mode: {}", executionMode); - SearchResponse response = client() - .prepareSearch("idx") - .setTypes("type") - .addAggregation( - terms("terms").executionHint(executionMode == null ? null : executionMode.toString()) - .field(SINGLE_VALUED_FIELD_NAME).collectMode(randomFrom(SubAggCollectionMode.values()))).execute() - .actionGet(); - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - for (int i = 0; i < 5; i++) { - Terms.Bucket bucket = terms.getBucketByKey("val" + i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("val" + i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - } - } - } - - public void testSingleValueFieldWithRegexFiltering() throws Exception { - // include without exclude - // we should be left with: val000, val001, val002, val003, val004, val005, val006, val007, val008, val009 - - SearchResponse response = client() - .prepareSearch("high_card_idx") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())).includeExclude(new IncludeExclude("val00.+", null))) - .execute().actionGet(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(10)); - - for (int i = 0; i < 10; i++) { - Terms.Bucket bucket = terms.getBucketByKey("val00" + i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("val00" + i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - } - - // include and exclude - // we should be left with: val002, val003, val004, val005, val006, val007, val008, val009 - - response = client() - .prepareSearch("high_card_idx") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .includeExclude(new IncludeExclude("val00.+", "(val000|val001)"))) - .execute().actionGet(); - - assertSearchResponse(response); - - terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(8)); - - for (int i = 2; i < 10; i++) { - Terms.Bucket bucket = terms.getBucketByKey("val00" + i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("val00" + i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - } - - // exclude without include - // we should be left with: val000, val001, val002, val003, val004, val005, val006, val007, val008, val009 - - response = client() - .prepareSearch("high_card_idx") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .includeExclude(new IncludeExclude(null, new RegExp("val0[1-9]+.+")))) - .execute().actionGet(); - - assertSearchResponse(response); - - terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(10)); - - for (int i = 0; i < 10; i++) { - Terms.Bucket bucket = terms.getBucketByKey("val00" + i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("val00" + i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - } - } - - public void testSingleValueFieldWithExactTermFiltering() throws Exception { - // include without exclude - String incVals[] = { "val000", "val001", "val002", "val003", "val004", "val005", "val006", "val007", "val008", "val009" }; - SearchResponse response = client() - .prepareSearch("high_card_idx") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())).includeExclude(new IncludeExclude(incVals, null))) - .execute().actionGet(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(incVals.length)); - - for (String incVal : incVals) { - Terms.Bucket bucket = terms.getBucketByKey(incVal); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo(incVal)); - assertThat(bucket.getDocCount(), equalTo(1L)); - } - - // include and exclude - // Slightly illogical example with exact terms below as include and exclude sets - // are made to overlap but the exclude set should have priority over matches. - // we should be left with: val002, val003, val004, val005, val006, val007, val008, val009 - String excVals[] = { "val000", "val001" }; - - response = client() - .prepareSearch("high_card_idx") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())).includeExclude(new IncludeExclude(incVals, excVals))) - .execute() - .actionGet(); - - assertSearchResponse(response); - - terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(8)); - - for (int i = 2; i < 10; i++) { - Terms.Bucket bucket = terms.getBucketByKey("val00" + i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("val00" + i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - } - - // Check case with only exact term exclude clauses - response = client() - .prepareSearch("high_card_idx") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())).includeExclude(new IncludeExclude(null, excVals))) - .execute().actionGet(); - - assertSearchResponse(response); - - terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(10)); - for (String key : excVals) { - Terms.Bucket bucket = terms.getBucketByKey(key); - assertThat(bucket, nullValue()); - } - NumberFormat nf = NumberFormat.getIntegerInstance(Locale.ENGLISH); - nf.setMinimumIntegerDigits(3); - for (int i = 2; i < 12; i++) { - Terms.Bucket bucket = terms.getBucketByKey("val" + nf.format(i)); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("val" + nf.format(i))); - assertThat(bucket.getDocCount(), equalTo(1L)); - } - - } - public void testSingleValueFieldWithPartitionedFiltering() throws Exception { runTestFieldWithPartitionedFiltering(SINGLE_VALUED_FIELD_NAME); } @@ -503,116 +278,6 @@ private void runTestFieldWithPartitionedFiltering(String field) throws Exception assertEquals(expectedCardinality, foundTerms.size()); } - - public void testSingleValueFieldWithMaxSize() throws Exception { - SearchResponse response = client() - .prepareSearch("high_card_idx") - .addAggregation( - terms("terms") - .executionHint(randomExecutionHint()) - .field(SINGLE_VALUED_FIELD_NAME).size(20) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .order(BucketOrder.key(true))) // we need to sort by terms cause we're checking the first 20 values - .execute().actionGet(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(20)); - - for (int i = 0; i < 20; i++) { - Terms.Bucket bucket = terms.getBucketByKey("val" + Strings.padStart(i + "", 3, '0')); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("val" + Strings.padStart(i + "", 3, '0'))); - assertThat(bucket.getDocCount(), equalTo(1L)); - } - } - - public void testSingleValueFieldOrderedByTermAsc() throws Exception { - SearchResponse response = client() - .prepareSearch("idx") - .setTypes("type") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.key(true))).execute() - .actionGet(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - int i = 0; - for (Terms.Bucket bucket : terms.getBuckets()) { - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("val" + i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - i++; - } - } - - public void testSingleValueFieldOrderedByTermDesc() throws Exception { - SearchResponse response = client() - .prepareSearch("idx") - .setTypes("type") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.key(false))).execute() - .actionGet(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - int i = 4; - for (Terms.Bucket bucket : terms.getBuckets()) { - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("val" + i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - i--; - } - } - - public void testSingleValuedFieldWithSubAggregation() throws Exception { - SearchResponse response = client() - .prepareSearch("idx") - .setTypes("type") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())) - .subAggregation(count("count").field(MULTI_VALUED_FIELD_NAME))).execute().actionGet(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - Object[] propertiesKeys = (Object[]) ((InternalAggregation)terms).getProperty("_key"); - Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)terms).getProperty("_count"); - Object[] propertiesCounts = (Object[]) ((InternalAggregation)terms).getProperty("count.value"); - - for (int i = 0; i < 5; i++) { - Terms.Bucket bucket = terms.getBucketByKey("val" + i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("val" + i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - ValueCount valueCount = bucket.getAggregations().get("count"); - assertThat(valueCount, notNullValue()); - assertThat(valueCount.getValue(), equalTo(2L)); - assertThat((String) propertiesKeys[i], equalTo("val" + i)); - assertThat((long) propertiesDocCounts[i], equalTo(1L)); - assertThat((double) propertiesCounts[i], equalTo(2.0)); - } - } - public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse response = client() .prepareSearch("idx") @@ -666,33 +331,6 @@ public void testMultiValuedFieldWithValueScriptNotUnique() throws Exception { assertThat(bucket.getDocCount(), equalTo(5L)); } - public void testMultiValuedField() throws Exception { - SearchResponse response = client() - .prepareSearch("idx") - .setTypes("type") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(MULTI_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values()))).execute().actionGet(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(6)); - - for (int i = 0; i < 6; i++) { - Terms.Bucket bucket = terms.getBucketByKey("val" + i); - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("val" + i)); - if (i == 0 || i == 5) { - assertThat(bucket.getDocCount(), equalTo(1L)); - } else { - assertThat(bucket.getDocCount(), equalTo(2L)); - } - } - } - public void testMultiValuedScript() throws Exception { SearchResponse response = client() .prepareSearch("idx") @@ -856,22 +494,6 @@ public void testScriptMultiValued() throws Exception { } } - public void testUnmapped() throws Exception { - SearchResponse response = client() - .prepareSearch("idx_unmapped") - .setTypes("type") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()).size(randomIntBetween(1, 5)).field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values()))).execute().actionGet(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(0)); - } - public void testPartiallyUnmapped() throws Exception { SearchResponse response = client() .prepareSearch("idx", "idx_unmapped") @@ -922,87 +544,6 @@ public void testStringTermsNestedIntoPerBucketAggregator() throws Exception { } } - public void testEmptyAggregation() throws Exception { - SearchResponse searchResponse = client() - .prepareSearch("empty_bucket_idx") - .setQuery(matchAllQuery()) - .addAggregation( - histogram("histo") - .field(SINGLE_VALUED_FIELD_NAME) - .interval(1L) - .minDocCount(0) - .subAggregation(terms("terms").field("value"))) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L)); - Histogram histo = searchResponse.getAggregations().get("histo"); - assertThat(histo, Matchers.notNullValue()); - Histogram.Bucket bucket = histo.getBuckets().get(1); - assertThat(bucket, Matchers.notNullValue()); - - Terms terms = bucket.getAggregations().get("terms"); - assertThat(terms, Matchers.notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().isEmpty(), is(true)); - } - - public void testSingleValuedFieldOrderedBySingleValueSubAggregationAsc() throws Exception { - boolean asc = true; - SearchResponse response = client() - .prepareSearch("idx") - .setTypes("type") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.aggregation("avg_i", asc)) - .subAggregation(avg("avg_i").field("i"))).execute().actionGet(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - int i = 0; - for (Terms.Bucket bucket : terms.getBuckets()) { - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("val" + i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - Avg avg = bucket.getAggregations().get("avg_i"); - assertThat(avg, notNullValue()); - assertThat(avg.getValue(), equalTo((double) i)); - i++; - } - } - - public void testSingleValuedFieldOrderedByTieBreaker() throws Exception { - SearchResponse response = client() - .prepareSearch("idx") - .setTypes("type") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.aggregation("max_constant", randomBoolean())) - .subAggregation(max("max_constant").field("constant"))).execute().actionGet(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - int i = 0; - for (Terms.Bucket bucket : terms.getBuckets()) { - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("val" + i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - Max max = bucket.getAggregations().get("max_constant"); - assertThat(max, notNullValue()); - assertThat(max.getValue(), equalTo((double) 1)); - i++; - } - } - public void testSingleValuedFieldOrderedByIllegalAgg() throws Exception { boolean asc = true; try { @@ -1330,37 +871,6 @@ public void testSingleValuedFieldOrderedByMultiValuedSubAggregationWithoutMetric } } - public void testSingleValuedFieldOrderedBySingleValueSubAggregationDesc() throws Exception { - boolean asc = false; - SearchResponse response = client() - .prepareSearch("idx") - .setTypes("type") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME) - .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.aggregation("avg_i", asc)) - .subAggregation(avg("avg_i").field("i"))).execute().actionGet(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(5)); - - int i = 4; - for (Terms.Bucket bucket : terms.getBuckets()) { - assertThat(bucket, notNullValue()); - assertThat(key(bucket), equalTo("val" + i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - - Avg avg = bucket.getAggregations().get("avg_i"); - assertThat(avg, notNullValue()); - assertThat(avg.getValue(), equalTo((double) i)); - i--; - } - - } - public void testSingleValuedFieldOrderedByMultiValueSubAggregationAsc() throws Exception { boolean asc = true; SearchResponse response = client() diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index 33e0c54d93425..cf8aa6ba657ea 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -19,32 +19,52 @@ package org.elasticsearch.search.aggregations.bucket.terms; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorTestCase; -import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.bucket.filter.Filter; +import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; import org.elasticsearch.search.aggregations.support.ValueType; import java.io.IOException; +import java.net.InetAddress; import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; import static org.hamcrest.Matchers.instanceOf; @@ -94,54 +114,754 @@ public void testGlobalOrdinalsExecutionHint() throws Exception { directory.close(); } - public void testTermsAggregator() throws Exception { - Directory directory = newDirectory(); - RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); - Document document = new Document(); - document.add(new SortedSetDocValuesField("string", new BytesRef("a"))); - document.add(new SortedSetDocValuesField("string", new BytesRef("b"))); - indexWriter.addDocument(document); - document = new Document(); - document.add(new SortedSetDocValuesField("string", new BytesRef("c"))); - document.add(new SortedSetDocValuesField("string", new BytesRef("a"))); - indexWriter.addDocument(document); - document = new Document(); - document.add(new SortedSetDocValuesField("string", new BytesRef("b"))); - document.add(new SortedSetDocValuesField("string", new BytesRef("d"))); - indexWriter.addDocument(document); - indexWriter.close(); + public void testSimple() throws Exception { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + Document document = new Document(); + document.add(new SortedSetDocValuesField("string", new BytesRef("a"))); + document.add(new SortedSetDocValuesField("string", new BytesRef("b"))); + indexWriter.addDocument(document); + document = new Document(); + document.add(new SortedSetDocValuesField("string", new BytesRef("c"))); + document.add(new SortedSetDocValuesField("string", new BytesRef("a"))); + indexWriter.addDocument(document); + document = new Document(); + document.add(new SortedSetDocValuesField("string", new BytesRef("b"))); + document.add(new SortedSetDocValuesField("string", new BytesRef("d"))); + indexWriter.addDocument(document); + try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) { + IndexSearcher indexSearcher = newIndexSearcher(indexReader); + for (TermsAggregatorFactory.ExecutionMode executionMode : TermsAggregatorFactory.ExecutionMode.values()) { + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + .executionHint(executionMode.toString()) + .field("string") + .order(BucketOrder.key(true)); + MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType(); + fieldType.setName("string"); + fieldType.setHasDocValues(true ); - IndexReader indexReader = DirectoryReader.open(directory); - // We do not use LuceneTestCase.newSearcher because we need a DirectoryReader - IndexSearcher indexSearcher = new IndexSearcher(indexReader); + TermsAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + Terms result = (Terms) aggregator.buildAggregation(0L); + assertEquals(4, result.getBuckets().size()); + assertEquals("a", result.getBuckets().get(0).getKeyAsString()); + assertEquals(2L, result.getBuckets().get(0).getDocCount()); + assertEquals("b", result.getBuckets().get(1).getKeyAsString()); + assertEquals(2L, result.getBuckets().get(1).getDocCount()); + assertEquals("c", result.getBuckets().get(2).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(2).getDocCount()); + assertEquals("d", result.getBuckets().get(3).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(3).getDocCount()); + } + } + } + } + } - for (TermsAggregatorFactory.ExecutionMode executionMode : TermsAggregatorFactory.ExecutionMode.values()) { - TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) - .executionHint(executionMode.toString()) - .field("string") - .order(BucketOrder.key(true)); - MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType(); - fieldType.setName("string"); - fieldType.setHasDocValues(true ); - - TermsAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - Terms result = (Terms) aggregator.buildAggregation(0L); - assertEquals(4, result.getBuckets().size()); - assertEquals("a", result.getBuckets().get(0).getKeyAsString()); - assertEquals(2L, result.getBuckets().get(0).getDocCount()); - assertEquals("b", result.getBuckets().get(1).getKeyAsString()); - assertEquals(2L, result.getBuckets().get(1).getDocCount()); - assertEquals("c", result.getBuckets().get(2).getKeyAsString()); - assertEquals(1L, result.getBuckets().get(2).getDocCount()); - assertEquals("d", result.getBuckets().get(3).getKeyAsString()); - assertEquals(1L, result.getBuckets().get(3).getDocCount()); + public void testStringIncludeExclude() throws Exception { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + Document document = new Document(); + document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val000"))); + document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val001"))); + document.add(new SortedDocValuesField("sv_field", new BytesRef("val001"))); + indexWriter.addDocument(document); + document = new Document(); + document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val002"))); + document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val003"))); + document.add(new SortedDocValuesField("sv_field", new BytesRef("val003"))); + indexWriter.addDocument(document); + document = new Document(); + document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val004"))); + document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val005"))); + document.add(new SortedDocValuesField("sv_field", new BytesRef("val005"))); + indexWriter.addDocument(document); + document = new Document(); + document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val006"))); + document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val007"))); + document.add(new SortedDocValuesField("sv_field", new BytesRef("val007"))); + indexWriter.addDocument(document); + document = new Document(); + document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val008"))); + document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val009"))); + document.add(new SortedDocValuesField("sv_field", new BytesRef("val009"))); + indexWriter.addDocument(document); + document = new Document(); + document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val010"))); + document.add(new SortedSetDocValuesField("mv_field", new BytesRef("val011"))); + document.add(new SortedDocValuesField("sv_field", new BytesRef("val011"))); + indexWriter.addDocument(document); + try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) { + IndexSearcher indexSearcher = newIndexSearcher(indexReader); + MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType(); + fieldType.setName("mv_field"); + fieldType.setHasDocValues(true); + + String executionHint = randomFrom(TermsAggregatorFactory.ExecutionMode.values()).toString(); + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + .executionHint(executionHint) + .includeExclude(new IncludeExclude("val00.+", null)) + .field("mv_field") + .size(12) + .order(BucketOrder.key(true)); + + Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + Terms result = (Terms) aggregator.buildAggregation(0L); + assertEquals(10, result.getBuckets().size()); + assertEquals("val000", result.getBuckets().get(0).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(0).getDocCount()); + assertEquals("val001", result.getBuckets().get(1).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(1).getDocCount()); + assertEquals("val002", result.getBuckets().get(2).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(2).getDocCount()); + assertEquals("val003", result.getBuckets().get(3).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(3).getDocCount()); + assertEquals("val004", result.getBuckets().get(4).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(4).getDocCount()); + assertEquals("val005", result.getBuckets().get(5).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(5).getDocCount()); + assertEquals("val006", result.getBuckets().get(6).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(6).getDocCount()); + assertEquals("val007", result.getBuckets().get(7).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(7).getDocCount()); + assertEquals("val008", result.getBuckets().get(8).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(8).getDocCount()); + assertEquals("val009", result.getBuckets().get(9).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(9).getDocCount()); + + MappedFieldType fieldType2 = new KeywordFieldMapper.KeywordFieldType(); + fieldType2.setName("sv_field"); + fieldType2.setHasDocValues(true); + aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + .executionHint(executionHint) + .includeExclude(new IncludeExclude("val00.+", null)) + .field("sv_field") + .order(BucketOrder.key(true)); + + aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType2); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + result = (Terms) aggregator.buildAggregation(0L); + assertEquals(5, result.getBuckets().size()); + assertEquals("val001", result.getBuckets().get(0).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(0).getDocCount()); + assertEquals("val003", result.getBuckets().get(1).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(1).getDocCount()); + assertEquals("val005", result.getBuckets().get(2).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(2).getDocCount()); + assertEquals("val007", result.getBuckets().get(3).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(3).getDocCount()); + assertEquals("val009", result.getBuckets().get(4).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(4).getDocCount()); + aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + .executionHint(executionHint) + .includeExclude(new IncludeExclude("val00.+", "(val000|val001)")) + .field("mv_field") + .order(BucketOrder.key(true)); + + aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + result = (Terms) aggregator.buildAggregation(0L); + assertEquals(8, result.getBuckets().size()); + assertEquals("val002", result.getBuckets().get(0).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(0).getDocCount()); + assertEquals("val003", result.getBuckets().get(1).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(1).getDocCount()); + assertEquals("val004", result.getBuckets().get(2).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(2).getDocCount()); + assertEquals("val005", result.getBuckets().get(3).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(3).getDocCount()); + assertEquals("val006", result.getBuckets().get(4).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(4).getDocCount()); + assertEquals("val007", result.getBuckets().get(5).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(5).getDocCount()); + assertEquals("val008", result.getBuckets().get(6).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(6).getDocCount()); + assertEquals("val009", result.getBuckets().get(7).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(7).getDocCount()); + + aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + .executionHint(executionHint) + .includeExclude(new IncludeExclude(null, "val00.+")) + .field("mv_field") + .order(BucketOrder.key(true)); + aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + result = (Terms) aggregator.buildAggregation(0L); + assertEquals(2, result.getBuckets().size()); + assertEquals("val010", result.getBuckets().get(0).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(0).getDocCount()); + assertEquals("val011", result.getBuckets().get(1).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(1).getDocCount()); + + aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + .executionHint(executionHint) + .includeExclude(new IncludeExclude(new String[]{"val000", "val010"}, null)) + .field("mv_field") + .order(BucketOrder.key(true)); + aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + result = (Terms) aggregator.buildAggregation(0L); + assertEquals(2, result.getBuckets().size()); + assertEquals("val000", result.getBuckets().get(0).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(0).getDocCount()); + assertEquals("val010", result.getBuckets().get(1).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(1).getDocCount()); + + aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + .executionHint(executionHint) + .includeExclude(new IncludeExclude(null, new String[]{"val001", "val002", "val003", "val004", + "val005", "val006", "val007", "val008", "val009", "val011"})) + .field("mv_field") + .order(BucketOrder.key(true)); + aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + result = (Terms) aggregator.buildAggregation(0L); + assertEquals(2, result.getBuckets().size()); + assertEquals("val000", result.getBuckets().get(0).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(0).getDocCount()); + assertEquals("val010", result.getBuckets().get(1).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(1).getDocCount()); + } + } + } + } + + public void testNumericIncludeExclude() throws Exception { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + Document document = new Document(); + document.add(new NumericDocValuesField("long_field", 0)); + document.add(new NumericDocValuesField("double_field", Double.doubleToRawLongBits(0.0))); + indexWriter.addDocument(document); + document = new Document(); + document.add(new NumericDocValuesField("long_field", 1)); + document.add(new NumericDocValuesField("double_field", Double.doubleToRawLongBits(1.0))); + indexWriter.addDocument(document); + document = new Document(); + document.add(new NumericDocValuesField("long_field", 2)); + document.add(new NumericDocValuesField("double_field", Double.doubleToRawLongBits(2.0))); + indexWriter.addDocument(document); + document = new Document(); + document.add(new NumericDocValuesField("long_field", 3)); + document.add(new NumericDocValuesField("double_field", Double.doubleToRawLongBits(3.0))); + indexWriter.addDocument(document); + document = new Document(); + document.add(new NumericDocValuesField("long_field", 4)); + document.add(new NumericDocValuesField("double_field", Double.doubleToRawLongBits(4.0))); + indexWriter.addDocument(document); + document = new Document(); + document.add(new NumericDocValuesField("long_field", 5)); + document.add(new NumericDocValuesField("double_field", Double.doubleToRawLongBits(5.0))); + indexWriter.addDocument(document); + try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) { + IndexSearcher indexSearcher = newIndexSearcher(indexReader); + MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); + fieldType.setName("long_field"); + fieldType.setHasDocValues(true ); + + String executionHint = randomFrom(TermsAggregatorFactory.ExecutionMode.values()).toString(); + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.LONG) + .executionHint(executionHint) + .includeExclude(new IncludeExclude(new long[]{0, 5}, null)) + .field("long_field") + .order(BucketOrder.key(true)); + Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + Terms result = (Terms) aggregator.buildAggregation(0L); + assertEquals(2, result.getBuckets().size()); + assertEquals(0L, result.getBuckets().get(0).getKey()); + assertEquals(1L, result.getBuckets().get(0).getDocCount()); + assertEquals(5L, result.getBuckets().get(1).getKey()); + assertEquals(1L, result.getBuckets().get(1).getDocCount()); + + aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.LONG) + .executionHint(executionHint) + .includeExclude(new IncludeExclude(null, new long[]{0, 5})) + .field("long_field") + .order(BucketOrder.key(true)); + aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + result = (Terms) aggregator.buildAggregation(0L); + assertEquals(4, result.getBuckets().size()); + assertEquals(1L, result.getBuckets().get(0).getKey()); + assertEquals(1L, result.getBuckets().get(0).getDocCount()); + assertEquals(2L, result.getBuckets().get(1).getKey()); + assertEquals(1L, result.getBuckets().get(1).getDocCount()); + assertEquals(3L, result.getBuckets().get(2).getKey()); + assertEquals(1L, result.getBuckets().get(2).getDocCount()); + assertEquals(4L, result.getBuckets().get(3).getKey()); + assertEquals(1L, result.getBuckets().get(3).getDocCount()); + + fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE); + fieldType.setName("double_field"); + fieldType.setHasDocValues(true ); + aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.DOUBLE) + .executionHint(executionHint) + .includeExclude(new IncludeExclude(new double[]{0.0, 5.0}, null)) + .field("double_field") + .order(BucketOrder.key(true)); + aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + result = (Terms) aggregator.buildAggregation(0L); + assertEquals(2, result.getBuckets().size()); + assertEquals(0.0, result.getBuckets().get(0).getKey()); + assertEquals(1L, result.getBuckets().get(0).getDocCount()); + assertEquals(5.0, result.getBuckets().get(1).getKey()); + assertEquals(1L, result.getBuckets().get(1).getDocCount()); + + aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.DOUBLE) + .executionHint(executionHint) + .includeExclude(new IncludeExclude(null, new double[]{0.0, 5.0})) + .field("double_field") + .order(BucketOrder.key(true)); + aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + result = (Terms) aggregator.buildAggregation(0L); + assertEquals(4, result.getBuckets().size()); + assertEquals(1.0, result.getBuckets().get(0).getKey()); + assertEquals(1L, result.getBuckets().get(0).getDocCount()); + assertEquals(2.0, result.getBuckets().get(1).getKey()); + assertEquals(1L, result.getBuckets().get(1).getDocCount()); + assertEquals(3.0, result.getBuckets().get(2).getKey()); + assertEquals(1L, result.getBuckets().get(2).getDocCount()); + assertEquals(4.0, result.getBuckets().get(3).getKey()); + assertEquals(1L, result.getBuckets().get(3).getDocCount()); + } + } + } + } + + public void testStringTermsAggregator() throws Exception { + BiFunction luceneFieldFactory = (val, mv) -> { + if (mv) { + return new SortedSetDocValuesField("field", new BytesRef(val)); + } else { + return new SortedDocValuesField("field", new BytesRef(val)); + } + }; + MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType(); + termsAggregator(ValueType.STRING, fieldType, i -> Integer.toString(i), + String::compareTo, luceneFieldFactory); + termsAggregatorWithNestedMaxAgg(ValueType.STRING, fieldType, i -> Integer.toString(i), + val -> new SortedDocValuesField("field", new BytesRef(val))); + } + + public void testLongTermsAggregator() throws Exception { + BiFunction luceneFieldFactory = (val, mv) -> { + if (mv) { + return new SortedNumericDocValuesField("field", val); + } else { + return new NumericDocValuesField("field", val); + } + }; + MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); + termsAggregator(ValueType.LONG, fieldType, Integer::longValue, Long::compareTo, luceneFieldFactory); + termsAggregatorWithNestedMaxAgg(ValueType.LONG, fieldType, Integer::longValue, val -> new NumericDocValuesField("field", val)); + } + + public void testDoubleTermsAggregator() throws Exception { + BiFunction luceneFieldFactory = (val, mv) -> { + if (mv) { + return new SortedNumericDocValuesField("field", Double.doubleToRawLongBits(val)); + } else { + return new NumericDocValuesField("field", Double.doubleToRawLongBits(val)); + } + }; + MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE); + termsAggregator(ValueType.DOUBLE, fieldType, Integer::doubleValue, Double::compareTo, luceneFieldFactory); + termsAggregatorWithNestedMaxAgg(ValueType.DOUBLE, fieldType, Integer::doubleValue, + val -> new NumericDocValuesField("field", Double.doubleToRawLongBits(val))); + } + + public void testIpTermsAggregator() throws Exception { + BiFunction luceneFieldFactory = (val, mv) -> { + if (mv) { + return new SortedSetDocValuesField("field", new BytesRef(InetAddressPoint.encode(val))); + } else { + return new SortedDocValuesField("field", new BytesRef(InetAddressPoint.encode(val))); + } + }; + InetAddress[] base = new InetAddress[] {InetAddresses.forString("192.168.0.0")}; + Comparator comparator = (o1, o2) -> { + BytesRef b1 = new BytesRef(InetAddressPoint.encode(o1)); + BytesRef b2 = new BytesRef(InetAddressPoint.encode(o2)); + return b1.compareTo(b2); + }; + termsAggregator(ValueType.IP, new IpFieldMapper.IpFieldType(), i -> {return base[0] = InetAddressPoint.nextUp(base[0]);}, + comparator, luceneFieldFactory); + } + + private void termsAggregator(ValueType valueType, MappedFieldType fieldType, + Function valueFactory, Comparator keyComparator, + BiFunction luceneFieldFactory) throws Exception { + final Map counts = new HashMap<>(); + final Map filteredCounts = new HashMap<>(); + int numTerms = scaledRandomIntBetween(8, 128); + for (int i = 0; i < numTerms; i++) { + int numDocs = scaledRandomIntBetween(2, 32); + T key = valueFactory.apply(i); + counts.put(key, numDocs); + filteredCounts.put(key, 0); + } + + try (Directory directory = newDirectory()) { + boolean multiValued = randomBoolean(); + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + if (multiValued == false) { + for (Map.Entry entry : counts.entrySet()) { + for (int i = 0; i < entry.getValue(); i++) { + Document document = new Document(); + document.add(luceneFieldFactory.apply(entry.getKey(), false)); + if (randomBoolean()) { + document.add(new StringField("include", "yes", Field.Store.NO)); + filteredCounts.computeIfPresent(entry.getKey(), (key, integer) -> integer + 1); + } + indexWriter.addDocument(document); + } + } + } else { + Iterator> iterator = counts.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry1 = iterator.next(); + Map.Entry entry2 = null; + if (randomBoolean() && iterator.hasNext()) { + entry2 = iterator.next(); + if (entry1.getValue().compareTo(entry2.getValue()) < 0) { + Map.Entry temp = entry1; + entry1 = entry2; + entry2 = temp; + } + } + + for (int i = 0; i < entry1.getValue(); i++) { + Document document = new Document(); + document.add(luceneFieldFactory.apply(entry1.getKey(), true)); + if (entry2 != null && i < entry2.getValue()) { + document.add(luceneFieldFactory.apply(entry2.getKey(), true)); + } + indexWriter.addDocument(document); + } + } + } + try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) { + boolean order = randomBoolean(); + List> expectedBuckets = new ArrayList<>(); + expectedBuckets.addAll(counts.entrySet()); + BucketOrder bucketOrder; + Comparator> comparator; + if (randomBoolean()) { + bucketOrder = BucketOrder.key(order); + comparator = Comparator.comparing(Map.Entry::getKey, keyComparator); + } else { + // if order by count then we need to use compound so that we can also sort by key as tie breaker: + bucketOrder = BucketOrder.compound(BucketOrder.count(order), BucketOrder.key(order)); + comparator = Comparator.comparing(Map.Entry::getValue); + comparator = comparator.thenComparing(Comparator.comparing(Map.Entry::getKey, keyComparator)); + } + if (order == false) { + comparator = comparator.reversed(); + } + expectedBuckets.sort(comparator); + int size = randomIntBetween(1, counts.size()); + + String executionHint = randomFrom(TermsAggregatorFactory.ExecutionMode.values()).toString(); + logger.info("bucket_order={} size={} execution_hint={}", bucketOrder, size, executionHint); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); + AggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", valueType) + .executionHint(executionHint) + .size(size) + .shardSize(size) + .field("field") + .order(bucketOrder); + fieldType.setName("field"); + fieldType.setHasDocValues(true); + + Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + Terms result = (Terms) aggregator.buildAggregation(0L); + assertEquals(size, result.getBuckets().size()); + for (int i = 0; i < size; i++) { + Map.Entry expected = expectedBuckets.get(i); + Terms.Bucket actual = result.getBuckets().get(i); + if (valueType == ValueType.IP) { + assertEquals(String.valueOf(expected.getKey()).substring(1), actual.getKey()); + } else { + assertEquals(expected.getKey(), actual.getKey()); + } + assertEquals(expected.getValue().longValue(), actual.getDocCount()); + } + + if (multiValued == false) { + aggregationBuilder = new FilterAggregationBuilder("_name1", QueryBuilders.termQuery("include", "yes")); + aggregationBuilder.subAggregation(new TermsAggregationBuilder("_name2", valueType) + .executionHint(executionHint) + .size(numTerms) + .collectMode(randomFrom(Aggregator.SubAggCollectionMode.values())) + .field("field")); + aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + result = ((Filter) aggregator.buildAggregation(0L)).getAggregations().get("_name2"); + int expectedFilteredCounts = 0; + for (Integer count : filteredCounts.values()) { + if (count > 0) { + expectedFilteredCounts++; + } + } + assertEquals(expectedFilteredCounts, result.getBuckets().size()); + for (Terms.Bucket actual : result.getBuckets()) { + Integer expectedCount; + if (valueType == ValueType.IP) { + expectedCount = filteredCounts.get(InetAddresses.forString((String)actual.getKey())); + } else { + expectedCount = filteredCounts.get(actual.getKey()); + } + assertEquals(expectedCount.longValue(), actual.getDocCount()); + } + } + } + } + } + } + + private void termsAggregatorWithNestedMaxAgg(ValueType valueType, MappedFieldType fieldType, + Function valueFactory, + Function luceneFieldFactory) throws Exception { + final Map counts = new HashMap<>(); + int numTerms = scaledRandomIntBetween(8, 128); + for (int i = 0; i < numTerms; i++) { + counts.put(valueFactory.apply(i), randomLong()); + } + + try (Directory directory = newDirectory()) { + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + for (Map.Entry entry : counts.entrySet()) { + Document document = new Document(); + document.add(luceneFieldFactory.apply(entry.getKey())); + document.add(new NumericDocValuesField("value", entry.getValue())); + indexWriter.addDocument(document); + } + try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) { + boolean order = randomBoolean(); + List> expectedBuckets = new ArrayList<>(); + expectedBuckets.addAll(counts.entrySet()); + BucketOrder bucketOrder = BucketOrder.aggregation("_max", order); + Comparator> comparator = Comparator.comparing(Map.Entry::getValue, Long::compareTo); + if (order == false) { + comparator = comparator.reversed(); + } + expectedBuckets.sort(comparator); + int size = randomIntBetween(1, counts.size()); + + String executionHint = randomFrom(TermsAggregatorFactory.ExecutionMode.values()).toString(); + Aggregator.SubAggCollectionMode collectionMode = randomFrom(Aggregator.SubAggCollectionMode.values()); + logger.info("bucket_order={} size={} execution_hint={}, collect_mode={}", + bucketOrder, size, executionHint, collectionMode); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); + AggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", valueType) + .executionHint(executionHint) + .collectMode(collectionMode) + .size(size) + .shardSize(size) + .field("field") + .order(bucketOrder) + .subAggregation(AggregationBuilders.max("_max").field("value")); + fieldType.setName("field"); + fieldType.setHasDocValues(true); + + MappedFieldType fieldType2 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); + fieldType2.setName("value"); + fieldType2.setHasDocValues(true); + Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType, fieldType2); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + Terms result = (Terms) aggregator.buildAggregation(0L); + assertEquals(size, result.getBuckets().size()); + for (int i = 0; i < size; i++) { + Map.Entry expected = expectedBuckets.get(i); + Terms.Bucket actual = result.getBuckets().get(i); + assertEquals(expected.getKey(), actual.getKey()); + } + } + } + } + } + + public void testEmpty() throws Exception { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + MappedFieldType fieldType1 = new KeywordFieldMapper.KeywordFieldType(); + fieldType1.setName("string"); + fieldType1.setHasDocValues(true); + + MappedFieldType fieldType2 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); + fieldType1.setName("long"); + fieldType1.setHasDocValues(true); + + MappedFieldType fieldType3 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE); + fieldType1.setName("double"); + fieldType1.setHasDocValues(true); + try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) { + IndexSearcher indexSearcher = newIndexSearcher(indexReader); + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING) + .field("string"); + Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType1); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + Terms result = (Terms) aggregator.buildAggregation(0L); + assertEquals("_name", result.getName()); + assertEquals(0, result.getBuckets().size()); + + aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.LONG) + .field("long"); + aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType2); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + result = (Terms) aggregator.buildAggregation(0L); + assertEquals("_name", result.getName()); + assertEquals(0, result.getBuckets().size()); + + aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.DOUBLE) + .field("double"); + aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType3); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + result = (Terms) aggregator.buildAggregation(0L); + assertEquals("_name", result.getName()); + assertEquals(0, result.getBuckets().size()); + } + } + } + } + + public void testUnmapped() throws Exception { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + Document document = new Document(); + document.add(new SortedDocValuesField("string", new BytesRef("a"))); + document.add(new NumericDocValuesField("long", 0L)); + document.add(new NumericDocValuesField("double", Double.doubleToRawLongBits(0L))); + indexWriter.addDocument(document); + MappedFieldType fieldType1 = new KeywordFieldMapper.KeywordFieldType(); + fieldType1.setName("another_string"); + fieldType1.setHasDocValues(true); + + MappedFieldType fieldType2 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); + fieldType1.setName("another_long"); + fieldType1.setHasDocValues(true); + + MappedFieldType fieldType3 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE); + fieldType1.setName("another_double"); + fieldType1.setHasDocValues(true); + try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) { + IndexSearcher indexSearcher = newIndexSearcher(indexReader); + ValueType[] valueTypes = new ValueType[]{ValueType.STRING, ValueType.LONG, ValueType.DOUBLE}; + String[] fieldNames = new String[]{"string", "long", "double"}; + for (int i = 0; i < fieldNames.length; i++) { + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", valueTypes[i]) + .field(fieldNames[i]); + Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType1, fieldType2, fieldType3); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + Terms result = (Terms) aggregator.buildAggregation(0L); + assertEquals("_name", result.getName()); + assertEquals(0, result.getBuckets().size()); + } + } + } + } + } + + public void testNestedTermsAgg() throws Exception { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + Document document = new Document(); + document.add(new SortedDocValuesField("field1", new BytesRef("a"))); + document.add(new SortedDocValuesField("field2", new BytesRef("b"))); + indexWriter.addDocument(document); + document = new Document(); + document.add(new SortedDocValuesField("field1", new BytesRef("c"))); + document.add(new SortedDocValuesField("field2", new BytesRef("d"))); + indexWriter.addDocument(document); + document = new Document(); + document.add(new SortedDocValuesField("field1", new BytesRef("e"))); + document.add(new SortedDocValuesField("field2", new BytesRef("f"))); + indexWriter.addDocument(document); + try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) { + IndexSearcher indexSearcher = newIndexSearcher(indexReader); + String executionHint = randomFrom(TermsAggregatorFactory.ExecutionMode.values()).toString(); + Aggregator.SubAggCollectionMode collectionMode = randomFrom(Aggregator.SubAggCollectionMode.values()); + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name1", ValueType.STRING) + .executionHint(executionHint) + .collectMode(collectionMode) + .field("field1") + .order(BucketOrder.key(true)) + .subAggregation(new TermsAggregationBuilder("_name2", ValueType.STRING) + .executionHint(executionHint) + .collectMode(collectionMode) + .field("field2") + .order(BucketOrder.key(true)) + ); + MappedFieldType fieldType1 = new KeywordFieldMapper.KeywordFieldType(); + fieldType1.setName("field1"); + fieldType1.setHasDocValues(true); + MappedFieldType fieldType2 = new KeywordFieldMapper.KeywordFieldType(); + fieldType2.setName("field2"); + fieldType2.setHasDocValues(true); + + Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType1, fieldType2); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + Terms result = (Terms) aggregator.buildAggregation(0L); + assertEquals(3, result.getBuckets().size()); + assertEquals("a", result.getBuckets().get(0).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(0).getDocCount()); + Terms.Bucket nestedBucket = ((Terms) result.getBuckets().get(0).getAggregations().get("_name2")).getBuckets().get(0); + assertEquals("b", nestedBucket.getKeyAsString()); + assertEquals("c", result.getBuckets().get(1).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(1).getDocCount()); + nestedBucket = ((Terms) result.getBuckets().get(1).getAggregations().get("_name2")).getBuckets().get(0); + assertEquals("d", nestedBucket.getKeyAsString()); + assertEquals("e", result.getBuckets().get(2).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(2).getDocCount()); + nestedBucket = ((Terms) result.getBuckets().get(2).getAggregations().get("_name2")).getBuckets().get(0); + assertEquals("f", nestedBucket.getKeyAsString()); + } + } } - indexReader.close(); - directory.close(); } public void testMixLongAndDouble() throws Exception { diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 32513a51c132e..074ce1f8c477b 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -18,10 +18,13 @@ */ package org.elasticsearch.search.aggregations; +import org.apache.lucene.index.AssertingDirectoryReader; import org.apache.lucene.index.CompositeReaderContext; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.AssertingIndexSearcher; import org.apache.lucene.search.Collector; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -298,6 +301,32 @@ protected static DirectoryReader wrap(DirectoryReader directoryReader) throws IO return ElasticsearchDirectoryReader.wrap(directoryReader, new ShardId(new Index("_index", "_na_"), 0)); } + /** + * Added to randomly run with more assertions on the index searcher level, + * like {@link org.apache.lucene.util.LuceneTestCase#newSearcher(IndexReader)}, which can't be used because it also + * wraps in the IndexSearcher's IndexReader with other implementations that we can't handle. (e.g. ParallelCompositeReader) + */ + protected static IndexSearcher newIndexSearcher(IndexReader indexReader) { + if (randomBoolean()) { + return new AssertingIndexSearcher(random(), indexReader); + } else { + return new IndexSearcher(indexReader); + } + } + + /** + * Added to randomly run with more assertions on the index reader level, + * like {@link org.apache.lucene.util.LuceneTestCase#wrapReader(IndexReader)}, which can't be used because it also + * wraps in the IndexReader with other implementations that we can't handle. (e.g. ParallelCompositeReader) + */ + protected static IndexReader maybeWrapReaderEs(DirectoryReader reader) throws IOException { + if (randomBoolean()) { + return new AssertingDirectoryReader(reader); + } else { + return reader; + } + } + @After private void cleanupReleasables() { Releasables.close(releasables);