diff --git a/docs/reference/cat/allocation.asciidoc b/docs/reference/cat/allocation.asciidoc index a9de182e3c00e..0ff745c3e5cb3 100644 --- a/docs/reference/cat/allocation.asciidoc +++ b/docs/reference/cat/allocation.asciidoc @@ -21,5 +21,6 @@ shards disk.indices disk.used disk.avail disk.total disk.percent host ip // TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/ s/46/\\d+/] // TESTRESPONSE[s/CSUXak2/.+/ _cat] -Here we can see that each node has been allocated a single shard and -that they're all using about the same amount of space. +Here we can see that the single shard created has been allocated to the single +node available. + diff --git a/docs/reference/query-dsl/dis-max-query.asciidoc b/docs/reference/query-dsl/dis-max-query.asciidoc index 1f9fc53d66d9f..f05f97107a0a4 100644 --- a/docs/reference/query-dsl/dis-max-query.asciidoc +++ b/docs/reference/query-dsl/dis-max-query.asciidoc @@ -20,7 +20,7 @@ of these DisjunctionMaxQuery's is combined into a BooleanQuery. The tie breaker capability allows results that include the same term in multiple fields to be judged better than results that include this term in only the best of those multiple fields, without confusing this with -the better case of two different terms in the multiple fields.The +the better case of two different terms in the multiple fields. The default `tie_breaker` is `0.0`. This query maps to Lucene `DisjunctionMaxQuery`. diff --git a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java index b4a6c49754869..bbc6a64dcdb2b 100644 --- a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java +++ b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java @@ -19,11 +19,12 @@ package org.elasticsearch.smoketest; -import org.apache.http.HttpHost; -import org.apache.lucene.util.BytesRef; - import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; +import org.apache.http.HttpHost; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.TimeUnits; import org.elasticsearch.Version; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.ParseField; @@ -48,12 +49,13 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; - import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +//The default 20 minutes timeout isn't always enough, please do not increase further than 30 before analyzing what makes this suite so slow +@TimeoutSuite(millis = 30 * TimeUnits.MINUTE) public class DocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { public DocsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/qa/multi-cluster-search/build.gradle b/qa/multi-cluster-search/build.gradle index 6942331c97c25..0835945499d34 100644 --- a/qa/multi-cluster-search/build.gradle +++ b/qa/multi-cluster-search/build.gradle @@ -21,6 +21,10 @@ import org.elasticsearch.gradle.test.RestIntegTestTask apply plugin: 'elasticsearch.standalone-test' +dependencies { + testCompile "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}" +} + task remoteClusterTest(type: RestIntegTestTask) { mustRunAfter(precommit) } @@ -53,6 +57,6 @@ task integTest { dependsOn = [mixedClusterTest] } -unitTest.enabled = false // no unit tests for multi-cluster-search, only the rest integration test +unitTest.enabled = false // no unit tests for multi-cluster-search, only integration tests check.dependsOn(integTest) diff --git a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java new file mode 100644 index 0000000000000..4a18ddbe1b696 --- /dev/null +++ b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java @@ -0,0 +1,847 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search; + +import com.carrotsearch.randomizedtesting.RandomizedContext; +import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; +import org.apache.lucene.search.join.ScoreMode; +import org.apache.lucene.util.TimeUnits; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.LatchedActionListener; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.bulk.BulkProcessor; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.index.query.InnerHitBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.indices.TermsLookup; +import org.elasticsearch.join.query.HasChildQueryBuilder; +import org.elasticsearch.join.query.HasParentQueryBuilder; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.support.ValueType; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.collapse.CollapseBuilder; +import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.elasticsearch.search.rescore.QueryRescoreMode; +import org.elasticsearch.search.rescore.QueryRescorerBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; +import org.elasticsearch.search.suggest.phrase.DirectCandidateGeneratorBuilder; +import org.elasticsearch.search.suggest.phrase.PhraseSuggestion; +import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder; +import org.elasticsearch.search.suggest.term.TermSuggestion; +import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; +import org.elasticsearch.test.NotEqualMessageBuilder; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.AfterClass; +import org.junit.Before; + +import java.io.IOException; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +/** + * This test class executes twice, first against the remote cluster, and then against another cluster that has the remote cluster + * registered. Given that each test gets executed against both clusters, {@link #assumeMultiClusterSetup()} needs to be used to run a test + * against the multi cluster setup only, which is required for testing cross-cluster search. + * The goal of this test is not to test correctness of CCS responses, but rather to verify that CCS returns the same responses when + * minimizeRoundTrips is set to either true or false. In fact the execution differs depending on + * such parameter, hence we want to verify that results are the same in both scenarios. + */ +@TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs +public class CCSDuelIT extends ESRestTestCase { + + private static final String INDEX_NAME = "ccs_duel_index"; + private static final String REMOTE_INDEX_NAME = "my_remote_cluster:" + INDEX_NAME; + private static final String[] TAGS = new String[]{"java", "xml", "sql", "html", "php", "ruby", "python", "perl"}; + + private static RestHighLevelClient restHighLevelClient; + + @Before + public void init() throws Exception { + super.initClient(); + if (restHighLevelClient == null) { + restHighLevelClient = new HighLevelClient(client()); + String destinationCluster = System.getProperty("tests.rest.suite"); + //we index docs with private randomness otherwise the two clusters end up with exactly the same documents + //given that this test class is run twice with same seed. + RandomizedContext.current().runWithPrivateRandomness(random().nextLong() + destinationCluster.hashCode(), + (Callable) () -> { + indexDocuments(destinationCluster + "-"); + return null; + }); + } + } + + private static class HighLevelClient extends RestHighLevelClient { + private HighLevelClient(RestClient restClient) { + super(restClient, (client) -> {}, Collections.emptyList()); + } + } + + @AfterClass + public static void cleanupClient() throws IOException { + IOUtils.close(restHighLevelClient); + restHighLevelClient = null; + } + + @Override + protected boolean preserveIndicesUponCompletion() { + return true; + } + + private static void indexDocuments(String idPrefix) throws IOException, InterruptedException { + //this index with a single document is used to test partial failures + IndexRequest indexRequest = new IndexRequest(INDEX_NAME + "_err"); + indexRequest.id("id"); + indexRequest.source("creationDate", "err"); + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); + IndexResponse indexResponse = restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT); + assertEquals(201, indexResponse.status().getStatus()); + + CreateIndexRequest createEmptyIndexRequest = new CreateIndexRequest(INDEX_NAME + "_empty"); + CreateIndexResponse response = restHighLevelClient.indices().create(createEmptyIndexRequest, RequestOptions.DEFAULT); + assertTrue(response.isAcknowledged()); + + int numShards = randomIntBetween(1, 5); + CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME); + createIndexRequest.settings(Settings.builder().put("index.number_of_shards", numShards).put("index.number_of_replicas", 0)); + createIndexRequest.mapping("{\"properties\":{" + + "\"suggest\":{\"type\":\"completion\"}," + + "\"join\":{\"type\":\"join\", \"relations\": {\"question\":\"answer\"}}}}", XContentType.JSON); + CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT); + assertTrue(createIndexResponse.isAcknowledged()); + + BulkProcessor bulkProcessor = BulkProcessor.builder((r, l) -> restHighLevelClient.bulkAsync(r, RequestOptions.DEFAULT, l), + new BulkProcessor.Listener() { + @Override + public void beforeBulk(long executionId, BulkRequest request) { + } + + @Override + public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + assertFalse(response.hasFailures()); + } + + @Override + public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + throw new AssertionError("Failed to execute bulk", failure); + } + }).build(); + + int numQuestions = randomIntBetween(50, 100); + for (int i = 0; i < numQuestions; i++) { + bulkProcessor.add(buildIndexRequest(idPrefix + i, "question", null)); + } + int numAnswers = randomIntBetween(100, 150); + for (int i = 0; i < numAnswers; i++) { + bulkProcessor.add(buildIndexRequest(idPrefix + (i + 1000), "answer", idPrefix + randomIntBetween(0, numQuestions - 1))); + } + assertTrue(bulkProcessor.awaitClose(30, TimeUnit.SECONDS)); + + RefreshResponse refreshResponse = restHighLevelClient.indices().refresh(new RefreshRequest(INDEX_NAME), RequestOptions.DEFAULT); + assertEquals(0, refreshResponse.getFailedShards()); + assertEquals(numShards, refreshResponse.getSuccessfulShards()); + } + + private static IndexRequest buildIndexRequest(String id, String type, String questionId) { + IndexRequest indexRequest = new IndexRequest(INDEX_NAME); + indexRequest.id(id); + if (questionId != null) { + indexRequest.routing(questionId); + } + indexRequest.create(true); + int numTags = randomIntBetween(1, 3); + Set tags = new HashSet<>(); + if (questionId == null) { + for (int i = 0; i < numTags; i++) { + tags.add(randomFrom(TAGS)); + } + } + String[] tagsArray = tags.toArray(new String[0]); + String date = LocalDate.of(2019, 1, randomIntBetween(1, 31)).format(DateTimeFormatter.ofPattern("yyyy/MM/dd", Locale.ROOT)); + Map joinField = new HashMap<>(); + joinField.put("name", type); + if (questionId != null) { + joinField.put("parent", questionId); + } + indexRequest.source(XContentType.JSON, + "type", type, + "votes", randomIntBetween(0, 30), + "questionId", questionId, + "tags", tagsArray, + "user", "user" + randomIntBetween(1, 10), + "suggest", Collections.singletonMap("input", tagsArray), + "creationDate", date, + "join", joinField); + return indexRequest; + } + + public void testMatchAll() throws Exception { + assumeMultiClusterSetup(); + //verify that the order in which documents are returned when they all have the same score is the same + SearchRequest searchRequest = initSearchRequest(); + duelSearch(searchRequest, CCSDuelIT::assertHits); + } + + public void testMatchQuery() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.size(50); + sourceBuilder.query(QueryBuilders.matchQuery("tags", "php")); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, CCSDuelIT::assertHits); + } + + public void testTrackTotalHitsUpTo() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.trackTotalHitsUpTo(5); + sourceBuilder.query(QueryBuilders.matchQuery("tags", "sql")); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, CCSDuelIT::assertHits); + } + + public void testTerminateAfter() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.terminateAfter(10); + sourceBuilder.query(QueryBuilders.matchQuery("tags", "perl")); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, CCSDuelIT::assertHits); + } + + public void testPagination() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.from(10); + sourceBuilder.size(20); + sourceBuilder.query(QueryBuilders.matchQuery("tags", "python")); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, response -> assertHits(response, 10)); + } + + public void testHighlighting() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.highlighter(new HighlightBuilder().field("tags")); + sourceBuilder.query(QueryBuilders.matchQuery("tags", "xml")); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, response -> { + assertHits(response); + assertFalse(response.getHits().getHits()[0].getHighlightFields().isEmpty()); + }); + } + + public void testFetchSource() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.fetchSource(new String[]{"tags"}, Strings.EMPTY_ARRAY); + sourceBuilder.query(QueryBuilders.matchQuery("tags", "ruby")); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, response -> { + assertHits(response); + assertEquals(1, response.getHits().getHits()[0].getSourceAsMap().size()); + }); + } + + public void testDocValueFields() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.docValueField("user.keyword"); + sourceBuilder.query(QueryBuilders.matchQuery("tags", "xml")); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, response -> { + assertHits(response); + assertEquals(1, response.getHits().getHits()[0].getFields().size()); + assertNotNull(response.getHits().getHits()[0].getFields().get("user.keyword")); + }); + } + + public void testScriptFields() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.scriptField("parent", new Script(ScriptType.INLINE, "painless", "doc['join#question']", Collections.emptyMap())); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, response -> { + assertHits(response); + assertEquals(1, response.getHits().getHits()[0].getFields().size()); + assertNotNull(response.getHits().getHits()[0].getFields().get("parent")); + }); + } + + public void testExplain() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.explain(true); + sourceBuilder.query(QueryBuilders.matchQuery("tags", "sql")); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, response -> { + assertHits(response); + assertNotNull(response.getHits().getHits()[0].getExplanation()); + }); + } + + public void testRescore() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.query(QueryBuilders.matchQuery("tags", "xml")); + QueryRescorerBuilder rescorerBuilder = new QueryRescorerBuilder(new MatchQueryBuilder("tags", "java")); + rescorerBuilder.setScoreMode(QueryRescoreMode.Multiply); + rescorerBuilder.setRescoreQueryWeight(5); + sourceBuilder.addRescorer(rescorerBuilder); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, CCSDuelIT::assertHits); + } + + public void testHasParentWithInnerHit() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + HasParentQueryBuilder hasParentQueryBuilder = new HasParentQueryBuilder("question", QueryBuilders.matchQuery("tags", "xml"), true); + hasParentQueryBuilder.innerHit(new InnerHitBuilder("inner")); + sourceBuilder.query(hasParentQueryBuilder); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, CCSDuelIT::assertHits); + } + + public void testHasChildWithInnerHit() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder("creationDate").gte("2019/01/01").lte("2019/01/31"); + HasChildQueryBuilder query = new HasChildQueryBuilder("answer", rangeQueryBuilder, ScoreMode.Total); + query.innerHit(new InnerHitBuilder("inner")); + sourceBuilder.query(query); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, CCSDuelIT::assertHits); + } + + public void testProfile() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.profile(true); + sourceBuilder.query(QueryBuilders.matchQuery("tags", "html")); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, response -> { + assertHits(response); + assertFalse(response.getProfileResults().isEmpty()); + }); + } + + public void testSortByField() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.from(30); + sourceBuilder.size(25); + sourceBuilder.query(QueryBuilders.matchQuery("tags", "php")); + sourceBuilder.sort("type.keyword", SortOrder.ASC); + sourceBuilder.sort("creationDate", SortOrder.DESC); + sourceBuilder.sort("user.keyword", SortOrder.ASC); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, response -> { + assertHits(response, 30); + if (response.getHits().getTotalHits().value > 30) { + assertEquals(3, response.getHits().getHits()[0].getSortValues().length); + } + }); + } + + public void testSortByFieldOneClusterHasNoResults() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + boolean onlyRemote = randomBoolean(); + sourceBuilder.query(new TermQueryBuilder("_index", onlyRemote ? REMOTE_INDEX_NAME : INDEX_NAME)); + sourceBuilder.sort("type.keyword", SortOrder.ASC); + sourceBuilder.sort("creationDate", SortOrder.DESC); + sourceBuilder.sort("user.keyword", SortOrder.ASC); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, response -> { + assertHits(response); + SearchHit[] hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + assertEquals(3, hit.getSortValues().length); + assertEquals(INDEX_NAME, hit.getIndex()); + if (onlyRemote) { + assertEquals("my_remote_cluster", hit.getClusterAlias()); + } else { + assertNull(hit.getClusterAlias()); + } + } + }); + } + + public void testFieldCollapsingOneClusterHasNoResults() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + boolean onlyRemote = randomBoolean(); + sourceBuilder.query(new TermQueryBuilder("_index", onlyRemote ? REMOTE_INDEX_NAME : INDEX_NAME)); + sourceBuilder.collapse(new CollapseBuilder("user.keyword")); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, response -> { + assertHits(response); + for (SearchHit hit : response.getHits().getHits()) { + assertEquals(INDEX_NAME, hit.getIndex()); + if (onlyRemote) { + assertEquals("my_remote_cluster", hit.getClusterAlias()); + } else { + assertNull(hit.getClusterAlias()); + } + } + }); + } + + public void testFieldCollapsingSortByScore() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + searchRequest.source(sourceBuilder); + sourceBuilder.query(QueryBuilders.matchQuery("tags", "ruby")); + sourceBuilder.collapse(new CollapseBuilder("user.keyword")); + duelSearch(searchRequest, CCSDuelIT::assertHits); + } + + public void testFieldCollapsingSortByField() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + searchRequest.source(sourceBuilder); + sourceBuilder.query(QueryBuilders.matchQuery("tags", "ruby")); + sourceBuilder.sort("creationDate", SortOrder.DESC); + sourceBuilder.sort(new ScoreSortBuilder()); + sourceBuilder.collapse(new CollapseBuilder("user.keyword")); + duelSearch(searchRequest, response -> { + assertHits(response); + assertEquals(2, response.getHits().getHits()[0].getSortValues().length); + }); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/40005") + public void testTermsAggs() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + searchRequest.source(buildTermsAggsSource()); + duelSearch(searchRequest, CCSDuelIT::assertAggs); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/40005") + public void testTermsAggsWithProfile() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + searchRequest.source(buildTermsAggsSource().profile(true)); + duelSearch(searchRequest, CCSDuelIT::assertAggs); + } + + private static SearchSourceBuilder buildTermsAggsSource() { + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.size(0); + TermsAggregationBuilder cluster = new TermsAggregationBuilder("cluster123", ValueType.STRING); + cluster.field("_index"); + TermsAggregationBuilder type = new TermsAggregationBuilder("type", ValueType.STRING); + type.field("type.keyword"); + type.showTermDocCountError(true); + type.order(BucketOrder.key(true)); + cluster.subAggregation(type); + sourceBuilder.aggregation(cluster); + + TermsAggregationBuilder tags = new TermsAggregationBuilder("tags", ValueType.STRING); + tags.field("tags.keyword"); + tags.showTermDocCountError(true); + tags.size(100); + sourceBuilder.aggregation(tags); + + TermsAggregationBuilder tags2 = new TermsAggregationBuilder("tags", ValueType.STRING); + tags2.field("tags.keyword"); + tags.subAggregation(tags2); + + FilterAggregationBuilder answers = new FilterAggregationBuilder("answers", new TermQueryBuilder("type", "answer")); + TermsAggregationBuilder answerPerQuestion = new TermsAggregationBuilder("answer_per_question", ValueType.STRING); + answerPerQuestion.showTermDocCountError(true); + answerPerQuestion.field("questionId.keyword"); + answers.subAggregation(answerPerQuestion); + TermsAggregationBuilder answerPerUser = new TermsAggregationBuilder("answer_per_user", ValueType.STRING); + answerPerUser.field("user.keyword"); + answerPerUser.size(30); + answerPerUser.showTermDocCountError(true); + answers.subAggregation(answerPerUser); + sourceBuilder.aggregation(answers); + return sourceBuilder; + } + + public void testDateHistogram() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.size(0); + searchRequest.source(sourceBuilder); + TermsAggregationBuilder tags = new TermsAggregationBuilder("tags", ValueType.STRING); + tags.field("tags.keyword"); + tags.showTermDocCountError(true); + DateHistogramAggregationBuilder creation = new DateHistogramAggregationBuilder("creation"); + creation.field("creationDate"); + creation.dateHistogramInterval(DateHistogramInterval.QUARTER); + creation.subAggregation(tags); + sourceBuilder.aggregation(creation); + duelSearch(searchRequest, CCSDuelIT::assertAggs); + } + + public void testCardinalityAgg() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.size(0); + searchRequest.source(sourceBuilder); + CardinalityAggregationBuilder tags = new CardinalityAggregationBuilder("tags", ValueType.STRING); + tags.field("tags.keyword"); + sourceBuilder.aggregation(tags); + duelSearch(searchRequest, CCSDuelIT::assertAggs); + } + + public void testPipelineAggs() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.query(new TermQueryBuilder("type", "answer")); + searchRequest.source(sourceBuilder); + sourceBuilder.size(0); + DateHistogramAggregationBuilder daily = new DateHistogramAggregationBuilder("daily"); + daily.field("creationDate"); + daily.dateHistogramInterval(DateHistogramInterval.DAY); + sourceBuilder.aggregation(daily); + daily.subAggregation(new DerivativePipelineAggregationBuilder("derivative", "_count")); + sourceBuilder.aggregation(new MaxBucketPipelineAggregationBuilder("biggest_day", "daily._count")); + daily.subAggregation(new SumAggregationBuilder("votes").field("votes")); + sourceBuilder.aggregation(new MaxBucketPipelineAggregationBuilder("most_voted", "daily>votes")); + duelSearch(searchRequest, response -> { + assertAggs(response); + assertNotNull(response.getAggregations().get("most_voted")); + }); + duelSearch(searchRequest, CCSDuelIT::assertAggs); + } + + public void testTopHits() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + searchRequest.source(sourceBuilder); + sourceBuilder.size(0); + TopHitsAggregationBuilder topHits = new TopHitsAggregationBuilder("top"); + topHits.from(10); + topHits.size(10); + topHits.sort("creationDate", SortOrder.DESC); + topHits.sort("_id", SortOrder.ASC); + TermsAggregationBuilder tags = new TermsAggregationBuilder("tags", ValueType.STRING); + tags.field("tags.keyword"); + tags.size(10); + tags.subAggregation(topHits); + sourceBuilder.aggregation(tags); + duelSearch(searchRequest, CCSDuelIT::assertAggs); + } + + public void testTermsLookup() throws Exception { + assumeMultiClusterSetup(); + IndexRequest indexRequest = new IndexRequest("lookup_index"); + indexRequest.id("id"); + indexRequest.source("tags", new String[]{"java", "sql", "html", "jax-ws"}); + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); + IndexResponse indexResponse = restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT); + assertEquals(201, indexResponse.status().getStatus()); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + TermsQueryBuilder termsQueryBuilder = new TermsQueryBuilder("tags", new TermsLookup("lookup_index", "id", "tags")); + sourceBuilder.query(termsQueryBuilder); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, CCSDuelIT::assertHits); + } + + public void testShardFailures() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = new SearchRequest(INDEX_NAME + "*", REMOTE_INDEX_NAME + "*"); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.query(QueryBuilders.matchQuery("creationDate", "err")); + searchRequest.source(sourceBuilder); + duelSearch(searchRequest, response -> { + assertMultiClusterSearchResponse(response); + assertThat(response.getHits().getTotalHits().value, greaterThan(0L)); + assertNull(response.getAggregations()); + assertNull(response.getSuggest()); + assertThat(response.getHits().getHits().length, greaterThan(0)); + assertThat(response.getFailedShards(), greaterThanOrEqualTo(2)); + }); + } + + public void testTermSuggester() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + searchRequest.source(sourceBuilder); + SuggestBuilder suggestBuilder = new SuggestBuilder(); + suggestBuilder.setGlobalText("jva hml"); + suggestBuilder.addSuggestion("tags", new TermSuggestionBuilder("tags") + .suggestMode(TermSuggestionBuilder.SuggestMode.POPULAR)); + sourceBuilder.suggest(suggestBuilder); + duelSearch(searchRequest, response -> { + assertMultiClusterSearchResponse(response); + assertEquals(1, response.getSuggest().size()); + TermSuggestion tags = response.getSuggest().getSuggestion("tags"); + assertThat(tags.getEntries().size(), greaterThan(0)); + }); + } + + public void testPhraseSuggester() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + searchRequest.source(sourceBuilder); + SuggestBuilder suggestBuilder = new SuggestBuilder(); + suggestBuilder.setGlobalText("jva and hml"); + suggestBuilder.addSuggestion("tags", new PhraseSuggestionBuilder("tags").addCandidateGenerator( + new DirectCandidateGeneratorBuilder("tags").suggestMode("always")).highlight("", "")); + sourceBuilder.suggest(suggestBuilder); + duelSearch(searchRequest, response -> { + assertMultiClusterSearchResponse(response); + assertEquals(1, response.getSuggest().size()); + PhraseSuggestion tags = response.getSuggest().getSuggestion("tags"); + assertThat(tags.getEntries().size(), greaterThan(0)); + }); + } + + public void testCompletionSuggester() throws Exception { + assumeMultiClusterSetup(); + SearchRequest searchRequest = initSearchRequest(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + searchRequest.source(sourceBuilder); + SuggestBuilder suggestBuilder = new SuggestBuilder(); + suggestBuilder.addSuggestion("python", new CompletionSuggestionBuilder("suggest").size(10).text("pyth")); + suggestBuilder.addSuggestion("java", new CompletionSuggestionBuilder("suggest").size(20).text("jav")); + suggestBuilder.addSuggestion("ruby", new CompletionSuggestionBuilder("suggest").size(30).text("rub")); + sourceBuilder.suggest(suggestBuilder); + duelSearch(searchRequest, response -> { + assertMultiClusterSearchResponse(response); + assertEquals(Strings.toString(response, true, true), 3, response.getSuggest().size()); + assertThat(response.getSuggest().getSuggestion("python").getEntries().size(), greaterThan(0)); + assertThat(response.getSuggest().getSuggestion("java").getEntries().size(), greaterThan(0)); + assertThat(response.getSuggest().getSuggestion("ruby").getEntries().size(), greaterThan(0)); + }); + } + + private static void assumeMultiClusterSetup() { + assumeTrue("must run only against the multi_cluster setup", "multi_cluster".equals(System.getProperty("tests.rest.suite"))); + } + + private static SearchRequest initSearchRequest() { + List indices = Arrays.asList(INDEX_NAME, "my_remote_cluster:" + INDEX_NAME); + Collections.shuffle(indices, random()); + return new SearchRequest(indices.toArray(new String[0])); + } + + private static void duelSearch(SearchRequest searchRequest, Consumer responseChecker) throws Exception { + CountDownLatch latch = new CountDownLatch(2); + AtomicReference exception1 = new AtomicReference<>(); + AtomicReference minimizeRoundtripsResponse = new AtomicReference<>(); + searchRequest.setCcsMinimizeRoundtrips(true); + restHighLevelClient.searchAsync(searchRequest, RequestOptions.DEFAULT, + new LatchedActionListener<>(ActionListener.wrap(minimizeRoundtripsResponse::set, exception1::set), latch)); + + AtomicReference exception2 = new AtomicReference<>(); + AtomicReference fanOutResponse = new AtomicReference<>(); + searchRequest.setCcsMinimizeRoundtrips(false); + restHighLevelClient.searchAsync(searchRequest, RequestOptions.DEFAULT, + new LatchedActionListener<>(ActionListener.wrap(fanOutResponse::set, exception2::set), latch)); + + latch.await(); + + if (exception1.get() != null && exception2.get() != null) { + exception1.get().addSuppressed(exception2.get()); + throw new AssertionError("both requests returned an exception", exception1.get()); + } else { + if (exception1.get() != null) { + throw new AssertionError("one of the two requests returned an exception", exception1.get()); + } + if (exception2.get() != null) { + throw new AssertionError("one of the two requests returned an exception", exception2.get()); + } + SearchResponse minimizeRoundtripsSearchResponse = minimizeRoundtripsResponse.get(); + responseChecker.accept(minimizeRoundtripsSearchResponse); + assertEquals(3, minimizeRoundtripsSearchResponse.getNumReducePhases()); + SearchResponse fanOutSearchResponse = fanOutResponse.get(); + responseChecker.accept(fanOutSearchResponse); + assertEquals(1, fanOutSearchResponse.getNumReducePhases()); + Map minimizeRoundtripsResponseMap = responseToMap(minimizeRoundtripsSearchResponse); + Map fanOutResponseMap = responseToMap(fanOutSearchResponse); + if (minimizeRoundtripsResponseMap.equals(fanOutResponseMap) == false) { + NotEqualMessageBuilder message = new NotEqualMessageBuilder(); + message.compareMaps(minimizeRoundtripsResponseMap, fanOutResponseMap); + throw new AssertionError("Didn't match expected value:\n" + message); + } + } + } + + private static void assertMultiClusterSearchResponse(SearchResponse searchResponse) { + assertEquals(2, searchResponse.getClusters().getTotal()); + assertEquals(2, searchResponse.getClusters().getSuccessful()); + assertThat(searchResponse.getTotalShards(), greaterThan(1)); + assertThat(searchResponse.getSuccessfulShards(), greaterThan(1)); + } + + private static void assertHits(SearchResponse response) { + assertHits(response, 0); + } + + private static void assertHits(SearchResponse response, int from) { + assertMultiClusterSearchResponse(response); + assertThat(response.getHits().getTotalHits().value, greaterThan(0L)); + assertEquals(0, response.getFailedShards()); + assertNull(response.getAggregations()); + assertNull(response.getSuggest()); + if (response.getHits().getTotalHits().value > from) { + assertThat(response.getHits().getHits().length, greaterThan(0)); + } else { + assertThat(response.getHits().getHits().length, equalTo(0)); + } + } + + private static void assertAggs(SearchResponse response) { + assertMultiClusterSearchResponse(response); + assertThat(response.getHits().getTotalHits().value, greaterThan(0L)); + assertEquals(0, response.getHits().getHits().length); + assertNull(response.getSuggest()); + assertNotNull(response.getAggregations()); + List aggregations = response.getAggregations().asList(); + for (Aggregation aggregation : aggregations) { + if (aggregation instanceof MultiBucketsAggregation) { + MultiBucketsAggregation multiBucketsAggregation = (MultiBucketsAggregation) aggregation; + assertThat("agg " + multiBucketsAggregation.getName() + " has 0 buckets", + multiBucketsAggregation.getBuckets().size(), greaterThan(0)); + } + } + } + + @SuppressWarnings("unchecked") + private static Map responseToMap(SearchResponse response) throws IOException { + BytesReference bytesReference = XContentHelper.toXContent(response, XContentType.JSON, false); + Map responseMap = XContentHelper.convertToMap(bytesReference, false, XContentType.JSON).v2(); + assertNotNull(responseMap.put("took", -1)); + responseMap.remove("num_reduce_phases"); + Map profile = (Map)responseMap.get("profile"); + if (profile != null) { + List> shards = (List >)profile.get("shards"); + for (Map shard : shards) { + replaceProfileTime(shard); + } + } + return responseMap; + } + + @SuppressWarnings("unchecked") + private static void replaceProfileTime(Map map) { + for (Map.Entry entry : map.entrySet()) { + if (entry.getKey().contains("time")) { + assertThat(entry.getValue(), instanceOf(Number.class)); + assertNotNull(entry.setValue(-1)); + } + if (entry.getKey().equals("breakdown")) { + Map breakdown = (Map) entry.getValue(); + for (String key : breakdown.keySet()) { + assertNotNull(breakdown.put(key, -1L)); + } + } + if (entry.getValue() instanceof Map) { + replaceProfileTime((Map) entry.getValue()); + } + if (entry.getValue() instanceof List) { + List list = (List) entry.getValue(); + for (Object obj : list) { + if (obj instanceof Map) { + replaceProfileTime((Map) obj); + } + } + } + } + } +} diff --git a/qa/multi-cluster-search/src/test/java/org/elasticsearch/upgrades/MultiClusterSearchYamlTestSuiteIT.java b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/MultiClusterSearchYamlTestSuiteIT.java similarity index 97% rename from qa/multi-cluster-search/src/test/java/org/elasticsearch/upgrades/MultiClusterSearchYamlTestSuiteIT.java rename to qa/multi-cluster-search/src/test/java/org/elasticsearch/search/MultiClusterSearchYamlTestSuiteIT.java index fe3a909883181..eb4f9a8e6a916 100644 --- a/qa/multi-cluster-search/src/test/java/org/elasticsearch/upgrades/MultiClusterSearchYamlTestSuiteIT.java +++ b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/MultiClusterSearchYamlTestSuiteIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.search; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; @@ -42,5 +42,4 @@ public MultiClusterSearchYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t public static Iterable parameters() throws Exception { return createParameters(); } - } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index f54f101041d1b..0125084c37099 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -486,7 +486,7 @@ private ReducedQueryPhase reducedQueryPhase(Collection pipelineAggregators = context.aggregations().factories().createPipelineAggregators(); - List siblingPipelineAggregators = new ArrayList<>(pipelineAggregators.size()); - for (PipelineAggregator pipelineAggregator : pipelineAggregators) { - if (pipelineAggregator instanceof SiblingPipelineAggregator) { - siblingPipelineAggregators.add((SiblingPipelineAggregator) pipelineAggregator); - } else { - throw new AggregationExecutionException("Invalid pipeline aggregation named [" + pipelineAggregator.name() - + "] of type [" + pipelineAggregator.getWriteableName() + "]. Only sibling pipeline aggregations are " - + "allowed at the top level"); - } + List pipelineAggregators = context.aggregations().factories().createPipelineAggregators(); + List siblingPipelineAggregators = new ArrayList<>(pipelineAggregators.size()); + for (PipelineAggregator pipelineAggregator : pipelineAggregators) { + if (pipelineAggregator instanceof SiblingPipelineAggregator) { + siblingPipelineAggregators.add((SiblingPipelineAggregator) pipelineAggregator); + } else { + throw new AggregationExecutionException("Invalid pipeline aggregation named [" + pipelineAggregator.name() + + "] of type [" + pipelineAggregator.getWriteableName() + "]. Only sibling pipeline aggregations are " + + "allowed at the top level"); } - context.queryResult().pipelineAggregators(siblingPipelineAggregators); - } catch (IOException e) { - throw new AggregationExecutionException("Failed to build top level pipeline aggregators", e); } + context.queryResult().aggregations(new InternalAggregations(aggregations, siblingPipelineAggregators)); // disable aggregations so that they don't run on next pages in case of scrolling context.aggregations(null); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java index 9683651391cc2..5c1120452f6c3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java @@ -175,7 +175,7 @@ private AggregatorFactories(AggregatorFactory[] factories, List createPipelineAggregators() throws IOException { + public List createPipelineAggregators() { List pipelineAggregators = new ArrayList<>(this.pipelineAggregatorFactories.size()); for (PipelineAggregationBuilder factory : this.pipelineAggregatorFactories) { pipelineAggregators.add(factory.create()); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java index 70135c2d51e73..08c389675ad72 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java @@ -78,7 +78,7 @@ public InternalAggregations(List aggregations, List getTopLevelPipelineAggregators() { + public List getTopLevelPipelineAggregators() { return topLevelPipelineAggregators; } @@ -92,20 +92,7 @@ public static InternalAggregations reduce(List aggregation if (aggregationsList.isEmpty()) { return null; } - InternalAggregations first = aggregationsList.get(0); - return reduce(aggregationsList, first.topLevelPipelineAggregators, context); - } - - /** - * Reduces the given list of aggregations as well as the provided top-level pipeline aggregators. - * Note that top-level pipeline aggregators are reduced only as part of the final reduction phase, otherwise they are left untouched. - */ - public static InternalAggregations reduce(List aggregationsList, - List topLevelPipelineAggregators, - ReduceContext context) { - if (aggregationsList.isEmpty()) { - return null; - } + List topLevelPipelineAggregators = aggregationsList.get(0).getTopLevelPipelineAggregators(); // first we collect all aggregations of the same type and list them together Map> aggByName = new HashMap<>(); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java index 8b66738848fda..08938a5b9b9fe 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java @@ -24,7 +24,6 @@ import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import java.io.IOException; import java.util.Collection; import java.util.Map; @@ -76,7 +75,7 @@ protected abstract void validate(AggregatorFactory parent, Collection parent, Collection metaData) throws IOException; + protected abstract PipelineAggregator createInternal(Map metaData); /** * Creates the pipeline aggregator @@ -97,7 +97,7 @@ public final void validate(AggregatorFactory parent, Collection metaData) throws IOException { + protected PipelineAggregator createInternal(Map metaData) { return new AvgBucketPipelineAggregator(name, bucketsPaths, gapPolicy(), formatter(), metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java index 27da9dea53099..eddc48c6fdcba 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java @@ -104,7 +104,7 @@ public GapPolicy gapPolicy() { } @Override - protected abstract PipelineAggregator createInternal(Map metaData) throws IOException; + protected abstract PipelineAggregator createInternal(Map metaData); @Override public void doValidate(AggregatorFactory parent, Collection aggBuilders, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java index db56779559a40..c8ea3553752f8 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java @@ -139,7 +139,7 @@ public GapPolicy gapPolicy() { } @Override - protected PipelineAggregator createInternal(Map metaData) throws IOException { + protected PipelineAggregator createInternal(Map metaData) { return new BucketScriptPipelineAggregator(name, bucketsPathsMap, script, formatter(), gapPolicy, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java index f0497932b21c5..a6627be1cfe6f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java @@ -108,7 +108,7 @@ public GapPolicy gapPolicy() { } @Override - protected PipelineAggregator createInternal(Map metaData) throws IOException { + protected PipelineAggregator createInternal(Map metaData) { return new BucketSelectorPipelineAggregator(name, bucketsPathsMap, script, gapPolicy, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java index 0ce4c08720649..4dcd42934fc96 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java @@ -139,7 +139,7 @@ public BucketSortPipelineAggregationBuilder gapPolicy(GapPolicy gapPolicy) { } @Override - protected PipelineAggregator createInternal(Map metaData) throws IOException { + protected PipelineAggregator createInternal(Map metaData) { return new BucketSortPipelineAggregator(name, sorts, from, size, gapPolicy, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java index 207b1e35bef7a..a8b51d9c1a5fe 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java @@ -88,7 +88,7 @@ protected DocValueFormat formatter() { } @Override - protected PipelineAggregator createInternal(Map metaData) throws IOException { + protected PipelineAggregator createInternal(Map metaData) { return new CumulativeSumPipelineAggregator(name, bucketsPaths, formatter(), metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java index 68ec9085df52a..25f30bc8343cb 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java @@ -129,7 +129,7 @@ public String unit() { } @Override - protected PipelineAggregator createInternal(Map metaData) throws IOException { + protected PipelineAggregator createInternal(Map metaData) { DocValueFormat formatter; if (format != null) { formatter = new DocValueFormat.Decimal(format); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java index 10347e40354a8..3d16cf91ee065 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java @@ -75,7 +75,7 @@ public double sigma() { } @Override - protected PipelineAggregator createInternal(Map metaData) throws IOException { + protected PipelineAggregator createInternal(Map metaData) { return new ExtendedStatsBucketPipelineAggregator(name, bucketsPaths, sigma, gapPolicy(), formatter(), metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java index 852a3e378d090..b335c15865d70 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java @@ -46,7 +46,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException { } @Override - protected PipelineAggregator createInternal(Map metaData) throws IOException { + protected PipelineAggregator createInternal(Map metaData) { return new MaxBucketPipelineAggregator(name, bucketsPaths, gapPolicy(), formatter(), metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java index b44ee869e2c4b..405285993c0ab 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java @@ -46,7 +46,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException { } @Override - protected PipelineAggregator createInternal(Map metaData) throws IOException { + protected PipelineAggregator createInternal(Map metaData) { return new MinBucketPipelineAggregator(name, bucketsPaths, gapPolicy(), formatter(), metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java index 02fa979563350..4a552f47d5359 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java @@ -250,7 +250,7 @@ public Boolean minimize() { } @Override - protected PipelineAggregator createInternal(Map metaData) throws IOException { + protected PipelineAggregator createInternal(Map metaData) { // If the user doesn't set a preference for cost minimization, ask // what the model prefers boolean minimize = this.minimize == null ? model.minimizeByDefault() : this.minimize; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java index 998f6e387a843..3f589d8a08949 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java @@ -179,7 +179,7 @@ public void doValidate(AggregatorFactory parent, Collection metaData) throws IOException { + protected PipelineAggregator createInternal(Map metaData) { return new MovFnPipelineAggregator(name, bucketsPathString, script, window, formatter(), gapPolicy, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java index a3ac220177716..31848d2159149 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java @@ -112,7 +112,7 @@ public boolean getKeyed() { } @Override - protected PipelineAggregator createInternal(Map metaData) throws IOException { + protected PipelineAggregator createInternal(Map metaData) { return new PercentilesBucketPipelineAggregator(name, percents, keyed, bucketsPaths, gapPolicy(), formatter(), metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java index f634493749da7..019026740f813 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java @@ -135,7 +135,7 @@ protected DocValueFormat formatter() { } @Override - protected PipelineAggregator createInternal(Map metaData) throws IOException { + protected PipelineAggregator createInternal(Map metaData) { return new SerialDiffPipelineAggregator(name, bucketsPaths, formatter(), gapPolicy, lag, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java index f943f3318fc84..904cc16c29076 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java @@ -47,7 +47,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException { } @Override - protected PipelineAggregator createInternal(Map metaData) throws IOException { + protected PipelineAggregator createInternal(Map metaData) { return new StatsBucketPipelineAggregator(name, bucketsPaths, gapPolicy(), formatter(), metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java index 920f7e9b0ac26..eb075d4368981 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java @@ -46,7 +46,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException { } @Override - protected PipelineAggregator createInternal(Map metaData) throws IOException { + protected PipelineAggregator createInternal(Map metaData) { return new SumBucketPipelineAggregator(name, bucketsPaths, gapPolicy(), formatter(), metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java index 55787dfc53a35..db8b67ca109db 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java @@ -29,6 +29,7 @@ import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator; @@ -38,7 +39,6 @@ import java.io.IOException; import java.util.Collections; import java.util.List; -import java.util.Objects; import java.util.stream.Collectors; import static org.elasticsearch.common.lucene.Lucene.readTopDocs; @@ -55,7 +55,6 @@ public final class QuerySearchResult extends SearchPhaseResult { private DocValueFormat[] sortValueFormats; private InternalAggregations aggregations; private boolean hasAggs; - private List pipelineAggregators = Collections.emptyList(); private Suggest suggest; private boolean searchTimedOut; private Boolean terminatedEarly = null; @@ -199,14 +198,6 @@ public void profileResults(ProfileShardResult shardResults) { hasProfileResults = shardResults != null; } - public List pipelineAggregators() { - return pipelineAggregators; - } - - public void pipelineAggregators(List pipelineAggregators) { - this.pipelineAggregators = Objects.requireNonNull(pipelineAggregators); - } - public Suggest suggest() { return suggest; } @@ -295,8 +286,18 @@ public void readFromWithId(long id, StreamInput in) throws IOException { if (hasAggs = in.readBoolean()) { aggregations = InternalAggregations.readAggregations(in); } - pipelineAggregators = in.readNamedWriteableList(PipelineAggregator.class).stream().map(a -> (SiblingPipelineAggregator) a) - .collect(Collectors.toList()); + if (in.getVersion().before(Version.V_7_1_0)) { + List pipelineAggregators = in.readNamedWriteableList(PipelineAggregator.class).stream() + .map(a -> (SiblingPipelineAggregator) a).collect(Collectors.toList()); + if (hasAggs && pipelineAggregators.isEmpty() == false) { + List internalAggs = aggregations.asList().stream() + .map(agg -> (InternalAggregation) agg).collect(Collectors.toList()); + //Earlier versions serialize sibling pipeline aggs separately as they used to be set to QuerySearchResult directly, while + //later versions include them in InternalAggregations. Note that despite serializing sibling pipeline aggs as part of + //InternalAggregations is supported since 6.7.0, the shards set sibling pipeline aggs to InternalAggregations only from 7.1. + this.aggregations = new InternalAggregations(internalAggs, pipelineAggregators); + } + } if (in.readBoolean()) { suggest = new Suggest(in); } @@ -338,7 +339,16 @@ public void writeToNoId(StreamOutput out) throws IOException { out.writeBoolean(true); aggregations.writeTo(out); } - out.writeNamedWriteableList(pipelineAggregators); + if (out.getVersion().before(Version.V_7_1_0)) { + //Earlier versions expect sibling pipeline aggs separately as they used to be set to QuerySearchResult directly, + //while later versions expect them in InternalAggregations. Note that despite serializing sibling pipeline aggs as part of + //InternalAggregations is supported since 6.7.0, the shards set sibling pipeline aggs to InternalAggregations only from 7.1 on. + if (aggregations == null) { + out.writeNamedWriteableList(Collections.emptyList()); + } else { + out.writeNamedWriteableList(aggregations.getTopLevelPipelineAggregators()); + } + } if (suggest == null) { out.writeBoolean(false); } else { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index 571843126f98c..228d05c51c462 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -580,7 +580,26 @@ public void testConcreteIndicesIgnoreIndicesEmptyRequest() { assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, new String[]{})), equalTo(newHashSet("kuku", "testXXX"))); } + public void testConcreteIndicesNoIndicesErrorMessage() { + MetaData.Builder mdBuilder = MetaData.builder(); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); + IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, + IndicesOptions.fromOptions(false, false, true, true)); + IndexNotFoundException infe = expectThrows(IndexNotFoundException.class, + () -> indexNameExpressionResolver.concreteIndices(context, new String[]{})); + assertThat(infe.getMessage(), is("no such index [null] and no indices exist")); + } + public void testConcreteIndicesNoIndicesErrorMessageNoExpand() { + MetaData.Builder mdBuilder = MetaData.builder(); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); + IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, + IndicesOptions.fromOptions(false, false, false, false)); + IndexNotFoundException infe = expectThrows(IndexNotFoundException.class, + () -> indexNameExpressionResolver.concreteIndices(context, new String[]{})); + assertThat(infe.getMessage(), is("no such index [_all] and no indices exist")); + } + public void testConcreteIndicesWildcardExpansion() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("testXXX").state(State.OPEN)) diff --git a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java index 2a54cda752a9d..3594022e2231c 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java @@ -442,7 +442,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { } @Override - protected PipelineAggregator createInternal(Map metaData) throws IOException { + protected PipelineAggregator createInternal(Map metaData) { return null; } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationsTests.java index 3212c18cf278f..0165a1b3ae0f8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationsTests.java @@ -50,23 +50,24 @@ public class InternalAggregationsTests extends ESTestCase { public void testReduceEmptyAggs() { List aggs = Collections.emptyList(); InternalAggregation.ReduceContext reduceContext = new InternalAggregation.ReduceContext(null, null, randomBoolean()); - assertNull(InternalAggregations.reduce(aggs, Collections.emptyList(), reduceContext)); + assertNull(InternalAggregations.reduce(aggs, reduceContext)); } - public void testNonFinalReduceTopLevelPipelineAggs() throws IOException { + public void testNonFinalReduceTopLevelPipelineAggs() { InternalAggregation terms = new StringTerms("name", BucketOrder.key(true), 10, 1, Collections.emptyList(), Collections.emptyMap(), DocValueFormat.RAW, 25, false, 10, Collections.emptyList(), 0); - List aggs = Collections.singletonList(new InternalAggregations(Collections.singletonList(terms))); List topLevelPipelineAggs = new ArrayList<>(); MaxBucketPipelineAggregationBuilder maxBucketPipelineAggregationBuilder = new MaxBucketPipelineAggregationBuilder("test", "test"); topLevelPipelineAggs.add((SiblingPipelineAggregator)maxBucketPipelineAggregationBuilder.create()); + List aggs = Collections.singletonList(new InternalAggregations(Collections.singletonList(terms), + topLevelPipelineAggs)); InternalAggregation.ReduceContext reduceContext = new InternalAggregation.ReduceContext(null, null, false); - InternalAggregations reducedAggs = InternalAggregations.reduce(aggs, topLevelPipelineAggs, reduceContext); + InternalAggregations reducedAggs = InternalAggregations.reduce(aggs, reduceContext); assertEquals(1, reducedAggs.getTopLevelPipelineAggregators().size()); assertEquals(1, reducedAggs.aggregations.size()); } - public void testFinalReduceTopLevelPipelineAggs() throws IOException { + public void testFinalReduceTopLevelPipelineAggs() { InternalAggregation terms = new StringTerms("name", BucketOrder.key(true), 10, 1, Collections.emptyList(), Collections.emptyMap(), DocValueFormat.RAW, 25, false, 10, Collections.emptyList(), 0); @@ -79,15 +80,15 @@ public void testFinalReduceTopLevelPipelineAggs() throws IOException { Collections.singletonList(siblingPipelineAggregator)); reducedAggs = InternalAggregations.reduce(Collections.singletonList(aggs), reduceContext); } else { - InternalAggregations aggs = new InternalAggregations(Collections.singletonList(terms)); - List topLevelPipelineAggs = Collections.singletonList(siblingPipelineAggregator); - reducedAggs = InternalAggregations.reduce(Collections.singletonList(aggs), topLevelPipelineAggs, reduceContext); + InternalAggregations aggs = new InternalAggregations(Collections.singletonList(terms), + Collections.singletonList(siblingPipelineAggregator)); + reducedAggs = InternalAggregations.reduce(Collections.singletonList(aggs), reduceContext); } assertEquals(0, reducedAggs.getTopLevelPipelineAggregators().size()); assertEquals(2, reducedAggs.aggregations.size()); } - public void testSerialization() throws Exception { + public static InternalAggregations createTestInstance() throws Exception { List aggsList = new ArrayList<>(); if (randomBoolean()) { StringTermsTests stringTermsTests = new StringTermsTests(); @@ -116,7 +117,11 @@ public void testSerialization() throws Exception { topLevelPipelineAggs.add((SiblingPipelineAggregator)new SumBucketPipelineAggregationBuilder("name3", "bucket3").create()); } } - InternalAggregations aggregations = new InternalAggregations(aggsList, topLevelPipelineAggs); + return new InternalAggregations(aggsList, topLevelPipelineAggs); + } + + public void testSerialization() throws Exception { + InternalAggregations aggregations = createTestInstance(); writeToAndReadFrom(aggregations, 0); } diff --git a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java new file mode 100644 index 0000000000000..29ee7c9238e9f --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java @@ -0,0 +1,134 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.query; + +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.Version; +import org.elasticsearch.action.OriginalIndices; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.SearchShardTarget; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.InternalAggregationsTests; +import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator; +import org.elasticsearch.search.suggest.SuggestTests; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; + +import java.io.IOException; +import java.util.Base64; +import java.util.List; + +import static java.util.Collections.emptyList; + +public class QuerySearchResultTests extends ESTestCase { + + private final NamedWriteableRegistry namedWriteableRegistry; + + public QuerySearchResultTests() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); + this.namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables()); + } + + private static QuerySearchResult createTestInstance() throws Exception { + ShardId shardId = new ShardId("index", "uuid", randomInt()); + QuerySearchResult result = new QuerySearchResult(randomLong(), new SearchShardTarget("node", shardId, null, OriginalIndices.NONE)); + if (randomBoolean()) { + result.terminatedEarly(randomBoolean()); + } + TopDocs topDocs = new TopDocs(new TotalHits(randomLongBetween(0, Long.MAX_VALUE), TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]); + result.topDocs(new TopDocsAndMaxScore(topDocs, randomBoolean() ? Float.NaN : randomFloat()), new DocValueFormat[0]); + result.size(randomInt()); + result.from(randomInt()); + if (randomBoolean()) { + result.suggest(SuggestTests.createTestItem()); + } + if (randomBoolean()) { + result.aggregations(InternalAggregationsTests.createTestInstance()); + } + return result; + } + + public void testSerialization() throws Exception { + QuerySearchResult querySearchResult = createTestInstance(); + Version version = VersionUtils.randomVersion(random()); + QuerySearchResult deserialized = copyStreamable(querySearchResult, namedWriteableRegistry, QuerySearchResult::new, version); + assertEquals(querySearchResult.getRequestId(), deserialized.getRequestId()); + assertNull(deserialized.getSearchShardTarget()); + assertEquals(querySearchResult.topDocs().maxScore, deserialized.topDocs().maxScore, 0f); + assertEquals(querySearchResult.topDocs().topDocs.totalHits, deserialized.topDocs().topDocs.totalHits); + assertEquals(querySearchResult.from(), deserialized.from()); + assertEquals(querySearchResult.size(), deserialized.size()); + assertEquals(querySearchResult.hasAggs(), deserialized.hasAggs()); + if (deserialized.hasAggs()) { + Aggregations aggs = querySearchResult.consumeAggs(); + Aggregations deserializedAggs = deserialized.consumeAggs(); + assertEquals(aggs.asList(), deserializedAggs.asList()); + List pipelineAggs = ((InternalAggregations) aggs).getTopLevelPipelineAggregators(); + List deserializedPipelineAggs = + ((InternalAggregations) deserializedAggs).getTopLevelPipelineAggregators(); + assertEquals(pipelineAggs.size(), deserializedPipelineAggs.size()); + for (int i = 0; i < pipelineAggs.size(); i++) { + SiblingPipelineAggregator pipelineAgg = pipelineAggs.get(i); + SiblingPipelineAggregator deserializedPipelineAgg = deserializedPipelineAggs.get(i); + assertArrayEquals(pipelineAgg.bucketsPaths(), deserializedPipelineAgg.bucketsPaths()); + assertEquals(pipelineAgg.name(), deserializedPipelineAgg.name()); + } + } + assertEquals(querySearchResult.terminatedEarly(), deserialized.terminatedEarly()); + } + + public void testReadFromPre_7_1_0() throws IOException { + String message = "AAAAAAAAAGQAAAEAAAB/wAAAAAEBBnN0ZXJtcwVJblhNRgoDBVNhdWpvAAVrS3l3cwVHSVVZaAAFZXRUbEUFZGN0WVoABXhzYnVrAAEDAfoN" + + "A3JhdwUBAAJRAAAAAAAAA30DBnN0ZXJtcwVNdVVFRwoAAAEDAfoNA3JhdwUBAAdDAAAAAAAAA30AAApQVkFhaUxSdHh5TAAAAAAAAAN9AAAKTVRUeUxnd1hyd" + + "y0AAAAAAAADfQAACnZRQXZ3cWp0SmwPAAAAAAAAA30AAApmYXNyUUhNVWZBCwAAAAAAAAN9AAAKT3FIQ2RMZ1JZUwUAAAAAAAADfQAACm9jT05aZmZ4ZmUmAA" + + "AAAAAAA30AAApvb0tJTkdvbHdzBnN0ZXJtcwVtRmlmZAoAAAEDAfoNA3JhdwUBAARXAAAAAAAAA30AAApZd3BwQlpBZEhpMQAAAAAAAAN9AAAKREZ3UVpTSXh" + + "DSE4AAAAAAAADfQAAClVMZW1YZGtkSHUUAAAAAAAAA30AAApBUVdKVk1kTlF1BnN0ZXJtcwVxbkJGVgoAAAEDAfoNA3JhdwUBAAYJAAAAAAAAA30AAApBS2NL" + + "U1ZVS25EIQAAAAAAAAN9AAAKWGpCbXZBZmduRhsAAAAAAAADfQAACk54TkJEV3pLRmI7AAAAAAAAA30AAApydkdaZnJycXhWSAAAAAAAAAN9AAAKSURVZ3JhQ" + + "lFHSy4AAAAAAAADfQAACmJmZ0x5YlFlVksAClRJZHJlSkpVc1Y4AAAAAAAAA30DBnN0ZXJtcwVNdVVFRwoAAAEDAfoNA3JhdwUBAAdDAAAAAAAAA30AAApQVk" + + "FhaUxSdHh5TAAAAAAAAAN9AAAKTVRUeUxnd1hydy0AAAAAAAADfQAACnZRQXZ3cWp0SmwPAAAAAAAAA30AAApmYXNyUUhNVWZBCwAAAAAAAAN9AAAKT3FIQ2R" + + "MZ1JZUwUAAAAAAAADfQAACm9jT05aZmZ4ZmUmAAAAAAAAA30AAApvb0tJTkdvbHdzBnN0ZXJtcwVtRmlmZAoAAAEDAfoNA3JhdwUBAARXAAAAAAAAA30AAApZ" + + "d3BwQlpBZEhpMQAAAAAAAAN9AAAKREZ3UVpTSXhDSE4AAAAAAAADfQAAClVMZW1YZGtkSHUUAAAAAAAAA30AAApBUVdKVk1kTlF1BnN0ZXJtcwVxbkJGVgoAA" + + "AEDAfoNA3JhdwUBAAYJAAAAAAAAA30AAApBS2NLU1ZVS25EIQAAAAAAAAN9AAAKWGpCbXZBZmduRhsAAAAAAAADfQAACk54TkJEV3pLRmI7AAAAAAAAA30AAA" + + "pydkdaZnJycXhWSAAAAAAAAAN9AAAKSURVZ3JhQlFHSy4AAAAAAAADfQAACmJmZ0x5YlFlVksACm5rdExLUHp3cGgBCm1heF9idWNrZXQFbmFtZTEBB2J1Y2t" + + "ldDH/A3JhdwEBCm1heF9idWNrZXQFbmFtZTEBB2J1Y2tldDH/A3JhdwEAAAIAAf////8AAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + byte[] bytes = Base64.getDecoder().decode(message); + try (NamedWriteableAwareStreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry)) { + in.setVersion(Version.V_7_0_0); + QuerySearchResult querySearchResult = new QuerySearchResult(); + querySearchResult.readFrom(in); + assertEquals(100, querySearchResult.getRequestId()); + assertTrue(querySearchResult.hasAggs()); + InternalAggregations aggs = (InternalAggregations)querySearchResult.consumeAggs(); + assertEquals(1, aggs.asList().size()); + //top-level pipeline aggs are retrieved as part of InternalAggregations although they were serialized separately + assertEquals(1, aggs.getTopLevelPipelineAggregators().size()); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java b/server/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java index d9f25a369d613..54d9a015b4e4a 100644 --- a/server/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java +++ b/server/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java @@ -160,7 +160,7 @@ public void testValidateEmptyCluster() { client().admin().indices().prepareValidateQuery().get(); fail("Expected IndexNotFoundException"); } catch (IndexNotFoundException e) { - assertThat(e.getMessage(), is("no such index [null]")); + assertThat(e.getMessage(), is("no such index [null] and no indices exist")); } } diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/Debug.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/Debug.java index c63d6ce19934c..cb10395e3925c 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/Debug.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/Debug.java @@ -91,12 +91,12 @@ static ResultSetMetaData proxy(ResultSetMetaDataProxy handler) { static Statement proxy(Object statement, StatementProxy handler) { Class i = Statement.class; - if (statement instanceof PreparedStatement) { - i = PreparedStatement.class; - } - else if (statement instanceof CallableStatement) { + if (statement instanceof CallableStatement) { i = CallableStatement.class; } + else if (statement instanceof PreparedStatement) { + i = PreparedStatement.class; + } return createProxy(i, handler); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java index 0fa2236a6bb5a..329eb9b566a05 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java @@ -287,16 +287,12 @@ private FieldExtraction topHitFieldRef(FieldAttribute fieldAttr) { } private Tuple nestedHitFieldRef(FieldAttribute attr) { - // Find the nested query for this field. If there isn't one then create it - List nestedRefs = new ArrayList<>(); - String name = aliasName(attr); Query q = rewriteToContainNestedField(query, attr.source(), attr.nestedParent().name(), name, attr.field().getDataType().format(), attr.field().isAggregatable()); SearchHitFieldRef nestedFieldRef = new SearchHitFieldRef(name, attr.field().getDataType(), attr.field().isAggregatable(), attr.parent().name()); - nestedRefs.add(nestedFieldRef); return new Tuple<>(new QueryContainer(q, aggs, fields, aliases, pseudoFunctions, scalarFunctions, sort, limit, trackHits), nestedFieldRef);