From 2e8f331dcc8f5e6e93f117df62261b29600eab99 Mon Sep 17 00:00:00 2001 From: liketic Date: Fri, 15 Dec 2017 17:24:37 +0800 Subject: [PATCH 1/2] Add version support for inner hits in field collapsing (#27822) --- .../action/search/ExpandSearchPhase.java | 1 + .../action/search/ExpandSearchPhaseTests.java | 6 +- .../action/search/InnerHitsCollapseIT.java | 121 ++++++++++++++++++ 3 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 core/src/test/java/org/elasticsearch/action/search/InnerHitsCollapseIT.java diff --git a/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index 53ce4299c546b..35df6212833c4 100644 --- a/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -165,6 +165,7 @@ private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder optio } groupSource.explain(options.isExplain()); groupSource.trackScores(options.isTrackScores()); + groupSource.version(options.isVersion()); return groupSource; } } diff --git a/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index 0951380fcf4aa..b580d48c11a43 100644 --- a/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.search; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.text.Text; @@ -248,6 +247,8 @@ public void run() throws IOException { public void testExpandRequestOptions() throws IOException { MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); + boolean version = randomBoolean(); + mockSearchPhaseContext.searchTransport = new SearchTransportService( Settings.builder().put("search.remote.connect", false).build(), null, null) { @@ -256,13 +257,14 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL final QueryBuilder postFilter = QueryBuilders.existsQuery("foo"); assertTrue(request.requests().stream().allMatch((r) -> "foo".equals(r.preference()))); assertTrue(request.requests().stream().allMatch((r) -> "baz".equals(r.routing()))); + assertTrue(request.requests().stream().allMatch((r) -> version == r.source().version())); assertTrue(request.requests().stream().allMatch((r) -> postFilter.equals(r.source().postFilter()))); } }; mockSearchPhaseContext.getRequest().source(new SearchSourceBuilder() .collapse( new CollapseBuilder("someField") - .setInnerHits(new InnerHitBuilder().setName("foobarbaz")) + .setInnerHits(new InnerHitBuilder().setName("foobarbaz").setVersion(version)) ) .postFilter(QueryBuilders.existsQuery("foo"))) .preference("foobar") diff --git a/core/src/test/java/org/elasticsearch/action/search/InnerHitsCollapseIT.java b/core/src/test/java/org/elasticsearch/action/search/InnerHitsCollapseIT.java new file mode 100644 index 0000000000000..13fa3f6d67ec2 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/search/InnerHitsCollapseIT.java @@ -0,0 +1,121 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.search; + +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.index.VersionType; +import org.elasticsearch.index.query.InnerHitBuilder; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.collapse.CollapseBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; +import static org.hamcrest.Matchers.equalTo; + + +public class InnerHitsCollapseIT extends ESIntegTestCase { + + + public void testSimpleCollapse() throws Exception { + final String index = randomAlphaOfLength(8).toLowerCase(Locale.ROOT); + final String type = randomAlphaOfLength(8); + final String collapseField = randomAlphaOfLength(8); + + assertAcked(prepareCreate(index).addMapping(type, + jsonBuilder().startObject() + .startObject("properties") + .startObject("title") + .field("type", "text") + .endObject() + .startObject(collapseField) + .field("type", "keyword") + .field("doc_values", "true") + .endObject() + .endObject().endObject())); + + List requests = new ArrayList<>(); + long version1 = randomLongBetween(0, 1000); + requests.add(client().prepareIndex(index, type, "1") + .setVersionType(VersionType.EXTERNAL) + .setVersion(version1) + .setSource(jsonBuilder().startObject().field("title", "quick brown fox") + .field(collapseField, "ck1").endObject())); + long version2 = randomLongBetween(0, 1000); + requests.add(client().prepareIndex(index, type, "2") + .setVersionType(VersionType.EXTERNAL) + .setVersion(version2) + .setSource(jsonBuilder().startObject().field("title", "big gray elephant") + .field(collapseField, "ck2").endObject())); + long version3 = randomLongBetween(0, 1000); + requests.add(client().prepareIndex(index, type, "3") + .setVersionType(VersionType.EXTERNAL) + .setVersion(version3) + .setSource(jsonBuilder().startObject().field("title", "big gray mice") + .field(collapseField, "ck1").endObject())); + indexRandom(true, requests); + + List storedFields = new ArrayList<>(2); + storedFields.add("*"); + storedFields.add("_source"); + CollapseBuilder cBuild = new CollapseBuilder(collapseField).setInnerHits( + new InnerHitBuilder() + .setName(collapseField) + .setVersion(true) + .setTrackScores(true) + .setStoredFieldNames(storedFields) + .addSort(new FieldSortBuilder("_id").order(SortOrder.ASC)) + .setFrom(0) + .setSize(22) + .setIgnoreUnmapped(false) + ); + SearchResponse response = client().prepareSearch(index) + .setQuery(matchAllQuery()) + .setCollapse(cBuild) + .setVersion(true) + .addSort("_id", SortOrder.ASC) + .get(); + assertNoFailures(response); + assertHitCount(response, 3); + assertSearchHit(response, 1, hasId("1")); + assertEquals(response.getHits().getAt(0).getVersion(), version1); + assertEquals(response.getHits().getAt(1).getVersion(), version2); + + assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1)); + SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get(collapseField); + assertThat(innerHits.getTotalHits(), equalTo(2L)); + assertThat(innerHits.getHits().length, equalTo(2)); + assertThat(innerHits.getAt(0).getId(), equalTo("1")); + assertEquals(innerHits.getAt(0).getVersion(), version1); + assertEquals(innerHits.getAt(1).getVersion(), version3); + } + +} From c9b8ee488ddabf807932d2441ac8ff9536f58a6d Mon Sep 17 00:00:00 2001 From: liketic Date: Fri, 15 Dec 2017 18:38:02 +0800 Subject: [PATCH 2/2] Add a new rest test for field collapsing --- .../action/search/InnerHitsCollapseIT.java | 121 ------------------ .../test/search/110_field_collapsing.yml | 65 ++++++++++ 2 files changed, 65 insertions(+), 121 deletions(-) delete mode 100644 core/src/test/java/org/elasticsearch/action/search/InnerHitsCollapseIT.java diff --git a/core/src/test/java/org/elasticsearch/action/search/InnerHitsCollapseIT.java b/core/src/test/java/org/elasticsearch/action/search/InnerHitsCollapseIT.java deleted file mode 100644 index 13fa3f6d67ec2..0000000000000 --- a/core/src/test/java/org/elasticsearch/action/search/InnerHitsCollapseIT.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.search; - -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.index.VersionType; -import org.elasticsearch.index.query.InnerHitBuilder; -import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.collapse.CollapseBuilder; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.test.ESIntegTestCase; - -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHit; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; -import static org.hamcrest.Matchers.equalTo; - - -public class InnerHitsCollapseIT extends ESIntegTestCase { - - - public void testSimpleCollapse() throws Exception { - final String index = randomAlphaOfLength(8).toLowerCase(Locale.ROOT); - final String type = randomAlphaOfLength(8); - final String collapseField = randomAlphaOfLength(8); - - assertAcked(prepareCreate(index).addMapping(type, - jsonBuilder().startObject() - .startObject("properties") - .startObject("title") - .field("type", "text") - .endObject() - .startObject(collapseField) - .field("type", "keyword") - .field("doc_values", "true") - .endObject() - .endObject().endObject())); - - List requests = new ArrayList<>(); - long version1 = randomLongBetween(0, 1000); - requests.add(client().prepareIndex(index, type, "1") - .setVersionType(VersionType.EXTERNAL) - .setVersion(version1) - .setSource(jsonBuilder().startObject().field("title", "quick brown fox") - .field(collapseField, "ck1").endObject())); - long version2 = randomLongBetween(0, 1000); - requests.add(client().prepareIndex(index, type, "2") - .setVersionType(VersionType.EXTERNAL) - .setVersion(version2) - .setSource(jsonBuilder().startObject().field("title", "big gray elephant") - .field(collapseField, "ck2").endObject())); - long version3 = randomLongBetween(0, 1000); - requests.add(client().prepareIndex(index, type, "3") - .setVersionType(VersionType.EXTERNAL) - .setVersion(version3) - .setSource(jsonBuilder().startObject().field("title", "big gray mice") - .field(collapseField, "ck1").endObject())); - indexRandom(true, requests); - - List storedFields = new ArrayList<>(2); - storedFields.add("*"); - storedFields.add("_source"); - CollapseBuilder cBuild = new CollapseBuilder(collapseField).setInnerHits( - new InnerHitBuilder() - .setName(collapseField) - .setVersion(true) - .setTrackScores(true) - .setStoredFieldNames(storedFields) - .addSort(new FieldSortBuilder("_id").order(SortOrder.ASC)) - .setFrom(0) - .setSize(22) - .setIgnoreUnmapped(false) - ); - SearchResponse response = client().prepareSearch(index) - .setQuery(matchAllQuery()) - .setCollapse(cBuild) - .setVersion(true) - .addSort("_id", SortOrder.ASC) - .get(); - assertNoFailures(response); - assertHitCount(response, 3); - assertSearchHit(response, 1, hasId("1")); - assertEquals(response.getHits().getAt(0).getVersion(), version1); - assertEquals(response.getHits().getAt(1).getVersion(), version2); - - assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1)); - SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get(collapseField); - assertThat(innerHits.getTotalHits(), equalTo(2L)); - assertThat(innerHits.getHits().length, equalTo(2)); - assertThat(innerHits.getAt(0).getId(), equalTo("1")); - assertEquals(innerHits.getAt(0).getVersion(), version1); - assertEquals(innerHits.getAt(1).getVersion(), version3); - } - -} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml index fa012ff3acb8c..48ca92c1ee999 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml @@ -7,36 +7,48 @@ setup: index: test type: test id: 1 + version_type: external + version: 11 body: { numeric_group: 1, sort: 10 } - do: index: index: test type: test id: 2 + version_type: external + version: 22 body: { numeric_group: 1, sort: 6 } - do: index: index: test type: test id: 3 + version_type: external + version: 33 body: { numeric_group: 1, sort: 24 } - do: index: index: test type: test id: 4 + version_type: external + version: 44 body: { numeric_group: 25, sort: 10 } - do: index: index: test type: test id: 5 + version_type: external + version: 55 body: { numeric_group: 25, sort: 5 } - do: index: index: test type: test id: 6 + version_type: external + version: 66 body: { numeric_group: 3, sort: 36 } - do: indices.refresh: @@ -322,3 +334,56 @@ setup: - match: { hits.hits.2.inner_hits.sub_hits_desc.hits.total: 2 } - length: { hits.hits.2.inner_hits.sub_hits_desc.hits.hits: 1 } - match: { hits.hits.2.inner_hits.sub_hits_desc.hits.hits.0._id: "4" } + +--- +"field collapsing, inner_hits and version": + + - skip: + version: " - 6.99.99" + reason: "bug fixed in 7.0.0" + + - do: + search: + index: test + type: test + body: + collapse: { field: numeric_group, inner_hits: { name: sub_hits, version: true, size: 2, sort: [{ sort: asc }] } } + sort: [{ sort: desc }] + version: true + + - match: { hits.total: 6 } + - length: { hits.hits: 3 } + - match: { hits.hits.0._index: test } + - match: { hits.hits.0._type: test } + - match: { hits.hits.0.fields.numeric_group: [3] } + - match: { hits.hits.0.sort: [36] } + - match: { hits.hits.0._id: "6" } + - match: { hits.hits.0._version: 66 } + - match: { hits.hits.0.inner_hits.sub_hits.hits.total: 1 } + - length: { hits.hits.0.inner_hits.sub_hits.hits.hits: 1 } + - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" } + - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._version: 66 } + - match: { hits.hits.1._index: test } + - match: { hits.hits.1._type: test } + - match: { hits.hits.1.fields.numeric_group: [1] } + - match: { hits.hits.1.sort: [24] } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.1._version: 33 } + - match: { hits.hits.1.inner_hits.sub_hits.hits.total: 3 } + - length: { hits.hits.1.inner_hits.sub_hits.hits.hits: 2 } + - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "2" } + - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._version: 22 } + - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" } + - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._version: 11 } + - match: { hits.hits.2._index: test } + - match: { hits.hits.2._type: test } + - match: { hits.hits.2.fields.numeric_group: [25] } + - match: { hits.hits.2.sort: [10] } + - match: { hits.hits.2._id: "4" } + - match: { hits.hits.2._version: 44 } + - match: { hits.hits.2.inner_hits.sub_hits.hits.total: 2 } + - length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 } + - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "5" } + - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._version: 55 } + - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" } + - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._version: 44 }