From 3033e2392c63b2375959b7f885abe79f61918be5 Mon Sep 17 00:00:00 2001 From: javanna Date: Thu, 2 Aug 2018 20:54:10 +0200 Subject: [PATCH 1/2] Expose `group_shard_failures` query_string parameter The search, count, search_template, msearch and msearch_template API are supposed to support a query_string parameter at REST, called `group_shard_failures` that allows to disable grouping of shard failures that have the same reason (see RestActions#buildBroadcastShardsHeader). The parameter cannot be provided at REST though to date, hence it is not supported by any of the above API. This commit restores support for the parameter, adds it to the REST spec and add tests for it. It also introduces support for it to the Java high-level REST client. --- .../client/RequestConverters.java | 17 +++- .../client/RequestConvertersTests.java | 27 +++++- .../mustache/MultiSearchTemplateRequest.java | 37 ++++++-- .../RestMultiSearchTemplateAction.java | 6 +- .../mustache/RestSearchTemplateAction.java | 6 +- .../test/lang_mustache/70_shard_failures.yml | 91 +++++++++++++++++++ .../resources/rest-api-spec/api/count.json | 4 + .../resources/rest-api-spec/api/msearch.json | 4 + .../rest-api-spec/api/msearch_template.json | 4 + .../resources/rest-api-spec/api/search.json | 4 + .../rest-api-spec/api/search_template.json | 4 + .../test/count/30_shard_failures.yml | 46 ++++++++++ .../test/msearch/20_typed_keys.yml | 2 +- .../test/msearch/30_shard_failures.yml | 50 ++++++++++ .../test/search/220_shard_failures.yml | 46 ++++++++++ .../action/search/MultiSearchRequest.java | 28 +++++- .../action/search/SearchRequest.java | 27 ++++++ .../rest/action/document/RestCountAction.java | 10 ++ .../action/search/RestMultiSearchAction.java | 7 +- .../rest/action/search/RestSearchAction.java | 5 +- 20 files changed, 398 insertions(+), 27 deletions(-) create mode 100644 modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/70_shard_failures.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/count/30_shard_failures.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/msearch/30_shard_failures.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/220_shard_failures.yml diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index ce6fd1c8c94bb..0c6209404ed6f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -39,12 +39,12 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; +import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; +import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; -import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; -import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; @@ -78,8 +78,8 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.DeletePipelineRequest; import org.elasticsearch.action.ingest.GetPipelineRequest; -import org.elasticsearch.action.ingest.SimulatePipelineRequest; import org.elasticsearch.action.ingest.PutPipelineRequest; +import org.elasticsearch.action.ingest.SimulatePipelineRequest; import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.SearchRequest; @@ -107,10 +107,10 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.index.rankeval.RankEvalRequest; import org.elasticsearch.protocol.xpack.XPackInfoRequest; -import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest; -import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest; import org.elasticsearch.protocol.xpack.XPackUsageRequest; import org.elasticsearch.protocol.xpack.license.PutLicenseRequest; +import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest; +import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.script.mustache.MultiSearchTemplateRequest; import org.elasticsearch.script.mustache.SearchTemplateRequest; @@ -556,6 +556,7 @@ static Request search(SearchRequest searchRequest) throws IOException { private static void addSearchRequestParams(Params params, SearchRequest searchRequest) { params.putParam(RestSearchAction.TYPED_KEYS_PARAM, "true"); + params.withGroupShardFailures(searchRequest.groupShardFailures()); params.withRouting(searchRequest.routing()); params.withPreference(searchRequest.preference()); params.withIndicesOptions(searchRequest.indicesOptions()); @@ -589,6 +590,7 @@ static Request multiSearch(MultiSearchRequest multiSearchRequest) throws IOExcep Params params = new Params(request); params.putParam(RestSearchAction.TYPED_KEYS_PARAM, "true"); + params.withGroupShardFailures(multiSearchRequest.groupShardFailures()); if (multiSearchRequest.maxConcurrentSearchRequests() != MultiSearchRequest.MAX_CONCURRENT_SEARCH_REQUESTS_DEFAULT) { params.putParam("max_concurrent_searches", Integer.toString(multiSearchRequest.maxConcurrentSearchRequests())); } @@ -622,6 +624,7 @@ static Request multiSearchTemplate(MultiSearchTemplateRequest multiSearchTemplat Params params = new Params(request); params.putParam(RestSearchAction.TYPED_KEYS_PARAM, "true"); + params.withGroupShardFailures(multiSearchTemplateRequest.groupShardFailures()); if (multiSearchTemplateRequest.maxConcurrentSearchRequests() != MultiSearchRequest.MAX_CONCURRENT_SEARCH_REQUESTS_DEFAULT) { params.putParam("max_concurrent_searches", Integer.toString(multiSearchTemplateRequest.maxConcurrentSearchRequests())); } @@ -1309,6 +1312,10 @@ Params withRouting(String routing) { return putParam("routing", routing); } + Params withGroupShardFailures(boolean groupShardFailures) { + return putParam(RestSearchAction.GROUP_SHARD_FAILURES_PARAM, Boolean.toString(groupShardFailures)); + } + Params withStoredFields(String[] storedFields) { if (storedFields != null && storedFields.length > 0) { return putParam("stored_fields", String.join(",", storedFields)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index e4aa690acb617..a7a596041b003 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -1307,6 +1307,7 @@ public void testMultiSearch() throws IOException { Map expectedParams = new HashMap<>(); expectedParams.put(RestSearchAction.TYPED_KEYS_PARAM, "true"); + setRandomGroupShardFailures(multiSearchRequest::groupShardFailures, expectedParams); if (randomBoolean()) { multiSearchRequest.maxConcurrentSearchRequests(randomIntBetween(1, 8)); expectedParams.put("max_concurrent_searches", Integer.toString(multiSearchRequest.maxConcurrentSearchRequests())); @@ -1360,7 +1361,6 @@ public void testClearScroll() throws IOException { } public void testSearchTemplate() throws Exception { - // Create a random request. String[] indices = randomIndicesNames(0, 5); SearchRequest searchRequest = new SearchRequest(indices); @@ -1369,7 +1369,6 @@ public void testSearchTemplate() throws Exception { setRandomIndicesOptions(searchRequest::indicesOptions, searchRequest::indicesOptions, expectedParams); SearchTemplateRequest searchTemplateRequest = new SearchTemplateRequest(searchRequest); - searchTemplateRequest.setScript("{\"query\": { \"match\" : { \"{{field}}\" : \"{{value}}\" }}}"); searchTemplateRequest.setScriptType(ScriptType.INLINE); searchTemplateRequest.setProfile(randomBoolean()); @@ -1421,21 +1420,28 @@ public void testRenderSearchTemplate() throws Exception { public void testMultiSearchTemplate() throws Exception { final int numSearchRequests = randomIntBetween(1, 10); MultiSearchTemplateRequest multiSearchTemplateRequest = new MultiSearchTemplateRequest(); + Map expectedParams2 = new HashMap<>(); + expectedParams2.put(RestSearchAction.TYPED_KEYS_PARAM, "true"); + setRandomGroupShardFailures(multiSearchTemplateRequest::groupShardFailures, expectedParams2); + if (randomBoolean()) { + multiSearchTemplateRequest.maxConcurrentSearchRequests(randomIntBetween(1, 8)); + expectedParams2.put("max_concurrent_searches", Integer.toString(multiSearchTemplateRequest.maxConcurrentSearchRequests())); + } for (int i = 0; i < numSearchRequests; i++) { // Create a random request. String[] indices = randomIndicesNames(0, 5); SearchRequest searchRequest = new SearchRequest(indices); - Map expectedParams = new HashMap<>(); - setRandomSearchParams(searchRequest, expectedParams); + Map ignored = new HashMap<>(); + setRandomSearchParams(searchRequest, ignored); // scroll is not supported in the current msearch or msearchtemplate api, so unset it: searchRequest.scroll((Scroll) null); // batched reduce size is currently not set-able on a per-request basis as it is a query string parameter only searchRequest.setBatchedReduceSize(SearchRequest.DEFAULT_BATCHED_REDUCE_SIZE); - setRandomIndicesOptions(searchRequest::indicesOptions, searchRequest::indicesOptions, expectedParams); + setRandomIndicesOptions(searchRequest::indicesOptions, searchRequest::indicesOptions, ignored); SearchTemplateRequest searchTemplateRequest = new SearchTemplateRequest(searchRequest); @@ -1455,6 +1461,7 @@ public void testMultiSearchTemplate() throws Exception { assertEquals(HttpPost.METHOD_NAME, multiRequest.getMethod()); assertEquals("/_msearch/template", multiRequest.getEndpoint()); + assertEquals(expectedParams2, multiRequest.getParameters()); List searchRequests = multiSearchTemplateRequest.requests(); assertEquals(numSearchRequests, searchRequests.size()); @@ -2624,6 +2631,7 @@ private static void randomizeFetchSourceContextParams(Consumer expectedParams) { expectedParams.put(RestSearchAction.TYPED_KEYS_PARAM, "true"); + setRandomGroupShardFailures(searchRequest::groupShardFailures, expectedParams); if (randomBoolean()) { searchRequest.routing(randomAlphaOfLengthBetween(3, 10)); expectedParams.put("routing", searchRequest.routing()); @@ -2683,6 +2691,15 @@ private static void setRandomIncludeDefaults(GetIndexRequest request, Map consumer, Map expectedParams) { + boolean groupShardFailures = true; + if (randomBoolean()) { + groupShardFailures = randomBoolean(); + consumer.accept(groupShardFailures); + } + expectedParams.put(RestSearchAction.GROUP_SHARD_FAILURES_PARAM, Boolean.toString(groupShardFailures)); + } + private static void setRandomHumanReadable(GetIndexRequest request, Map expectedParams) { if (randomBoolean()) { boolean humanReadable = randomBoolean(); diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java index caa9fa4831add..71f3bbb25e7cc 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java @@ -46,6 +46,13 @@ public class MultiSearchTemplateRequest extends ActionRequest implements Composi private int maxConcurrentSearchRequests = 0; private List requests = new ArrayList<>(); + /** + * Flag that controls whether shard failures are grouped by reason when outputting them + * using {@link org.elasticsearch.action.search.SearchResponse#toXContent(XContentBuilder, ToXContent.Params)}. This flag is used + * in the high-level REST client while it has no effect in the transport client. + */ + private boolean groupShardFailures = true; + private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed(); /** @@ -66,7 +73,6 @@ public MultiSearchTemplateRequest add(SearchTemplateRequest request) { return this; } - /** * Returns the amount of search requests specified in this multi search requests are allowed to be ran concurrently. */ @@ -86,6 +92,25 @@ public MultiSearchTemplateRequest maxConcurrentSearchRequests(int maxConcurrentS return this; } + /** + * Controls whether shard failures should be grouped by reason when outputting them + * using {@link org.elasticsearch.action.search.SearchResponse#toXContent(XContentBuilder, ToXContent.Params)}. + * This flag is used in the high-level REST client while it has no effect in the transport client. + */ + public MultiSearchTemplateRequest groupShardFailures(boolean groupShardFailures) { + this.groupShardFailures = groupShardFailures; + return this; + } + + /** + * Returns whether shard failures will be grouped by reason when outputting them + * using {@link org.elasticsearch.action.search.SearchResponse#toXContent(XContentBuilder, ToXContent.Params)}. + * This flag is used in the high-level REST client while it has no effect in the transport client. + */ + public boolean groupShardFailures() { + return this.groupShardFailures; + } + public List requests() { return this.requests; } @@ -134,7 +159,7 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeStreamableList(requests); } - + @Override public boolean equals(Object o) { if (this == o) return true; @@ -148,9 +173,9 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hash(maxConcurrentSearchRequests, requests, indicesOptions); - } - - public static byte[] writeMultiLineFormat(MultiSearchTemplateRequest multiSearchTemplateRequest, + } + + public static byte[] writeMultiLineFormat(MultiSearchTemplateRequest multiSearchTemplateRequest, XContent xContent) throws IOException { ByteArrayOutputStream output = new ByteArrayOutputStream(); for (SearchTemplateRequest templateRequest : multiSearchTemplateRequest.requests()) { @@ -168,5 +193,5 @@ public static byte[] writeMultiLineFormat(MultiSearchTemplateRequest multiSearch } return output.toByteArray(); } - + } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java index 3aef9c32dc3d7..6e39f4f80dd91 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java @@ -29,7 +29,8 @@ import org.elasticsearch.rest.action.search.RestSearchAction; import java.io.IOException; -import java.util.Collections; +import java.util.Arrays; +import java.util.HashSet; import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -37,7 +38,8 @@ public class RestMultiSearchTemplateAction extends BaseRestHandler { - private static final Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM); + private static final Set RESPONSE_PARAMS = new HashSet<>(Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, + RestSearchAction.GROUP_SHARD_FAILURES_PARAM)); private final boolean allowExplicitIndex; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java index e9bbc7aeeecf2..f20e77144ee56 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java @@ -30,7 +30,8 @@ import org.elasticsearch.rest.action.search.RestSearchAction; import java.io.IOException; -import java.util.Collections; +import java.util.Arrays; +import java.util.HashSet; import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -38,7 +39,8 @@ public class RestSearchTemplateAction extends BaseRestHandler { - private static final Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM); + private static final Set RESPONSE_PARAMS = new HashSet<>(Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, + RestSearchAction.GROUP_SHARD_FAILURES_PARAM)); public RestSearchTemplateAction(Settings settings, RestController controller) { super(settings); diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/70_shard_failures.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/70_shard_failures.yml new file mode 100644 index 0000000000000..bdb1418aa98c3 --- /dev/null +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/70_shard_failures.yml @@ -0,0 +1,91 @@ +--- +"shard failures": + - do: + indices.create: + index: text + body: + settings: + number_of_replicas: 0 + number_of_shards: 1 + mappings: + doc: + properties: + creation_date: + type: keyword + + - do: + indices.create: + index: date + body: + settings: + number_of_replicas: 0 + number_of_shards: 3 + mappings: + doc: + properties: + creation_date: + type: date + + - do: + search_template: + body: + source: + query: + match: + creation_date: "{{text_value}}" + params: + text_value: "text" + + - length: { _shards.failures: 1 } + - match: { _shards.failed: 3} + - match: { _shards.total: 4} + - match: { _shards.successful: 1} + + - do: + search_template: + group_shard_failures: false + body: + source: + query: + match: + creation_date: "{{text_value}}" + params: + text_value: "text" + + - length: { _shards.failures: 3 } + - match: { _shards.failed: 3} + - match: { _shards.total: 4} + - match: { _shards.successful: 1} + + - do: + msearch_template: + body: + - index: text,date + - source: + query: + match: + creation_date: "{{text_value}}" + params: + text_value: "text" + + - length: { responses.0._shards.failures: 1 } + - match: { responses.0._shards.failed: 3} + - match: { responses.0._shards.total: 4} + - match: { responses.0._shards.successful: 1} + + - do: + msearch_template: + group_shard_failures: false + body: + - index: text,date + - source: + query: + match: + creation_date: "{{text_value}}" + params: + text_value: "text" + + - length: { responses.0._shards.failures: 3 } + - match: { responses.0._shards.failed: 3} + - match: { responses.0._shards.total: 4} + - match: { responses.0._shards.successful: 1} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json index 96fa4daf12b95..3236438f36067 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json @@ -71,6 +71,10 @@ "terminate_after" : { "type" : "number", "description" : "The maximum count for each shard, upon reaching which the query execution will terminate early" + }, + "group_shard_failures" : { + "type" : "boolean", + "description" : "Specify whether shard failures should be grouped by reason, default is true" } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json index 090c429fd82c0..45acdd8d7fb95 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json @@ -33,6 +33,10 @@ "type" : "number", "description" : "A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint.", "default" : 128 + }, + "group_shard_failures" : { + "type" : "boolean", + "description" : "Specify whether shard failures should be grouped by reason, default is true" } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json index e742093293255..f66a33057902d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json @@ -28,6 +28,10 @@ "max_concurrent_searches" : { "type" : "number", "description" : "Controls the maximum number of concurrent searches the multi search api will execute" + }, + "group_shard_failures" : { + "type" : "boolean", + "description" : "Specify whether shard failures should be grouped by reason, default is true" } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json index 751c2797b9deb..0e8bdf6b21995 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json @@ -182,6 +182,10 @@ "type" : "number", "description" : "A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint.", "default" : 128 + }, + "group_shard_failures" : { + "type" : "boolean", + "description" : "Specify whether shard failures should be grouped by reason, default is true" } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json index a78295dd4f5a3..b29b6c2bfdc03 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json @@ -58,6 +58,10 @@ "typed_keys": { "type" : "boolean", "description" : "Specify whether aggregation and suggester names should be prefixed by their respective types in the response" + }, + "group_shard_failures" : { + "type" : "boolean", + "description" : "Specify whether shard failures should be grouped by reason, default is true" } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/count/30_shard_failures.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/count/30_shard_failures.yml new file mode 100644 index 0000000000000..e3d3cb6a2cf3d --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/count/30_shard_failures.yml @@ -0,0 +1,46 @@ +--- +"shard failures": + - do: + indices.create: + index: text + body: + settings: + number_of_replicas: 0 + number_of_shards: 1 + mappings: + doc: + properties: + creation_date: + type: keyword + + - do: + indices.create: + index: date + body: + settings: + number_of_replicas: 0 + number_of_shards: 3 + mappings: + doc: + properties: + creation_date: + type: date + + - do: + count: + body: { query: { match: { creation_date: text} } } + + - length: { _shards.failures: 1 } + - match: { _shards.failed: 3} + - match: { _shards.total: 4} + - match: { _shards.successful: 1} + + - do: + count: + group_shard_failures: false + body: { query: { match: { creation_date: text} } } + + - length: { _shards.failures: 3 } + - match: { _shards.failed: 3} + - match: { _shards.total: 4} + - match: { _shards.successful: 1} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yml index 7348cd4d14bb6..bf13c762c0340 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yml @@ -61,7 +61,7 @@ setup: msearch: typed_keys: true body: - # Testing aggegrations + # Testing aggregations - index: test-* - {query: {match: {bool: true} }, size: 0, aggs: {test_filter: {filter: {range: {integer: {gte: 20} } } } } } - index: test-1 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/30_shard_failures.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/30_shard_failures.yml new file mode 100644 index 0000000000000..5dd9ee599e4ac --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/30_shard_failures.yml @@ -0,0 +1,50 @@ +--- +"shard failures": + - do: + indices.create: + index: text + body: + settings: + number_of_replicas: 0 + number_of_shards: 1 + mappings: + doc: + properties: + creation_date: + type: keyword + + - do: + indices.create: + index: date + body: + settings: + number_of_replicas: 0 + number_of_shards: 3 + mappings: + doc: + properties: + creation_date: + type: date + + - do: + msearch: + body: + - index: text,date + - { query: { match: { creation_date: text} } } + + - length: { responses.0._shards.failures: 1 } + - match: { responses.0._shards.failed: 3} + - match: { responses.0._shards.total: 4} + - match: { responses.0._shards.successful: 1} + + - do: + msearch: + group_shard_failures: false + body: + - index: text,date + - { query: { match: { creation_date: text} } } + + - length: { responses.0._shards.failures: 3 } + - match: { responses.0._shards.failed: 3} + - match: { responses.0._shards.total: 4} + - match: { responses.0._shards.successful: 1} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/220_shard_failures.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/220_shard_failures.yml new file mode 100644 index 0000000000000..bd64408db9699 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/220_shard_failures.yml @@ -0,0 +1,46 @@ +--- +"shard failures": + - do: + indices.create: + index: text + body: + settings: + number_of_replicas: 0 + number_of_shards: 1 + mappings: + doc: + properties: + creation_date: + type: keyword + + - do: + indices.create: + index: date + body: + settings: + number_of_replicas: 0 + number_of_shards: 3 + mappings: + doc: + properties: + creation_date: + type: date + + - do: + search: + body: { query: { match: { creation_date: text} } } + + - length: { _shards.failures: 1 } + - match: { _shards.failed: 3} + - match: { _shards.total: 4} + - match: { _shards.successful: 1} + + - do: + search: + group_shard_failures: false + body: { query: { match: { creation_date: text} } } + + - length: { _shards.failures: 3 } + - match: { _shards.failed: 3} + - match: { _shards.total: 4} + - match: { _shards.successful: 1} diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java index 056c4c29c7a3d..b1e8273eabcad 100644 --- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java @@ -55,6 +55,13 @@ public class MultiSearchRequest extends ActionRequest implements CompositeIndice public static final int MAX_CONCURRENT_SEARCH_REQUESTS_DEFAULT = 0; + /** + * Flag that controls whether shard failures are grouped by reason when outputting them + * using {@link SearchResponse#toXContent(XContentBuilder, ToXContent.Params)}. This flag is used + * in the high-level REST client while it has no effect in the transport client. + */ + private boolean groupShardFailures = true; + private int maxConcurrentSearchRequests = 0; private List requests = new ArrayList<>(); @@ -129,6 +136,25 @@ public MultiSearchRequest indicesOptions(IndicesOptions indicesOptions) { return this; } + /** + * Controls whether shard failures should be grouped by reason when outputting them + * using {@link SearchResponse#toXContent(XContentBuilder, ToXContent.Params)}. This flag is used in the high-level REST client while + * it has no effect in the transport client. + */ + public MultiSearchRequest groupShardFailures(boolean groupShardFailures) { + this.groupShardFailures = groupShardFailures; + return this; + } + + /** + * Returns whether shard failures will be grouped by reason when outputting them + * using {@link SearchResponse#toXContent(XContentBuilder, ToXContent.Params)}. This flag is used in the high-level REST client while + * it has no effect in the transport client. + */ + public boolean groupShardFailures() { + return this.groupShardFailures; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); @@ -287,7 +313,7 @@ public static byte[] writeMultiLineFormat(MultiSearchRequest multiSearchRequest, } return output.toByteArray(); } - + public static void writeSearchRequestParams(SearchRequest request, XContentBuilder xContentBuilder) throws IOException { xContentBuilder.startObject(); if (request.indices() != null) { diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index e67517c4852b8..299abc53c52e4 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.tasks.Task; @@ -66,6 +67,13 @@ public final class SearchRequest extends ActionRequest implements IndicesRequest private String[] indices = Strings.EMPTY_ARRAY; + /** + * Flag that controls whether shard failures are grouped by reason when outputting them + * using {@link SearchResponse#toXContent(XContentBuilder, ToXContent.Params)}. This flag is used + * in the high-level REST client while it has no effect in the transport client. + */ + private boolean groupShardFailures = true; + @Nullable private String routing; @Nullable @@ -214,6 +222,25 @@ public SearchRequest indicesOptions(IndicesOptions indicesOptions) { return this; } + /** + * Controls whether shard failures should be grouped by reason when outputting them + * using {@link SearchResponse#toXContent(XContentBuilder, ToXContent.Params)}. This flag is used in the high-level REST client while + * it has no effect in the transport client. + */ + public SearchRequest groupShardFailures(boolean groupShardFailures) { + this.groupShardFailures = groupShardFailures; + return this; + } + + /** + * Returns whether shard failures will be grouped by reason when outputting them + * using {@link SearchResponse#toXContent(XContentBuilder, ToXContent.Params)}. This flag is used in the high-level REST client while + * it has no effect in the transport client. + */ + public boolean groupShardFailures() { + return this.groupShardFailures; + } + /** * The document types to execute the search against. Defaults to be executed against * all types. diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestCountAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestCountAction.java index a66b17cc5a5e5..6e88e724411ff 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestCountAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestCountAction.java @@ -34,9 +34,12 @@ import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.rest.action.RestBuilderListener; +import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; +import java.util.Collections; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -44,6 +47,9 @@ import static org.elasticsearch.search.internal.SearchContext.DEFAULT_TERMINATE_AFTER; public class RestCountAction extends BaseRestHandler { + + private static final Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.GROUP_SHARD_FAILURES_PARAM); + public RestCountAction(Settings settings, RestController controller) { super(settings); controller.registerHandler(POST, "/_count", this); @@ -106,4 +112,8 @@ public RestResponse buildResponse(SearchResponse response, XContentBuilder build }); } + @Override + protected Set responseParams() { + return RESPONSE_PARAMS; + } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java index 2a60262b32f57..1101053157ba0 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java @@ -38,7 +38,8 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; -import java.util.Collections; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -47,7 +48,8 @@ public class RestMultiSearchAction extends BaseRestHandler { - private static final Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM); + private static final Set RESPONSE_PARAMS = new HashSet<>(Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, + RestSearchAction.GROUP_SHARD_FAILURES_PARAM)); private final boolean allowExplicitIndex; @@ -86,7 +88,6 @@ public static MultiSearchRequest parseRequest(RestRequest restRequest, boolean a int preFilterShardSize = restRequest.paramAsInt("pre_filter_shard_size", SearchRequest.DEFAULT_PRE_FILTER_SHARD_SIZE); - parseMultiLineRequest(restRequest, multiRequest.indicesOptions(), allowExplicitIndex, (searchRequest, parser) -> { searchRequest.source(SearchSourceBuilder.fromXContent(parser, false)); multiRequest.add(searchRequest); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 11c91703bdcdb..8f00ba0a0e9f4 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -44,7 +44,7 @@ import java.io.IOException; import java.util.Arrays; -import java.util.Collections; +import java.util.HashSet; import java.util.Set; import java.util.function.IntConsumer; @@ -56,7 +56,8 @@ public class RestSearchAction extends BaseRestHandler { public static final String TYPED_KEYS_PARAM = "typed_keys"; - private static final Set RESPONSE_PARAMS = Collections.singleton(TYPED_KEYS_PARAM); + public static final String GROUP_SHARD_FAILURES_PARAM = "group_shard_failures"; + private static final Set RESPONSE_PARAMS = new HashSet<>(Arrays.asList(TYPED_KEYS_PARAM, GROUP_SHARD_FAILURES_PARAM)); private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(RestSearchAction.class)); public RestSearchAction(Settings settings, RestController controller) { From dedf78af026f0a75753bf9a3919368298cdc10da Mon Sep 17 00:00:00 2001 From: javanna Date: Thu, 2 Aug 2018 21:00:06 +0200 Subject: [PATCH 2/2] rename variable --- .../elasticsearch/client/RequestConvertersTests.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index a7a596041b003..837028fbed5a8 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -1420,12 +1420,12 @@ public void testRenderSearchTemplate() throws Exception { public void testMultiSearchTemplate() throws Exception { final int numSearchRequests = randomIntBetween(1, 10); MultiSearchTemplateRequest multiSearchTemplateRequest = new MultiSearchTemplateRequest(); - Map expectedParams2 = new HashMap<>(); - expectedParams2.put(RestSearchAction.TYPED_KEYS_PARAM, "true"); - setRandomGroupShardFailures(multiSearchTemplateRequest::groupShardFailures, expectedParams2); + Map expectedParams = new HashMap<>(); + expectedParams.put(RestSearchAction.TYPED_KEYS_PARAM, "true"); + setRandomGroupShardFailures(multiSearchTemplateRequest::groupShardFailures, expectedParams); if (randomBoolean()) { multiSearchTemplateRequest.maxConcurrentSearchRequests(randomIntBetween(1, 8)); - expectedParams2.put("max_concurrent_searches", Integer.toString(multiSearchTemplateRequest.maxConcurrentSearchRequests())); + expectedParams.put("max_concurrent_searches", Integer.toString(multiSearchTemplateRequest.maxConcurrentSearchRequests())); } for (int i = 0; i < numSearchRequests; i++) { @@ -1461,7 +1461,7 @@ public void testMultiSearchTemplate() throws Exception { assertEquals(HttpPost.METHOD_NAME, multiRequest.getMethod()); assertEquals("/_msearch/template", multiRequest.getEndpoint()); - assertEquals(expectedParams2, multiRequest.getParameters()); + assertEquals(expectedParams, multiRequest.getParameters()); List searchRequests = multiSearchTemplateRequest.requests(); assertEquals(numSearchRequests, searchRequests.size());