From 6400d7faba6fc8205fa81b05f993cc98ca51d197 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 4 Oct 2018 14:35:31 +0200 Subject: [PATCH 1/5] Prevent throttled indices to be searched though wildcards by default Today if a wildcard, date-math expression or alias expands/resolves to an index that is search-throttled we still search it. This is likely not the desired behavior since it can unexpectedly slow down searches significantly. This change adds a new indices option that allows `search`, `count` and `msearch` to ignore throttled indices by default. Users can force expansion to throttled indices by using `ignore_throttled=true` on the rest request to expand also to throttled indices. Relates to #34352 --- .../org/elasticsearch/client/RankEvalIT.java | 2 +- .../client/RequestConvertersTests.java | 3 +- .../mustache/MultiSearchTemplateRequest.java | 2 +- .../index/rankeval/RankEvalRequestIT.java | 14 ++--- .../index/rankeval/RankEvalRequestTests.java | 3 +- .../resources/rest-api-spec/api/count.json | 4 ++ .../resources/rest-api-spec/api/search.json | 4 ++ .../rest-api-spec/api/search_template.json | 4 ++ .../indices/alias/IndicesAliasesRequest.java | 2 +- .../indices/delete/DeleteIndexRequest.java | 2 +- .../action/search/MultiSearchRequest.java | 4 +- .../search/MultiSearchRequestBuilder.java | 7 ++- .../action/search/SearchRequest.java | 2 +- .../action/support/IndicesOptions.java | 54 +++++++++++++--- .../metadata/IndexNameExpressionResolver.java | 21 ++++++- .../CanMatchPreFilterSearchPhaseTests.java | 7 +-- .../search/MultiSearchRequestTests.java | 16 ++--- .../action/search/SearchAsyncActionTests.java | 7 +-- .../search/TransportSearchActionTests.java | 8 +-- .../action/support/IndicesOptionsTests.java | 62 ++++++++++++------- .../IndexNameExpressionResolverTests.java | 58 +++++++++++++++-- .../WildcardExpressionResolverTests.java | 11 ++-- .../search/SearchServiceTests.java | 15 +++++ 23 files changed, 233 insertions(+), 79 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java index 15272ad80a655..9a0a861cc132d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java @@ -108,7 +108,7 @@ public void testRankEvalRequest() throws IOException { // now try this when test2 is closed client().performRequest(new Request("POST", "index2/_close")); - rankEvalRequest.indicesOptions(IndicesOptions.fromParameters(null, "true", null, SearchRequest.DEFAULT_INDICES_OPTIONS)); + rankEvalRequest.indicesOptions(IndicesOptions.fromParameters(null, "true", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 3801dfe71de9c..6b586c02bb741 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -971,7 +971,8 @@ public void testMultiSearch() throws IOException { IndicesOptions msearchDefault = new MultiSearchRequest().indicesOptions(); searchRequest.indicesOptions(IndicesOptions.fromOptions(randomlyGenerated.ignoreUnavailable(), randomlyGenerated.allowNoIndices(), randomlyGenerated.expandWildcardsOpen(), randomlyGenerated.expandWildcardsClosed(), - msearchDefault.allowAliasesToMultipleIndices(), msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases())); + msearchDefault.allowAliasesToMultipleIndices(), msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases(), + msearchDefault.ignoreThrottled())); multiSearchRequest.add(searchRequest); } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java index eea9e31d4a79d..a685c3ba5ba7c 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java @@ -45,7 +45,7 @@ public class MultiSearchTemplateRequest extends ActionRequest implements Composi private int maxConcurrentSearchRequests = 0; private List requests = new ArrayList<>(); - private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed(); + private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled(); /** * Add a search template request to execute. Note, the order is important, the search response will be returned in the diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index cdad280fd9a29..cc5b554e39e93 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -286,7 +286,7 @@ public void testIndicesOptions() { // test that ignore_unavailable=true works but returns one result less assertTrue(client().admin().indices().prepareClose("test2").get().isAcknowledged()); - request.indicesOptions(IndicesOptions.fromParameters(null, "true", null, SearchRequest.DEFAULT_INDICES_OPTIONS)); + request.indicesOptions(IndicesOptions.fromParameters(null, "true", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails(); assertEquals(6, details.getRetrieved()); @@ -294,37 +294,37 @@ public void testIndicesOptions() { // test that ignore_unavailable=false or default settings throw an IndexClosedException assertTrue(client().admin().indices().prepareClose("test2").get().isAcknowledged()); - request.indicesOptions(IndicesOptions.fromParameters(null, "false", null, SearchRequest.DEFAULT_INDICES_OPTIONS)); + request.indicesOptions(IndicesOptions.fromParameters(null, "false", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); assertEquals(1, response.getFailures().size()); assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexClosedException.class)); // test expand_wildcards request = new RankEvalRequest(task, new String[] { "tes*" }); - request.indicesOptions(IndicesOptions.fromParameters("none", null, null, SearchRequest.DEFAULT_INDICES_OPTIONS)); + request.indicesOptions(IndicesOptions.fromParameters("none", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails(); assertEquals(0, details.getRetrieved()); - request.indicesOptions(IndicesOptions.fromParameters("open", null, null, SearchRequest.DEFAULT_INDICES_OPTIONS)); + request.indicesOptions(IndicesOptions.fromParameters("open", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails(); assertEquals(6, details.getRetrieved()); assertEquals(5, details.getRelevantRetrieved()); - request.indicesOptions(IndicesOptions.fromParameters("closed", null, null, SearchRequest.DEFAULT_INDICES_OPTIONS)); + request.indicesOptions(IndicesOptions.fromParameters("closed", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); assertEquals(1, response.getFailures().size()); assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexClosedException.class)); // test allow_no_indices request = new RankEvalRequest(task, new String[] { "bad*" }); - request.indicesOptions(IndicesOptions.fromParameters(null, null, "true", SearchRequest.DEFAULT_INDICES_OPTIONS)); + request.indicesOptions(IndicesOptions.fromParameters(null, null, "true", "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails(); assertEquals(0, details.getRetrieved()); - request.indicesOptions(IndicesOptions.fromParameters(null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); + request.indicesOptions(IndicesOptions.fromParameters(null, null, "false", "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); assertEquals(1, response.getFailures().size()); assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexNotFoundException.class)); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index 10e3611b30d31..1a16c311fcf3a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -59,7 +59,8 @@ protected RankEvalRequest createTestInstance() { } RankEvalRequest rankEvalRequest = new RankEvalRequest(RankEvalSpecTests.createTestItem(), indices); IndicesOptions indicesOptions = IndicesOptions.fromOptions( - randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()); + randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), + randomBoolean()); rankEvalRequest.indicesOptions(indicesOptions); return rankEvalRequest; } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json index 96fa4daf12b95..d14f4ab784a57 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json @@ -20,6 +20,10 @@ "type" : "boolean", "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)" }, + "ignore_throttled": { + "type" : "boolean", + "description" : "Whether specified concrete, expanded or aliased indices should be ignored when throttled" + }, "allow_no_indices": { "type" : "boolean", "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json index 751c2797b9deb..c211aae2db324 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json @@ -58,6 +58,10 @@ "type" : "boolean", "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)" }, + "ignore_throttled": { + "type" : "boolean", + "description" : "Whether specified concrete, expanded or aliased indices should be ignored when throttled" + }, "allow_no_indices": { "type" : "boolean", "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json index a78295dd4f5a3..df1fc1b079923 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json @@ -20,6 +20,10 @@ "type" : "boolean", "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)" }, + "ignore_throttled": { + "type" : "boolean", + "description" : "Whether specified concrete, expanded or aliased indices should be ignored when throttled" + }, "allow_no_indices": { "type" : "boolean", "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)" diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java index 22e8554ed6aa6..0e34fb3b39015 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java @@ -67,7 +67,7 @@ public class IndicesAliasesRequest extends AcknowledgedRequest private String[] indices; // Delete index should work by default on both open and closed indices. - private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true); + private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); public DeleteIndexRequest() { } diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java index 056c4c29c7a3d..0cff8aadb5268 100644 --- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java @@ -58,7 +58,7 @@ public class MultiSearchRequest extends ActionRequest implements CompositeIndice private int maxConcurrentSearchRequests = 0; private List requests = new ArrayList<>(); - private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed(); + private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled(); /** * Add a search request to execute. Note, the order is important, the search response will be returned in the @@ -287,7 +287,7 @@ public static byte[] writeMultiLineFormat(MultiSearchRequest multiSearchRequest, } return output.toByteArray(); } - + public static void writeSearchRequestParams(SearchRequest request, XContentBuilder xContentBuilder) throws IOException { xContentBuilder.startObject(); if (request.indices() != null) { diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java index 64d512f4be05f..ce43f47a4979b 100644 --- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java @@ -28,6 +28,7 @@ */ public class MultiSearchRequestBuilder extends ActionRequestBuilder { + public MultiSearchRequestBuilder(ElasticsearchClient client, MultiSearchAction action) { super(client, action, new MultiSearchRequest()); } @@ -40,7 +41,8 @@ public MultiSearchRequestBuilder(ElasticsearchClient client, MultiSearchAction a * will not be used (if set). */ public MultiSearchRequestBuilder add(SearchRequest request) { - if (request.indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) { + if (request.indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() + && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) { request.indicesOptions(request().indicesOptions()); } @@ -53,7 +55,8 @@ public MultiSearchRequestBuilder add(SearchRequest request) { * same order as the search requests. */ public MultiSearchRequestBuilder add(SearchRequestBuilder request) { - if (request.request().indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) { + if (request.request().indicesOptions() == SearchRequest.DEFAULT_INDICES_OPTIONS + && request().indicesOptions() != SearchRequest.DEFAULT_INDICES_OPTIONS) { request.request().indicesOptions(request().indicesOptions()); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index dd7f687294383..f4b501b769796 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -88,7 +88,7 @@ public final class SearchRequest extends ActionRequest implements IndicesRequest private String[] types = Strings.EMPTY_ARRAY; - public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosed(); + public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled(); private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS; diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java index f2cf0b5444d52..1a983d1b5fe64 100644 --- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java +++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java @@ -81,7 +81,8 @@ public enum Option { IGNORE_ALIASES, ALLOW_NO_INDICES, FORBID_ALIASES_TO_MULTIPLE_INDICES, - FORBID_CLOSED_INDICES; + FORBID_CLOSED_INDICES, + IGNORE_THROTTLED; public static final EnumSet