From a63bb0a28b162779692de8a5959b18bc31074e1d Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 16 Jun 2020 11:44:26 -0400 Subject: [PATCH 1/3] [7.x] EQL: Adds an ability to execute an asynchronous EQL search Adds async support to EQL searches Closes #49638 Co-authored-by: James Rodewig --- .../client/eql/EqlSearchRequest.java | 52 ++- .../client/eql/EqlSearchResponse.java | 54 ++- .../client/eql/EqlSearchResponseTests.java | 21 +- .../eql/delete-async-eql-search-api.asciidoc | 47 +++ docs/reference/eql/eql-search-api.asciidoc | 170 +++++++++ .../eql/get-async-eql-search-api.asciidoc | 82 ++++ docs/reference/eql/search.asciidoc | 262 +++++++++++++ docs/reference/search.asciidoc | 4 + .../action/support/ListenerTimeouts.java | 36 +- x-pack/plugin/async-search/build.gradle | 1 + .../xpack/search/AsyncSearchSecurityIT.java | 4 +- .../xpack/search/AsyncSearchActionIT.java | 3 +- .../xpack/search/AsyncSearch.java | 52 +-- .../search/AsyncSearchMaintenanceService.java | 36 -- .../xpack/search/AsyncSearchTask.java | 7 + .../search/RestDeleteAsyncSearchAction.java | 8 +- .../search/RestGetAsyncSearchAction.java | 3 +- .../TransportDeleteAsyncSearchAction.java | 115 ------ .../search/TransportGetAsyncSearchAction.java | 138 ++----- .../TransportSubmitAsyncSearchAction.java | 7 +- .../search/AsyncSearchIntegTestCase.java | 28 +- .../search/AsyncSearchResponseTests.java | 2 +- .../search/DeleteAsyncSearchRequestTests.java | 14 +- .../search/GetAsyncSearchRequestTests.java | 47 --- x-pack/plugin/async/build.gradle | 29 ++ .../xpack/async/AsyncResultsIndexPlugin.java | 88 +++++ .../plugin-metadata/plugin-security.policy | 0 .../async/AsyncResultsIndexPluginTests.java | 20 + .../xpack/core/XPackClientPlugin.java | 6 +- .../elasticsearch/xpack/core/XPackPlugin.java | 4 + .../xpack/core/async/AsyncResultsService.java | 172 +++++++++ .../xpack/core/async/AsyncTask.java | 17 + .../core/async/AsyncTaskIndexService.java | 59 ++- .../async/AsyncTaskMaintenanceService.java | 24 +- .../core/async/DeleteAsyncResultAction.java | 25 ++ .../core/async/DeleteAsyncResultRequest.java | 56 +++ .../core/async/DeleteAsyncResultsService.java | 88 +++++ .../core/async/GetAsyncResultRequest.java | 96 +++++ .../TransportDeleteAsyncResultAction.java | 60 +++ .../xpack/core/eql/EqlAsyncActionNames.java | 14 + .../action/DeleteAsyncSearchAction.java | 72 ---- .../search/action/GetAsyncSearchAction.java | 102 ----- .../core/async/AsyncExecutionIdTests.java | 2 +- .../core/async/AsyncResultsServiceTests.java | 276 ++++++++++++++ .../async/AsyncSearchIndexServiceTests.java | 6 +- .../async/GetAsyncResultRequestTests.java | 35 ++ x-pack/plugin/eql/build.gradle | 3 + .../rest-api-spec/test/eql/10_basic.yml | 35 ++ x-pack/plugin/eql/qa/security/build.gradle | 27 ++ x-pack/plugin/eql/qa/security/roles.yml | 33 ++ .../xpack/eql/AsyncEqlSecurityIT.java | 168 +++++++++ .../AbstractEqlBlockingIntegTestCase.java | 240 ++++++++++++ .../eql/action/AbstractEqlIntegTestCase.java | 0 .../eql/action/AsyncEqlSearchActionIT.java | 349 ++++++++++++++++++ .../xpack/eql/action/EqlCancellationIT.java | 193 +--------- .../eql/action/RestEqlCancellationIT.java | 147 ++++++++ .../xpack/eql/action/EqlSearchRequest.java | 94 ++++- .../xpack/eql/action/EqlSearchResponse.java | 79 +++- .../xpack/eql/action/EqlSearchTask.java | 28 +- .../eql/async/AsyncTaskManagementService.java | 265 +++++++++++++ .../xpack/eql/async/StoredAsyncResponse.java | 103 ++++++ .../xpack/eql/async/StoredAsyncTask.java | 102 +++++ .../eql/plugin/EqlAsyncGetResultAction.java | 18 + .../xpack/eql/plugin/EqlPlugin.java | 12 +- .../RestEqlDeleteAsyncResultAction.java | 36 ++ .../plugin/RestEqlGetAsyncResultAction.java | 41 ++ .../xpack/eql/plugin/RestEqlSearchAction.java | 28 +- .../TransportEqlAsyncGetResultAction.java | 84 +++++ .../eql/plugin/TransportEqlSearchAction.java | 71 +++- .../xpack/eql/session/EqlConfiguration.java | 12 +- .../elasticsearch/xpack/eql/EqlTestUtils.java | 10 +- .../eql/action/EqlSearchResponseTests.java | 35 +- .../AsyncTaskManagementServiceTests.java | 287 ++++++++++++++ .../eql/async/StoredAsyncResponseTests.java | 78 ++++ .../xpack/security/authz/RBACEngine.java | 10 +- .../rest-api-spec/api/eql.delete.json | 25 ++ .../resources/rest-api-spec/api/eql.get.json | 36 ++ .../rest-api-spec/api/eql.search.json | 17 +- 78 files changed, 4209 insertions(+), 901 deletions(-) create mode 100644 docs/reference/eql/delete-async-eql-search-api.asciidoc create mode 100644 docs/reference/eql/get-async-eql-search-api.asciidoc delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java delete mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java create mode 100644 x-pack/plugin/async/build.gradle create mode 100644 x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java create mode 100644 x-pack/plugin/async/src/main/plugin-metadata/plugin-security.policy create mode 100644 x-pack/plugin/async/src/test/java/org/elasticsearch/xpack/async/AsyncResultsIndexPluginTests.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncResultsService.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/DeleteAsyncResultAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/DeleteAsyncResultRequest.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/DeleteAsyncResultsService.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/GetAsyncResultRequest.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/TransportDeleteAsyncResultAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/eql/EqlAsyncActionNames.java delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncResultsServiceTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/GetAsyncResultRequestTests.java create mode 100644 x-pack/plugin/eql/qa/security/build.gradle create mode 100644 x-pack/plugin/eql/qa/security/roles.yml create mode 100644 x-pack/plugin/eql/qa/security/src/test/java/org/elasticsearch/xpack/eql/AsyncEqlSecurityIT.java create mode 100644 x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlBlockingIntegTestCase.java rename x-pack/plugin/eql/src/{test => internalClusterTest}/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java (100%) create mode 100644 x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AsyncEqlSearchActionIT.java create mode 100644 x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/RestEqlCancellationIT.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/async/AsyncTaskManagementService.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/async/StoredAsyncResponse.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/async/StoredAsyncTask.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlAsyncGetResultAction.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlDeleteAsyncResultAction.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlGetAsyncResultAction.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlAsyncGetResultAction.java create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/async/AsyncTaskManagementServiceTests.java create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/async/StoredAsyncResponseTests.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/eql.delete.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/eql.get.json diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java index 6068e31d296be..7be4251c7f081 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Validatable; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -45,6 +46,11 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { private String query; private String tiebreakerField; + // Async settings + private TimeValue waitForCompletionTimeout; + private boolean keepOnCompletion; + private TimeValue keepAlive; + static final String KEY_FILTER = "filter"; static final String KEY_TIMESTAMP_FIELD = "timestamp_field"; static final String KEY_TIEBREAKER_FIELD = "tiebreaker_field"; @@ -53,6 +59,9 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { static final String KEY_SIZE = "size"; static final String KEY_SEARCH_AFTER = "search_after"; static final String KEY_QUERY = "query"; + static final String KEY_WAIT_FOR_COMPLETION_TIMEOUT = "wait_for_completion_timeout"; + static final String KEY_KEEP_ALIVE = "keep_alive"; + static final String KEY_KEEP_ON_COMPLETION = "keep_on_completion"; public EqlSearchRequest(String indices, String query) { indices(indices); @@ -80,6 +89,13 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par } builder.field(KEY_QUERY, query); + if (waitForCompletionTimeout != null) { + builder.field(KEY_WAIT_FOR_COMPLETION_TIMEOUT, waitForCompletionTimeout); + } + if (keepAlive != null) { + builder.field(KEY_KEEP_ALIVE, keepAlive); + } + builder.field(KEY_KEEP_ON_COMPLETION, keepOnCompletion); builder.endObject(); return builder; } @@ -181,6 +197,32 @@ public EqlSearchRequest query(String query) { return this; } + public TimeValue waitForCompletionTimeout() { + return waitForCompletionTimeout; + } + + public EqlSearchRequest waitForCompletionTimeout(TimeValue waitForCompletionTimeout) { + this.waitForCompletionTimeout = waitForCompletionTimeout; + return this; + } + + public Boolean keepOnCompletion() { + return keepOnCompletion; + } + + public void keepOnCompletion(Boolean keepOnCompletion) { + this.keepOnCompletion = keepOnCompletion; + } + + public TimeValue keepAlive() { + return keepAlive; + } + + public EqlSearchRequest keepAlive(TimeValue keepAlive) { + this.keepAlive = keepAlive; + return this; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -199,7 +241,10 @@ public boolean equals(Object o) { Objects.equals(eventCategoryField, that.eventCategoryField) && Objects.equals(implicitJoinKeyField, that.implicitJoinKeyField) && Objects.equals(searchAfterBuilder, that.searchAfterBuilder) && - Objects.equals(query, that.query); + Objects.equals(query, that.query) && + Objects.equals(waitForCompletionTimeout, that.waitForCompletionTimeout) && + Objects.equals(keepAlive, that.keepAlive) && + Objects.equals(keepOnCompletion, that.keepOnCompletion); } @Override @@ -214,7 +259,10 @@ public int hashCode() { eventCategoryField, implicitJoinKeyField, searchAfterBuilder, - query); + query, + waitForCompletionTimeout, + keepAlive, + keepOnCompletion); } public String[] indices() { diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchResponse.java index 76d224342739c..f359f3813107a 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchResponse.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.InstantiatingObjectParser; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -32,43 +33,56 @@ import java.util.List; import java.util.Objects; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + public class EqlSearchResponse { private final Hits hits; private final long tookInMillis; private final boolean isTimeout; + private final String asyncExecutionId; + private final boolean isRunning; + private final boolean isPartial; private static final class Fields { static final String TOOK = "took"; static final String TIMED_OUT = "timed_out"; static final String HITS = "hits"; + static final String ID = "id"; + static final String IS_RUNNING = "is_running"; + static final String IS_PARTIAL = "is_partial"; } private static final ParseField TOOK = new ParseField(Fields.TOOK); private static final ParseField TIMED_OUT = new ParseField(Fields.TIMED_OUT); private static final ParseField HITS = new ParseField(Fields.HITS); + private static final ParseField ID = new ParseField(Fields.ID); + private static final ParseField IS_RUNNING = new ParseField(Fields.IS_RUNNING); + private static final ParseField IS_PARTIAL = new ParseField(Fields.IS_PARTIAL); - private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("eql/search_response", true, - args -> { - int i = 0; - Hits hits = (Hits) args[i++]; - Long took = (Long) args[i++]; - Boolean timeout = (Boolean) args[i]; - return new EqlSearchResponse(hits, took, timeout); - }); - + private static final InstantiatingObjectParser PARSER; static { - PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> Hits.fromXContent(p), HITS); - PARSER.declareLong(ConstructingObjectParser.constructorArg(), TOOK); - PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), TIMED_OUT); + InstantiatingObjectParser.Builder parser = + InstantiatingObjectParser.builder("eql/search_response", true, EqlSearchResponse.class); + parser.declareObject(constructorArg(), (p, c) -> Hits.fromXContent(p), HITS); + parser.declareLong(constructorArg(), TOOK); + parser.declareBoolean(constructorArg(), TIMED_OUT); + parser.declareString(optionalConstructorArg(), ID); + parser.declareBoolean(constructorArg(), IS_RUNNING); + parser.declareBoolean(constructorArg(), IS_PARTIAL); + PARSER = parser.build(); } - public EqlSearchResponse(Hits hits, long tookInMillis, boolean isTimeout) { + public EqlSearchResponse(Hits hits, long tookInMillis, boolean isTimeout, String asyncExecutionId, + boolean isRunning, boolean isPartial) { super(); this.hits = hits == null ? Hits.EMPTY : hits; this.tookInMillis = tookInMillis; this.isTimeout = isTimeout; + this.asyncExecutionId = asyncExecutionId; + this.isRunning = isRunning; + this.isPartial = isPartial; } public static EqlSearchResponse fromXContent(XContentParser parser) { @@ -87,6 +101,18 @@ public Hits hits() { return hits; } + public String id() { + return asyncExecutionId; + } + + public boolean isRunning() { + return isRunning; + } + + public boolean isPartial() { + return isPartial; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java index 2cc82656f20cd..65f20f4c5364b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java @@ -57,7 +57,12 @@ public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomE if (randomBoolean()) { hits = new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Hits(randomEvents(), null, null, totalHits); } - return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + if (randomBoolean()) { + return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + } else { + return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean(), + randomAlphaOfLength(10), randomBoolean(), randomBoolean()); + } } public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomSequencesResponse(TotalHits totalHits) { @@ -77,7 +82,12 @@ public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomS if (randomBoolean()) { hits = new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Hits(null, seq, null, totalHits); } - return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + if (randomBoolean()) { + return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + } else { + return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean(), + randomAlphaOfLength(10), randomBoolean(), randomBoolean()); + } } public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomCountResponse(TotalHits totalHits) { @@ -97,7 +107,12 @@ public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomC if (randomBoolean()) { hits = new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Hits(null, null, cn, totalHits); } - return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + if (randomBoolean()) { + return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + } else { + return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean(), + randomAlphaOfLength(10), randomBoolean(), randomBoolean()); + } } public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomInstance(TotalHits totalHits) { diff --git a/docs/reference/eql/delete-async-eql-search-api.asciidoc b/docs/reference/eql/delete-async-eql-search-api.asciidoc new file mode 100644 index 0000000000000..9b585c28c5515 --- /dev/null +++ b/docs/reference/eql/delete-async-eql-search-api.asciidoc @@ -0,0 +1,47 @@ +[role="xpack"] +[testenv="basic"] + +[[delete-async-eql-search-api]] +=== Delete async EQL search API +++++ +Delete async EQL search +++++ + +dev::[] + +Deletes an <> or a +<>. The API also +deletes results for the search. + +[source,console] +---- +DELETE /_eql/search/FkpMRkJGS1gzVDRlM3g4ZzMyRGlLbkEaTXlJZHdNT09TU2VTZVBoNDM3cFZMUToxMDM= +---- +// TEST[skip: no access to search ID] + +[[delete-async-eql-search-api-request]] +==== {api-request-title} + +`DELETE /_eql/search/` + +[[delete-async-eql-search-api-prereqs]] +==== {api-prereq-title} + +See <>. + +[[delete-async-eql-search-api-limitations]] +===== Limitations + +See <>. + +[[delete-async-eql-search-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +Identifier for the search to delete. ++ +A search ID is provided in the <>'s response for +an <>. A search ID is also provided if the +request's <> parameter +is `true`. diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc index 186228fcb152d..a7a04a8f2d28e 100644 --- a/docs/reference/eql/eql-search-api.asciidoc +++ b/docs/reference/eql/eql-search-api.asciidoc @@ -81,6 +81,68 @@ Defaults to `open`. include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +`keep_alive`:: ++ +-- +(Optional, <>) +Period for which the search and its results are stored on the cluster. Defaults +to `5d` (five days). + +When this period expires, the search and its results are deleted, even if the +search is still ongoing. + +If the <> parameter is +`false`, {es} only stores <<> that do not +complete within the period set by the +<> +parameter, regardless of this value. + +[IMPORTANT] +==== +You can also specify this value using the `keep_alive` request body parameter. +If both parameters are specified, only the query parameter is used. +==== +-- + +`keep_on_completion`:: ++ +-- +(Optional, boolean) +If `true`, the search and its results are stored on the cluster. + +If `false`, the search and its results are stored on the cluster only if the +request does not complete during the period set by the +<> +parameter. Defaults to `false`. + +[IMPORTANT] +==== +You can also specify this value using the `keep_on_completion` request body +parameter. If both parameters are specified, only the query parameter is used. +==== +-- + +`wait_for_completion_timeout`:: ++ +-- +(Optional, <>) +Timeout duration to wait for the request to finish. Defaults to no +timeout, meaning the request waits for complete search results. + +If this parameter is specified and the request completes during this period, +complete search results are returned. + +If the request does not complete during this period, the search becomes an +<>. + +[IMPORTANT] +==== +You can also specify this value using the `wait_for_completion_timeout` request +body parameter. If both parameters are specified, only the query parameter is +used. +==== +-- + [[eql-search-api-request-body]] ==== {api-request-body-title} @@ -107,6 +169,48 @@ runs. (Optional, string) Reserved for future use. +`keep_alive`:: ++ +-- +(Optional, <>) +Period for which the search and its results are stored on the cluster. Defaults +to `5d` (five days). + +When this period expires, the search and its results are deleted, even if the +search is still ongoing. + +If the <> parameter is +`false`, {es} only stores <<> that do not +complete within the period set by the +<> +parameter, regardless of this value. + +[IMPORTANT] +==== +You can also specify this value using the `keep_alive` query parameter. +If both parameters are specified, only the query parameter is used. +==== +-- + +[[eql-search-api-keep-on-completion]] +`keep_on_completion`:: ++ +-- +(Optional, boolean) +If `true`, the search and its results are stored on the cluster. + +If `false`, the search and its results are stored on the cluster only if the +request does not complete during the period set by the +<> +parameter. Defaults to `false`. + +[IMPORTANT] +==== +You can also specify this value using the `keep_on_completion` query parameter. +If both parameters are specified, only the query parameter is used. +==== +-- + [[eql-search-api-request-query-param]] `query`:: (Required, string) @@ -145,10 +249,72 @@ milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. -- +[[eql-search-api-wait-for-completion-timeout]] +`wait_for_completion_timeout`:: ++ +-- +(Optional, <>) +Timeout duration to wait for the request to finish. Defaults to no +timeout, meaning the request waits for complete search results. + +If this parameter is specified and the request completes during this period, +complete search results are returned. + +If the request does not complete during this period, the search becomes an +<>. + +[IMPORTANT] +==== +You can also specify this value using the `wait_for_completion_timeout` query +parameter. If both parameters are specified, only the query parameter is used. +==== +-- + [role="child_attributes"] [[eql-search-api-response-body]] ==== {api-response-body-title} +[[eql-search-api-response-body-search-id]] +`id`:: ++ +-- +Identifier for the search. + +This search ID is only provided if one of the following conditions is met: + +* A search request does not return complete results during the + <> + parameter's timeout period, becoming an <>. + +* The search request's <> + parameter is `true`. + +You can use this ID with the <> to get the current status and available results for the search. +-- + +`is_partial`:: +(boolean) +If `true`, the response does not contain complete search results. + +`is_running`:: ++ +-- +(boolean) +If `true`, the search request is still executing. + +[IMPORTANT] +==== +If this parameter and the `is_partial` parameter are `true`, the search is an +<>. If the `keep_alive` period does not +pass, the complete search results will be available when the search completes. + +If `is_partial` is `true` but `is_running` is `false`, the search returned +partial results due to a failure. Only some shards returned results or the node +coordinating the search failed. +==== +-- + `took`:: + -- @@ -332,6 +498,8 @@ in ascending order. [source,console-result] ---- { + "is_partial": false, + "is_running": false, "took": 6, "timed_out": false, "hits": { @@ -447,6 +615,8 @@ the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. [source,console-result] ---- { + "is_partial": false, + "is_running": false, "took": 6, "timed_out": false, "hits": { diff --git a/docs/reference/eql/get-async-eql-search-api.asciidoc b/docs/reference/eql/get-async-eql-search-api.asciidoc new file mode 100644 index 0000000000000..f88b19090742d --- /dev/null +++ b/docs/reference/eql/get-async-eql-search-api.asciidoc @@ -0,0 +1,82 @@ +[role="xpack"] +[testenv="basic"] + +[[get-async-eql-search-api]] +=== Get async EQL search API +++++ +Get async EQL search +++++ + +dev::[] + +Returns the current status and available results for an <> or a <>. + +[source,console] +---- +GET /_eql/search/FkpMRkJGS1gzVDRlM3g4ZzMyRGlLbkEaTXlJZHdNT09TU2VTZVBoNDM3cFZMUToxMDM= +---- +// TEST[skip: no access to search ID] + +[[get-async-eql-search-api-request]] +==== {api-request-title} + +`GET /_eql/search/` + +[[get-async-eql-search-api-prereqs]] +==== {api-prereq-title} + +See <>. + +[[get-async-eql-search-api-limitations]] +===== Limitations + +See <>. + +[[get-async-eql-search-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +Identifier for the search. ++ +A search ID is provided in the <>'s response for +an <>. A search ID is also provided if the +request's <> parameter +is `true`. + +[[get-async-eql-search-api-query-params]] +==== {api-query-parms-title} + +`keep_alive`:: +(Optional, <>) +Period for which the search and its results are stored on the cluster. Defaults +to the `keep_alive` value set by the search's <> request. ++ +If specified, this parameter sets a new `keep_alive` period for the search, +starting when the get async EQL search API request executes. This new period +overwrites the one specified in the EQL search API request. ++ +When this period expires, the search and its results are deleted, even if the +search is ongoing. + +`wait_for_completion_timeout`:: +(Optional, <>) +Timeout duration to wait for the request to finish. Defaults to no timeout, +meaning the request waits for complete search results. ++ +If this parameter is specified and the request completes during this period, +complete search results are returned. ++ +If the request does not complete during this period, partial results, if +available, are returned. + +[role="child_attributes"] +[[get-async-eql-search-api-response-body]] +==== {api-response-body-title} + +The async EQL search API returns the same response body as the EQL search API. +See the EQL search API's <>. \ No newline at end of file diff --git a/docs/reference/eql/search.asciidoc b/docs/reference/eql/search.asciidoc index b39339dbdc063..b339a645cb8f9 100644 --- a/docs/reference/eql/search.asciidoc +++ b/docs/reference/eql/search.asciidoc @@ -70,6 +70,8 @@ https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. [source,console-result] ---- { + "is_partial": false, + "is_running": false, "took": 60, "timed_out": false, "hits": { @@ -174,6 +176,8 @@ the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. [source,console-result] ---- { + "is_partial": false, + "is_running": false, "took": 60, "timed_out": false, "hits": { @@ -292,6 +296,8 @@ contains the shared `agent.id` value for each matching event. [source,console-result] ---- { + "is_partial": false, + "is_running": false, "took": 60, "timed_out": false, "hits": { @@ -462,6 +468,261 @@ GET /sec_logs/_eql/search ---- ==== +[discrete] +[[eql-search-async]] +=== Run an async EQL search + +EQL searches in {es} are designed to run on large volumes of data quickly, +often returning results in milliseconds. Because of this, the EQL search API +runs _synchronous_ searches by default. This means the search request waits for +complete results before returning a response. + +However, complete results can take longer for searches across: + +* <> +* <> +* Many shards + +To avoid long waits, you can use the EQL search API's +`wait_for_completion_timeout` parameter to run an _asynchronous_, or _async_, +search. + +Set the `wait_for_completion_timeout` parameter to a duration you'd like to wait +for complete search results. If the search request does not finish within this +period, the search becomes an async search. The EQL search +API returns a response that includes: + +* A search ID, which can be used to monitor the progress of the async search and + retrieve complete results when it finishes. +* An `is_partial` value of `true`, indicating the response does not contain + complete search results. +* An `is_running` value of `true`, indicating the search is async and ongoing. +* Partial search results, if available, in the `hits` property. + +The async search continues to run in the background without blocking +other requests. + +[%collapsible] +.*Example* +==== +The following request searches the `frozen_sec_logs` index, which has been +<> for storage and is rarely searched. + +Because searches on frozen indices are expected to take longer to complete, the +request contains a `wait_for_completion_timeout` parameter value of `2s` +(two seconds). + +If the request does not return complete results in two seconds, the search +becomes an async search and a search ID is returned. + +[source,console] +---- +GET /frozen_sec_logs/_eql/search +{ + "wait_for_completion_timeout": "2s", + "query": """ + process where process.name == "cmd.exe" + """ +} +---- +// TEST[s/frozen_sec_logs/sec_logs/] + +After two seconds, the request returns the following response. Note the +`is_partial` and `is_running` properties are `true`, indicating an ongoing async +search. + +[source,console-result] +---- +{ + "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=", + "is_partial": true, + "is_running": true, + "took": 2000, + "timed_out": false, + "hits": ... +} +---- +// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/] +// TESTRESPONSE[s/"is_partial": true/"is_partial": $body.is_partial/] +// TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/] +// TESTRESPONSE[s/"took": 2000/"took": $body.took/] +// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] +==== + +You can use the the returned search ID and the <> to check the progress of an ongoing async search. + +The get async EQL search API also accepts a `wait_for_completion_timeout` query +parameter. Set the `wait_for_completion_timeout` parameter to a duration you'd +like to wait for complete search results. If the search does not finish during +this period, partial search results, if available, are returned. + +[%collapsible] +.*Example* +==== +The following get async EQL search API request checks the progress of the +previous async EQL search. The request specifies a `wait_for_completion_timeout` +query parameter value of `2s` (two seconds). + +[source,console] +---- +GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?wait_for_completion_timeout=2s +---- +// TEST[skip: no access to search ID] + +The request returns the following response. Note the `is_partial` and +`is_running` properties are `false`, indicating the async EQL search has +finished and the search results in the `hits` property are complete. + +[source,console-result] +---- +{ + "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=", + "is_partial": false, + "is_running": false, + "took": 2000, + "timed_out": false, + "hits": ... +} +---- +// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/] +// TESTRESPONSE[s/"took": 2000/"took": $body.took/] +// TESTRESPONSE[s/"_index": "frozen_sec_logs"/"_index": "sec_logs"/] +// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] +==== + +[discrete] +[[eql-search-store-async-eql-search]] +=== Change the search retention period + +By default, the EQL search API only stores async searches and their results for +five days. After this period, any ongoing searches or saved results are deleted. + +You can use the EQL search API's `keep_alive` parameter to change the duration +of this period. + +.*Example* +[%collapsible] +==== +In the following EQL search API request, the `keep_alive` parameter is `2d` (two +days). This means that if the search becomes async, its results +are stored on the cluster for two days. After two days, the async +search and its results are deleted, even if it's still ongoing. + +[source,console] +---- +GET /sec_logs/_eql/search +{ + "keep_alive": "2d", + "wait_for_completion_timeout": "2s", + "query": """ + process where process.name == "cmd.exe" + """ +} +---- +==== + +You can use the <>'s +`keep_alive` query parameter to later change the retention period. The new +retention period starts after the get async EQL search API request executes. + +.*Example* +[%collapsible] +==== +The following get async EQL search API request sets the `keep_alive` query +parameter to `5d` (five days). The async search and its results are deleted five +days after the get async EQL search API request executes. + +[source,console] +---- +GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d +---- +// TEST[skip: no access to search ID] +==== + +You can use the <> to +manually delete an async EQL search before the `keep_alive` period ends. If the +search is still ongoing, this cancels the search request. + +.*Example* +[%collapsible] +==== +The following delete async EQL search API request deletes an async EQL search +and its results. + +[source,console] +---- +DELETE /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d +---- +// TEST[skip: no access to search ID] +==== + +[discrete] +[[eql-search-store-sync-eql-search]] +=== Store synchronous EQL searches + +By default, the EQL search API only stores async searches that cannot be +completed within the period set by the `wait_for_completion_timeout` parameter. + +To save the results of searches that complete during this period, set the +`keep_on_completion` parameter to `true`. + +[%collapsible] +.*Example* +==== +In the following EQL search API request, the `keep_on_completion` parameter is +`true`. This means the search results are stored on the cluster, even if +the search completes within the `2s` (two-second) period set by the +`wait_for_completion_timeout` parameter. + +[source,console] +---- +GET /sec_logs/_eql/search +{ + "keep_on_completion": true, + "wait_for_completion_timeout": "2s", + "query": """ + process where process.name == "cmd.exe" + """ +} +---- + +The API returns the following response. Note that a search ID is provided in the +`id` property. The `is_partial` and `is_running` properties are `false`, +indicating the EQL search was synchronous and returned complete search results. + +[source,console-result] +---- +{ + "id": "FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=", + "is_partial": false, + "is_running": false, + "took": 52, + "timed_out": false, + "hits": ... +} +---- +// TESTRESPONSE[s/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=/$body.id/] +// TESTRESPONSE[s/"took": 52/"took": $body.took/] +// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] + +You can use the search ID and the <> to retrieve the same results later. + +[source,console] +---- +GET /_eql/search/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY= +---- +// TEST[skip: no access to search ID] +==== + +Saved synchronous searches are still subject to the storage retention period set +by the `keep_alive` parameter. After this period, the search and its saved +results are deleted. + +You can also manually delete saved synchronous searches using the +<>. + [discrete] [[eql-search-case-sensitive]] === Run a case-sensitive EQL search @@ -484,6 +745,7 @@ query. ---- GET /sec_logs/_eql/search { + "keep_on_completion": true, "case_sensitive": true, "query": """ process where stringContains(process.path, "System32") diff --git a/docs/reference/search.asciidoc b/docs/reference/search.asciidoc index b2e5c50f11741..9f7f36e179ff7 100644 --- a/docs/reference/search.asciidoc +++ b/docs/reference/search.asciidoc @@ -172,6 +172,10 @@ ifdef::permanently-unreleased-branch[] include::eql/eql-search-api.asciidoc[] +include::eql/get-async-eql-search-api.asciidoc[] + +include::eql/delete-async-eql-search-api.asciidoc[] + endif::[] include::search/count.asciidoc[] diff --git a/server/src/main/java/org/elasticsearch/action/support/ListenerTimeouts.java b/server/src/main/java/org/elasticsearch/action/support/ListenerTimeouts.java index df9afd32ca21c..3305ae891a802 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ListenerTimeouts.java +++ b/server/src/main/java/org/elasticsearch/action/support/ListenerTimeouts.java @@ -26,6 +26,7 @@ import org.elasticsearch.threadpool.ThreadPool; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; public class ListenerTimeouts { @@ -41,9 +42,29 @@ public class ListenerTimeouts { * @param listenerName name of the listener for timeout exception * @return the wrapped listener that will timeout */ - public static ActionListener wrapWithTimeout(ThreadPool threadPool, ActionListener listener, + public static ActionListener wrapWithTimeout(ThreadPool threadPool, ActionListener listener, TimeValue timeout, String executor, String listenerName) { - TimeoutableListener wrappedListener = new TimeoutableListener<>(listener, timeout, listenerName); + return wrapWithTimeout(threadPool, timeout, executor, listener, (ignore) -> { + String timeoutMessage = "[" + listenerName + "]" + " timed out after [" + timeout + "]"; + listener.onFailure(new ElasticsearchTimeoutException(timeoutMessage)); + }); + } + + /** + * Wraps a listener with a listener that can timeout. After the timeout period the + * onTimeout Runnable will be called. + * + * @param threadPool used to schedule the timeout + * @param timeout period before listener failed + * @param executor to use for scheduling timeout + * @param listener to that can timeout + * @param onTimeout consumer will be called and the resulting wrapper will be passed to it as a parameter + * @return the wrapped listener that will timeout + */ + public static ActionListener wrapWithTimeout(ThreadPool threadPool, TimeValue timeout, String executor, + ActionListener listener, + Consumer> onTimeout) { + TimeoutableListener wrappedListener = new TimeoutableListener<>(listener, onTimeout); wrappedListener.cancellable = threadPool.schedule(wrappedListener, timeout, executor); return wrappedListener; } @@ -52,14 +73,12 @@ private static class TimeoutableListener implements ActionListener delegate; - private final TimeValue timeout; - private final String listenerName; + private final Consumer> onTimeout; private volatile Scheduler.ScheduledCancellable cancellable; - private TimeoutableListener(ActionListener delegate, TimeValue timeout, String listenerName) { + private TimeoutableListener(ActionListener delegate, Consumer> onTimeout) { this.delegate = delegate; - this.timeout = timeout; - this.listenerName = listenerName; + this.onTimeout = onTimeout; } @Override @@ -81,8 +100,7 @@ public void onFailure(Exception e) { @Override public void run() { if (isDone.compareAndSet(false, true)) { - String timeoutMessage = "[" + listenerName + "]" + " timed out after [" + timeout + "]"; - delegate.onFailure(new ElasticsearchTimeoutException(timeoutMessage)); + onTimeout.accept(this); } } } diff --git a/x-pack/plugin/async-search/build.gradle b/x-pack/plugin/async-search/build.gradle index 4f5ed0b9c6706..4e501784cc219 100644 --- a/x-pack/plugin/async-search/build.gradle +++ b/x-pack/plugin/async-search/build.gradle @@ -28,6 +28,7 @@ dependencies { compileOnly project(path: xpackModule('core'), configuration: 'default') testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') testImplementation project(path: xpackModule('ilm')) + testImplementation project(path: xpackModule('async')) } dependencyLicenses { diff --git a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java index 60102798f6fee..9db013e7efda1 100644 --- a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java +++ b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java @@ -29,7 +29,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.core.security.authc.AuthenticationServiceField.RUN_AS_USER_HEADER; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -import static org.elasticsearch.xpack.search.AsyncSearch.INDEX; +import static org.elasticsearch.xpack.core.XPackPlugin.ASYNC_RESULTS_INDEX; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -84,7 +84,7 @@ private void testCase(String user, String other) throws Exception { // other and user cannot access the result from direct get calls AsyncExecutionId searchId = AsyncExecutionId.decode(id); for (String runAs : new String[] {user, other}) { - exc = expectThrows(ResponseException.class, () -> get(INDEX, searchId.getDocId(), runAs)); + exc = expectThrows(ResponseException.class, () -> get(ASYNC_RESULTS_INDEX, searchId.getDocId(), runAs)); assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(403)); assertThat(exc.getMessage(), containsString("unauthorized")); } diff --git a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchActionIT.java b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchActionIT.java index eae3be39a9890..3cdac5a24b688 100644 --- a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchActionIT.java +++ b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchActionIT.java @@ -18,6 +18,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase.SuiteScopeTestCase; import org.elasticsearch.test.junit.annotations.TestIssueLogging; +import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; @@ -369,7 +370,7 @@ public void testRemoveAsyncIndex() throws Exception { assertThat(response.getExpirationTime(), greaterThan(now)); // remove the async search index - client().admin().indices().prepareDelete(AsyncSearch.INDEX).get(); + client().admin().indices().prepareDelete(XPackPlugin.ASYNC_RESULTS_INDEX).get(); Exception exc = expectThrows(Exception.class, () -> getAsyncSearch(response.getId())); Throwable cause = exc instanceof ExecutionException ? diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java index eda9c84b9a71f..f21e3628c6878 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java @@ -7,57 +7,34 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xpack.core.async.AsyncTaskIndexService; -import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; -import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.function.Supplier; -import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; -import static org.elasticsearch.xpack.search.AsyncSearchMaintenanceService.ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING; +import static org.elasticsearch.xpack.core.async.AsyncTaskMaintenanceService.ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING; public final class AsyncSearch extends Plugin implements ActionPlugin { - public static final String INDEX = ".async-search"; - private final Settings settings; - - public AsyncSearch(Settings settings) { - this.settings = settings; - } @Override public List> getActions() { return Arrays.asList( new ActionHandler<>(SubmitAsyncSearchAction.INSTANCE, TransportSubmitAsyncSearchAction.class), - new ActionHandler<>(GetAsyncSearchAction.INSTANCE, TransportGetAsyncSearchAction.class), - new ActionHandler<>(DeleteAsyncSearchAction.INSTANCE, TransportDeleteAsyncSearchAction.class) + new ActionHandler<>(GetAsyncSearchAction.INSTANCE, TransportGetAsyncSearchAction.class) ); } @@ -73,31 +50,6 @@ public List getRestHandlers(Settings settings, RestController restC ); } - @Override - public Collection createComponents(Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier) { - if (DiscoveryNode.isDataNode(environment.settings())) { - // only data nodes should be eligible to run the maintenance service. - AsyncTaskIndexService indexService = - new AsyncTaskIndexService<>(AsyncSearch.INDEX, clusterService, threadPool.getThreadContext(), client, ASYNC_SEARCH_ORIGIN, - AsyncSearchResponse::new, namedWriteableRegistry); - AsyncSearchMaintenanceService maintenanceService = - new AsyncSearchMaintenanceService(clusterService, nodeEnvironment.nodeId(), settings, threadPool, indexService); - return Collections.singletonList(maintenanceService); - } else { - return Collections.emptyList(); - } - } - @Override public List> getSettings() { return Collections.singletonList(ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING); diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java deleted file mode 100644 index 65be6b6ba1471..0000000000000 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.search; - -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.async.AsyncTaskIndexService; -import org.elasticsearch.xpack.core.async.AsyncTaskMaintenanceService; - -public class AsyncSearchMaintenanceService extends AsyncTaskMaintenanceService { - - /** - * Controls the interval at which the cleanup is scheduled. - * Defaults to 1h. It is an undocumented/expert setting that - * is mainly used by integration tests to make the garbage - * collection of search responses more reactive. - */ - public static final Setting ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING = - Setting.timeSetting("async_search.index_cleanup_interval", TimeValue.timeValueHours(1), Setting.Property.NodeScope); - - AsyncSearchMaintenanceService(ClusterService clusterService, - String localNodeId, - Settings nodeSettings, - ThreadPool threadPool, - AsyncTaskIndexService indexService) { - super(clusterService, AsyncSearch.INDEX, localNodeId, threadPool, indexService, - ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING.get(nodeSettings)); - } -} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java index 8bb254a21743a..68fcbf091b2ad 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java @@ -24,6 +24,7 @@ import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.threadpool.Scheduler.Cancellable; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.async.AsyncExecutionId; @@ -127,10 +128,16 @@ Listener getSearchProgressActionListener() { /** * Update the expiration time of the (partial) response. */ + @Override public void setExpirationTime(long expirationTimeMillis) { this.expirationTimeMillis = expirationTimeMillis; } + @Override + public void cancelTask(TaskManager taskManager, Runnable runnable, String reason) { + cancelTask(runnable, reason); + } + /** * Cancels the running task and its children. */ diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java index faab51dc3af73..96ac890345bc1 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java @@ -7,10 +7,10 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestHandler.Route; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; -import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; +import org.elasticsearch.xpack.core.async.DeleteAsyncResultRequest; +import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; import java.io.IOException; @@ -34,7 +34,7 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - DeleteAsyncSearchAction.Request delete = new DeleteAsyncSearchAction.Request(request.param("id")); - return channel -> client.execute(DeleteAsyncSearchAction.INSTANCE, delete, new RestToXContentListener<>(channel)); + DeleteAsyncResultRequest delete = new DeleteAsyncResultRequest(request.param("id")); + return channel -> client.execute(DeleteAsyncResultAction.INSTANCE, delete, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java index 313d73e078c9f..8d8eab8764645 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java @@ -9,6 +9,7 @@ import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestStatusToXContentListener; +import org.elasticsearch.xpack.core.async.GetAsyncResultRequest; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import java.util.List; @@ -33,7 +34,7 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { - GetAsyncSearchAction.Request get = new GetAsyncSearchAction.Request(request.param("id")); + GetAsyncResultRequest get = new GetAsyncResultRequest(request.param("id")); if (request.hasParam("wait_for_completion_timeout")) { get.setWaitForCompletionTimeout(request.paramAsTime("wait_for_completion_timeout", get.getWaitForCompletionTimeout())); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java deleted file mode 100644 index aa68ff53047a3..0000000000000 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.search; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.ResourceNotFoundException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionListenerResponseHandler; -import org.elasticsearch.action.delete.DeleteResponse; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportRequestOptions; -import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.async.AsyncExecutionId; -import org.elasticsearch.xpack.core.async.AsyncTaskIndexService; -import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; -import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; - -import java.io.IOException; - -import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; - -public class TransportDeleteAsyncSearchAction extends HandledTransportAction { - private static final Logger logger = LogManager.getLogger(TransportDeleteAsyncSearchAction.class); - - private final ClusterService clusterService; - private final TransportService transportService; - private final AsyncTaskIndexService store; - - @Inject - public TransportDeleteAsyncSearchAction(TransportService transportService, - ActionFilters actionFilters, - ClusterService clusterService, - ThreadPool threadPool, - NamedWriteableRegistry registry, - Client client) { - super(DeleteAsyncSearchAction.NAME, transportService, actionFilters, DeleteAsyncSearchAction.Request::new); - this.store = new AsyncTaskIndexService<>(AsyncSearch.INDEX, clusterService, threadPool.getThreadContext(), client, - ASYNC_SEARCH_ORIGIN, AsyncSearchResponse::new, registry); - this.clusterService = clusterService; - this.transportService = transportService; - } - - @Override - protected void doExecute(Task task, DeleteAsyncSearchAction.Request request, ActionListener listener) { - try { - AsyncExecutionId searchId = AsyncExecutionId.decode(request.getId()); - DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId()); - if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId()) || node == null) { - cancelTaskAndDeleteResult(searchId, listener); - } else { - TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); - transportService.sendRequest(node, DeleteAsyncSearchAction.NAME, request, builder.build(), - new ActionListenerResponseHandler<>(listener, AcknowledgedResponse::new, ThreadPool.Names.SAME)); - } - } catch (Exception exc) { - listener.onFailure(exc); - } - } - - void cancelTaskAndDeleteResult(AsyncExecutionId searchId, ActionListener listener) throws IOException { - AsyncSearchTask task = store.getTask(taskManager, searchId, AsyncSearchTask.class); - if (task != null) { - //the task was found and gets cancelled. The response may or may not be found, but we will return 200 anyways. - task.cancelTask(() -> store.deleteResponse(searchId, - ActionListener.wrap( - r -> listener.onResponse(new AcknowledgedResponse(true)), - exc -> { - RestStatus status = ExceptionsHelper.status(ExceptionsHelper.unwrapCause(exc)); - //the index may not be there (no initial async search response stored yet?): we still want to return 200 - //note that index missing comes back as 200 hence it's handled in the onResponse callback - if (status == RestStatus.NOT_FOUND) { - listener.onResponse(new AcknowledgedResponse(true)); - } else { - logger.error(() -> new ParameterizedMessage("failed to clean async-search [{}]", searchId.getEncoded()), exc); - listener.onFailure(exc); - } - })), "cancelled by user"); - } else { - // the task was not found (already cancelled, already completed, or invalid id?) - // we fail if the response is not found in the index - ActionListener deleteListener = ActionListener.wrap( - resp -> { - if (resp.status() == RestStatus.NOT_FOUND) { - listener.onFailure(new ResourceNotFoundException(searchId.getEncoded())); - } else { - listener.onResponse(new AcknowledgedResponse(true)); - } - }, - exc -> { - logger.error(() -> new ParameterizedMessage("failed to clean async-search [{}]", searchId.getEncoded()), exc); - listener.onFailure(exc); - } - ); - //we get before deleting to verify that the user is authorized - store.getResponse(searchId, false, - ActionListener.wrap(res -> store.deleteResponse(searchId, deleteListener), listener::onFailure)); - } - } -} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index 8cb157dfe297c..14b01b5259f0f 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -5,11 +5,6 @@ */ package org.elasticsearch.xpack.search; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.ActionFilters; @@ -19,23 +14,22 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.async.AsyncExecutionId; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.core.async.AsyncResultsService; import org.elasticsearch.xpack.core.async.AsyncTaskIndexService; +import org.elasticsearch.xpack.core.async.GetAsyncResultRequest; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; -public class TransportGetAsyncSearchAction extends HandledTransportAction { - private final Logger logger = LogManager.getLogger(TransportGetAsyncSearchAction.class); - private final ClusterService clusterService; +public class TransportGetAsyncSearchAction extends HandledTransportAction { + private final AsyncResultsService resultsService; private final TransportService transportService; - private final AsyncTaskIndexService store; @Inject public TransportGetAsyncSearchAction(TransportService transportService, @@ -44,113 +38,31 @@ public TransportGetAsyncSearchAction(TransportService transportService, NamedWriteableRegistry registry, Client client, ThreadPool threadPool) { - super(GetAsyncSearchAction.NAME, transportService, actionFilters, GetAsyncSearchAction.Request::new); - this.clusterService = clusterService; + super(GetAsyncSearchAction.NAME, transportService, actionFilters, GetAsyncResultRequest::new); this.transportService = transportService; - this.store = new AsyncTaskIndexService<>(AsyncSearch.INDEX, clusterService, threadPool.getThreadContext(), client, - ASYNC_SEARCH_ORIGIN, AsyncSearchResponse::new, registry); + this.resultsService = createResultsService(transportService, clusterService, registry, client, threadPool); } - @Override - protected void doExecute(Task task, GetAsyncSearchAction.Request request, ActionListener listener) { - try { - long nowInMillis = System.currentTimeMillis(); - AsyncExecutionId searchId = AsyncExecutionId.decode(request.getId()); - DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId()); - if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId()) || node == null) { - if (request.getKeepAlive().getMillis() > 0) { - long expirationTime = nowInMillis + request.getKeepAlive().getMillis(); - store.updateExpirationTime(searchId.getDocId(), expirationTime, - ActionListener.wrap( - p -> getSearchResponseFromTask(searchId, request, nowInMillis, expirationTime, listener), - exc -> { - //don't log when: the async search document or its index is not found. That can happen if an invalid - //search id is provided or no async search initial response has been stored yet. - RestStatus status = ExceptionsHelper.status(ExceptionsHelper.unwrapCause(exc)); - if (status != RestStatus.NOT_FOUND) { - logger.error(() -> new ParameterizedMessage("failed to update expiration time for async-search [{}]", - searchId.getEncoded()), exc); - } - listener.onFailure(new ResourceNotFoundException(searchId.getEncoded())); - } - )); - } else { - getSearchResponseFromTask(searchId, request, nowInMillis, -1, listener); - } - } else { - TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); - transportService.sendRequest(node, GetAsyncSearchAction.NAME, request, builder.build(), - new ActionListenerResponseHandler<>(listener, AsyncSearchResponse::new, ThreadPool.Names.SAME)); - } - } catch (Exception exc) { - listener.onFailure(exc); - } + static AsyncResultsService createResultsService(TransportService transportService, + ClusterService clusterService, + NamedWriteableRegistry registry, + Client client, + ThreadPool threadPool) { + AsyncTaskIndexService store = new AsyncTaskIndexService<>(XPackPlugin.ASYNC_RESULTS_INDEX, clusterService, + threadPool.getThreadContext(), client, ASYNC_SEARCH_ORIGIN, AsyncSearchResponse::new, registry); + return new AsyncResultsService<>(store, true, AsyncSearchTask.class, AsyncSearchTask::addCompletionListener, + transportService.getTaskManager(), clusterService); } - private void getSearchResponseFromTask(AsyncExecutionId searchId, - GetAsyncSearchAction.Request request, - long nowInMillis, - long expirationTimeMillis, - ActionListener listener) { - try { - final AsyncSearchTask task = store.getTask(taskManager, searchId, AsyncSearchTask.class); - if (task == null) { - getSearchResponseFromIndex(searchId, request, nowInMillis, listener); - return; - } - - if (task.isCancelled()) { - listener.onFailure(new ResourceNotFoundException(searchId.getEncoded())); - return; - } - - if (expirationTimeMillis != -1) { - task.setExpirationTime(expirationTimeMillis); - } - task.addCompletionListener(new ActionListener() { - @Override - public void onResponse(AsyncSearchResponse response) { - sendFinalResponse(request, response, nowInMillis, listener); - } - - @Override - public void onFailure(Exception exc) { - listener.onFailure(exc); - } - }, request.getWaitForCompletionTimeout()); - } catch (Exception exc) { - listener.onFailure(exc); - } - } - - private void getSearchResponseFromIndex(AsyncExecutionId searchId, - GetAsyncSearchAction.Request request, - long nowInMillis, - ActionListener listener) { - store.getResponse(searchId, true, - new ActionListener() { - @Override - public void onResponse(AsyncSearchResponse response) { - sendFinalResponse(request, response, nowInMillis, listener); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); - } - - private void sendFinalResponse(GetAsyncSearchAction.Request request, - AsyncSearchResponse response, - long nowInMillis, - ActionListener listener) { - // check if the result has expired - if (response.getExpirationTime() < nowInMillis) { - listener.onFailure(new ResourceNotFoundException(request.getId())); - return; + @Override + protected void doExecute(Task task, GetAsyncResultRequest request, ActionListener listener) { + DiscoveryNode node = resultsService.getNode(request.getId()); + if (node == null || resultsService.isLocalNode(node)) { + resultsService.retrieveResult(request, listener); + } else { + TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); + transportService.sendRequest(node, GetAsyncSearchAction.NAME, request, builder.build(), + new ActionListenerResponseHandler<>(listener, AsyncSearchResponse::new, ThreadPool.Names.SAME)); } - - listener.onResponse(response); } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index ea93598f68e6e..b8894e5bba778 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -34,6 +34,7 @@ import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.async.AsyncExecutionId; import org.elasticsearch.xpack.core.async.AsyncTaskIndexService; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; @@ -69,7 +70,7 @@ public TransportSubmitAsyncSearchAction(ClusterService clusterService, this.requestToAggReduceContextBuilder = request -> searchService.aggReduceContextBuilder(request).forFinalReduction(); this.searchAction = searchAction; this.threadContext = transportService.getThreadPool().getThreadContext(); - this.store = new AsyncTaskIndexService<>(AsyncSearch.INDEX, clusterService, threadContext, client, + this.store = new AsyncTaskIndexService<>(XPackPlugin.ASYNC_RESULTS_INDEX, clusterService, threadContext, client, ASYNC_SEARCH_ORIGIN, AsyncSearchResponse::new, registry); } @@ -98,7 +99,7 @@ public void onResponse(AsyncSearchResponse searchResponse) { // creates the fallback response if the node crashes/restarts in the middle of the request // TODO: store intermediate results ? AsyncSearchResponse initialResp = searchResponse.clone(searchResponse.getId()); - store.storeInitialResponse(docId, searchTask.getOriginHeaders(), initialResp, + store.createResponse(docId, searchTask.getOriginHeaders(), initialResp, new ActionListener() { @Override public void onResponse(IndexResponse r) { @@ -194,7 +195,7 @@ private void onFinalResponse(CancellableTask submitTask, } try { - store.storeFinalResponse(searchTask.getExecutionId().getDocId(), threadContext.getResponseHeaders(),response, + store.updateResponse(searchTask.getExecutionId().getDocId(), threadContext.getResponseHeaders(),response, ActionListener.wrap(resp -> unregisterTaskAndMoveOn(searchTask, nextAction), exc -> { Throwable cause = ExceptionsHelper.unwrapCause(exc); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java index e4377414dc056..911288674a523 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.xpack.async.AsyncResultsIndexPlugin; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -30,8 +31,11 @@ import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.core.async.AsyncExecutionId; +import org.elasticsearch.xpack.core.async.AsyncTaskMaintenanceService; +import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; +import org.elasticsearch.xpack.core.async.DeleteAsyncResultRequest; +import org.elasticsearch.xpack.core.async.GetAsyncResultRequest; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; -import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; @@ -49,8 +53,8 @@ import java.util.List; import java.util.concurrent.ExecutionException; -import static org.elasticsearch.xpack.search.AsyncSearch.INDEX; -import static org.elasticsearch.xpack.search.AsyncSearchMaintenanceService.ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING; +import static org.elasticsearch.xpack.core.XPackPlugin.ASYNC_RESULTS_INDEX; +import static org.elasticsearch.xpack.core.async.AsyncTaskMaintenanceService.ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -79,7 +83,7 @@ public List getAggregations() { @Before public void startMaintenanceService() { - for (AsyncSearchMaintenanceService service : internalCluster().getDataNodeInstances(AsyncSearchMaintenanceService.class)) { + for (AsyncTaskMaintenanceService service : internalCluster().getDataNodeInstances(AsyncTaskMaintenanceService.class)) { if (service.lifecycleState() == Lifecycle.State.STOPPED) { // force the service to start again service.start(); @@ -91,7 +95,7 @@ public void startMaintenanceService() { @After public void stopMaintenanceService() { - for (AsyncSearchMaintenanceService service : internalCluster().getDataNodeInstances(AsyncSearchMaintenanceService.class)) { + for (AsyncTaskMaintenanceService service : internalCluster().getDataNodeInstances(AsyncTaskMaintenanceService.class)) { service.stop(); } } @@ -103,7 +107,7 @@ public void releaseQueryLatch() { @Override protected Collection> nodePlugins() { - return Arrays.asList(LocalStateCompositeXPackPlugin.class, AsyncSearch.class, IndexLifecycle.class, + return Arrays.asList(LocalStateCompositeXPackPlugin.class, AsyncSearch.class, AsyncResultsIndexPlugin.class, IndexLifecycle.class, SearchTestPlugin.class, ReindexPlugin.class); } @@ -137,7 +141,7 @@ protected void restartTaskNode(String id, String indexName) throws Exception { stopMaintenanceService(); internalCluster().restartNode(node.getName(), new InternalTestCluster.RestartCallback() {}); startMaintenanceService(); - ensureYellow(INDEX, indexName); + ensureYellow(ASYNC_RESULTS_INDEX, indexName); } protected AsyncSearchResponse submitAsyncSearch(SubmitAsyncSearchRequest request) throws ExecutionException, InterruptedException { @@ -145,15 +149,15 @@ protected AsyncSearchResponse submitAsyncSearch(SubmitAsyncSearchRequest request } protected AsyncSearchResponse getAsyncSearch(String id) throws ExecutionException, InterruptedException { - return client().execute(GetAsyncSearchAction.INSTANCE, new GetAsyncSearchAction.Request(id)).get(); + return client().execute(GetAsyncSearchAction.INSTANCE, new GetAsyncResultRequest(id)).get(); } protected AsyncSearchResponse getAsyncSearch(String id, TimeValue keepAlive) throws ExecutionException, InterruptedException { - return client().execute(GetAsyncSearchAction.INSTANCE, new GetAsyncSearchAction.Request(id).setKeepAlive(keepAlive)).get(); + return client().execute(GetAsyncSearchAction.INSTANCE, new GetAsyncResultRequest(id).setKeepAlive(keepAlive)).get(); } protected AcknowledgedResponse deleteAsyncSearch(String id) throws ExecutionException, InterruptedException { - return client().execute(DeleteAsyncSearchAction.INSTANCE, new DeleteAsyncSearchAction.Request(id)).get(); + return client().execute(DeleteAsyncResultAction.INSTANCE, new DeleteAsyncResultRequest(id)).get(); } /** @@ -163,7 +167,7 @@ protected void ensureTaskRemoval(String id) throws Exception { AsyncExecutionId searchId = AsyncExecutionId.decode(id); assertBusy(() -> { GetResponse resp = client().prepareGet() - .setIndex(INDEX) + .setIndex(ASYNC_RESULTS_INDEX) .setId(searchId.getDocId()) .get(); assertFalse(resp.isExists()); @@ -249,7 +253,7 @@ private AsyncSearchResponse doNext() throws Exception { } queryLatch.countDownAndReset(); AsyncSearchResponse newResponse = client().execute(GetAsyncSearchAction.INSTANCE, - new GetAsyncSearchAction.Request(response.getId()) + new GetAsyncResultRequest(response.getId()) .setWaitForCompletionTimeout(TimeValue.timeValueMillis(10))).get(); if (newResponse.isRunning()) { diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java index 15e4df6686a01..c7ff71e5f64c9 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java @@ -36,7 +36,7 @@ import java.util.List; import static java.util.Collections.emptyList; -import static org.elasticsearch.xpack.search.GetAsyncSearchRequestTests.randomSearchId; +import static org.elasticsearch.xpack.core.async.GetAsyncResultRequestTests.randomSearchId; public class AsyncSearchResponseTests extends ESTestCase { private SearchResponse searchResponse = randomSearchResponse(); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java index f71d859f648a3..b92d300da45ac 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java @@ -7,18 +7,18 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; +import org.elasticsearch.xpack.core.async.DeleteAsyncResultRequest; -import static org.elasticsearch.xpack.search.GetAsyncSearchRequestTests.randomSearchId; +import static org.elasticsearch.xpack.core.async.GetAsyncResultRequestTests.randomSearchId; -public class DeleteAsyncSearchRequestTests extends AbstractWireSerializingTestCase { +public class DeleteAsyncSearchRequestTests extends AbstractWireSerializingTestCase { @Override - protected Writeable.Reader instanceReader() { - return DeleteAsyncSearchAction.Request::new; + protected Writeable.Reader instanceReader() { + return DeleteAsyncResultRequest::new; } @Override - protected DeleteAsyncSearchAction.Request createTestInstance() { - return new DeleteAsyncSearchAction.Request(randomSearchId()); + protected DeleteAsyncResultRequest createTestInstance() { + return new DeleteAsyncResultRequest(randomSearchId()); } } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java deleted file mode 100644 index 84c5a2aa880f4..0000000000000 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.search; - -import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xpack.core.async.AsyncExecutionId; -import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; - -import java.util.Collections; - -public class GetAsyncSearchRequestTests extends AbstractWireSerializingTestCase { - @Override - protected Writeable.Reader instanceReader() { - return GetAsyncSearchAction.Request::new; - } - - @Override - protected GetAsyncSearchAction.Request createTestInstance() { - GetAsyncSearchAction.Request req = new GetAsyncSearchAction.Request(randomSearchId()); - if (randomBoolean()) { - req.setWaitForCompletionTimeout(TimeValue.timeValueMillis(randomIntBetween(1, 10000))); - } - if (randomBoolean()) { - req.setKeepAlive(TimeValue.timeValueMillis(randomIntBetween(1, 10000))); - } - return req; - } - - static String randomSearchId() { - return AsyncExecutionId.encode(UUIDs.randomBase64UUID(), - new TaskId(randomAlphaOfLengthBetween(10, 20), randomLongBetween(0, Long.MAX_VALUE))); - } - - public void testTaskDescription() { - GetAsyncSearchAction.Request request = new GetAsyncSearchAction.Request("abcdef"); - Task task = request.createTask(1, "type", "action", null, Collections.emptyMap()); - assertEquals("id[abcdef], waitForCompletionTimeout[-1], keepAlive[-1]", task.getDescription()); - } -} diff --git a/x-pack/plugin/async/build.gradle b/x-pack/plugin/async/build.gradle new file mode 100644 index 0000000000000..36d919bbfb43a --- /dev/null +++ b/x-pack/plugin/async/build.gradle @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'x-pack-async' + description 'A module which handles common async operations' + classname 'org.elasticsearch.xpack.async.AsyncResultsIndexPlugin' + extendedPlugins = ['x-pack-core'] +} +archivesBaseName = 'x-pack-async' + +dependencies { + compileOnly project(":server") + compileOnly project(path: xpackModule('core'), configuration: 'default') +} + +dependencyLicenses { + ignoreSha 'x-pack-core' +} + +integTest.enabled = false + diff --git a/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java b/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java new file mode 100644 index 0000000000000..faa1628cfbd68 --- /dev/null +++ b/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.async; + +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.core.async.AsyncTaskIndexService; +import org.elasticsearch.xpack.core.async.AsyncTaskMaintenanceService; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; + +public class AsyncResultsIndexPlugin extends Plugin implements SystemIndexPlugin { + + protected final Settings settings; + + public AsyncResultsIndexPlugin(Settings settings) { + this.settings = settings; + } + + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return Collections.singletonList(new SystemIndexDescriptor(XPackPlugin.ASYNC_RESULTS_INDEX, this.getClass().getSimpleName())); + } + + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + List components = new ArrayList<>(); + if (DiscoveryNode.isDataNode(environment.settings())) { + // only data nodes should be eligible to run the maintenance service. + AsyncTaskIndexService indexService = new AsyncTaskIndexService<>( + XPackPlugin.ASYNC_RESULTS_INDEX, + clusterService, + threadPool.getThreadContext(), + client, + ASYNC_SEARCH_ORIGIN, + AsyncSearchResponse::new, + namedWriteableRegistry + ); + AsyncTaskMaintenanceService maintenanceService = new AsyncTaskMaintenanceService( + clusterService, + nodeEnvironment.nodeId(), + settings, + threadPool, + indexService + ); + components.add(maintenanceService); + } + return components; + } +} diff --git a/x-pack/plugin/async/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/async/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugin/async/src/test/java/org/elasticsearch/xpack/async/AsyncResultsIndexPluginTests.java b/x-pack/plugin/async/src/test/java/org/elasticsearch/xpack/async/AsyncResultsIndexPluginTests.java new file mode 100644 index 0000000000000..c2f48fdc24851 --- /dev/null +++ b/x-pack/plugin/async/src/test/java/org/elasticsearch/xpack/async/AsyncResultsIndexPluginTests.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.async; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; + +public class AsyncResultsIndexPluginTests extends ESTestCase { + + public void testDummy() { + // This is a dummy test case to satisfy the conventions + AsyncResultsIndexPlugin plugin = new AsyncResultsIndexPlugin(Settings.EMPTY); + assertThat(plugin.getSystemIndexDescriptors(Settings.EMPTY), Matchers.hasSize(1)); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index c15e19e52001f..2ae783222e022 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -188,7 +188,7 @@ import org.elasticsearch.xpack.core.rollup.action.StopRollupJobAction; import org.elasticsearch.xpack.core.rollup.job.RollupJob; import org.elasticsearch.xpack.core.rollup.job.RollupJobStatus; -import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; +import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; import org.elasticsearch.xpack.core.security.SecurityFeatureSetUsage; @@ -480,8 +480,8 @@ public List> getClientActions() { // Async Search SubmitAsyncSearchAction.INSTANCE, GetAsyncSearchAction.INSTANCE, - DeleteAsyncSearchAction.INSTANCE - ); + DeleteAsyncResultAction.INSTANCE + ); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index 4d60d194747fc..a732523b6bc26 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -61,6 +61,8 @@ import org.elasticsearch.xpack.core.action.TransportXPackUsageAction; import org.elasticsearch.xpack.core.action.XPackInfoAction; import org.elasticsearch.xpack.core.action.XPackUsageAction; +import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; +import org.elasticsearch.xpack.core.async.TransportDeleteAsyncResultAction; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.rest.action.RestReloadAnalyzersAction; import org.elasticsearch.xpack.core.rest.action.RestXPackInfoAction; @@ -91,6 +93,7 @@ public class XPackPlugin extends XPackClientPlugin implements ExtensiblePlugin, private static final Logger logger = LogManager.getLogger(XPackPlugin.class); private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger); + public static final String ASYNC_RESULTS_INDEX = ".async-search"; public static final String XPACK_INSTALLED_NODE_ATTR = "xpack.installed"; // TODO: clean up this library to not ask for write access to all system properties! @@ -279,6 +282,7 @@ public Collection createComponents(Client client, ClusterService cluster actions.add(new ActionHandler<>(XPackUsageAction.INSTANCE, TransportXPackUsageAction.class)); actions.addAll(licensing.getActions()); actions.add(new ActionHandler<>(ReloadAnalyzerAction.INSTANCE, TransportReloadAnalyzersAction.class)); + actions.add(new ActionHandler<>(DeleteAsyncResultAction.INSTANCE, TransportDeleteAsyncResultAction.class)); return actions; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncResultsService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncResultsService.java new file mode 100644 index 0000000000000..8b208ba3065ca --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncResultsService.java @@ -0,0 +1,172 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.async; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.TriConsumer; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.TaskManager; + +import java.util.Objects; + +/** + * Service that is capable of retrieving and cleaning up AsyncTasks regardless of their state. It works with the TaskManager, if a task + * is still running and AsyncTaskIndexService if task results already stored there. + */ +public class AsyncResultsService> { + private final Logger logger = LogManager.getLogger(AsyncResultsService.class); + private final Class asyncTaskClass; + private final TaskManager taskManager; + private final ClusterService clusterService; + private final AsyncTaskIndexService store; + private final boolean updateInitialResultsInStore; + private final TriConsumer, TimeValue> addCompletionListener; + + /** + * Creates async results service + * + * @param store AsyncTaskIndexService for the response we are working with + * @param updateInitialResultsInStore true if initial results are stored (Async Search) or false otherwise (EQL Search) + * @param asyncTaskClass async task class + * @param addCompletionListener function that registers a completion listener with the task + * @param taskManager task manager + * @param clusterService cluster service + */ + public AsyncResultsService(AsyncTaskIndexService store, + boolean updateInitialResultsInStore, + Class asyncTaskClass, + TriConsumer, TimeValue> addCompletionListener, + TaskManager taskManager, + ClusterService clusterService) { + this.updateInitialResultsInStore = updateInitialResultsInStore; + this.asyncTaskClass = asyncTaskClass; + this.addCompletionListener = addCompletionListener; + this.taskManager = taskManager; + this.clusterService = clusterService; + this.store = store; + + } + + public DiscoveryNode getNode(String id) { + AsyncExecutionId searchId = AsyncExecutionId.decode(id); + return clusterService.state().nodes().get(searchId.getTaskId().getNodeId()); + } + + public boolean isLocalNode(DiscoveryNode node) { + return Objects.requireNonNull(node).equals(clusterService.localNode()); + } + + public void retrieveResult(GetAsyncResultRequest request, ActionListener listener) { + try { + long nowInMillis = System.currentTimeMillis(); + AsyncExecutionId searchId = AsyncExecutionId.decode(request.getId()); + long expirationTime; + if (request.getKeepAlive() != null && request.getKeepAlive().getMillis() > 0) { + expirationTime = nowInMillis + request.getKeepAlive().getMillis(); + } else { + expirationTime = -1; + } + // EQL doesn't store initial or intermediate results so we only need to update expiration time in store for only in case of + // async search + if (updateInitialResultsInStore & expirationTime > 0) { + store.updateExpirationTime(searchId.getDocId(), expirationTime, + ActionListener.wrap( + p -> getSearchResponseFromTask(searchId, request, nowInMillis, expirationTime, listener), + exc -> { + //don't log when: the async search document or its index is not found. That can happen if an invalid + //search id is provided or no async search initial response has been stored yet. + RestStatus status = ExceptionsHelper.status(ExceptionsHelper.unwrapCause(exc)); + if (status != RestStatus.NOT_FOUND) { + logger.error(() -> new ParameterizedMessage("failed to update expiration time for async-search [{}]", + searchId.getEncoded()), exc); + } + listener.onFailure(new ResourceNotFoundException(searchId.getEncoded())); + } + )); + } else { + getSearchResponseFromTask(searchId, request, nowInMillis, expirationTime, listener); + } + } catch (Exception exc) { + listener.onFailure(exc); + } + } + + private void getSearchResponseFromTask(AsyncExecutionId searchId, + GetAsyncResultRequest request, + long nowInMillis, + long expirationTimeMillis, + ActionListener listener) { + try { + final Task task = store.getTask(taskManager, searchId, asyncTaskClass); + if (task == null) { + getSearchResponseFromIndex(searchId, request, nowInMillis, listener); + return; + } + + if (task.isCancelled()) { + listener.onFailure(new ResourceNotFoundException(searchId.getEncoded())); + return; + } + + if (expirationTimeMillis != -1) { + task.setExpirationTime(expirationTimeMillis); + } + addCompletionListener.apply(task, new ActionListener() { + @Override + public void onResponse(Response response) { + sendFinalResponse(request, response, nowInMillis, listener); + } + + @Override + public void onFailure(Exception exc) { + listener.onFailure(exc); + } + }, request.getWaitForCompletionTimeout()); + } catch (Exception exc) { + listener.onFailure(exc); + } + } + + private void getSearchResponseFromIndex(AsyncExecutionId searchId, + GetAsyncResultRequest request, + long nowInMillis, + ActionListener listener) { + store.getResponse(searchId, true, + new ActionListener() { + @Override + public void onResponse(Response response) { + sendFinalResponse(request, response, nowInMillis, listener); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + + private void sendFinalResponse(GetAsyncResultRequest request, + Response response, + long nowInMillis, + ActionListener listener) { + // check if the result has expired + if (response.getExpirationTime() < nowInMillis) { + listener.onFailure(new ResourceNotFoundException(request.getId())); + return; + } + + listener.onResponse(response); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTask.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTask.java index 18ce4a0fc68bf..db8393e74a493 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTask.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTask.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.core.async; +import org.elasticsearch.tasks.TaskManager; + import java.util.Map; /** @@ -21,4 +23,19 @@ public interface AsyncTask { * Returns the {@link AsyncExecutionId} of the task */ AsyncExecutionId getExecutionId(); + + /** + * Returns true if the task is cancelled + */ + boolean isCancelled(); + + /** + * Update the expiration time of the (partial) response. + */ + void setExpirationTime(long expirationTimeMillis); + + /** + * Performs necessary checks, cancels the task and calls the runnable upon completion + */ + void cancelTask(TaskManager taskManager, Runnable runnable, String reason); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java index 4c27ee371c44b..69cbb300cdc84 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; @@ -163,10 +164,10 @@ void createIndexIfNecessary(ActionListener listener) { * Stores the initial response with the original headers of the authenticated user * and the expected expiration time. */ - public void storeInitialResponse(String docId, - Map headers, - R response, - ActionListener listener) throws IOException { + public void createResponse(String docId, + Map headers, + R response, + ActionListener listener) throws IOException { Map source = new HashMap<>(); source.put(HEADERS_FIELD, headers); source.put(EXPIRATION_TIME_FIELD, response.getExpirationTime()); @@ -181,10 +182,10 @@ public void storeInitialResponse(String docId, /** * Stores the final response if the place-holder document is still present (update). */ - public void storeFinalResponse(String docId, - Map> responseHeaders, - R response, - ActionListener listener) throws IOException { + public void updateResponse(String docId, + Map> responseHeaders, + R response, + ActionListener listener) throws IOException { Map source = new HashMap<>(); source.put(RESPONSE_HEADERS_FIELD, responseHeaders); source.put(RESULT_FIELD, encodeResponse(response)); @@ -243,15 +244,9 @@ public T getTask(TaskManager taskManager, AsyncExecutionId return asyncTask; } - /** - * Gets the response from the index if present, or delegate a {@link ResourceNotFoundException} - * failure to the provided listener if not. - * When the provided restoreResponseHeaders is true, this method also restores the - * response headers of the original request in the current thread context. - */ - public void getResponse(AsyncExecutionId asyncExecutionId, - boolean restoreResponseHeaders, - ActionListener listener) { + private void getEncodedResponse(AsyncExecutionId asyncExecutionId, + boolean restoreResponseHeaders, + ActionListener> listener) { final Authentication current = securityContext.getAuthentication(); GetRequest internalGet = new GetRequest(index) .preference(asyncExecutionId.getEncoded()) @@ -280,7 +275,7 @@ public void getResponse(AsyncExecutionId asyncExecutionId, long expirationTime = (long) get.getSource().get(EXPIRATION_TIME_FIELD); String encoded = (String) get.getSource().get(RESULT_FIELD); if (encoded != null) { - listener.onResponse(decodeResponse(encoded).withExpirationTime(expirationTime)); + listener.onResponse(new Tuple<>(encoded, expirationTime)); } else { listener.onResponse(null); } @@ -289,6 +284,34 @@ public void getResponse(AsyncExecutionId asyncExecutionId, )); } + /** + * Gets the response from the index if present, or delegate a {@link ResourceNotFoundException} + * failure to the provided listener if not. + * When the provided restoreResponseHeaders is true, this method also restores the + * response headers of the original request in the current thread context. + */ + public void getResponse(AsyncExecutionId asyncExecutionId, + boolean restoreResponseHeaders, + ActionListener listener) { + getEncodedResponse(asyncExecutionId, restoreResponseHeaders, ActionListener.wrap( + (t) -> listener.onResponse(decodeResponse(t.v1()).withExpirationTime(t.v2())), + listener::onFailure + )); + } + + /** + * Ensures that the current user can read the specified response without actually reading it + */ + public void authorizeResponse(AsyncExecutionId asyncExecutionId, + boolean restoreResponseHeaders, + ActionListener listener) { + getEncodedResponse(asyncExecutionId, restoreResponseHeaders, ActionListener.wrap( + (t) -> listener.onResponse(null), + listener::onFailure + )); + } + + /** * Extracts the authentication from the original headers and checks that it matches * the current user. This function returns always true if the provided diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskMaintenanceService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskMaintenanceService.java index 5dfd77c0f46f7..7838ba5adfb33 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskMaintenanceService.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskMaintenanceService.java @@ -15,6 +15,8 @@ import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.gateway.GatewayService; @@ -23,6 +25,7 @@ import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.XPackPlugin; import java.io.IOException; @@ -34,7 +37,17 @@ * Since we will have several injected implementation of this class injected into different transports, and we bind components created * by {@linkplain org.elasticsearch.plugins.Plugin#createComponents} to their classes, we need to implement one class per binding. */ -public abstract class AsyncTaskMaintenanceService extends AbstractLifecycleComponent implements ClusterStateListener { +public class AsyncTaskMaintenanceService extends AbstractLifecycleComponent implements ClusterStateListener { + + /** + * Controls the interval at which the cleanup is scheduled. + * Defaults to 1h. It is an undocumented/expert setting that + * is mainly used by integration tests to make the garbage + * collection of search responses more reactive. + */ + public static final Setting ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING = + Setting.timeSetting("async_search.index_cleanup_interval", TimeValue.timeValueHours(1), Setting.Property.NodeScope); + private static final Logger logger = LogManager.getLogger(AsyncTaskMaintenanceService.class); private final ClusterService clusterService; @@ -48,17 +61,16 @@ public abstract class AsyncTaskMaintenanceService extends AbstractLifecycleCompo private volatile Scheduler.Cancellable cancellable; public AsyncTaskMaintenanceService(ClusterService clusterService, - String index, String localNodeId, + Settings nodeSettings, ThreadPool threadPool, - AsyncTaskIndexService indexService, - TimeValue delay) { + AsyncTaskIndexService indexService) { this.clusterService = clusterService; - this.index = index; + this.index = XPackPlugin.ASYNC_RESULTS_INDEX; this.localNodeId = localNodeId; this.threadPool = threadPool; this.indexService = indexService; - this.delay = delay; + this.delay = ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING.get(nodeSettings); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/DeleteAsyncResultAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/DeleteAsyncResultAction.java new file mode 100644 index 0000000000000..86986630f031f --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/DeleteAsyncResultAction.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.async; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.io.stream.Writeable; + +public class DeleteAsyncResultAction extends ActionType { + public static final DeleteAsyncResultAction INSTANCE = new DeleteAsyncResultAction(); + public static final String NAME = "indices:data/read/async_search/delete"; + + private DeleteAsyncResultAction() { + super(NAME, AcknowledgedResponse::new); + } + + @Override + public Writeable.Reader getResponseReader() { + return AcknowledgedResponse::new; + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/DeleteAsyncResultRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/DeleteAsyncResultRequest.java new file mode 100644 index 0000000000000..63158e07c4f16 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/DeleteAsyncResultRequest.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.async; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Objects; + +public class DeleteAsyncResultRequest extends ActionRequest { + private final String id; + + public DeleteAsyncResultRequest(String id) { + this.id = id; + } + + public DeleteAsyncResultRequest(StreamInput in) throws IOException { + super(in); + this.id = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(id); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + public String getId() { + return id; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DeleteAsyncResultRequest request = (DeleteAsyncResultRequest) o; + return id.equals(request.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/DeleteAsyncResultsService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/DeleteAsyncResultsService.java new file mode 100644 index 0000000000000..caefd180d9402 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/DeleteAsyncResultsService.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.async; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.TaskManager; + +/** + * Service that is capable of retrieving and cleaning up AsyncTasks regardless of their state. It works with the TaskManager, if a task + * is still running and AsyncTaskIndexService if task results already stored there. + */ +public class DeleteAsyncResultsService { + private final Logger logger = LogManager.getLogger(DeleteAsyncResultsService.class); + private final TaskManager taskManager; + private final AsyncTaskIndexService> store; + + /** + * Creates async results service + * + * @param store AsyncTaskIndexService for the response we are working with + * @param taskManager task manager + */ + public DeleteAsyncResultsService(AsyncTaskIndexService> store, + TaskManager taskManager) { + this.taskManager = taskManager; + this.store = store; + + } + + public void deleteResult(DeleteAsyncResultRequest request, ActionListener listener) { + try { + AsyncExecutionId searchId = AsyncExecutionId.decode(request.getId()); + AsyncTask task = store.getTask(taskManager, searchId, AsyncTask.class); + if (task != null) { + //the task was found and gets cancelled. The response may or may not be found, but we will return 200 anyways. + task.cancelTask(taskManager, () -> store.deleteResponse(searchId, + ActionListener.wrap( + r -> listener.onResponse(new AcknowledgedResponse(true)), + exc -> { + RestStatus status = ExceptionsHelper.status(ExceptionsHelper.unwrapCause(exc)); + //the index may not be there (no initial async search response stored yet?): we still want to return 200 + //note that index missing comes back as 200 hence it's handled in the onResponse callback + if (status == RestStatus.NOT_FOUND) { + listener.onResponse(new AcknowledgedResponse(true)); + } else { + logger.error(() -> new ParameterizedMessage("failed to clean async result [{}]", + searchId.getEncoded()), exc); + listener.onFailure(exc); + } + })), "cancelled by user" + ); + } else { + // the task was not found (already cancelled, already completed, or invalid id?) + // we fail if the response is not found in the index + ActionListener deleteListener = ActionListener.wrap( + resp -> { + if (resp.status() == RestStatus.NOT_FOUND) { + listener.onFailure(new ResourceNotFoundException(searchId.getEncoded())); + } else { + listener.onResponse(new AcknowledgedResponse(true)); + } + }, + exc -> { + logger.error(() -> new ParameterizedMessage("failed to clean async-search [{}]", searchId.getEncoded()), exc); + listener.onFailure(exc); + } + ); + //we get before deleting to verify that the user is authorized + store.authorizeResponse(searchId, false, + ActionListener.wrap(res -> store.deleteResponse(searchId, deleteListener), listener::onFailure)); + } + } catch (Exception exc) { + listener.onFailure(exc); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/GetAsyncResultRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/GetAsyncResultRequest.java new file mode 100644 index 0000000000000..c01f24e5ea01b --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/GetAsyncResultRequest.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.async; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.TimeValue; + +import java.io.IOException; +import java.util.Objects; + +public class GetAsyncResultRequest extends ActionRequest { + private final String id; + private TimeValue waitForCompletionTimeout = TimeValue.MINUS_ONE; + private TimeValue keepAlive = TimeValue.MINUS_ONE; + + /** + * Creates a new request + * + * @param id The id of the search progress request. + */ + public GetAsyncResultRequest(String id) { + this.id = id; + } + + public GetAsyncResultRequest(StreamInput in) throws IOException { + super(in); + this.id = in.readString(); + this.waitForCompletionTimeout = TimeValue.timeValueMillis(in.readLong()); + this.keepAlive = in.readTimeValue(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(id); + out.writeLong(waitForCompletionTimeout.millis()); + out.writeTimeValue(keepAlive); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + /** + * Returns the id of the async search. + */ + public String getId() { + return id; + } + + /** + * Sets the minimum time that the request should wait before returning a partial result (defaults to no wait). + */ + public GetAsyncResultRequest setWaitForCompletionTimeout(TimeValue timeValue) { + this.waitForCompletionTimeout = timeValue; + return this; + } + + public TimeValue getWaitForCompletionTimeout() { + return waitForCompletionTimeout; + } + + /** + * Extends the amount of time after which the result will expire (defaults to no extension). + */ + public GetAsyncResultRequest setKeepAlive(TimeValue timeValue) { + this.keepAlive = timeValue; + return this; + } + + public TimeValue getKeepAlive() { + return keepAlive; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GetAsyncResultRequest request = (GetAsyncResultRequest) o; + return Objects.equals(id, request.id) && + waitForCompletionTimeout.equals(request.waitForCompletionTimeout) && + keepAlive.equals(request.keepAlive); + } + + @Override + public int hashCode() { + return Objects.hash(id, waitForCompletionTimeout, keepAlive); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/TransportDeleteAsyncResultAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/TransportDeleteAsyncResultAction.java new file mode 100644 index 0000000000000..0448d210fdd94 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/TransportDeleteAsyncResultAction.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.async; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionListenerResponseHandler; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackPlugin; + +import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; + +public class TransportDeleteAsyncResultAction extends HandledTransportAction { + private final DeleteAsyncResultsService deleteResultsService; + private final ClusterService clusterService; + private final TransportService transportService; + + @Inject + public TransportDeleteAsyncResultAction(TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + NamedWriteableRegistry registry, + Client client, + ThreadPool threadPool) { + super(DeleteAsyncResultAction.NAME, transportService, actionFilters, DeleteAsyncResultRequest::new); + this.transportService = transportService; + this.clusterService = clusterService; + AsyncTaskIndexService store = new AsyncTaskIndexService<>(XPackPlugin.ASYNC_RESULTS_INDEX, clusterService, + threadPool.getThreadContext(), client, ASYNC_SEARCH_ORIGIN, + (in) -> {throw new UnsupportedOperationException("Reading is not supported during deletion");}, registry); + this.deleteResultsService = new DeleteAsyncResultsService(store, transportService.getTaskManager()); + } + + + @Override + protected void doExecute(Task task, DeleteAsyncResultRequest request, ActionListener listener) { + AsyncExecutionId searchId = AsyncExecutionId.decode(request.getId()); + DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId()); + if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId()) || node == null) { + deleteResultsService.deleteResult(request, listener); + } else { + TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); + transportService.sendRequest(node, DeleteAsyncResultAction.NAME, request, builder.build(), + new ActionListenerResponseHandler<>(listener, AcknowledgedResponse::new, ThreadPool.Names.SAME)); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/eql/EqlAsyncActionNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/eql/EqlAsyncActionNames.java new file mode 100644 index 0000000000000..e84655052451c --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/eql/EqlAsyncActionNames.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.eql; + +/** + * Exposes EQL async action names for RBACEngine + */ +public final class EqlAsyncActionNames { + public static final String EQL_ASYNC_GET_RESULT_ACTION_NAME = "indices:data/read/eql/async/get"; +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java deleted file mode 100644 index d69de80d2293e..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.core.search.action; - -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; - -import java.io.IOException; -import java.util.Objects; - -public class DeleteAsyncSearchAction extends ActionType { - public static final DeleteAsyncSearchAction INSTANCE = new DeleteAsyncSearchAction(); - public static final String NAME = "indices:data/read/async_search/delete"; - - private DeleteAsyncSearchAction() { - super(NAME, AcknowledgedResponse::new); - } - - @Override - public Writeable.Reader getResponseReader() { - return AcknowledgedResponse::new; - } - - public static class Request extends ActionRequest { - private final String id; - - public Request(String id) { - this.id = id; - } - - public Request(StreamInput in) throws IOException { - super(in); - this.id = in.readString(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(id); - } - - @Override - public ActionRequestValidationException validate() { - return null; - } - - public String getId() { - return id; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Request request = (Request) o; - return id.equals(request.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - } -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java index 934e7aacfa615..3ef53f712bc72 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java @@ -5,16 +5,7 @@ */ package org.elasticsearch.xpack.core.search.action; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.unit.TimeValue; - -import java.io.IOException; -import java.util.Objects; public class GetAsyncSearchAction extends ActionType { public static final GetAsyncSearchAction INSTANCE = new GetAsyncSearchAction(); @@ -23,97 +14,4 @@ public class GetAsyncSearchAction extends ActionType { private GetAsyncSearchAction() { super(NAME, AsyncSearchResponse::new); } - - @Override - public Writeable.Reader getResponseReader() { - return AsyncSearchResponse::new; - } - - public static class Request extends ActionRequest { - private final String id; - private TimeValue waitForCompletionTimeout = TimeValue.MINUS_ONE; - private TimeValue keepAlive = TimeValue.MINUS_ONE; - - /** - * Creates a new request - * - * @param id The id of the search progress request. - */ - public Request(String id) { - this.id = id; - } - - public Request(StreamInput in) throws IOException { - super(in); - this.id = in.readString(); - this.waitForCompletionTimeout = TimeValue.timeValueMillis(in.readLong()); - this.keepAlive = in.readTimeValue(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(id); - out.writeLong(waitForCompletionTimeout.millis()); - out.writeTimeValue(keepAlive); - } - - @Override - public ActionRequestValidationException validate() { - return null; - } - - /** - * Returns the id of the async search. - */ - public String getId() { - return id; - } - - /** - * Sets the minimum time that the request should wait before returning a partial result (defaults to no wait). - */ - public Request setWaitForCompletionTimeout(TimeValue timeValue) { - this.waitForCompletionTimeout = timeValue; - return this; - } - - public TimeValue getWaitForCompletionTimeout() { - return waitForCompletionTimeout; - } - - /** - * Extends the amount of time after which the result will expire (defaults to no extension). - */ - public Request setKeepAlive(TimeValue timeValue) { - this.keepAlive = timeValue; - return this; - } - - public TimeValue getKeepAlive() { - return keepAlive; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Request request = (Request) o; - return Objects.equals(id, request.id) && - waitForCompletionTimeout.equals(request.waitForCompletionTimeout) && - keepAlive.equals(request.keepAlive); - } - - @Override - public int hashCode() { - return Objects.hash(id, waitForCompletionTimeout, keepAlive); - } - - @Override - public String getDescription() { - return "id[" + id + - "], waitForCompletionTimeout[" + waitForCompletionTimeout + - "], keepAlive[" + keepAlive + "]"; - } - } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncExecutionIdTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncExecutionIdTests.java index 6b02edf069749..f4c0496deae96 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncExecutionIdTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncExecutionIdTests.java @@ -28,7 +28,7 @@ public void testEncodeAndDecode() { } } - private static AsyncExecutionId randomAsyncId() { + public static AsyncExecutionId randomAsyncId() { return new AsyncExecutionId(UUIDs.randomBase64UUID(), new TaskId(randomAlphaOfLengthBetween(5, 20), randomNonNegativeLong())); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncResultsServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncResultsServiceTests.java new file mode 100644 index 0000000000000..bd5a1d66daec4 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncResultsServiceTests.java @@ -0,0 +1,276 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.async; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.async.AsyncSearchIndexServiceTests.TestAsyncResponse; +import org.junit.Before; + +import java.util.HashMap; +import java.util.Map; + +import static java.util.Collections.emptyMap; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFutureThrows; +import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; +import static org.elasticsearch.xpack.core.async.AsyncExecutionIdTests.randomAsyncId; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class AsyncResultsServiceTests extends ESSingleNodeTestCase { + private ClusterService clusterService; + private TaskManager taskManager; + private AsyncTaskIndexService indexService; + + public static class TestTask extends CancellableTask implements AsyncTask { + private final AsyncExecutionId executionId; + private final Map, TimeValue> listeners = new HashMap<>(); + private long expirationTimeMillis; + + public TestTask(AsyncExecutionId executionId, long id, String type, String action, String description, TaskId parentTaskId, + Map headers) { + super(id, type, action, description, parentTaskId, headers); + this.executionId = executionId; + } + + @Override + public boolean shouldCancelChildrenOnCancellation() { + return false; + } + + @Override + public Map getOriginHeaders() { + return null; + } + + @Override + public AsyncExecutionId getExecutionId() { + return executionId; + } + + @Override + public void setExpirationTime(long expirationTimeMillis) { + this.expirationTimeMillis = expirationTimeMillis; + } + + @Override + public void cancelTask(TaskManager taskManager, Runnable runnable, String reason) { + taskManager.cancelTaskAndDescendants(this, reason, true, ActionListener.wrap(runnable)); + } + + public long getExpirationTime() { + return this.expirationTimeMillis; + } + + public synchronized void addListener(ActionListener listener, TimeValue timeout) { + if (timeout.getMillis() < 0) { + listener.onResponse(new TestAsyncResponse(null, expirationTimeMillis)); + } else { + assertThat(listeners.put(listener, timeout), nullValue()); + } + } + + private synchronized void onResponse(String response) { + TestAsyncResponse r = new TestAsyncResponse(response, expirationTimeMillis); + for (ActionListener listener : listeners.keySet()) { + listener.onResponse(r); + } + } + + private synchronized void onFailure(Exception e) { + for (ActionListener listener : listeners.keySet()) { + listener.onFailure(e); + } + } + } + + public class TestRequest extends TransportRequest { + private final String string; + + public TestRequest(String string) { + this.string = string; + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + AsyncExecutionId asyncExecutionId = new AsyncExecutionId(randomAlphaOfLength(10), + new TaskId(clusterService.localNode().getId(), id)); + return new TestTask(asyncExecutionId, id, type, action, string, parentTaskId, headers); + } + } + + @Before + public void setup() { + clusterService = getInstanceFromNode(ClusterService.class); + TransportService transportService = getInstanceFromNode(TransportService.class); + taskManager = transportService.getTaskManager(); + indexService = new AsyncTaskIndexService<>("test", clusterService, transportService.getThreadPool().getThreadContext(), + client(), ASYNC_SEARCH_ORIGIN, TestAsyncResponse::new, writableRegistry()); + + } + + private AsyncResultsService createResultsService(boolean updateInitialResultsInStore) { + return new AsyncResultsService<>(indexService, updateInitialResultsInStore, TestTask.class, TestTask::addListener, + taskManager, clusterService); + } + + private DeleteAsyncResultsService createDeleteResultsService() { + return new DeleteAsyncResultsService(indexService, taskManager); + } + + public void testRecordNotFound() { + AsyncResultsService service = createResultsService(randomBoolean()); + DeleteAsyncResultsService deleteService = createDeleteResultsService(); + PlainActionFuture listener = new PlainActionFuture<>(); + service.retrieveResult(new GetAsyncResultRequest(randomAsyncId().getEncoded()), listener); + assertFutureThrows(listener, ResourceNotFoundException.class); + PlainActionFuture deleteListener = new PlainActionFuture<>(); + deleteService.deleteResult(new DeleteAsyncResultRequest(randomAsyncId().getEncoded()), deleteListener); + assertFutureThrows(listener, ResourceNotFoundException.class); + } + + public void testRetrieveFromMemoryWithExpiration() throws Exception { + boolean updateInitialResultsInStore = randomBoolean(); + AsyncResultsService service = createResultsService(updateInitialResultsInStore); + TestTask task = (TestTask) taskManager.register("test", "test", new TestRequest("test request")); + try { + boolean shouldExpire = randomBoolean(); + long expirationTime = System.currentTimeMillis() + randomLongBetween(1000, 10000) * (shouldExpire ? -1 : 1); + task.setExpirationTime(expirationTime); + + if (updateInitialResultsInStore) { + // we need to store initial result + PlainActionFuture future = new PlainActionFuture<>(); + indexService.createResponse(task.getExecutionId().getDocId(), task.getOriginHeaders(), + new TestAsyncResponse(null, task.getExpirationTime()), future); + future.actionGet(TimeValue.timeValueSeconds(10)); + } + + PlainActionFuture listener = new PlainActionFuture<>(); + service.retrieveResult(new GetAsyncResultRequest(task.getExecutionId().getEncoded()) + .setWaitForCompletionTimeout(TimeValue.timeValueSeconds(5)), listener); + if (randomBoolean()) { + // Test success + String expectedResponse = randomAlphaOfLength(10); + task.onResponse(expectedResponse); + if (shouldExpire) { + assertFutureThrows(listener, ResourceNotFoundException.class); + } else { + TestAsyncResponse response = listener.actionGet(TimeValue.timeValueSeconds(10)); + assertThat(response, notNullValue()); + assertThat(response.test, equalTo(expectedResponse)); + assertThat(response.expirationTimeMillis, equalTo(expirationTime)); + } + } else { + // Test Failure + task.onFailure(new IllegalArgumentException("test exception")); + assertFutureThrows(listener, IllegalArgumentException.class); + } + } finally { + taskManager.unregister(task); + } + } + + public void testAssertExpirationPropagation() throws Exception { + boolean updateInitialResultsInStore = randomBoolean(); + AsyncResultsService service = createResultsService(updateInitialResultsInStore); + TestRequest request = new TestRequest("test request"); + TestTask task = (TestTask) taskManager.register("test", "test", request); + try { + long startTime = System.currentTimeMillis(); + task.setExpirationTime(startTime + TimeValue.timeValueMinutes(1).getMillis()); + + if (updateInitialResultsInStore) { + // we need to store initial result + PlainActionFuture future = new PlainActionFuture<>(); + indexService.createResponse(task.getExecutionId().getDocId(), task.getOriginHeaders(), + new TestAsyncResponse(null, task.getExpirationTime()), future); + future.actionGet(TimeValue.timeValueSeconds(10)); + } + + TimeValue newKeepAlive = TimeValue.timeValueDays(1); + PlainActionFuture listener = new PlainActionFuture<>(); + // not waiting for completion, so should return immediately with timeout + service.retrieveResult(new GetAsyncResultRequest(task.getExecutionId().getEncoded()).setKeepAlive(newKeepAlive), listener); + listener.actionGet(TimeValue.timeValueSeconds(10)); + assertThat(task.getExpirationTime(), greaterThanOrEqualTo(startTime + newKeepAlive.getMillis())); + assertThat(task.getExpirationTime(), lessThanOrEqualTo(System.currentTimeMillis() + newKeepAlive.getMillis())); + + if (updateInitialResultsInStore) { + PlainActionFuture future = new PlainActionFuture<>(); + indexService.getResponse(task.executionId, randomBoolean(), future); + TestAsyncResponse response = future.actionGet(TimeValue.timeValueMinutes(10)); + assertThat(response.getExpirationTime(), greaterThanOrEqualTo(startTime + newKeepAlive.getMillis())); + assertThat(response.getExpirationTime(), lessThanOrEqualTo(System.currentTimeMillis() + newKeepAlive.getMillis())); + } + } finally { + taskManager.unregister(task); + } + } + + public void testRetrieveFromDisk() throws Exception { + boolean updateInitialResultsInStore = randomBoolean(); + AsyncResultsService service = createResultsService(updateInitialResultsInStore); + DeleteAsyncResultsService deleteService = createDeleteResultsService(); + TestRequest request = new TestRequest("test request"); + TestTask task = (TestTask) taskManager.register("test", "test", request); + try { + long startTime = System.currentTimeMillis(); + task.setExpirationTime(startTime + TimeValue.timeValueMinutes(1).getMillis()); + + if (updateInitialResultsInStore) { + // we need to store initial result + PlainActionFuture futureCreate = new PlainActionFuture<>(); + indexService.createResponse(task.getExecutionId().getDocId(), task.getOriginHeaders(), + new TestAsyncResponse(null, task.getExpirationTime()), futureCreate); + futureCreate.actionGet(TimeValue.timeValueSeconds(10)); + + PlainActionFuture futureUpdate = new PlainActionFuture<>(); + indexService.updateResponse(task.getExecutionId().getDocId(), emptyMap(), + new TestAsyncResponse("final_response", task.getExpirationTime()), futureUpdate); + futureUpdate.actionGet(TimeValue.timeValueSeconds(10)); + } else { + PlainActionFuture futureCreate = new PlainActionFuture<>(); + indexService.createResponse(task.getExecutionId().getDocId(), task.getOriginHeaders(), + new TestAsyncResponse("final_response", task.getExpirationTime()), futureCreate); + futureCreate.actionGet(TimeValue.timeValueSeconds(10)); + } + + } finally { + taskManager.unregister(task); + } + + PlainActionFuture listener = new PlainActionFuture<>(); + // not waiting for completion, so should return immediately with timeout + service.retrieveResult(new GetAsyncResultRequest(task.getExecutionId().getEncoded()), listener); + TestAsyncResponse response = listener.actionGet(TimeValue.timeValueSeconds(10)); + assertThat(response.test, equalTo("final_response")); + + PlainActionFuture deleteListener = new PlainActionFuture<>(); + deleteService.deleteResult(new DeleteAsyncResultRequest(task.getExecutionId().getEncoded()), deleteListener); + assertThat(deleteListener.actionGet().isAcknowledged(), equalTo(true)); + + deleteListener = new PlainActionFuture<>(); + deleteService.deleteResult(new DeleteAsyncResultRequest(task.getExecutionId().getEncoded()), deleteListener); + assertFutureThrows(deleteListener, ResourceNotFoundException.class); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncSearchIndexServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncSearchIndexServiceTests.java index 7457acfefcac3..84e101cc0b3d5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncSearchIndexServiceTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncSearchIndexServiceTests.java @@ -23,8 +23,8 @@ public class AsyncSearchIndexServiceTests extends ESSingleNodeTestCase { private AsyncTaskIndexService indexService; public static class TestAsyncResponse implements AsyncResponse { - private final String test; - private final long expirationTimeMillis; + public final String test; + public final long expirationTimeMillis; public TestAsyncResponse(String test, long expirationTimeMillis) { this.test = test; @@ -38,7 +38,7 @@ public TestAsyncResponse(StreamInput input) throws IOException { @Override public long getExpirationTime() { - return 0; + return expirationTimeMillis; } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/GetAsyncResultRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/GetAsyncResultRequestTests.java new file mode 100644 index 0000000000000..b9d57d7cc3452 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/GetAsyncResultRequestTests.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.async; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import static org.elasticsearch.xpack.core.async.AsyncExecutionIdTests.randomAsyncId; + +public class GetAsyncResultRequestTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return GetAsyncResultRequest::new; + } + + @Override + protected GetAsyncResultRequest createTestInstance() { + GetAsyncResultRequest req = new GetAsyncResultRequest(randomSearchId()); + if (randomBoolean()) { + req.setWaitForCompletionTimeout(TimeValue.timeValueMillis(randomIntBetween(1, 10000))); + } + if (randomBoolean()) { + req.setKeepAlive(TimeValue.timeValueMillis(randomIntBetween(1, 10000))); + } + return req; + } + + public static String randomSearchId() { + return randomAsyncId().getEncoded(); + } +} diff --git a/x-pack/plugin/eql/build.gradle b/x-pack/plugin/eql/build.gradle index f5a0a2772c117..eadf6605393db 100644 --- a/x-pack/plugin/eql/build.gradle +++ b/x-pack/plugin/eql/build.gradle @@ -34,6 +34,7 @@ dependencies { } compile "org.antlr:antlr4-runtime:${antlrVersion}" compileOnly project(path: xpackModule('ql'), configuration: 'default') + testImplementation project(':test:framework') testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') testImplementation project(path: xpackModule('security'), configuration: 'testArtifacts') @@ -41,6 +42,8 @@ dependencies { testImplementation project(path: ':modules:reindex', configuration: 'runtime') testImplementation project(path: ':modules:parent-join', configuration: 'runtime') testImplementation project(path: ':modules:analysis-common', configuration: 'runtime') + testImplementation project(path: ':modules:transport-netty4', configuration: 'runtime') // for http in RestEqlCancellationIT + testImplementation project(path: ':plugins:transport-nio', configuration: 'runtime') // for http in RestEqlCancellationIT } diff --git a/x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/10_basic.yml b/x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/10_basic.yml index 79610f784c670..ef233b286a881 100644 --- a/x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/10_basic.yml +++ b/x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/10_basic.yml @@ -26,3 +26,38 @@ setup: - match: {hits.total.relation: "eq"} - match: {hits.events.0._source.user: "SYSTEM"} +--- +"Execute some EQL in async mode": + - do: + eql.search: + index: eql_test + wait_for_completion_timeout: "0ms" + body: + query: "process where user = 'SYSTEM'" + + - match: {is_running: true} + - match: {is_partial: true} + - is_true: id + - set: {id: id} + + - do: + eql.get: + id: $id + wait_for_completion_timeout: "10s" + + - match: {is_running: false} + - match: {is_partial: false} + - match: {timed_out: false} + - match: {hits.total.value: 1} + - match: {hits.total.relation: "eq"} + - match: {hits.events.0._source.user: "SYSTEM"} + + - do: + eql.delete: + id: $id + - match: {acknowledged: true} + + - do: + catch: missing + eql.delete: + id: $id diff --git a/x-pack/plugin/eql/qa/security/build.gradle b/x-pack/plugin/eql/qa/security/build.gradle new file mode 100644 index 0000000000000..3a402682ec04f --- /dev/null +++ b/x-pack/plugin/eql/qa/security/build.gradle @@ -0,0 +1,27 @@ +import org.elasticsearch.gradle.info.BuildParams + +apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' + +dependencies { +// testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') + testCompile project(path: xpackModule('eql'), configuration: 'runtime') + testCompile project(path: xpackModule('eql:qa:common'), configuration: 'runtime') + testCompile project(':x-pack:plugin:async-search:qa') +} + +testClusters.integTest { + testDistribution = 'DEFAULT' + if (BuildParams.isSnapshotBuild()) { + setting 'xpack.eql.enabled', 'true' + } + setting 'xpack.license.self_generated.type', 'basic' + setting 'xpack.monitoring.collection.enabled', 'true' + setting 'xpack.security.enabled', 'true' + numberOfNodes = 2 + extraConfigFile 'roles.yml', file('roles.yml') + user username: "test-admin", password: 'x-pack-test-password', role: "test-admin" + user username: "user1", password: 'x-pack-test-password', role: "user1" + user username: "user2", password: 'x-pack-test-password', role: "user2" +} diff --git a/x-pack/plugin/eql/qa/security/roles.yml b/x-pack/plugin/eql/qa/security/roles.yml new file mode 100644 index 0000000000000..4ab3be5ff0571 --- /dev/null +++ b/x-pack/plugin/eql/qa/security/roles.yml @@ -0,0 +1,33 @@ +# All cluster rights +# All operations on all indices +# Run as all users +test-admin: + cluster: + - all + indices: + - names: '*' + privileges: [ all ] + run_as: + - '*' + +user1: + cluster: + - cluster:monitor/main + indices: + - names: ['index-user1', 'index' ] + privileges: + - read + - write + - create_index + - indices:admin/refresh + +user2: + cluster: + - cluster:monitor/main + indices: + - names: [ 'index-user2', 'index' ] + privileges: + - read + - write + - create_index + - indices:admin/refresh diff --git a/x-pack/plugin/eql/qa/security/src/test/java/org/elasticsearch/xpack/eql/AsyncEqlSecurityIT.java b/x-pack/plugin/eql/qa/security/src/test/java/org/elasticsearch/xpack/eql/AsyncEqlSecurityIT.java new file mode 100644 index 0000000000000..40db5f99354b1 --- /dev/null +++ b/x-pack/plugin/eql/qa/security/src/test/java/org/elasticsearch/xpack/eql/AsyncEqlSecurityIT.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql; + +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.core.async.AsyncExecutionId; +import org.junit.Before; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationServiceField.RUN_AS_USER_HEADER; +import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class AsyncEqlSecurityIT extends ESRestTestCase { + /** + * All tests run as a superuser but use es-security-runas-user to become a less privileged user. + */ + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue("test-admin", new SecureString("x-pack-test-password".toCharArray())); + return Settings.builder() + .put(ThreadContext.PREFIX + ".Authorization", token) + .build(); + } + + @Before + public void indexDocuments() throws IOException { + createIndex("index", Settings.EMPTY); + index("index", "0", "event_type", "my_event", "@timestamp", "2020-04-09T12:35:48Z", "val", 0); + refresh("index"); + + createIndex("index-user1", Settings.EMPTY); + index("index-user1", "0", "event_type", "my_event", "@timestamp", "2020-04-09T12:35:48Z", "val", 0); + refresh("index-user1"); + + createIndex("index-user2", Settings.EMPTY); + index("index-user2", "0", "event_type", "my_event", "@timestamp", "2020-04-09T12:35:48Z", "val", 0); + refresh("index-user2"); + } + + public void testWithUsers() throws Exception { + testCase("user1", "user2"); + testCase("user2", "user1"); + } + + private void testCase(String user, String other) throws Exception { + for (String indexName : new String[] {"index", "index-" + user}) { + Response submitResp = submitAsyncEqlSearch(indexName, "my_event where val=0", TimeValue.timeValueSeconds(10), user); + assertOK(submitResp); + String id = extractResponseId(submitResp); + Response getResp = getAsyncEqlSearch(id, user); + assertOK(getResp); + + // other cannot access the result + ResponseException exc = expectThrows(ResponseException.class, () -> getAsyncEqlSearch(id, other)); + assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + + // other cannot delete the result + exc = expectThrows(ResponseException.class, () -> deleteAsyncEqlSearch(id, other)); + assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + + // other and user cannot access the result from direct get calls + AsyncExecutionId searchId = AsyncExecutionId.decode(id); + for (String runAs : new String[] {user, other}) { + exc = expectThrows(ResponseException.class, () -> get(XPackPlugin.ASYNC_RESULTS_INDEX, searchId.getDocId(), runAs)); + assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(403)); + assertThat(exc.getMessage(), containsString("unauthorized")); + } + + Response delResp = deleteAsyncEqlSearch(id, user); + assertOK(delResp); + } + ResponseException exc = expectThrows(ResponseException.class, + () -> submitAsyncEqlSearch("index-" + other, "*", TimeValue.timeValueSeconds(10), user)); + assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(403)); + assertThat(exc.getMessage(), containsString("unauthorized")); + } + + static String extractResponseId(Response response) throws IOException { + Map map = toMap(response); + return (String) map.get("id"); + } + + static void index(String index, String id, Object... fields) throws IOException { + XContentBuilder document = jsonBuilder().startObject(); + for (int i = 0; i < fields.length; i += 2) { + document.field((String) fields[i], fields[i + 1]); + } + document.endObject(); + final Request request = new Request("POST", "/" + index + "/_doc/" + id); + request.setJsonEntity(Strings.toString(document)); + assertOK(client().performRequest(request)); + } + + static void refresh(String index) throws IOException { + assertOK(adminClient().performRequest(new Request("POST", "/" + index + "/_refresh"))); + } + + static Response get(String index, String id, String user) throws IOException { + final Request request = new Request("GET", "/" + index + "/_doc/" + id); + setRunAsHeader(request, user); + return client().performRequest(request); + } + + static Response submitAsyncEqlSearch(String indexName, String query, TimeValue waitForCompletion, String user) throws IOException { + final Request request = new Request("POST", indexName + "/_eql/search"); + setRunAsHeader(request, user); + request.setJsonEntity(Strings.toString(JsonXContent.contentBuilder() + .startObject() + .field("event_category_field", "event_type") + .field("query", query) + .endObject())); + request.addParameter("wait_for_completion_timeout", waitForCompletion.toString()); + // we do the cleanup explicitly + request.addParameter("keep_on_completion", "true"); + return client().performRequest(request); + } + + static Response getAsyncEqlSearch(String id, String user) throws IOException { + final Request request = new Request("GET", "/_eql/search/" + id); + setRunAsHeader(request, user); + request.addParameter("wait_for_completion_timeout", "0ms"); + return client().performRequest(request); + } + + static Response deleteAsyncEqlSearch(String id, String user) throws IOException { + final Request request = new Request("DELETE", "/_eql/search/" + id); + setRunAsHeader(request, user); + return client().performRequest(request); + } + + static Map toMap(Response response) throws IOException { + return toMap(EntityUtils.toString(response.getEntity())); + } + + static Map toMap(String response) { + return XContentHelper.convertToMap(JsonXContent.jsonXContent, response, false); + } + + static void setRunAsHeader(Request request, String user) { + final RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder(); + builder.addHeader(RUN_AS_USER_HEADER, user); + request.setOptions(builder); + } + +} diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlBlockingIntegTestCase.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlBlockingIntegTestCase.java new file mode 100644 index 0000000000000..8213e7f1fc471 --- /dev/null +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlBlockingIntegTestCase.java @@ -0,0 +1,240 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; +import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesAction; +import org.elasticsearch.action.support.ActionFilter; +import org.elasticsearch.action.support.ActionFilterChain; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.shard.SearchOperationListener; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskInfo; +import org.elasticsearch.test.ESIntegTestCase; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; + +/** + * IT tests that can block EQL execution at different places + */ +@ESIntegTestCase.ClusterScope(scope = SUITE, numDataNodes = 0, numClientNodes = 0, maxNumDataNodes = 0, transportClientRatio = 0) +public abstract class AbstractEqlBlockingIntegTestCase extends AbstractEqlIntegTestCase { + + protected List initBlockFactory(boolean searchBlock, boolean fieldCapsBlock) { + List plugins = new ArrayList<>(); + for (PluginsService pluginsService : internalCluster().getInstances(PluginsService.class)) { + plugins.addAll(pluginsService.filterPlugins(SearchBlockPlugin.class)); + } + for (SearchBlockPlugin plugin : plugins) { + plugin.reset(); + if (searchBlock) { + plugin.enableSearchBlock(); + } + if (fieldCapsBlock) { + plugin.enableFieldCapBlock(); + } + } + return plugins; + } + + protected void disableBlocks(List plugins) { + disableFieldCapBlocks(plugins); + disableSearchBlocks(plugins); + } + + protected void disableSearchBlocks(List plugins) { + for (SearchBlockPlugin plugin : plugins) { + plugin.disableSearchBlock(); + } + } + + protected void disableFieldCapBlocks(List plugins) { + for (SearchBlockPlugin plugin : plugins) { + plugin.disableFieldCapBlock(); + } + } + + protected void awaitForBlockedSearches(List plugins, String index) throws Exception { + int numberOfShards = getNumShards(index).numPrimaries; + assertBusy(() -> { + int numberOfBlockedPlugins = getNumberOfContexts(plugins); + logger.trace("The plugin blocked on {} out of {} shards", numberOfBlockedPlugins, numberOfShards); + assertThat(numberOfBlockedPlugins, greaterThan(0)); + }); + } + + protected int getNumberOfContexts(List plugins) throws Exception { + int count = 0; + for (SearchBlockPlugin plugin : plugins) { + count += plugin.contexts.get(); + } + return count; + } + + protected int getNumberOfFieldCaps(List plugins) throws Exception { + int count = 0; + for (SearchBlockPlugin plugin : plugins) { + count += plugin.fieldCaps.get(); + } + return count; + } + + protected void awaitForBlockedFieldCaps(List plugins) throws Exception { + assertBusy(() -> { + int numberOfBlockedPlugins = getNumberOfFieldCaps(plugins); + logger.trace("The plugin blocked on {} nodes", numberOfBlockedPlugins); + assertThat(numberOfBlockedPlugins, greaterThan(0)); + }); + } + + public static class SearchBlockPlugin extends Plugin implements ActionPlugin { + protected final Logger logger = LogManager.getLogger(getClass()); + + private final AtomicInteger contexts = new AtomicInteger(); + + private final AtomicInteger fieldCaps = new AtomicInteger(); + + private final AtomicBoolean shouldBlockOnSearch = new AtomicBoolean(false); + + private final AtomicBoolean shouldBlockOnFieldCapabilities = new AtomicBoolean(false); + + private final String nodeId; + + public void reset() { + contexts.set(0); + fieldCaps.set(0); + } + + public void disableSearchBlock() { + shouldBlockOnSearch.set(false); + } + + public void enableSearchBlock() { + shouldBlockOnSearch.set(true); + } + + + public void disableFieldCapBlock() { + shouldBlockOnFieldCapabilities.set(false); + } + + public void enableFieldCapBlock() { + shouldBlockOnFieldCapabilities.set(true); + } + + public SearchBlockPlugin(Settings settings, Path configPath) throws Exception { + nodeId = settings.get("node.name"); + } + + @Override + public void onIndexModule(IndexModule indexModule) { + super.onIndexModule(indexModule); + indexModule.addSearchOperationListener(new SearchOperationListener() { + @Override + public void onNewContext(SearchContext context) { + contexts.incrementAndGet(); + try { + logger.trace("blocking search on " + nodeId); + assertBusy(() -> assertFalse(shouldBlockOnSearch.get())); + logger.trace("unblocking search on " + nodeId); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + } + + @Override + public List getActionFilters() { + List list = new ArrayList<>(); + list.add(new ActionFilter() { + @Override + public int order() { + return 0; + } + + @Override + public void apply( + Task task, String action, Request request, ActionListener listener, + ActionFilterChain chain) { + if (action.equals(FieldCapabilitiesAction.NAME)) { + try { + fieldCaps.incrementAndGet(); + logger.trace("blocking field caps on " + nodeId); + assertBusy(() -> assertFalse(shouldBlockOnFieldCapabilities.get())); + logger.trace("unblocking field caps on " + nodeId); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + chain.proceed(task, action, request, listener); + } + }); + return list; + } + } + + @Override + protected Collection> nodePlugins() { + List> plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(SearchBlockPlugin.class); + return plugins; + } + + protected TaskId findTaskWithXOpaqueId(String id, String action) { + TaskInfo taskInfo = getTaskInfoWithXOpaqueId(id, action); + if (taskInfo != null) { + return taskInfo.getTaskId(); + } else { + return null; + } + } + + protected TaskInfo getTaskInfoWithXOpaqueId(String id, String action) { + ListTasksResponse tasks = client().admin().cluster().prepareListTasks().setActions(action).get(); + for (TaskInfo task : tasks.getTasks()) { + if (id.equals(task.getHeaders().get(Task.X_OPAQUE_ID))) { + return task; + } + } + return null; + } + + protected TaskId cancelTaskWithXOpaqueId(String id, String action) { + TaskId taskId = findTaskWithXOpaqueId(id, action); + assertNotNull(taskId); + logger.trace("Cancelling task " + taskId); + CancelTasksResponse response = client().admin().cluster().prepareCancelTasks().setTaskId(taskId).get(); + assertThat(response.getTasks(), hasSize(1)); + assertThat(response.getTasks().get(0).getAction(), equalTo(action)); + logger.trace("Task is cancelled " + taskId); + return taskId; + } + +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java similarity index 100% rename from x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java rename to x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AsyncEqlSearchActionIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AsyncEqlSearchActionIT.java new file mode 100644 index 0000000000000..f696bf2bf6479 --- /dev/null +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AsyncEqlSearchActionIT.java @@ -0,0 +1,349 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.action; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.NoShardAvailableActionException; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.ByteBufferStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.core.async.AsyncExecutionId; +import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; +import org.elasticsearch.xpack.core.async.DeleteAsyncResultRequest; +import org.elasticsearch.xpack.core.async.GetAsyncResultRequest; +import org.elasticsearch.xpack.eql.async.StoredAsyncResponse; +import org.elasticsearch.xpack.eql.plugin.EqlAsyncGetResultAction; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.junit.After; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Function; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFutureThrows; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class AsyncEqlSearchActionIT extends AbstractEqlBlockingIntegTestCase { + + private final ExecutorService executorService = Executors.newFixedThreadPool(1); + + NamedWriteableRegistry registry = new NamedWriteableRegistry(new SearchModule(Settings.EMPTY, true, + Collections.emptyList()).getNamedWriteables()); + + /** + * Shutdown the executor so we don't leak threads into other test runs. + */ + @After + public void shutdownExec() { + executorService.shutdown(); + } + + private void prepareIndex() throws Exception { + assertAcked(client().admin().indices().prepareCreate("test") + .addMapping("_doc", "val", "type=integer", "event_type", "type=keyword", "@timestamp", "type=date", "i", "type=integer") + .get()); + createIndex("idx_unmapped"); + + int numDocs = randomIntBetween(6, 20); + + List builders = new ArrayList<>(); + + for (int i = 0; i < numDocs; i++) { + int fieldValue = randomIntBetween(0, 10); + builders.add(client().prepareIndex("test", "_doc").setSource( + jsonBuilder().startObject() + .field("val", fieldValue) + .field("event_type", "my_event") + .field("@timestamp", "2020-04-09T12:35:48Z") + .field("i", i) + .endObject())); + } + indexRandom(true, builders); + } + + public void testBasicAsyncExecution() throws Exception { + prepareIndex(); + + boolean success = randomBoolean(); + String query = success ? "my_event where i=1" : "my_event where 10/i=1"; + EqlSearchRequest request = new EqlSearchRequest().indices("test").query(query).eventCategoryField("event_type") + .waitForCompletionTimeout(TimeValue.timeValueMillis(1)); + + List plugins = initBlockFactory(true, false); + + logger.trace("Starting async search"); + EqlSearchResponse response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.isRunning(), is(true)); + assertThat(response.isPartial(), is(true)); + assertThat(response.id(), notNullValue()); + + logger.trace("Waiting for block to be established"); + awaitForBlockedSearches(plugins, "test"); + logger.trace("Block is established"); + + if (randomBoolean()) { + // let's timeout first + GetAsyncResultRequest getResultsRequest = new GetAsyncResultRequest(response.id()) + .setWaitForCompletionTimeout(TimeValue.timeValueMillis(10)); + EqlSearchResponse responseWithTimeout = client().execute(EqlAsyncGetResultAction.INSTANCE, getResultsRequest).get(); + assertThat(responseWithTimeout.isRunning(), is(true)); + assertThat(responseWithTimeout.isPartial(), is(true)); + assertThat(responseWithTimeout.id(), equalTo(response.id())); + } + + // Now we wait + GetAsyncResultRequest getResultsRequest = new GetAsyncResultRequest(response.id()) + .setWaitForCompletionTimeout(TimeValue.timeValueSeconds(10)); + ActionFuture future = client().execute(EqlAsyncGetResultAction.INSTANCE, getResultsRequest); + disableBlocks(plugins); + if (success) { + response = future.get(); + assertThat(response, notNullValue()); + assertThat(response.hits().events().size(), equalTo(1)); + } else { + Exception ex = expectThrows(Exception.class, future::actionGet); + assertThat(ex.getCause().getMessage(), containsString("by zero")); + } + + AcknowledgedResponse deleteResponse = + client().execute(DeleteAsyncResultAction.INSTANCE, new DeleteAsyncResultRequest(response.id())).actionGet(); + assertThat(deleteResponse.isAcknowledged(), equalTo(true)); + } + + public void testGoingAsync() throws Exception { + prepareIndex(); + + boolean success = randomBoolean(); + String query = success ? "my_event where i=1" : "my_event where 10/i=1"; + EqlSearchRequest request = new EqlSearchRequest().indices("test").query(query).eventCategoryField("event_type") + .waitForCompletionTimeout(TimeValue.timeValueMillis(1)); + + boolean customKeepAlive = randomBoolean(); + TimeValue keepAliveValue; + if (customKeepAlive) { + keepAliveValue = TimeValue.parseTimeValue(randomTimeValue(1, 5, "d"), "test"); + request.keepAlive(keepAliveValue); + } else { + keepAliveValue = EqlSearchRequest.DEFAULT_KEEP_ALIVE; + } + + List plugins = initBlockFactory(true, false); + + String opaqueId = randomAlphaOfLength(10); + logger.trace("Starting async search"); + EqlSearchResponse response = client().filterWithHeader(Collections.singletonMap(Task.X_OPAQUE_ID, opaqueId)) + .execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.isRunning(), is(true)); + assertThat(response.isPartial(), is(true)); + assertThat(response.id(), notNullValue()); + + logger.trace("Waiting for block to be established"); + awaitForBlockedSearches(plugins, "test"); + logger.trace("Block is established"); + + String id = response.id(); + TaskId taskId = findTaskWithXOpaqueId(opaqueId, EqlSearchAction.NAME + "[a]"); + assertThat(taskId, notNullValue()); + + disableBlocks(plugins); + + assertBusy(() -> assertThat(findTaskWithXOpaqueId(opaqueId, EqlSearchAction.NAME + "[a]"), nullValue())); + StoredAsyncResponse doc = getStoredRecord(id); + // Make sure that the expiration time is not more than 1 min different from the current time + keep alive + assertThat(System.currentTimeMillis() + keepAliveValue.getMillis() - doc.getExpirationTime(), + lessThan(doc.getExpirationTime() + TimeValue.timeValueMinutes(1).getMillis())); + if (success) { + assertThat(doc.getException(), nullValue()); + assertThat(doc.getResponse(), notNullValue()); + assertThat(doc.getResponse().hits().events().size(), equalTo(1)); + } else { + assertThat(doc.getException(), notNullValue()); + assertThat(doc.getResponse(), nullValue()); + assertThat(doc.getException().getCause().getMessage(), containsString("by zero")); + } + } + + public void testAsyncCancellation() throws Exception { + prepareIndex(); + + boolean success = randomBoolean(); + String query = success ? "my_event where i=1" : "my_event where 10/i=1"; + EqlSearchRequest request = new EqlSearchRequest().indices("test").query(query).eventCategoryField("event_type") + .waitForCompletionTimeout(TimeValue.timeValueMillis(1)); + + boolean customKeepAlive = randomBoolean(); + final TimeValue keepAliveValue; + if (customKeepAlive) { + keepAliveValue = TimeValue.parseTimeValue(randomTimeValue(1, 5, "d"), "test"); + request.keepAlive(keepAliveValue); + } + + List plugins = initBlockFactory(true, false); + + String opaqueId = randomAlphaOfLength(10); + logger.trace("Starting async search"); + EqlSearchResponse response = client().filterWithHeader(Collections.singletonMap(Task.X_OPAQUE_ID, opaqueId)) + .execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.isRunning(), is(true)); + assertThat(response.isPartial(), is(true)); + assertThat(response.id(), notNullValue()); + + logger.trace("Waiting for block to be established"); + awaitForBlockedSearches(plugins, "test"); + logger.trace("Block is established"); + + ActionFuture deleteResponse = + client().execute(DeleteAsyncResultAction.INSTANCE, new DeleteAsyncResultRequest(response.id())); + disableBlocks(plugins); + assertThat(deleteResponse.actionGet().isAcknowledged(), equalTo(true)); + + deleteResponse = client().execute(DeleteAsyncResultAction.INSTANCE, new DeleteAsyncResultRequest(response.id())); + assertFutureThrows(deleteResponse, ResourceNotFoundException.class); + } + + public void testFinishingBeforeTimeout() throws Exception { + prepareIndex(); + + boolean success = randomBoolean(); + boolean keepOnCompletion = randomBoolean(); + String query = success ? "my_event where i=1" : "my_event where 10/i=1"; + EqlSearchRequest request = new EqlSearchRequest().indices("test").query(query).eventCategoryField("event_type") + .waitForCompletionTimeout(TimeValue.timeValueSeconds(10)); + if (keepOnCompletion || randomBoolean()) { + request.keepOnCompletion(keepOnCompletion); + } + + if (success) { + EqlSearchResponse response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.isRunning(), is(false)); + assertThat(response.isPartial(), is(false)); + assertThat(response.id(), notNullValue()); + assertThat(response.hits().events().size(), equalTo(1)); + if (keepOnCompletion) { + StoredAsyncResponse doc = getStoredRecord(response.id()); + assertThat(doc, notNullValue()); + assertThat(doc.getException(), nullValue()); + assertThat(doc.getResponse(), notNullValue()); + assertThat(doc.getResponse().hits().events().size(), equalTo(1)); + EqlSearchResponse storedResponse = client().execute(EqlAsyncGetResultAction.INSTANCE, + new GetAsyncResultRequest(response.id())).actionGet(); + assertThat(storedResponse, equalTo(response)); + + AcknowledgedResponse deleteResponse = + client().execute(DeleteAsyncResultAction.INSTANCE, new DeleteAsyncResultRequest(response.id())).actionGet(); + assertThat(deleteResponse.isAcknowledged(), equalTo(true)); + } + } else { + Exception ex = expectThrows(Exception.class, + () -> client().execute(EqlSearchAction.INSTANCE, request).get()); + assertThat(ex.getMessage(), containsString("by zero")); + } + } + + public StoredAsyncResponse getStoredRecord(String id) throws Exception { + try { + GetResponse doc = client().prepareGet(XPackPlugin.ASYNC_RESULTS_INDEX, "_doc", AsyncExecutionId.decode(id).getDocId()).get(); + if (doc.isExists()) { + String value = doc.getSource().get("result").toString(); + try (ByteBufferStreamInput buf = new ByteBufferStreamInput(ByteBuffer.wrap(Base64.getDecoder().decode(value)))) { + try (StreamInput in = new NamedWriteableAwareStreamInput(buf, registry)) { + in.setVersion(Version.readVersion(in)); + return new StoredAsyncResponse<>(EqlSearchResponse::new, in); + } + } + } + return null; + } catch (IndexNotFoundException | NoShardAvailableActionException ex) { + return null; + } + } + + public static org.hamcrest.Matcher eqlSearchResponseMatcherEqualTo(EqlSearchResponse eqlSearchResponse) { + return new BaseMatcher() { + + @Override + public void describeTo(Description description) { + description.appendText(Strings.toString(eqlSearchResponse)); + } + + @Override + public boolean matches(Object o) { + if (eqlSearchResponse == o) { + return true; + } + if (o == null || EqlSearchResponse.class != o.getClass()) { + return false; + } + EqlSearchResponse that = (EqlSearchResponse) o; + // We don't compare took since it might deffer + return Objects.equals(eqlSearchResponse.hits(), that.hits()) + && Objects.equals(eqlSearchResponse.isTimeout(), that.isTimeout()); + } + }; + } + + public static class FakePainlessScriptPlugin extends MockScriptPlugin { + + @Override + protected Map, Object>> pluginScripts() { + Map, Object>> scripts = new HashMap<>(); + scripts.put("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.div(" + + "params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", FakePainlessScriptPlugin::fail); + return scripts; + } + + public static Object fail(Map arg) { + throw new ArithmeticException("Division by zero"); + } + + public String pluginScriptLang() { + // Faking painless + return "painless"; + } + } + + @Override + protected Collection> nodePlugins() { + List> plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(FakePainlessScriptPlugin.class); + return plugins; + } + +} diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/EqlCancellationIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/EqlCancellationIT.java index e8f1e082f9508..4d85defc034b0 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/EqlCancellationIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/EqlCancellationIT.java @@ -6,50 +6,26 @@ package org.elasticsearch.xpack.eql.action; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; -import org.elasticsearch.action.fieldcaps.FieldCapabilitiesAction; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; -import org.elasticsearch.action.support.ActionFilter; -import org.elasticsearch.action.support.ActionFilterChain; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexModule; -import org.elasticsearch.index.shard.SearchOperationListener; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.PluginsService; -import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; -import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.tasks.TaskInfo; import org.junit.After; -import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; -public class EqlCancellationIT extends AbstractEqlIntegTestCase { +public class EqlCancellationIT extends AbstractEqlBlockingIntegTestCase { private final ExecutorService executorService = Executors.newFixedThreadPool(1); @@ -96,20 +72,8 @@ public void testCancellation() throws Exception { awaitForBlockedFieldCaps(plugins); } logger.trace("Block is established"); - ListTasksResponse tasks = client().admin().cluster().prepareListTasks().setActions(EqlSearchAction.NAME).get(); - TaskId taskId = null; - for (TaskInfo task : tasks.getTasks()) { - if (id.equals(task.getHeaders().get(Task.X_OPAQUE_ID))) { - taskId = task.getTaskId(); - break; - } - } - assertNotNull(taskId); - logger.trace("Cancelling task " + taskId); - CancelTasksResponse response = client().admin().cluster().prepareCancelTasks().setTaskId(taskId).get(); - assertThat(response.getTasks(), hasSize(1)); - assertThat(response.getTasks().get(0).getAction(), equalTo(EqlSearchAction.NAME)); - logger.trace("Task is cancelled " + taskId); + cancelTaskWithXOpaqueId(id, EqlSearchAction.NAME); + disableBlocks(plugins); Exception exception = expectThrows(Exception.class, future::get); Throwable inner = ExceptionsHelper.unwrap(exception, SearchPhaseExecutionException.class); @@ -126,155 +90,4 @@ public void testCancellation() throws Exception { assertNotNull(cancellationException); } } - - private List initBlockFactory(boolean searchBlock, boolean fieldCapsBlock) { - List plugins = new ArrayList<>(); - for (PluginsService pluginsService : internalCluster().getDataNodeInstances(PluginsService.class)) { - plugins.addAll(pluginsService.filterPlugins(SearchBlockPlugin.class)); - } - for (SearchBlockPlugin plugin : plugins) { - plugin.reset(); - if (searchBlock) { - plugin.enableSearchBlock(); - } - if (fieldCapsBlock) { - plugin.enableFieldCapBlock(); - } - } - return plugins; - } - - private void disableBlocks(List plugins) { - for (SearchBlockPlugin plugin : plugins) { - plugin.disableSearchBlock(); - plugin.disableFieldCapBlock(); - } - } - - private void awaitForBlockedSearches(List plugins, String index) throws Exception { - int numberOfShards = getNumShards(index).numPrimaries; - assertBusy(() -> { - int numberOfBlockedPlugins = getNumberOfContexts(plugins); - logger.trace("The plugin blocked on {} out of {} shards", numberOfBlockedPlugins, numberOfShards); - assertThat(numberOfBlockedPlugins, greaterThan(0)); - }); - } - - private int getNumberOfContexts(List plugins) throws Exception { - int count = 0; - for (SearchBlockPlugin plugin : plugins) { - count += plugin.contexts.get(); - } - return count; - } - - private int getNumberOfFieldCaps(List plugins) throws Exception { - int count = 0; - for (SearchBlockPlugin plugin : plugins) { - count += plugin.fieldCaps.get(); - } - return count; - } - - private void awaitForBlockedFieldCaps(List plugins) throws Exception { - assertBusy(() -> { - int numberOfBlockedPlugins = getNumberOfFieldCaps(plugins); - logger.trace("The plugin blocked on {} nodes", numberOfBlockedPlugins); - assertThat(numberOfBlockedPlugins, greaterThan(0)); - }); - } - - public static class SearchBlockPlugin extends LocalStateEQLXPackPlugin { - protected final Logger logger = LogManager.getLogger(getClass()); - - private final AtomicInteger contexts = new AtomicInteger(); - - private final AtomicInteger fieldCaps = new AtomicInteger(); - - private final AtomicBoolean shouldBlockOnSearch = new AtomicBoolean(false); - - private final AtomicBoolean shouldBlockOnFieldCapabilities = new AtomicBoolean(false); - - private final String nodeId; - - public void reset() { - contexts.set(0); - fieldCaps.set(0); - } - - public void disableSearchBlock() { - shouldBlockOnSearch.set(false); - } - - public void enableSearchBlock() { - shouldBlockOnSearch.set(true); - } - - - public void disableFieldCapBlock() { - shouldBlockOnFieldCapabilities.set(false); - } - - public void enableFieldCapBlock() { - shouldBlockOnFieldCapabilities.set(true); - } - - public SearchBlockPlugin(Settings settings, Path configPath) throws Exception { - super(settings, configPath); - nodeId = settings.get("node.name"); - } - - @Override - public void onIndexModule(IndexModule indexModule) { - super.onIndexModule(indexModule); - indexModule.addSearchOperationListener(new SearchOperationListener() { - @Override - public void onNewContext(SearchContext context) { - contexts.incrementAndGet(); - try { - logger.trace("blocking search on " + nodeId); - assertBusy(() -> assertFalse(shouldBlockOnSearch.get())); - logger.trace("unblocking search on " + nodeId); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - }); - } - - @Override - public List getActionFilters() { - List list = new ArrayList<>(super.getActionFilters()); - list.add(new ActionFilter() { - @Override - public int order() { - return 0; - } - - @Override - public void apply( - Task task, String action, Request request, ActionListener listener, - ActionFilterChain chain) { - if (action.equals(FieldCapabilitiesAction.NAME)) { - try { - fieldCaps.incrementAndGet(); - logger.trace("blocking field caps on " + nodeId); - assertBusy(() -> assertFalse(shouldBlockOnFieldCapabilities.get())); - logger.trace("unblocking field caps on " + nodeId); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - chain.proceed(task, action, request, listener); - } - }); - return list; - } - } - - @Override - protected Collection> nodePlugins() { - return Collections.singletonList(SearchBlockPlugin.class); - } - } diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/RestEqlCancellationIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/RestEqlCancellationIT.java new file mode 100644 index 0000000000000..27c003ec18f44 --- /dev/null +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/RestEqlCancellationIT.java @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.action; + +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.client.Cancellable; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseListener; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.Netty4Plugin; +import org.elasticsearch.transport.nio.NioTransportPlugin; +import org.junit.BeforeClass; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class RestEqlCancellationIT extends AbstractEqlBlockingIntegTestCase { + + private static String nodeHttpTypeKey; + + @SuppressWarnings("unchecked") + @BeforeClass + public static void setUpTransport() { + nodeHttpTypeKey = getHttpTypeKey(randomFrom(Netty4Plugin.class, NioTransportPlugin.class)); + } + + @Override + protected boolean addMockHttpTransport() { + return false; // enable http + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(NetworkModule.HTTP_TYPE_KEY, nodeHttpTypeKey).build(); + } + + private static String getHttpTypeKey(Class clazz) { + if (clazz.equals(NioTransportPlugin.class)) { + return NioTransportPlugin.NIO_HTTP_TRANSPORT_NAME; + } else { + assert clazz.equals(Netty4Plugin.class); + return Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME; + } + } + + @Override + protected Collection> nodePlugins() { + List> plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(getTestTransportPlugin()); + plugins.add(Netty4Plugin.class); + plugins.add(NioTransportPlugin.class); + return plugins; + } + + public void testRestCancellation() throws Exception { + assertAcked(client().admin().indices().prepareCreate("test") + .addMapping("_doc", "val", "type=integer", "event_type", "type=keyword", "@timestamp", "type=date") + .get()); + createIndex("idx_unmapped"); + + int numDocs = randomIntBetween(6, 20); + + List builders = new ArrayList<>(); + + for (int i = 0; i < numDocs; i++) { + int fieldValue = randomIntBetween(0, 10); + builders.add(client().prepareIndex("test", "_doc").setSource( + jsonBuilder().startObject() + .field("val", fieldValue).field("event_type", "my_event").field("@timestamp", "2020-04-09T12:35:48Z") + .endObject())); + } + + indexRandom(true, builders); + + // We are cancelling during both mapping and searching but we cancel during mapping so we should never reach the second block + List plugins = initBlockFactory(true, true); + org.elasticsearch.client.eql.EqlSearchRequest eqlSearchRequest = + new org.elasticsearch.client.eql.EqlSearchRequest("test", "my_event where val=1").eventCategoryField("event_type"); + String id = randomAlphaOfLength(10); + + Request request = new Request("GET", "/test/_eql/search"); + request.setJsonEntity(Strings.toString(eqlSearchRequest)); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader(Task.X_OPAQUE_ID, id)); + logger.trace("Preparing search"); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference error = new AtomicReference<>(); + Cancellable cancellable = getRestClient().performRequestAsync(request, new ResponseListener() { + @Override + public void onSuccess(Response response) { + latch.countDown(); + } + + @Override + public void onFailure(Exception exception) { + error.set(exception); + latch.countDown(); + } + }); + + logger.trace("Waiting for block to be established"); + awaitForBlockedFieldCaps(plugins); + logger.trace("Block is established"); + assertThat(getTaskInfoWithXOpaqueId(id, EqlSearchAction.NAME), notNullValue()); + cancellable.cancel(); + logger.trace("Request is cancelled"); + disableFieldCapBlocks(plugins); + // The task should be cancelled before ever reaching search blocks + assertBusy(() -> { + assertThat(getTaskInfoWithXOpaqueId(id, EqlSearchAction.NAME), nullValue()); + }); + // Make sure it didn't reach search blocks + assertThat(getNumberOfContexts(plugins), equalTo(0)); + disableSearchBlocks(plugins); + + latch.await(); + assertThat(error.get(), instanceOf(CancellationException.class)); + } + + @Override + protected boolean ignoreExternalCluster() { + return true; + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java index 2439d3df407ff..a4f49ff24f98b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.eql.action; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; @@ -13,6 +14,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -37,6 +39,9 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Replaceable, ToXContent { + public static long MIN_KEEP_ALIVE = TimeValue.timeValueMinutes(1).millis(); + public static TimeValue DEFAULT_KEEP_ALIVE = TimeValue.timeValueDays(5); + private String[] indices; private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, false); @@ -51,6 +56,11 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re private String query; private boolean isCaseSensitive = false; + // Async settings + private TimeValue waitForCompletionTimeout = null; + private TimeValue keepAlive = DEFAULT_KEEP_ALIVE; + private boolean keepOnCompletion; + static final String KEY_FILTER = "filter"; static final String KEY_TIMESTAMP_FIELD = "timestamp_field"; static final String KEY_TIEBREAKER_FIELD = "tiebreaker_field"; @@ -59,6 +69,9 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re static final String KEY_SIZE = "size"; static final String KEY_SEARCH_AFTER = "search_after"; static final String KEY_QUERY = "query"; + static final String KEY_WAIT_FOR_COMPLETION_TIMEOUT = "wait_for_completion_timeout"; + static final String KEY_KEEP_ALIVE = "keep_alive"; + static final String KEY_KEEP_ON_COMPLETION = "keep_on_completion"; static final String KEY_CASE_SENSITIVE = "case_sensitive"; static final ParseField FILTER = new ParseField(KEY_FILTER); @@ -69,6 +82,9 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re static final ParseField SIZE = new ParseField(KEY_SIZE); static final ParseField SEARCH_AFTER = new ParseField(KEY_SEARCH_AFTER); static final ParseField QUERY = new ParseField(KEY_QUERY); + static final ParseField WAIT_FOR_COMPLETION_TIMEOUT = new ParseField(KEY_WAIT_FOR_COMPLETION_TIMEOUT); + static final ParseField KEEP_ALIVE = new ParseField(KEY_KEEP_ALIVE); + static final ParseField KEEP_ON_COMPLETION = new ParseField(KEY_KEEP_ON_COMPLETION); static final ParseField CASE_SENSITIVE = new ParseField(KEY_CASE_SENSITIVE); private static final ObjectParser PARSER = objectParser(EqlSearchRequest::new); @@ -89,6 +105,11 @@ public EqlSearchRequest(StreamInput in) throws IOException { fetchSize = in.readVInt(); searchAfterBuilder = in.readOptionalWriteable(SearchAfterBuilder::new); query = in.readString(); + if (in.getVersion().onOrAfter(Version.V_7_9_0)) { + this.waitForCompletionTimeout = in.readOptionalTimeValue(); + this.keepAlive = in.readOptionalTimeValue(); + this.keepOnCompletion = in.readBoolean(); + } isCaseSensitive = in.readBoolean(); } @@ -131,6 +152,11 @@ public ActionRequestValidationException validate() { validationException = addValidationError("size must be greater than 0", validationException); } + if (keepAlive != null && keepAlive.getMillis() < MIN_KEEP_ALIVE) { + validationException = + addValidationError("[keep_alive] must be greater than 1 minute, got:" + keepAlive.toString(), validationException); + } + return validationException; } @@ -154,6 +180,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.field(KEY_QUERY, query); + if (waitForCompletionTimeout != null) { + builder.field(KEY_WAIT_FOR_COMPLETION_TIMEOUT, waitForCompletionTimeout); + } + if (keepAlive != null) { + builder.field(KEY_KEEP_ALIVE, keepAlive); + } + builder.field(KEY_KEEP_ON_COMPLETION, keepOnCompletion); builder.field(KEY_CASE_SENSITIVE, isCaseSensitive); return builder; @@ -175,6 +208,12 @@ protected static ObjectParser objectParser parser.declareField(EqlSearchRequest::setSearchAfter, SearchAfterBuilder::fromXContent, SEARCH_AFTER, ObjectParser.ValueType.OBJECT_ARRAY); parser.declareString(EqlSearchRequest::query, QUERY); + parser.declareField(EqlSearchRequest::waitForCompletionTimeout, + (p, c) -> TimeValue.parseTimeValue(p.text(), KEY_WAIT_FOR_COMPLETION_TIMEOUT), WAIT_FOR_COMPLETION_TIMEOUT, + ObjectParser.ValueType.VALUE); + parser.declareField(EqlSearchRequest::keepAlive, + (p, c) -> TimeValue.parseTimeValue(p.text(), KEY_KEEP_ALIVE), KEEP_ALIVE, ObjectParser.ValueType.VALUE); + parser.declareBoolean(EqlSearchRequest::keepOnCompletion, KEEP_ON_COMPLETION); parser.declareBoolean(EqlSearchRequest::isCaseSensitive, CASE_SENSITIVE); return parser; } @@ -251,6 +290,33 @@ public EqlSearchRequest query(String query) { return this; } + public TimeValue waitForCompletionTimeout() { + return waitForCompletionTimeout; + } + + public EqlSearchRequest waitForCompletionTimeout(TimeValue waitForCompletionTimeout) { + this.waitForCompletionTimeout = waitForCompletionTimeout; + return this; + } + + public TimeValue keepAlive() { + return keepAlive; + } + + public EqlSearchRequest keepAlive(TimeValue keepAlive) { + this.keepAlive = keepAlive; + return this; + } + + public boolean keepOnCompletion() { + return keepOnCompletion; + } + + public EqlSearchRequest keepOnCompletion(boolean keepOnCompletion) { + this.keepOnCompletion = keepOnCompletion; + return this; + } + public boolean isCaseSensitive() { return this.isCaseSensitive; } public EqlSearchRequest isCaseSensitive(boolean isCaseSensitive) { @@ -271,6 +337,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(fetchSize); out.writeOptionalWriteable(searchAfterBuilder); out.writeString(query); + if (out.getVersion().onOrAfter(Version.V_7_9_0)) { + out.writeOptionalTimeValue(waitForCompletionTimeout); + out.writeOptionalTimeValue(keepAlive); + out.writeBoolean(keepOnCompletion); + } out.writeBoolean(isCaseSensitive); } @@ -293,6 +364,8 @@ public boolean equals(Object o) { Objects.equals(implicitJoinKeyField, that.implicitJoinKeyField) && Objects.equals(searchAfterBuilder, that.searchAfterBuilder) && Objects.equals(query, that.query) && + Objects.equals(waitForCompletionTimeout, that.waitForCompletionTimeout) && + Objects.equals(keepAlive, that.keepAlive) && Objects.equals(isCaseSensitive, that.isCaseSensitive); } @@ -309,6 +382,8 @@ public int hashCode() { implicitJoinKeyField, searchAfterBuilder, query, + waitForCompletionTimeout, + keepAlive, isCaseSensitive); } @@ -329,13 +404,16 @@ public IndicesOptions indicesOptions() { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { - return new EqlSearchTask(id, type, action, () -> { - StringBuilder sb = new StringBuilder(); - sb.append("indices["); - Strings.arrayToDelimitedString(indices, ",", sb); - sb.append("], "); - sb.append(query); - return sb.toString(); - }, parentTaskId, headers); + return new EqlSearchTask(id, type, action, getDescription(), parentTaskId, headers, null, null, keepAlive); + } + + @Override + public String getDescription() { + StringBuilder sb = new StringBuilder(); + sb.append("indices["); + Strings.arrayToDelimitedString(indices, ",", sb); + sb.append("], "); + sb.append(query); + return sb.toString(); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java index e88e6d6b8f40e..ff088bde50727 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.InstantiatingObjectParser; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -27,43 +28,61 @@ import java.util.List; import java.util.Objects; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + public class EqlSearchResponse extends ActionResponse implements ToXContentObject { private final Hits hits; private final long tookInMillis; private final boolean isTimeout; + private final String asyncExecutionId; + private final boolean isRunning; + private final boolean isPartial; + private static final class Fields { static final String TOOK = "took"; static final String TIMED_OUT = "timed_out"; static final String HITS = "hits"; + static final String ID = "id"; + static final String IS_RUNNING = "is_running"; + static final String IS_PARTIAL = "is_partial"; } private static final ParseField TOOK = new ParseField(Fields.TOOK); private static final ParseField TIMED_OUT = new ParseField(Fields.TIMED_OUT); private static final ParseField HITS = new ParseField(Fields.HITS); + private static final ParseField ID = new ParseField(Fields.ID); + private static final ParseField IS_RUNNING = new ParseField(Fields.IS_RUNNING); + private static final ParseField IS_PARTIAL = new ParseField(Fields.IS_PARTIAL); - private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("eql/search_response", true, - args -> { - int i = 0; - Hits hits = (Hits) args[i++]; - Long took = (Long) args[i++]; - Boolean timeout = (Boolean) args[i]; - return new EqlSearchResponse(hits, took, timeout); - }); - + private static final InstantiatingObjectParser PARSER; static { - PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> Hits.fromXContent(p), HITS); - PARSER.declareLong(ConstructingObjectParser.constructorArg(), TOOK); - PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), TIMED_OUT); + InstantiatingObjectParser.Builder parser = + InstantiatingObjectParser.builder("eql/search_response", true, EqlSearchResponse.class); + parser.declareObject(constructorArg(), (p, c) -> Hits.fromXContent(p), HITS); + parser.declareLong(constructorArg(), TOOK); + parser.declareBoolean(constructorArg(), TIMED_OUT); + parser.declareString(optionalConstructorArg(), ID); + parser.declareBoolean(constructorArg(), IS_RUNNING); + parser.declareBoolean(constructorArg(), IS_PARTIAL); + PARSER = parser.build(); } public EqlSearchResponse(Hits hits, long tookInMillis, boolean isTimeout) { + this(hits, tookInMillis, isTimeout, null, false, false); + } + + public EqlSearchResponse(Hits hits, long tookInMillis, boolean isTimeout, String asyncExecutionId, + boolean isRunning, boolean isPartial) { super(); this.hits = hits == null ? Hits.EMPTY : hits; this.tookInMillis = tookInMillis; this.isTimeout = isTimeout; + this.asyncExecutionId = asyncExecutionId; + this.isRunning = isRunning; + this.isPartial = isPartial; } public EqlSearchResponse(StreamInput in) throws IOException { @@ -71,6 +90,9 @@ public EqlSearchResponse(StreamInput in) throws IOException { tookInMillis = in.readVLong(); isTimeout = in.readBoolean(); hits = new Hits(in); + asyncExecutionId = in.readOptionalString(); + isPartial = in.readBoolean(); + isRunning = in.readBoolean(); } public static EqlSearchResponse fromXContent(XContentParser parser) { @@ -82,6 +104,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(tookInMillis); out.writeBoolean(isTimeout); hits.writeTo(out); + out.writeOptionalString(asyncExecutionId); + out.writeBoolean(isPartial); + out.writeBoolean(isRunning); } @Override @@ -92,6 +117,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } private XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { + if (asyncExecutionId != null) { + builder.field(ID.getPreferredName(), asyncExecutionId); + } + builder.field(IS_PARTIAL.getPreferredName(), isPartial); + builder.field(IS_RUNNING.getPreferredName(), isRunning); builder.field(TOOK.getPreferredName(), tookInMillis); builder.field(TIMED_OUT.getPreferredName(), isTimeout); hits.toXContent(builder, params); @@ -110,6 +140,18 @@ public Hits hits() { return hits; } + public String id() { + return asyncExecutionId; + } + + public boolean isRunning() { + return isRunning; + } + + public boolean isPartial() { + return isPartial; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -121,12 +163,13 @@ public boolean equals(Object o) { EqlSearchResponse that = (EqlSearchResponse) o; return Objects.equals(hits, that.hits) && Objects.equals(tookInMillis, that.tookInMillis) - && Objects.equals(isTimeout, that.isTimeout); + && Objects.equals(isTimeout, that.isTimeout) + && Objects.equals(asyncExecutionId, that.asyncExecutionId); } @Override public int hashCode() { - return Objects.hash(hits, tookInMillis, isTimeout); + return Objects.hash(hits, tookInMillis, isTimeout, asyncExecutionId); } @Override @@ -253,9 +296,9 @@ private static final class Fields { }); static { - PARSER.declareInt(ConstructingObjectParser.constructorArg(), COUNT); - PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), KEYS); - PARSER.declareFloat(ConstructingObjectParser.constructorArg(), PERCENT); + PARSER.declareInt(constructorArg(), COUNT); + PARSER.declareStringArray(constructorArg(), KEYS); + PARSER.declareFloat(constructorArg(), PERCENT); } public Count(int count, List keys, float percent) { diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchTask.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchTask.java index e0c813718cd13..ad97e8a6cebca 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchTask.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchTask.java @@ -6,28 +6,26 @@ package org.elasticsearch.xpack.eql.action; -import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.xpack.core.async.AsyncExecutionId; +import org.elasticsearch.xpack.eql.async.StoredAsyncTask; import java.util.Map; -import java.util.function.Supplier; +import java.util.concurrent.atomic.AtomicReference; -public class EqlSearchTask extends CancellableTask { - private final Supplier descriptionSupplier; +public class EqlSearchTask extends StoredAsyncTask { + public volatile AtomicReference finalResponse = new AtomicReference<>(); - public EqlSearchTask(long id, String type, String action, Supplier descriptionSupplier, TaskId parentTaskId, - Map headers) { - super(id, type, action, null, parentTaskId, headers); - this.descriptionSupplier = descriptionSupplier; + public EqlSearchTask(long id, String type, String action, String description, TaskId parentTaskId, Map headers, + Map originHeaders, AsyncExecutionId asyncExecutionId, TimeValue keepAlive) { + super(id, type, action, description, parentTaskId, headers, originHeaders, asyncExecutionId, keepAlive); } @Override - public boolean shouldCancelChildrenOnCancellation() { - return true; - } - - @Override - public String getDescription() { - return descriptionSupplier.get(); + public EqlSearchResponse getCurrentResult() { + EqlSearchResponse response = finalResponse.get(); + return response != null ? response : new EqlSearchResponse(EqlSearchResponse.Hits.EMPTY, + System.currentTimeMillis() - getStartTime(), false, getExecutionId().getEncoded(), true, true); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/async/AsyncTaskManagementService.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/async/AsyncTaskManagementService.java new file mode 100644 index 0000000000000..51d74475d309d --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/async/AsyncTaskManagementService.java @@ -0,0 +1,265 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.async; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ListenerTimeouts; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.engine.DocumentMissingException; +import org.elasticsearch.index.engine.VersionConflictEngineException; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskAwareRequest; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.threadpool.Scheduler; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.async.AsyncExecutionId; +import org.elasticsearch.xpack.core.async.AsyncTask; +import org.elasticsearch.xpack.core.async.AsyncTaskIndexService; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Service for managing EQL requests + */ +public class AsyncTaskManagementService> { + + private static final Logger logger = LogManager.getLogger(AsyncTaskManagementService.class); + + private final TaskManager taskManager; + private final String action; + private final AsyncTaskIndexService> asyncTaskIndexService; + private final AsyncOperation operation; + private final ThreadPool threadPool; + private final ClusterService clusterService; + private final Class taskClass; + + public interface AsyncOperation { + + T createTask(Request request, long id, String type, String action, TaskId parentTaskId, Map headers, + Map originHeaders, AsyncExecutionId asyncExecutionId); + + void execute(Request request, T task, ActionListener listener); + + Response initialResponse(T task); + + Response readResponse(StreamInput inputStream) throws IOException; + } + + /** + * Wrapper for EqlSearchRequest that creates an async version of EqlSearchTask + */ + private class AsyncRequestWrapper implements TaskAwareRequest { + private final Request request; + private final String doc; + private final String node; + + AsyncRequestWrapper(Request request, String node) { + this.request = request; + this.doc = UUIDs.randomBase64UUID(); + this.node = node; + } + + @Override + public void setParentTask(TaskId taskId) { + request.setParentTask(taskId); + } + + @Override + public TaskId getParentTask() { + return request.getParentTask(); + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return operation.createTask(request, id, type, action, parentTaskId, headers, threadPool.getThreadContext().getHeaders(), + new AsyncExecutionId(doc, new TaskId(node, id))); + } + + @Override + public String getDescription() { + return request.getDescription(); + } + } + + public AsyncTaskManagementService(String index, Client client, String origin, NamedWriteableRegistry registry, TaskManager taskManager, + String action, AsyncOperation operation, Class taskClass, + ClusterService clusterService, + ThreadPool threadPool) { + this.taskManager = taskManager; + this.action = action; + this.operation = operation; + this.taskClass = taskClass; + this.asyncTaskIndexService = new AsyncTaskIndexService<>(index, clusterService, threadPool.getThreadContext(), client, + origin, i -> new StoredAsyncResponse<>(operation::readResponse, i), registry); + this.clusterService = clusterService; + this.threadPool = threadPool; + } + + public void asyncExecute(Request request, TimeValue waitForCompletionTimeout, TimeValue keepAlive, boolean keepOnCompletion, + ActionListener listener) { + String nodeId = clusterService.localNode().getId(); + @SuppressWarnings("unchecked") + T searchTask = (T) taskManager.register("transport", action + "[a]", new AsyncRequestWrapper(request, nodeId)); + boolean operationStarted = false; + try { + operation.execute(request, searchTask, + wrapStoringListener(searchTask, waitForCompletionTimeout, keepAlive, keepOnCompletion, listener)); + operationStarted = true; + } finally { + // If we didn't start operation for any reason, we need to clean up the task that we have created + if (operationStarted == false) { + taskManager.unregister(searchTask); + } + } + } + + private ActionListener wrapStoringListener(T searchTask, + TimeValue waitForCompletionTimeout, + TimeValue keepAlive, + boolean keepOnCompletion, + ActionListener listener) { + AtomicReference> exclusiveListener = new AtomicReference<>(listener); + // This is will performed in case of timeout + Scheduler.ScheduledCancellable timeoutHandler = threadPool.schedule(() -> { + ActionListener acquiredListener = exclusiveListener.getAndSet(null); + if (acquiredListener != null) { + acquiredListener.onResponse(operation.initialResponse(searchTask)); + } + }, waitForCompletionTimeout, ThreadPool.Names.SEARCH); + // This will be performed at the end of normal execution + return ActionListener.wrap(response -> { + ActionListener acquiredListener = exclusiveListener.getAndSet(null); + if (acquiredListener != null) { + // We finished before timeout + timeoutHandler.cancel(); + if (keepOnCompletion) { + storeResults(searchTask, + new StoredAsyncResponse<>(response, threadPool.absoluteTimeInMillis() + keepAlive.getMillis()), + ActionListener.wrap(() -> acquiredListener.onResponse(response))); + } else { + taskManager.unregister(searchTask); + searchTask.onResponse(response); + acquiredListener.onResponse(response); + } + } else { + // We finished after timeout - saving results + storeResults(searchTask, new StoredAsyncResponse<>(response, threadPool.absoluteTimeInMillis() + keepAlive.getMillis())); + } + }, e -> { + ActionListener acquiredListener = exclusiveListener.getAndSet(null); + if (acquiredListener != null) { + // We finished before timeout + timeoutHandler.cancel(); + if (keepOnCompletion) { + storeResults(searchTask, + new StoredAsyncResponse<>(e, threadPool.absoluteTimeInMillis() + keepAlive.getMillis()), + ActionListener.wrap(() -> acquiredListener.onFailure(e))); + } else { + taskManager.unregister(searchTask); + searchTask.onFailure(e); + acquiredListener.onFailure(e); + } + } else { + // We finished after timeout - saving exception + storeResults(searchTask, new StoredAsyncResponse<>(e, threadPool.absoluteTimeInMillis() + keepAlive.getMillis())); + } + }); + } + + private void storeResults(T searchTask, StoredAsyncResponse storedResponse) { + storeResults(searchTask, storedResponse, null); + } + + private void storeResults(T searchTask, StoredAsyncResponse storedResponse, ActionListener finalListener) { + try { + asyncTaskIndexService.createResponse(searchTask.getExecutionId().getDocId(), + threadPool.getThreadContext().getHeaders(), storedResponse, ActionListener.wrap( + // We should only unregister after the result is saved + resp -> { + logger.trace(() -> new ParameterizedMessage("stored eql search results for [{}]", + searchTask.getExecutionId().getEncoded())); + taskManager.unregister(searchTask); + if (storedResponse.getException() != null) { + searchTask.onFailure(storedResponse.getException()); + } else { + searchTask.onResponse(storedResponse.getResponse()); + } + if (finalListener != null) { + finalListener.onResponse(null); + } + }, + exc -> { + taskManager.unregister(searchTask); + searchTask.onFailure(exc); + Throwable cause = ExceptionsHelper.unwrapCause(exc); + if (cause instanceof DocumentMissingException == false && + cause instanceof VersionConflictEngineException == false) { + logger.error(() -> new ParameterizedMessage("failed to store eql search results for [{}]", + searchTask.getExecutionId().getEncoded()), exc); + } + if (finalListener != null) { + finalListener.onFailure(exc); + } + })); + } catch (Exception exc) { + taskManager.unregister(searchTask); + searchTask.onFailure(exc); + logger.error(() -> new ParameterizedMessage("failed to store eql search results for [{}]", + searchTask.getExecutionId().getEncoded()), exc); + } + } + + /** + * Adds a self-unregistering listener to a task. It works as a normal listener except it retrieves a partial response and unregister + * itself from the task if timeout occurs. + */ + public static > void addCompletionListener( + ThreadPool threadPool, + Task task, + ActionListener> listener, + TimeValue timeout) { + if (timeout.getMillis() <= 0) { + getCurrentResult(task, listener); + } else { + task.addCompletionListener(ListenerTimeouts.wrapWithTimeout(threadPool, timeout, ThreadPool.Names.SEARCH, ActionListener.wrap( + r -> listener.onResponse(new StoredAsyncResponse<>(r, task.getExpirationTimeMillis())), + e -> listener.onResponse(new StoredAsyncResponse<>(e, task.getExpirationTimeMillis())) + ), wrapper -> { + // Timeout was triggered + task.removeCompletionListener(wrapper); + getCurrentResult(task, listener); + })); + } + } + + private static > void getCurrentResult( + Task task, + ActionListener> listener + ) { + try { + listener.onResponse(new StoredAsyncResponse<>(task.getCurrentResult(), task.getExpirationTimeMillis())); + } catch (Exception ex) { + listener.onFailure(ex); + } + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/async/StoredAsyncResponse.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/async/StoredAsyncResponse.java new file mode 100644 index 0000000000000..d027d22c0c417 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/async/StoredAsyncResponse.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.async; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.async.AsyncResponse; + +import java.io.IOException; +import java.util.Objects; + +/** + * Internal class for temporary storage of eql search results + */ +public class StoredAsyncResponse extends ActionResponse + implements AsyncResponse>, ToXContentObject { + private final R response; + private final Exception exception; + private final long expirationTimeMillis; + + public StoredAsyncResponse(R response, long expirationTimeMillis) { + this(response, null, expirationTimeMillis); + } + + public StoredAsyncResponse(Exception exception, long expirationTimeMillis) { + this(null, exception, expirationTimeMillis); + } + + public StoredAsyncResponse(Writeable.Reader reader, StreamInput input) throws IOException { + expirationTimeMillis = input.readLong(); + this.response = input.readOptionalWriteable(reader); + this.exception = input.readException(); + } + + private StoredAsyncResponse(R response, Exception exception, long expirationTimeMillis) { + this.response = response; + this.exception = exception; + this.expirationTimeMillis = expirationTimeMillis; + } + + @Override + public long getExpirationTime() { + return expirationTimeMillis; + } + + @Override + public StoredAsyncResponse withExpirationTime(long expirationTimeMillis) { + return new StoredAsyncResponse<>(this.response, this.exception, expirationTimeMillis); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(expirationTimeMillis); + out.writeOptionalWriteable(response); + out.writeException(exception); + } + + public R getResponse() { + return response; + } + + public Exception getException() { + return exception; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + StoredAsyncResponse response1 = (StoredAsyncResponse) o; + if (exception != null && response1.exception != null) { + if (Objects.equals(exception.getClass(), response1.exception.getClass()) == false || + Objects.equals(exception.getMessage(), response1.exception.getMessage()) == false) { + return false; + } + } else { + if (Objects.equals(exception, response1.exception) == false) { + return false; + } + } + return expirationTimeMillis == response1.expirationTimeMillis && + Objects.equals(response, response1.response); + } + + @Override + public int hashCode() { + return Objects.hash(response, exception == null ? null : exception.getClass(), + exception == null ? null : exception.getMessage(), expirationTimeMillis); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return null; + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/async/StoredAsyncTask.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/async/StoredAsyncTask.java new file mode 100644 index 0000000000000..f7d4e11259b09 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/async/StoredAsyncTask.java @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.async; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.xpack.core.async.AsyncExecutionId; +import org.elasticsearch.xpack.core.async.AsyncTask; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + + +public abstract class StoredAsyncTask extends CancellableTask implements AsyncTask { + + private final AsyncExecutionId asyncExecutionId; + private final Map originHeaders; + private volatile long expirationTimeMillis; + private final List> completionListeners; + + public StoredAsyncTask(long id, String type, String action, String description, TaskId parentTaskId, + Map headers, Map originHeaders, AsyncExecutionId asyncExecutionId, + TimeValue keepAlive) { + super(id, type, action, description, parentTaskId, headers); + this.asyncExecutionId = asyncExecutionId; + this.originHeaders = originHeaders; + this.expirationTimeMillis = getStartTime() + keepAlive.getMillis(); + this.completionListeners = new ArrayList<>(); + } + + @Override + public boolean shouldCancelChildrenOnCancellation() { + return true; + } + + @Override + public Map getOriginHeaders() { + return originHeaders; + } + + @Override + public AsyncExecutionId getExecutionId() { + return asyncExecutionId; + } + + /** + * Update the expiration time of the (partial) response. + */ + @Override + public void setExpirationTime(long expirationTimeMillis) { + this.expirationTimeMillis = expirationTimeMillis; + } + + public long getExpirationTimeMillis() { + return expirationTimeMillis; + } + + public synchronized void addCompletionListener(ActionListener listener) { + completionListeners.add(listener); + } + + public synchronized void removeCompletionListener(ActionListener listener) { + completionListeners.remove(listener); + } + + /** + * This method is called when the task is finished successfully before unregistering the task and storing the results + */ + protected synchronized void onResponse(Response response) { + for (ActionListener listener : completionListeners) { + listener.onResponse(response); + } + } + + /** + * This method is called when the task failed before unregistering the task and storing the results + */ + protected synchronized void onFailure(Exception e) { + for (ActionListener listener : completionListeners) { + listener.onFailure(e); + } + } + + /** + * Return currently available partial or the final results + */ + protected abstract Response getCurrentResult(); + + @Override + public void cancelTask(TaskManager taskManager, Runnable runnable, String reason) { + taskManager.cancelTaskAndDescendants(this, reason, true, ActionListener.wrap(runnable)); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlAsyncGetResultAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlAsyncGetResultAction.java new file mode 100644 index 0000000000000..02f5b1b3f7bad --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlAsyncGetResultAction.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.plugin; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.xpack.core.eql.EqlAsyncActionNames; +import org.elasticsearch.xpack.eql.action.EqlSearchResponse; + +public class EqlAsyncGetResultAction extends ActionType { + public static final EqlAsyncGetResultAction INSTANCE = new EqlAsyncGetResultAction(); + + private EqlAsyncGetResultAction() { + super(EqlAsyncActionNames.EQL_ASYNC_GET_RESULT_ACTION_NAME, EqlSearchResponse::new); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java index 495829d15a0a8..5802875a2dec7 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java @@ -118,7 +118,8 @@ public List> getSettings() { if (enabled) { return Arrays.asList( new ActionHandler<>(EqlSearchAction.INSTANCE, TransportEqlSearchAction.class), - new ActionHandler<>(EqlStatsAction.INSTANCE, TransportEqlStatsAction.class) + new ActionHandler<>(EqlStatsAction.INSTANCE, TransportEqlStatsAction.class), + new ActionHandler<>(EqlAsyncGetResultAction.INSTANCE, TransportEqlAsyncGetResultAction.class) ); } return Collections.emptyList(); @@ -143,7 +144,12 @@ public List getRestHandlers(Settings settings, Supplier nodesInCluster) { if (enabled) { - return Arrays.asList(new RestEqlSearchAction(), new RestEqlStatsAction()); + return Arrays.asList( + new RestEqlSearchAction(), + new RestEqlStatsAction(), + new RestEqlGetAsyncResultAction(), + new RestEqlDeleteAsyncResultAction() + ); } return Collections.emptyList(); } @@ -152,4 +158,4 @@ public List getRestHandlers(Settings settings, protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlDeleteAsyncResultAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlDeleteAsyncResultAction.java new file mode 100644 index 0000000000000..128c310e7f572 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlDeleteAsyncResultAction.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.plugin; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; +import org.elasticsearch.xpack.core.async.DeleteAsyncResultRequest; + +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.DELETE; + +public class RestEqlDeleteAsyncResultAction extends BaseRestHandler { + @Override + public List routes() { + return Collections.singletonList(new Route(DELETE, "/_eql/search/{id}")); + } + + @Override + public String getName() { + return "eql_delete_async_result"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + DeleteAsyncResultRequest delete = new DeleteAsyncResultRequest(request.param("id")); + return channel -> client.execute(DeleteAsyncResultAction.INSTANCE, delete, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlGetAsyncResultAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlGetAsyncResultAction.java new file mode 100644 index 0000000000000..a9875ac0dd91e --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlGetAsyncResultAction.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.plugin; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.async.GetAsyncResultRequest; + +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +public class RestEqlGetAsyncResultAction extends BaseRestHandler { + @Override + public List routes() { + return Collections.singletonList(new Route(GET, "/_eql/search/{id}")); + } + + @Override + public String getName() { + return "eql_get_async_result"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + GetAsyncResultRequest get = new GetAsyncResultRequest(request.param("id")); + if (request.hasParam("wait_for_completion_timeout")) { + get.setWaitForCompletionTimeout(request.paramAsTime("wait_for_completion_timeout", get.getWaitForCompletionTimeout())); + } + if (request.hasParam("keep_alive")) { + get.setKeepAlive(request.paramAsTime("keep_alive", get.getKeepAlive())); + } + return channel -> client.execute(EqlAsyncGetResultAction.INSTANCE, get, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java index 9f615d34f19b0..548b869186e1b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestResponseListener; import org.elasticsearch.xpack.eql.action.EqlSearchAction; import org.elasticsearch.xpack.eql.action.EqlSearchRequest; @@ -48,16 +49,27 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli eqlRequest = EqlSearchRequest.fromXContent(parser); eqlRequest.indices(Strings.splitStringByCommaToArray(request.param("index"))); eqlRequest.indicesOptions(IndicesOptions.fromRequest(request, eqlRequest.indicesOptions())); + if (request.hasParam("wait_for_completion_timeout")) { + eqlRequest.waitForCompletionTimeout( + request.paramAsTime("wait_for_completion_timeout", eqlRequest.waitForCompletionTimeout())); + } + if (request.hasParam("keep_alive")) { + eqlRequest.keepAlive(request.paramAsTime("keep_alive", eqlRequest.keepAlive())); + } + eqlRequest.keepOnCompletion(request.paramAsBoolean("keep_on_completion", eqlRequest.keepOnCompletion())); } - return channel -> client.execute(EqlSearchAction.INSTANCE, eqlRequest, new RestResponseListener(channel) { - @Override - public RestResponse buildResponse(EqlSearchResponse response) throws Exception { - XContentBuilder builder = channel.newBuilder(request.getXContentType(), XContentType.JSON, true); - response.toXContent(builder, request); - return new BytesRestResponse(RestStatus.OK, builder); - } - }); + return channel -> { + RestCancellableNodeClient cancellableClient = new RestCancellableNodeClient(client, request.getHttpChannel()); + cancellableClient.execute(EqlSearchAction.INSTANCE, eqlRequest, new RestResponseListener(channel) { + @Override + public RestResponse buildResponse(EqlSearchResponse response) throws Exception { + XContentBuilder builder = channel.newBuilder(request.getXContentType(), XContentType.JSON, true); + response.toXContent(builder, request); + return new BytesRestResponse(RestStatus.OK, builder); + } + }); + }; } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlAsyncGetResultAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlAsyncGetResultAction.java new file mode 100644 index 0000000000000..f484d8e036fd0 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlAsyncGetResultAction.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.plugin; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionListenerResponseHandler; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.core.async.AsyncResultsService; +import org.elasticsearch.xpack.core.async.AsyncTaskIndexService; +import org.elasticsearch.xpack.core.async.GetAsyncResultRequest; +import org.elasticsearch.xpack.core.eql.EqlAsyncActionNames; +import org.elasticsearch.xpack.eql.action.EqlSearchResponse; +import org.elasticsearch.xpack.eql.action.EqlSearchTask; +import org.elasticsearch.xpack.eql.async.AsyncTaskManagementService; +import org.elasticsearch.xpack.eql.async.StoredAsyncResponse; + +import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; + +public class TransportEqlAsyncGetResultAction extends HandledTransportAction { + private final AsyncResultsService> resultsService; + private final TransportService transportService; + + @Inject + public TransportEqlAsyncGetResultAction(TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + NamedWriteableRegistry registry, + Client client, + ThreadPool threadPool) { + super(EqlAsyncActionNames.EQL_ASYNC_GET_RESULT_ACTION_NAME, transportService, actionFilters, GetAsyncResultRequest::new); + this.transportService = transportService; + this.resultsService = createResultsService(transportService, clusterService, registry, client, threadPool); + } + + static AsyncResultsService> createResultsService( + TransportService transportService, + ClusterService clusterService, + NamedWriteableRegistry registry, + Client client, + ThreadPool threadPool) { + Writeable.Reader> reader = in -> new StoredAsyncResponse<>(EqlSearchResponse::new, in); + AsyncTaskIndexService> store = new AsyncTaskIndexService<>(XPackPlugin.ASYNC_RESULTS_INDEX, + clusterService, threadPool.getThreadContext(), client, ASYNC_SEARCH_ORIGIN, reader, registry); + return new AsyncResultsService<>(store, true, EqlSearchTask.class, + (task, listener, timeout) -> AsyncTaskManagementService.addCompletionListener(threadPool, task, listener, timeout), + transportService.getTaskManager(), clusterService); + } + + @Override + protected void doExecute(Task task, GetAsyncResultRequest request, ActionListener listener) { + DiscoveryNode node = resultsService.getNode(request.getId()); + if (node == null || resultsService.isLocalNode(node)) { + resultsService.retrieveResult(request, ActionListener.wrap( + r -> { + if (r.getException() != null) { + listener.onFailure(r.getException()); + } else { + listener.onResponse(r.getResponse()); + } + }, + listener::onFailure + )); + } else { + TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); + transportService.sendRequest(node, EqlAsyncActionNames.EQL_ASYNC_GET_RESULT_ACTION_NAME, request, builder.build(), + new ActionListenerResponseHandler<>(listener, EqlSearchResponse::new, ThreadPool.Names.SAME)); + } + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java index 43754c156d95f..51b41f74808b2 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java @@ -8,8 +8,11 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.unit.TimeValue; @@ -18,7 +21,10 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.async.AsyncExecutionId; +import org.elasticsearch.xpack.eql.async.AsyncTaskManagementService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.eql.action.EqlSearchAction; import org.elasticsearch.xpack.eql.action.EqlSearchRequest; @@ -29,32 +35,73 @@ import org.elasticsearch.xpack.eql.session.EqlConfiguration; import org.elasticsearch.xpack.eql.session.Results; +import java.io.IOException; import java.time.ZoneId; +import java.util.Map; import static org.elasticsearch.action.ActionListener.wrap; +import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; + +public class TransportEqlSearchAction extends HandledTransportAction + implements AsyncTaskManagementService.AsyncOperation { -public class TransportEqlSearchAction extends HandledTransportAction { private final SecurityContext securityContext; private final ClusterService clusterService; private final PlanExecutor planExecutor; + private final ThreadPool threadPool; + private final AsyncTaskManagementService asyncTaskManagementService; @Inject public TransportEqlSearchAction(Settings settings, ClusterService clusterService, TransportService transportService, - ThreadPool threadPool, ActionFilters actionFilters, PlanExecutor planExecutor) { + ThreadPool threadPool, ActionFilters actionFilters, PlanExecutor planExecutor, + NamedWriteableRegistry registry, Client client) { super(EqlSearchAction.NAME, transportService, actionFilters, EqlSearchRequest::new); this.securityContext = XPackSettings.SECURITY_ENABLED.get(settings) ? new SecurityContext(settings, threadPool.getThreadContext()) : null; this.clusterService = clusterService; this.planExecutor = planExecutor; + this.threadPool = threadPool; + + this.asyncTaskManagementService = new AsyncTaskManagementService<>(XPackPlugin.ASYNC_RESULTS_INDEX, client, ASYNC_SEARCH_ORIGIN, + registry, taskManager, EqlSearchAction.INSTANCE.name(), this, EqlSearchTask.class, clusterService, threadPool); } @Override - protected void doExecute(Task task, EqlSearchRequest request, ActionListener listener) { - operation(planExecutor, (EqlSearchTask) task, request, username(securityContext), clusterName(clusterService), + public EqlSearchTask createTask(EqlSearchRequest request, long id, String type, String action, TaskId parentTaskId, + Map headers, Map originHeaders, AsyncExecutionId asyncExecutionId) { + return new EqlSearchTask(id, type, action, request.getDescription(), parentTaskId, headers, originHeaders, asyncExecutionId, + request.keepAlive()); + } + + @Override + public void execute(EqlSearchRequest request, EqlSearchTask task, ActionListener listener) { + operation(planExecutor, task, request, username(securityContext), clusterName(clusterService), clusterService.localNode().getId(), listener); } + @Override + public EqlSearchResponse initialResponse(EqlSearchTask task) { + return new EqlSearchResponse(EqlSearchResponse.Hits.EMPTY, + threadPool.relativeTimeInMillis() - task.getStartTime(), false, task.getExecutionId().getEncoded(), true, true); + } + + @Override + public EqlSearchResponse readResponse(StreamInput inputStream) throws IOException { + return new EqlSearchResponse(inputStream); + } + + @Override + protected void doExecute(Task task, EqlSearchRequest request, ActionListener listener) { + if (request.waitForCompletionTimeout() != null && request.waitForCompletionTimeout().getMillis() >= 0) { + asyncTaskManagementService.asyncExecute(request, request.waitForCompletionTimeout(), request.keepAlive(), + request.keepOnCompletion(), listener); + } else { + operation(planExecutor, (EqlSearchTask) task, request, username(securityContext), clusterName(clusterService), + clusterService.localNode().getId(), listener); + } + } + public static void operation(PlanExecutor planExecutor, EqlSearchTask task, EqlSearchRequest request, String username, String clusterName, String nodeId, ActionListener listener) { // TODO: these should be sent by the client @@ -71,15 +118,19 @@ public static void operation(PlanExecutor planExecutor, EqlSearchTask task, EqlS .implicitJoinKey(request.implicitJoinKeyField()); EqlConfiguration cfg = new EqlConfiguration(request.indices(), zoneId, username, clusterName, filter, timeout, request.fetchSize(), - includeFrozen, request.isCaseSensitive(), clientId, new TaskId(nodeId, task.getId()), task::isCancelled); - planExecutor.eql(cfg, request.query(), params, wrap(r -> listener.onResponse(createResponse(r)), listener::onFailure)); + includeFrozen, request.isCaseSensitive(), clientId, new TaskId(nodeId, task.getId()), task); + planExecutor.eql(cfg, request.query(), params, wrap(r -> listener.onResponse(createResponse(r, task.getExecutionId())), + listener::onFailure)); } - static EqlSearchResponse createResponse(Results results) { + static EqlSearchResponse createResponse(Results results, AsyncExecutionId id) { EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(results.searchHits(), results.sequences(), results.counts(), results - .totalHits()); - - return new EqlSearchResponse(hits, results.tookTime().getMillis(), results.timedOut()); + .totalHits()); + if (id != null) { + return new EqlSearchResponse(hits, results.tookTime().getMillis(), results.timedOut(), id.getEncoded(), false, false); + } else { + return new EqlSearchResponse(hits, results.tookTime().getMillis(), results.timedOut()); + } } static String username(SecurityContext securityContext) { diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java index e606cb202f941..53fde26794b58 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java @@ -11,9 +11,9 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.xpack.eql.action.EqlSearchTask; import java.time.ZoneId; -import java.util.function.Supplier; public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configuration { @@ -22,17 +22,15 @@ public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu private final int size; private final String clientId; private final boolean includeFrozenIndices; - private final Supplier isCancelled; private final TaskId taskId; + private final EqlSearchTask task; private final boolean isCaseSensitive; @Nullable private final QueryBuilder filter; public EqlConfiguration(String[] indices, ZoneId zi, String username, String clusterName, QueryBuilder filter, TimeValue requestTimeout, - int size, boolean includeFrozen, boolean isCaseSensitive, String clientId, TaskId taskId, - Supplier isCancelled) { - + int size, boolean includeFrozen, boolean isCaseSensitive, String clientId, TaskId taskId, EqlSearchTask task) { super(zi, username, clusterName); this.indices = indices; @@ -43,7 +41,7 @@ public EqlConfiguration(String[] indices, ZoneId zi, String username, String clu this.includeFrozenIndices = includeFrozen; this.isCaseSensitive = isCaseSensitive; this.taskId = taskId; - this.isCancelled = isCancelled; + this.task = task; } public String[] indices() { @@ -79,7 +77,7 @@ public boolean isCaseSensitive() { } public boolean isCancelled() { - return isCancelled.get(); + return task.isCancelled(); } public TaskId getTaskId() { diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java index b2cacf9775347..d9e67dbfc133f 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java @@ -8,6 +8,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.xpack.core.async.AsyncExecutionId; import org.elasticsearch.xpack.eql.action.EqlSearchAction; import org.elasticsearch.xpack.eql.action.EqlSearchTask; import org.elasticsearch.xpack.eql.session.EqlConfiguration; @@ -28,7 +29,7 @@ private EqlTestUtils() { public static final EqlConfiguration TEST_CFG = new EqlConfiguration(new String[]{"none"}, org.elasticsearch.xpack.ql.util.DateUtils.UTC, "nobody", "cluster", null, TimeValue.timeValueSeconds(30), -1, false, false, "", - new TaskId(randomAlphaOfLength(10), randomNonNegativeLong()), () -> false); + new TaskId(randomAlphaOfLength(10), randomNonNegativeLong()), randomTask()); public static EqlConfiguration randomConfiguration() { return new EqlConfiguration(new String[]{randomAlphaOfLength(16)}, @@ -42,7 +43,7 @@ public static EqlConfiguration randomConfiguration() { randomBoolean(), randomAlphaOfLength(16), new TaskId(randomAlphaOfLength(10), randomNonNegativeLong()), - () -> false); + randomTask()); } public static EqlConfiguration randomConfigurationWithCaseSensitive(boolean isCaseSensitive) { @@ -57,10 +58,11 @@ public static EqlConfiguration randomConfigurationWithCaseSensitive(boolean isCa isCaseSensitive, randomAlphaOfLength(16), new TaskId(randomAlphaOfLength(10), randomNonNegativeLong()), - () -> false); + randomTask()); } public static EqlSearchTask randomTask() { - return new EqlSearchTask(randomLong(), "transport", EqlSearchAction.NAME, () -> "", null, Collections.emptyMap()); + return new EqlSearchTask(randomLong(), "transport", EqlSearchAction.NAME, "", null, Collections.emptyMap(), Collections.emptyMap(), + new AsyncExecutionId("", new TaskId(randomAlphaOfLength(10), 1)), TimeValue.timeValueDays(5)); } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java index 36f5ff0c98619..b1a52331c2a2e 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java @@ -35,11 +35,7 @@ static List randomEvents() { @Override protected EqlSearchResponse createTestInstance() { - TotalHits totalHits = null; - if (randomBoolean()) { - totalHits = new TotalHits(randomIntBetween(100, 1000), TotalHits.Relation.EQUAL_TO); - } - return createRandomInstance(totalHits); + return randomEqlSearchResponse(); } @Override @@ -47,12 +43,25 @@ protected Writeable.Reader instanceReader() { return EqlSearchResponse::new; } + public static EqlSearchResponse randomEqlSearchResponse() { + TotalHits totalHits = null; + if (randomBoolean()) { + totalHits = new TotalHits(randomIntBetween(100, 1000), TotalHits.Relation.EQUAL_TO); + } + return createRandomInstance(totalHits); + } + public static EqlSearchResponse createRandomEventsResponse(TotalHits totalHits) { EqlSearchResponse.Hits hits = null; if (randomBoolean()) { hits = new EqlSearchResponse.Hits(randomEvents(), null, null, totalHits); } - return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + if (randomBoolean()) { + return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + } else { + return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean(), + randomAlphaOfLength(10), randomBoolean(), randomBoolean()); + } } public static EqlSearchResponse createRandomSequencesResponse(TotalHits totalHits) { @@ -72,7 +81,12 @@ public static EqlSearchResponse createRandomSequencesResponse(TotalHits totalHit if (randomBoolean()) { hits = new EqlSearchResponse.Hits(null, seq, null, totalHits); } - return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + if (randomBoolean()) { + return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + } else { + return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean(), + randomAlphaOfLength(10), randomBoolean(), randomBoolean()); + } } public static EqlSearchResponse createRandomCountResponse(TotalHits totalHits) { @@ -92,7 +106,12 @@ public static EqlSearchResponse createRandomCountResponse(TotalHits totalHits) { if (randomBoolean()) { hits = new EqlSearchResponse.Hits(null, null, cn, totalHits); } - return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + if (randomBoolean()) { + return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + } else { + return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean(), + randomAlphaOfLength(10), randomBoolean(), randomBoolean()); + } } public static EqlSearchResponse createRandomInstance(TotalHits totalHits) { diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/async/AsyncTaskManagementServiceTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/async/AsyncTaskManagementServiceTests.java new file mode 100644 index 0000000000000..bda31d14d71c4 --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/async/AsyncTaskManagementServiceTests.java @@ -0,0 +1,287 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.async; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.async.AsyncExecutionId; +import org.elasticsearch.xpack.core.async.AsyncResultsService; +import org.elasticsearch.xpack.core.async.AsyncTaskIndexService; +import org.elasticsearch.xpack.core.async.GetAsyncResultRequest; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.xpack.eql.async.AsyncTaskManagementService.addCompletionListener; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class AsyncTaskManagementServiceTests extends ESSingleNodeTestCase { + private ClusterService clusterService; + private TransportService transportService; + private AsyncResultsService> results; + + private final ExecutorService executorService = Executors.newFixedThreadPool(1); + + public static class TestRequest extends ActionRequest { + private final String string; + + public TestRequest(String string) { + this.string = string; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + } + + public static class TestResponse extends ActionResponse { + private final String string; + private final String id; + + public TestResponse(String string, String id) { + this.string = string; + this.id = id; + } + + public TestResponse(StreamInput input) throws IOException { + this.string = input.readOptionalString(); + this.id = input.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(string); + out.writeOptionalString(id); + } + } + + public static class TestTask extends StoredAsyncTask { + public volatile AtomicReference finalResponse = new AtomicReference<>(); + + public TestTask(long id, String type, String action, String description, TaskId parentTaskId, Map headers, + Map originHeaders, AsyncExecutionId asyncExecutionId, TimeValue keepAlive) { + super(id, type, action, description, parentTaskId, headers, originHeaders, asyncExecutionId, keepAlive); + } + + @Override + public TestResponse getCurrentResult() { + TestResponse response = finalResponse.get(); + return response != null ? response : new TestResponse(null, getExecutionId().getEncoded()); + } + } + + public static class TestOperation implements AsyncTaskManagementService.AsyncOperation { + + @Override + public TestTask createTask(TestRequest request, long id, String type, String action, TaskId parentTaskId, + Map headers, Map originHeaders, AsyncExecutionId asyncExecutionId) { + return new TestTask(id, type, action, request.getDescription(), parentTaskId, headers, originHeaders, asyncExecutionId, + TimeValue.timeValueDays(5)); + } + + @Override + public void execute(TestRequest request, TestTask task, ActionListener listener) { + if (request.string.equals("die")) { + listener.onFailure(new IllegalArgumentException("test exception")); + } else { + listener.onResponse(new TestResponse("response for [" + request.string + "]", task.getExecutionId().getEncoded())); + } + } + + @Override + public TestResponse initialResponse(TestTask task) { + return new TestResponse(null, task.getExecutionId().getEncoded()); + } + + @Override + public TestResponse readResponse(StreamInput inputStream) throws IOException { + return new TestResponse(inputStream); + } + } + + public String index = "test-index"; + + @Before + public void setup() { + clusterService = getInstanceFromNode(ClusterService.class); + transportService = getInstanceFromNode(TransportService.class); + AsyncTaskIndexService> store = + new AsyncTaskIndexService<>(index, clusterService, transportService.getThreadPool().getThreadContext(), client(), "test", + in -> new StoredAsyncResponse<>(TestResponse::new, in), writableRegistry()); + results = new AsyncResultsService<>(store, true, TestTask.class, + (task, listener, timeout) -> addCompletionListener(transportService.getThreadPool(), task, listener, timeout), + transportService.getTaskManager(), clusterService); + } + + /** + * Shutdown the executor so we don't leak threads into other test runs. + */ + @After + public void shutdownExec() { + executorService.shutdown(); + } + + private AsyncTaskManagementService createManagementService( + AsyncTaskManagementService.AsyncOperation operation) { + return new AsyncTaskManagementService<>(index, client(), "test_origin", writableRegistry(), + transportService.getTaskManager(), "test_action", operation, TestTask.class, clusterService, transportService.getThreadPool()); + } + + public void testReturnBeforeTimeout() throws Exception { + AsyncTaskManagementService service = createManagementService(new TestOperation()); + boolean success = randomBoolean(); + boolean keepOnCompletion = randomBoolean(); + CountDownLatch latch = new CountDownLatch(1); + TestRequest request = new TestRequest(success ? randomAlphaOfLength(10) : "die"); + service.asyncExecute(request, TimeValue.timeValueMinutes(1), TimeValue.timeValueMinutes(10), keepOnCompletion, + ActionListener.wrap(r -> { + assertThat(success, equalTo(true)); + assertThat(r.string, equalTo("response for [" + request.string + "]")); + assertThat(r.id, notNullValue()); + latch.countDown(); + }, e -> { + assertThat(success, equalTo(false)); + assertThat(e.getMessage(), equalTo("test exception")); + latch.countDown(); + })); + assertThat(latch.await(10, TimeUnit.SECONDS), equalTo(true)); + } + + public void testReturnAfterTimeout() throws Exception { + CountDownLatch executionLatch = new CountDownLatch(1); + AsyncTaskManagementService service = createManagementService(new TestOperation() { + @Override + public void execute(TestRequest request, TestTask task, ActionListener listener) { + executorService.submit(() -> { + try { + assertThat(executionLatch.await(10, TimeUnit.SECONDS), equalTo(true)); + } catch (InterruptedException ex) { + fail("Shouldn't be here"); + } + super.execute(request, task, listener); + }); + } + }); + boolean success = randomBoolean(); + boolean keepOnCompletion = randomBoolean(); + boolean timeoutOnFirstAttempt = randomBoolean(); + boolean waitForCompletion = randomBoolean(); + CountDownLatch latch = new CountDownLatch(1); + TestRequest request = new TestRequest(success ? randomAlphaOfLength(10) : "die"); + AtomicReference responseHolder = new AtomicReference<>(); + service.asyncExecute(request, TimeValue.timeValueMillis(1), TimeValue.timeValueMinutes(10), keepOnCompletion, + ActionListener.wrap(r -> { + assertThat(r.string, nullValue()); + assertThat(r.id, notNullValue()); + assertThat(responseHolder.getAndSet(r), nullValue()); + latch.countDown(); + }, e -> fail("Shouldn't be here"))); + assertThat(latch.await(20, TimeUnit.SECONDS), equalTo(true)); + + if (timeoutOnFirstAttempt) { + logger.trace("Getting an in-flight response"); + // try getting results, but fail with timeout because it is not ready yet + StoredAsyncResponse response = getResponse(responseHolder.get().id, TimeValue.timeValueMillis(2)); + assertThat(response.getException(), nullValue()); + assertThat(response.getResponse(), notNullValue()); + assertThat(response.getResponse().id, equalTo(responseHolder.get().id)); + assertThat(response.getResponse().string, nullValue()); + } + + if (waitForCompletion) { + // now we are waiting for the task to finish + logger.trace("Waiting for response to complete"); + AtomicReference> responseRef = new AtomicReference<>(); + CountDownLatch getResponseCountDown = getResponse(responseHolder.get().id, TimeValue.timeValueSeconds(5), + ActionListener.wrap(responseRef::set, e -> fail("Shouldn't be here"))); + + executionLatch.countDown(); + assertThat(getResponseCountDown.await(10, TimeUnit.SECONDS), equalTo(true)); + + StoredAsyncResponse response = responseRef.get(); + if (success) { + assertThat(response.getException(), nullValue()); + assertThat(response.getResponse(), notNullValue()); + assertThat(response.getResponse().id, equalTo(responseHolder.get().id)); + assertThat(response.getResponse().string, equalTo("response for [" + request.string + "]")); + } else { + assertThat(response.getException(), notNullValue()); + assertThat(response.getResponse(), nullValue()); + assertThat(response.getException().getMessage(), equalTo("test exception")); + } + } else { + executionLatch.countDown(); + } + + // finally wait until the task disappears and get the response from the index + logger.trace("Wait for task to disappear "); + assertBusy(() -> { + Task task = transportService.getTaskManager().getTask(AsyncExecutionId.decode(responseHolder.get().id).getTaskId().getId()); + assertThat(task, nullValue()); + }); + + logger.trace("Getting the the final response from the index"); + StoredAsyncResponse response = getResponse(responseHolder.get().id, TimeValue.ZERO); + if (success) { + assertThat(response.getException(), nullValue()); + assertThat(response.getResponse(), notNullValue()); + assertThat(response.getResponse().string, equalTo("response for [" + request.string + "]")); + } else { + assertThat(response.getException(), notNullValue()); + assertThat(response.getResponse(), nullValue()); + assertThat(response.getException().getMessage(), equalTo("test exception")); + } + } + + private StoredAsyncResponse getResponse(String id, TimeValue timeout) throws InterruptedException { + AtomicReference> response = new AtomicReference<>(); + assertThat( + getResponse(id, timeout, ActionListener.wrap(response::set, e -> fail("Shouldn't be here"))).await(10, TimeUnit.SECONDS), + equalTo(true) + ); + return response.get(); + } + + private CountDownLatch getResponse(String id, + TimeValue timeout, + ActionListener> listener) { + CountDownLatch responseLatch = new CountDownLatch(1); + GetAsyncResultRequest getResultsRequest = new GetAsyncResultRequest(id) + .setWaitForCompletionTimeout(timeout); + results.retrieveResult(getResultsRequest, ActionListener.wrap( + r -> { + listener.onResponse(r); + responseLatch.countDown(); + }, + e -> { + listener.onFailure(e); + responseLatch.countDown(); + } + )); + return responseLatch; + } + +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/async/StoredAsyncResponseTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/async/StoredAsyncResponseTests.java new file mode 100644 index 0000000000000..b57b797cf6150 --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/async/StoredAsyncResponseTests.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.async; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.Collections; +import java.util.Objects; + +public class StoredAsyncResponseTests extends AbstractWireSerializingTestCase> { + + public static class TestResponse implements Writeable { + private final String string; + + public TestResponse(String string) { + this.string = string; + } + + public TestResponse(StreamInput input) throws IOException { + this.string = input.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(string); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TestResponse that = (TestResponse) o; + return Objects.equals(string, that.string); + } + + @Override + public int hashCode() { + return Objects.hash(string); + } + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + return new NamedWriteableRegistry(searchModule.getNamedWriteables()); + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + return new NamedXContentRegistry(searchModule.getNamedXContents()); + } + + @Override + protected StoredAsyncResponse createTestInstance() { + if (randomBoolean()) { + return new StoredAsyncResponse<>(new IllegalArgumentException(randomAlphaOfLength(10)), randomNonNegativeLong()); + } else { + return new StoredAsyncResponse<>(new TestResponse(randomAlphaOfLength(10)), randomNonNegativeLong()); + } + } + + @Override + protected Writeable.Reader> instanceReader() { + return in -> new StoredAsyncResponse<>(TestResponse::new, in); + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index 81b35e4dfb163..8098235c0ff33 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -32,7 +32,8 @@ import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.transport.TransportActionProxy; import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; +import org.elasticsearch.xpack.core.eql.EqlAsyncActionNames; +import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; import org.elasticsearch.xpack.core.security.action.GetApiKeyAction; @@ -266,7 +267,7 @@ public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo auth // information such as the index and the incoming address of the request listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES)); } - } else if (isAsyncSearchRelatedAction(action)) { + } else if (isAsyncRelatedAction(action)) { if (SubmitAsyncSearchAction.NAME.equals(action)) { // we check if the user has any indices permission when submitting an async-search request in order to be // able to fail the request early. Fine grained index-level permissions are handled by the search action @@ -587,9 +588,10 @@ private static boolean isScrollRelatedAction(String action) { action.equals(SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME); } - private static boolean isAsyncSearchRelatedAction(String action) { + private static boolean isAsyncRelatedAction(String action) { return action.equals(SubmitAsyncSearchAction.NAME) || action.equals(GetAsyncSearchAction.NAME) || - action.equals(DeleteAsyncSearchAction.NAME); + action.equals(DeleteAsyncResultAction.NAME) || + action.equals(EqlAsyncActionNames.EQL_ASYNC_GET_RESULT_ACTION_NAME); } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/eql.delete.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/eql.delete.json new file mode 100644 index 0000000000000..47b3990adcb0a --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/eql.delete.json @@ -0,0 +1,25 @@ +{ + "eql.delete":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/eql-search-api.html", + "description": "Deletes an async EQL search by ID. If the search is still running, the search request will be cancelled. Otherwise, the saved search results are deleted." + }, + "stability":"beta", + "url":{ + "paths":[ + { + "path":"/_eql/search/{id}", + "methods":[ + "DELETE" + ], + "parts":{ + "id":{ + "type":"string", + "description":"The async search ID" + } + } + } + ] + } + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/eql.get.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/eql.get.json new file mode 100644 index 0000000000000..9271f43edf736 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/eql.get.json @@ -0,0 +1,36 @@ +{ + "eql.get":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/eql-search-api.html", + "description": "Returns async results from previously executed Event Query Language (EQL) search" + }, + "stability": "beta", + "url":{ + "paths":[ + { + "path":"/_eql/search/{id}", + "methods":[ + "GET" + ], + "parts":{ + "id":{ + "type":"string", + "description":"The async search ID" + } + } + } + ] + }, + "params":{ + "wait_for_completion_timeout":{ + "type":"time", + "description":"Specify the time that the request should block waiting for the final response" + }, + "keep_alive": { + "type": "time", + "description": "Update the time interval in which the results (partial or final) for this search will be available", + "default": "5d" + } + } + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/eql.search.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/eql.search.json index b9ba460d6a997..c371851deeb53 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/eql.search.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/eql.search.json @@ -22,7 +22,22 @@ } ] }, - "params":{}, + "params":{ + "wait_for_completion_timeout":{ + "type":"time", + "description":"Specify the time that the request should block waiting for the final response" + }, + "keep_on_completion":{ + "type":"boolean", + "description":"Control whether the response should be stored in the cluster if it completed within the provided [wait_for_completion] time (default: false)", + "default":false + }, + "keep_alive": { + "type": "time", + "description": "Update the time interval in which the results (partial or final) for this search will be available", + "default": "5d" + } + }, "body":{ "description":"Eql request body. Use the `query` to limit the query scope.", "required":true From c76e32eee508fbc60ff901a5ef489b245e8e530f Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 17 Jun 2020 16:05:36 -0400 Subject: [PATCH 2/3] EQL: Disable Eql Security IT tests in release builds (#58293) Fixes #58268 Relates to #51613 --- x-pack/plugin/eql/qa/security/build.gradle | 6 ++---- .../org/elasticsearch/xpack/eql/AsyncEqlSecurityIT.java | 8 ++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/eql/qa/security/build.gradle b/x-pack/plugin/eql/qa/security/build.gradle index 3a402682ec04f..4a20f5adc63d6 100644 --- a/x-pack/plugin/eql/qa/security/build.gradle +++ b/x-pack/plugin/eql/qa/security/build.gradle @@ -5,10 +5,8 @@ apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' dependencies { -// testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') - testCompile project(path: xpackModule('eql'), configuration: 'runtime') - testCompile project(path: xpackModule('eql:qa:common'), configuration: 'runtime') - testCompile project(':x-pack:plugin:async-search:qa') + testImplementation project(path: xpackModule('eql'), configuration: 'runtime') + testImplementation project(path: xpackModule('eql:qa:common'), configuration: 'runtime') } testClusters.integTest { diff --git a/x-pack/plugin/eql/qa/security/src/test/java/org/elasticsearch/xpack/eql/AsyncEqlSecurityIT.java b/x-pack/plugin/eql/qa/security/src/test/java/org/elasticsearch/xpack/eql/AsyncEqlSecurityIT.java index 40db5f99354b1..34d1e09db9cb9 100644 --- a/x-pack/plugin/eql/qa/security/src/test/java/org/elasticsearch/xpack/eql/AsyncEqlSecurityIT.java +++ b/x-pack/plugin/eql/qa/security/src/test/java/org/elasticsearch/xpack/eql/AsyncEqlSecurityIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.eql; import org.apache.http.util.EntityUtils; +import org.elasticsearch.Build; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; @@ -23,6 +24,7 @@ import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.async.AsyncExecutionId; import org.junit.Before; +import org.junit.BeforeClass; import java.io.IOException; import java.util.Map; @@ -34,6 +36,12 @@ import static org.hamcrest.Matchers.equalTo; public class AsyncEqlSecurityIT extends ESRestTestCase { + + @BeforeClass + public static void checkForSnapshot() { + assumeTrue("Only works on snapshot builds for now", Build.CURRENT.isSnapshot()); + } + /** * All tests run as a superuser but use es-security-runas-user to become a less privileged user. */ From 966707d6005414559971842fdb331bd0fbcf7596 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 25 Jun 2020 09:26:53 -0400 Subject: [PATCH 3/3] EQL: Fix race condition in RestEqlCancellationIT (#58493) Makes RestEqlCancellationIT more deterministic by adding an assertBusy for the cancellation propagation. It also refactors the SearchBlockPlugin to block after the field caps are received, which prevents the test to block on the transport thread. Blocking on transport thread was preventing the cancellation on disconnect from cancelling the task until after the block on the transport thread was released. Since I cannot reporduce this issue, I will leave the trace logging for now and will remove it after observing this test for a while in CI. Relates to #58270 --- .../AbstractEqlBlockingIntegTestCase.java | 24 ++++++++++------- .../eql/action/RestEqlCancellationIT.java | 26 ++++++++++++++++++- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlBlockingIntegTestCase.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlBlockingIntegTestCase.java index 8213e7f1fc471..5e400bc529a23 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlBlockingIntegTestCase.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlBlockingIntegTestCase.java @@ -183,17 +183,23 @@ public int order() { public void apply( Task task, String action, Request request, ActionListener listener, ActionFilterChain chain) { + ActionListener listenerWrapper = listener; if (action.equals(FieldCapabilitiesAction.NAME)) { - try { - fieldCaps.incrementAndGet(); - logger.trace("blocking field caps on " + nodeId); - assertBusy(() -> assertFalse(shouldBlockOnFieldCapabilities.get())); - logger.trace("unblocking field caps on " + nodeId); - } catch (Exception e) { - throw new RuntimeException(e); - } + listenerWrapper = ActionListener.wrap(resp -> { + try { + fieldCaps.incrementAndGet(); + logger.trace("blocking field caps on " + nodeId); + assertBusy(() -> assertFalse(shouldBlockOnFieldCapabilities.get())); + logger.trace("unblocking field caps on " + nodeId); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + listener.onResponse(resp); + } + }, listener::onFailure); + } - chain.proceed(task, action, request, listener); + chain.proceed(task, action, request, listenerWrapper); } }); return list; diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/RestEqlCancellationIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/RestEqlCancellationIT.java index 27c003ec18f44..47c391b3ccc3d 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/RestEqlCancellationIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/RestEqlCancellationIT.java @@ -17,7 +17,10 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskInfo; +import org.elasticsearch.test.junit.annotations.TestIssueLogging; import org.elasticsearch.transport.Netty4Plugin; +import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.nio.NioTransportPlugin; import org.junit.BeforeClass; @@ -35,6 +38,8 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +@TestIssueLogging(value = "org.elasticsearch.xpack.eql.action:TRACE", + issueUrl = "https://github.com/elastic/elasticsearch/issues/58270") public class RestEqlCancellationIT extends AbstractEqlBlockingIntegTestCase { private static String nodeHttpTypeKey; @@ -124,9 +129,28 @@ public void onFailure(Exception exception) { logger.trace("Waiting for block to be established"); awaitForBlockedFieldCaps(plugins); logger.trace("Block is established"); - assertThat(getTaskInfoWithXOpaqueId(id, EqlSearchAction.NAME), notNullValue()); + TaskInfo blockedTaskInfo = getTaskInfoWithXOpaqueId(id, EqlSearchAction.NAME); + assertThat(blockedTaskInfo, notNullValue()); cancellable.cancel(); logger.trace("Request is cancelled"); + + assertBusy(() -> { + for (TransportService transportService : internalCluster().getInstances(TransportService.class)) { + if (transportService.getLocalNode().getId().equals(blockedTaskInfo.getTaskId().getNodeId())) { + Task task = transportService.getTaskManager().getTask(blockedTaskInfo.getId()); + if (task != null) { + assertThat(task, instanceOf(EqlSearchTask.class)); + EqlSearchTask eqlSearchTask = (EqlSearchTask) task; + logger.trace("Waiting for cancellation to be propagated {} ", eqlSearchTask.isCancelled()); + assertThat(eqlSearchTask.isCancelled(), equalTo(true)); + } + return; + } + } + fail("Task not found"); + }); + + logger.trace("Disabling field cap blocks"); disableFieldCapBlocks(plugins); // The task should be cancelled before ever reaching search blocks assertBusy(() -> {