[7.x] EQL: Adds an ability to execute an asynchronous EQL search (#58192)

Adds async support to EQL searches

Closes #49638

Co-authored-by: James Rodewig [email protected]

Author: imotov

client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java

@@ -21,6 +21,7 @@
 
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Validatable;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -45,6 +46,11 @@ public class EqlSearchRequest implements Validatable, ToXContentObject {
     private String query;
     private String tiebreakerField;
 
+    // Async settings
+    private TimeValue waitForCompletionTimeout;
+    private boolean keepOnCompletion;
+    private TimeValue keepAlive;
+
     static final String KEY_FILTER = "filter";
     static final String KEY_TIMESTAMP_FIELD = "timestamp_field";
     static final String KEY_TIEBREAKER_FIELD = "tiebreaker_field";
@@ -53,6 +59,9 @@ public class EqlSearchRequest implements Validatable, ToXContentObject {
     static final String KEY_SIZE = "size";
     static final String KEY_SEARCH_AFTER = "search_after";
     static final String KEY_QUERY = "query";
+    static final String KEY_WAIT_FOR_COMPLETION_TIMEOUT = "wait_for_completion_timeout";
+    static final String KEY_KEEP_ALIVE = "keep_alive";
+    static final String KEY_KEEP_ON_COMPLETION = "keep_on_completion";
 
     public EqlSearchRequest(String indices, String query) {
         indices(indices);
@@ -80,6 +89,13 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
         }
 
         builder.field(KEY_QUERY, query);
+        if (waitForCompletionTimeout != null) {
+            builder.field(KEY_WAIT_FOR_COMPLETION_TIMEOUT, waitForCompletionTimeout);
+        }
+        if (keepAlive != null) {
+            builder.field(KEY_KEEP_ALIVE, keepAlive);
+        }
+        builder.field(KEY_KEEP_ON_COMPLETION, keepOnCompletion);
         builder.endObject();
         return builder;
     }
@@ -181,6 +197,32 @@ public EqlSearchRequest query(String query) {
         return this;
     }
 
+    public TimeValue waitForCompletionTimeout() {
+        return waitForCompletionTimeout;
+    }
+
+    public EqlSearchRequest waitForCompletionTimeout(TimeValue waitForCompletionTimeout) {
+        this.waitForCompletionTimeout = waitForCompletionTimeout;
+        return this;
+    }
+
+    public Boolean keepOnCompletion() {
+        return keepOnCompletion;
+    }
+
+    public void keepOnCompletion(Boolean keepOnCompletion) {
+        this.keepOnCompletion = keepOnCompletion;
+    }
+
+    public TimeValue keepAlive() {
+        return keepAlive;
+    }
+
+    public EqlSearchRequest keepAlive(TimeValue keepAlive) {
+        this.keepAlive = keepAlive;
+        return this;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {
@@ -199,7 +241,10 @@ public boolean equals(Object o) {
                 Objects.equals(eventCategoryField, that.eventCategoryField) &&
                 Objects.equals(implicitJoinKeyField, that.implicitJoinKeyField) &&
                 Objects.equals(searchAfterBuilder, that.searchAfterBuilder) &&
-                Objects.equals(query, that.query);
+                Objects.equals(query, that.query) &&
+                Objects.equals(waitForCompletionTimeout, that.waitForCompletionTimeout) &&
+                Objects.equals(keepAlive, that.keepAlive) &&
+                Objects.equals(keepOnCompletion, that.keepOnCompletion);
     }
 
     @Override
@@ -214,7 +259,10 @@ public int hashCode() {
             eventCategoryField,
             implicitJoinKeyField,
             searchAfterBuilder,
-            query);
+            query,
+            waitForCompletionTimeout,
+            keepAlive,
+            keepOnCompletion);
     }
 
     public String[] indices() {

client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchResponse.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.InstantiatingObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
@@ -32,43 +33,56 @@
 import java.util.List;
 import java.util.Objects;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
 public class EqlSearchResponse {
 
     private final Hits hits;
     private final long tookInMillis;
     private final boolean isTimeout;
+    private final String asyncExecutionId;
+    private final boolean isRunning;
+    private final boolean isPartial;
 
     private static final class Fields {
         static final String TOOK = "took";
         static final String TIMED_OUT = "timed_out";
         static final String HITS = "hits";
+        static final String ID = "id";
+        static final String IS_RUNNING = "is_running";
+        static final String IS_PARTIAL = "is_partial";
     }
 
     private static final ParseField TOOK = new ParseField(Fields.TOOK);
     private static final ParseField TIMED_OUT = new ParseField(Fields.TIMED_OUT);
     private static final ParseField HITS = new ParseField(Fields.HITS);
+    private static final ParseField ID = new ParseField(Fields.ID);
+    private static final ParseField IS_RUNNING = new ParseField(Fields.IS_RUNNING);
+    private static final ParseField IS_PARTIAL = new ParseField(Fields.IS_PARTIAL);
 
-    private static final ConstructingObjectParser<EqlSearchResponse, Void> PARSER =
-        new ConstructingObjectParser<>("eql/search_response", true,
-            args -> {
-                int i = 0;
-                Hits hits = (Hits) args[i++];
-                Long took = (Long) args[i++];
-                Boolean timeout = (Boolean) args[i];
-                return new EqlSearchResponse(hits, took, timeout);
-            });
-
+    private static final InstantiatingObjectParser<EqlSearchResponse, Void> PARSER;
     static {
-        PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> Hits.fromXContent(p), HITS);
-        PARSER.declareLong(ConstructingObjectParser.constructorArg(), TOOK);
-        PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), TIMED_OUT);
+        InstantiatingObjectParser.Builder<EqlSearchResponse, Void> parser =
+            InstantiatingObjectParser.builder("eql/search_response", true, EqlSearchResponse.class);
+        parser.declareObject(constructorArg(), (p, c) -> Hits.fromXContent(p), HITS);
+        parser.declareLong(constructorArg(), TOOK);
+        parser.declareBoolean(constructorArg(), TIMED_OUT);
+        parser.declareString(optionalConstructorArg(), ID);
+        parser.declareBoolean(constructorArg(), IS_RUNNING);
+        parser.declareBoolean(constructorArg(), IS_PARTIAL);
+        PARSER = parser.build();
     }
 
-    public EqlSearchResponse(Hits hits, long tookInMillis, boolean isTimeout) {
+    public EqlSearchResponse(Hits hits, long tookInMillis, boolean isTimeout, String asyncExecutionId,
+                             boolean isRunning, boolean isPartial) {
         super();
         this.hits = hits == null ? Hits.EMPTY : hits;
         this.tookInMillis = tookInMillis;
         this.isTimeout = isTimeout;
+        this.asyncExecutionId = asyncExecutionId;
+        this.isRunning = isRunning;
+        this.isPartial = isPartial;
     }
 
     public static EqlSearchResponse fromXContent(XContentParser parser) {
@@ -87,6 +101,18 @@ public Hits hits() {
         return hits;
     }
 
+    public String id() {
+        return asyncExecutionId;
+    }
+
+    public boolean isRunning() {
+        return isRunning;
+    }
+
+    public boolean isPartial() {
+        return isPartial;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {

client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java

@@ -57,7 +57,12 @@ public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomE
         if (randomBoolean()) {
             hits = new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Hits(randomEvents(), null, null, totalHits);
         }
-        return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean());
+        if (randomBoolean()) {
+            return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean());
+        } else {
+            return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean(),
+                randomAlphaOfLength(10), randomBoolean(), randomBoolean());
+        }
     }
 
     public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomSequencesResponse(TotalHits totalHits) {
@@ -77,7 +82,12 @@ public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomS
         if (randomBoolean()) {
             hits = new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Hits(null, seq, null, totalHits);
         }
-        return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean());
+        if (randomBoolean()) {
+            return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean());
+        } else {
+            return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean(),
+                randomAlphaOfLength(10), randomBoolean(), randomBoolean());
+        }
     }
 
     public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomCountResponse(TotalHits totalHits) {
@@ -97,7 +107,12 @@ public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomC
         if (randomBoolean()) {
             hits = new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Hits(null, null, cn, totalHits);
         }
-        return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean());
+        if (randomBoolean()) {
+            return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean());
+        } else {
+            return new org.elasticsearch.xpack.eql.action.EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean(),
+                randomAlphaOfLength(10), randomBoolean(), randomBoolean());
+        }
     }
 
     public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomInstance(TotalHits totalHits) {

docs/reference/eql/delete-async-eql-search-api.asciidoc

@@ -0,0 +1,47 @@
+[role="xpack"]
+[testenv="basic"]
+
+[[delete-async-eql-search-api]]
+=== Delete async EQL search API
+++++
+<titleabbrev>Delete async EQL search</titleabbrev>
+++++
+
+dev::[]
+
+Deletes an <<eql-search-async,async EQL search>> or a
+<<eql-search-store-sync-eql-search,stored synchronous EQL search>>. The API also
+deletes results for the search.
+
+[source,console]
+----
+DELETE /_eql/search/FkpMRkJGS1gzVDRlM3g4ZzMyRGlLbkEaTXlJZHdNT09TU2VTZVBoNDM3cFZMUToxMDM=
+----
+// TEST[skip: no access to search ID]
+
+[[delete-async-eql-search-api-request]]
+==== {api-request-title}
+
+`DELETE /_eql/search/<search_id>`
+
+[[delete-async-eql-search-api-prereqs]]
+==== {api-prereq-title}
+
+See <<eql-requirements,EQL requirements>>.
+
+[[delete-async-eql-search-api-limitations]]
+===== Limitations
+
+See <<eql-limitations,EQL limitations>>.
+
+[[delete-async-eql-search-api-path-params]]
+==== {api-path-parms-title}
+
+`<search_id>`::
+(Required, string)
+Identifier for the search to delete.
++
+A search ID is provided in the <<eql-search-api,EQL search API>>'s response for
+an <<eql-search-async,async search>>. A search ID is also provided if the
+request's <<eql-search-api-keep-on-completion,`keep_on_completion`>> parameter
+is `true`.