EQL: Disable field extraction for returned events (#52884)

Return the whole source of matching events

Author: costin

client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java

@@ -23,8 +23,13 @@
 import org.apache.http.client.methods.HttpPut;
 import org.elasticsearch.client.eql.EqlSearchRequest;
 import org.elasticsearch.client.eql.EqlSearchResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.time.DateUtils;
+import org.elasticsearch.index.IndexSettings;
 import org.junit.Before;
 
+import java.time.format.DateTimeFormatter;
+
 import static org.hamcrest.Matchers.equalTo;
 
 public class EqlIT extends ESRestHighLevelClientTestCase {
@@ -35,7 +40,6 @@ public void setupRemoteClusterConfig() throws Exception {
     }
 
     public void testBasicSearch() throws Exception {
-
         Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1");
         doc1.setJsonEntity("{\"event_subtype_full\": \"already_running\", " +
                 "\"event_type\": \"process\", " +
@@ -61,4 +65,33 @@ public void testBasicSearch() throws Exception {
         assertNotNull(response.hits().events());
         assertThat(response.hits().events().size(), equalTo(1));
     }
+
+    public void testLargeMapping() throws Exception {
+        Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1");
+        // use more exact fields (dates) than the default to verify that retrieval works and requesting doc values
+        // would fail
+        int PASS_DEFAULT_DOC_VALUES = IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING.get(Settings.EMPTY) + 50;
+        String now = DateUtils.nowWithMillisResolution().format(DateTimeFormatter.ISO_DATE_TIME);
+        StringBuilder sb = new StringBuilder();
+        sb.append("{");
+        for (int i = 0; i < PASS_DEFAULT_DOC_VALUES; i++) {
+            sb.append("\"datetime" + i + "\":\"" + now + "\"");
+            sb.append(",");
+        }
+        sb.append("\"event_type\": \"process\",");
+        sb.append("\"serial_event_id\": 1");
+        sb.append("}");
+        doc1.setJsonEntity(sb.toString());
+
+        client().performRequest(doc1);
+        client().performRequest(new Request(HttpPost.METHOD_NAME, "/_refresh"));
+
+
+        EqlClient eql = highLevelClient().eql();
+        EqlSearchRequest request = new EqlSearchRequest("index", "process where true");
+        EqlSearchResponse response = execute(request, eql::search, eql::searchAsync);
+        assertNotNull(response);
+        assertNotNull(response.hits());
+        assertThat(response.hits().events().size(), equalTo(1));
+    }
 }

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java

@@ -10,7 +10,6 @@
 import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer;
-import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder;
 
 import java.util.List;
 
@@ -41,14 +40,7 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB
         final SearchSourceBuilder source = new SearchSourceBuilder();
         source.query(finalQuery);
 
-        QlSourceBuilder sortBuilder = new QlSourceBuilder();
-        // Iterate through all the columns requested, collecting the fields that
-        // need to be retrieved from the result documents
-
-        // NB: the sortBuilder takes care of eliminating duplicates
-        container.fields().forEach(f -> f.v1().collectFields(sortBuilder));
-        sortBuilder.build(source);
-        optimize(sortBuilder, source);
+        source.fetchSource(FetchSourceContext.FETCH_SOURCE);
 
         // set fetch size
         if (size != null) {
@@ -62,22 +54,9 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB
         return source;
     }
 
-    private static void optimize(QlSourceBuilder qlSource, SearchSourceBuilder builder) {
-        if (qlSource.noSource()) {
-            disableSource(builder);
-        }
-    }
-
     private static void optimize(QueryContainer query, SearchSourceBuilder builder) {
         if (query.shouldTrackHits()) {
             builder.trackTotalHits(true);
         }
     }
-
-    private static void disableSource(SearchSourceBuilder builder) {
-        builder.fetchSource(FetchSourceContext.DO_NOT_FETCH_SOURCE);
-        if (builder.storedFields() == null) {
-            builder.storedFields(NO_STORED_FIELD);
-        }
-    }
 }

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java

@@ -51,10 +51,6 @@ public void testBasicPlan() {
         // test query term
         assertThat(query, containsString("\"term\":{\"event_type\":{\"value\":\"process\""));
         // test field source extraction
-        assertThat(query, containsString("\"_source\":{\"includes\":["));
-        assertThat(query, containsString("\"pid\""));
-        // test docvalue extraction
-        assertThat(query, containsString("{\"field\":\"command_line\"}"));
-        assertThat(query, containsString("{\"field\":\"timestamp\",\"format\":\"epoch_millis\"}"));
+        assertThat(query, containsString("\"_source\":{\"includes\":[],\"excludes\":[]"));
     }
 }