From f012304cb5fa6411317b3626c222cd6914815703 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Thu, 27 Feb 2020 14:18:14 +0200 Subject: [PATCH 1/3] EQL: Disable field extraction for source return --- .../java/org/elasticsearch/client/EqlIT.java | 31 ++++++++++++++++++- .../eql/execution/search/SourceGenerator.java | 18 +---------- .../xpack/eql/planner/QueryFolderTests.java | 6 +--- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java index ae577ef1adb21..049ceadc76f4d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java @@ -23,8 +23,11 @@ import org.apache.http.client.methods.HttpPut; import org.elasticsearch.client.eql.EqlSearchRequest; import org.elasticsearch.client.eql.EqlSearchResponse; +import org.elasticsearch.common.time.DateUtils; import org.junit.Before; +import java.time.format.DateTimeFormatter; + import static org.hamcrest.Matchers.equalTo; public class EqlIT extends ESRestHighLevelClientTestCase { @@ -35,7 +38,6 @@ public void setupRemoteClusterConfig() throws Exception { } public void testBasicSearch() throws Exception { - Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1"); doc1.setJsonEntity("{\"event_subtype_full\": \"already_running\", " + "\"event_type\": \"process\", " + @@ -61,4 +63,31 @@ public void testBasicSearch() throws Exception { assertNotNull(response.hits().events()); assertThat(response.hits().events().size(), equalTo(1)); } + + public void testLargeMapping() throws Exception { + Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1"); + + String now = DateUtils.nowWithMillisResolution().format(DateTimeFormatter.ISO_DATE_TIME); + StringBuilder sb = new StringBuilder(); + sb.append("{"); + for (int i = 0; i < 250; i++) { + sb.append("\"datetime" + i + "\":\"" + now + "\""); + sb.append(","); + } + sb.append("\"event_type\": \"process\","); + sb.append("\"serial_event_id\": 1"); + sb.append("}"); + doc1.setJsonEntity(sb.toString()); + + client().performRequest(doc1); + client().performRequest(new Request(HttpPost.METHOD_NAME, "/_refresh")); + + + EqlClient eql = highLevelClient().eql(); + EqlSearchRequest request = new EqlSearchRequest("index", "process where true"); + EqlSearchResponse response = execute(request, eql::search, eql::searchAsync); + assertNotNull(response); + assertNotNull(response.hits()); + assertThat(response.hits().events().size(), equalTo(1)); + } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java index e85e01db3ea50..62a77997aadf6 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java @@ -45,10 +45,7 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB // Iterate through all the columns requested, collecting the fields that // need to be retrieved from the result documents - // NB: the sortBuilder takes care of eliminating duplicates - container.fields().forEach(f -> f.v1().collectFields(sortBuilder)); - sortBuilder.build(source); - optimize(sortBuilder, source); + source.fetchSource(FetchSourceContext.FETCH_SOURCE); // set fetch size if (size != null) { @@ -62,22 +59,9 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB return source; } - private static void optimize(QlSourceBuilder qlSource, SearchSourceBuilder builder) { - if (qlSource.noSource()) { - disableSource(builder); - } - } - private static void optimize(QueryContainer query, SearchSourceBuilder builder) { if (query.shouldTrackHits()) { builder.trackTotalHits(true); } } - - private static void disableSource(SearchSourceBuilder builder) { - builder.fetchSource(FetchSourceContext.DO_NOT_FETCH_SOURCE); - if (builder.storedFields() == null) { - builder.storedFields(NO_STORED_FIELD); - } - } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java index ad2b42880c4df..9e9cc96cdfd66 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderTests.java @@ -51,10 +51,6 @@ public void testBasicPlan() { // test query term assertThat(query, containsString("\"term\":{\"event_type\":{\"value\":\"process\"")); // test field source extraction - assertThat(query, containsString("\"_source\":{\"includes\":[")); - assertThat(query, containsString("\"pid\"")); - // test docvalue extraction - assertThat(query, containsString("{\"field\":\"command_line\"}")); - assertThat(query, containsString("{\"field\":\"timestamp\",\"format\":\"epoch_millis\"}")); + assertThat(query, containsString("\"_source\":{\"includes\":[],\"excludes\":[]")); } } From ffb90b0ed708c6e31df4fa51ce1aeaf5b94f3b99 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Fri, 28 Feb 2020 11:45:21 +0200 Subject: [PATCH 2/3] Address feedback --- .../src/test/java/org/elasticsearch/client/EqlIT.java | 5 ++++- .../xpack/eql/execution/search/SourceGenerator.java | 5 ----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java index 049ceadc76f4d..6a3c71b04f09b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java @@ -23,7 +23,9 @@ import org.apache.http.client.methods.HttpPut; import org.elasticsearch.client.eql.EqlSearchRequest; import org.elasticsearch.client.eql.EqlSearchResponse; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateUtils; +import org.elasticsearch.index.IndexSettings; import org.junit.Before; import java.time.format.DateTimeFormatter; @@ -67,10 +69,11 @@ public void testBasicSearch() throws Exception { public void testLargeMapping() throws Exception { Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1"); + int PASS_DEFAULT_DOC_VALUES = IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING.get(Settings.EMPTY) + 50; String now = DateUtils.nowWithMillisResolution().format(DateTimeFormatter.ISO_DATE_TIME); StringBuilder sb = new StringBuilder(); sb.append("{"); - for (int i = 0; i < 250; i++) { + for (int i = 0; i < PASS_DEFAULT_DOC_VALUES; i++) { sb.append("\"datetime" + i + "\":\"" + now + "\""); sb.append(","); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java index 62a77997aadf6..aed496434527d 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java @@ -10,7 +10,6 @@ import org.elasticsearch.search.fetch.StoredFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer; -import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder; import java.util.List; @@ -41,10 +40,6 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB final SearchSourceBuilder source = new SearchSourceBuilder(); source.query(finalQuery); - QlSourceBuilder sortBuilder = new QlSourceBuilder(); - // Iterate through all the columns requested, collecting the fields that - // need to be retrieved from the result documents - source.fetchSource(FetchSourceContext.FETCH_SOURCE); // set fetch size From 5445289902e15887dec6802f72bd3a9c3a9adffd Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Fri, 28 Feb 2020 12:02:46 +0200 Subject: [PATCH 3/3] Add comments --- .../src/test/java/org/elasticsearch/client/EqlIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java index 6a3c71b04f09b..9a472e13842cd 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java @@ -68,7 +68,8 @@ public void testBasicSearch() throws Exception { public void testLargeMapping() throws Exception { Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1"); - + // use more exact fields (dates) than the default to verify that retrieval works and requesting doc values + // would fail int PASS_DEFAULT_DOC_VALUES = IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING.get(Settings.EMPTY) + 50; String now = DateUtils.nowWithMillisResolution().format(DateTimeFormatter.ISO_DATE_TIME); StringBuilder sb = new StringBuilder();