diff --git a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml index 13d936a1ae95b..576e2db411864 100644 --- a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml +++ b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml @@ -88,6 +88,25 @@ setup: - match: {hits.events.2.fields.valid: [true]} - match: {hits.events.2.fields.day_of_week: ["Wednesday"]} +--- +"Execute EQL events query with wildcard (*) fields filtering.": + - do: + eql.search: + index: eql_test + body: + query: 'process where user == "SYSTEM"' + fields: [{"field":"*"}] + + - match: {timed_out: false} + - match: {hits.total.value: 3} + - match: {hits.total.relation: "eq"} + - match: {hits.events.0._id: "1"} + - match: {hits.events.0.fields:{"valid":[false],"@timestamp":["2020-02-03T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Monday"]}} + - match: {hits.events.1._id: "2"} + - match: {hits.events.1.fields:{"valid":[true],"@timestamp":["2020-02-04T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Tuesday"]}} + - match: {hits.events.2._id: "3"} + - match: {hits.events.2.fields:{"valid":[true],"@timestamp":["2020-02-05T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Wednesday"]}} + --- "Execute EQL events query with filter_path": - do: @@ -218,6 +237,50 @@ setup: - match: {hits.sequences.1.events.1.fields.valid: [true]} - match: {hits.sequences.1.events.1.fields.day_of_week: ["Wednesday"]} +--- +"Execute EQL sequence with wildcard (*) fields filtering.": + - do: + eql.search: + index: eql_test + body: + query: 'sequence by user [process where user == "SYSTEM"] [process where true]' + fields: [{"field":"*"}] + - match: {timed_out: false} + - match: {hits.total.value: 2} + - match: {hits.total.relation: "eq"} + - match: {hits.sequences.0.join_keys.0: "SYSTEM"} + - match: {hits.sequences.0.events.0._id: "1"} + - match: {hits.sequences.0.events.0.fields:{"valid":[false],"@timestamp":["2020-02-03T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Monday"]}} + - match: {hits.sequences.0.events.1._id: "2"} + - match: {hits.sequences.0.events.1.fields:{"valid":[true],"@timestamp":["2020-02-04T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Tuesday"]}} + - match: {hits.sequences.1.join_keys.0: "SYSTEM"} + - match: {hits.sequences.1.events.0._id: "2"} + - match: {hits.sequences.1.events.0.fields:{"valid":[true],"@timestamp":["2020-02-04T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Tuesday"]}} + - match: {hits.sequences.1.events.1._id: "3"} + - match: {hits.sequences.1.events.1.fields:{"valid":[true],"@timestamp":["2020-02-05T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Wednesday"]}} + +--- +"Execute EQL sequence with custom format for timestamp field.": + - do: + eql.search: + index: eql_test + body: + query: 'sequence by user [process where user == "SYSTEM"] [process where true]' + fields: [{"field":"@timestamp","format":"yyyy"},{"field":"day_of_week"}] + - match: {timed_out: false} + - match: {hits.total.value: 2} + - match: {hits.total.relation: "eq"} + - match: {hits.sequences.0.join_keys.0: "SYSTEM"} + - match: {hits.sequences.0.events.0._id: "1"} + - match: {hits.sequences.0.events.0.fields:{"@timestamp":["2020"],"day_of_week":["Monday"]}} + - match: {hits.sequences.0.events.1._id: "2"} + - match: {hits.sequences.0.events.1.fields:{"@timestamp":["2020"],"day_of_week":["Tuesday"]}} + - match: {hits.sequences.1.join_keys.0: "SYSTEM"} + - match: {hits.sequences.1.events.0._id: "2"} + - match: {hits.sequences.1.events.0.fields:{"@timestamp":["2020"],"day_of_week":["Tuesday"]}} + - match: {hits.sequences.1.events.1._id: "3"} + - match: {hits.sequences.1.events.1.fields:{"@timestamp":["2020"],"day_of_week":["Wednesday"]}} + --- "Execute EQL sequence with filter_path": - do: diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java index 80ba6b5c08c95..1c0aba3a585ad 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java @@ -92,7 +92,7 @@ public Executable assemble(List> listOfKeys, PhysicalPlan query = plans.get(i); // search query if (query instanceof EsQueryExec) { - SearchSourceBuilder source = ((EsQueryExec) query).source(session); + SearchSourceBuilder source = ((EsQueryExec) query).source(session, false); QueryRequest original = () -> source; BoxedQueryRequest boxedRequest = new BoxedQueryRequest(original, timestampName, keyFields); Criterion criterion = diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java index 906de0c4add05..cc62efa558a46 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java @@ -52,16 +52,22 @@ public List output() { return output; } - public SearchSourceBuilder source(EqlSession session) { + + /* + * {@param includeFetchFields} should be true for event queries and false for in progress sequence queries + * Fetching fields during in progress sequence queries is unnecessary. + */ + public SearchSourceBuilder source(EqlSession session, boolean includeFetchFields) { EqlConfiguration cfg = session.configuration(); // by default use the configuration size - return SourceGenerator.sourceBuilder(queryContainer, cfg.filter(), cfg.fetchFields(), cfg.runtimeMappings()); + return SourceGenerator.sourceBuilder(queryContainer, cfg.filter(), includeFetchFields ? cfg.fetchFields() : null, + cfg.runtimeMappings()); } @Override public void execute(EqlSession session, ActionListener listener) { // endpoint - fetch all source - QueryRequest request = () -> source(session).fetchSource(FetchSourceContext.FETCH_SOURCE); + QueryRequest request = () -> source(session, true).fetchSource(FetchSourceContext.FETCH_SOURCE); listener = shouldReverse(request) ? new ReverseListener(listener) : listener; new BasicQueryClient(session).query(request, new AsEventListener(listener)); }