From 647e7b351e3c08836e9a53db7fedded2cfe4bb23 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Wed, 16 Sep 2020 15:51:47 +0300 Subject: [PATCH 1/4] EQL: Forbid usage of ['] for string literals The usage of single quotes to wrap a string literal is forbidden and an error encouraging the user to user double quotes is returned. Tests are properly adjusted. Related to #61659 --- .../test/eql/stats/RestEqlUsageTestCase.java | 4 +- .../resources/additional_test_queries.toml | 16 +-- .../src/main/resources/test_queries.toml | 128 +++++++++--------- .../resources/test_queries_unsupported.toml | 66 ++++----- .../rest-api-spec/test/eql/10_basic.yml | 10 +- .../xpack/eql/parser/AbstractBuilder.java | 19 ++- .../xpack/eql/parser/ExpressionBuilder.java | 3 +- .../xpack/eql/analysis/VerifierTests.java | 102 +++++++------- .../xpack/eql/optimizer/OptimizerTests.java | 10 +- .../xpack/eql/parser/ExpressionTests.java | 80 ++++++----- .../xpack/eql/parser/LogicalPlanTests.java | 14 +- .../eql/planner/QueryFolderFailTests.java | 10 +- .../src/test/resources/queries-supported.eql | 68 +++++----- .../src/test/resources/queryfolder_tests.txt | 22 +-- .../eql/src/test/resources/test_folding.toml | 6 +- .../test/resources/test_string_functions.toml | 67 ++++----- .../src/test/resources/test_unsupported.toml | 21 +-- 17 files changed, 324 insertions(+), 322 deletions(-) diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/stats/RestEqlUsageTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/stats/RestEqlUsageTestCase.java index a202aecd7f994..d7cab363e4eea 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/stats/RestEqlUsageTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/stats/RestEqlUsageTestCase.java @@ -178,7 +178,7 @@ public void testEqlRestUsage() throws IOException { " [process where serial_event_id < 4] by exit_code" + " [process where opcode == 1] by user" + " [process where opcode == 2] by user" + - " [file where parent_process_name == 'file_delete_event'] by exit_code" + + " [file where parent_process_name == \\\"file_delete_event\\\"] by exit_code" + " until [process where opcode=1] by ppid" + " | head 4" + " | tail 2"); @@ -216,7 +216,7 @@ public void testEqlRestUsage() throws IOException { runEql("sequence by user, ppid, exit_code with maxspan=1m" + " [process where serial_event_id < 4]" + " [process where opcode == 1]" + - " [file where parent_process_name == 'file_delete_event']" + + " [file where parent_process_name == \\\"file_delete_event\\\"]" + " [process where serial_event_id < 4]" + " [process where opcode == 1]" + "| tail 4"); diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml b/x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml index e330d1c84b148..1588d1b2d08f1 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml @@ -62,7 +62,7 @@ network where cidrMatch(source_address, "0.0.0.0/0") == true name = "concatEquals1" description = "test string concatenation. update test to avoid case-sensitivity issues" query = ''' -process where concat(serial_event_id, '::', process_name, '::', opcode) == '5::wininit.exe::3' +process where concat(serial_event_id, "::", process_name, "::", opcode) == "5::wininit.exe::3" ''' expected_event_ids = [5] @@ -138,15 +138,15 @@ expected_event_ids = [98] notes = "regexp doesn't support character classes" query = ''' // -// ?'.*?net1\s+localgroup.*?') -process where match(command_line, ?'.*?net1[ ]+localgroup.*?') +// ?".*?net1\s+localgroup.*?") +process where match(command_line, ?".*?net1[ ]+localgroup.*?") ''' [[queries]] name = "matchLiteAdditional" expected_event_ids = [98] query = ''' -process where matchLite(command_line, ?'.*?net1.*?') +process where matchLite(command_line, ?".*?net1.*?") ''' [[queries]] @@ -154,8 +154,8 @@ name = "matchWithCharacterClasses2" expected_event_ids = [98] notes = "regexp doesn't support predefined character classes (like \\s)" query = ''' -// ?'.*?net1\s+\w{4,15}\s+.*?' -process where match(command_line, ?'.*?net1[ ]+[a-z]{4,15}[ ]+.*?') +// ?".*?net1\s+\w{4,15}\s+.*?" +process where match(command_line, ?".*?net1[ ]+[a-z]{4,15}[ ]+.*?") ''' @@ -164,14 +164,14 @@ process where match(command_line, ?'.*?net1[ ]+[a-z]{4,15}[ ]+.*?') name = "multiPatternMatch" expected_event_ids = [50, 97, 98] query = ''' -process where match(command_line, '.*?net[1]? localgroup.*?', '.*? myappserver.py .*?') +process where match(command_line, ".*?net[1]? localgroup.*?", ".*? myappserver.py .*?") ''' [[queries]] name = "matchWithSubstring" expected_event_ids = [50, 98] query = ''' -process where match(substring(command_line, 5), '.*?net[1]? localgroup.*?', '.*? myappserver.py .*?') +process where match(substring(command_line, 5), ".*?net[1]? localgroup.*?", ".*? myappserver.py .*?") ''' [[queries]] diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml index 952dbae710e52..e9b2df6baedf9 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml @@ -762,7 +762,7 @@ expected_event_ids = [1, 2, name = "sequencesOnDifferentEventTypes1" query = ''' sequence by unique_pid - [process where opcode=1 and process_name == 'MSBuild.exe'] + [process where opcode=1 and process_name == "MSBuild.exe"] [network where true] ''' expected_event_ids = [75273, 75304] @@ -1322,7 +1322,7 @@ name = "arrayContainsCaseInsensitive1" case_insensitive = true expected_event_ids = [57] query = ''' -registry where arrayContains(bytes_written_string_list, 'En-uS') +registry where arrayContains(bytes_written_string_list, "En-uS") ''' [[queries]] @@ -1330,7 +1330,7 @@ name = "arrayContainsCaseInsensitive2" case_insensitive = true expected_event_ids = [57] query = ''' -registry where arrayContains(bytes_written_string_list, 'En') +registry where arrayContains(bytes_written_string_list, "En") ''' [[queries]] @@ -1338,7 +1338,7 @@ name = "lengthCaseInsensitive2" case_insensitive = true expected_event_ids = [57] query = ''' -registry where length(bytes_written_string_list) > 0 and bytes_written_string_list[0] == 'EN-us' +registry where length(bytes_written_string_list) > 0 and bytes_written_string_list[0] == "EN-us" ''' [[queries]] @@ -1346,7 +1346,7 @@ name = "arrayCaseInsensitive1" case_insensitive = true expected_event_ids = [57] query = ''' -registry where bytes_written_string_list[0] == 'EN-us' +registry where bytes_written_string_list[0] == "EN-us" ''' [[queries]] @@ -1354,7 +1354,7 @@ name = "arrayCaseInsensitive2" case_insensitive = true expected_event_ids = [57] query = ''' -registry where bytes_written_string_list[1] == 'EN' +registry where bytes_written_string_list[1] == "EN" ''' @@ -1363,7 +1363,7 @@ name = "arrayContainsCaseInsensitive3" case_insensitive = true expected_event_ids = [57] query = ''' -registry where arrayContains(bytes_written_string_list, 'en-US') +registry where arrayContains(bytes_written_string_list, "en-US") ''' [[queries]] @@ -1371,7 +1371,7 @@ name = "arrayContainsCaseInsensitive4" case_insensitive = true expected_event_ids = [57] query = ''' -registry where arrayContains(bytes_written_string_list, 'en') +registry where arrayContains(bytes_written_string_list, "en") ''' [[queries]] @@ -1379,41 +1379,41 @@ name = "arrayCaseInsensitive3" case_insensitive = true expected_event_ids = [57] query = ''' -registry where length(bytes_written_string_list) > 0 and bytes_written_string_list[0] == 'en-US' +registry where length(bytes_written_string_list) > 0 and bytes_written_string_list[0] == "en-US" ''' [[queries]] name = "arrayEquality1" expected_event_ids = [57] query = ''' -registry where bytes_written_string_list[0] == 'en-US' +registry where bytes_written_string_list[0] == "en-US" ''' [[queries]] name = "arrayEquality2" expected_event_ids = [57] query = ''' -registry where bytes_written_string_list[1] == 'en' +registry where bytes_written_string_list[1] == "en" ''' [[queries]] name = "matchLite1" query = ''' -process where matchLite(command_line, ?'.*?net1\s+localgroup\s+.*?') +process where matchLite(command_line, ?".*?net1\s+localgroup\s+.*?") ''' expected_event_ids = [98] [[queries]] name = "matchLite2" query = ''' -process where matchLite(command_line, ?'.*?net1\s+\w+\s+.*?') +process where matchLite(command_line, ?".*?net1\s+\w+\s+.*?") ''' expected_event_ids = [98] [[queries]] name = "matchLite3" query = ''' -process where matchLite(command_line, ?'.*?net1\s+\w{4,15}\s+.*?') +process where matchLite(command_line, ?".*?net1\s+\w{4,15}\s+.*?") ''' expected_event_ids = [98] @@ -1421,13 +1421,13 @@ expected_event_ids = [98] name = "match1" expected_event_ids = [98] query = ''' -process where match(command_line, ?'.*?net1\s+\w{4,15}\s+.*?') +process where match(command_line, ?".*?net1\s+\w{4,15}\s+.*?") ''' [[queries]] name = "matchLite4" query = ''' -process where matchLite(command_line, ?'.*?net1\s+[localgrup]{4,15}\s+.*?') +process where matchLite(command_line, ?".*?net1\s+[localgrup]{4,15}\s+.*?") ''' expected_event_ids = [98] @@ -1435,7 +1435,7 @@ expected_event_ids = [98] name = "stringEqualsCaseInsensitive1" case_insensitive = true query = ''' -process where 'net.EXE' == original_file_name +process where "net.EXE" == original_file_name | filter process_name="net*.exe" ''' expected_event_ids = [97] @@ -1445,7 +1445,7 @@ note = "check that case insensitive comparisons are performed even for lhs strin name = "stringEqualsCaseInsensitive2" case_insensitive = true query = ''' -process where process_name == original_file_name and process_name='net*.exe' +process where process_name == original_file_name and process_name="net*.exe" ''' expected_event_ids = [97, 98] note = "check that case insensitive comparisons are performed for fields." @@ -1463,7 +1463,7 @@ description = "check that case insensitive comparisons are performed for fields. name = "startsWithCaseSensitive" case_sensitive = true query = ''' -file where opcode=0 and startsWith(file_name, 'explorer.') +file where opcode=0 and startsWith(file_name, "explorer.") ''' expected_event_ids = [88] description = "check built-in string functions" @@ -1473,7 +1473,7 @@ description = "check built-in string functions" name = "startsWithCaseInsensitive1" case_insensitive = true query = ''' -file where opcode=0 and startsWith(file_name, 'explorer.') +file where opcode=0 and startsWith(file_name, "explorer.") ''' expected_event_ids = [88, 92] description = "check built-in string functions" @@ -1483,7 +1483,7 @@ description = "check built-in string functions" name = "startsWithCaseInsensitive2" case_insensitive = true query = ''' -file where opcode=0 and startsWith(file_name, 'exploRER.') +file where opcode=0 and startsWith(file_name, "exploRER.") ''' expected_event_ids = [88, 92] description = "check built-in string functions" @@ -1492,7 +1492,7 @@ description = "check built-in string functions" name = "startsWithCaseInsensitive3" case_insensitive = true query = ''' -file where opcode=0 and startsWith(file_name, 'expLORER.exe') +file where opcode=0 and startsWith(file_name, "expLORER.exe") ''' expected_event_ids = [88, 92] description = "check built-in string functions" @@ -1500,7 +1500,7 @@ description = "check built-in string functions" [[queries]] name = "endsWith1" query = ''' -file where opcode=0 and endsWith(file_name, 'lorer.exe') +file where opcode=0 and endsWith(file_name, "lorer.exe") ''' expected_event_ids = [88] description = "check built-in string functions" @@ -1510,7 +1510,7 @@ description = "check built-in string functions" name = "endsWithCaseInsensitive" case_insensitive = true query = ''' -file where opcode=0 and endsWith(file_name, 'loREr.exe') +file where opcode=0 and endsWith(file_name, "loREr.exe") ''' expected_event_ids = [88] description = "check built-in string functions" @@ -1518,7 +1518,7 @@ description = "check built-in string functions" [[queries]] name = "endsWith2" query = ''' -file where opcode=0 and startsWith('explorer.exeaaaaaaaa', file_name) +file where opcode=0 and startsWith("explorer.exeaaaaaaaa", file_name) ''' expected_event_ids = [88] description = "check built-in string functions" @@ -1527,7 +1527,7 @@ description = "check built-in string functions" name = "endsWithAndCondition" case_insensitive = true query = ''' -file where opcode=0 and serial_event_id = 88 and startsWith('explorer.exeaAAAA', 'EXPLORER.exe') +file where opcode=0 and serial_event_id = 88 and startsWith("explorer.exeaAAAA", "EXPLORER.exe") ''' expected_event_ids = [88] description = "check built-in string functions" @@ -1535,7 +1535,7 @@ description = "check built-in string functions" [[queries]] name = "stringContains2" query = ''' -file where opcode=0 and stringContains('ABCDEFGHIexplorer.exeJKLMNOP', file_name) +file where opcode=0 and stringContains("ABCDEFGHIexplorer.exeJKLMNOP", file_name) ''' expected_event_ids = [88] description = "check built-in string functions" @@ -1544,7 +1544,7 @@ description = "check built-in string functions" name = "indexOfCaseInsensitive" case_insensitive = true query = ''' -file where opcode=0 and indexOf(file_name, 'plore') == 2 and indexOf(file_name, '.pf') == null +file where opcode=0 and indexOf(file_name, "plore") == 2 and indexOf(file_name, ".pf") == null ''' expected_event_ids = [88] description = "check built-in string functions" @@ -1552,7 +1552,7 @@ description = "check built-in string functions" [[queries]] name = "indexOf1" query = ''' -file where opcode=0 and indexOf(file_name, 'explorer.') > 0 and indexOf(file_name, 'plore', 100) > 0 +file where opcode=0 and indexOf(file_name, "explorer.") > 0 and indexOf(file_name, "plore", 100) > 0 ''' expected_event_ids = [] description = "check built-in string functions" @@ -1561,7 +1561,7 @@ description = "check built-in string functions" name = "indexOf2" case_sensitive = true query = ''' -file where opcode=0 and indexOf(file_name, 'plorer.', 0) == 2 +file where opcode=0 and indexOf(file_name, "plorer.", 0) == 2 ''' expected_event_ids = [88] description = "check built-in string functions" @@ -1570,7 +1570,7 @@ description = "check built-in string functions" name = "indexOf3" case_insensitive = true query = ''' -file where opcode=0 and indexOf(file_name, 'plorer.', 0) == 2 +file where opcode=0 and indexOf(file_name, "plorer.", 0) == 2 ''' expected_event_ids = [88, 92] description = "check built-in string functions" @@ -1579,7 +1579,7 @@ description = "check built-in string functions" name = "indexOf4" case_sensitive = true query = ''' -file where opcode=0 and indexOf(file_name, 'plorer.', 2) != null +file where opcode=0 and indexOf(file_name, "plorer.", 2) != null ''' expected_event_ids = [88] description = "check built-in string functions" @@ -1588,7 +1588,7 @@ description = "check built-in string functions" name = "indexOf5" case_insensitive = true query = ''' -file where opcode=0 and indexOf(file_name, 'plorer.', 2) != null +file where opcode=0 and indexOf(file_name, "plorer.", 2) != null ''' expected_event_ids = [88, 92] description = "check built-in string functions" @@ -1596,7 +1596,7 @@ description = "check built-in string functions" [[queries]] name = "indexOf6" query = ''' -file where opcode=0 and indexOf(file_name, 'plorer.', 4) != null +file where opcode=0 and indexOf(file_name, "plorer.", 4) != null ''' expected_event_ids = [] description = "check built-in string functions" @@ -1604,7 +1604,7 @@ description = "check built-in string functions" [[queries]] name = "indexOf7" query = ''' -file where opcode=0 and indexOf(file_name, 'thing that never happened') != null +file where opcode=0 and indexOf(file_name, "thing that never happened") != null ''' expected_event_ids = [] description = "check built-in string functions" @@ -1613,7 +1613,7 @@ description = "check built-in string functions" name = "indexOf8" case_insensitive = true query = ''' -file where opcode=0 and indexOf(file_name, 'plorer.', 2) == 2 +file where opcode=0 and indexOf(file_name, "plorer.", 2) == 2 ''' expected_event_ids = [88, 92] description = "check substring ranges" @@ -1622,7 +1622,7 @@ description = "check substring ranges" name = "indexOf9" case_sensitive = true query = ''' -file where opcode=0 and indexOf(file_name, 'plorer.', 2) == 2 +file where opcode=0 and indexOf(file_name, "plorer.", 2) == 2 ''' expected_event_ids = [88] description = "check substring ranges" @@ -1631,7 +1631,7 @@ description = "check substring ranges" name = "indexOf10" case_sensitive = true query = ''' -file where opcode=0 and indexOf(file_name, 'explorer.', 0) == 0 +file where opcode=0 and indexOf(file_name, "explorer.", 0) == 0 ''' expected_event_ids = [88] description = "check substring ranges" @@ -1640,7 +1640,7 @@ description = "check substring ranges" name = "indexOf11" case_insensitive = true query = ''' -file where opcode=0 and indexOf(file_name, 'explorer.', 0) == 0 +file where opcode=0 and indexOf(file_name, "explorer.", 0) == 0 ''' expected_event_ids = [88, 92] description = "check substring ranges" @@ -1649,7 +1649,7 @@ description = "check substring ranges" name = "substring1" case_insensitive = true query = ''' -file where serial_event_id=88 and substring(file_name, 0, 4) == 'expl' +file where serial_event_id=88 and substring(file_name, 0, 4) == "expl" ''' expected_event_ids = [88] description = "check substring ranges" @@ -1658,7 +1658,7 @@ description = "check substring ranges" name = "substring2" case_sensitive = true query = ''' -file where substring(file_name, 1, 3) == 'xp' +file where substring(file_name, 1, 3) == "xp" ''' expected_event_ids = [88, 91] description = "check substring ranges" @@ -1667,7 +1667,7 @@ description = "check substring ranges" name = "substring3" case_insensitive = true query = ''' -file where substring(file_name, 1, 3) == 'xp' +file where substring(file_name, 1, 3) == "xp" ''' expected_event_ids = [88, 91, 92] description = "check substring ranges" @@ -1675,7 +1675,7 @@ description = "check substring ranges" [[queries]] name = "substring4" query = ''' -file where substring(file_name, -4) == '.exe' +file where substring(file_name, -4) == ".exe" ''' expected_event_ids = [55, 59, 61, 65, 67, 70, 72, 75, 76, 81, 83, 86, 88, 91] description = "check substring ranges" @@ -1683,7 +1683,7 @@ description = "check substring ranges" [[queries]] name = "substring5" query = ''' -file where substring(file_name, -4, -1) == '.ex' +file where substring(file_name, -4, -1) == ".ex" ''' expected_event_ids = [55, 59, 61, 65, 67, 70, 72, 75, 76, 81, 83, 86, 88, 91] description = "check substring ranges" @@ -1723,7 +1723,7 @@ description = "test built-in math functions" [[queries]] name = "stringNumberConversion1" query = ''' -process where serial_event_id == number('5') +process where serial_event_id == number("5") ''' expected_event_ids = [5] description = "test string/number conversions" @@ -1733,7 +1733,7 @@ name = "stringNumberConversion2" expected_event_ids = [50] description = "test string/number conversions" query = ''' -process where serial_event_id == number('0x32', 16) +process where serial_event_id == number("0x32", 16) ''' [[queries]] @@ -1741,13 +1741,13 @@ name = "stringNumberConversion3" expected_event_ids = [50] description = "test string/number conversions" query = ''' -process where serial_event_id == number('32', 16) +process where serial_event_id == number("32", 16) ''' [[queries]] name = "concat1" query = ''' -process where concat(serial_event_id, ':', process_name, opcode) == '5:wininit.exe3' +process where concat(serial_event_id, ":", process_name, opcode) == "5:wininit.exe3" ''' expected_event_ids = [5] description = "test string concatenation" @@ -1756,7 +1756,7 @@ description = "test string concatenation" name = "concatCaseInsensitive" case_insensitive = true query = ''' -process where concat(serial_event_id, ':', process_name, opcode) == '5:winINIT.exe3' +process where concat(serial_event_id, ":", process_name, opcode) == "5:winINIT.exe3" ''' expected_event_ids = [5] description = "test string concatenation" @@ -1776,7 +1776,7 @@ name = "arraySearch1" expected_event_ids = [57] description = "test arraySearch functionality for lists of strings, and lists of objects" query = ''' -registry where arraySearch(bytes_written_string_list, a, a == 'en-US') +registry where arraySearch(bytes_written_string_list, a, a == "en-US") ''' [[queries]] @@ -1785,7 +1785,7 @@ case_sensitive = true expected_event_ids = [] description = "test arraySearch functionality for lists of strings, and lists of objects" query = ''' -registry where arraySearch(bytes_written_string_list, a, a == 'EN-US') +registry where arraySearch(bytes_written_string_list, a, a == "EN-US") ''' [[queries]] @@ -1794,7 +1794,7 @@ case_insensitive = true expected_event_ids = [57] description = "test arraySearch functionality for lists of strings, and lists of objects" query = ''' -registry where arraySearch(bytes_written_string_list, a, a == 'en-us') +registry where arraySearch(bytes_written_string_list, a, a == "en-us") ''' [[queries]] @@ -1803,7 +1803,7 @@ case_insensitive = true expected_event_ids = [57] description = "test arraySearch functionality for lists of strings, and lists of objects" query = ''' -registry where arraySearch(bytes_written_string_list, a, endsWith(a, '-us')) +registry where arraySearch(bytes_written_string_list, a, endsWith(a, "-us")) ''' [[queries]] @@ -1828,7 +1828,7 @@ name = "arraySearchWithMysteriousField3" expected_event_ids = [75305] description = "test arraySearch - conditional" query = ''' -network where mysterious_field and arraySearch(mysterious_field.subarray, s, s.a == 's0-*') +network where mysterious_field and arraySearch(mysterious_field.subarray, s, s.a == "s0-*") ''' [[queries]] @@ -1836,7 +1836,7 @@ name = "arraySearchWithMysteriousField4" expected_event_ids = [75305] description = "test arraySearch - conditional" query = ''' -network where mysterious_field and arraySearch(mysterious_field.subarray, s, s.a != 's0-*') +network where mysterious_field and arraySearch(mysterious_field.subarray, s, s.a != "s0-*") ''' [[queries]] @@ -1846,7 +1846,7 @@ description = "test arraySearch - nested" query = ''' network where mysterious_field and arraySearch(mysterious_field.subarray, sub1, - arraySearch(sub1.c, nested, nested.x.y == '*')) + arraySearch(sub1.c, nested, nested.x.y == "*")) ''' [[queries]] @@ -1856,7 +1856,7 @@ description = "test arraySearch - nested with cross-check pass" query = ''' network where mysterious_field and arraySearch(mysterious_field.subarray, sub1, - sub1.a == 's0-a' and arraySearch(sub1.c, nested, nested.z == 's0-c1-x-z')) + sub1.a == "s0-a" and arraySearch(sub1.c, nested, nested.z == "s0-c1-x-z")) ''' [[queries]] @@ -1866,7 +1866,7 @@ description = "test arraySearch - nested with cross-check pass" query = ''' network where mysterious_field and arraySearch(mysterious_field.subarray, sub1, - sub1.a == 's0-a' and arraySearch(sub1.c, nested, nested.z == sub1.cross_match)) + sub1.a == "s0-a" and arraySearch(sub1.c, nested, nested.z == sub1.cross_match)) ''' [[queries]] @@ -1891,7 +1891,7 @@ network where safe(divide(process_name, process_name)) name = "nestedSetComparisons" case_insensitive = true query = ''' -file where serial_event_id == 82 and (true == (process_name in ('svchost.EXE', 'bad.exe', 'bad2.exe'))) +file where serial_event_id == 82 and (true == (process_name in ("svchost.EXE", "bad.exe", "bad2.exe"))) ''' expected_event_ids = [82] description = "nested set comparisons" @@ -1901,7 +1901,7 @@ name = "arrayCount1" case_insensitive = true expected_event_ids = [57] query = ''' -registry where arrayCount(bytes_written_string_list, s, s == '*-us') == 1 +registry where arrayCount(bytes_written_string_list, s, s == "*-us") == 1 ''' [[queries]] @@ -1909,7 +1909,7 @@ name = "arrayCount2" case_insensitive = true expected_event_ids = [57] query = ''' -registry where arrayCount(bytes_written_string_list, s, s == '*en*') == 2 +registry where arrayCount(bytes_written_string_list, s, s == "*en*") == 2 ''' [[queries]] @@ -2037,7 +2037,7 @@ name = "lengthCaseSensitive" case_sensitive = true expected_event_ids = [7, 14, 29, 44] query = ''' -process where length(between(process_name, 'g', 'e')) > 0 +process where length(between(process_name, "g", "e")) > 0 ''' [[queries]] @@ -2045,12 +2045,12 @@ name = "lengthCaseInsensitiveAndBetween" case_insensitive = true expected_event_ids = [7, 14, 22, 29, 44] query = ''' -process where length(between(process_name, 'g', 'e')) > 0 +process where length(between(process_name, "g", "e")) > 0 ''' [[queries]] name = "length1" expected_event_ids = [] query = ''' -process where length(between(process_name, 'g', 'z')) > 0 +process where length(between(process_name, "g", "z")) > 0 ''' diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml index f307d8dc4c2d1..bcfa2a9549b16 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml @@ -719,7 +719,7 @@ name = "arrayContainsCaseInsensitive1" case_insensitive = true expected_event_ids = [57] query = ''' -registry where arrayContains(bytes_written_string_list, 'En-uS') +registry where arrayContains(bytes_written_string_list, "En-uS") ''' [[queries]] @@ -727,7 +727,7 @@ name = "arrayContainsCaseInsensitive2" case_insensitive = true expected_event_ids = [57] query = ''' -registry where arrayContains(bytes_written_string_list, 'En') +registry where arrayContains(bytes_written_string_list, "En") ''' [[queries]] @@ -735,7 +735,7 @@ name = "lengthCaseInsensitive" case_insensitive = true expected_event_ids = [57] query = ''' -registry where length(bytes_written_string_list) > 0 and bytes_written_string_list[0] == 'EN-us' +registry where length(bytes_written_string_list) > 0 and bytes_written_string_list[0] == "EN-us" ''' [[queries]] @@ -743,7 +743,7 @@ name = "arrayCaseInsensitive1" case_insensitive = true expected_event_ids = [57] query = ''' -registry where bytes_written_string_list[0] == 'EN-us' +registry where bytes_written_string_list[0] == "EN-us" ''' [[queries]] @@ -751,7 +751,7 @@ name = "arrayCaseInsensitive2" case_insensitive = true expected_event_ids = [57] query = ''' -registry where bytes_written_string_list[1] == 'EN' +registry where bytes_written_string_list[1] == "EN" ''' @@ -760,7 +760,7 @@ name = "arrayContainsCaseInsensitive3" case_insensitive = true expected_event_ids = [57] query = ''' -registry where arrayContains(bytes_written_string_list, 'en-US') +registry where arrayContains(bytes_written_string_list, "en-US") ''' [[queries]] @@ -768,7 +768,7 @@ name = "arrayContainsCaseInsensitive4" case_insensitive = true expected_event_ids = [57] query = ''' -registry where arrayContains(bytes_written_string_list, 'en') +registry where arrayContains(bytes_written_string_list, "en") ''' [[queries]] @@ -776,42 +776,42 @@ name = "arrayCaseInsensitive3" case_insensitive = true expected_event_ids = [57] query = ''' -registry where length(bytes_written_string_list) > 0 and bytes_written_string_list[0] == 'en-US' +registry where length(bytes_written_string_list) > 0 and bytes_written_string_list[0] == "en-US" ''' [[queries]] name = "arrayEquality1" expected_event_ids = [57] query = ''' -registry where bytes_written_string_list[0] == 'en-US' +registry where bytes_written_string_list[0] == "en-US" ''' [[queries]] name = "arrayEquality2" expected_event_ids = [57] query = ''' -registry where bytes_written_string_list[1] == 'en' +registry where bytes_written_string_list[1] == "en" ''' # character classes aren't supported. custom tests made in test_queries_supported.toml [[queries]] name = "matchLite1" query = ''' -process where matchLite(command_line, ?'.*?net1\s+localgroup\s+.*?') +process where matchLite(command_line, ?".*?net1\s+localgroup\s+.*?") ''' expected_event_ids = [98] [[queries]] name = "matchLite2" query = ''' -process where matchLite(command_line, ?'.*?net1\s+\w+\s+.*?') +process where matchLite(command_line, ?".*?net1\s+\w+\s+.*?") ''' expected_event_ids = [98] [[queries]] name = "matchLite3" query = ''' -process where matchLite(command_line, ?'.*?net1\s+\w{4,15}\s+.*?') +process where matchLite(command_line, ?".*?net1\s+\w{4,15}\s+.*?") ''' expected_event_ids = [98] @@ -819,13 +819,13 @@ expected_event_ids = [98] name = "match1" expected_event_ids = [98] query = ''' -process where match(command_line, ?'.*?net1\s+\w{4,15}\s+.*?') +process where match(command_line, ?".*?net1\s+\w{4,15}\s+.*?") ''' [[queries]] name = "matchLite4" query = ''' -process where matchLite(command_line, ?'.*?net1\s+[localgrup]{4,15}\s+.*?') +process where matchLite(command_line, ?".*?net1\s+[localgrup]{4,15}\s+.*?") ''' expected_event_ids = [98] @@ -833,7 +833,7 @@ expected_event_ids = [98] name = "stringEqualsCaseInsensitive1" case_insensitive = true query = ''' -process where 'net.EXE' == original_file_name +process where "net.EXE" == original_file_name | filter process_name="net*.exe" ''' expected_event_ids = [97] @@ -843,7 +843,7 @@ note = "check that case insensitive comparisons are performed even for lhs strin name = "stringEqualsCaseInsensitive2" case_insensitive = true query = ''' -process where process_name == original_file_name and process_name='net*.exe' +process where process_name == original_file_name and process_name="net*.exe" ''' expected_event_ids = [97, 98] note = "check that case insensitive comparisons are performed for fields." @@ -861,7 +861,7 @@ description = "check that case insensitive comparisons are performed for fields. name = "substring3" case_insensitive = true query = ''' -file where substring(file_name, 1, 3) == 'xp' +file where substring(file_name, 1, 3) == "xp" ''' expected_event_ids = [88, 91, 92] description = "check substring ranges" @@ -878,7 +878,7 @@ description = "test built-in math functions" name = "concatCaseInsensitive" case_insensitive = true query = ''' -process where concat(serial_event_id, ':', process_name, opcode) == '5:winINIT.exe3' +process where concat(serial_event_id, ":", process_name, opcode) == "5:winINIT.exe3" ''' expected_event_ids = [5] description = "test string concatenation" @@ -898,7 +898,7 @@ name = "arraySearch1" expected_event_ids = [57] description = "test arraySearch functionality for lists of strings, and lists of objects" query = ''' -registry where arraySearch(bytes_written_string_list, a, a == 'en-US') +registry where arraySearch(bytes_written_string_list, a, a == "en-US") ''' [[queries]] @@ -907,7 +907,7 @@ case_sensitive = true expected_event_ids = [] description = "test arraySearch functionality for lists of strings, and lists of objects" query = ''' -registry where arraySearch(bytes_written_string_list, a, a == 'EN-US') +registry where arraySearch(bytes_written_string_list, a, a == "EN-US") ''' [[queries]] @@ -916,7 +916,7 @@ case_insensitive = true expected_event_ids = [57] description = "test arraySearch functionality for lists of strings, and lists of objects" query = ''' -registry where arraySearch(bytes_written_string_list, a, a == 'en-us') +registry where arraySearch(bytes_written_string_list, a, a == "en-us") ''' [[queries]] @@ -925,7 +925,7 @@ case_insensitive = true expected_event_ids = [57] description = "test arraySearch functionality for lists of strings, and lists of objects" query = ''' -registry where arraySearch(bytes_written_string_list, a, endsWith(a, '-us')) +registry where arraySearch(bytes_written_string_list, a, endsWith(a, "-us")) ''' [[queries]] @@ -950,7 +950,7 @@ name = "arraySearchWithMysteriousField3" expected_event_ids = [75305] description = "test arraySearch - conditional" query = ''' -network where mysterious_field and arraySearch(mysterious_field.subarray, s, s.a == 's0-*') +network where mysterious_field and arraySearch(mysterious_field.subarray, s, s.a == "s0-*") ''' [[queries]] @@ -958,7 +958,7 @@ name = "arraySearchWithMysteriousField4" expected_event_ids = [75305] description = "test arraySearch - conditional" query = ''' -network where mysterious_field and arraySearch(mysterious_field.subarray, s, s.a != 's0-*') +network where mysterious_field and arraySearch(mysterious_field.subarray, s, s.a != "s0-*") ''' [[queries]] @@ -968,7 +968,7 @@ description = "test arraySearch - nested" query = ''' network where mysterious_field and arraySearch(mysterious_field.subarray, sub1, - arraySearch(sub1.c, nested, nested.x.y == '*')) + arraySearch(sub1.c, nested, nested.x.y == "*")) ''' [[queries]] @@ -978,7 +978,7 @@ description = "test arraySearch - nested with cross-check pass" query = ''' network where mysterious_field and arraySearch(mysterious_field.subarray, sub1, - sub1.a == 's0-a' and arraySearch(sub1.c, nested, nested.z == 's0-c1-x-z')) + sub1.a == "s0-a" and arraySearch(sub1.c, nested, nested.z == "s0-c1-x-z")) ''' [[queries]] @@ -988,7 +988,7 @@ description = "test arraySearch - nested with cross-check pass" query = ''' network where mysterious_field and arraySearch(mysterious_field.subarray, sub1, - sub1.a == 's0-a' and arraySearch(sub1.c, nested, nested.z == sub1.cross_match)) + sub1.a == "s0-a" and arraySearch(sub1.c, nested, nested.z == sub1.cross_match)) ''' [[queries]] @@ -1013,7 +1013,7 @@ network where safe(divide(process_name, process_name)) name = "nestedSetComparisons" case_insensitive = true query = ''' -file where serial_event_id == 82 and (true == (process_name in ('svchost.EXE', 'bad.exe', 'bad2.exe'))) +file where serial_event_id == 82 and (true == (process_name in ("svchost.EXE", "bad.exe", "bad2.exe"))) ''' expected_event_ids = [82] description = "nested set comparisons" @@ -1023,7 +1023,7 @@ name = "arrayCount1" case_insensitive = true expected_event_ids = [57] query = ''' -registry where arrayCount(bytes_written_string_list, s, s == '*-us') == 1 +registry where arrayCount(bytes_written_string_list, s, s == "*-us") == 1 ''' [[queries]] @@ -1031,7 +1031,7 @@ name = "arrayCount2" case_insensitive = true expected_event_ids = [57] query = ''' -registry where arrayCount(bytes_written_string_list, s, s == '*en*') == 2 +registry where arrayCount(bytes_written_string_list, s, s == "*en*") == 2 ''' [[queries]] @@ -1081,7 +1081,7 @@ name = "lengthCaseSensitive" case_sensitive = true expected_event_ids = [7, 14, 29, 44] query = ''' -process where length(between(process_name, 'g', 'e')) > 0 +process where length(between(process_name, "g", "e")) > 0 ''' # TODO: add toggles to this function so it's not always insensitive @@ -1090,6 +1090,6 @@ process where length(between(process_name, 'g', 'e')) > 0 #case_insensitive = true #expected_event_ids = [7, 14, 22, 29, 44] #query = ''' -#process where length(between(process_name, 'g', 'e')) > 0 +#process where length(between(process_name, "g", "e")) > 0 #''' diff --git a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml index 120ddb33c41f5..fd74fa282cc9b 100644 --- a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml +++ b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml @@ -39,7 +39,7 @@ setup: eql.search: index: eql_test body: - query: "process where user = 'SYSTEM'" + query: 'process where user = "SYSTEM"' - match: {timed_out: false} - match: {hits.total.value: 3} @@ -55,7 +55,7 @@ setup: eql.search: index: eql_test body: - query: "sequence by user [process where user = 'SYSTEM'] [process where true]" + query: 'sequence by user [process where user = "SYSTEM"] [process where true]' - match: {timed_out: false} - match: {hits.total.value: 2} - match: {hits.total.relation: "eq"} @@ -72,7 +72,7 @@ setup: eql.search: index: eql_test body: - query: "sequence by id [process where user = 'SYSTEM'] [process where true]" + query: 'sequence by id [process where user = "SYSTEM"] [process where true]' - match: {timed_out: false} - match: {hits.total.value: 2} - match: {hits.total.relation: "eq"} @@ -89,7 +89,7 @@ setup: eql.search: index: eql_test body: - query: "sequence by valid [process where user = 'SYSTEM'] [process where true]" + query: 'sequence by valid [process where user = "SYSTEM"] [process where true]' - match: {timed_out: false} - match: {hits.total.value: 1} - match: {hits.total.relation: "eq"} @@ -105,7 +105,7 @@ setup: wait_for_completion_timeout: "0ms" keep_on_completion: true body: - query: "process where user = 'SYSTEM'" + query: 'process where user = "SYSTEM"' - is_true: id - set: {id: id} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AbstractBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AbstractBuilder.java index c2baf39929313..0293313cc432e 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AbstractBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AbstractBuilder.java @@ -115,26 +115,30 @@ static String text(ParseTree node) { return node == null ? null : node.getText(); } - public static String unquoteString(String text) { + public static String unquoteString(Source source) { // remove leading and trailing ' for strings and also eliminate escaped single quotes + String text = source.text(); if (text == null) { return null; } // unescaped strings can be interpreted directly if (text.startsWith("?")) { + checkForSingleQuotedString(source, text, 1); return text.substring(2, text.length() - 1); } + checkForSingleQuotedString(source, text, 0); + text = text.substring(1, text.length() - 1); StringBuffer resultString = new StringBuffer(); Matcher regexMatcher = slashPattern.matcher(text); while (regexMatcher.find()) { - String source = regexMatcher.group(); + String group = regexMatcher.group(); String replacement; - switch (source) { + switch (group) { case "\\t": replacement = "\t"; break; @@ -162,7 +166,7 @@ public static String unquoteString(String text) { break; default: // unknown escape sequence, pass through as-is - replacement = source; + replacement = group; } regexMatcher.appendReplacement(resultString, replacement); @@ -173,6 +177,13 @@ public static String unquoteString(String text) { return resultString.toString(); } + private static void checkForSingleQuotedString(Source source, String text, int i) { + if (text.charAt(i) == '\'') { + throw new ParsingException(source, + "Single quotes ['] are not supported around literal strings, please use double quotes [\"]"); + } + } + @Override public Object visitTerminal(TerminalNode node) { Source source = source(node); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java index 66612d7b347f6..8e7edf62bcdd4 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java @@ -253,6 +253,7 @@ public Expression visitParenthesizedExpression(EqlBaseParser.ParenthesizedExpres @Override public Literal visitString(EqlBaseParser.StringContext ctx) { - return new Literal(source(ctx), unquoteString(ctx.getText()), DataTypes.KEYWORD); + Source source = source(ctx); + return new Literal(source, unquoteString(source), DataTypes.KEYWORD); } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java index 6f33fe961246b..df7d772158da2 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java @@ -73,7 +73,7 @@ public void testBasicQuery() { public void testQueryStartsWithNumber() { assertEquals("1:1: no viable alternative at input '42'", errorParsing("42 where true")); - assertEquals("1:1: no viable alternative at input ''42''", errorParsing("'42' where true")); + assertEquals("1:1: no viable alternative at input '\"42\"'", errorParsing("\"42\" where true")); } public void testMissingColumn() { @@ -100,7 +100,7 @@ public void testProcessRelationshipsUnsupported() { // Some functions fail with "Unsupported" message at the parse stage public void testArrayFunctionsUnsupported() { assertEquals("1:16: Unknown function [arrayContains], did you mean [stringcontains]?", - error("registry where arrayContains(bytes_written_string_list, 'En')")); + error("registry where arrayContains(bytes_written_string_list, \"En\")")); assertEquals("1:16: Unknown function [arraySearch]", error("registry where arraySearch(bytes_written_string_list, bytes_written_string, true)")); assertEquals("1:16: Unknown function [arrayCount]", @@ -116,13 +116,13 @@ public void testFunctionParsingUnknown() { // Test unsupported array indexes public void testArrayIndexesUnsupported() { assertEquals("1:84: Array indexes are not supported", - errorParsing("registry where length(bytes_written_string_list) > 0 and bytes_written_string_list[0] == 'EN-us")); + errorParsing("registry where length(bytes_written_string_list) > 0 and bytes_written_string_list[0] == \"EN-us")); } // Test valid/supported queries public void testQueryOk() { // Mismatched type, still ok - accept("process where serial_event_id = 'abcdef'"); + accept("process where serial_event_id = \"abcdef\""); // Equals condition accept("process where serial_event_id = 1"); @@ -143,7 +143,7 @@ public void testQueryOk() { " and serial_event_id in (58, 64, 69, 74, 80, 85, 90, 93, 94)"); // Combination - accept("file where serial_event_id == 82 and (true == (process_name in ('svchost.EXE', 'bad.exe', 'bad2.exe')))"); + accept("file where serial_event_id == 82 and (true == (process_name in (\"svchost.EXE\", \"bad.exe\", \"bad2.exe\")))"); // String handling accept("process where process_path == \"*\\\\MACHINE\\\\SAM\\\\SAM\\\\*\\\\Account\\\\Us*ers\\\\00*03E9\\\\F\""); @@ -156,7 +156,7 @@ public void testQueryOk() { accept("file where serial_event_id % 40 == 2"); } - // Test mapping that doesn't have property event.category defined + // Test mapping that doesn\"t have property event.category defined public void testMissingEventCategory() { final IndexResolution idxr = loadIndexResolution("mapping-missing-event-category.json"); assertEquals("1:1: Unknown column [event.category]", error(idxr, "foo where true")); @@ -167,11 +167,11 @@ public void testAliasErrors() { // Check unsupported assertEquals("1:11: Cannot use field [user_name_alias] with unsupported type [alias]", - error(idxr, "foo where user_name_alias == 'bob'")); + error(idxr, "foo where user_name_alias == \"bob\"")); // Check alias name typo assertEquals("1:11: Unknown column [user_name_alia], did you mean any of [user_name, user_domain]?", - error(idxr, "foo where user_name_alia == 'bob'")); + error(idxr, "foo where user_name_alia == \"bob\"")); } // Test all elasticsearch numeric field types @@ -193,45 +193,45 @@ public void testNumeric() { public void testNoDoc() { final IndexResolution idxr = loadIndexResolution("mapping-nodoc.json"); - accept(idxr, "foo where description_nodoc == ''"); + accept(idxr, "foo where description_nodoc == \"\""); // TODO: add sort test on nodoc field once we have pipes support } public void testDate() { final IndexResolution idxr = loadIndexResolution("mapping-date.json"); - accept(idxr, "foo where date == ''"); - accept(idxr, "foo where date == '2020-02-02'"); - accept(idxr, "foo where date == '2020-02-41'"); - accept(idxr, "foo where date == '20200241'"); - - accept(idxr, "foo where date_with_format == ''"); - accept(idxr, "foo where date_with_format == '2020-02-02'"); - accept(idxr, "foo where date_with_format == '2020-02-41'"); - accept(idxr, "foo where date_with_format == '20200241'"); - - accept(idxr, "foo where date_with_multi_format == ''"); - accept(idxr, "foo where date_with_multi_format == '2020-02-02'"); - accept(idxr, "foo where date_with_multi_format == '2020-02-41'"); - accept(idxr, "foo where date_with_multi_format == '20200241'"); - accept(idxr, "foo where date_with_multi_format == '11:12:13'"); + accept(idxr, "foo where date == \"\""); + accept(idxr, "foo where date == \"2020-02-02\""); + accept(idxr, "foo where date == \"2020-02-41\""); + accept(idxr, "foo where date == \"20200241\""); + + accept(idxr, "foo where date_with_format == \"\""); + accept(idxr, "foo where date_with_format == \"2020-02-02\""); + accept(idxr, "foo where date_with_format == \"2020-02-41\""); + accept(idxr, "foo where date_with_format == \"20200241\""); + + accept(idxr, "foo where date_with_multi_format == \"\""); + accept(idxr, "foo where date_with_multi_format == \"2020-02-02\""); + accept(idxr, "foo where date_with_multi_format == \"2020-02-41\""); + accept(idxr, "foo where date_with_multi_format == \"20200241\""); + accept(idxr, "foo where date_with_multi_format == \"11:12:13\""); // Test query against unsupported field type date_nanos assertEquals("1:11: Cannot use field [date_nanos_field] with unsupported type [date_nanos]", - error(idxr, "foo where date_nanos_field == ''")); + error(idxr, "foo where date_nanos_field == \"\"")); } public void testBoolean() { final IndexResolution idxr = loadIndexResolution("mapping-boolean.json"); accept(idxr, "foo where boolean_field == true"); - accept(idxr, "foo where boolean_field == 'bar'"); + accept(idxr, "foo where boolean_field == \"bar\""); accept(idxr, "foo where boolean_field == 0"); accept(idxr, "foo where boolean_field == 123456"); } public void testBinary() { final IndexResolution idxr = loadIndexResolution("mapping-binary.json"); - accept(idxr, "foo where blob == ''"); - accept(idxr, "foo where blob == 'bar'"); + accept(idxr, "foo where blob == \"\""); + accept(idxr, "foo where blob == \"bar\""); accept(idxr, "foo where blob == 0"); accept(idxr, "foo where blob == 123456"); } @@ -239,23 +239,23 @@ public void testBinary() { public void testRange() { final IndexResolution idxr = loadIndexResolution("mapping-range.json"); assertEquals("1:11: Cannot use field [integer_range_field] with unsupported type [integer_range]", - error(idxr, "foo where integer_range_field == ''")); + error(idxr, "foo where integer_range_field == \"\"")); assertEquals("1:11: Cannot use field [float_range_field] with unsupported type [float_range]", - error(idxr, "foo where float_range_field == ''")); + error(idxr, "foo where float_range_field == \"\"")); assertEquals("1:11: Cannot use field [long_range_field] with unsupported type [long_range]", - error(idxr, "foo where long_range_field == ''")); + error(idxr, "foo where long_range_field == \"\"")); assertEquals("1:11: Cannot use field [double_range_field] with unsupported type [double_range]", - error(idxr, "foo where double_range_field == ''")); + error(idxr, "foo where double_range_field == \"\"")); assertEquals("1:11: Cannot use field [date_range_field] with unsupported type [date_range]", - error(idxr, "foo where date_range_field == ''")); + error(idxr, "foo where date_range_field == \"\"")); assertEquals("1:11: Cannot use field [ip_range_field] with unsupported type [ip_range]", - error(idxr, "foo where ip_range_field == ''")); + error(idxr, "foo where ip_range_field == \"\"")); } public void testMixedSet() { final IndexResolution idxr = loadIndexResolution("mapping-numeric.json"); - assertEquals("1:11: 2nd argument of [long_field in (1, 'string')] must be [long], found value ['string'] type [keyword]", - error(idxr, "foo where long_field in (1, 'string')")); + assertEquals("1:11: 2nd argument of [long_field in (1, \"string\")] must be [long], found value [\"string\"] type [keyword]", + error(idxr, "foo where long_field in (1, \"string\")")); } public void testObject() { @@ -297,32 +297,32 @@ public void testJoin() { public void testMultiField() { final IndexResolution idxr = loadIndexResolution("mapping-multi-field.json"); - accept(idxr, "foo where multi_field.raw == 'bar'"); + accept(idxr, "foo where multi_field.raw == \"bar\""); - assertEquals("1:11: [multi_field.english == 'bar'] cannot operate on first argument field of data type [text]: " + + assertEquals("1:11: [multi_field.english == \"bar\"] cannot operate on first argument field of data type [text]: " + "No keyword/multi-field defined exact matches for [english]; define one or use MATCH/QUERY instead", - error(idxr, "foo where multi_field.english == 'bar'")); + error(idxr, "foo where multi_field.english == \"bar\"")); - accept(idxr, "foo where multi_field_options.raw == 'bar'"); - accept(idxr, "foo where multi_field_options.key == 'bar'"); - accept(idxr, "foo where multi_field_ambiguous.one == 'bar'"); - accept(idxr, "foo where multi_field_ambiguous.two == 'bar'"); + accept(idxr, "foo where multi_field_options.raw == \"bar\""); + accept(idxr, "foo where multi_field_options.key == \"bar\""); + accept(idxr, "foo where multi_field_ambiguous.one == \"bar\""); + accept(idxr, "foo where multi_field_ambiguous.two == \"bar\""); - assertEquals("1:11: [multi_field_ambiguous.normalized == 'bar'] cannot operate on first argument field of data type [keyword]: " + + assertEquals("1:11: [multi_field_ambiguous.normalized == \"bar\"] cannot operate on first argument field of data type [keyword]: " + "Normalized keyword field cannot be used for exact match operations", - error(idxr, "foo where multi_field_ambiguous.normalized == 'bar'")); + error(idxr, "foo where multi_field_ambiguous.normalized == \"bar\"")); assertEquals("1:11: Cannot use field [multi_field_nested.dep_name] type [text] with unsupported nested type in hierarchy " + "(field [multi_field_nested])", - error(idxr, "foo where multi_field_nested.dep_name == 'bar'")); + error(idxr, "foo where multi_field_nested.dep_name == \"bar\"")); assertEquals("1:11: Cannot use field [multi_field_nested.dep_id.keyword] type [keyword] with unsupported nested type in " + "hierarchy (field [multi_field_nested])", - error(idxr, "foo where multi_field_nested.dep_id.keyword == 'bar'")); + error(idxr, "foo where multi_field_nested.dep_id.keyword == \"bar\"")); assertEquals("1:11: Cannot use field [multi_field_nested.end_date] type [datetime] with unsupported nested type in " + "hierarchy (field [multi_field_nested])", - error(idxr, "foo where multi_field_nested.end_date == ''")); + error(idxr, "foo where multi_field_nested.end_date == \"\"")); assertEquals("1:11: Cannot use field [multi_field_nested.start_date] type [datetime] with unsupported nested type in " + "hierarchy (field [multi_field_nested])", - error(idxr, "foo where multi_field_nested.start_date == 'bar'")); + error(idxr, "foo where multi_field_nested.start_date == \"bar\"")); } public void testStringFunctionWithText() { @@ -330,6 +330,6 @@ public void testStringFunctionWithText() { assertEquals("1:15: [string(multi_field.english)] cannot operate on field " + "of data type [text]: No keyword/multi-field defined exact matches for [english]; " + "define one or use MATCH/QUERY instead", - error(idxr, "process where string(multi_field.english) == 'foo'")); + error(idxr, "process where string(multi_field.english) == \"foo\"")); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java index 6516e0501b973..940c98af4a145 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java @@ -135,8 +135,8 @@ public void testIsNotNull() { public void testEqualsWildcard() { List tests = Arrays.asList( - "foo where command_line == '* bar *'", - "foo where '* bar *' == command_line" + "foo where command_line == \"* bar *\"", + "foo where \"* bar *\" == command_line" ); for (String q : tests) { @@ -157,8 +157,8 @@ public void testEqualsWildcard() { public void testNotEqualsWildcard() { List tests = Arrays.asList( - "foo where command_line != '* baz *'", - "foo where '* baz *' != command_line" + "foo where command_line != \"* baz *\"", + "foo where \"* baz *\" != command_line" ); for (String q : tests) { @@ -180,7 +180,7 @@ public void testNotEqualsWildcard() { } public void testWildcardEscapes() { - LogicalPlan plan = defaultPipes(accept("foo where command_line == '* %bar_ * \\\\ \\n \\r \\t'")); + LogicalPlan plan = defaultPipes(accept("foo where command_line == \"* %bar_ * \\\\ \\n \\r \\t\"")); assertTrue(plan instanceof Filter); Filter filter = (Filter) plan; diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java index 51326dc97198f..d1e8d690640a5 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; import java.util.ArrayList; @@ -40,6 +41,10 @@ public class ExpressionTests extends ESTestCase { private final EqlParser parser = new EqlParser(); + private static Source source(String text) { + return new Source(0, 0, text); + } + private Expression expr(String source) { return parser.createExpression(source); } @@ -52,22 +57,21 @@ private List exprs(String... sources) { return results; } - public void testStrings() { - assertEquals("hello\"world", unquoteString("'hello\"world'")); - assertEquals("hello'world", unquoteString("\"hello'world\"")); - assertEquals("hello\nworld", unquoteString("'hello\\nworld'")); - assertEquals("hello\\\nworld", unquoteString("'hello\\\\\\nworld'")); - assertEquals("hello\\\"world", unquoteString("'hello\\\\\\\"world'")); - - // test for unescaped strings: ?"...." or ?'....' - assertEquals("hello\"world", unquoteString("?'hello\"world'")); - assertEquals("hello\\\"world", unquoteString("?'hello\\\"world'")); - assertEquals("hello'world", unquoteString("?\"hello'world\"")); - assertEquals("hello\\nworld", unquoteString("?'hello\\nworld'")); - assertEquals("hello\\\\nworld", unquoteString("?'hello\\\\nworld'")); - assertEquals("hello\\\\\\nworld", unquoteString("?'hello\\\\\\nworld'")); - assertEquals("hello\\\\\\\"world", unquoteString("?'hello\\\\\\\"world'")); + assertEquals("hello\"world", unquoteString(source("\"hello\"world\""))); + assertEquals("hello'world", unquoteString(source("\"hello'world\""))); + assertEquals("hello\nworld", unquoteString(source("\"hello\\nworld\""))); + assertEquals("hello\\\nworld", unquoteString(source("\"hello\\\\\\nworld\""))); + assertEquals("hello\\\"world", unquoteString(source("\"hello\\\\\\\"world\""))); + + // test for unescaped strings: ?"...." + assertEquals("hello\"world", unquoteString(source("?\"hello\"world\""))); + assertEquals("hello\\\"world", unquoteString(source("?\"hello\\\"world\""))); + assertEquals("hello'world", unquoteString(source("?\"hello'world\""))); + assertEquals("hello\\nworld", unquoteString(source("?\"hello\\nworld\""))); + assertEquals("hello\\\\nworld", unquoteString(source("?\"hello\\\\nworld\""))); + assertEquals("hello\\\\\\nworld", unquoteString(source("?\"hello\\\\\\nworld\""))); + assertEquals("hello\\\\\\\"world", unquoteString(source("?\"hello\\\\\\\"world\""))); } public void testLiterals() { @@ -76,11 +80,13 @@ public void testLiterals() { assertEquals(Literal.NULL, expr("null")); } - public void testSingleQuotedString() { - // "hello \" world" - Expression parsed = expr("'hello \\' world!'"); - Expression expected = new Literal(null, "hello ' world!", DataTypes.KEYWORD); - assertEquals(expected, parsed); + public void testSingleQuotedStringForbidden() { + ParsingException e = expectThrows(ParsingException.class, () -> expr("'hello world'")); + assertEquals("line 1:2: Single quotes ['] are not supported around literal strings, please use double quotes [\"]", + e.getMessage()); + e = expectThrows(ParsingException.class, () -> parser.createStatement("process where name='hello world'")); + assertEquals("line 1:21: Single quotes ['] are not supported around literal strings, please use double quotes [\"]", + e.getMessage()); } public void testDoubleQuotedString() { @@ -90,11 +96,13 @@ public void testDoubleQuotedString() { assertEquals(expected, parsed); } - public void testSingleQuotedUnescapedString() { - // "hello \" world" - Expression parsed = expr("?'hello \\' world!'"); - Expression expected = new Literal(null, "hello \\' world!", DataTypes.KEYWORD); - assertEquals(expected, parsed); + public void testSingleQuotedUnescapedStringForbidden() { + ParsingException e = expectThrows(ParsingException.class, () -> expr("?'hello world'")); + assertEquals("line 1:2: Single quotes ['] are not supported around literal strings, please use double quotes [\"]", + e.getMessage()); + e = expectThrows(ParsingException.class, () -> parser.createStatement("process where name=?'hello world'")); + assertEquals("line 1:21: Single quotes ['] are not supported around literal strings, please use double quotes [\"]", + e.getMessage()); } public void testDoubleQuotedUnescapedString() { @@ -135,7 +143,7 @@ public void testFunctions() { UnresolvedFunction.ResolutionType resolutionType = UnresolvedFunction.ResolutionType.STANDARD; Expression expected = new UnresolvedFunction(null, "concat", resolutionType, arguments); - assertEquals(expected, expr("concat(some.field, 'test string')")); + assertEquals(expected, expr("concat(some.field, \"test string\")")); } public void testComparison() { @@ -154,8 +162,8 @@ public void testComparison() { } public void testBoolean() { - String leftText = "process_name == 'net.exe'"; - String rightText = "command_line == '* localgroup*'"; + String leftText = "process_name == \"net.exe\""; + String rightText = "command_line == \"* localgroup*\""; Expression lhs = expr(leftText); Expression rhs = expr(rightText); @@ -178,13 +186,13 @@ public void testInSet() { new In(null, expr("name"), exprs("2", "1")) ); assertEquals( - expr("name in ('net.exe')"), - new In(null, expr("name"), exprs("'net.exe'")) + expr("name in (\"net.exe\")"), + new In(null, expr("name"), exprs("\"net.exe\"")) ); assertEquals( - expr("name in ('net.exe', 'whoami.exe', 'hostname.exe')"), - new In(null, expr("name"), exprs("'net.exe'", "'whoami.exe'", "'hostname.exe'")) + expr("name in (\"net.exe\", \"whoami.exe\", \"hostname.exe\")"), + new In(null, expr("name"), exprs("\"net.exe\"", "\"whoami.exe\"", "\"hostname.exe\"")) ); } @@ -195,17 +203,17 @@ public void testInSetDuplicates() { ); assertEquals( - expr("name in ('net.exe', 'net.exe')"), - new In(null, expr("name"), exprs("'net.exe'", "'net.exe'")) + expr("name in (\"net.exe\", \"net.exe\")"), + new In(null, expr("name"), exprs("\"net.exe\"", "\"net.exe\"")) ); } public void testNotInSet() { assertEquals( - expr("name not in ('net.exe', 'whoami.exe', 'hostname.exe')"), + expr("name not in (\"net.exe\", \"whoami.exe\", \"hostname.exe\")"), new Not(null, new In(null, expr("name"), - exprs("'net.exe'", "'whoami.exe'", "'hostname.exe'"))) + exprs("\"net.exe\"", "\"whoami.exe\"", "\"hostname.exe\""))) ); } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java index e28f6cc74df22..70be3009bb8e3 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java @@ -53,23 +53,23 @@ private static LogicalPlan relation() { } public void testAnyQuery() { - LogicalPlan fullQuery = parser.createStatement("any where process_name == 'net.exe'"); - Expression fullExpression = expr("process_name == 'net.exe'"); + LogicalPlan fullQuery = parser.createStatement("any where process_name == \"net.exe\""); + Expression fullExpression = expr("process_name == \"net.exe\""); assertEquals(wrapFilter(fullExpression), fullQuery); } public void testEventQuery() { - LogicalPlan fullQuery = parser.createStatement("process where process_name == 'net.exe'"); - Expression fullExpression = expr("event.category == 'process' and process_name == 'net.exe'"); + LogicalPlan fullQuery = parser.createStatement("process where process_name == \"net.exe\""); + Expression fullExpression = expr("event.category == \"process\" and process_name == \"net.exe\""); assertEquals(wrapFilter(fullExpression), fullQuery); } public void testParameterizedEventQuery() { ParserParams params = new ParserParams(UTC).fieldEventCategory("myCustomEvent"); - LogicalPlan fullQuery = parser.createStatement("process where process_name == 'net.exe'", params); - Expression fullExpression = expr("myCustomEvent == 'process' and process_name == 'net.exe'"); + LogicalPlan fullQuery = parser.createStatement("process where process_name == \"net.exe\"", params); + Expression fullExpression = expr("myCustomEvent == \"process\" and process_name == \"net.exe\""); assertEquals(wrapFilter(fullExpression), fullQuery); } @@ -150,4 +150,4 @@ private LogicalPlan defaultPipes(LogicalPlan plan) { assertTrue(plan instanceof OrderBy); return ((OrderBy) plan).child(); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java index 9d14ddde1689b..356036abcf050 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java @@ -133,11 +133,11 @@ public void testLengthFunctionWithInexact() { public void testMatchWithText() { VerificationException e = expectThrows(VerificationException.class, - () -> plan("process where match(plain_text, 'foo.*')")); + () -> plan("process where match(plain_text, \"foo.*\")")); String msg = e.getMessage(); assertEquals("Found 1 problem\n" + - "line 1:15: [match(plain_text, 'foo.*')] cannot operate on first argument field of data type [text]: No keyword/multi-field " + - "defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg); + "line 1:15: [match(plain_text, \"foo.*\")] cannot operate on first argument field of data type [text]: " + + "No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg); } public void testMatchWithNonString() { @@ -253,9 +253,9 @@ public void testWildcardWithNumericPattern() { public void testWildcardWithNumericField() { VerificationException e = expectThrows(VerificationException.class, - () -> plan("process where wildcard(pid, '*.exe')")); + () -> plan("process where wildcard(pid, \"*.exe\")")); String msg = e.getMessage(); assertEquals("Found 1 problem\n" + - "line 1:15: first argument of [wildcard(pid, '*.exe')] must be [string], found value [pid] type [long]", msg); + "line 1:15: first argument of [wildcard(pid, \"*.exe\")] must be [string], found value [pid] type [long]", msg); } } diff --git a/x-pack/plugin/eql/src/test/resources/queries-supported.eql b/x-pack/plugin/eql/src/test/resources/queries-supported.eql index 068149f10663d..02b5fb80f3aa6 100644 --- a/x-pack/plugin/eql/src/test/resources/queries-supported.eql +++ b/x-pack/plugin/eql/src/test/resources/queries-supported.eql @@ -1,5 +1,5 @@ process where process_name == "svchost.exe" and command_line != "* -k *"; -process where process_name in ('ipconfig.exe', 'netstat.exe', 'systeminfo.exe', 'route.exe'); +process where process_name in ("ipconfig.exe", "netstat.exe", "systeminfo.exe", "route.exe"); process where subtype.create and wildcard(command_line, "*.ost *", "*.pst *") ; @@ -178,66 +178,66 @@ process where command_line == "*%*%*" ; process where command_line == "%*%*" ; -process where match(?'.*?net1\s+localgroup\s+.*?', command_line) +process where match(?".*?net1\s+localgroup\s+.*?", command_line) ; -process where match(?'.*?net1\s+\w+\s+.*?', command_line) +process where match(?".*?net1\s+\w+\s+.*?", command_line) ; -process where match(?'.*?net1\s+\w{4,15}\s+.*?', command_line) +process where match(?".*?net1\s+\w{4,15}\s+.*?", command_line) ; -process where match(?'.*?net1\s+\w{4,15}\s+.*?', command_line) +process where match(?".*?net1\s+\w{4,15}\s+.*?", command_line) ; -process where match(?'.*?net1\s+[localgrup]{4,15}\s+.*?', command_line) +process where match(?".*?net1\s+[localgrup]{4,15}\s+.*?", command_line) ; -file where opcode=0 and startsWith(file_name, 'exploRER.') +file where opcode=0 and startsWith(file_name, "exploRER.") ; -file where opcode=0 and startsWith(file_name, 'expLORER.exe') +file where opcode=0 and startsWith(file_name, "expLORER.exe") ; -file where opcode=0 and endsWith(file_name, 'loREr.exe'); +file where opcode=0 and endsWith(file_name, "loREr.exe"); -file where opcode=0 and startsWith(file_name, 'explORER.EXE'); +file where opcode=0 and startsWith(file_name, "explORER.EXE"); -file where opcode=0 and startsWith('explorer.exeaaaaaaaa', file_name); +file where opcode=0 and startsWith("explorer.exeaaaaaaaa", file_name); -file where opcode=0 and serial_event_id = 88 and startsWith('explorer.exeaAAAA', 'EXPLORER.exe'); +file where opcode=0 and serial_event_id = 88 and startsWith("explorer.exeaAAAA", "EXPLORER.exe"); -file where opcode=0 and stringContains('ABCDEFGHIexplorer.exeJKLMNOP', file_name) +file where opcode=0 and stringContains("ABCDEFGHIexplorer.exeJKLMNOP", file_name) ; -file where opcode=0 and indexOf(file_name, 'plore') == 2 and not indexOf(file_name, '.pf') +file where opcode=0 and indexOf(file_name, "plore") == 2 and not indexOf(file_name, ".pf") ; -file where opcode=0 and indexOf(file_name, 'explorer.') and indexOf(file_name, 'plore', 100) +file where opcode=0 and indexOf(file_name, "explorer.") and indexOf(file_name, "plore", 100) ; -file where opcode=0 and indexOf(file_name, 'plorer.', 0) == 2; +file where opcode=0 and indexOf(file_name, "plorer.", 0) == 2; -file where opcode=0 and indexOf(file_name, 'plorer.', 2); +file where opcode=0 and indexOf(file_name, "plorer.", 2); -file where opcode=0 and indexOf(file_name, 'plorer.', 4); +file where opcode=0 and indexOf(file_name, "plorer.", 4); -file where opcode=0 and indexOf(file_name, 'thing that never happened'); +file where opcode=0 and indexOf(file_name, "thing that never happened"); -file where opcode=0 and indexOf(file_name, 'plorer.', 2) == 2; +file where opcode=0 and indexOf(file_name, "plorer.", 2) == 2; -file where opcode=0 and indexOf(file_name, 'explorer.', 0) == 0; +file where opcode=0 and indexOf(file_name, "explorer.", 0) == 0; -file where serial_event_id=88 and substring(file_name, 0, 4) == 'expl' +file where serial_event_id=88 and substring(file_name, 0, 4) == "expl" ; -file where serial_event_id=88 and substring(file_name, 1, 3) == 'xp' +file where serial_event_id=88 and substring(file_name, 1, 3) == "xp" ; -file where serial_event_id=88 and substring(file_name, -4) == '.exe' +file where serial_event_id=88 and substring(file_name, -4) == ".exe" ; -file where serial_event_id=88 and substring(file_name, -4, -1) == '.ex' +file where serial_event_id=88 and substring(file_name, -4, -1) == ".ex" ; process where add(serial_event_id, 0) == 1 and add(0, 1) == serial_event_id; @@ -248,15 +248,15 @@ process where multiply(6, serial_event_id) == 30 and divide(30, 4.0) == 7.5; process where modulo(11, add(serial_event_id, 1)) == serial_event_id; -process where serial_event_id == number('5'); +process where serial_event_id == number("5"); -process where serial_event_id == number('0x32', 16); +process where serial_event_id == number("0x32", 16); -process where serial_event_id == number('32', 16); +process where serial_event_id == number("32", 16); process where number(serial_event_id) == number(5); -process where concat(serial_event_id, ':', process_name, opcode) == '5:winINIT.exe3' +process where concat(serial_event_id, ":", process_name, opcode) == "5:winINIT.exe3" ; @@ -264,7 +264,7 @@ process where concat(serial_event_id, ':', process_name, opcode) == '5:winINIT.e // network where safe(divide(process_name, process_name)) //; -file where serial_event_id == 82 and (true == (process_name in ('svchost.EXE', 'bad.exe', 'bad2.exe'))) +file where serial_event_id == 82 and (true == (process_name in ("svchost.EXE", "bad.exe", "bad2.exe"))) ; @@ -315,10 +315,10 @@ network where cidrMatch(source_address, "192.168.0.0/16", "10.6.48.157/8") network where cidrMatch(source_address, "0.0.0.0/0") ; -process where length(between(process_name, 'g', 'e')) > 0 +process where length(between(process_name, "g", "e")) > 0 ; -process where length(between(process_name, 'g', 'z')) > 0 +process where length(between(process_name, "g", "z")) > 0 ; @@ -542,7 +542,7 @@ until [process where opcode=2] by ppid,process_path ; sequence by unique_pid - [process where opcode=1 and process_name == 'msbuild.exe'] + [process where opcode=1 and process_name == "msbuild.exe"] [network where true] ; @@ -686,4 +686,4 @@ process where fake_field != "*" | head 4; process where not (fake_field == "*") -| head 4; \ No newline at end of file +| head 4; diff --git a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt index 6d16a650a907e..5abd9108f1f52 100644 --- a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt +++ b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt @@ -103,14 +103,14 @@ process where cidrMatch(source_address, "10.0.0.0/8") != false ; twoFunctionsEqualsBooleanLiterals-caseSensitive -process where endsWith(process_path, 'x') == true and endsWith(process_path, 'yx') == false +process where endsWith(process_path, "x") == true and endsWith(process_path, "yx") == false ; {"bool":{"must":[{"wildcard":{"process_path":{"wildcard":"*x","boost":1.0}}}, {"bool":{"must_not":[{"wildcard":{"process_path":{"wildcard":"*yx","boost":1.0}}}],"boost":1.0}}],"boost":1.0}} ; twoFunctionsEqualsBooleanLiterals-caseInsensitive -process where endsWith(process_path, 'x') == true and endsWith(process_path, 'yx') == false +process where endsWith(process_path, "x") == true and endsWith(process_path, "yx") == false ; "bool":{"must":[{"term":{"event.category":{"value":"process" "must":[{"script":{"script":{"source":"InternalQlScriptUtils.nullSafeFilter( @@ -122,21 +122,21 @@ InternalEqlScriptUtils.endsWith(InternalQlScriptUtils.docValue(doc,params.v0),pa ; endsWithKeywordFieldFunction-caseSensitive -process where endsWith(user_name, 'c') +process where endsWith(user_name, "c") ; {"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0}}}, {"wildcard":{"user_name":{"wildcard":"*c","boost":1.0}}}],"boost":1.0}} ; endsWithWildcardSubFieldFunction-caseSensitive -process where endsWith(hostname, 'c') +process where endsWith(hostname, "c") ; {"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0}}}, {"wildcard":{"hostname.keyword":{"wildcard":"*c","boost":1.0}}}],"boost":1.0}} ; endsWithFunction-caseInsensitive -process where endsWith(user_name, 'c') +process where endsWith(user_name, "c") ; "script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.endsWith( InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2))", @@ -168,7 +168,7 @@ InternalEqlScriptUtils.length(InternalQlScriptUtils.docValue(doc,params.v0)),par ; startsWithFunction-caseInsensitive -process where startsWith(user_name, 'A') +process where startsWith(user_name, "A") ; "script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.startsWith( InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2))", @@ -176,14 +176,14 @@ InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2))", ; startsWithFunctionSimple-caseSensitive -process where startsWith(user_name, 'A') +process where startsWith(user_name, "A") ; {"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0}}}, {"prefix":{"user_name":{"value":"A","boost":1.0}}}],"boost":1.0}} ; startsWithFunctionWithCondition-caseSensitive -process where startsWith(user_name, 'A') or startsWith(user_name, 'B') +process where startsWith(user_name, "A") or startsWith(user_name, "B") ; {"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0}}}, {"bool":{"should":[{"prefix":{"user_name":{"value":"A","boost":1.0}}}, @@ -222,7 +222,7 @@ InternalEqlScriptUtils.string(InternalQlScriptUtils.docValue(doc,params.v0)),par ; indexOfFunction-caseSensitive -process where indexOf(user_name, 'A', 2) > 0 +process where indexOf(user_name, "A", 2) > 0 ; "script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt( InternalEqlScriptUtils.indexOf(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2,params.v3),params.v4))", @@ -230,7 +230,7 @@ InternalEqlScriptUtils.indexOf(InternalQlScriptUtils.docValue(doc,params.v0),par ; indexOfFunction-caseInsensitive -process where indexOf(user_name, 'A', 2) > 0 +process where indexOf(user_name, "A", 2) > 0 ; "script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt( InternalEqlScriptUtils.indexOf(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2,params.v3),params.v4))", @@ -238,7 +238,7 @@ InternalEqlScriptUtils.indexOf(InternalQlScriptUtils.docValue(doc,params.v0),par ; substringFunction -process where substring(file_name, -4) == '.exe' +process where substring(file_name, -4) == ".exe" ; "script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( InternalEqlScriptUtils.substring(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))", diff --git a/x-pack/plugin/eql/src/test/resources/test_folding.toml b/x-pack/plugin/eql/src/test/resources/test_folding.toml index 7a5dcc6df79bc..119315e515180 100644 --- a/x-pack/plugin/eql/src/test/resources/test_folding.toml +++ b/x-pack/plugin/eql/src/test/resources/test_folding.toml @@ -337,15 +337,15 @@ description = "Fold constant check in set" expected = false [[in_set.fold.tests]] - expression = "'foo' not in ('foo', 'bar', 'baz')" + expression = '"foo" not in ("foo", "bar", "baz")' expected = false [[in_set.fold.tests]] - expression = "'foo' in ('foo', 'bar', 'baz')" + expression = '"foo" in ("foo", "bar", "baz")' expected = true [[in_set.fold.tests]] - expression = "'foo' in ('bar', 'baz')" + expression = '"foo" in ("bar", "baz")' expected = false [or] diff --git a/x-pack/plugin/eql/src/test/resources/test_string_functions.toml b/x-pack/plugin/eql/src/test/resources/test_string_functions.toml index 08c5a3eec2396..c7b059895c354 100644 --- a/x-pack/plugin/eql/src/test/resources/test_string_functions.toml +++ b/x-pack/plugin/eql/src/test/resources/test_string_functions.toml @@ -94,11 +94,11 @@ description = "Test the `concat` function" description = "Test the `endsWith` function with case matching" [[endswith.fold.tests]] - expression = "endsWith('FooBarBaz', 'Baz')" + expression = 'endsWith("FooBarBaz", "Baz")' expected = true [[endswith.fold.tests]] - expression = "endsWith('FooBarBaz', 'Foo')" + expression = 'endsWith("FooBarBaz", "Foo")' expected = false @@ -107,7 +107,7 @@ description = "Test the `endsWith` function with case insensitivity" case_insensitive = true [[endswith_insensitive.fold.tests]] - expression = "endsWith('FooBarBaz', 'baz')" + expression = 'endsWith("FooBarBaz", "baz")' expected = true @@ -116,7 +116,7 @@ description = "Test the `endsWith` function with case sensitivity" case_sensitive = true [[endswith_sensitive.fold.tests]] - expression = "endsWith('FooBarBaz', 'baz')" + expression = 'endsWith("FooBarBaz", "baz")' expected = false @@ -295,37 +295,37 @@ description = "Test the `startsWith` function" expression = "startsWith()" [[startswith.verifier.failures]] - expression = "startsWith(1, 'FOO')" + expression = '''startsWith(1, "FOO")''' [[startswith.verifier.failures]] - expression = "startsWith('FOO', 123)" + expression = '''startsWith("FOO", 123)''' [[startswith.fold.tests]] - expression = "startsWith('FooBarBaz', null)" + expression = '''startsWith("FooBarBaz", null)''' # expected = null [[startswith.fold.tests]] - expression = "startsWith(null, 'Foo')" + expression = '''startsWith(null, "Foo")''' # expected = null [[startswith.fold.tests]] - expression = "startsWith('FooBarBaz', 'Foo')" + expression = '''startsWith("FooBarBaz", "Foo")''' expected = true [[startswith.fold.tests]] - expression = "startsWith('FooBarBaz', '')" + expression = '''startsWith("FooBarBaz", "")''' expected = true [[startswith.fold.tests]] - expression = "startsWith('FooBarBaz', 'FooBar')" + expression = '''startsWith("FooBarBaz", "FooBar")''' expected = true [[startswith.fold.tests]] - expression = "startsWith('FooBarBaz', 'Bar')" + expression = '''startsWith("FooBarBaz", "Bar")''' expected = false [[startswith.fold.tests]] - expression = "startsWith('FooBarBaz', 'Bar')" + expression = '''startsWith("FooBarBaz", "Bar")''' expected = false @@ -334,31 +334,32 @@ description = "Test the `startsWith` function with case-sensitive matching" case_sensitive = true [[startswith_case_sensitive.tests]] - expression = "startsWith('FooBar', 'Foo')" + expression = '''startsWith("FooBar", "Foo")''' expected = true [[startswith_case_sensitive.tests]] - expression = "startsWith('FooBar', 'foo')" + expression = '''startsWith("FooBar", "foo")''' expected = false [startswith_case_insensitive] description = "Test the `startsWith` function with case-insensitive matching" +case_insensitive = true [[startswith_case_insensitive.fold.tests]] - expression = "startsWith('FooBar', 'Foo')" + expression = '''startsWith("FooBar", "Foo")''' expected = true [[startswith_case_insensitive.fold.tests]] - expression = "startsWith('FooBar', 'FOO')" + expression = '''startsWith("FooBar", "FOO")''' expected = true [[startswith_case_insensitive.fold.tests]] - expression = "startsWith('FooBar', 'foo')" + expression = '''startsWith("FooBar", "foo")''' expected = true [[startswith_case_insensitive.fold.tests]] - expression = "startsWith('FooBar', 'Bar')" + expression = '''startsWith("FooBar", "Bar")''' expected = false @@ -382,7 +383,7 @@ description = "Test the `string` function" expected = "2.1828" [[string.fold.tests]] - expression = "string('hello')" + expression = '''string("hello")''' expected = "hello" [substring] @@ -397,55 +398,55 @@ description = "Test the `substring` function when the case already matches" # expected = null [[substring.fold.tests]] - expression = "substring('hello world', null, null)" + expression = '''substring("hello world", null, null)''' expected = "hello world" [[substring.fold.tests]] - expression = "substring('hello world', 6, null)" + expression = '''substring("hello world", 6, null)''' expected = "world" [[substring.fold.tests]] - expression = "substring('hello world', null, 5)" + expression = '''substring("hello world", null, 5)''' expected = "hello" [[substring.fold.tests]] - expression = "substring('hello world', 0, 5)" + expression = '''substring("hello world", 0, 5)''' expected = "hello" [[substring.fold.tests]] - expression = "substring('hello world', 0, 500)" + expression = '''substring("hello world", 0, 500)''' expected = "hello world" [[substring.fold.tests]] - expression = "substring('hello world', 5, -1)" + expression = '''substring("hello world", 5, -1)''' expected = " worl" [[substring.fold.tests]] - expression = "substring('hello world', 3, 9)" + expression = '''substring("hello world", 3, 9)''' expected = "lo wor" [[substring.fold.tests]] - expression = "substring('hello world', -5, null)" + expression = '''substring("hello world", -5, null)''' expected = "world" [[substring.fold.tests]] - expression = "substring('hello world', -5, 11)" + expression = '''substring("hello world", -5, 11)''' expected = "world" [[substring.fold.tests]] - expression = "substring('hello world', -5, 100)" + expression = '''substring("hello world", -5, 100)''' expected = "world" [[substring.fold.tests]] - expression = "substring('hello world', -60, 5)" + expression = '''substring("hello world", -60, 5)''' expected = "hello" [[substring.fold.tests]] - expression = "substring('hello world', -5)" + expression = '''substring("hello world", -5)''' expected = "world" [[substring.fold.tests]] - expression = "substring('hello world', -5, -1)" + expression = '''substring("hello world", -5, -1)''' expected = "worl" [wildcard] diff --git a/x-pack/plugin/eql/src/test/resources/test_unsupported.toml b/x-pack/plugin/eql/src/test/resources/test_unsupported.toml index 2690a73a13285..27972065c4757 100644 --- a/x-pack/plugin/eql/src/test/resources/test_unsupported.toml +++ b/x-pack/plugin/eql/src/test/resources/test_unsupported.toml @@ -127,27 +127,8 @@ case_insensitive = true expected = false - -[endswith_sensitive] -case_sensitive = true - - [[endswith_sensitive.fold.tests]] - expression = "endsWith('FooBarBaz', 'baz')" - expected = false - - -[startswith_case_insensitive] - - [[startswith_case_insensitive.fold.tests]] - expression = "startsWith('FooBar', 'FOO')" - expected = true - - [[startswith_case_insensitive.fold.tests]] - expression = "startsWith('FooBar', 'foo')" - expected = true - [substring] [[substring.fold.tests]] - expression = "substring('hello world', null, 5)" + expression = '''substring("hello world", null, 5)''' expected = "hello" From 8f7c305baebb91ef12d9788b6e5e60b4a5e47181 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Wed, 16 Sep 2020 16:42:10 +0300 Subject: [PATCH 2/4] fix tests --- docs/reference/eql/detect-threats-with-eql.asciidoc | 4 ++-- .../test/data_stream/10_data_stream_resolvability.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/reference/eql/detect-threats-with-eql.asciidoc b/docs/reference/eql/detect-threats-with-eql.asciidoc index 86d6877432100..ffbbc75fe3fcb 100644 --- a/docs/reference/eql/detect-threats-with-eql.asciidoc +++ b/docs/reference/eql/detect-threats-with-eql.asciidoc @@ -299,8 +299,8 @@ GET /my-index-000001/_eql/search { "query": """ sequence by process.pid - [process where process.name == 'regsvr32.exe'] - [library where dll.name == 'scrobj.dll'] + [process where process.name == "regsvr32.exe"] + [library where dll.name == "scrobj.dll"] [network where true] """ } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml index 5c3da419df2a5..d42eb7cb45c6d 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml @@ -516,7 +516,7 @@ eql.search: index: simple-data-stream1 body: - query: "process where user = 'SYSTEM'" + query: 'process where user = "SYSTEM"' - match: {timed_out: false} - match: {hits.total.value: 1} @@ -527,7 +527,7 @@ eql.search: index: simple-data-s* body: - query: "process where user = 'SYSTEM'" + query: 'process where user = "SYSTEM"' - match: {timed_out: false} - match: {hits.total.value: 1} From efddcbbf446fda3d6eeda3d4def7937bc8195f34 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Wed, 16 Sep 2020 18:43:10 +0300 Subject: [PATCH 3/4] rephrase error message --- .../elasticsearch/xpack/eql/parser/AbstractBuilder.java | 2 +- .../elasticsearch/xpack/eql/parser/ExpressionTests.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AbstractBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AbstractBuilder.java index 0293313cc432e..56d687584d166 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AbstractBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AbstractBuilder.java @@ -180,7 +180,7 @@ public static String unquoteString(Source source) { private static void checkForSingleQuotedString(Source source, String text, int i) { if (text.charAt(i) == '\'') { throw new ParsingException(source, - "Single quotes ['] are not supported around literal strings, please use double quotes [\"]"); + "Use double quotes [\"] to define string literals, not single quotes [']"); } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java index d1e8d690640a5..b69f12563be69 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java @@ -82,10 +82,10 @@ public void testLiterals() { public void testSingleQuotedStringForbidden() { ParsingException e = expectThrows(ParsingException.class, () -> expr("'hello world'")); - assertEquals("line 1:2: Single quotes ['] are not supported around literal strings, please use double quotes [\"]", + assertEquals("line 1:2: Use double quotes [\"] to define string literals, not single quotes [']", e.getMessage()); e = expectThrows(ParsingException.class, () -> parser.createStatement("process where name='hello world'")); - assertEquals("line 1:21: Single quotes ['] are not supported around literal strings, please use double quotes [\"]", + assertEquals("line 1:21: Use double quotes [\"] to define string literals, not single quotes [']", e.getMessage()); } @@ -98,10 +98,10 @@ public void testDoubleQuotedString() { public void testSingleQuotedUnescapedStringForbidden() { ParsingException e = expectThrows(ParsingException.class, () -> expr("?'hello world'")); - assertEquals("line 1:2: Single quotes ['] are not supported around literal strings, please use double quotes [\"]", + assertEquals("line 1:2: Use double quotes [\"] to define string literals, not single quotes [']", e.getMessage()); e = expectThrows(ParsingException.class, () -> parser.createStatement("process where name=?'hello world'")); - assertEquals("line 1:21: Single quotes ['] are not supported around literal strings, please use double quotes [\"]", + assertEquals("line 1:21: Use double quotes [\"] to define string literals, not single quotes [']", e.getMessage()); } From 6ee279cb2a971f80a46c63b57f9788d449de22de Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Wed, 16 Sep 2020 19:03:17 +0300 Subject: [PATCH 4/4] fix tests --- x-pack/plugin/eql/src/test/resources/queries-unsupported.eql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql b/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql index 1ffc81d483ab1..956007b2b5ff6 100644 --- a/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql +++ b/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql @@ -298,7 +298,7 @@ sequence ; -process where 'net.EXE' == original_file_name +process where "net.EXE" == original_file_name | filter process_name="net*.exe" ;