Skip to content

Commit f04fb68

Browse files
rw-accessrw-access
authored
EQL: Add wildcard function (#54020)
* EQL: Add wildcard function * EQL: Cleanup Wildcard.getArguments * EQL: Cleanup Wildcard and rearrange methods * EQL: Wildcard newline lint * EQL: Make StringUtils function final * EQL: Make Wildcard.asLikes return ScalarFunction * QL: Restore BinaryLogic.java * EQL: Add Wildcard PR feedback * EQL: Add Wildcard verification tests * EQL: Switch wildcard to isFoldable test * EQL: Change wildcard test to numeric field * EQL: Remove Wildcard.get_arguments
1 parent 01dc110 commit f04fb68

File tree

10 files changed

+267
-20
lines changed

10 files changed

+267
-20
lines changed

‎x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,14 @@ registry where length(bytes_written_string_list) == 2 and bytes_written_string_l
209209

210210
[[queries]]
211211
query = '''
212-
registry where key_path == "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F"'''
212+
registry where key_path == "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F"
213+
'''
214+
expected_event_ids = [79]
215+
216+
[[queries]]
217+
query = '''
218+
registry where wildcard(key_path, "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F")
219+
'''
213220
expected_event_ids = [79]
214221

215222
[[queries]]

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length;
1111
import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWith;
1212
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring;
13+
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Wildcard;
1314
import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition;
1415
import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
1516

@@ -20,7 +21,7 @@ public class EqlFunctionRegistry extends FunctionRegistry {
2021
public EqlFunctionRegistry() {
2122
super(functions());
2223
}
23-
24+
2425
private static FunctionDefinition[][] functions() {
2526
return new FunctionDefinition[][] {
2627
// Scalar functions
@@ -29,7 +30,8 @@ private static FunctionDefinition[][] functions() {
2930
def(EndsWith.class, EndsWith::new, "endswith"),
3031
def(Length.class, Length::new, "length"),
3132
def(StartsWith.class, StartsWith::new, "startswith"),
32-
def(Substring.class, Substring::new, "substring")
33+
def(Substring.class, Substring::new, "substring"),
34+
def(Wildcard.class, Wildcard::new, "wildcard"),
3335
}
3436
};
3537
}

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Wildcard.java‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
8+
9+
import org.elasticsearch.xpack.eql.EqlIllegalArgumentException;
10+
import org.elasticsearch.xpack.eql.util.StringUtils;
11+
import org.elasticsearch.xpack.ql.expression.Expression;
12+
import org.elasticsearch.xpack.ql.expression.Expressions;
13+
import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal;
14+
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
15+
import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
16+
import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
17+
import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
18+
import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
19+
import org.elasticsearch.xpack.ql.tree.NodeInfo;
20+
import org.elasticsearch.xpack.ql.tree.Source;
21+
import org.elasticsearch.xpack.ql.type.DataType;
22+
import org.elasticsearch.xpack.ql.type.DataTypes;
23+
import org.elasticsearch.xpack.ql.util.CollectionUtils;
24+
25+
import java.util.Collections;
26+
import java.util.List;
27+
28+
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isFoldable;
29+
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isString;
30+
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact;
31+
32+
/**
33+
* EQL wildcard function. Matches the form:
34+
* wildcard(field, "*wildcard*pattern*", ...)
35+
*/
36+
public class Wildcard extends ScalarFunction {
37+
38+
private final Expression field;
39+
private final List<Expression> patterns;
40+
41+
public Wildcard(Source source, Expression field, List<Expression> patterns) {
42+
super(source, CollectionUtils.combine(Collections.singletonList(field), patterns));
43+
this.field = field;
44+
this.patterns = patterns;
45+
}
46+
47+
@Override
48+
protected NodeInfo<? extends Expression> info() {
49+
return NodeInfo.create(this, Wildcard::new, field, patterns);
50+
}
51+
52+
@Override
53+
public Expression replaceChildren(List<Expression> newChildren) {
54+
if (newChildren.size() < 2) {
55+
throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]");
56+
}
57+
58+
return new Wildcard(source(), newChildren.get(0), newChildren.subList(1, newChildren.size()));
59+
}
60+
61+
@Override
62+
public DataType dataType() {
63+
return DataTypes.BOOLEAN;
64+
}
65+
66+
@Override
67+
protected TypeResolution resolveType() {
68+
if (childrenResolved() == false) {
69+
return new TypeResolution("Unresolved children");
70+
}
71+
72+
TypeResolution lastResolution = isStringAndExact(field, sourceText(), ParamOrdinal.FIRST);
73+
if (lastResolution.unresolved()) {
74+
return lastResolution;
75+
}
76+
77+
int index = 1;
78+
79+
for (Expression p: patterns) {
80+
81+
lastResolution = isFoldable(p, sourceText(), ParamOrdinal.fromIndex(index));
82+
if (lastResolution.unresolved()) {
83+
break;
84+
}
85+
86+
lastResolution = isString(p, sourceText(), ParamOrdinal.fromIndex(index));
87+
if (lastResolution.unresolved()) {
88+
break;
89+
}
90+
91+
index++;
92+
}
93+
94+
return lastResolution;
95+
}
96+
97+
@Override
98+
public boolean foldable() {
99+
return Expressions.foldable(children()) && asLikes().foldable();
100+
}
101+
102+
@Override
103+
public Object fold() {
104+
return asLikes().fold();
105+
}
106+
107+
@Override
108+
protected Pipe makePipe() {
109+
throw new EqlIllegalArgumentException("Wildcard.makePipe() should not be called directly");
110+
}
111+
112+
@Override
113+
public ScriptTemplate asScript() {
114+
throw new EqlIllegalArgumentException("Wildcard.asScript() should not be called directly");
115+
}
116+
117+
public ScalarFunction asLikes() {
118+
ScalarFunction result = null;
119+
120+
for (Expression pattern: patterns) {
121+
String wcString = pattern.fold().toString();
122+
Like like = new Like(source(), field, StringUtils.toLikePattern(wcString));
123+
result = result == null ? like : new Or(source(), result, like);
124+
}
125+
126+
return result;
127+
}
128+
}

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66

77
package org.elasticsearch.xpack.eql.optimizer;
88

9+
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Wildcard;
10+
import org.elasticsearch.xpack.eql.util.StringUtils;
911
import org.elasticsearch.xpack.ql.expression.Expression;
1012
import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
1113
import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull;
@@ -14,7 +16,6 @@
1416
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
1517
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
1618
import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
17-
import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern;
1819
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight;
1920
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification;
2021
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineBinaryComparisons;
@@ -48,6 +49,7 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
4849
new ReplaceNullChecks(),
4950
new PropagateEquals(),
5051
new CombineBinaryComparisons(),
52+
new ReplaceWildcardFunction(),
5153
// prune/elimination
5254
new PruneFilters(),
5355
new PruneLiteralsInOrderBy()
@@ -60,6 +62,14 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
6062
}
6163

6264

65+
private static class ReplaceWildcardFunction extends OptimizerRule<Filter> {
66+
67+
@Override
68+
protected LogicalPlan rule(Filter filter) {
69+
return filter.transformExpressionsUp(e -> e instanceof Wildcard ? ((Wildcard) e).asLikes() : e);
70+
}
71+
}
72+
6373
private static class ReplaceWildcards extends OptimizerRule<Filter> {
6474

6575
private static boolean isWildcard(Expression expr) {
@@ -70,18 +80,6 @@ private static boolean isWildcard(Expression expr) {
7080
return false;
7181
}
7282

73-
private static LikePattern toLikePattern(String s) {
74-
// pick a character that is guaranteed not to be in the string, because it isn't allowed to escape itself
75-
char escape = 1;
76-
77-
// replace wildcards with % and escape special characters
78-
String likeString = s.replace("%", escape + "%")
79-
.replace("_", escape + "_")
80-
.replace("*", "%");
81-
82-
return new LikePattern(likeString, escape);
83-
}
84-
8583
@Override
8684
protected LogicalPlan rule(Filter filter) {
8785
return filter.transformExpressionsUp(e -> {
@@ -91,7 +89,7 @@ protected LogicalPlan rule(Filter filter) {
9189

9290
if (isWildcard(cmp.right())) {
9391
String wcString = cmp.right().fold().toString();
94-
Expression like = new Like(e.source(), cmp.left(), toLikePattern(wcString));
92+
Expression like = new Like(e.source(), cmp.left(), StringUtils.toLikePattern(wcString));
9593

9694
if (e instanceof NotEquals) {
9795
like = new Not(e.source(), like);

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/StringUtils.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.eql.util;
8+
9+
import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern;
10+
11+
public final class StringUtils {
12+
13+
private StringUtils() {}
14+
15+
/**
16+
* Convert an EQL wildcard string to a LikePattern.
17+
*/
18+
public static LikePattern toLikePattern(String s) {
19+
// pick a character that is guaranteed not to be in the string, because it isn't allowed to escape itself
20+
char escape = 1;
21+
22+
// replace wildcards with % and escape special characters
23+
String likeString = s.replace("%", escape + "%")
24+
.replace("_", escape + "_")
25+
.replace("*", "%");
26+
27+
return new LikePattern(likeString, escape);
28+
}
29+
}

‎x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ private List<Expression> exprs(String... sources) {
5252
}
5353

5454

55-
public void testStrings() throws Exception {
55+
public void testStrings() {
5656
assertEquals("hello\"world", unquoteString("'hello\"world'"));
5757
assertEquals("hello'world", unquoteString("\"hello'world\""));
5858
assertEquals("hello\nworld", unquoteString("'hello\\nworld'"));

‎x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
package org.elasticsearch.xpack.eql.planner;
88

99
import org.elasticsearch.xpack.eql.analysis.VerificationException;
10+
import org.elasticsearch.xpack.ql.ParsingException;
1011
import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
1112

1213
public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
@@ -48,4 +49,35 @@ public void testStartsWithFunctionWithInexact() {
4849
assertEquals("Found 1 problem\nline 1:15: [startsWith(plain_text, \"foo\")] cannot operate on first argument field of data type "
4950
+ "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
5051
}
52+
53+
public void testWildcardNotEnoughArguments() {
54+
ParsingException e = expectThrows(ParsingException.class,
55+
() -> plan("process where wildcard(process_name)"));
56+
String msg = e.getMessage();
57+
assertEquals("line 1:16: error building [wildcard]: expects at least two arguments", msg);
58+
}
59+
60+
public void testWildcardAgainstVariable() {
61+
VerificationException e = expectThrows(VerificationException.class,
62+
() -> plan("process where wildcard(process_name, parent_process_name)"));
63+
String msg = e.getMessage();
64+
assertEquals("Found 1 problem\nline 1:15: second argument of [wildcard(process_name, parent_process_name)] " +
65+
"must be a constant, received [parent_process_name]", msg);
66+
}
67+
68+
public void testWildcardWithNumericPattern() {
69+
VerificationException e = expectThrows(VerificationException.class,
70+
() -> plan("process where wildcard(process_name, 1)"));
71+
String msg = e.getMessage();
72+
assertEquals("Found 1 problem\n" +
73+
"line 1:15: second argument of [wildcard(process_name, 1)] must be [string], found value [1] type [integer]", msg);
74+
}
75+
76+
public void testWildcardWithNumericField() {
77+
VerificationException e = expectThrows(VerificationException.class,
78+
() -> plan("process where wildcard(pid, '*.exe')"));
79+
String msg = e.getMessage();
80+
assertEquals("Found 1 problem\n" +
81+
"line 1:15: first argument of [wildcard(pid, '*.exe')] must be [string], found value [pid] type [long]", msg);
82+
}
5183
}

‎x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,3 +102,21 @@ process where substring(file_name, -4) == '.exe'
102102
InternalEqlScriptUtils.substring(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))",
103103
"params":{"v0":"file_name.keyword","v1":-4,"v2":null,"v3":".exe"}
104104

105+
106+
wildcardFunctionSingleArgument
107+
process where wildcard(process_path, "*\\red_ttp\\wininit.*")
108+
"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*"
109+
110+
111+
wildcardFunctionTwoArguments
112+
process where wildcard(process_path, "*\\red_ttp\\wininit.*", "*\\abc\\*")
113+
"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*"
114+
"wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*"
115+
116+
117+
wildcardFunctionThreeArguments
118+
process where wildcard(process_path, "*\\red_ttp\\wininit.*", "*\\abc\\*", "*def*")
119+
"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*"
120+
"wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*"
121+
"wildcard":{"process_path":{"wildcard":"*def*"
122+

‎x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,20 @@ public enum ParamOrdinal {
3131
FIRST,
3232
SECOND,
3333
THIRD,
34-
FOURTH
34+
FOURTH;
35+
36+
public static ParamOrdinal fromIndex(int index) {
37+
switch (index) {
38+
case 0: return ParamOrdinal.FIRST;
39+
case 1: return ParamOrdinal.SECOND;
40+
case 2: return ParamOrdinal.THIRD;
41+
case 3: return ParamOrdinal.FOURTH;
42+
default: return ParamOrdinal.DEFAULT;
43+
}
44+
}
3545
}
3646

47+
3748
private Expressions() {}
3849

3950
public static NamedExpression wrapAsNamed(Expression exp) {
@@ -205,4 +216,4 @@ public static List<Pipe> pipe(List<Expression> expressions) {
205216
public static String id(Expression e) {
206217
return Integer.toHexString(e.hashCode());
207218
}
208-
}
219+
}

0 commit comments

Comments
 (0)