From 4548b0fb35550b1fdf8fa93c279debabb57153cf Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Thu, 6 Feb 2020 21:09:20 +0800 Subject: [PATCH 1/8] Support 'like any' and 'like all' operators --- .../spark/sql/catalyst/parser/SqlBase.g4 | 1 + .../sql/catalyst/parser/AstBuilder.scala | 25 ++-- .../resources/sql-tests/inputs/like-all.sql | 39 ++++++ .../resources/sql-tests/inputs/like-any.sql | 39 ++++++ .../sql-tests/results/like-all.sql.out | 109 +++++++++++++++++ .../sql-tests/results/like-any.sql.out | 115 ++++++++++++++++++ 6 files changed, 320 insertions(+), 8 deletions(-) create mode 100644 sql/core/src/test/resources/sql-tests/inputs/like-all.sql create mode 100644 sql/core/src/test/resources/sql-tests/inputs/like-any.sql create mode 100644 sql/core/src/test/resources/sql-tests/results/like-all.sql.out create mode 100644 sql/core/src/test/resources/sql-tests/results/like-any.sql.out diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 6f2bb7a9a7536..cd8ea09ddd225 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -748,6 +748,7 @@ predicate | NOT? kind=IN '(' expression (',' expression)* ')' | NOT? kind=IN '(' query ')' | NOT? kind=RLIKE pattern=valueExpression + | NOT? kind=LIKE operator=(ANY | ALL) '(' expression (',' expression)* ')' | NOT? kind=LIKE pattern=valueExpression (ESCAPE escapeChar=STRING)? | IS NOT? kind=NULL | IS NOT? kind=(TRUE | FALSE | UNKNOWN) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index e9ad84472904d..5d24e2a6fdea1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1387,14 +1387,23 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging case SqlBaseParser.IN => invertIfNotDefined(In(e, ctx.expression.asScala.map(expression))) case SqlBaseParser.LIKE => - val escapeChar = Option(ctx.escapeChar).map(string).map { str => - if (str.length != 1) { - throw new ParseException("Invalid escape string." + - "Escape string must contains only one character.", ctx) - } - str - }.getOrElse('\\') - invertIfNotDefined(Like(e, expression(ctx.pattern), Literal(escapeChar))) + Option(ctx.operator).map(_.getType) match { + case Some(SqlBaseParser.ANY) if !ctx.expression.isEmpty => + ctx.expression.asScala.map(expression).map(p => invertIfNotDefined(new Like(e, p))) + .reduceLeft(Or) + case Some(SqlBaseParser.ALL) if !ctx.expression.isEmpty => + ctx.expression.asScala.map(expression).map(p => invertIfNotDefined(new Like(e, p))) + .reduceLeft(And) + case _ => + val escapeChar = Option(ctx.escapeChar).map(string).map { str => + if (str.length != 1) { + throw new ParseException("Invalid escape string." + + "Escape string must contains only one character.", ctx) + } + str + }.getOrElse('\\') + invertIfNotDefined(Like(e, expression(ctx.pattern), Literal(escapeChar))) + } case SqlBaseParser.RLIKE => invertIfNotDefined(RLike(e, expression(ctx.pattern))) case SqlBaseParser.NULL if ctx.NOT != null => diff --git a/sql/core/src/test/resources/sql-tests/inputs/like-all.sql b/sql/core/src/test/resources/sql-tests/inputs/like-all.sql new file mode 100644 index 0000000000000..de59ef162b40b --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/like-all.sql @@ -0,0 +1,39 @@ +CREATE OR REPLACE TEMPORARY VIEW like_all_table AS SELECT * FROM (VALUES + ('google', '%oo%'), + ('facebook', '%oo%'), + ('linkedin', '%in')) + as t1(company, pat); + +select company from like_all_table where company like all ('%oo%','%go%') ; + +select company from like_all_table where company like all ('microsoft','%yoo%') ; + +select + company, + CASE + WHEN company like all ('%oo%','%go%') THEN 'Y' + ELSE 'N' + END AS is_available, + CASE + WHEN company like all ('%oo%','go%') OR company like all ('%in','ms%') THEN 'Y' + ELSE 'N' + END AS mix +From like_all_table ; + +--Mix test with constant pattern and column value +select company from like_all_table where company like all ('%oo%',pat) ; + +-- not like all test + +select company from like_all_table where company not like all ('%oo%','%in','fa%') ; +select company from like_all_table where company not like all ('microsoft','%yoo%') ; + +-- null test + +select company from like_all_table where company like all ('%oo%',null) ; + +select company from like_all_table where company not like all ('%oo%',null) ; + +select company from like_all_table where company not like all (null,null) ; + +select company from like_all_table where company not like all (null,null) ; \ No newline at end of file diff --git a/sql/core/src/test/resources/sql-tests/inputs/like-any.sql b/sql/core/src/test/resources/sql-tests/inputs/like-any.sql new file mode 100644 index 0000000000000..3ea9f3c217eb7 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/like-any.sql @@ -0,0 +1,39 @@ +CREATE OR REPLACE TEMPORARY VIEW like_any_table AS SELECT * FROM (VALUES + ('google', '%oo%'), + ('facebook', '%oo%'), + ('linkedin', '%in')) + as t1(company, pat); + +select company from like_any_table where company like any ('%oo%','%in','fa%') ; + +select company from like_any_table where company like any ('microsoft','%yoo%') ; + +select + company, + CASE + WHEN company like any ('%oo%','%in','fa%') THEN 'Y' + ELSE 'N' + END AS is_available, + CASE + WHEN company like any ('%oo%','fa%') OR company like any ('%in','ms%') THEN 'Y' + ELSE 'N' + END AS mix +From like_any_table; + +--Mix test with constant pattern and column value +select company from like_any_table where company like any ('%zz%',pat) ; + +-- not like any test + +select company from like_any_table where company not like any ('%oo%','%in','fa%') ; +select company from like_any_table where company not like any ('microsoft','%yoo%') ; + +-- null test + +select company from like_any_table where company like any ('%oo%',null) ; + +select company from like_any_table where company not like any ('%oo%',null) ; + +select company from like_any_table where company like any (null,null) ; + +select company from like_any_table where company not like any (null,null) ; diff --git a/sql/core/src/test/resources/sql-tests/results/like-all.sql.out b/sql/core/src/test/resources/sql-tests/results/like-all.sql.out new file mode 100644 index 0000000000000..239b7f4906ea1 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/like-all.sql.out @@ -0,0 +1,109 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 11 + + +-- !query +CREATE OR REPLACE TEMPORARY VIEW like_all_table AS SELECT * FROM (VALUES + ('google', '%oo%'), + ('facebook', '%oo%'), + ('linkedin', '%in')) + as t1(company, pat) +-- !query schema +struct<> +-- !query output + + + +-- !query +select company from like_all_table where company like all ('%oo%','%go%') +-- !query schema +struct +-- !query output +google + + +-- !query +select company from like_all_table where company like all ('microsoft','%yoo%') +-- !query schema +struct +-- !query output + + + +-- !query +select + company, + CASE + WHEN company like all ('%oo%','%go%') THEN 'Y' + ELSE 'N' + END AS is_available, + CASE + WHEN company like all ('%oo%','go%') OR company like all ('%in','ms%') THEN 'Y' + ELSE 'N' + END AS mix +From like_all_table +-- !query schema +struct +-- !query output +facebook N N +google Y Y +linkedin N N + + +-- !query +select company from like_all_table where company like all ('%oo%',pat) +-- !query schema +struct +-- !query output +facebook +google + + +-- !query +select company from like_all_table where company not like all ('%oo%','%in','fa%') +-- !query schema +struct +-- !query output + + + +-- !query +select company from like_all_table where company not like all ('microsoft','%yoo%') +-- !query schema +struct +-- !query output +facebook +google +linkedin + + +-- !query +select company from like_all_table where company like all ('%oo%',null) +-- !query schema +struct +-- !query output + + + +-- !query +select company from like_all_table where company not like all ('%oo%',null) +-- !query schema +struct +-- !query output + + + +-- !query +select company from like_all_table where company not like all (null,null) +-- !query schema +struct +-- !query output + + + +-- !query +select company from like_all_table where company not like all (null,null) +-- !query schema +struct +-- !query output + diff --git a/sql/core/src/test/resources/sql-tests/results/like-any.sql.out b/sql/core/src/test/resources/sql-tests/results/like-any.sql.out new file mode 100644 index 0000000000000..92b4cff1506da --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/like-any.sql.out @@ -0,0 +1,115 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 11 + + +-- !query +CREATE OR REPLACE TEMPORARY VIEW like_any_table AS SELECT * FROM (VALUES + ('google', '%oo%'), + ('facebook', '%oo%'), + ('linkedin', '%in')) + as t1(company, pat) +-- !query schema +struct<> +-- !query output + + + +-- !query +select company from like_any_table where company like any ('%oo%','%in','fa%') +-- !query schema +struct +-- !query output +facebook +google +linkedin + + +-- !query +select company from like_any_table where company like any ('microsoft','%yoo%') +-- !query schema +struct +-- !query output + + + +-- !query +select + company, + CASE + WHEN company like any ('%oo%','%in','fa%') THEN 'Y' + ELSE 'N' + END AS is_available, + CASE + WHEN company like any ('%oo%','fa%') OR company like any ('%in','ms%') THEN 'Y' + ELSE 'N' + END AS mix +From like_any_table +-- !query schema +struct +-- !query output +facebook Y Y +google Y Y +linkedin Y Y + + +-- !query +select company from like_any_table where company like any ('%zz%',pat) +-- !query schema +struct +-- !query output +facebook +google +linkedin + + +-- !query +select company from like_any_table where company not like any ('%oo%','%in','fa%') +-- !query schema +struct +-- !query output +facebook +google +linkedin + + +-- !query +select company from like_any_table where company not like any ('microsoft','%yoo%') +-- !query schema +struct +-- !query output +facebook +google +linkedin + + +-- !query +select company from like_any_table where company like any ('%oo%',null) +-- !query schema +struct +-- !query output +facebook +google + + +-- !query +select company from like_any_table where company not like any ('%oo%',null) +-- !query schema +struct +-- !query output +linkedin + + +-- !query +select company from like_any_table where company like any (null,null) +-- !query schema +struct +-- !query output + + + +-- !query +select company from like_any_table where company not like any (null,null) +-- !query schema +struct +-- !query output + From 5475675617e7575b1c0f27544e2f7c100a551c5d Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Thu, 6 Feb 2020 22:02:19 +0800 Subject: [PATCH 2/8] operator -> quantifier --- .../main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 | 2 +- .../scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index cd8ea09ddd225..cf732201b3c99 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -748,7 +748,7 @@ predicate | NOT? kind=IN '(' expression (',' expression)* ')' | NOT? kind=IN '(' query ')' | NOT? kind=RLIKE pattern=valueExpression - | NOT? kind=LIKE operator=(ANY | ALL) '(' expression (',' expression)* ')' + | NOT? kind=LIKE quantifier=(ANY | ALL) '(' expression (',' expression)* ')' | NOT? kind=LIKE pattern=valueExpression (ESCAPE escapeChar=STRING)? | IS NOT? kind=NULL | IS NOT? kind=(TRUE | FALSE | UNKNOWN) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 5d24e2a6fdea1..3c8d8c77350a0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1387,7 +1387,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging case SqlBaseParser.IN => invertIfNotDefined(In(e, ctx.expression.asScala.map(expression))) case SqlBaseParser.LIKE => - Option(ctx.operator).map(_.getType) match { + Option(ctx.quantifier).map(_.getType) match { case Some(SqlBaseParser.ANY) if !ctx.expression.isEmpty => ctx.expression.asScala.map(expression).map(p => invertIfNotDefined(new Like(e, p))) .reduceLeft(Or) From de7c398b25008004dc3feeb3be2475ba0639baa1 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 7 Feb 2020 13:21:01 +0800 Subject: [PATCH 3/8] address comments --- .../spark/sql/catalyst/parser/SqlBase.g4 | 2 +- .../sql/catalyst/parser/AstBuilder.scala | 20 ++++++---- .../resources/sql-tests/inputs/like-all.sql | 27 +++++++------ .../resources/sql-tests/inputs/like-any.sql | 27 +++++++------ .../sql-tests/results/like-all.sql.out | 38 +++++++++++++------ .../sql-tests/results/like-any.sql.out | 38 +++++++++++++------ 6 files changed, 96 insertions(+), 56 deletions(-) diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index cf732201b3c99..a81af84669e46 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -748,7 +748,7 @@ predicate | NOT? kind=IN '(' expression (',' expression)* ')' | NOT? kind=IN '(' query ')' | NOT? kind=RLIKE pattern=valueExpression - | NOT? kind=LIKE quantifier=(ANY | ALL) '(' expression (',' expression)* ')' + | NOT? kind=LIKE quantifier=(ANY | ALL) ('('')' | '(' expression (',' expression)* ')') | NOT? kind=LIKE pattern=valueExpression (ESCAPE escapeChar=STRING)? | IS NOT? kind=NULL | IS NOT? kind=(TRUE | FALSE | UNKNOWN) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 3c8d8c77350a0..fef332ef94b13 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1357,7 +1357,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Add a predicate to the given expression. Supported expressions are: * - (NOT) BETWEEN * - (NOT) IN - * - (NOT) LIKE + * - (NOT) LIKE (ANY | SOME) * - (NOT) RLIKE * - IS (NOT) NULL. * - IS (NOT) (TRUE | FALSE | UNKNOWN) @@ -1375,6 +1375,14 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging case other => Seq(other) } + def getLikeQuantifierExps(expressions: java.util.List[ExpressionContext]): Seq[Expression] = { + if (expressions.isEmpty) { + throw new ParseException("Syntax error: expected something between '(' and ')'.", ctx) + } else { + expressions.asScala.map(expression).map(p => invertIfNotDefined(new Like(e, p))) + } + } + // Create the predicate. ctx.kind.getType match { case SqlBaseParser.BETWEEN => @@ -1388,12 +1396,10 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging invertIfNotDefined(In(e, ctx.expression.asScala.map(expression))) case SqlBaseParser.LIKE => Option(ctx.quantifier).map(_.getType) match { - case Some(SqlBaseParser.ANY) if !ctx.expression.isEmpty => - ctx.expression.asScala.map(expression).map(p => invertIfNotDefined(new Like(e, p))) - .reduceLeft(Or) - case Some(SqlBaseParser.ALL) if !ctx.expression.isEmpty => - ctx.expression.asScala.map(expression).map(p => invertIfNotDefined(new Like(e, p))) - .reduceLeft(And) + case Some(SqlBaseParser.ANY) => + getLikeQuantifierExps(ctx.expression).reduceLeft(Or) + case Some(SqlBaseParser.ALL) => + getLikeQuantifierExps(ctx.expression).reduceLeft(And) case _ => val escapeChar = Option(ctx.escapeChar).map(string).map { str => if (str.length != 1) { diff --git a/sql/core/src/test/resources/sql-tests/inputs/like-all.sql b/sql/core/src/test/resources/sql-tests/inputs/like-all.sql index de59ef162b40b..6f125d5b761b4 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/like-all.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/like-all.sql @@ -4,36 +4,39 @@ CREATE OR REPLACE TEMPORARY VIEW like_all_table AS SELECT * FROM (VALUES ('linkedin', '%in')) as t1(company, pat); -select company from like_all_table where company like all ('%oo%','%go%') ; +select company from like_all_table where company like all ('%oo%', '%go%'); -select company from like_all_table where company like all ('microsoft','%yoo%') ; +select company from like_all_table where company like all ('microsoft', '%yoo%'); select company, CASE - WHEN company like all ('%oo%','%go%') THEN 'Y' + WHEN company like all ('%oo%', '%go%') THEN 'Y' ELSE 'N' END AS is_available, CASE - WHEN company like all ('%oo%','go%') OR company like all ('%in','ms%') THEN 'Y' + WHEN company like all ('%oo%', 'go%') OR company like all ('%in', 'ms%') THEN 'Y' ELSE 'N' END AS mix From like_all_table ; ---Mix test with constant pattern and column value -select company from like_all_table where company like all ('%oo%',pat) ; +-- Mix test with constant pattern and column value +select company from like_all_table where company like all ('%oo%', pat); -- not like all test -select company from like_all_table where company not like all ('%oo%','%in','fa%') ; -select company from like_all_table where company not like all ('microsoft','%yoo%') ; +select company from like_all_table where company not like all ('%oo%', '%in', 'fa%'); +select company from like_all_table where company not like all ('microsoft', '%yoo%'); -- null test -select company from like_all_table where company like all ('%oo%',null) ; +select company from like_all_table where company like all ('%oo%', null); -select company from like_all_table where company not like all ('%oo%',null) ; +select company from like_all_table where company not like all ('%oo%', null); -select company from like_all_table where company not like all (null,null) ; +select company from like_all_table where company not like all (null, null); -select company from like_all_table where company not like all (null,null) ; \ No newline at end of file +select company from like_all_table where company not like all (null, null); + +-- negative case +select company from like_any_table where company like all (); diff --git a/sql/core/src/test/resources/sql-tests/inputs/like-any.sql b/sql/core/src/test/resources/sql-tests/inputs/like-any.sql index 3ea9f3c217eb7..fb3bcd1e9b88b 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/like-any.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/like-any.sql @@ -4,36 +4,39 @@ CREATE OR REPLACE TEMPORARY VIEW like_any_table AS SELECT * FROM (VALUES ('linkedin', '%in')) as t1(company, pat); -select company from like_any_table where company like any ('%oo%','%in','fa%') ; +select company from like_any_table where company like any ('%oo%', '%in', 'fa%'); -select company from like_any_table where company like any ('microsoft','%yoo%') ; +select company from like_any_table where company like any ('microsoft', '%yoo%'); select company, CASE - WHEN company like any ('%oo%','%in','fa%') THEN 'Y' + WHEN company like any ('%oo%', '%in', 'fa%') THEN 'Y' ELSE 'N' END AS is_available, CASE - WHEN company like any ('%oo%','fa%') OR company like any ('%in','ms%') THEN 'Y' + WHEN company like any ('%oo%', 'fa%') OR company like any ('%in', 'ms%') THEN 'Y' ELSE 'N' END AS mix From like_any_table; ---Mix test with constant pattern and column value -select company from like_any_table where company like any ('%zz%',pat) ; +-- Mix test with constant pattern and column value +select company from like_any_table where company like any ('%zz%', pat); -- not like any test -select company from like_any_table where company not like any ('%oo%','%in','fa%') ; -select company from like_any_table where company not like any ('microsoft','%yoo%') ; +select company from like_any_table where company not like any ('%oo%', '%in', 'fa%'); +select company from like_any_table where company not like any ('microsoft', '%yoo%'); -- null test -select company from like_any_table where company like any ('%oo%',null) ; +select company from like_any_table where company like any ('%oo%', null); -select company from like_any_table where company not like any ('%oo%',null) ; +select company from like_any_table where company not like any ('%oo%', null); -select company from like_any_table where company like any (null,null) ; +select company from like_any_table where company like any (null, null); -select company from like_any_table where company not like any (null,null) ; +select company from like_any_table where company not like any (null, null); + +-- negative case +select company from like_any_table where company like any (); diff --git a/sql/core/src/test/resources/sql-tests/results/like-all.sql.out b/sql/core/src/test/resources/sql-tests/results/like-all.sql.out index 239b7f4906ea1..da35a0ca407e3 100644 --- a/sql/core/src/test/resources/sql-tests/results/like-all.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/like-all.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 11 +-- Number of queries: 12 -- !query @@ -15,7 +15,7 @@ struct<> -- !query -select company from like_all_table where company like all ('%oo%','%go%') +select company from like_all_table where company like all ('%oo%', '%go%') -- !query schema struct -- !query output @@ -23,7 +23,7 @@ google -- !query -select company from like_all_table where company like all ('microsoft','%yoo%') +select company from like_all_table where company like all ('microsoft', '%yoo%') -- !query schema struct -- !query output @@ -34,11 +34,11 @@ struct select company, CASE - WHEN company like all ('%oo%','%go%') THEN 'Y' + WHEN company like all ('%oo%', '%go%') THEN 'Y' ELSE 'N' END AS is_available, CASE - WHEN company like all ('%oo%','go%') OR company like all ('%in','ms%') THEN 'Y' + WHEN company like all ('%oo%', 'go%') OR company like all ('%in', 'ms%') THEN 'Y' ELSE 'N' END AS mix From like_all_table @@ -51,7 +51,7 @@ linkedin N N -- !query -select company from like_all_table where company like all ('%oo%',pat) +select company from like_all_table where company like all ('%oo%', pat) -- !query schema struct -- !query output @@ -60,7 +60,7 @@ google -- !query -select company from like_all_table where company not like all ('%oo%','%in','fa%') +select company from like_all_table where company not like all ('%oo%', '%in', 'fa%') -- !query schema struct -- !query output @@ -68,7 +68,7 @@ struct -- !query -select company from like_all_table where company not like all ('microsoft','%yoo%') +select company from like_all_table where company not like all ('microsoft', '%yoo%') -- !query schema struct -- !query output @@ -78,7 +78,7 @@ linkedin -- !query -select company from like_all_table where company like all ('%oo%',null) +select company from like_all_table where company like all ('%oo%', null) -- !query schema struct -- !query output @@ -86,7 +86,7 @@ struct -- !query -select company from like_all_table where company not like all ('%oo%',null) +select company from like_all_table where company not like all ('%oo%', null) -- !query schema struct -- !query output @@ -94,7 +94,7 @@ struct -- !query -select company from like_all_table where company not like all (null,null) +select company from like_all_table where company not like all (null, null) -- !query schema struct -- !query output @@ -102,8 +102,22 @@ struct -- !query -select company from like_all_table where company not like all (null,null) +select company from like_all_table where company not like all (null, null) -- !query schema struct -- !query output + + +-- !query +select company from like_any_table where company like all () +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.parser.ParseException + +Syntax error: expected something between '(' and ')'.(line 1, pos 49) + +== SQL == +select company from like_any_table where company like all () +-------------------------------------------------^^^ diff --git a/sql/core/src/test/resources/sql-tests/results/like-any.sql.out b/sql/core/src/test/resources/sql-tests/results/like-any.sql.out index 92b4cff1506da..533e05ffae1de 100644 --- a/sql/core/src/test/resources/sql-tests/results/like-any.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/like-any.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 11 +-- Number of queries: 12 -- !query @@ -15,7 +15,7 @@ struct<> -- !query -select company from like_any_table where company like any ('%oo%','%in','fa%') +select company from like_any_table where company like any ('%oo%', '%in', 'fa%') -- !query schema struct -- !query output @@ -25,7 +25,7 @@ linkedin -- !query -select company from like_any_table where company like any ('microsoft','%yoo%') +select company from like_any_table where company like any ('microsoft', '%yoo%') -- !query schema struct -- !query output @@ -36,11 +36,11 @@ struct select company, CASE - WHEN company like any ('%oo%','%in','fa%') THEN 'Y' + WHEN company like any ('%oo%', '%in', 'fa%') THEN 'Y' ELSE 'N' END AS is_available, CASE - WHEN company like any ('%oo%','fa%') OR company like any ('%in','ms%') THEN 'Y' + WHEN company like any ('%oo%', 'fa%') OR company like any ('%in', 'ms%') THEN 'Y' ELSE 'N' END AS mix From like_any_table @@ -53,7 +53,7 @@ linkedin Y Y -- !query -select company from like_any_table where company like any ('%zz%',pat) +select company from like_any_table where company like any ('%zz%', pat) -- !query schema struct -- !query output @@ -63,7 +63,7 @@ linkedin -- !query -select company from like_any_table where company not like any ('%oo%','%in','fa%') +select company from like_any_table where company not like any ('%oo%', '%in', 'fa%') -- !query schema struct -- !query output @@ -73,7 +73,7 @@ linkedin -- !query -select company from like_any_table where company not like any ('microsoft','%yoo%') +select company from like_any_table where company not like any ('microsoft', '%yoo%') -- !query schema struct -- !query output @@ -83,7 +83,7 @@ linkedin -- !query -select company from like_any_table where company like any ('%oo%',null) +select company from like_any_table where company like any ('%oo%', null) -- !query schema struct -- !query output @@ -92,7 +92,7 @@ google -- !query -select company from like_any_table where company not like any ('%oo%',null) +select company from like_any_table where company not like any ('%oo%', null) -- !query schema struct -- !query output @@ -100,7 +100,7 @@ linkedin -- !query -select company from like_any_table where company like any (null,null) +select company from like_any_table where company like any (null, null) -- !query schema struct -- !query output @@ -108,8 +108,22 @@ struct -- !query -select company from like_any_table where company not like any (null,null) +select company from like_any_table where company not like any (null, null) -- !query schema struct -- !query output + + +-- !query +select company from like_any_table where company like any () +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.parser.ParseException + +Syntax error: expected something between '(' and ')'.(line 1, pos 49) + +== SQL == +select company from like_any_table where company like any () +-------------------------------------------------^^^ From caea82a5a8108b5012071b54b07691bca226e52d Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 7 Feb 2020 16:57:15 +0800 Subject: [PATCH 4/8] SOME -> ALL --- .../scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index fef332ef94b13..4faeb36e3d8db 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1357,7 +1357,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Add a predicate to the given expression. Supported expressions are: * - (NOT) BETWEEN * - (NOT) IN - * - (NOT) LIKE (ANY | SOME) + * - (NOT) LIKE (ANY | ALL) * - (NOT) RLIKE * - IS (NOT) NULL. * - IS (NOT) (TRUE | FALSE | UNKNOWN) From 88ca4c2de2fe64a29092f92198baea28674cd1c6 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 7 Feb 2020 22:24:33 +0800 Subject: [PATCH 5/8] Capitalize SQL keywords --- .../resources/sql-tests/inputs/like-all.sql | 33 ++++++++----------- .../resources/sql-tests/inputs/like-any.sql | 31 ++++++++--------- .../sql-tests/results/like-all.sql.out | 30 ++++++++--------- .../sql-tests/results/like-any.sql.out | 28 ++++++++-------- 4 files changed, 56 insertions(+), 66 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/like-all.sql b/sql/core/src/test/resources/sql-tests/inputs/like-all.sql index 6f125d5b761b4..4bcdd4a65da10 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/like-all.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/like-all.sql @@ -4,39 +4,34 @@ CREATE OR REPLACE TEMPORARY VIEW like_all_table AS SELECT * FROM (VALUES ('linkedin', '%in')) as t1(company, pat); -select company from like_all_table where company like all ('%oo%', '%go%'); +SELECT company FROM like_all_table WHERE company LIKE ALL ('%oo%', '%go%'); -select company from like_all_table where company like all ('microsoft', '%yoo%'); +SELECT company FROM like_all_table WHERE company LIKE ALL ('microsoft', '%yoo%'); -select +SELECT company, CASE - WHEN company like all ('%oo%', '%go%') THEN 'Y' + WHEN company LIKE ALL ('%oo%', '%go%') THEN 'Y' ELSE 'N' END AS is_available, CASE - WHEN company like all ('%oo%', 'go%') OR company like all ('%in', 'ms%') THEN 'Y' + WHEN company LIKE ALL ('%oo%', 'go%') OR company LIKE ALL ('%in', 'ms%') THEN 'Y' ELSE 'N' END AS mix -From like_all_table ; +FROM like_all_table ; -- Mix test with constant pattern and column value -select company from like_all_table where company like all ('%oo%', pat); +SELECT company FROM like_all_table WHERE company LIKE ALL ('%oo%', pat); -- not like all test - -select company from like_all_table where company not like all ('%oo%', '%in', 'fa%'); -select company from like_all_table where company not like all ('microsoft', '%yoo%'); +SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('%oo%', '%in', 'fa%'); +SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('microsoft', '%yoo%'); -- null test - -select company from like_all_table where company like all ('%oo%', null); - -select company from like_all_table where company not like all ('%oo%', null); - -select company from like_all_table where company not like all (null, null); - -select company from like_all_table where company not like all (null, null); +SELECT company FROM like_all_table WHERE company LIKE ALL ('%oo%', NULL); +SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('%oo%', NULL); +SELECT company FROM like_all_table WHERE company LIKE ALL (NULL, NULL); +SELECT company FROM like_all_table WHERE company NOT LIKE ALL (NULL, NULL); -- negative case -select company from like_any_table where company like all (); +SELECT company FROM like_any_table WHERE company LIKE ALL (); diff --git a/sql/core/src/test/resources/sql-tests/inputs/like-any.sql b/sql/core/src/test/resources/sql-tests/inputs/like-any.sql index fb3bcd1e9b88b..7b857003ef2fc 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/like-any.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/like-any.sql @@ -4,39 +4,34 @@ CREATE OR REPLACE TEMPORARY VIEW like_any_table AS SELECT * FROM (VALUES ('linkedin', '%in')) as t1(company, pat); -select company from like_any_table where company like any ('%oo%', '%in', 'fa%'); +SELECT company FROM like_any_table WHERE company LIKE ANY ('%oo%', '%in', 'fa%'); -select company from like_any_table where company like any ('microsoft', '%yoo%'); +SELECT company FROM like_any_table WHERE company LIKE ANY ('microsoft', '%yoo%'); select company, CASE - WHEN company like any ('%oo%', '%in', 'fa%') THEN 'Y' + WHEN company LIKE ANY ('%oo%', '%in', 'fa%') THEN 'Y' ELSE 'N' END AS is_available, CASE - WHEN company like any ('%oo%', 'fa%') OR company like any ('%in', 'ms%') THEN 'Y' + WHEN company LIKE ANY ('%oo%', 'fa%') OR company LIKE ANY ('%in', 'ms%') THEN 'Y' ELSE 'N' END AS mix -From like_any_table; +FROM like_any_table; -- Mix test with constant pattern and column value -select company from like_any_table where company like any ('%zz%', pat); +SELECT company FROM like_any_table WHERE company LIKE ANY ('%zz%', pat); -- not like any test - -select company from like_any_table where company not like any ('%oo%', '%in', 'fa%'); -select company from like_any_table where company not like any ('microsoft', '%yoo%'); +SELECT company FROM like_any_table WHERE company NOT LIKE ANY ('%oo%', '%in', 'fa%'); +SELECT company FROM like_any_table WHERE company NOT LIKE ANY ('microsoft', '%yoo%'); -- null test - -select company from like_any_table where company like any ('%oo%', null); - -select company from like_any_table where company not like any ('%oo%', null); - -select company from like_any_table where company like any (null, null); - -select company from like_any_table where company not like any (null, null); +SELECT company FROM like_any_table WHERE company LIKE ANY ('%oo%', NULL); +SELECT company FROM like_any_table WHERE company NOT LIKE ANY ('%oo%', NULL); +SELECT company FROM like_any_table WHERE company LIKE ANY (NULL, NULL); +SELECT company FROM like_any_table WHERE company NOT LIKE ANY (NULL, NULL); -- negative case -select company from like_any_table where company like any (); +SELECT company FROM like_any_table WHERE company LIKE ANY (); diff --git a/sql/core/src/test/resources/sql-tests/results/like-all.sql.out b/sql/core/src/test/resources/sql-tests/results/like-all.sql.out index da35a0ca407e3..84e6571c2951f 100644 --- a/sql/core/src/test/resources/sql-tests/results/like-all.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/like-all.sql.out @@ -15,7 +15,7 @@ struct<> -- !query -select company from like_all_table where company like all ('%oo%', '%go%') +SELECT company FROM like_all_table WHERE company LIKE ALL ('%oo%', '%go%') -- !query schema struct -- !query output @@ -23,7 +23,7 @@ google -- !query -select company from like_all_table where company like all ('microsoft', '%yoo%') +SELECT company FROM like_all_table WHERE company LIKE ALL ('microsoft', '%yoo%') -- !query schema struct -- !query output @@ -31,17 +31,17 @@ struct -- !query -select +SELECT company, CASE - WHEN company like all ('%oo%', '%go%') THEN 'Y' + WHEN company LIKE ALL ('%oo%', '%go%') THEN 'Y' ELSE 'N' END AS is_available, CASE - WHEN company like all ('%oo%', 'go%') OR company like all ('%in', 'ms%') THEN 'Y' + WHEN company LIKE ALL ('%oo%', 'go%') OR company LIKE ALL ('%in', 'ms%') THEN 'Y' ELSE 'N' END AS mix -From like_all_table +FROM like_all_table -- !query schema struct -- !query output @@ -51,7 +51,7 @@ linkedin N N -- !query -select company from like_all_table where company like all ('%oo%', pat) +SELECT company FROM like_all_table WHERE company LIKE ALL ('%oo%', pat) -- !query schema struct -- !query output @@ -60,7 +60,7 @@ google -- !query -select company from like_all_table where company not like all ('%oo%', '%in', 'fa%') +SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('%oo%', '%in', 'fa%') -- !query schema struct -- !query output @@ -68,7 +68,7 @@ struct -- !query -select company from like_all_table where company not like all ('microsoft', '%yoo%') +SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('microsoft', '%yoo%') -- !query schema struct -- !query output @@ -78,7 +78,7 @@ linkedin -- !query -select company from like_all_table where company like all ('%oo%', null) +SELECT company FROM like_all_table WHERE company LIKE ALL ('%oo%', NULL) -- !query schema struct -- !query output @@ -86,7 +86,7 @@ struct -- !query -select company from like_all_table where company not like all ('%oo%', null) +SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('%oo%', NULL) -- !query schema struct -- !query output @@ -94,7 +94,7 @@ struct -- !query -select company from like_all_table where company not like all (null, null) +SELECT company FROM like_all_table WHERE company LIKE ALL (NULL, NULL) -- !query schema struct -- !query output @@ -102,7 +102,7 @@ struct -- !query -select company from like_all_table where company not like all (null, null) +SELECT company FROM like_all_table WHERE company NOT LIKE ALL (NULL, NULL) -- !query schema struct -- !query output @@ -110,7 +110,7 @@ struct -- !query -select company from like_any_table where company like all () +SELECT company FROM like_any_table WHERE company LIKE ALL () -- !query schema struct<> -- !query output @@ -119,5 +119,5 @@ org.apache.spark.sql.catalyst.parser.ParseException Syntax error: expected something between '(' and ')'.(line 1, pos 49) == SQL == -select company from like_any_table where company like all () +SELECT company FROM like_any_table WHERE company LIKE ALL () -------------------------------------------------^^^ diff --git a/sql/core/src/test/resources/sql-tests/results/like-any.sql.out b/sql/core/src/test/resources/sql-tests/results/like-any.sql.out index 533e05ffae1de..2433d948c8980 100644 --- a/sql/core/src/test/resources/sql-tests/results/like-any.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/like-any.sql.out @@ -15,7 +15,7 @@ struct<> -- !query -select company from like_any_table where company like any ('%oo%', '%in', 'fa%') +SELECT company FROM like_any_table WHERE company LIKE ANY ('%oo%', '%in', 'fa%') -- !query schema struct -- !query output @@ -25,7 +25,7 @@ linkedin -- !query -select company from like_any_table where company like any ('microsoft', '%yoo%') +SELECT company FROM like_any_table WHERE company LIKE ANY ('microsoft', '%yoo%') -- !query schema struct -- !query output @@ -36,14 +36,14 @@ struct select company, CASE - WHEN company like any ('%oo%', '%in', 'fa%') THEN 'Y' + WHEN company LIKE ANY ('%oo%', '%in', 'fa%') THEN 'Y' ELSE 'N' END AS is_available, CASE - WHEN company like any ('%oo%', 'fa%') OR company like any ('%in', 'ms%') THEN 'Y' + WHEN company LIKE ANY ('%oo%', 'fa%') OR company LIKE ANY ('%in', 'ms%') THEN 'Y' ELSE 'N' END AS mix -From like_any_table +FROM like_any_table -- !query schema struct -- !query output @@ -53,7 +53,7 @@ linkedin Y Y -- !query -select company from like_any_table where company like any ('%zz%', pat) +SELECT company FROM like_any_table WHERE company LIKE ANY ('%zz%', pat) -- !query schema struct -- !query output @@ -63,7 +63,7 @@ linkedin -- !query -select company from like_any_table where company not like any ('%oo%', '%in', 'fa%') +SELECT company FROM like_any_table WHERE company NOT LIKE ANY ('%oo%', '%in', 'fa%') -- !query schema struct -- !query output @@ -73,7 +73,7 @@ linkedin -- !query -select company from like_any_table where company not like any ('microsoft', '%yoo%') +SELECT company FROM like_any_table WHERE company NOT LIKE ANY ('microsoft', '%yoo%') -- !query schema struct -- !query output @@ -83,7 +83,7 @@ linkedin -- !query -select company from like_any_table where company like any ('%oo%', null) +SELECT company FROM like_any_table WHERE company LIKE ANY ('%oo%', NULL) -- !query schema struct -- !query output @@ -92,7 +92,7 @@ google -- !query -select company from like_any_table where company not like any ('%oo%', null) +SELECT company FROM like_any_table WHERE company NOT LIKE ANY ('%oo%', NULL) -- !query schema struct -- !query output @@ -100,7 +100,7 @@ linkedin -- !query -select company from like_any_table where company like any (null, null) +SELECT company FROM like_any_table WHERE company LIKE ANY (NULL, NULL) -- !query schema struct -- !query output @@ -108,7 +108,7 @@ struct -- !query -select company from like_any_table where company not like any (null, null) +SELECT company FROM like_any_table WHERE company NOT LIKE ANY (NULL, NULL) -- !query schema struct -- !query output @@ -116,7 +116,7 @@ struct -- !query -select company from like_any_table where company like any () +SELECT company FROM like_any_table WHERE company LIKE ANY () -- !query schema struct<> -- !query output @@ -125,5 +125,5 @@ org.apache.spark.sql.catalyst.parser.ParseException Syntax error: expected something between '(' and ')'.(line 1, pos 49) == SQL == -select company from like_any_table where company like any () +SELECT company FROM like_any_table WHERE company LIKE ANY () -------------------------------------------------^^^ From 0656b051a3b85dd0ba02f57b9dc30a99848e7bc0 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Sat, 11 Apr 2020 16:23:10 +0800 Subject: [PATCH 6/8] Add a test case from our customers --- .../parser/ExpressionParserSuite.scala | 9 +++++++++ .../resources/sql-tests/inputs/like-all.sql | 2 ++ .../resources/sql-tests/inputs/like-any.sql | 2 ++ .../sql-tests/results/like-all.sql.out | 19 ++++++++++++++++++- .../sql-tests/results/like-any.sql.out | 19 ++++++++++++++++++- 5 files changed, 49 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala index 522d49e448f1f..0fbe6f90c4212 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala @@ -208,6 +208,15 @@ class ExpressionParserSuite extends AnalysisTest { assertEqual("a rlike 'pattern\\t\\n'", 'a rlike "pattern\\t\\n", parser) } + test("(NOT) LIKE (ANY | ALL) expressions") { + assertEqual("a like any ('foo%', 'bar%')", ('a like "foo%") || ('a like "bar%")) + assertEqual("a not like any ('foo%', 'bar%')", !('a like "foo%") || !('a like "bar%")) + assertEqual("not (a like any ('foo%', 'bar%'))", !(('a like "foo%") || ('a like "bar%"))) + assertEqual("a like all ('foo%', 'bar%')", ('a like "foo%") && ('a like "bar%")) + assertEqual("a not like all ('foo%', 'bar%')", !('a like "foo%") && !('a like "bar%")) + assertEqual("not (a like all ('foo%', 'bar%'))", !(('a like "foo%") && ('a like "bar%"))) + } + test("is null expressions") { assertEqual("a is null", 'a.isNull) assertEqual("a is not null", 'a.isNotNull) diff --git a/sql/core/src/test/resources/sql-tests/inputs/like-all.sql b/sql/core/src/test/resources/sql-tests/inputs/like-all.sql index 4bcdd4a65da10..a084dbef61a0c 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/like-all.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/like-all.sql @@ -26,6 +26,8 @@ SELECT company FROM like_all_table WHERE company LIKE ALL ('%oo%', pat); -- not like all test SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('%oo%', '%in', 'fa%'); SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('microsoft', '%yoo%'); +SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('%oo%', 'fa%'); +SELECT company FROM like_all_table WHERE NOT company LIKE ALL ('%oo%', 'fa%'); -- null test SELECT company FROM like_all_table WHERE company LIKE ALL ('%oo%', NULL); diff --git a/sql/core/src/test/resources/sql-tests/inputs/like-any.sql b/sql/core/src/test/resources/sql-tests/inputs/like-any.sql index 7b857003ef2fc..5758a2a494944 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/like-any.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/like-any.sql @@ -26,6 +26,8 @@ SELECT company FROM like_any_table WHERE company LIKE ANY ('%zz%', pat); -- not like any test SELECT company FROM like_any_table WHERE company NOT LIKE ANY ('%oo%', '%in', 'fa%'); SELECT company FROM like_any_table WHERE company NOT LIKE ANY ('microsoft', '%yoo%'); +SELECT company FROM like_any_table WHERE company NOT LIKE ANY ('%oo%', 'fa%'); +SELECT company FROM like_any_table WHERE NOT company LIKE ANY ('%oo%', 'fa%'); -- null test SELECT company FROM like_any_table WHERE company LIKE ANY ('%oo%', NULL); diff --git a/sql/core/src/test/resources/sql-tests/results/like-all.sql.out b/sql/core/src/test/resources/sql-tests/results/like-all.sql.out index 84e6571c2951f..30bd10920d401 100644 --- a/sql/core/src/test/resources/sql-tests/results/like-all.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/like-all.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 12 +-- Number of queries: 14 -- !query @@ -77,6 +77,23 @@ google linkedin +-- !query +SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('%oo%', 'fa%') +-- !query schema +struct +-- !query output +linkedin + + +-- !query +SELECT company FROM like_all_table WHERE NOT company LIKE ALL ('%oo%', 'fa%') +-- !query schema +struct +-- !query output +google +linkedin + + -- !query SELECT company FROM like_all_table WHERE company LIKE ALL ('%oo%', NULL) -- !query schema diff --git a/sql/core/src/test/resources/sql-tests/results/like-any.sql.out b/sql/core/src/test/resources/sql-tests/results/like-any.sql.out index 2433d948c8980..8c2a1c72d10bb 100644 --- a/sql/core/src/test/resources/sql-tests/results/like-any.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/like-any.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 12 +-- Number of queries: 14 -- !query @@ -82,6 +82,23 @@ google linkedin +-- !query +SELECT company FROM like_any_table WHERE company NOT LIKE ANY ('%oo%', 'fa%') +-- !query schema +struct +-- !query output +google +linkedin + + +-- !query +SELECT company FROM like_any_table WHERE NOT company LIKE ANY ('%oo%', 'fa%') +-- !query schema +struct +-- !query output +linkedin + + -- !query SELECT company FROM like_any_table WHERE company LIKE ANY ('%oo%', NULL) -- !query schema From cf4666ff2c80c4cc34185970054346ab06db4b05 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Thu, 23 Apr 2020 23:37:03 +0800 Subject: [PATCH 7/8] Address comments --- .../spark/sql/catalyst/parser/SqlBase.g4 | 2 +- .../sql/catalyst/parser/AstBuilder.scala | 14 ++++++------ .../parser/ExpressionParserSuite.scala | 22 ++++++++++++------- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 1508079b9ff6f..e49bc07f18c59 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -766,7 +766,7 @@ predicate | NOT? kind=IN '(' expression (',' expression)* ')' | NOT? kind=IN '(' query ')' | NOT? kind=RLIKE pattern=valueExpression - | NOT? kind=LIKE quantifier=(ANY | ALL) ('('')' | '(' expression (',' expression)* ')') + | NOT? kind=LIKE quantifier=(ANY | SOME | ALL) ('('')' | '(' expression (',' expression)* ')') | NOT? kind=LIKE pattern=valueExpression (ESCAPE escapeChar=STRING)? | IS NOT? kind=NULL | IS NOT? kind=(TRUE | FALSE | UNKNOWN) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index d1a07ca848557..fe02c967bcce7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1373,7 +1373,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Add a predicate to the given expression. Supported expressions are: * - (NOT) BETWEEN * - (NOT) IN - * - (NOT) LIKE (ANY | ALL) + * - (NOT) LIKE (ANY | SOME | ALL) * - (NOT) RLIKE * - IS (NOT) NULL. * - IS (NOT) (TRUE | FALSE | UNKNOWN) @@ -1391,9 +1391,9 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging case other => Seq(other) } - def getLikeQuantifierExps(expressions: java.util.List[ExpressionContext]): Seq[Expression] = { + def getLikeQuantifierExprs(expressions: java.util.List[ExpressionContext]): Seq[Expression] = { if (expressions.isEmpty) { - throw new ParseException("Syntax error: expected something between '(' and ')'.", ctx) + throw new ParseException("Expected something between '(' and ')'.", ctx) } else { expressions.asScala.map(expression).map(p => invertIfNotDefined(new Like(e, p))) } @@ -1412,15 +1412,15 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging invertIfNotDefined(In(e, ctx.expression.asScala.map(expression))) case SqlBaseParser.LIKE => Option(ctx.quantifier).map(_.getType) match { - case Some(SqlBaseParser.ANY) => - getLikeQuantifierExps(ctx.expression).reduceLeft(Or) + case Some(SqlBaseParser.ANY) | Some(SqlBaseParser.SOME) => + getLikeQuantifierExprs(ctx.expression).reduceLeft(Or) case Some(SqlBaseParser.ALL) => - getLikeQuantifierExps(ctx.expression).reduceLeft(And) + getLikeQuantifierExprs(ctx.expression).reduceLeft(And) case _ => val escapeChar = Option(ctx.escapeChar).map(string).map { str => if (str.length != 1) { throw new ParseException("Invalid escape string." + - "Escape string must contains only one character.", ctx) + "Escape string must contain only one character.", ctx) } str.charAt(0) }.getOrElse('\\') diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala index 0fbe6f90c4212..b583e1b79c8d8 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala @@ -188,7 +188,7 @@ class ExpressionParserSuite extends AnalysisTest { } test("like escape expressions") { - val message = "Escape string must contains only one character." + val message = "Escape string must contain only one character." assertEqual("a like 'pattern%' escape '#'", 'a.like("pattern%", '#')) assertEqual("a like 'pattern%' escape '\"'", 'a.like("pattern%", '\"')) intercept("a like 'pattern%' escape '##'", message) @@ -208,13 +208,19 @@ class ExpressionParserSuite extends AnalysisTest { assertEqual("a rlike 'pattern\\t\\n'", 'a rlike "pattern\\t\\n", parser) } - test("(NOT) LIKE (ANY | ALL) expressions") { - assertEqual("a like any ('foo%', 'bar%')", ('a like "foo%") || ('a like "bar%")) - assertEqual("a not like any ('foo%', 'bar%')", !('a like "foo%") || !('a like "bar%")) - assertEqual("not (a like any ('foo%', 'bar%'))", !(('a like "foo%") || ('a like "bar%"))) - assertEqual("a like all ('foo%', 'bar%')", ('a like "foo%") && ('a like "bar%")) - assertEqual("a not like all ('foo%', 'bar%')", !('a like "foo%") && !('a like "bar%")) - assertEqual("not (a like all ('foo%', 'bar%'))", !(('a like "foo%") && ('a like "bar%"))) + test("(NOT) LIKE (ANY | SOME | ALL) expressions") { + Seq("any", "some").foreach { quantifier => + assertEqual(s"a like $quantifier ('foo%', 'b%')", ('a like "foo%") || ('a like "b%")) + assertEqual(s"a not like $quantifier ('foo%', 'b%')", !('a like "foo%") || !('a like "b%")) + assertEqual(s"not (a like $quantifier ('foo%', 'b%'))", !(('a like "foo%") || ('a like "b%"))) + } + assertEqual("a like all ('foo%', 'b%')", ('a like "foo%") && ('a like "b%")) + assertEqual("a not like all ('foo%', 'b%')", !('a like "foo%") && !('a like "b%")) + assertEqual("not (a like all ('foo%', 'b%'))", !(('a like "foo%") && ('a like "b%"))) + + Seq("any", "some", "all").foreach { quantifier => + intercept(s"a like $quantifier()", "Expected something between '(' and ')'") + } } test("is null expressions") { From f9af131410fe52ca36453ceb9a5636d9d48b6dc5 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 24 Apr 2020 10:54:35 +0800 Subject: [PATCH 8/8] Fix test error --- sql/core/src/test/resources/sql-tests/results/like-all.sql.out | 2 +- sql/core/src/test/resources/sql-tests/results/like-any.sql.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/like-all.sql.out b/sql/core/src/test/resources/sql-tests/results/like-all.sql.out index 30bd10920d401..b4bb69c9e511c 100644 --- a/sql/core/src/test/resources/sql-tests/results/like-all.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/like-all.sql.out @@ -133,7 +133,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -Syntax error: expected something between '(' and ')'.(line 1, pos 49) +Expected something between '(' and ')'.(line 1, pos 49) == SQL == SELECT company FROM like_any_table WHERE company LIKE ALL () diff --git a/sql/core/src/test/resources/sql-tests/results/like-any.sql.out b/sql/core/src/test/resources/sql-tests/results/like-any.sql.out index 8c2a1c72d10bb..e46ac6d858931 100644 --- a/sql/core/src/test/resources/sql-tests/results/like-any.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/like-any.sql.out @@ -139,7 +139,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -Syntax error: expected something between '(' and ')'.(line 1, pos 49) +Expected something between '(' and ')'.(line 1, pos 49) == SQL == SELECT company FROM like_any_table WHERE company LIKE ANY ()