From 3d2a3ba7d7fb2c8368c5dc5c91b21daf14392562 Mon Sep 17 00:00:00 2001 From: samelamin Date: Thu, 30 Mar 2017 22:59:27 +0100 Subject: [PATCH 01/11] add failing test --- .../sql/catalyst/parser/PlanParserSuite.scala | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala index 411777d6e85a..1c0d8d8c0ffe 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala @@ -471,6 +471,20 @@ class PlanParserSuite extends PlanTest { UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) } + test("table valued function case insensitive") { + assertEqual( + "select * from RangE(2)", + UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) + + assertEqual( + "select * from rAnGe(2)", + UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) + + assertEqual( + "select * from RANGE(2)", + UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) + } + test("inline table") { assertEqual("values 1, 2, 3, 4", UnresolvedInlineTable(Seq("col1"), Seq(1, 2, 3, 4).map(x => Seq(Literal(x))))) From ee15dc0bed3818b11321b8dd60c5e0499964d343 Mon Sep 17 00:00:00 2001 From: samelamin Date: Thu, 30 Mar 2017 23:07:36 +0100 Subject: [PATCH 02/11] SPARK-20145: lower case value passed into map to make the statement case insensitive --- .../sql/catalyst/analysis/ResolveTableValuedFunctions.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala index 6b3bb68538dd..63540e2cc935 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala @@ -105,7 +105,7 @@ object ResolveTableValuedFunctions extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators { case u: UnresolvedTableValuedFunction if u.functionArgs.forall(_.resolved) => - builtinFunctions.get(u.functionName) match { + builtinFunctions.get(u.functionName.toLowerCase) match { case Some(tvf) => val resolved = tvf.flatMap { case (argList, resolver) => argList.implicitCast(u.functionArgs) match { From 407bdbf1ae66b73d47611477b9ce0f03dc37ff7b Mon Sep 17 00:00:00 2001 From: samelamin Date: Sun, 2 Apr 2017 00:23:36 +0100 Subject: [PATCH 03/11] use sql conf to get case sensitivity --- .../sql/catalyst/analysis/unresolved.scala | 9 +++++- .../sql/catalyst/parser/AstBuilder.scala | 9 ++++-- .../sql/catalyst/parser/PlanParserSuite.scala | 28 +++++++++---------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala index 262b894e2a0a..ea3fb9fa0933 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala @@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, Codege import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan} import org.apache.spark.sql.catalyst.trees.TreeNode import org.apache.spark.sql.catalyst.util.quoteIdentifier +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, Metadata, StructType} /** @@ -69,9 +70,15 @@ case class UnresolvedInlineTable( * select * from range(10); * }}} */ -case class UnresolvedTableValuedFunction(functionName: String, functionArgs: Seq[Expression]) +case class UnresolvedTableValuedFunction(conf: SQLConf, + var functionName: String, + functionArgs: Seq[Expression]) extends LeafNode { + if (!conf.caseSensitiveAnalysis) { + functionName = functionName.toLowerCase + } + override def output: Seq[Attribute] = Nil override lazy val resolved = false diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 162051a8c0e4..6ebbdddaca5f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -29,6 +29,7 @@ import org.antlr.v4.runtime.tree.{ParseTree, RuleNode, TerminalNode} import org.apache.spark.internal.Logging import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} +import org.apache.spark.sql.catalyst.SimpleCatalystConf import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.parser.SqlBaseParser._ @@ -44,6 +45,7 @@ import org.apache.spark.util.random.RandomSampler */ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging { import ParserUtils._ + val sqlConf = SimpleCatalystConf(caseSensitiveAnalysis = false) protected def typedVisit[T](ctx: ParseTree): T = { ctx.accept(this).asInstanceOf[T] @@ -674,12 +676,15 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging { /** * Create a table-valued function call with arguments, e.g. range(1000) - */ + */ override def visitTableValuedFunction(ctx: TableValuedFunctionContext) : LogicalPlan = withOrigin(ctx) { - UnresolvedTableValuedFunction(ctx.identifier.getText, ctx.expression.asScala.map(expression)) + UnresolvedTableValuedFunction(sqlConf, + ctx.identifier.getText, + ctx.expression.asScala.map(expression)) } + /** * Create an inline table (a virtual table in Hive parlance). */ diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala index 1c0d8d8c0ffe..843799f99f68 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala @@ -17,11 +17,12 @@ package org.apache.spark.sql.catalyst.parser -import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.analysis.{UnresolvedGenerator, UnresolvedInlineTable, UnresolvedTableValuedFunction} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.catalyst.FunctionIdentifier +import org.apache.spark.sql.catalyst.SimpleCatalystConf import org.apache.spark.sql.types.IntegerType /** @@ -33,6 +34,8 @@ class PlanParserSuite extends PlanTest { import CatalystSqlParser._ import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.dsl.plans._ + override val conf = SimpleCatalystConf(caseSensitiveAnalysis = false) + private def assertEqual(sqlCommand: String, plan: LogicalPlan): Unit = { comparePlans(parsePlan(sqlCommand), plan) @@ -52,6 +55,14 @@ class PlanParserSuite extends PlanTest { assertEqual("SELECT * FROM a", plan) } + + test("case insensitive range queries") { + val plan = UnresolvedTableValuedFunction(conf, "range", Literal(2) :: Nil).select(star()) + assertEqual("select * from rAngE(2)", plan) + assertEqual("select * from RaNgE(2)", plan) + assertEqual("select * from RANGE(2)", plan) + } + test("explain") { intercept("EXPLAIN logical SELECT 1", "Unsupported SQL statement") intercept("EXPLAIN formatted SELECT 1", "Unsupported SQL statement") @@ -468,22 +479,9 @@ class PlanParserSuite extends PlanTest { test("table valued function") { assertEqual( "select * from range(2)", - UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) + UnresolvedTableValuedFunction(conf, "range", Literal(2) :: Nil).select(star())) } - test("table valued function case insensitive") { - assertEqual( - "select * from RangE(2)", - UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) - - assertEqual( - "select * from rAnGe(2)", - UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) - - assertEqual( - "select * from RANGE(2)", - UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) - } test("inline table") { assertEqual("values 1, 2, 3, 4", From ccbfcb32c1e5ccf83c3aea473e309ec1a9b9fbf9 Mon Sep 17 00:00:00 2001 From: samelamin Date: Sun, 2 Apr 2017 12:58:43 +0100 Subject: [PATCH 04/11] pass failing test --- .../sql-tests/results/sql-compatibility-functions.sql.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out index 9f0b95994be5..5f600a8b542e 100644 --- a/sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out @@ -75,7 +75,7 @@ struct -- !query 8 output == Parsed Logical Plan == 'Project [unresolvedalias('ifnull('id, x), None), unresolvedalias('nullif('id, x), None), unresolvedalias('nvl('id, x), None), unresolvedalias('nvl2('id, x, y), None)] -+- 'UnresolvedTableValuedFunction range, [2] ++- 'UnresolvedTableValuedFunction SimpleCatalystConf(false,true,true,100,10,20,1000,true,false,false,false,12,false,/user/hive/warehouse,America/Los_Angeles,100,true), range, [2] == Analyzed Logical Plan == ifnull(`id`, 'x'): string, nullif(`id`, 'x'): bigint, nvl(`id`, 'x'): string, nvl2(`id`, 'x', 'y'): string From e1541b4a88367aa28e75d859b806a0ca24452155 Mon Sep 17 00:00:00 2001 From: samelamin Date: Sun, 2 Apr 2017 18:07:48 +0100 Subject: [PATCH 05/11] pass failing test (reverted from commit ccbfcb32c1e5ccf83c3aea473e309ec1a9b9fbf9) --- .../sql-tests/results/sql-compatibility-functions.sql.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out index 5f600a8b542e..9f0b95994be5 100644 --- a/sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out @@ -75,7 +75,7 @@ struct -- !query 8 output == Parsed Logical Plan == 'Project [unresolvedalias('ifnull('id, x), None), unresolvedalias('nullif('id, x), None), unresolvedalias('nvl('id, x), None), unresolvedalias('nvl2('id, x, y), None)] -+- 'UnresolvedTableValuedFunction SimpleCatalystConf(false,true,true,100,10,20,1000,true,false,false,false,12,false,/user/hive/warehouse,America/Los_Angeles,100,true), range, [2] ++- 'UnresolvedTableValuedFunction range, [2] == Analyzed Logical Plan == ifnull(`id`, 'x'): string, nullif(`id`, 'x'): bigint, nvl(`id`, 'x'): string, nvl2(`id`, 'x', 'y'): string From de7b8a6cbc2867977d019358d13cb98d53e8821f Mon Sep 17 00:00:00 2001 From: samelamin Date: Sun, 2 Apr 2017 18:07:49 +0100 Subject: [PATCH 06/11] use sql conf to get case sensitivity (reverted from commit 407bdbf1ae66b73d47611477b9ce0f03dc37ff7b) --- .../sql/catalyst/analysis/unresolved.scala | 9 +----- .../sql/catalyst/parser/AstBuilder.scala | 9 ++---- .../sql/catalyst/parser/PlanParserSuite.scala | 28 ++++++++++--------- 3 files changed, 18 insertions(+), 28 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala index ea3fb9fa0933..262b894e2a0a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala @@ -25,7 +25,6 @@ import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, Codege import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan} import org.apache.spark.sql.catalyst.trees.TreeNode import org.apache.spark.sql.catalyst.util.quoteIdentifier -import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, Metadata, StructType} /** @@ -70,15 +69,9 @@ case class UnresolvedInlineTable( * select * from range(10); * }}} */ -case class UnresolvedTableValuedFunction(conf: SQLConf, - var functionName: String, - functionArgs: Seq[Expression]) +case class UnresolvedTableValuedFunction(functionName: String, functionArgs: Seq[Expression]) extends LeafNode { - if (!conf.caseSensitiveAnalysis) { - functionName = functionName.toLowerCase - } - override def output: Seq[Attribute] = Nil override lazy val resolved = false diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 6ebbdddaca5f..162051a8c0e4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -29,7 +29,6 @@ import org.antlr.v4.runtime.tree.{ParseTree, RuleNode, TerminalNode} import org.apache.spark.internal.Logging import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} -import org.apache.spark.sql.catalyst.SimpleCatalystConf import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.parser.SqlBaseParser._ @@ -45,7 +44,6 @@ import org.apache.spark.util.random.RandomSampler */ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging { import ParserUtils._ - val sqlConf = SimpleCatalystConf(caseSensitiveAnalysis = false) protected def typedVisit[T](ctx: ParseTree): T = { ctx.accept(this).asInstanceOf[T] @@ -676,15 +674,12 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging { /** * Create a table-valued function call with arguments, e.g. range(1000) - */ + */ override def visitTableValuedFunction(ctx: TableValuedFunctionContext) : LogicalPlan = withOrigin(ctx) { - UnresolvedTableValuedFunction(sqlConf, - ctx.identifier.getText, - ctx.expression.asScala.map(expression)) + UnresolvedTableValuedFunction(ctx.identifier.getText, ctx.expression.asScala.map(expression)) } - /** * Create an inline table (a virtual table in Hive parlance). */ diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala index 843799f99f68..1c0d8d8c0ffe 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala @@ -17,12 +17,11 @@ package org.apache.spark.sql.catalyst.parser +import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.analysis.{UnresolvedGenerator, UnresolvedInlineTable, UnresolvedTableValuedFunction} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.catalyst.FunctionIdentifier -import org.apache.spark.sql.catalyst.SimpleCatalystConf import org.apache.spark.sql.types.IntegerType /** @@ -34,8 +33,6 @@ class PlanParserSuite extends PlanTest { import CatalystSqlParser._ import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.dsl.plans._ - override val conf = SimpleCatalystConf(caseSensitiveAnalysis = false) - private def assertEqual(sqlCommand: String, plan: LogicalPlan): Unit = { comparePlans(parsePlan(sqlCommand), plan) @@ -55,14 +52,6 @@ class PlanParserSuite extends PlanTest { assertEqual("SELECT * FROM a", plan) } - - test("case insensitive range queries") { - val plan = UnresolvedTableValuedFunction(conf, "range", Literal(2) :: Nil).select(star()) - assertEqual("select * from rAngE(2)", plan) - assertEqual("select * from RaNgE(2)", plan) - assertEqual("select * from RANGE(2)", plan) - } - test("explain") { intercept("EXPLAIN logical SELECT 1", "Unsupported SQL statement") intercept("EXPLAIN formatted SELECT 1", "Unsupported SQL statement") @@ -479,9 +468,22 @@ class PlanParserSuite extends PlanTest { test("table valued function") { assertEqual( "select * from range(2)", - UnresolvedTableValuedFunction(conf, "range", Literal(2) :: Nil).select(star())) + UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) } + test("table valued function case insensitive") { + assertEqual( + "select * from RangE(2)", + UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) + + assertEqual( + "select * from rAnGe(2)", + UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) + + assertEqual( + "select * from RANGE(2)", + UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) + } test("inline table") { assertEqual("values 1, 2, 3, 4", From 8bce1da346294a7aec405ca6eb9c59474f0880a3 Mon Sep 17 00:00:00 2001 From: samelamin Date: Sun, 2 Apr 2017 18:10:18 +0100 Subject: [PATCH 07/11] revert based on comments from pr --- .../sql/catalyst/analysis/ResolveTableValuedFunctions.scala | 2 +- .../org/apache/spark/sql/catalyst/analysis/unresolved.scala | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala index 63540e2cc935..6b3bb68538dd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala @@ -105,7 +105,7 @@ object ResolveTableValuedFunctions extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators { case u: UnresolvedTableValuedFunction if u.functionArgs.forall(_.resolved) => - builtinFunctions.get(u.functionName.toLowerCase) match { + builtinFunctions.get(u.functionName) match { case Some(tvf) => val resolved = tvf.flatMap { case (argList, resolver) => argList.implicitCast(u.functionArgs) match { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala index 262b894e2a0a..87a60a8facb2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala @@ -69,9 +69,10 @@ case class UnresolvedInlineTable( * select * from range(10); * }}} */ -case class UnresolvedTableValuedFunction(functionName: String, functionArgs: Seq[Expression]) +case class UnresolvedTableValuedFunction(var functionName: String, functionArgs: Seq[Expression]) extends LeafNode { + functionName = functionName.toLowerCase() override def output: Seq[Attribute] = Nil override lazy val resolved = false From ad5780a6467560bfe73c8408616929a7c3009afb Mon Sep 17 00:00:00 2001 From: samelamin Date: Sun, 2 Apr 2017 18:17:41 +0100 Subject: [PATCH 08/11] code cleanup --- .../sql/catalyst/analysis/ResolveTableValuedFunctions.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala index 6b3bb68538dd..bc9699f0c707 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala @@ -17,9 +17,7 @@ package org.apache.spark.sql.catalyst.analysis -import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Range} import org.apache.spark.sql.catalyst.rules._ import org.apache.spark.sql.types.{DataType, IntegerType, LongType} From 6efb25482ace898beeb94fca2a358f49650ebadb Mon Sep 17 00:00:00 2001 From: samelamin Date: Sun, 2 Apr 2017 18:18:03 +0100 Subject: [PATCH 09/11] remove empty spaces --- .../apache/spark/sql/catalyst/parser/PlanParserSuite.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala index 1c0d8d8c0ffe..2bfb1e6f429a 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala @@ -471,15 +471,13 @@ class PlanParserSuite extends PlanTest { UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) } - test("table valued function case insensitive") { + test("case insensitive range queries.") { assertEqual( "select * from RangE(2)", UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) - assertEqual( "select * from rAnGe(2)", UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) - assertEqual( "select * from RANGE(2)", UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) From a01aba6d5646be61b047d312aca2a4b27887db2d Mon Sep 17 00:00:00 2001 From: samelamin Date: Mon, 3 Apr 2017 14:14:49 +0100 Subject: [PATCH 10/11] remove vars and use actual SQL --- .../ResolveTableValuedFunctions.scala | 2 +- .../sql/catalyst/analysis/unresolved.scala | 3 +-- .../inputs/table-valued-functions.sql | 6 ++++++ .../results/table-valued-functions.sql.out | 20 ++++++++++++++++++- 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala index bc9699f0c707..8841309939c2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableValuedFunctions.scala @@ -103,7 +103,7 @@ object ResolveTableValuedFunctions extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators { case u: UnresolvedTableValuedFunction if u.functionArgs.forall(_.resolved) => - builtinFunctions.get(u.functionName) match { + builtinFunctions.get(u.functionName.toLowerCase()) match { case Some(tvf) => val resolved = tvf.flatMap { case (argList, resolver) => argList.implicitCast(u.functionArgs) match { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala index 87a60a8facb2..262b894e2a0a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala @@ -69,10 +69,9 @@ case class UnresolvedInlineTable( * select * from range(10); * }}} */ -case class UnresolvedTableValuedFunction(var functionName: String, functionArgs: Seq[Expression]) +case class UnresolvedTableValuedFunction(functionName: String, functionArgs: Seq[Expression]) extends LeafNode { - functionName = functionName.toLowerCase() override def output: Seq[Attribute] = Nil override lazy val resolved = false diff --git a/sql/core/src/test/resources/sql-tests/inputs/table-valued-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/table-valued-functions.sql index 2e6dcd538b7a..d0d2df7b243d 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/table-valued-functions.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/table-valued-functions.sql @@ -18,3 +18,9 @@ select * from range(1, 1, 1, 1, 1); -- range call with null select * from range(1, null); + +-- range call with a mixed-case function name +select * from RaNgE(2); + +-- Explain +EXPLAIN select * from RaNgE(2); diff --git a/sql/core/src/test/resources/sql-tests/results/table-valued-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/table-valued-functions.sql.out index d769bcef0aca..acd4ecf14617 100644 --- a/sql/core/src/test/resources/sql-tests/results/table-valued-functions.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/table-valued-functions.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 7 +-- Number of queries: 9 -- !query 0 @@ -85,3 +85,21 @@ struct<> -- !query 6 output java.lang.IllegalArgumentException Invalid arguments for resolved function: 1, null + + +-- !query 7 +select * from RaNgE(2) +-- !query 7 schema +struct +-- !query 7 output +0 +1 + + +-- !query 8 +EXPLAIN select * from RaNgE(2) +-- !query 8 schema +struct +-- !query 8 output +== Physical Plan == +*Range (0, 2, step=1, splits=None) From 406eba8f2e0e5cf1d6a349a1a3f6c35e99ed6f15 Mon Sep 17 00:00:00 2001 From: samelamin Date: Mon, 3 Apr 2017 14:18:03 +0100 Subject: [PATCH 11/11] remove unused test --- .../spark/sql/catalyst/parser/PlanParserSuite.scala | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala index 2bfb1e6f429a..411777d6e85a 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala @@ -471,18 +471,6 @@ class PlanParserSuite extends PlanTest { UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) } - test("case insensitive range queries.") { - assertEqual( - "select * from RangE(2)", - UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) - assertEqual( - "select * from rAnGe(2)", - UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) - assertEqual( - "select * from RANGE(2)", - UnresolvedTableValuedFunction("range", Literal(2) :: Nil).select(star())) - } - test("inline table") { assertEqual("values 1, 2, 3, 4", UnresolvedInlineTable(Seq("col1"), Seq(1, 2, 3, 4).map(x => Seq(Literal(x)))))