From d49fbdbacd6b3ced5801ff62afc7ec12a8831028 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 24 Aug 2016 19:52:20 +0900 Subject: [PATCH 1/3] TypeCoercion supports widening conversion between DateType and TimestampType --- .../sql/catalyst/analysis/TypeCoercion.scala | 4 + .../catalyst/analysis/TypeCoercionSuite.scala | 97 +++++++++++++++++-- 2 files changed, 93 insertions(+), 8 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala index 193c3ec4e585..acf0e1efa843 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala @@ -134,6 +134,8 @@ object TypeCoercion { Some(DecimalPrecision.widerDecimalType(DecimalType.forType(t), d)) case (_: FractionalType, _: DecimalType) | (_: DecimalType, _: FractionalType) => Some(DoubleType) + case (_: TimestampType, _: DateType) | (_: DateType, _: TimestampType) => + Some(TimestampType) case _ => findTightestCommonTypeToString(t1, t2) } @@ -161,6 +163,8 @@ object TypeCoercion { Some(DecimalPrecision.widerDecimalType(DecimalType.forType(t), d)) case (_: FractionalType, _: DecimalType) | (_: DecimalType, _: FractionalType) => Some(DoubleType) + case (_: TimestampType, _: DateType) | (_: DateType, _: TimestampType) => + Some(TimestampType) case _ => None }) case None => None diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala index 9560563a8ca5..8fba81826be2 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala @@ -261,6 +261,13 @@ class TypeCoercionSuite extends PlanTest { :: Cast(Literal(1), DecimalType(22, 0)) :: Cast(Literal(new java.math.BigDecimal("1000000000000000000000")), DecimalType(22, 0)) :: Nil)) + ruleTest(TypeCoercion.FunctionArgumentConversion, + Coalesce(Literal.create(null, TimestampType) + :: Literal.create(null, DateType) + :: Nil), + Coalesce(Literal.create(null, TimestampType).cast(TimestampType) + :: Literal.create(null, DateType).cast(TimestampType) + :: Nil)) } test("CreateArray casts") { @@ -301,6 +308,14 @@ class TypeCoercionSuite extends PlanTest { :: Literal.create(null, DecimalType(22, 10)).cast(DecimalType(38, 38)) :: Literal.create(null, DecimalType(38, 38)).cast(DecimalType(38, 38)) :: Nil)) + + ruleTest(TypeCoercion.FunctionArgumentConversion, + CreateArray(Literal.create(null, TimestampType) + :: Literal.create(null, DateType) + :: Nil), + CreateArray(Literal.create(null, TimestampType).cast(TimestampType) + :: Literal.create(null, DateType).cast(TimestampType) + :: Nil)) } test("CreateMap casts") { @@ -327,6 +342,18 @@ class TypeCoercionSuite extends PlanTest { :: Literal.create(2.0, FloatType).cast(DoubleType) :: Literal("b") :: Nil)) + ruleTest(TypeCoercion.FunctionArgumentConversion, + CreateMap(Literal.create(null, TimestampType) + :: Literal("a") + :: Literal.create(null, DateType) + :: Literal("b") + :: Nil), + CreateMap(Literal.create(null, TimestampType).cast(TimestampType) + :: Literal("a") + :: Literal.create(null, DateType).cast(TimestampType) + :: Literal("b") + :: Nil)) + // type coercion for map values ruleTest(TypeCoercion.FunctionArgumentConversion, CreateMap(Literal(1) @@ -350,6 +377,17 @@ class TypeCoercionSuite extends PlanTest { :: Literal(2) :: Literal.create(null, DecimalType(38, 38)).cast(DecimalType(38, 38)) :: Nil)) + ruleTest(TypeCoercion.FunctionArgumentConversion, + CreateMap(Literal(1) + :: Literal.create(null, TimestampType) + :: Literal(2) + :: Literal.create(null, DateType) + :: Nil), + CreateMap(Literal(1) + :: Literal.create(null, TimestampType).cast(TimestampType) + :: Literal(2) + :: Literal.create(null, DateType).cast(TimestampType) + :: Nil)) // type coercion for both map keys and values ruleTest(TypeCoercion.FunctionArgumentConversion, CreateMap(Literal(1) @@ -362,6 +400,17 @@ class TypeCoercionSuite extends PlanTest { :: Cast(Literal(2.0), DoubleType) :: Cast(Literal(3.0), StringType) :: Nil)) + ruleTest(TypeCoercion.FunctionArgumentConversion, + CreateMap(Literal.create(null, DateType) + :: Literal.create(null, TimestampType) + :: Literal.create(null, TimestampType) + :: Literal.create(null, DateType) + :: Nil), + CreateMap(Literal.create(null, DateType).cast(TimestampType) + :: Literal.create(null, TimestampType).cast(TimestampType) + :: Literal.create(null, TimestampType).cast(TimestampType) + :: Literal.create(null, DateType).cast(TimestampType) + :: Nil)) } test("greatest/least cast") { @@ -411,6 +460,13 @@ class TypeCoercionSuite extends PlanTest { :: Literal(1).cast(DecimalType(25, 5)) :: Literal.create(null, DecimalType(10, 5)).cast(DecimalType(25, 5)) :: Nil)) + ruleTest(TypeCoercion.FunctionArgumentConversion, + operator(Literal.create(null, TimestampType) + :: Literal.create(null, DateType) + :: Nil), + operator(Literal.create(null, TimestampType).cast(TimestampType) + :: Literal.create(null, DateType).cast(TimestampType) + :: Nil)) } } @@ -444,6 +500,14 @@ class TypeCoercionSuite extends PlanTest { ruleTest(rule, If(AssertTrue(Literal.create(false, BooleanType)), Literal(1), Literal(2)), If(Cast(AssertTrue(Literal.create(false, BooleanType)), BooleanType), Literal(1), Literal(2))) + + ruleTest(rule, + If(Literal(true), + Literal.create(null, DateType), + Literal.create(null, TimestampType)), + If(Literal(true), + Literal.create(null, DateType).cast(TimestampType), + Literal.create(null, TimestampType))) } test("type coercion for CaseKeyWhen") { @@ -465,6 +529,15 @@ class TypeCoercionSuite extends PlanTest { CaseWhen(Seq((Literal(true), Cast(Literal(100L), DecimalType(22, 2)))), Cast(Literal.create(1, DecimalType(7, 2)), DecimalType(22, 2))) ) + + ruleTest(TypeCoercion.CaseWhenCoercion, + CaseWhen( + Seq((Literal(true), Literal.create(null, DateType))), + Literal.create(null, TimestampType)), + CaseWhen( + Seq((Literal(true), Literal.create(null, DateType).cast(TimestampType))), + Literal.create(null, TimestampType)) + ) } test("BooleanEquality type cast") { @@ -546,15 +619,18 @@ class TypeCoercionSuite extends PlanTest { AttributeReference("i", IntegerType)(), AttributeReference("u", DecimalType.SYSTEM_DEFAULT)(), AttributeReference("b", ByteType)(), - AttributeReference("d", DoubleType)()) + AttributeReference("d", DoubleType)(), + AttributeReference("e", DateType)()) val secondTable = LocalRelation( AttributeReference("s", StringType)(), AttributeReference("d", DecimalType(2, 1))(), AttributeReference("f", FloatType)(), - AttributeReference("l", LongType)()) + AttributeReference("l", LongType)(), + AttributeReference("q", TimestampType)()) val wt = TypeCoercion.WidenSetOperationTypes - val expectedTypes = Seq(StringType, DecimalType.SYSTEM_DEFAULT, FloatType, DoubleType) + val expectedTypes = + Seq(StringType, DecimalType.SYSTEM_DEFAULT, FloatType, DoubleType, TimestampType) val r1 = wt(Except(firstTable, secondTable)).asInstanceOf[Except] val r2 = wt(Intersect(firstTable, secondTable)).asInstanceOf[Intersect] @@ -575,25 +651,30 @@ class TypeCoercionSuite extends PlanTest { AttributeReference("i", IntegerType)(), AttributeReference("u", DecimalType.SYSTEM_DEFAULT)(), AttributeReference("b", ByteType)(), - AttributeReference("d", DoubleType)()) + AttributeReference("d", DoubleType)(), + AttributeReference("e", DateType)()) val secondTable = LocalRelation( AttributeReference("s", StringType)(), AttributeReference("d", DecimalType(2, 1))(), AttributeReference("f", FloatType)(), - AttributeReference("l", LongType)()) + AttributeReference("l", LongType)(), + AttributeReference("e", DateType)()) val thirdTable = LocalRelation( AttributeReference("m", StringType)(), AttributeReference("n", DecimalType.SYSTEM_DEFAULT)(), AttributeReference("p", FloatType)(), - AttributeReference("q", DoubleType)()) + AttributeReference("q", DoubleType)(), + AttributeReference("e", TimestampType)()) val forthTable = LocalRelation( AttributeReference("m", StringType)(), AttributeReference("n", DecimalType.SYSTEM_DEFAULT)(), AttributeReference("p", ByteType)(), - AttributeReference("q", DoubleType)()) + AttributeReference("q", DoubleType)(), + AttributeReference("e", TimestampType)()) val wt = TypeCoercion.WidenSetOperationTypes - val expectedTypes = Seq(StringType, DecimalType.SYSTEM_DEFAULT, FloatType, DoubleType) + val expectedTypes = + Seq(StringType, DecimalType.SYSTEM_DEFAULT, FloatType, DoubleType, TimestampType) val unionRelation = wt( Union(firstTable :: secondTable :: thirdTable :: forthTable :: Nil)).asInstanceOf[Union] From ab754fa8eb537dcd6ce3f4f3b256f0fba2f2fdcd Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 25 Aug 2016 17:45:11 +0900 Subject: [PATCH 2/3] Address comments and add some more tests --- .../spark/sql/catalyst/analysis/TypeCoercion.scala | 7 +++---- .../sql/catalyst/analysis/TypeCoercionSuite.scala | 1 + .../spark/sql/SQLCompatibilityFunctionSuite.scala | 12 ++++++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala index acf0e1efa843..01b04c036d15 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala @@ -96,6 +96,9 @@ object TypeCoercion { val index = numericPrecedence.lastIndexWhere(t => t == t1 || t == t2) Some(numericPrecedence(index)) + case (_: TimestampType, _: DateType) | (_: DateType, _: TimestampType) => + Some(TimestampType) + case _ => None } @@ -134,8 +137,6 @@ object TypeCoercion { Some(DecimalPrecision.widerDecimalType(DecimalType.forType(t), d)) case (_: FractionalType, _: DecimalType) | (_: DecimalType, _: FractionalType) => Some(DoubleType) - case (_: TimestampType, _: DateType) | (_: DateType, _: TimestampType) => - Some(TimestampType) case _ => findTightestCommonTypeToString(t1, t2) } @@ -163,8 +164,6 @@ object TypeCoercion { Some(DecimalPrecision.widerDecimalType(DecimalType.forType(t), d)) case (_: FractionalType, _: DecimalType) | (_: DecimalType, _: FractionalType) => Some(DoubleType) - case (_: TimestampType, _: DateType) | (_: DateType, _: TimestampType) => - Some(TimestampType) case _ => None }) case None => None diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala index 8fba81826be2..b3fc77485c3b 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala @@ -188,6 +188,7 @@ class TypeCoercionSuite extends PlanTest { // TimestampType widenTest(NullType, TimestampType, Some(TimestampType)) widenTest(TimestampType, TimestampType, Some(TimestampType)) + widenTest(DateType, TimestampType, Some(TimestampType)) widenTest(IntegerType, TimestampType, None) widenTest(StringType, TimestampType, None) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLCompatibilityFunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLCompatibilityFunctionSuite.scala index 27b60e0d9def..c0769c2ba4fe 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLCompatibilityFunctionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLCompatibilityFunctionSuite.scala @@ -39,6 +39,9 @@ class SQLCompatibilityFunctionSuite extends QueryTest with SharedSQLContext { checkAnswer( sql("SELECT ifnull(1, 2.1d), ifnull(null, 2.1d)"), Row(1.0, 2.1)) + checkAnswer( + sql("SELECT ifnull(CAST('1990-02-25' AS DATE), CAST('1990-02-24 12:00:00' AS TIMESTAMP))"), + Row(Timestamp.valueOf("1990-02-25 00:00:00"))) } test("nullif") { @@ -50,6 +53,9 @@ class SQLCompatibilityFunctionSuite extends QueryTest with SharedSQLContext { checkAnswer( sql("SELECT nullif(1, 2.1d), nullif(1, 1.0d)"), Row(1.0, null)) + checkAnswer( + sql("SELECT nullif(CAST('1990-02-25' AS DATE), CAST('1990-02-24 12:00:00' AS TIMESTAMP))"), + Row(Timestamp.valueOf("1990-02-25 00:00:00"))) } test("nvl") { @@ -61,6 +67,9 @@ class SQLCompatibilityFunctionSuite extends QueryTest with SharedSQLContext { checkAnswer( sql("SELECT nvl(1, 2.1d), nvl(null, 2.1d)"), Row(1.0, 2.1)) + checkAnswer( + sql("SELECT nvl(CAST('1990-02-25' AS DATE), CAST('1990-02-24 12:00:00' AS TIMESTAMP))"), + Row(Timestamp.valueOf("1990-02-25 00:00:00"))) } test("nvl2") { @@ -72,6 +81,9 @@ class SQLCompatibilityFunctionSuite extends QueryTest with SharedSQLContext { checkAnswer( sql("SELECT nvl2(null, 1, 2.1d), nvl2('n', 1, 2.1d)"), Row(2.1, 1.0)) + checkAnswer( + sql("SELECT nvl2('n', CAST('1990-02-25' AS DATE), CAST('1990-02-24 12:00:00' AS TIMESTAMP))"), + Row(Timestamp.valueOf("1990-02-25 00:00:00"))) } test("SPARK-16730 cast alias functions for Hive compatibility") { From d035eb3ba725250f6238a5b8a189b6749065cf95 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Thu, 25 Aug 2016 22:24:27 +0900 Subject: [PATCH 3/3] Address comments --- .../catalyst/analysis/TypeCoercionSuite.scala | 97 ++----------------- .../sql/SQLCompatibilityFunctionSuite.scala | 12 --- 2 files changed, 8 insertions(+), 101 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala index b3fc77485c3b..6f69613f8531 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala @@ -262,13 +262,6 @@ class TypeCoercionSuite extends PlanTest { :: Cast(Literal(1), DecimalType(22, 0)) :: Cast(Literal(new java.math.BigDecimal("1000000000000000000000")), DecimalType(22, 0)) :: Nil)) - ruleTest(TypeCoercion.FunctionArgumentConversion, - Coalesce(Literal.create(null, TimestampType) - :: Literal.create(null, DateType) - :: Nil), - Coalesce(Literal.create(null, TimestampType).cast(TimestampType) - :: Literal.create(null, DateType).cast(TimestampType) - :: Nil)) } test("CreateArray casts") { @@ -309,14 +302,6 @@ class TypeCoercionSuite extends PlanTest { :: Literal.create(null, DecimalType(22, 10)).cast(DecimalType(38, 38)) :: Literal.create(null, DecimalType(38, 38)).cast(DecimalType(38, 38)) :: Nil)) - - ruleTest(TypeCoercion.FunctionArgumentConversion, - CreateArray(Literal.create(null, TimestampType) - :: Literal.create(null, DateType) - :: Nil), - CreateArray(Literal.create(null, TimestampType).cast(TimestampType) - :: Literal.create(null, DateType).cast(TimestampType) - :: Nil)) } test("CreateMap casts") { @@ -343,18 +328,6 @@ class TypeCoercionSuite extends PlanTest { :: Literal.create(2.0, FloatType).cast(DoubleType) :: Literal("b") :: Nil)) - ruleTest(TypeCoercion.FunctionArgumentConversion, - CreateMap(Literal.create(null, TimestampType) - :: Literal("a") - :: Literal.create(null, DateType) - :: Literal("b") - :: Nil), - CreateMap(Literal.create(null, TimestampType).cast(TimestampType) - :: Literal("a") - :: Literal.create(null, DateType).cast(TimestampType) - :: Literal("b") - :: Nil)) - // type coercion for map values ruleTest(TypeCoercion.FunctionArgumentConversion, CreateMap(Literal(1) @@ -378,17 +351,6 @@ class TypeCoercionSuite extends PlanTest { :: Literal(2) :: Literal.create(null, DecimalType(38, 38)).cast(DecimalType(38, 38)) :: Nil)) - ruleTest(TypeCoercion.FunctionArgumentConversion, - CreateMap(Literal(1) - :: Literal.create(null, TimestampType) - :: Literal(2) - :: Literal.create(null, DateType) - :: Nil), - CreateMap(Literal(1) - :: Literal.create(null, TimestampType).cast(TimestampType) - :: Literal(2) - :: Literal.create(null, DateType).cast(TimestampType) - :: Nil)) // type coercion for both map keys and values ruleTest(TypeCoercion.FunctionArgumentConversion, CreateMap(Literal(1) @@ -401,17 +363,6 @@ class TypeCoercionSuite extends PlanTest { :: Cast(Literal(2.0), DoubleType) :: Cast(Literal(3.0), StringType) :: Nil)) - ruleTest(TypeCoercion.FunctionArgumentConversion, - CreateMap(Literal.create(null, DateType) - :: Literal.create(null, TimestampType) - :: Literal.create(null, TimestampType) - :: Literal.create(null, DateType) - :: Nil), - CreateMap(Literal.create(null, DateType).cast(TimestampType) - :: Literal.create(null, TimestampType).cast(TimestampType) - :: Literal.create(null, TimestampType).cast(TimestampType) - :: Literal.create(null, DateType).cast(TimestampType) - :: Nil)) } test("greatest/least cast") { @@ -461,13 +412,6 @@ class TypeCoercionSuite extends PlanTest { :: Literal(1).cast(DecimalType(25, 5)) :: Literal.create(null, DecimalType(10, 5)).cast(DecimalType(25, 5)) :: Nil)) - ruleTest(TypeCoercion.FunctionArgumentConversion, - operator(Literal.create(null, TimestampType) - :: Literal.create(null, DateType) - :: Nil), - operator(Literal.create(null, TimestampType).cast(TimestampType) - :: Literal.create(null, DateType).cast(TimestampType) - :: Nil)) } } @@ -501,14 +445,6 @@ class TypeCoercionSuite extends PlanTest { ruleTest(rule, If(AssertTrue(Literal.create(false, BooleanType)), Literal(1), Literal(2)), If(Cast(AssertTrue(Literal.create(false, BooleanType)), BooleanType), Literal(1), Literal(2))) - - ruleTest(rule, - If(Literal(true), - Literal.create(null, DateType), - Literal.create(null, TimestampType)), - If(Literal(true), - Literal.create(null, DateType).cast(TimestampType), - Literal.create(null, TimestampType))) } test("type coercion for CaseKeyWhen") { @@ -530,15 +466,6 @@ class TypeCoercionSuite extends PlanTest { CaseWhen(Seq((Literal(true), Cast(Literal(100L), DecimalType(22, 2)))), Cast(Literal.create(1, DecimalType(7, 2)), DecimalType(22, 2))) ) - - ruleTest(TypeCoercion.CaseWhenCoercion, - CaseWhen( - Seq((Literal(true), Literal.create(null, DateType))), - Literal.create(null, TimestampType)), - CaseWhen( - Seq((Literal(true), Literal.create(null, DateType).cast(TimestampType))), - Literal.create(null, TimestampType)) - ) } test("BooleanEquality type cast") { @@ -620,18 +547,15 @@ class TypeCoercionSuite extends PlanTest { AttributeReference("i", IntegerType)(), AttributeReference("u", DecimalType.SYSTEM_DEFAULT)(), AttributeReference("b", ByteType)(), - AttributeReference("d", DoubleType)(), - AttributeReference("e", DateType)()) + AttributeReference("d", DoubleType)()) val secondTable = LocalRelation( AttributeReference("s", StringType)(), AttributeReference("d", DecimalType(2, 1))(), AttributeReference("f", FloatType)(), - AttributeReference("l", LongType)(), - AttributeReference("q", TimestampType)()) + AttributeReference("l", LongType)()) val wt = TypeCoercion.WidenSetOperationTypes - val expectedTypes = - Seq(StringType, DecimalType.SYSTEM_DEFAULT, FloatType, DoubleType, TimestampType) + val expectedTypes = Seq(StringType, DecimalType.SYSTEM_DEFAULT, FloatType, DoubleType) val r1 = wt(Except(firstTable, secondTable)).asInstanceOf[Except] val r2 = wt(Intersect(firstTable, secondTable)).asInstanceOf[Intersect] @@ -652,30 +576,25 @@ class TypeCoercionSuite extends PlanTest { AttributeReference("i", IntegerType)(), AttributeReference("u", DecimalType.SYSTEM_DEFAULT)(), AttributeReference("b", ByteType)(), - AttributeReference("d", DoubleType)(), - AttributeReference("e", DateType)()) + AttributeReference("d", DoubleType)()) val secondTable = LocalRelation( AttributeReference("s", StringType)(), AttributeReference("d", DecimalType(2, 1))(), AttributeReference("f", FloatType)(), - AttributeReference("l", LongType)(), - AttributeReference("e", DateType)()) + AttributeReference("l", LongType)()) val thirdTable = LocalRelation( AttributeReference("m", StringType)(), AttributeReference("n", DecimalType.SYSTEM_DEFAULT)(), AttributeReference("p", FloatType)(), - AttributeReference("q", DoubleType)(), - AttributeReference("e", TimestampType)()) + AttributeReference("q", DoubleType)()) val forthTable = LocalRelation( AttributeReference("m", StringType)(), AttributeReference("n", DecimalType.SYSTEM_DEFAULT)(), AttributeReference("p", ByteType)(), - AttributeReference("q", DoubleType)(), - AttributeReference("e", TimestampType)()) + AttributeReference("q", DoubleType)()) val wt = TypeCoercion.WidenSetOperationTypes - val expectedTypes = - Seq(StringType, DecimalType.SYSTEM_DEFAULT, FloatType, DoubleType, TimestampType) + val expectedTypes = Seq(StringType, DecimalType.SYSTEM_DEFAULT, FloatType, DoubleType) val unionRelation = wt( Union(firstTable :: secondTable :: thirdTable :: forthTable :: Nil)).asInstanceOf[Union] diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLCompatibilityFunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLCompatibilityFunctionSuite.scala index c0769c2ba4fe..27b60e0d9def 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLCompatibilityFunctionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLCompatibilityFunctionSuite.scala @@ -39,9 +39,6 @@ class SQLCompatibilityFunctionSuite extends QueryTest with SharedSQLContext { checkAnswer( sql("SELECT ifnull(1, 2.1d), ifnull(null, 2.1d)"), Row(1.0, 2.1)) - checkAnswer( - sql("SELECT ifnull(CAST('1990-02-25' AS DATE), CAST('1990-02-24 12:00:00' AS TIMESTAMP))"), - Row(Timestamp.valueOf("1990-02-25 00:00:00"))) } test("nullif") { @@ -53,9 +50,6 @@ class SQLCompatibilityFunctionSuite extends QueryTest with SharedSQLContext { checkAnswer( sql("SELECT nullif(1, 2.1d), nullif(1, 1.0d)"), Row(1.0, null)) - checkAnswer( - sql("SELECT nullif(CAST('1990-02-25' AS DATE), CAST('1990-02-24 12:00:00' AS TIMESTAMP))"), - Row(Timestamp.valueOf("1990-02-25 00:00:00"))) } test("nvl") { @@ -67,9 +61,6 @@ class SQLCompatibilityFunctionSuite extends QueryTest with SharedSQLContext { checkAnswer( sql("SELECT nvl(1, 2.1d), nvl(null, 2.1d)"), Row(1.0, 2.1)) - checkAnswer( - sql("SELECT nvl(CAST('1990-02-25' AS DATE), CAST('1990-02-24 12:00:00' AS TIMESTAMP))"), - Row(Timestamp.valueOf("1990-02-25 00:00:00"))) } test("nvl2") { @@ -81,9 +72,6 @@ class SQLCompatibilityFunctionSuite extends QueryTest with SharedSQLContext { checkAnswer( sql("SELECT nvl2(null, 1, 2.1d), nvl2('n', 1, 2.1d)"), Row(2.1, 1.0)) - checkAnswer( - sql("SELECT nvl2('n', CAST('1990-02-25' AS DATE), CAST('1990-02-24 12:00:00' AS TIMESTAMP))"), - Row(Timestamp.valueOf("1990-02-25 00:00:00"))) } test("SPARK-16730 cast alias functions for Hive compatibility") {