From 34fddc7a10aa395a01687e040ef523df0bad1610 Mon Sep 17 00:00:00 2001 From: petermaxlee Date: Wed, 17 Aug 2016 20:47:00 -0700 Subject: [PATCH 1/2] [SPARK-17117][SQL] 1 / NULL should not fail analysis --- .../spark/sql/catalyst/analysis/TypeCoercion.scala | 6 ++++-- .../spark/sql/catalyst/analysis/TypeCoercionSuite.scala | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala index 021952e7166f9..469513b26337e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala @@ -543,11 +543,13 @@ object TypeCoercion { // Decimal and Double remain the same case d: Divide if d.dataType == DoubleType => d case d: Divide if d.dataType.isInstanceOf[DecimalType] => d - case Divide(left, right) if isNumeric(left) && isNumeric(right) => + case Divide(left, right) if isNumericOrNull(left) && isNumericOrNull(right) => Divide(Cast(left, DoubleType), Cast(right, DoubleType)) } - private def isNumeric(ex: Expression): Boolean = ex.dataType.isInstanceOf[NumericType] + private def isNumericOrNull(ex: Expression): Boolean = { + ex.dataType.isInstanceOf[NumericType] || ex.dataType == NullType + } } /** diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala index a13c45fe2ffee..9560563a8ca56 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.analysis import java.sql.Timestamp -import org.apache.spark.sql.catalyst.analysis.TypeCoercion.{Division, FunctionArgumentConversion} +import org.apache.spark.sql.catalyst.analysis.TypeCoercion._ import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.PlanTest @@ -730,6 +730,13 @@ class TypeCoercionSuite extends PlanTest { // the right expression to Decimal. ruleTest(rules, sum(Divide(Decimal(4.0), 3)), sum(Divide(Decimal(4.0), 3))) } + + test("SPARK-17117 null type coercion in divide") { + val rules = Seq(FunctionArgumentConversion, Division, ImplicitTypeCasts) + val nullLit = Literal.create(null, NullType) + ruleTest(rules, Divide(1L, nullLit), Divide(Cast(1L, DoubleType), Cast(nullLit, DoubleType))) + ruleTest(rules, Divide(nullLit, 1L), Divide(Cast(nullLit, DoubleType), Cast(1L, DoubleType))) + } } From a946269811540d6cdb2237c62f095f847b461cee Mon Sep 17 00:00:00 2001 From: petermaxlee Date: Wed, 17 Aug 2016 20:54:15 -0700 Subject: [PATCH 2/2] Add end-to-end tests --- .../sql/catalyst/analysis/TypeCoercion.scala | 1 + .../resources/sql-tests/inputs/arithmetic.sql | 12 ++- .../sql-tests/results/arithmetic.sql.out | 84 +++++++++++++++---- 3 files changed, 77 insertions(+), 20 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala index 469513b26337e..21e96aaf53844 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala @@ -548,6 +548,7 @@ object TypeCoercion { } private def isNumericOrNull(ex: Expression): Boolean = { + // We need to handle null types in case a query contains null literals. ex.dataType.isInstanceOf[NumericType] || ex.dataType == NullType } } diff --git a/sql/core/src/test/resources/sql-tests/inputs/arithmetic.sql b/sql/core/src/test/resources/sql-tests/inputs/arithmetic.sql index cbe40410cdc10..f62b10ca0037b 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/arithmetic.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/arithmetic.sql @@ -16,11 +16,19 @@ select + + 100; select - - max(key) from testdata; select + - key from testdata where key = 33; +-- div +select 5 / 2; +select 5 / 0; +select 5 / null; +select null / 5; +select 5 div 2; +select 5 div 0; +select 5 div null; +select null div 5; + -- other arithmetics select 1 + 2; select 1 - 2; select 2 * 5; -select 5 / 2; -select 5 div 2; select 5 % 3; select pmod(-7, 3); diff --git a/sql/core/src/test/resources/sql-tests/results/arithmetic.sql.out b/sql/core/src/test/resources/sql-tests/results/arithmetic.sql.out index f2b40a00d062d..6abe048af477d 100644 --- a/sql/core/src/test/resources/sql-tests/results/arithmetic.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/arithmetic.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 22 +-- Number of queries: 28 -- !query 0 @@ -123,35 +123,35 @@ struct<(- key):int> -- !query 15 -select 1 + 2 +select 5 / 2 -- !query 15 schema -struct<(1 + 2):int> +struct<(CAST(5 AS DOUBLE) / CAST(2 AS DOUBLE)):double> -- !query 15 output -3 +2.5 -- !query 16 -select 1 - 2 +select 5 / 0 -- !query 16 schema -struct<(1 - 2):int> +struct<(CAST(5 AS DOUBLE) / CAST(0 AS DOUBLE)):double> -- !query 16 output --1 +NULL -- !query 17 -select 2 * 5 +select 5 / null -- !query 17 schema -struct<(2 * 5):int> +struct<(CAST(5 AS DOUBLE) / CAST(NULL AS DOUBLE)):double> -- !query 17 output -10 +NULL -- !query 18 -select 5 / 2 +select null / 5 -- !query 18 schema -struct<(CAST(5 AS DOUBLE) / CAST(2 AS DOUBLE)):double> +struct<(CAST(NULL AS DOUBLE) / CAST(5 AS DOUBLE)):double> -- !query 18 output -2.5 +NULL -- !query 19 @@ -163,16 +163,64 @@ struct -- !query 20 -select 5 % 3 +select 5 div 0 -- !query 20 schema -struct<(5 % 3):int> +struct -- !query 20 output -2 +NULL -- !query 21 -select pmod(-7, 3) +select 5 div null -- !query 21 schema -struct +struct -- !query 21 output +NULL + + +-- !query 22 +select null div 5 +-- !query 22 schema +struct +-- !query 22 output +NULL + + +-- !query 23 +select 1 + 2 +-- !query 23 schema +struct<(1 + 2):int> +-- !query 23 output +3 + + +-- !query 24 +select 1 - 2 +-- !query 24 schema +struct<(1 - 2):int> +-- !query 24 output +-1 + + +-- !query 25 +select 2 * 5 +-- !query 25 schema +struct<(2 * 5):int> +-- !query 25 output +10 + + +-- !query 26 +select 5 % 3 +-- !query 26 schema +struct<(5 % 3):int> +-- !query 26 output +2 + + +-- !query 27 +select pmod(-7, 3) +-- !query 27 schema +struct +-- !query 27 output 2