From b0c3be317d0b681e183543de97093120a51a6222 Mon Sep 17 00:00:00 2001 From: Travis Hegner Date: Wed, 16 Sep 2015 16:12:23 -0400 Subject: [PATCH 1/3] Proposed bug fix when oracle returns -127 as a scale to a numeric type --- .../apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala index 730d88b024cb..313dbbdaf33d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala @@ -67,7 +67,7 @@ private[sql] object JDBCRDD extends Logging { case java.sql.Types.DATALINK => null case java.sql.Types.DATE => DateType case java.sql.Types.DECIMAL - if precision != 0 || scale != 0 => DecimalType.bounded(precision, scale) + if precision > 0 || scale > 0 => DecimalType.bounded(precision, scale) case java.sql.Types.DECIMAL => DecimalType.SYSTEM_DEFAULT case java.sql.Types.DISTINCT => null case java.sql.Types.DOUBLE => DoubleType @@ -81,7 +81,7 @@ private[sql] object JDBCRDD extends Logging { case java.sql.Types.NCLOB => StringType case java.sql.Types.NULL => null case java.sql.Types.NUMERIC - if precision != 0 || scale != 0 => DecimalType.bounded(precision, scale) + if precision > 0 || scale > 0 => DecimalType.bounded(precision, scale) case java.sql.Types.NUMERIC => DecimalType.SYSTEM_DEFAULT case java.sql.Types.NVARCHAR => StringType case java.sql.Types.OTHER => null From 9b1d407641270f788f31274437577ddcd672872a Mon Sep 17 00:00:00 2001 From: Travis Hegner Date: Mon, 21 Sep 2015 10:42:39 -0400 Subject: [PATCH 2/3] only testing for precision > 0, rather than scale also, since it is legal for scale to be < 0 --- .../apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala index 313dbbdaf33d..3a3d87a3ec65 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala @@ -67,7 +67,7 @@ private[sql] object JDBCRDD extends Logging { case java.sql.Types.DATALINK => null case java.sql.Types.DATE => DateType case java.sql.Types.DECIMAL - if precision > 0 || scale > 0 => DecimalType.bounded(precision, scale) + if precision > 0 => DecimalType.bounded(precision, scale) case java.sql.Types.DECIMAL => DecimalType.SYSTEM_DEFAULT case java.sql.Types.DISTINCT => null case java.sql.Types.DOUBLE => DoubleType @@ -81,7 +81,7 @@ private[sql] object JDBCRDD extends Logging { case java.sql.Types.NCLOB => StringType case java.sql.Types.NULL => null case java.sql.Types.NUMERIC - if precision > 0 || scale > 0 => DecimalType.bounded(precision, scale) + if precision > 0 => DecimalType.bounded(precision, scale) case java.sql.Types.NUMERIC => DecimalType.SYSTEM_DEFAULT case java.sql.Types.NVARCHAR => StringType case java.sql.Types.OTHER => null From 8110d308f388d0cabd13e8c6614f4f4bde9f1898 Mon Sep 17 00:00:00 2001 From: Travis Hegner Date: Tue, 22 Sep 2015 08:51:47 -0400 Subject: [PATCH 3/3] removed any validity checking for decimal/numeric types from JDBCRDD.scala, and put them into bounded function --- .../scala/org/apache/spark/sql/types/DecimalType.scala | 7 ++++++- .../spark/sql/execution/datasources/jdbc/JDBCRDD.scala | 8 ++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala index 0cd352d0fa92..f8e6685f3d89 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala @@ -140,7 +140,12 @@ object DecimalType extends AbstractDataType { } private[sql] def bounded(precision: Int, scale: Int): DecimalType = { - DecimalType(min(precision, MAX_PRECISION), min(scale, MAX_SCALE)) + if (precision <= 0) + DecimalType.SYSTEM_DEFAULT + else if (scale > precision) + DecimalType(min(precision, MAX_PRECISION), min(precision, MAX_SCALE)) + else + DecimalType(min(precision, MAX_PRECISION), min(scale, MAX_SCALE)) } override private[sql] def defaultConcreteType: DataType = SYSTEM_DEFAULT diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala index 3a3d87a3ec65..5f369c0f99bd 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala @@ -66,9 +66,7 @@ private[sql] object JDBCRDD extends Logging { case java.sql.Types.CLOB => StringType case java.sql.Types.DATALINK => null case java.sql.Types.DATE => DateType - case java.sql.Types.DECIMAL - if precision > 0 => DecimalType.bounded(precision, scale) - case java.sql.Types.DECIMAL => DecimalType.SYSTEM_DEFAULT + case java.sql.Types.DECIMAL => DecimalType.bounded(precision, scale) case java.sql.Types.DISTINCT => null case java.sql.Types.DOUBLE => DoubleType case java.sql.Types.FLOAT => FloatType @@ -80,9 +78,7 @@ private[sql] object JDBCRDD extends Logging { case java.sql.Types.NCHAR => StringType case java.sql.Types.NCLOB => StringType case java.sql.Types.NULL => null - case java.sql.Types.NUMERIC - if precision > 0 => DecimalType.bounded(precision, scale) - case java.sql.Types.NUMERIC => DecimalType.SYSTEM_DEFAULT + case java.sql.Types.NUMERIC => DecimalType.bounded(precision, scale) case java.sql.Types.NVARCHAR => StringType case java.sql.Types.OTHER => null case java.sql.Types.REAL => DoubleType