From ea7c2d3d1b8d71bacc905d01ca327f465ca61c86 Mon Sep 17 00:00:00 2001 From: Wojtek Szymanski Date: Mon, 27 Feb 2017 02:42:37 +0100 Subject: [PATCH 1/7] Fix for SQL round function modifying original column --- .../sql/catalyst/expressions/mathExpressions.scala | 2 +- .../org/apache/spark/sql/MathFunctionsSuite.scala | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index 65273a77b1054..fad348a6e50e7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -1023,7 +1023,7 @@ abstract class RoundBase(child: Expression, scale: Expression, def nullSafeEval(input1: Any): Any = { child.dataType match { case _: DecimalType => - val decimal = input1.asInstanceOf[Decimal] + val decimal = input1.asInstanceOf[Decimal].clone() if (decimal.changePrecision(decimal.precision, _scale, mode)) decimal else null case ByteType => BigDecimal(input1.asInstanceOf[Byte]).setScale(_scale, mode).toByte diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala index 37443d0342980..57c58ce92c77d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala @@ -233,6 +233,18 @@ class MathFunctionsSuite extends QueryTest with SharedSQLContext { ) } + test("round/bround with data frame from a local Seq of Product") { + val df = spark.createDataFrame(Seq(NumericRow(BigDecimal("5.9")))) + checkAnswer( + df.withColumn("value_rounded", round('value)), + Seq(Row(BigDecimal("5.9"), BigDecimal("6"))) + ) + checkAnswer( + df.withColumn("value_rounded", bround('value)), + Seq(Row(BigDecimal("5.9"), BigDecimal("6"))) + ) + } + test("exp") { testOneToOneMathFunction(exp, math.exp) } @@ -422,3 +434,4 @@ class MathFunctionsSuite extends QueryTest with SharedSQLContext { checkAnswer(df.selectExpr("positive(b)"), Row(-1)) } } +case class NumericRow(value : BigDecimal) From f1d902db2bafe3baa61e34e60eee29aba28ba7d3 Mon Sep 17 00:00:00 2001 From: Wojtek Szymanski Date: Wed, 1 Mar 2017 01:23:57 +0100 Subject: [PATCH 2/7] Decimal.toPrecision introduced --- .../sql/catalyst/CatalystTypeConverters.scala | 6 +---- .../spark/sql/catalyst/expressions/Cast.scala | 22 +++++++--------- .../expressions/decimalExpressions.scala | 10 ++----- .../expressions/mathExpressions.scala | 4 +-- .../org/apache/spark/sql/types/Decimal.scala | 26 ++++++++++++------- .../apache/spark/sql/types/DecimalSuite.scala | 5 +++- 6 files changed, 36 insertions(+), 37 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala index 5b9161551a7af..d4ebdb139fe0f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala @@ -310,11 +310,7 @@ object CatalystTypeConverters { case d: JavaBigInteger => Decimal(d) case d: Decimal => d } - if (decimal.changePrecision(dataType.precision, dataType.scale)) { - decimal - } else { - null - } + decimal.toPrecision(dataType.precision, dataType.scale).orNull } override def toScala(catalystValue: Decimal): JavaBigDecimal = { if (catalystValue == null) null diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index a36d3507d92ec..70e3921a1428c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -339,36 +339,34 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String } /** - * Change the precision / scale in a given decimal to those set in `decimalType` (if any), - * returning null if it overflows or modifying `value` in-place and returning it if successful. + * Create new `Decimal` with precision and scale given in `decimalType` (if any), + * returning null if it overflows or creating a new `value` and returning it if successful. * - * NOTE: this modifies `value` in-place, so don't call it on external data. */ - private[this] def changePrecision(value: Decimal, decimalType: DecimalType): Decimal = { - if (value.changePrecision(decimalType.precision, decimalType.scale)) value else null - } + private[this] def toPrecision(value: Decimal, decimalType: DecimalType): Decimal = + value.toPrecision(decimalType.precision, decimalType.scale).orNull private[this] def castToDecimal(from: DataType, target: DecimalType): Any => Any = from match { case StringType => buildCast[UTF8String](_, s => try { - changePrecision(Decimal(new JavaBigDecimal(s.toString)), target) + toPrecision(Decimal(new JavaBigDecimal(s.toString)), target) } catch { case _: NumberFormatException => null }) case BooleanType => - buildCast[Boolean](_, b => changePrecision(if (b) Decimal.ONE else Decimal.ZERO, target)) + buildCast[Boolean](_, b => toPrecision(if (b) Decimal.ONE else Decimal.ZERO, target)) case DateType => buildCast[Int](_, d => null) // date can't cast to decimal in Hive case TimestampType => // Note that we lose precision here. - buildCast[Long](_, t => changePrecision(Decimal(timestampToDouble(t)), target)) + buildCast[Long](_, t => toPrecision(Decimal(timestampToDouble(t)), target)) case dt: DecimalType => - b => changePrecision(b.asInstanceOf[Decimal].clone(), target) + b => toPrecision(b.asInstanceOf[Decimal], target) case t: IntegralType => - b => changePrecision(Decimal(t.integral.asInstanceOf[Integral[Any]].toLong(b)), target) + b => toPrecision(Decimal(t.integral.asInstanceOf[Integral[Any]].toLong(b)), target) case x: FractionalType => b => try { - changePrecision(Decimal(x.fractional.asInstanceOf[Fractional[Any]].toDouble(b)), target) + toPrecision(Decimal(x.fractional.asInstanceOf[Fractional[Any]].toDouble(b)), target) } catch { case _: NumberFormatException => null } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala index fa5dea6841149..c2211ae5d594b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala @@ -84,14 +84,8 @@ case class CheckOverflow(child: Expression, dataType: DecimalType) extends Unary override def nullable: Boolean = true - override def nullSafeEval(input: Any): Any = { - val d = input.asInstanceOf[Decimal].clone() - if (d.changePrecision(dataType.precision, dataType.scale)) { - d - } else { - null - } - } + override def nullSafeEval(input: Any): Any = + input.asInstanceOf[Decimal].toPrecision(dataType.precision, dataType.scale).orNull override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { nullSafeCodeGen(ctx, ev, eval => { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index fad348a6e50e7..dea5f85cb08cc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -1023,8 +1023,8 @@ abstract class RoundBase(child: Expression, scale: Expression, def nullSafeEval(input1: Any): Any = { child.dataType match { case _: DecimalType => - val decimal = input1.asInstanceOf[Decimal].clone() - if (decimal.changePrecision(decimal.precision, _scale, mode)) decimal else null + val decimal = input1.asInstanceOf[Decimal] + decimal.toPrecision(decimal.precision, _scale, mode).orNull case ByteType => BigDecimal(input1.asInstanceOf[Byte]).setScale(_scale, mode).toByte case ShortType => diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala index 089c84d5f7736..8505b8e01e4dd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala @@ -222,13 +222,25 @@ final class Decimal extends Ordered[Decimal] with Serializable { case java.math.BigDecimal.ROUND_HALF_EVEN => changePrecision(precision, scale, ROUND_HALF_EVEN) } + /** + * Create new `Decimal` with given precision and scale. + * + * @return `Some(decimal)` if successful or `None` if overflow would occur + */ + private[sql] def toPrecision(precision: Int, scale: Int, + roundMode: BigDecimal.RoundingMode.Value = + ROUND_HALF_UP): Option[Decimal] = { + val copy = clone() + if (copy.changePrecision(precision, scale, roundMode)) Some(copy) else None + } + /** * Update precision and scale while keeping our value the same, and return true if successful. * * @return true if successful, false if overflow would occur */ private[sql] def changePrecision(precision: Int, scale: Int, - roundMode: BigDecimal.RoundingMode.Value): Boolean = { + roundMode: BigDecimal.RoundingMode.Value): Boolean = { // fast path for UnsafeProjection if (precision == this.precision && scale == this.scale) { return true @@ -362,17 +374,13 @@ final class Decimal extends Ordered[Decimal] with Serializable { def abs: Decimal = if (this.compare(Decimal.ZERO) < 0) this.unary_- else this def floor: Decimal = if (scale == 0) this else { - val value = this.clone() - value.changePrecision( - DecimalType.bounded(precision - scale + 1, 0).precision, 0, ROUND_FLOOR) - value + toPrecision(DecimalType.bounded(precision - scale + 1, 0).precision, 0, ROUND_FLOOR) + .getOrElse(clone()) } def ceil: Decimal = if (scale == 0) this else { - val value = this.clone() - value.changePrecision( - DecimalType.bounded(precision - scale + 1, 0).precision, 0, ROUND_CEILING) - value + toPrecision(DecimalType.bounded(precision - scale + 1, 0).precision, 0, ROUND_CEILING) + .getOrElse(clone()) } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala index 52d0692524d0f..7a20f5fe733fc 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala @@ -193,14 +193,17 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester { assert(Decimal(Long.MaxValue, 100, 0).toUnscaledLong === Long.MaxValue) } - test("changePrecision() on compact decimal should respect rounding mode") { + test("changePrecision/toPrecission on compact decimal should respect rounding mode") { Seq(ROUND_FLOOR, ROUND_CEILING, ROUND_HALF_UP, ROUND_HALF_EVEN).foreach { mode => Seq("0.4", "0.5", "0.6", "1.0", "1.1", "1.6", "2.5", "5.5").foreach { n => Seq("", "-").foreach { sign => val bd = BigDecimal(sign + n) val unscaled = (bd * 10).toLongExact val d = Decimal(unscaled, 8, 1) + val copy = d.toPrecision(10, 0, mode) assert(d.changePrecision(10, 0, mode)) + assert(copy.isDefined) + assert(d.ne(copy.get)) assert(d.toString === bd.setScale(0, mode).toString(), s"num: $sign$n, mode: $mode") } } From d207d0ea077f9b8ce016fd778c24fcf79ab1b27e Mon Sep 17 00:00:00 2001 From: Wojtek Szymanski Date: Wed, 1 Mar 2017 01:40:12 +0100 Subject: [PATCH 3/7] Tests assertions for Decimal.toPrecision --- .../scala/org/apache/spark/sql/types/DecimalSuite.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala index 7a20f5fe733fc..a3fa30a6596ac 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala @@ -200,11 +200,14 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester { val bd = BigDecimal(sign + n) val unscaled = (bd * 10).toLongExact val d = Decimal(unscaled, 8, 1) - val copy = d.toPrecision(10, 0, mode) assert(d.changePrecision(10, 0, mode)) - assert(copy.isDefined) - assert(d.ne(copy.get)) assert(d.toString === bd.setScale(0, mode).toString(), s"num: $sign$n, mode: $mode") + + val copy = d.toPrecision(10, 0, mode).orNull + assert(copy !== null) + assert(d.ne(copy)) + assert(d === copy) + assert(copy.toString === bd.setScale(0, mode).toString(), s"num: $sign$n, mode: $mode") } } } From 51a0aea30b3b8d33b2274ec2d30a2a2b724dd3af Mon Sep 17 00:00:00 2001 From: Wojtek Szymanski Date: Thu, 2 Mar 2017 08:12:00 +0100 Subject: [PATCH 4/7] code review fixes --- .../spark/sql/catalyst/expressions/Cast.scala | 19 +++++++++++++++---- .../org/apache/spark/sql/types/Decimal.scala | 19 +++++++++++-------- .../apache/spark/sql/types/DecimalSuite.scala | 2 +- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 70e3921a1428c..51c592f946eb4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -338,6 +338,16 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String b => x.numeric.asInstanceOf[Numeric[Any]].toInt(b).toByte } + /** + * Change the precision / scale in a given decimal to those set in `decimalType` (if any), + * returning null if it overflows or modifying `value` in-place and returning it if successful. + * + * NOTE: this modifies `value` in-place, so don't call it on external data. + */ + private[this] def changePrecision(value: Decimal, decimalType: DecimalType): Decimal = { + if (value.changePrecision(decimalType.precision, decimalType.scale)) value else null + } + /** * Create new `Decimal` with precision and scale given in `decimalType` (if any), * returning null if it overflows or creating a new `value` and returning it if successful. @@ -346,10 +356,11 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String private[this] def toPrecision(value: Decimal, decimalType: DecimalType): Decimal = value.toPrecision(decimalType.precision, decimalType.scale).orNull + private[this] def castToDecimal(from: DataType, target: DecimalType): Any => Any = from match { case StringType => buildCast[UTF8String](_, s => try { - toPrecision(Decimal(new JavaBigDecimal(s.toString)), target) + changePrecision(Decimal(new JavaBigDecimal(s.toString)), target) } catch { case _: NumberFormatException => null }) @@ -359,14 +370,14 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String buildCast[Int](_, d => null) // date can't cast to decimal in Hive case TimestampType => // Note that we lose precision here. - buildCast[Long](_, t => toPrecision(Decimal(timestampToDouble(t)), target)) + buildCast[Long](_, t => changePrecision(Decimal(timestampToDouble(t)), target)) case dt: DecimalType => b => toPrecision(b.asInstanceOf[Decimal], target) case t: IntegralType => - b => toPrecision(Decimal(t.integral.asInstanceOf[Integral[Any]].toLong(b)), target) + b => changePrecision(Decimal(t.integral.asInstanceOf[Integral[Any]].toLong(b)), target) case x: FractionalType => b => try { - toPrecision(Decimal(x.fractional.asInstanceOf[Fractional[Any]].toDouble(b)), target) + changePrecision(Decimal(x.fractional.asInstanceOf[Fractional[Any]].toDouble(b)), target) } catch { case _: NumberFormatException => null } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala index 8505b8e01e4dd..047639db6c13e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala @@ -21,6 +21,7 @@ import java.lang.{Long => JLong} import java.math.{BigInteger, MathContext, RoundingMode} import org.apache.spark.annotation.InterfaceStability +import org.apache.spark.sql.AnalysisException /** * A mutable implementation of BigDecimal that can hold a Long if values are small enough. @@ -227,9 +228,9 @@ final class Decimal extends Ordered[Decimal] with Serializable { * * @return `Some(decimal)` if successful or `None` if overflow would occur */ - private[sql] def toPrecision(precision: Int, scale: Int, - roundMode: BigDecimal.RoundingMode.Value = - ROUND_HALF_UP): Option[Decimal] = { + private[sql] def toPrecision( + precision: Int, scale: Int, + roundMode: BigDecimal.RoundingMode.Value = ROUND_HALF_UP): Option[Decimal] = { val copy = clone() if (copy.changePrecision(precision, scale, roundMode)) Some(copy) else None } @@ -240,7 +241,7 @@ final class Decimal extends Ordered[Decimal] with Serializable { * @return true if successful, false if overflow would occur */ private[sql] def changePrecision(precision: Int, scale: Int, - roundMode: BigDecimal.RoundingMode.Value): Boolean = { + roundMode: BigDecimal.RoundingMode.Value): Boolean = { // fast path for UnsafeProjection if (precision == this.precision && scale == this.scale) { return true @@ -374,13 +375,15 @@ final class Decimal extends Ordered[Decimal] with Serializable { def abs: Decimal = if (this.compare(Decimal.ZERO) < 0) this.unary_- else this def floor: Decimal = if (scale == 0) this else { - toPrecision(DecimalType.bounded(precision - scale + 1, 0).precision, 0, ROUND_FLOOR) - .getOrElse(clone()) + val newPrecision = DecimalType.bounded(precision - scale + 1, 0).precision + toPrecision(newPrecision, 0, ROUND_FLOOR).getOrElse( + throw new AnalysisException(s"Overflow when setting precision to $newPrecision")) } def ceil: Decimal = if (scale == 0) this else { - toPrecision(DecimalType.bounded(precision - scale + 1, 0).precision, 0, ROUND_CEILING) - .getOrElse(clone()) + val newPrecision = DecimalType.bounded(precision - scale + 1, 0).precision + toPrecision(newPrecision, 0, ROUND_CEILING).getOrElse( + throw new AnalysisException(s"Overflow when setting precision to $newPrecision")) } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala index a3fa30a6596ac..714883a4099cf 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala @@ -193,7 +193,7 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester { assert(Decimal(Long.MaxValue, 100, 0).toUnscaledLong === Long.MaxValue) } - test("changePrecision/toPrecission on compact decimal should respect rounding mode") { + test("changePrecision/toPrecision on compact decimal should respect rounding mode") { Seq(ROUND_FLOOR, ROUND_CEILING, ROUND_HALF_UP, ROUND_HALF_EVEN).foreach { mode => Seq("0.4", "0.5", "0.6", "1.0", "1.1", "1.6", "2.5", "5.5").foreach { n => Seq("", "-").foreach { sign => From 6557cbaace19c85889e7f0a33c3c7acdc935481e Mon Sep 17 00:00:00 2001 From: Wojtek Szymanski Date: Sat, 4 Mar 2017 14:36:17 +0100 Subject: [PATCH 5/7] code style and tests improvements --- .../src/main/scala/org/apache/spark/sql/types/Decimal.scala | 3 ++- .../test/scala/org/apache/spark/sql/MathFunctionsSuite.scala | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala index 047639db6c13e..44ca628bd578f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala @@ -229,7 +229,8 @@ final class Decimal extends Ordered[Decimal] with Serializable { * @return `Some(decimal)` if successful or `None` if overflow would occur */ private[sql] def toPrecision( - precision: Int, scale: Int, + precision: Int, + scale: Int, roundMode: BigDecimal.RoundingMode.Value = ROUND_HALF_UP): Option[Decimal] = { val copy = clone() if (copy.changePrecision(precision, scale, roundMode)) Some(copy) else None diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala index 57c58ce92c77d..a09cb900acb0e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala @@ -240,7 +240,7 @@ class MathFunctionsSuite extends QueryTest with SharedSQLContext { Seq(Row(BigDecimal("5.9"), BigDecimal("6"))) ) checkAnswer( - df.withColumn("value_rounded", bround('value)), + df.withColumn("value_brounded", bround('value)), Seq(Row(BigDecimal("5.9"), BigDecimal("6"))) ) } From c03a876dda11a28db84f9a8030a2bd7078c60be5 Mon Sep 17 00:00:00 2001 From: Wojtek Szymanski Date: Mon, 6 Mar 2017 07:47:43 +0100 Subject: [PATCH 6/7] code style, NumericRow replaced with Tuple1 --- .../src/main/scala/org/apache/spark/sql/types/Decimal.scala | 6 +++--- .../scala/org/apache/spark/sql/MathFunctionsSuite.scala | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala index 44ca628bd578f..f2801ad452707 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala @@ -229,9 +229,9 @@ final class Decimal extends Ordered[Decimal] with Serializable { * @return `Some(decimal)` if successful or `None` if overflow would occur */ private[sql] def toPrecision( - precision: Int, - scale: Int, - roundMode: BigDecimal.RoundingMode.Value = ROUND_HALF_UP): Option[Decimal] = { + precision: Int, + scale: Int, + roundMode: BigDecimal.RoundingMode.Value = ROUND_HALF_UP): Option[Decimal] = { val copy = clone() if (copy.changePrecision(precision, scale, roundMode)) Some(copy) else None } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala index a09cb900acb0e..328c5395ec91e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala @@ -234,7 +234,7 @@ class MathFunctionsSuite extends QueryTest with SharedSQLContext { } test("round/bround with data frame from a local Seq of Product") { - val df = spark.createDataFrame(Seq(NumericRow(BigDecimal("5.9")))) + val df = spark.createDataFrame(Seq(Tuple1(BigDecimal("5.9")))).toDF("value") checkAnswer( df.withColumn("value_rounded", round('value)), Seq(Row(BigDecimal("5.9"), BigDecimal("6"))) @@ -434,4 +434,3 @@ class MathFunctionsSuite extends QueryTest with SharedSQLContext { checkAnswer(df.selectExpr("positive(b)"), Row(-1)) } } -case class NumericRow(value : BigDecimal) From fc0f2d11f60dc1eade4c32d5a56292766061886e Mon Sep 17 00:00:00 2001 From: Wojtek Szymanski Date: Mon, 6 Mar 2017 08:15:59 +0100 Subject: [PATCH 7/7] code style - 4 spaces indention in method args --- .../src/main/scala/org/apache/spark/sql/types/Decimal.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala index f2801ad452707..e8f6884c025c2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala @@ -229,9 +229,9 @@ final class Decimal extends Ordered[Decimal] with Serializable { * @return `Some(decimal)` if successful or `None` if overflow would occur */ private[sql] def toPrecision( - precision: Int, - scale: Int, - roundMode: BigDecimal.RoundingMode.Value = ROUND_HALF_UP): Option[Decimal] = { + precision: Int, + scale: Int, + roundMode: BigDecimal.RoundingMode.Value = ROUND_HALF_UP): Option[Decimal] = { val copy = clone() if (copy.changePrecision(precision, scale, roundMode)) Some(copy) else None }