Skip to content

[SPARK-2209][SQL] Cast shouldn't do null check twice. #1143

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

[SPARK-2209][SQL] Cast shouldn't do null check twice. #1143

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c2b88ae
[SPARK-2209][SQL] Cast shouldn't do null check twice.
rxin Jun 20, 2014
dd006cb
Code review feedback.
rxin Jun 20, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
274 changes: 159 additions & 115 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,72 +24,87 @@ import org.apache.spark.sql.catalyst.types._
/** Cast the child expression to the target data type. */
case class Cast(child: Expression, dataType: DataType) extends UnaryExpression {
override def foldable = child.foldable
def nullable = (child.dataType, dataType) match {

override def nullable = (child.dataType, dataType) match {
case (StringType, _: NumericType) => true
case (StringType, TimestampType) => true
case _ => child.nullable
}

override def toString = s"CAST($child, $dataType)"

type EvaluatedType = Any

def nullOrCast[T](a: Any, func: T => Any): Any = if(a == null) {
null
} else {
func(a.asInstanceOf[T])
}
// [[func]] assumes the input is no longer null because eval already does the null check.
@inline private[this] def buildCast[T](a: Any, func: T => Any): Any = func(a.asInstanceOf[T])

// UDFToString
def castToString: Any => Any = child.dataType match {
case BinaryType => nullOrCast[Array[Byte]](_, new String(_, "UTF-8"))
case _ => nullOrCast[Any](_, _.toString)
private[this] def castToString: Any => Any = child.dataType match {
case BinaryType => buildCast[Array[Byte]](_, new String(_, "UTF-8"))
case _ => buildCast[Any](_, _.toString)
}

// BinaryConverter
def castToBinary: Any => Any = child.dataType match {
case StringType => nullOrCast[String](_, _.getBytes("UTF-8"))
private[this] def castToBinary: Any => Any = child.dataType match {
case StringType => buildCast[String](_, _.getBytes("UTF-8"))
}

// UDFToBoolean
def castToBoolean: Any => Any = child.dataType match {
case StringType => nullOrCast[String](_, _.length() != 0)
case TimestampType => nullOrCast[Timestamp](_, b => {(b.getTime() != 0 || b.getNanos() != 0)})
case LongType => nullOrCast[Long](_, _ != 0)
case IntegerType => nullOrCast[Int](_, _ != 0)
case ShortType => nullOrCast[Short](_, _ != 0)
case ByteType => nullOrCast[Byte](_, _ != 0)
case DecimalType => nullOrCast[BigDecimal](_, _ != 0)
case DoubleType => nullOrCast[Double](_, _ != 0)
case FloatType => nullOrCast[Float](_, _ != 0)
private[this] def castToBoolean: Any => Any = child.dataType match {
case StringType =>
buildCast[String](_, _.length() != 0)
case TimestampType =>
buildCast[Timestamp](_, b => b.getTime() != 0 || b.getNanos() != 0)
case LongType =>
buildCast[Long](_, _ != 0)
case IntegerType =>
buildCast[Int](_, _ != 0)
case ShortType =>
buildCast[Short](_, _ != 0)
case ByteType =>
buildCast[Byte](_, _ != 0)
case DecimalType =>
buildCast[BigDecimal](_, _ != 0)
case DoubleType =>
buildCast[Double](_, _ != 0)
case FloatType =>
buildCast[Float](_, _ != 0)
}

// TimestampConverter
def castToTimestamp: Any => Any = child.dataType match {
case StringType => nullOrCast[String](_, s => {
// Throw away extra if more than 9 decimal places
val periodIdx = s.indexOf(".");
var n = s
if (periodIdx != -1) {
if (n.length() - periodIdx > 9) {
private[this] def castToTimestamp: Any => Any = child.dataType match {
case StringType =>
buildCast[String](_, s => {
// Throw away extra if more than 9 decimal places
val periodIdx = s.indexOf(".")
var n = s
if (periodIdx != -1 && n.length() - periodIdx > 9) {
n = n.substring(0, periodIdx + 10)
}
}
try Timestamp.valueOf(n) catch { case _: java.lang.IllegalArgumentException => null}
})
case BooleanType => nullOrCast[Boolean](_, b => new Timestamp((if(b) 1 else 0) * 1000))
case LongType => nullOrCast[Long](_, l => new Timestamp(l * 1000))
case IntegerType => nullOrCast[Int](_, i => new Timestamp(i * 1000))
case ShortType => nullOrCast[Short](_, s => new Timestamp(s * 1000))
case ByteType => nullOrCast[Byte](_, b => new Timestamp(b * 1000))
try Timestamp.valueOf(n) catch { case _: java.lang.IllegalArgumentException => null }
})
case BooleanType =>
buildCast[Boolean](_, b => new Timestamp((if (b) 1 else 0) * 1000))
case LongType =>
buildCast[Long](_, l => new Timestamp(l * 1000))
case IntegerType =>
buildCast[Int](_, i => new Timestamp(i * 1000))
case ShortType =>
buildCast[Short](_, s => new Timestamp(s * 1000))
case ByteType =>
buildCast[Byte](_, b => new Timestamp(b * 1000))
// TimestampWritable.decimalToTimestamp
case DecimalType => nullOrCast[BigDecimal](_, d => decimalToTimestamp(d))
case DecimalType =>
buildCast[BigDecimal](_, d => decimalToTimestamp(d))
// TimestampWritable.doubleToTimestamp
case DoubleType => nullOrCast[Double](_, d => decimalToTimestamp(d))
case DoubleType =>
buildCast[Double](_, d => decimalToTimestamp(d))
// TimestampWritable.floatToTimestamp
case FloatType => nullOrCast[Float](_, f => decimalToTimestamp(f))
case FloatType =>
buildCast[Float](_, f => decimalToTimestamp(f))
}

private def decimalToTimestamp(d: BigDecimal) = {
private[this] def decimalToTimestamp(d: BigDecimal) = {
val seconds = d.longValue()
Copy link
Contributor

@liancheng liancheng Jun 20, 2014

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

According to the API documentation, d.longValue() and d.intValue() may be negative:

scala> var d: BigDecimal = 1000000000
d: BigDecimal = 1000000000

scala> d = d * d * d * d
d: BigDecimal = 1.000000000000000000000000000000000E+36

scala> d.longValue
res0: Long = -5527149226598858752

Copy link
Contributor Author

@rxin rxin Jun 20, 2014

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you file a jira ticket? I'm tempted to not fix that in this PR.

val bd = (d - seconds) * 1000000000
val nanos = bd.intValue()
Expand All @@ -104,85 +119,118 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression {
}

// Timestamp to long, converting milliseconds to seconds
private def timestampToLong(ts: Timestamp) = ts.getTime / 1000
private[this] def timestampToLong(ts: Timestamp) = ts.getTime / 1000

private def timestampToDouble(ts: Timestamp) = {
private[this] def timestampToDouble(ts: Timestamp) = {
// First part is the seconds since the beginning of time, followed by nanosecs.
ts.getTime / 1000 + ts.getNanos.toDouble / 1000000000
}

def castToLong: Any => Any = child.dataType match {
case StringType => nullOrCast[String](_, s => try s.toLong catch {
case _: NumberFormatException => null
})
case BooleanType => nullOrCast[Boolean](_, b => if(b) 1L else 0L)
case TimestampType => nullOrCast[Timestamp](_, t => timestampToLong(t))
case DecimalType => nullOrCast[BigDecimal](_, _.toLong)
case x: NumericType => b => x.numeric.asInstanceOf[Numeric[Any]].toLong(b)
}

def castToInt: Any => Any = child.dataType match {
case StringType => nullOrCast[String](_, s => try s.toInt catch {
case _: NumberFormatException => null
})
case BooleanType => nullOrCast[Boolean](_, b => if(b) 1 else 0)
case TimestampType => nullOrCast[Timestamp](_, t => timestampToLong(t).toInt)
case DecimalType => nullOrCast[BigDecimal](_, _.toInt)
case x: NumericType => b => x.numeric.asInstanceOf[Numeric[Any]].toInt(b)
}

def castToShort: Any => Any = child.dataType match {
case StringType => nullOrCast[String](_, s => try s.toShort catch {
case _: NumberFormatException => null
})
case BooleanType => nullOrCast[Boolean](_, b => if(b) 1.toShort else 0.toShort)
case TimestampType => nullOrCast[Timestamp](_, t => timestampToLong(t).toShort)
case DecimalType => nullOrCast[BigDecimal](_, _.toShort)
case x: NumericType => b => x.numeric.asInstanceOf[Numeric[Any]].toInt(b).toShort
}

def castToByte: Any => Any = child.dataType match {
case StringType => nullOrCast[String](_, s => try s.toByte catch {
case _: NumberFormatException => null
})
case BooleanType => nullOrCast[Boolean](_, b => if(b) 1.toByte else 0.toByte)
case TimestampType => nullOrCast[Timestamp](_, t => timestampToLong(t).toByte)
case DecimalType => nullOrCast[BigDecimal](_, _.toByte)
case x: NumericType => b => x.numeric.asInstanceOf[Numeric[Any]].toInt(b).toByte
}

def castToDecimal: Any => Any = child.dataType match {
case StringType => nullOrCast[String](_, s => try BigDecimal(s.toDouble) catch {
case _: NumberFormatException => null
})
case BooleanType => nullOrCast[Boolean](_, b => if(b) BigDecimal(1) else BigDecimal(0))
private[this] def castToLong: Any => Any = child.dataType match {
case StringType =>
buildCast[String](_, s => try s.toLong catch {
case _: NumberFormatException => null
})
Copy link
Contributor

@liancheng liancheng Jun 20, 2014

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we can simplify this to:

Try(s.toLong).getOrElse(null)

(s.toLong throws NumerFormatException only.)

Copy link
Contributor Author

@rxin rxin Jun 20, 2014

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Try is really slow though.

case BooleanType =>
buildCast[Boolean](_, b => if (b) 1L else 0L)
case TimestampType =>
buildCast[Timestamp](_, t => timestampToLong(t))
case DecimalType =>
buildCast[BigDecimal](_, _.toLong)
case x: NumericType =>
b => x.numeric.asInstanceOf[Numeric[Any]].toLong(b)
}

private[this] def castToInt: Any => Any = child.dataType match {
case StringType =>
buildCast[String](_, s => try s.toInt catch {
case _: NumberFormatException => null
})
case BooleanType =>
buildCast[Boolean](_, b => if (b) 1 else 0)
case TimestampType =>
buildCast[Timestamp](_, t => timestampToLong(t).toInt)
case DecimalType =>
buildCast[BigDecimal](_, _.toInt)
case x: NumericType =>
b => x.numeric.asInstanceOf[Numeric[Any]].toInt(b)
}

private[this] def castToShort: Any => Any = child.dataType match {
case StringType =>
buildCast[String](_, s => try s.toShort catch {
case _: NumberFormatException => null
})
case BooleanType =>
buildCast[Boolean](_, b => if (b) 1.toShort else 0.toShort)
case TimestampType =>
buildCast[Timestamp](_, t => timestampToLong(t).toShort)
case DecimalType =>
buildCast[BigDecimal](_, _.toShort)
case x: NumericType =>
b => x.numeric.asInstanceOf[Numeric[Any]].toInt(b).toShort
}

private[this] def castToByte: Any => Any = child.dataType match {
case StringType =>
buildCast[String](_, s => try s.toByte catch {
case _: NumberFormatException => null
})
case BooleanType =>
buildCast[Boolean](_, b => if (b) 1.toByte else 0.toByte)
case TimestampType =>
buildCast[Timestamp](_, t => timestampToLong(t).toByte)
case DecimalType =>
buildCast[BigDecimal](_, _.toByte)
case x: NumericType =>
b => x.numeric.asInstanceOf[Numeric[Any]].toInt(b).toByte
}

private[this] def castToDecimal: Any => Any = child.dataType match {
case StringType =>
buildCast[String](_, s => try BigDecimal(s.toDouble) catch {
case _: NumberFormatException => null
})
case BooleanType =>
buildCast[Boolean](_, b => if (b) BigDecimal(1) else BigDecimal(0))
case TimestampType =>
// Note that we lose precision here.
nullOrCast[Timestamp](_, t => BigDecimal(timestampToDouble(t)))
case x: NumericType => b => BigDecimal(x.numeric.asInstanceOf[Numeric[Any]].toDouble(b))
}

def castToDouble: Any => Any = child.dataType match {
case StringType => nullOrCast[String](_, s => try s.toDouble catch {
case _: NumberFormatException => null
})
case BooleanType => nullOrCast[Boolean](_, b => if(b) 1d else 0d)
case TimestampType => nullOrCast[Timestamp](_, t => timestampToDouble(t))
case DecimalType => nullOrCast[BigDecimal](_, _.toDouble)
case x: NumericType => b => x.numeric.asInstanceOf[Numeric[Any]].toDouble(b)
}

def castToFloat: Any => Any = child.dataType match {
case StringType => nullOrCast[String](_, s => try s.toFloat catch {
case _: NumberFormatException => null
})
case BooleanType => nullOrCast[Boolean](_, b => if(b) 1f else 0f)
case TimestampType => nullOrCast[Timestamp](_, t => timestampToDouble(t).toFloat)
case DecimalType => nullOrCast[BigDecimal](_, _.toFloat)
case x: NumericType => b => x.numeric.asInstanceOf[Numeric[Any]].toFloat(b)
buildCast[Timestamp](_, t => BigDecimal(timestampToDouble(t)))
case x: NumericType =>
b => BigDecimal(x.numeric.asInstanceOf[Numeric[Any]].toDouble(b))
}

private[this] def castToDouble: Any => Any = child.dataType match {
case StringType =>
buildCast[String](_, s => try s.toDouble catch {
case _: NumberFormatException => null
})
case BooleanType =>
buildCast[Boolean](_, b => if (b) 1d else 0d)
case TimestampType =>
buildCast[Timestamp](_, t => timestampToDouble(t))
case DecimalType =>
buildCast[BigDecimal](_, _.toDouble)
case x: NumericType =>
b => x.numeric.asInstanceOf[Numeric[Any]].toDouble(b)
}

private[this] def castToFloat: Any => Any = child.dataType match {
case StringType =>
buildCast[String](_, s => try s.toFloat catch {
case _: NumberFormatException => null
})
case BooleanType =>
buildCast[Boolean](_, b => if (b) 1f else 0f)
case TimestampType =>
buildCast[Timestamp](_, t => timestampToDouble(t).toFloat)
case DecimalType =>
buildCast[BigDecimal](_, _.toFloat)
case x: NumericType =>
b => x.numeric.asInstanceOf[Numeric[Any]].toFloat(b)
}

private lazy val cast: Any => Any = dataType match {
private[this] lazy val cast: Any => Any = dataType match {
case StringType => castToString
case BinaryType => castToBinary
case DecimalType => castToDecimal
Expand All @@ -198,10 +246,6 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression {

override def eval(input: Row): Any = {
val evaluated = child.eval(input)
if (evaluated == null) {
null
} else {
cast(evaluated)
}
if (evaluated == null) null else cast(evaluated)
}
}