From 5e60ec5047d99efae061f4c716f33cb2146f78ed Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Tue, 19 May 2020 20:30:08 +0800 Subject: [PATCH 01/18] SPARK-31710: TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS and TIMESTAMP_MICROSECONDS to timestamp transfer --- .../catalyst/analysis/FunctionRegistry.scala | 3 + .../expressions/datetimeExpressions.scala | 86 +++++++++++++++++++ .../org/apache/spark/sql/SQLQuerySuite.scala | 22 +++++ 3 files changed, 111 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index 26d7b3ef3dcd..22bb51667928 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -424,6 +424,9 @@ object FunctionRegistry { expression[MakeInterval]("make_interval"), expression[DatePart]("date_part"), expression[Extract]("extract"), + expression[SecondsToTimestamp]("timestamp_seconds"), + expression[MilliSecondsToTimestamp]("timestamp_milliseconds"), + expression[MicroSecondsToTimestamp]("timestamp_microseconds"), // collection functions expression[CreateArray]("array"), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index ccedcb41fc28..1b36606350e2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -401,6 +401,92 @@ case class DayOfYear(child: Expression) extends UnaryExpression with ImplicitCas } } +@ExpressionDescription( + usage = "_FUNC_(date) - Returns timestamp from seconds.", + examples = """ + Examples: + > SELECT _FUNC_(1230219000); + "2008-12-25 07:30:00.0" + """, + group = "datetime_funcs", + since = "3.1.0") +case class SecondsToTimestamp(child: Expression) + extends NumberToTimestampBase { + + override def upScaleFactor: SQLTimestamp = MICROS_PER_SECOND + + override def prettyName: String = "timestamp_seconds" +} + +@ExpressionDescription( + usage = "_FUNC_(date) - Returns timestamp from milliseconds.", + examples = """ + Examples: + > SELECT _FUNC_(1230219000000); + "2008-12-25 07:30:00.0" + """, + group = "datetime_funcs", + since = "3.1.0") +case class MilliSecondsToTimestamp(child: Expression) + extends NumberToTimestampBase { + + override def upScaleFactor: SQLTimestamp = MICROS_PER_MILLIS + + override def prettyName: String = "timestamp_milliseconds" +} + +@ExpressionDescription( + usage = "_FUNC_(date) - Returns timestamp from microseconds.", + examples = """ + Examples: + > SELECT _FUNC_(1230219000000000); + "2008-12-25 07:30:00.0" + """, + group = "datetime_funcs", + since = "3.1.0") +case class MicroSecondsToTimestamp(child: Expression) + extends NumberToTimestampBase { + + override def upScaleFactor: SQLTimestamp = 1L + + override def prettyName: String = "timestamp_microseconds" +} + +abstract class NumberToTimestampBase extends UnaryExpression + with ImplicitCastInputTypes{ + + protected def upScaleFactor: Long + + override def inputTypes: Seq[AbstractDataType] = Seq(LongType, IntegerType) + + override def dataType: DataType = TimestampType + + override def eval(input: InternalRow): Any = { + val t = child.eval(input) + if (t == null) { + null + } else { + child.dataType match { + case IntegerType => + Math.multiplyExact(t.asInstanceOf[Int].toLong, upScaleFactor) + case LongType => + Math.multiplyExact(t.asInstanceOf[Long], upScaleFactor) + } + } + } + + override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + child.dataType match { + case IntegerType => + defineCodeGen(ctx, ev, _ => s"java.lang.Math.multiplyExact(" + + s"${ev.value.asInstanceOf[Integer].toLong}, ${upScaleFactor})") + case LongType => + defineCodeGen(ctx, ev, _ => s"java.lang.Math.multiplyExact(" + + s"${ev.value.asInstanceOf[Long]}, ${upScaleFactor})") + } + } +} + @ExpressionDescription( usage = "_FUNC_(date) - Returns the year component of the date/timestamp.", examples = """ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 788a07370195..1d4df680133e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -3495,6 +3495,28 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark assert(df4.schema.head.name === "randn(1)") checkIfSeedExistsInExplain(df2) } + + test("SPARK-31710: " + + "TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS and TIMESTAMP_MICROSECONDS to timestamp transfer") { + val df1 = sql("select TIMESTAMP_SECONDS(1230219000) as timestamp") + checkAnswer(df1, Row(Timestamp.valueOf("2008-12-25 07:30:00.0"))) + + val df2 = sql("select TIMESTAMP_MILLISECONDS(1230219000000) as timestamp") + checkAnswer(df2, Row(Timestamp.valueOf("2008-12-25 07:30:00.0"))) + + val df3 = sql("select TIMESTAMP_MICROSECONDS(1230219000000000) as timestamp") + checkAnswer(df3, Row(Timestamp.valueOf("2008-12-25 07:30:00.0"))) + + val df4 = sql("select TIMESTAMP_SECONDS(-1230219000) as timestamp") + checkAnswer(df4, Row(Timestamp.valueOf("1931-01-07 00:30:00.0"))) + + val df5 = sql("select TIMESTAMP_MILLISECONDS(-1230219000000) as timestamp") + checkAnswer(df5, Row(Timestamp.valueOf("1931-01-07 00:30:00.0"))) + + val df6 = sql("select TIMESTAMP_MICROSECONDS(-1230219000000000) as timestamp") + checkAnswer(df6, Row(Timestamp.valueOf("1931-01-07 00:30:00.0"))) + + } } case class Foo(bar: Option[String]) From 27a162b32ecdd824f720c757ee6e5090ebf43669 Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Wed, 20 May 2020 14:18:13 +0800 Subject: [PATCH 02/18] improve code quality according to advices --- .../expressions/datetimeExpressions.scala | 34 +++++++------------ .../resources/sql-tests/inputs/datetime.sql | 5 +++ .../sql-tests/results/ansi/datetime.sql.out | 24 +++++++++++++ .../sql-tests/results/datetime.sql.out | 24 +++++++++++++ .../org/apache/spark/sql/SQLQuerySuite.scala | 32 ++++++++--------- 5 files changed, 80 insertions(+), 39 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 1b36606350e2..1532a5b68e99 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -402,11 +402,11 @@ case class DayOfYear(child: Expression) extends UnaryExpression with ImplicitCas } @ExpressionDescription( - usage = "_FUNC_(date) - Returns timestamp from seconds.", + usage = "_FUNC_(seconds) - Creates timestamp from the number of seconds since UTC epoch.", examples = """ Examples: > SELECT _FUNC_(1230219000); - "2008-12-25 07:30:00.0" + "2008-12-25 07:30:00" """, group = "datetime_funcs", since = "3.1.0") @@ -419,11 +419,12 @@ case class SecondsToTimestamp(child: Expression) } @ExpressionDescription( - usage = "_FUNC_(date) - Returns timestamp from milliseconds.", + usage = "_FUNC_(milliseconds) - " + + "Creates timestamp from the number of milliseconds since UTC epoch.", examples = """ Examples: > SELECT _FUNC_(1230219000000); - "2008-12-25 07:30:00.0" + "2008-12-25 07:30:00" """, group = "datetime_funcs", since = "3.1.0") @@ -436,11 +437,12 @@ case class MilliSecondsToTimestamp(child: Expression) } @ExpressionDescription( - usage = "_FUNC_(date) - Returns timestamp from microseconds.", + usage = "_FUNC_(microseconds) - " + + "Creates timestamp from the number of microseconds since UTC epoch.", examples = """ Examples: > SELECT _FUNC_(1230219000000000); - "2008-12-25 07:30:00.0" + "2008-12-25 07:30:00" """, group = "datetime_funcs", since = "3.1.0") @@ -457,29 +459,19 @@ abstract class NumberToTimestampBase extends UnaryExpression protected def upScaleFactor: Long - override def inputTypes: Seq[AbstractDataType] = Seq(LongType, IntegerType) + override def inputTypes: Seq[AbstractDataType] = Seq(LongType) override def dataType: DataType = TimestampType - override def eval(input: InternalRow): Any = { - val t = child.eval(input) - if (t == null) { - null - } else { - child.dataType match { - case IntegerType => - Math.multiplyExact(t.asInstanceOf[Int].toLong, upScaleFactor) - case LongType => - Math.multiplyExact(t.asInstanceOf[Long], upScaleFactor) - } + override def nullSafeEval(input: Any): Any = { + child.dataType match { + case LongType => + Math.multiplyExact(input.asInstanceOf[Long], upScaleFactor) } } override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { child.dataType match { - case IntegerType => - defineCodeGen(ctx, ev, _ => s"java.lang.Math.multiplyExact(" + - s"${ev.value.asInstanceOf[Integer].toLong}, ${upScaleFactor})") case LongType => defineCodeGen(ctx, ev, _ => s"java.lang.Math.multiplyExact(" + s"${ev.value.asInstanceOf[Long]}, ${upScaleFactor})") diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql index 9be857ef767d..c2bdc7ef92ca 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql @@ -1,5 +1,10 @@ -- date time functions +-- [SPARK-31710] TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS and TIMESTAMP_MICROSECONDS to timestamp transfer +select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null); +select TIMESTAMP_MILLISECONDS(1230219000000),TIMESTAMP_MILLISECONDS(-1230219000000),TIMESTAMP_MILLISECONDS(null); +select TIMESTAMP_MICROSECONDS(1230219000000000),TIMESTAMP_MICROSECONDS(-1230219000000000),TIMESTAMP_MICROSECONDS(null); + -- [SPARK-16836] current_date and current_timestamp literals select current_date = current_date(), current_timestamp = current_timestamp(); diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out index ee155acb9b26..6258733464c8 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out @@ -2,6 +2,30 @@ -- Number of queries: 91 +-- !query +select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null) +-- !query schema +struct +-- !query output +2008-12-25 07:30:00 1931-01-07 00:30:00 NULL + + +-- !query +select TIMESTAMP_MILLISECONDS(1230219000000),TIMESTAMP_MILLISECONDS(-1230219000000),TIMESTAMP_MILLISECONDS(null) +-- !query schema +struct +-- !query output +2008-12-25 07:30:00 1931-01-07 00:30:00 NULL + + +-- !query +select TIMESTAMP_MICROSECONDS(1230219000000000),TIMESTAMP_MICROSECONDS(-1230219000000000),TIMESTAMP_MICROSECONDS(null) +-- !query schema +struct +-- !query output +2008-12-25 07:30:00 1931-01-07 00:30:00 NULL + + -- !query select current_date = current_date(), current_timestamp = current_timestamp() -- !query schema diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out index f3fc6a51e0a5..c120db7fa2b7 100755 --- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out @@ -2,6 +2,30 @@ -- Number of queries: 91 +-- !query +select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null) +-- !query schema +struct +-- !query output +2008-12-25 07:30:00 1931-01-07 00:30:00 NULL + + +-- !query +select TIMESTAMP_MILLISECONDS(1230219000000),TIMESTAMP_MILLISECONDS(-1230219000000),TIMESTAMP_MILLISECONDS(null) +-- !query schema +struct +-- !query output +2008-12-25 07:30:00 1931-01-07 00:30:00 NULL + + +-- !query +select TIMESTAMP_MICROSECONDS(1230219000000000),TIMESTAMP_MICROSECONDS(-1230219000000000),TIMESTAMP_MICROSECONDS(null) +-- !query schema +struct +-- !query output +2008-12-25 07:30:00 1931-01-07 00:30:00 NULL + + -- !query select current_date = current_date(), current_timestamp = current_timestamp() -- !query schema diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 1d4df680133e..e4eebb5954ba 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -3498,24 +3498,20 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark test("SPARK-31710: " + "TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS and TIMESTAMP_MICROSECONDS to timestamp transfer") { - val df1 = sql("select TIMESTAMP_SECONDS(1230219000) as timestamp") - checkAnswer(df1, Row(Timestamp.valueOf("2008-12-25 07:30:00.0"))) - - val df2 = sql("select TIMESTAMP_MILLISECONDS(1230219000000) as timestamp") - checkAnswer(df2, Row(Timestamp.valueOf("2008-12-25 07:30:00.0"))) - - val df3 = sql("select TIMESTAMP_MICROSECONDS(1230219000000000) as timestamp") - checkAnswer(df3, Row(Timestamp.valueOf("2008-12-25 07:30:00.0"))) - - val df4 = sql("select TIMESTAMP_SECONDS(-1230219000) as timestamp") - checkAnswer(df4, Row(Timestamp.valueOf("1931-01-07 00:30:00.0"))) - - val df5 = sql("select TIMESTAMP_MILLISECONDS(-1230219000000) as timestamp") - checkAnswer(df5, Row(Timestamp.valueOf("1931-01-07 00:30:00.0"))) - - val df6 = sql("select TIMESTAMP_MICROSECONDS(-1230219000000000) as timestamp") - checkAnswer(df6, Row(Timestamp.valueOf("1931-01-07 00:30:00.0"))) - + val df1 = sql("select TIMESTAMP_SECONDS(1230219000)," + + "TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)") + checkAnswer(df1, Row(Timestamp.valueOf("2008-12-25 07:30:00"), + Timestamp.valueOf("1931-01-07 00:30:00"), null)) + + val df2 = sql("select TIMESTAMP_MILLISECONDS(1230219000000)," + + "TIMESTAMP_MILLISECONDS(-1230219000000),TIMESTAMP_MILLISECONDS(null)") + checkAnswer(df2, Row(Timestamp.valueOf("2008-12-25 07:30:00"), + Timestamp.valueOf("1931-01-07 00:30:00"), null)) + + val df3 = sql("select TIMESTAMP_MICROSECONDS(1230219000000000)," + + "TIMESTAMP_MICROSECONDS(-1230219000000000),TIMESTAMP_MICROSECONDS(null)") + checkAnswer(df3, Row(Timestamp.valueOf("2008-12-25 07:30:00"), + Timestamp.valueOf("1931-01-07 00:30:00"), null)) } } From f879987c662a789dba56673900d699564c63b225 Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Wed, 20 May 2020 15:59:52 +0800 Subject: [PATCH 03/18] code style improved and remove bad asInstanceOf etc --- .../catalyst/expressions/datetimeExpressions.scala | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 1532a5b68e99..cc38d71ba77c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -455,7 +455,7 @@ case class MicroSecondsToTimestamp(child: Expression) } abstract class NumberToTimestampBase extends UnaryExpression - with ImplicitCastInputTypes{ + with ImplicitCastInputTypes { protected def upScaleFactor: Long @@ -464,18 +464,12 @@ abstract class NumberToTimestampBase extends UnaryExpression override def dataType: DataType = TimestampType override def nullSafeEval(input: Any): Any = { - child.dataType match { - case LongType => - Math.multiplyExact(input.asInstanceOf[Long], upScaleFactor) - } + Math.multiplyExact(input.asInstanceOf[Long], upScaleFactor) } override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - child.dataType match { - case LongType => - defineCodeGen(ctx, ev, _ => s"java.lang.Math.multiplyExact(" + - s"${ev.value.asInstanceOf[Long]}, ${upScaleFactor})") - } + defineCodeGen(ctx, ev, _ => s"java.lang.Math.multiplyExact(" + + s"${ev.value}, ${upScaleFactor})") } } From 4c17d4937286521719abaeb86220cfb1f1e4853f Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Wed, 20 May 2020 16:31:22 +0800 Subject: [PATCH 04/18] From a readability perspective it may be better to put the base class before the subclasses. --- .../expressions/datetimeExpressions.scala | 38 +++++++++---------- .../org/apache/spark/sql/SQLQuerySuite.scala | 18 --------- 2 files changed, 19 insertions(+), 37 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index cc38d71ba77c..0188a41be223 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -401,6 +401,25 @@ case class DayOfYear(child: Expression) extends UnaryExpression with ImplicitCas } } +abstract class NumberToTimestampBase extends UnaryExpression + with ImplicitCastInputTypes { + + protected def upScaleFactor: Long + + override def inputTypes: Seq[AbstractDataType] = Seq(LongType) + + override def dataType: DataType = TimestampType + + override def nullSafeEval(input: Any): Any = { + Math.multiplyExact(input.asInstanceOf[Long], upScaleFactor) + } + + override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + defineCodeGen(ctx, ev, _ => s"java.lang.Math.multiplyExact(" + + s"${ev.value}, ${upScaleFactor})") + } +} + @ExpressionDescription( usage = "_FUNC_(seconds) - Creates timestamp from the number of seconds since UTC epoch.", examples = """ @@ -454,25 +473,6 @@ case class MicroSecondsToTimestamp(child: Expression) override def prettyName: String = "timestamp_microseconds" } -abstract class NumberToTimestampBase extends UnaryExpression - with ImplicitCastInputTypes { - - protected def upScaleFactor: Long - - override def inputTypes: Seq[AbstractDataType] = Seq(LongType) - - override def dataType: DataType = TimestampType - - override def nullSafeEval(input: Any): Any = { - Math.multiplyExact(input.asInstanceOf[Long], upScaleFactor) - } - - override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - defineCodeGen(ctx, ev, _ => s"java.lang.Math.multiplyExact(" + - s"${ev.value}, ${upScaleFactor})") - } -} - @ExpressionDescription( usage = "_FUNC_(date) - Returns the year component of the date/timestamp.", examples = """ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index e4eebb5954ba..788a07370195 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -3495,24 +3495,6 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark assert(df4.schema.head.name === "randn(1)") checkIfSeedExistsInExplain(df2) } - - test("SPARK-31710: " + - "TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS and TIMESTAMP_MICROSECONDS to timestamp transfer") { - val df1 = sql("select TIMESTAMP_SECONDS(1230219000)," + - "TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)") - checkAnswer(df1, Row(Timestamp.valueOf("2008-12-25 07:30:00"), - Timestamp.valueOf("1931-01-07 00:30:00"), null)) - - val df2 = sql("select TIMESTAMP_MILLISECONDS(1230219000000)," + - "TIMESTAMP_MILLISECONDS(-1230219000000),TIMESTAMP_MILLISECONDS(null)") - checkAnswer(df2, Row(Timestamp.valueOf("2008-12-25 07:30:00"), - Timestamp.valueOf("1931-01-07 00:30:00"), null)) - - val df3 = sql("select TIMESTAMP_MICROSECONDS(1230219000000000)," + - "TIMESTAMP_MICROSECONDS(-1230219000000000),TIMESTAMP_MICROSECONDS(null)") - checkAnswer(df3, Row(Timestamp.valueOf("2008-12-25 07:30:00"), - Timestamp.valueOf("1931-01-07 00:30:00"), null)) - } } case class Foo(bar: Option[String]) From 34119358af3be2938b6ab85d0eb17eeca0602469 Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Wed, 20 May 2020 17:34:26 +0800 Subject: [PATCH 05/18] new exception check when overflow occur when multiply long and more precision used in example and test cases to milliseconds and microseconds for more details --- .../expressions/datetimeExpressions.scala | 16 +++++----- .../resources/sql-tests/inputs/datetime.sql | 4 +-- .../sql-tests/results/ansi/datetime.sql.out | 12 ++++---- .../sql-tests/results/datetime.sql.out | 12 ++++---- .../org/apache/spark/sql/SQLQuerySuite.scala | 30 +++++++++++++++++++ 5 files changed, 52 insertions(+), 22 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 0188a41be223..87e6ff783750 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -416,7 +416,7 @@ abstract class NumberToTimestampBase extends UnaryExpression override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { defineCodeGen(ctx, ev, _ => s"java.lang.Math.multiplyExact(" + - s"${ev.value}, ${upScaleFactor})") + s"${ev.value}, $upScaleFactor)") } } @@ -432,7 +432,7 @@ abstract class NumberToTimestampBase extends UnaryExpression case class SecondsToTimestamp(child: Expression) extends NumberToTimestampBase { - override def upScaleFactor: SQLTimestamp = MICROS_PER_SECOND + override def upScaleFactor: Long = MICROS_PER_SECOND override def prettyName: String = "timestamp_seconds" } @@ -442,15 +442,15 @@ case class SecondsToTimestamp(child: Expression) "Creates timestamp from the number of milliseconds since UTC epoch.", examples = """ Examples: - > SELECT _FUNC_(1230219000000); - "2008-12-25 07:30:00" + > SELECT _FUNC_(1230219000123); + "2008-12-25 07:30:00.123" """, group = "datetime_funcs", since = "3.1.0") case class MilliSecondsToTimestamp(child: Expression) extends NumberToTimestampBase { - override def upScaleFactor: SQLTimestamp = MICROS_PER_MILLIS + override def upScaleFactor: Long = MICROS_PER_MILLIS override def prettyName: String = "timestamp_milliseconds" } @@ -460,15 +460,15 @@ case class MilliSecondsToTimestamp(child: Expression) "Creates timestamp from the number of microseconds since UTC epoch.", examples = """ Examples: - > SELECT _FUNC_(1230219000000000); - "2008-12-25 07:30:00" + > SELECT _FUNC_(1230219000123123); + "2008-12-25 07:30:00.123123" """, group = "datetime_funcs", since = "3.1.0") case class MicroSecondsToTimestamp(child: Expression) extends NumberToTimestampBase { - override def upScaleFactor: SQLTimestamp = 1L + override def upScaleFactor: Long = 1L override def prettyName: String = "timestamp_microseconds" } diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql index c2bdc7ef92ca..9ae02203179c 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql @@ -2,8 +2,8 @@ -- [SPARK-31710] TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS and TIMESTAMP_MICROSECONDS to timestamp transfer select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null); -select TIMESTAMP_MILLISECONDS(1230219000000),TIMESTAMP_MILLISECONDS(-1230219000000),TIMESTAMP_MILLISECONDS(null); -select TIMESTAMP_MICROSECONDS(1230219000000000),TIMESTAMP_MICROSECONDS(-1230219000000000),TIMESTAMP_MICROSECONDS(null); +select TIMESTAMP_MILLISECONDS(1230219000123),TIMESTAMP_MILLISECONDS(-1230219000123),TIMESTAMP_MILLISECONDS(null); +select TIMESTAMP_MICROSECONDS(1230219000123123),TIMESTAMP_MICROSECONDS(-1230219000123123),TIMESTAMP_MICROSECONDS(null); -- [SPARK-16836] current_date and current_timestamp literals select current_date = current_date(), current_timestamp = current_timestamp(); diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out index 6258733464c8..94a20c5a697a 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out @@ -11,19 +11,19 @@ struct +struct -- !query output -2008-12-25 07:30:00 1931-01-07 00:30:00 NULL +2008-12-25 07:30:00.123 1931-01-07 00:29:59.877 NULL -- !query -select TIMESTAMP_MICROSECONDS(1230219000000000),TIMESTAMP_MICROSECONDS(-1230219000000000),TIMESTAMP_MICROSECONDS(null) +select TIMESTAMP_MICROSECONDS(1230219000123123),TIMESTAMP_MICROSECONDS(-1230219000123123),TIMESTAMP_MICROSECONDS(null) -- !query schema -struct +struct -- !query output -2008-12-25 07:30:00 1931-01-07 00:30:00 NULL +2008-12-25 07:30:00.123123 1931-01-07 00:29:59.876877 NULL -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out index c120db7fa2b7..58c70ec9e33d 100755 --- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out @@ -11,19 +11,19 @@ struct +struct -- !query output -2008-12-25 07:30:00 1931-01-07 00:30:00 NULL +2008-12-25 07:30:00.123 1931-01-07 00:29:59.877 NULL -- !query -select TIMESTAMP_MICROSECONDS(1230219000000000),TIMESTAMP_MICROSECONDS(-1230219000000000),TIMESTAMP_MICROSECONDS(null) +select TIMESTAMP_MICROSECONDS(1230219000123123),TIMESTAMP_MICROSECONDS(-1230219000123123),TIMESTAMP_MICROSECONDS(null) -- !query schema -struct +struct -- !query output -2008-12-25 07:30:00 1931-01-07 00:30:00 NULL +2008-12-25 07:30:00.123123 1931-01-07 00:29:59.876877 NULL -- !query diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 788a07370195..d9eaef49585c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -3495,6 +3495,36 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark assert(df4.schema.head.name === "randn(1)") checkIfSeedExistsInExplain(df2) } + + test("SPARK-31710: " + + "TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS and TIMESTAMP_MICROSECONDS to timestamp transfer") { + val df1 = sql("select TIMESTAMP_SECONDS(1230219000)," + + "TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)") + checkAnswer(df1, Row(Timestamp.valueOf("2008-12-25 07:30:00"), + Timestamp.valueOf("1931-01-07 00:30:00"), null)) + + val df2 = sql("select TIMESTAMP_MILLISECONDS(1230219000123)," + + "TIMESTAMP_MILLISECONDS(-1230219000123),TIMESTAMP_MILLISECONDS(null)") + checkAnswer(df2, Row(Timestamp.valueOf("2008-12-25 07:30:00.123"), + Timestamp.valueOf("1931-01-07 00:29:59.877"), null)) + + val df3 = sql("select TIMESTAMP_MICROSECONDS(1230219000123123)," + + "TIMESTAMP_MICROSECONDS(-1230219000123123),TIMESTAMP_MICROSECONDS(null)") + checkAnswer(df3, Row(Timestamp.valueOf("2008-12-25 07:30:00.123123"), + Timestamp.valueOf("1931-01-07 00:29:59.876877"), null)) + + intercept[ArithmeticException]( + sql("select TIMESTAMP_SECONDS(1230219000123123)," + + "TIMESTAMP_SECONDS(-1230219000123123),TIMESTAMP_SECONDS(null)").show + ) + + intercept[ArithmeticException]( + sql("select TIMESTAMP_MILLISECONDS(92233720368547758)," + + "TIMESTAMP_MILLISECONDS(-92233720368547758),TIMESTAMP_MILLISECONDS(null)").show + ) + + } + } case class Foo(bar: Option[String]) From 17a33f52392af8989df9521b665de005885a70a3 Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Wed, 20 May 2020 17:49:22 +0800 Subject: [PATCH 06/18] long overflow in datetime.sql added and pass the local test --- sql/core/src/test/resources/sql-tests/inputs/datetime.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql index 9ae02203179c..5bb156483ecd 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql @@ -4,6 +4,9 @@ select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null); select TIMESTAMP_MILLISECONDS(1230219000123),TIMESTAMP_MILLISECONDS(-1230219000123),TIMESTAMP_MILLISECONDS(null); select TIMESTAMP_MICROSECONDS(1230219000123123),TIMESTAMP_MICROSECONDS(-1230219000123123),TIMESTAMP_MICROSECONDS(null); +-- overflow exception: +select TIMESTAMP_SECONDS(1230219000123123),TIMESTAMP_SECONDS(-1230219000123123),TIMESTAMP_SECONDS(null); +select TIMESTAMP_MILLISECONDS(92233720368547758),TIMESTAMP_MILLISECONDS(-92233720368547758),TIMESTAMP_MILLISECONDS(null); -- [SPARK-16836] current_date and current_timestamp literals select current_date = current_date(), current_timestamp = current_timestamp(); From 344e5e101aedf5176e97739d3a99298892adaffe Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Wed, 20 May 2020 18:04:33 +0800 Subject: [PATCH 07/18] test overflow condition fix, just small unit for per test --- .../resources/sql-tests/inputs/datetime.sql | 6 +++-- .../sql-tests/results/ansi/datetime.sql.out | 27 +------------------ .../sql-tests/results/datetime.sql.out | 25 ----------------- .../org/apache/spark/sql/SQLQuerySuite.scala | 14 +++++++--- 4 files changed, 15 insertions(+), 57 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql index 5bb156483ecd..0fe9bf8bd20a 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql @@ -5,8 +5,10 @@ select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SE select TIMESTAMP_MILLISECONDS(1230219000123),TIMESTAMP_MILLISECONDS(-1230219000123),TIMESTAMP_MILLISECONDS(null); select TIMESTAMP_MICROSECONDS(1230219000123123),TIMESTAMP_MICROSECONDS(-1230219000123123),TIMESTAMP_MICROSECONDS(null); -- overflow exception: -select TIMESTAMP_SECONDS(1230219000123123),TIMESTAMP_SECONDS(-1230219000123123),TIMESTAMP_SECONDS(null); -select TIMESTAMP_MILLISECONDS(92233720368547758),TIMESTAMP_MILLISECONDS(-92233720368547758),TIMESTAMP_MILLISECONDS(null); +select TIMESTAMP_SECONDS(1230219000123123); +select TIMESTAMP_SECONDS(-1230219000123123); +select TIMESTAMP_MILLISECONDS(92233720368547758); +select TIMESTAMP_MILLISECONDS(-92233720368547758); -- [SPARK-16836] current_date and current_timestamp literals select current_date = current_date(), current_timestamp = current_timestamp(); diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out index 94a20c5a697a..828f714b8870 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out @@ -1,30 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 91 - - --- !query -select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null) --- !query schema -struct --- !query output -2008-12-25 07:30:00 1931-01-07 00:30:00 NULL - - --- !query -select TIMESTAMP_MILLISECONDS(1230219000123),TIMESTAMP_MILLISECONDS(-1230219000123),TIMESTAMP_MILLISECONDS(null) --- !query schema -struct --- !query output -2008-12-25 07:30:00.123 1931-01-07 00:29:59.877 NULL - - --- !query -select TIMESTAMP_MICROSECONDS(1230219000123123),TIMESTAMP_MICROSECONDS(-1230219000123123),TIMESTAMP_MICROSECONDS(null) --- !query schema -struct --- !query output -2008-12-25 07:30:00.123123 1931-01-07 00:29:59.876877 NULL - +-- Number of queries: 92 -- !query select current_date = current_date(), current_timestamp = current_timestamp() diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out index 58c70ec9e33d..2441830c8e50 100755 --- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out @@ -2,31 +2,6 @@ -- Number of queries: 91 --- !query -select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null) --- !query schema -struct --- !query output -2008-12-25 07:30:00 1931-01-07 00:30:00 NULL - - --- !query -select TIMESTAMP_MILLISECONDS(1230219000123),TIMESTAMP_MILLISECONDS(-1230219000123),TIMESTAMP_MILLISECONDS(null) --- !query schema -struct --- !query output -2008-12-25 07:30:00.123 1931-01-07 00:29:59.877 NULL - - --- !query -select TIMESTAMP_MICROSECONDS(1230219000123123),TIMESTAMP_MICROSECONDS(-1230219000123123),TIMESTAMP_MICROSECONDS(null) --- !query schema -struct --- !query output -2008-12-25 07:30:00.123123 1931-01-07 00:29:59.876877 NULL - - --- !query select current_date = current_date(), current_timestamp = current_timestamp() -- !query schema struct<(current_date() = current_date()):boolean,(current_timestamp() = current_timestamp()):boolean> diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index d9eaef49585c..25afa27865ef 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -3514,13 +3514,19 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark Timestamp.valueOf("1931-01-07 00:29:59.876877"), null)) intercept[ArithmeticException]( - sql("select TIMESTAMP_SECONDS(1230219000123123)," + - "TIMESTAMP_SECONDS(-1230219000123123),TIMESTAMP_SECONDS(null)").show + sql("select TIMESTAMP_SECONDS(1230219000123123)").show ) intercept[ArithmeticException]( - sql("select TIMESTAMP_MILLISECONDS(92233720368547758)," + - "TIMESTAMP_MILLISECONDS(-92233720368547758),TIMESTAMP_MILLISECONDS(null)").show + sql("select TIMESTAMP_SECONDS(-1230219000123123)").show + ) + + intercept[ArithmeticException]( + sql("select TIMESTAMP_MILLISECONDS(92233720368547758)").show + ) + + intercept[ArithmeticException]( + sql("select TIMESTAMP_MILLISECONDS(-92233720368547758)").show ) } From 407dcfc0cb15b0d36ec197ab67ba1272b42259ca Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Wed, 20 May 2020 19:37:50 +0800 Subject: [PATCH 08/18] fix unit test failed due to string value in ExpressionDescription --- .../sql/catalyst/expressions/datetimeExpressions.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 87e6ff783750..2ccf9a8fed5a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -425,7 +425,7 @@ abstract class NumberToTimestampBase extends UnaryExpression examples = """ Examples: > SELECT _FUNC_(1230219000); - "2008-12-25 07:30:00" + 2008-12-25 07:30:00 """, group = "datetime_funcs", since = "3.1.0") @@ -443,7 +443,7 @@ case class SecondsToTimestamp(child: Expression) examples = """ Examples: > SELECT _FUNC_(1230219000123); - "2008-12-25 07:30:00.123" + 2008-12-25 07:30:00.123 """, group = "datetime_funcs", since = "3.1.0") @@ -461,7 +461,7 @@ case class MilliSecondsToTimestamp(child: Expression) examples = """ Examples: > SELECT _FUNC_(1230219000123123); - "2008-12-25 07:30:00.123123" + 2008-12-25 07:30:00.123123 """, group = "datetime_funcs", since = "3.1.0") From 944c6441fe292a78e46ab0f9cf56b4eb93eea2db Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Wed, 20 May 2020 20:05:00 +0800 Subject: [PATCH 09/18] regenerate sql-expression-schema.md --- .../test/resources/sql-functions/sql-expression-schema.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md index 3570fb61e288..f43bb010c257 100644 --- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md +++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md @@ -1,6 +1,6 @@ ## Summary - - Number of queries: 333 + - Number of queries: 336 - Number of expressions that missing example: 34 - Expressions missing examples: and,string,tinyint,double,smallint,date,decimal,boolean,float,binary,bigint,int,timestamp,cume_dist,dense_rank,input_file_block_length,input_file_block_start,input_file_name,lag,lead,monotonically_increasing_id,ntile,struct,!,not,or,percent_rank,rank,row_number,spark_partition_id,version,window,positive,count_min_sketch ## Schema of Built-in Functions @@ -175,6 +175,8 @@ | org.apache.spark.sql.catalyst.expressions.MapValues | map_values | SELECT map_values(map(1, 'a', 2, 'b')) | struct> | | org.apache.spark.sql.catalyst.expressions.MapZipWith | map_zip_with | SELECT map_zip_with(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2)) | struct> | | org.apache.spark.sql.catalyst.expressions.Md5 | md5 | SELECT md5('Spark') | struct | +| org.apache.spark.sql.catalyst.expressions.MicroSecondsToTimestamp | timestamp_microseconds | SELECT timestamp_microseconds(1230219000123123) | struct | +| org.apache.spark.sql.catalyst.expressions.MilliSecondsToTimestamp | timestamp_milliseconds | SELECT timestamp_milliseconds(1230219000123) | struct | | org.apache.spark.sql.catalyst.expressions.Minute | minute | SELECT minute('2009-07-30 12:58:59') | struct | | org.apache.spark.sql.catalyst.expressions.MonotonicallyIncreasingID | monotonically_increasing_id | N/A | N/A | | org.apache.spark.sql.catalyst.expressions.Month | month | SELECT month('2016-07-30') | struct | @@ -223,6 +225,7 @@ | org.apache.spark.sql.catalyst.expressions.SchemaOfCsv | schema_of_csv | SELECT schema_of_csv('1,abc') | struct | | org.apache.spark.sql.catalyst.expressions.SchemaOfJson | schema_of_json | SELECT schema_of_json('[{"col":0}]') | struct | | org.apache.spark.sql.catalyst.expressions.Second | second | SELECT second('2009-07-30 12:58:59') | struct | +| org.apache.spark.sql.catalyst.expressions.SecondsToTimestamp | timestamp_seconds | SELECT timestamp_seconds(1230219000) | struct | | org.apache.spark.sql.catalyst.expressions.Sentences | sentences | SELECT sentences('Hi there! Good morning.') | struct>> | | org.apache.spark.sql.catalyst.expressions.Sequence | sequence | SELECT sequence(1, 5) | struct> | | org.apache.spark.sql.catalyst.expressions.Sha1 | sha1 | SELECT sha1('Spark') | struct | From 642668ca5a537a21f422bbace57f66ff42fd9c26 Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Wed, 20 May 2020 20:21:00 +0800 Subject: [PATCH 10/18] minimize code gen by judge equal 1 and remove duplicate unit test --- .../expressions/datetimeExpressions.scala | 7 ++-- .../org/apache/spark/sql/SQLQuerySuite.scala | 36 ------------------- 2 files changed, 5 insertions(+), 38 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 2ccf9a8fed5a..0cec07d302f3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -415,8 +415,11 @@ abstract class NumberToTimestampBase extends UnaryExpression } override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - defineCodeGen(ctx, ev, _ => s"java.lang.Math.multiplyExact(" + - s"${ev.value}, $upScaleFactor)") + if (upScaleFactor == 1) { + defineCodeGen(ctx, ev, c => c) + } else { + defineCodeGen(ctx, ev, c => s"java.lang.Math.multiplyExact($c, $upScaleFactor)") + } } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 25afa27865ef..788a07370195 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -3495,42 +3495,6 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark assert(df4.schema.head.name === "randn(1)") checkIfSeedExistsInExplain(df2) } - - test("SPARK-31710: " + - "TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS and TIMESTAMP_MICROSECONDS to timestamp transfer") { - val df1 = sql("select TIMESTAMP_SECONDS(1230219000)," + - "TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)") - checkAnswer(df1, Row(Timestamp.valueOf("2008-12-25 07:30:00"), - Timestamp.valueOf("1931-01-07 00:30:00"), null)) - - val df2 = sql("select TIMESTAMP_MILLISECONDS(1230219000123)," + - "TIMESTAMP_MILLISECONDS(-1230219000123),TIMESTAMP_MILLISECONDS(null)") - checkAnswer(df2, Row(Timestamp.valueOf("2008-12-25 07:30:00.123"), - Timestamp.valueOf("1931-01-07 00:29:59.877"), null)) - - val df3 = sql("select TIMESTAMP_MICROSECONDS(1230219000123123)," + - "TIMESTAMP_MICROSECONDS(-1230219000123123),TIMESTAMP_MICROSECONDS(null)") - checkAnswer(df3, Row(Timestamp.valueOf("2008-12-25 07:30:00.123123"), - Timestamp.valueOf("1931-01-07 00:29:59.876877"), null)) - - intercept[ArithmeticException]( - sql("select TIMESTAMP_SECONDS(1230219000123123)").show - ) - - intercept[ArithmeticException]( - sql("select TIMESTAMP_SECONDS(-1230219000123123)").show - ) - - intercept[ArithmeticException]( - sql("select TIMESTAMP_MILLISECONDS(92233720368547758)").show - ) - - intercept[ArithmeticException]( - sql("select TIMESTAMP_MILLISECONDS(-92233720368547758)").show - ) - - } - } case class Foo(bar: Option[String]) From 62bf8b51701cc4287604f146f0ec7688d04876eb Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Thu, 21 May 2020 12:35:25 +0800 Subject: [PATCH 11/18] correct style of usage content --- .../sql/catalyst/expressions/datetimeExpressions.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 0cec07d302f3..a986e5737f09 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -440,9 +440,9 @@ case class SecondsToTimestamp(child: Expression) override def prettyName: String = "timestamp_seconds" } +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(milliseconds) - " + - "Creates timestamp from the number of milliseconds since UTC epoch.", + usage = "_FUNC_(milliseconds) - Creates timestamp from the number of milliseconds since UTC epoch.", examples = """ Examples: > SELECT _FUNC_(1230219000123); @@ -450,6 +450,7 @@ case class SecondsToTimestamp(child: Expression) """, group = "datetime_funcs", since = "3.1.0") +// scalastyle:on line.size.limit case class MilliSecondsToTimestamp(child: Expression) extends NumberToTimestampBase { @@ -458,9 +459,9 @@ case class MilliSecondsToTimestamp(child: Expression) override def prettyName: String = "timestamp_milliseconds" } +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(microseconds) - " + - "Creates timestamp from the number of microseconds since UTC epoch.", + usage = "_FUNC_(microseconds) - Creates timestamp from the number of microseconds since UTC epoch.", examples = """ Examples: > SELECT _FUNC_(1230219000123123); @@ -468,6 +469,7 @@ case class MilliSecondsToTimestamp(child: Expression) """, group = "datetime_funcs", since = "3.1.0") +// scalastyle:on line.size.limit case class MicroSecondsToTimestamp(child: Expression) extends NumberToTimestampBase { From 7da94b52a73e365d5478467d8f1e83a59d0ddabb Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Thu, 21 May 2020 13:34:54 +0800 Subject: [PATCH 12/18] short name to MillisToTimestamp and MicrosToTimestamp --- .../apache/spark/sql/catalyst/analysis/FunctionRegistry.scala | 4 ++-- .../spark/sql/catalyst/expressions/datetimeExpressions.scala | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index 22bb51667928..9f65ef1337f4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -425,8 +425,8 @@ object FunctionRegistry { expression[DatePart]("date_part"), expression[Extract]("extract"), expression[SecondsToTimestamp]("timestamp_seconds"), - expression[MilliSecondsToTimestamp]("timestamp_milliseconds"), - expression[MicroSecondsToTimestamp]("timestamp_microseconds"), + expression[MillisToTimestamp]("timestamp_milliseconds"), + expression[MicrosToTimestamp]("timestamp_microseconds"), // collection functions expression[CreateArray]("array"), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index a986e5737f09..dcd7ddb55bf7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -451,7 +451,7 @@ case class SecondsToTimestamp(child: Expression) group = "datetime_funcs", since = "3.1.0") // scalastyle:on line.size.limit -case class MilliSecondsToTimestamp(child: Expression) +case class MillisToTimestamp(child: Expression) extends NumberToTimestampBase { override def upScaleFactor: Long = MICROS_PER_MILLIS @@ -470,7 +470,7 @@ case class MilliSecondsToTimestamp(child: Expression) group = "datetime_funcs", since = "3.1.0") // scalastyle:on line.size.limit -case class MicroSecondsToTimestamp(child: Expression) +case class MicrosToTimestamp(child: Expression) extends NumberToTimestampBase { override def upScaleFactor: Long = 1L From 3704fee9ef9f77e8216bacbed734ab4ef0f660e1 Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Thu, 21 May 2020 14:26:26 +0800 Subject: [PATCH 13/18] correct short name in sql-expression-schema.md --- .../src/test/resources/sql-functions/sql-expression-schema.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md index f43bb010c257..fd2d608be302 100644 --- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md +++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md @@ -175,8 +175,8 @@ | org.apache.spark.sql.catalyst.expressions.MapValues | map_values | SELECT map_values(map(1, 'a', 2, 'b')) | struct> | | org.apache.spark.sql.catalyst.expressions.MapZipWith | map_zip_with | SELECT map_zip_with(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2)) | struct> | | org.apache.spark.sql.catalyst.expressions.Md5 | md5 | SELECT md5('Spark') | struct | -| org.apache.spark.sql.catalyst.expressions.MicroSecondsToTimestamp | timestamp_microseconds | SELECT timestamp_microseconds(1230219000123123) | struct | -| org.apache.spark.sql.catalyst.expressions.MilliSecondsToTimestamp | timestamp_milliseconds | SELECT timestamp_milliseconds(1230219000123) | struct | +| org.apache.spark.sql.catalyst.expressions.MicrosToTimestamp | timestamp_microseconds | SELECT timestamp_microseconds(1230219000123123) | struct | +| org.apache.spark.sql.catalyst.expressions.MillisToTimestamp | timestamp_milliseconds | SELECT timestamp_milliseconds(1230219000123) | struct | | org.apache.spark.sql.catalyst.expressions.Minute | minute | SELECT minute('2009-07-30 12:58:59') | struct | | org.apache.spark.sql.catalyst.expressions.MonotonicallyIncreasingID | monotonically_increasing_id | N/A | N/A | | org.apache.spark.sql.catalyst.expressions.Month | month | SELECT month('2016-07-30') | struct | From f5b0a5a2eb48c28d9c9cdbf98116357e5c0b5886 Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Thu, 21 May 2020 15:21:14 +0800 Subject: [PATCH 14/18] function rename timestamp_microseconds to timestamp_micros and timestamp_milliseconds to timestamp_millis --- .../spark/sql/catalyst/analysis/FunctionRegistry.scala | 4 ++-- .../sql/catalyst/expressions/datetimeExpressions.scala | 4 ++-- .../test/resources/sql-functions/sql-expression-schema.md | 4 ++-- sql/core/src/test/resources/sql-tests/inputs/datetime.sql | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index 9f65ef1337f4..5e53927885ca 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -425,8 +425,8 @@ object FunctionRegistry { expression[DatePart]("date_part"), expression[Extract]("extract"), expression[SecondsToTimestamp]("timestamp_seconds"), - expression[MillisToTimestamp]("timestamp_milliseconds"), - expression[MicrosToTimestamp]("timestamp_microseconds"), + expression[MillisToTimestamp]("timestamp_millis"), + expression[MicrosToTimestamp]("timestamp_micros"), // collection functions expression[CreateArray]("array"), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index dcd7ddb55bf7..448a354b2eb3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -456,7 +456,7 @@ case class MillisToTimestamp(child: Expression) override def upScaleFactor: Long = MICROS_PER_MILLIS - override def prettyName: String = "timestamp_milliseconds" + override def prettyName: String = "timestamp_millis" } // scalastyle:off line.size.limit @@ -475,7 +475,7 @@ case class MicrosToTimestamp(child: Expression) override def upScaleFactor: Long = 1L - override def prettyName: String = "timestamp_microseconds" + override def prettyName: String = "timestamp_micros" } @ExpressionDescription( diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md index fd2d608be302..6b57e6b95522 100644 --- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md +++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md @@ -175,8 +175,8 @@ | org.apache.spark.sql.catalyst.expressions.MapValues | map_values | SELECT map_values(map(1, 'a', 2, 'b')) | struct> | | org.apache.spark.sql.catalyst.expressions.MapZipWith | map_zip_with | SELECT map_zip_with(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2)) | struct> | | org.apache.spark.sql.catalyst.expressions.Md5 | md5 | SELECT md5('Spark') | struct | -| org.apache.spark.sql.catalyst.expressions.MicrosToTimestamp | timestamp_microseconds | SELECT timestamp_microseconds(1230219000123123) | struct | -| org.apache.spark.sql.catalyst.expressions.MillisToTimestamp | timestamp_milliseconds | SELECT timestamp_milliseconds(1230219000123) | struct | +| org.apache.spark.sql.catalyst.expressions.MicrosToTimestamp | timestamp_micros | SELECT timestamp_micros(1230219000123123) | struct | +| org.apache.spark.sql.catalyst.expressions.MillisToTimestamp | timestamp_millis | SELECT timestamp_millis(1230219000123) | struct | | org.apache.spark.sql.catalyst.expressions.Minute | minute | SELECT minute('2009-07-30 12:58:59') | struct | | org.apache.spark.sql.catalyst.expressions.MonotonicallyIncreasingID | monotonically_increasing_id | N/A | N/A | | org.apache.spark.sql.catalyst.expressions.Month | month | SELECT month('2016-07-30') | struct | diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql index 0fe9bf8bd20a..0fb373f419e7 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql @@ -2,13 +2,13 @@ -- [SPARK-31710] TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS and TIMESTAMP_MICROSECONDS to timestamp transfer select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null); -select TIMESTAMP_MILLISECONDS(1230219000123),TIMESTAMP_MILLISECONDS(-1230219000123),TIMESTAMP_MILLISECONDS(null); -select TIMESTAMP_MICROSECONDS(1230219000123123),TIMESTAMP_MICROSECONDS(-1230219000123123),TIMESTAMP_MICROSECONDS(null); +select TIMESTAMP_MILLIS(1230219000123),TIMESTAMP_MILLIS(-1230219000123),TIMESTAMP_MILLIS(null); +select TIMESTAMP_MICROS(1230219000123123),TIMESTAMP_MICROS(-1230219000123123),TIMESTAMP_MICROS(null); -- overflow exception: select TIMESTAMP_SECONDS(1230219000123123); select TIMESTAMP_SECONDS(-1230219000123123); -select TIMESTAMP_MILLISECONDS(92233720368547758); -select TIMESTAMP_MILLISECONDS(-92233720368547758); +select TIMESTAMP_MILLIS(92233720368547758); +select TIMESTAMP_MILLIS(-92233720368547758); -- [SPARK-16836] current_date and current_timestamp literals select current_date = current_date(), current_timestamp = current_timestamp(); From f4f7c35db8fdfb5568e13fc102b5555bcd689937 Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Thu, 21 May 2020 16:16:35 +0800 Subject: [PATCH 15/18] disable cast string to long in number to timestamp --- .../sql/catalyst/expressions/datetimeExpressions.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 448a354b2eb3..58eacec0cd49 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -402,16 +402,16 @@ case class DayOfYear(child: Expression) extends UnaryExpression with ImplicitCas } abstract class NumberToTimestampBase extends UnaryExpression - with ImplicitCastInputTypes { + with ExpectsInputTypes { protected def upScaleFactor: Long - override def inputTypes: Seq[AbstractDataType] = Seq(LongType) + override def inputTypes: Seq[AbstractDataType] = Seq(IntegralType) override def dataType: DataType = TimestampType override def nullSafeEval(input: Any): Any = { - Math.multiplyExact(input.asInstanceOf[Long], upScaleFactor) + Math.multiplyExact(input.asInstanceOf[Number].longValue(), upScaleFactor) } override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { From 3c949acc2e0012ff57ea9d1b055fa861bf18bfad Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Fri, 22 May 2020 14:02:28 +0800 Subject: [PATCH 16/18] regenerate sql-expression-schema.md, UT added in DateExpressionsSuite --- .../expressions/datetimeExpressions.scala | 2 +- .../expressions/DateExpressionsSuite.scala | 21 +++++++ .../sql-functions/sql-expression-schema.md | 2 +- .../sql-tests/results/ansi/datetime.sql.out | 61 +++++++++++++++++++ .../sql-tests/results/datetime.sql.out | 61 +++++++++++++++++++ 5 files changed, 145 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 58eacec0cd49..afc57aa546fe 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -418,7 +418,7 @@ abstract class NumberToTimestampBase extends UnaryExpression if (upScaleFactor == 1) { defineCodeGen(ctx, ev, c => c) } else { - defineCodeGen(ctx, ev, c => s"java.lang.Math.multiplyExact($c, $upScaleFactor)") + defineCodeGen(ctx, ev, c => s"java.lang.Math.multiplyExact($c, ${upScaleFactor}L)") } } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 6e8397d12da7..7a311b9fa7d0 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -1146,4 +1146,25 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { Literal("yyyy-MM-dd'T'HH:mm:ss.SSSz")), "Fail to parse") } } + + test("[SPARK-31710][SQL] Adds TIMESTAMP_SECONDS, " + + "TIMESTAMP_MILLIS and TIMESTAMP_MICROS functions") { + checkEvaluation(SecondsToTimestamp(Literal(1230219000)), 1230219000000000L) + checkEvaluation(SecondsToTimestamp(Literal(-1230219000)), -1230219000000000L) + checkEvaluation(SecondsToTimestamp(Literal.create(null, IntegerType)), null) + checkEvaluation(MillisToTimestamp(Literal(1230219000123L)), 1230219000123000L) + checkEvaluation(MillisToTimestamp(Literal(-1230219000123L)), -1230219000123000L) + checkEvaluation(MillisToTimestamp(Literal.create(null, IntegerType)), null) + checkEvaluation(MicrosToTimestamp(Literal(1230219000123123L)), 1230219000123123L) + checkEvaluation(MicrosToTimestamp(Literal(-1230219000123123L)), -1230219000123123L) + checkEvaluation(MicrosToTimestamp(Literal.create(null, IntegerType)), null) + checkExceptionInExpression[ArithmeticException]( + SecondsToTimestamp(Literal(1230219000123123L)), "long overflow") + checkExceptionInExpression[ArithmeticException]( + SecondsToTimestamp(Literal(-1230219000123123L)), "long overflow") + checkExceptionInExpression[ArithmeticException]( + MillisToTimestamp(Literal(92233720368547758L)), "long overflow") + checkExceptionInExpression[ArithmeticException]( + MillisToTimestamp(Literal(-92233720368547758L)), "long overflow") + } } diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md index 6b57e6b95522..0efc2a6e0c02 100644 --- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md +++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md @@ -225,7 +225,7 @@ | org.apache.spark.sql.catalyst.expressions.SchemaOfCsv | schema_of_csv | SELECT schema_of_csv('1,abc') | struct | | org.apache.spark.sql.catalyst.expressions.SchemaOfJson | schema_of_json | SELECT schema_of_json('[{"col":0}]') | struct | | org.apache.spark.sql.catalyst.expressions.Second | second | SELECT second('2009-07-30 12:58:59') | struct | -| org.apache.spark.sql.catalyst.expressions.SecondsToTimestamp | timestamp_seconds | SELECT timestamp_seconds(1230219000) | struct | +| org.apache.spark.sql.catalyst.expressions.SecondsToTimestamp | timestamp_seconds | SELECT timestamp_seconds(1230219000) | struct | | org.apache.spark.sql.catalyst.expressions.Sentences | sentences | SELECT sentences('Hi there! Good morning.') | struct>> | | org.apache.spark.sql.catalyst.expressions.Sequence | sequence | SELECT sequence(1, 5) | struct> | | org.apache.spark.sql.catalyst.expressions.Sha1 | sha1 | SELECT sha1('Spark') | struct | diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out index 828f714b8870..2e61cb8cb8c3 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out @@ -1,6 +1,67 @@ -- Automatically generated by SQLQueryTestSuite -- Number of queries: 92 + +-- !query +select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null) +-- !query schema +struct +-- !query output +2008-12-25 07:30:00 1931-01-07 00:30:00 NULL + + +-- !query +select TIMESTAMP_MILLIS(1230219000123),TIMESTAMP_MILLIS(-1230219000123),TIMESTAMP_MILLIS(null) +-- !query schema +struct +-- !query output +2008-12-25 07:30:00.123 1931-01-07 00:29:59.877 NULL + + +-- !query +select TIMESTAMP_MICROS(1230219000123123),TIMESTAMP_MICROS(-1230219000123123),TIMESTAMP_MICROS(null) +-- !query schema +struct +-- !query output +2008-12-25 07:30:00.123123 1931-01-07 00:29:59.876877 NULL + + +-- !query +select TIMESTAMP_SECONDS(1230219000123123) +-- !query schema +struct<> +-- !query output +java.lang.ArithmeticException +long overflow + + +-- !query +select TIMESTAMP_SECONDS(-1230219000123123) +-- !query schema +struct<> +-- !query output +java.lang.ArithmeticException +long overflow + + +-- !query +select TIMESTAMP_MILLIS(92233720368547758) +-- !query schema +struct<> +-- !query output +java.lang.ArithmeticException +long overflow + + +-- !query +select TIMESTAMP_MILLIS(-92233720368547758) +-- !query schema +struct<> +-- !query output +java.lang.ArithmeticException +long overflow + + -- !query select current_date = current_date(), current_timestamp = current_timestamp() -- !query schema diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out index 2441830c8e50..4b879fcfbfc5 100755 --- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out @@ -2,6 +2,67 @@ -- Number of queries: 91 +-- !query +select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null) +-- !query schema +struct +-- !query output +2008-12-25 07:30:00 1931-01-07 00:30:00 NULL + + +-- !query +select TIMESTAMP_MILLIS(1230219000123),TIMESTAMP_MILLIS(-1230219000123),TIMESTAMP_MILLIS(null) +-- !query schema +struct +-- !query output +2008-12-25 07:30:00.123 1931-01-07 00:29:59.877 NULL + + +-- !query +select TIMESTAMP_MICROS(1230219000123123),TIMESTAMP_MICROS(-1230219000123123),TIMESTAMP_MICROS(null) +-- !query schema +struct +-- !query output +2008-12-25 07:30:00.123123 1931-01-07 00:29:59.876877 NULL + + +-- !query +select TIMESTAMP_SECONDS(1230219000123123) +-- !query schema +struct<> +-- !query output +java.lang.ArithmeticException +long overflow + + +-- !query +select TIMESTAMP_SECONDS(-1230219000123123) +-- !query schema +struct<> +-- !query output +java.lang.ArithmeticException +long overflow + + +-- !query +select TIMESTAMP_MILLIS(92233720368547758) +-- !query schema +struct<> +-- !query output +java.lang.ArithmeticException +long overflow + + +-- !query +select TIMESTAMP_MILLIS(-92233720368547758) +-- !query schema +struct<> +-- !query output +java.lang.ArithmeticException +long overflow + + +-- !query select current_date = current_date(), current_timestamp = current_timestamp() -- !query schema struct<(current_date() = current_date()):boolean,(current_timestamp() = current_timestamp()):boolean> From fdfd75c7ea78631520ad334ef72047e366a16a25 Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Fri, 22 May 2020 14:51:39 +0800 Subject: [PATCH 17/18] UT style improve --- .../expressions/DateExpressionsSuite.scala | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 7a311b9fa7d0..4848f6659d40 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -1147,17 +1147,17 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { } } - test("[SPARK-31710][SQL] Adds TIMESTAMP_SECONDS, " + - "TIMESTAMP_MILLIS and TIMESTAMP_MICROS functions") { - checkEvaluation(SecondsToTimestamp(Literal(1230219000)), 1230219000000000L) - checkEvaluation(SecondsToTimestamp(Literal(-1230219000)), -1230219000000000L) - checkEvaluation(SecondsToTimestamp(Literal.create(null, IntegerType)), null) - checkEvaluation(MillisToTimestamp(Literal(1230219000123L)), 1230219000123000L) - checkEvaluation(MillisToTimestamp(Literal(-1230219000123L)), -1230219000123000L) - checkEvaluation(MillisToTimestamp(Literal.create(null, IntegerType)), null) + test("[SPARK-31710][SQL] Adds TIMESTAMP_SECONDS,...") { + checkEvaluation(SecondsToTimestamp(Literal(1230219000)), 1230219000L * MICROS_PER_SECOND) + checkEvaluation(SecondsToTimestamp(Literal(-1230219000)), -1230219000L * MICROS_PER_SECOND) + checkEvaluation(SecondsToTimestamp(Literal(null, IntegerType)), null) + checkEvaluation(MillisToTimestamp(Literal(1230219000123L)), 1230219000123L * MICROS_PER_MILLIS) + checkEvaluation(MillisToTimestamp(Literal(-1230219000123L)), + -1230219000123L * MICROS_PER_MILLIS) + checkEvaluation(MillisToTimestamp(Literal(null, IntegerType)), null) checkEvaluation(MicrosToTimestamp(Literal(1230219000123123L)), 1230219000123123L) checkEvaluation(MicrosToTimestamp(Literal(-1230219000123123L)), -1230219000123123L) - checkEvaluation(MicrosToTimestamp(Literal.create(null, IntegerType)), null) + checkEvaluation(MicrosToTimestamp(Literal(null, IntegerType)), null) checkExceptionInExpression[ArithmeticException]( SecondsToTimestamp(Literal(1230219000123123L)), "long overflow") checkExceptionInExpression[ArithmeticException]( From f0cc63140f867699154f0b4cc88334c3af8aba45 Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Fri, 22 May 2020 15:04:27 +0800 Subject: [PATCH 18/18] UT style improve --- .../sql/catalyst/expressions/DateExpressionsSuite.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 4848f6659d40..87062f2d4ef3 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -1147,13 +1147,14 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { } } - test("[SPARK-31710][SQL] Adds TIMESTAMP_SECONDS,...") { + test("SPARK-31710:Adds TIMESTAMP_SECONDS, " + + "TIMESTAMP_MILLIS and TIMESTAMP_MICROS functions") { checkEvaluation(SecondsToTimestamp(Literal(1230219000)), 1230219000L * MICROS_PER_SECOND) checkEvaluation(SecondsToTimestamp(Literal(-1230219000)), -1230219000L * MICROS_PER_SECOND) checkEvaluation(SecondsToTimestamp(Literal(null, IntegerType)), null) checkEvaluation(MillisToTimestamp(Literal(1230219000123L)), 1230219000123L * MICROS_PER_MILLIS) - checkEvaluation(MillisToTimestamp(Literal(-1230219000123L)), - -1230219000123L * MICROS_PER_MILLIS) + checkEvaluation(MillisToTimestamp( + Literal(-1230219000123L)), -1230219000123L * MICROS_PER_MILLIS) checkEvaluation(MillisToTimestamp(Literal(null, IntegerType)), null) checkEvaluation(MicrosToTimestamp(Literal(1230219000123123L)), 1230219000123123L) checkEvaluation(MicrosToTimestamp(Literal(-1230219000123123L)), -1230219000123123L)