From 4127a1cfae610ebd7087558f0b1e2b90da407ad4 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Tue, 6 Aug 2019 14:45:09 +0500 Subject: [PATCH 1/4] Support DayOfYear --- .../scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala | 2 ++ sql/core/src/test/resources/sql-tests/inputs/extract.sql | 2 ++ 2 files changed, 4 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index e727abd503de..c638a50a5962 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1408,6 +1408,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging DayOfMonth(expression(ctx.source)) case "DAYOFWEEK" => DayOfWeek(expression(ctx.source)) + case "DOY" => + DayOfYear(expression(ctx.source)) case "HOUR" => Hour(expression(ctx.source)) case "MINUTE" => diff --git a/sql/core/src/test/resources/sql-tests/inputs/extract.sql b/sql/core/src/test/resources/sql-tests/inputs/extract.sql index 9adf5d70056e..10e401eff843 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/extract.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/extract.sql @@ -12,6 +12,8 @@ select extract(day from c) from t; select extract(dayofweek from c) from t; +select extract(doy from c) from t; + select extract(hour from c) from t; select extract(minute from c) from t; From 15f57514559466203f2307e3a2ea91f8675d5066 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Tue, 6 Aug 2019 15:03:38 +0500 Subject: [PATCH 2/4] Support isodow --- .../expressions/datetimeExpressions.scala | 26 +++++++++++++++++++ .../sql/catalyst/parser/AstBuilder.scala | 2 ++ .../resources/sql-tests/inputs/extract.sql | 2 ++ 3 files changed, 30 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index e50abebe5798..fda2eb20af72 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -474,6 +474,32 @@ case class WeekDay(child: Expression) extends DayWeek { } } +// scalastyle:off line.size.limit +@ExpressionDescription( + usage = "_FUNC_(date) - Returns the day of the week for date/timestamp numbered according to ISO 8601 (1 = Monday, 2 = Tuesday, ..., 7 = Sunday).", + examples = """ + Examples: + > SELECT _FUNC_('2019-08-06'); + 2 + """, + since = "3.0.0") +// scalastyle:on line.size.limit +case class IsoDayOfWeek(child: Expression) extends DayWeek { + + override protected def nullSafeEval(date: Any): Any = { + val localDate = LocalDate.ofEpochDay(date.asInstanceOf[Int]) + localDate.getDayOfWeek.getValue + } + + override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + nullSafeCodeGen(ctx, ev, days => { + s""" + ${ev.value} = java.time.LocalDate.ofEpochDay($days).getDayOfWeek().getValue(); + """ + }) + } +} + abstract class DayWeek extends UnaryExpression with ImplicitCastInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(DateType) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index c638a50a5962..0847566cebff 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1408,6 +1408,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging DayOfMonth(expression(ctx.source)) case "DAYOFWEEK" => DayOfWeek(expression(ctx.source)) + case "ISODOW" => + IsoDayOfWeek(expression(ctx.source)) case "DOY" => DayOfYear(expression(ctx.source)) case "HOUR" => diff --git a/sql/core/src/test/resources/sql-tests/inputs/extract.sql b/sql/core/src/test/resources/sql-tests/inputs/extract.sql index 10e401eff843..a3aefbdbc1a8 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/extract.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/extract.sql @@ -12,6 +12,8 @@ select extract(day from c) from t; select extract(dayofweek from c) from t; +select extract(isodow from c) from t; + select extract(doy from c) from t; select extract(hour from c) from t; From f6a6ed57656e96c722461fd0aef21b40780a5a26 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Tue, 6 Aug 2019 15:32:33 +0500 Subject: [PATCH 3/4] Avoid IsoDayOfWeek --- .../expressions/datetimeExpressions.scala | 26 ------------------- .../sql/catalyst/parser/AstBuilder.scala | 4 ++- .../resources/sql-tests/inputs/extract.sql | 2 ++ 3 files changed, 5 insertions(+), 27 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index fda2eb20af72..e50abebe5798 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -474,32 +474,6 @@ case class WeekDay(child: Expression) extends DayWeek { } } -// scalastyle:off line.size.limit -@ExpressionDescription( - usage = "_FUNC_(date) - Returns the day of the week for date/timestamp numbered according to ISO 8601 (1 = Monday, 2 = Tuesday, ..., 7 = Sunday).", - examples = """ - Examples: - > SELECT _FUNC_('2019-08-06'); - 2 - """, - since = "3.0.0") -// scalastyle:on line.size.limit -case class IsoDayOfWeek(child: Expression) extends DayWeek { - - override protected def nullSafeEval(date: Any): Any = { - val localDate = LocalDate.ofEpochDay(date.asInstanceOf[Int]) - localDate.getDayOfWeek.getValue - } - - override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - nullSafeCodeGen(ctx, ev, days => { - s""" - ${ev.value} = java.time.LocalDate.ofEpochDay($days).getDayOfWeek().getValue(); - """ - }) - } -} - abstract class DayWeek extends UnaryExpression with ImplicitCastInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(DateType) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 0847566cebff..ef813d4fd157 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1408,8 +1408,10 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging DayOfMonth(expression(ctx.source)) case "DAYOFWEEK" => DayOfWeek(expression(ctx.source)) + case "DOW" => + Subtract(DayOfWeek(expression(ctx.source)), Literal(1)) case "ISODOW" => - IsoDayOfWeek(expression(ctx.source)) + Add(WeekDay(expression(ctx.source)), Literal(1)) case "DOY" => DayOfYear(expression(ctx.source)) case "HOUR" => diff --git a/sql/core/src/test/resources/sql-tests/inputs/extract.sql b/sql/core/src/test/resources/sql-tests/inputs/extract.sql index a3aefbdbc1a8..5dd3e6686f1c 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/extract.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/extract.sql @@ -12,6 +12,8 @@ select extract(day from c) from t; select extract(dayofweek from c) from t; +select extract(dow from c) from t; + select extract(isodow from c) from t; select extract(doy from c) from t; From ef3552beb2698f28fe4a68e2143c761380552e7a Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Tue, 6 Aug 2019 15:56:39 +0500 Subject: [PATCH 4/4] Re-gen extract.sql.out --- .../sql-tests/results/extract.sql.out | 48 ++++++++++++++----- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/extract.sql.out b/sql/core/src/test/resources/sql-tests/results/extract.sql.out index 160e4c7d7845..0ca7bdc09b01 100644 --- a/sql/core/src/test/resources/sql-tests/results/extract.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/extract.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 11 +-- Number of queries: 14 -- !query 0 @@ -59,34 +59,58 @@ struct -- !query 7 -select extract(hour from c) from t +select extract(dow from c) from t -- !query 7 schema -struct +struct<(dayofweek(CAST(c AS DATE)) - 1):int> -- !query 7 output -7 +5 -- !query 8 -select extract(minute from c) from t +select extract(isodow from c) from t -- !query 8 schema -struct +struct<(weekday(CAST(c AS DATE)) + 1):int> -- !query 8 output -8 +5 -- !query 9 -select extract(second from c) from t +select extract(doy from c) from t -- !query 9 schema -struct +struct -- !query 9 output -9 +126 -- !query 10 -select extract(not_supported from c) from t +select extract(hour from c) from t -- !query 10 schema -struct<> +struct -- !query 10 output +7 + + +-- !query 11 +select extract(minute from c) from t +-- !query 11 schema +struct +-- !query 11 output +8 + + +-- !query 12 +select extract(second from c) from t +-- !query 12 schema +struct +-- !query 12 output +9 + + +-- !query 13 +select extract(not_supported from c) from t +-- !query 13 schema +struct<> +-- !query 13 output org.apache.spark.sql.catalyst.parser.ParseException Literals of type 'NOT_SUPPORTED' are currently not supported.(line 1, pos 7)