Skip to content

[SPARK-24673][SQL] scala sql function from_utc_timestamp second argument could be Column instead of String #21693

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

[SPARK-24673][SQL] scala sql function from_utc_timestamp second argument could be Column instead of String #21693

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8bec904
[SPARK-24673] Add overloaded functions
agilelab-tmnd1991 Jun 28, 2018
3a376bc
[SPARK-24673] Add tests to to/from_utc_timestamp and fix doc
agilelab-tmnd1991 Jul 2, 2018
d4ebc8f
[SPARK-24673] Update @since doc tags
tmnd1991 Jul 3, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions sql/core/src/main/scala/org/apache/spark/sql/functions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2934,6 +2934,17 @@ object functions {
FromUTCTimestamp(ts.expr, Literal(tz))
}

/**
* Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders
* that time as a timestamp in the given time zone. For example, 'GMT+1' would yield
* '2017-07-14 03:40:00.0'.
* @group datetime_funcs
* @since 2.4.0
*/
def from_utc_timestamp(ts: Column, tz: Column): Column = withExpr {
Copy link
Member

@HyukjinKwon HyukjinKwon Jul 5, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is being merged since tz column can differ from the constant tz string for each record which makes sense in usecase, and arguably it's a common function.

Copy link
Member

@gatorsmile gatorsmile Jul 5, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For these changes, we also need to add the corresponding changes in PySpark functions.

FromUTCTimestamp(ts.expr, tz.expr)
}

/**
* Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time
* zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield
Expand All @@ -2945,6 +2956,17 @@ object functions {
ToUTCTimestamp(ts.expr, Literal(tz))
}

/**
* Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time
* zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield
* '2017-07-14 01:40:00.0'.
* @group datetime_funcs
* @since 2.4.0
*/
def to_utc_timestamp(ts: Column, tz: Column): Column = withExpr {
ToUTCTimestamp(ts.expr, tz.expr)
}

/**
* Bucketize rows into one or more time windows given a timestamp specifying column. Window
* starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window
Expand Down
38 changes: 36 additions & 2 deletions sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -663,7 +663,7 @@ class DateFunctionsSuite extends QueryTest with SharedSQLContext {
checkAnswer(df.selectExpr("datediff(a, d)"), Seq(Row(1), Row(1)))
}

test("from_utc_timestamp") {
test("from_utc_timestamp with literal zone") {
val df = Seq(
(Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
(Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
Expand All @@ -680,7 +680,24 @@ class DateFunctionsSuite extends QueryTest with SharedSQLContext {
Row(Timestamp.valueOf("2015-07-24 17:00:00"))))
}

test("to_utc_timestamp") {
test("from_utc_timestamp with column zone") {
val df = Seq(
(Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "CET"),
(Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "PST")
).toDF("a", "b", "c")
checkAnswer(
df.select(from_utc_timestamp(col("a"), col("c"))),
Seq(
Row(Timestamp.valueOf("2015-07-24 02:00:00")),
Row(Timestamp.valueOf("2015-07-24 17:00:00"))))
checkAnswer(
df.select(from_utc_timestamp(col("b"), col("c"))),
Seq(
Row(Timestamp.valueOf("2015-07-24 02:00:00")),
Row(Timestamp.valueOf("2015-07-24 17:00:00"))))
}

test("to_utc_timestamp with literal zone") {
val df = Seq(
(Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
(Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
Expand All @@ -697,6 +714,23 @@ class DateFunctionsSuite extends QueryTest with SharedSQLContext {
Row(Timestamp.valueOf("2015-07-25 07:00:00"))))
}

test("to_utc_timestamp with column zone") {
val df = Seq(
(Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "PST"),
(Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "CET")
).toDF("a", "b", "c")
checkAnswer(
df.select(to_utc_timestamp(col("a"), col("c"))),
Seq(
Row(Timestamp.valueOf("2015-07-24 07:00:00")),
Row(Timestamp.valueOf("2015-07-24 22:00:00"))))
checkAnswer(
df.select(to_utc_timestamp(col("b"), col("c"))),
Seq(
Row(Timestamp.valueOf("2015-07-24 07:00:00")),
Row(Timestamp.valueOf("2015-07-24 22:00:00"))))
}

test("SPARK-23715: to/from_utc_timestamp can retain the previous behavior") {
withSQLConf(SQLConf.REJECT_TIMEZONE_IN_STRING.key -> "false") {
checkAnswer(
Expand Down