Skip to content

Commit 56ded53

Browse files
adrian-wangadrian-wang
committed
rebase and address comments
1 parent 14a8b37 commit 56ded53

File tree

4 files changed

+52
-36
lines changed

4 files changed

+52
-36
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeFunctions.scala‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -276,16 +276,16 @@ case class DateFormatClass(left: Expression, right: Expression) extends BinaryEx
276276
* If the first parameter is a Date or Timestamp instead of String, we will ignore the
277277
* second parameter.
278278
*/
279-
case class UnixTimestamp(nodes: Seq[Expression])
279+
case class UnixTimestamp(left: Expression, right: Expression)
280280
extends BinaryExpression with ExpectsInputTypes {
281281

282-
override def children: Seq[Expression] = left :: right :: Nil
283-
override def left: Expression =
284-
if (childrenLength == 0) CurrentTimestamp() else nodes.head
285-
override def right: Expression =
286-
if (childrenLength > 1) nodes(1) else Literal("yyyy-MM-dd HH:mm:ss")
282+
def this(time: Expression) = {
283+
this(time, Literal("yyyy-MM-dd HH:mm:ss"))
284+
}
287285

288-
private lazy val childrenLength = nodes.length
286+
def this() = {
287+
this(CurrentTimestamp())
288+
}
289289

290290
override def inputTypes: Seq[AbstractDataType] =
291291
Seq(TypeCollection(StringType, DateType, TimestampType), StringType)
@@ -337,14 +337,12 @@ case class UnixTimestamp(nodes: Seq[Expression])
337337
* representing the timestamp of that moment in the current system time zone in the given
338338
* format. If the format is missing, using format like "1970-01-01 00:00:00".
339339
*/
340-
case class FromUnixTime(nodes: Seq[Expression])
340+
case class FromUnixTime(left: Expression, right: Expression)
341341
extends BinaryExpression with ImplicitCastInputTypes {
342342

343-
assert(nodes.length <= 2 && nodes.nonEmpty)
344-
override def children: Seq[Expression] = left :: right :: Nil
345-
override def left: Expression = nodes.head
346-
override def right: Expression =
347-
if (nodes.length > 1) nodes(1) else Literal("yyyy-MM-dd HH:mm:ss")
343+
def this(unix: Expression) = {
344+
this(unix, Literal("yyyy-MM-dd HH:mm:ss"))
345+
}
348346

349347
override def dataType: DataType = StringType
350348

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala‎

Lines changed: 29 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ import java.util.Calendar
2424
import org.apache.spark.SparkFunSuite
2525
import org.apache.spark.sql.catalyst.InternalRow
2626
import org.apache.spark.sql.catalyst.util.DateTimeUtils
27-
import org.apache.spark.sql.types.{StringType, TimestampType, DateType}
27+
import org.apache.spark.sql.types._
2828

2929
class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
3030

@@ -280,10 +280,17 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
280280
val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
281281
val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
282282
val sdf2 = new SimpleDateFormat(fmt2)
283-
checkEvaluation(FromUnixTime(Seq(Literal(0L))), sdf1.format(new Timestamp(0)))
284-
checkEvaluation(FromUnixTime(Seq(Literal(1000L))), sdf1.format(new Timestamp(1000000)))
285283
checkEvaluation(
286-
FromUnixTime(Seq(Literal(-1000L), Literal(fmt2))), sdf2.format(new Timestamp(-1000000)))
284+
FromUnixTime(Literal(0L), Literal("yyyy-MM-dd HH:mm:ss")), sdf1.format(new Timestamp(0)))
285+
checkEvaluation(FromUnixTime(
286+
Literal(1000L), Literal("yyyy-MM-dd HH:mm:ss")), sdf1.format(new Timestamp(1000000)))
287+
checkEvaluation(
288+
FromUnixTime(Literal(-1000L), Literal(fmt2)), sdf2.format(new Timestamp(-1000000)))
289+
checkEvaluation(
290+
FromUnixTime(Literal.create(null, LongType), Literal.create(null, StringType)), null)
291+
checkEvaluation(
292+
FromUnixTime(Literal.create(null, LongType), Literal("yyyy-MM-dd HH:mm:ss")), null)
293+
checkEvaluation(FromUnixTime(Literal(1000L), Literal.create(null, StringType)), null)
287294
}
288295

289296
test("unix_timestamp") {
@@ -293,20 +300,30 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
293300
val fmt3 = "yy-MM-dd"
294301
val sdf3 = new SimpleDateFormat(fmt3)
295302
val date1 = Date.valueOf("2015-07-24")
296-
checkEvaluation(UnixTimestamp(Seq(Literal(sdf1.format(new Timestamp(0))))), 0L)
297-
checkEvaluation(UnixTimestamp(Seq(Literal(sdf1.format(new Timestamp(1000000))))), 1000L)
298-
checkEvaluation(UnixTimestamp(Seq(Literal(new Timestamp(1000000)))), 1000L)
299303
checkEvaluation(
300-
UnixTimestamp(Seq(Literal(date1))),
304+
UnixTimestamp(Literal(sdf1.format(new Timestamp(0))), Literal("yyyy-MM-dd HH:mm:ss")), 0L)
305+
checkEvaluation(UnixTimestamp(
306+
Literal(sdf1.format(new Timestamp(1000000))), Literal("yyyy-MM-dd HH:mm:ss")), 1000L)
307+
checkEvaluation(
308+
UnixTimestamp(Literal(new Timestamp(1000000)), Literal("yyyy-MM-dd HH:mm:ss")), 1000L)
309+
checkEvaluation(
310+
UnixTimestamp(Literal(date1), Literal("yyyy-MM-dd HH:mm:ss")),
301311
DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1)) / 1000L)
302312
checkEvaluation(
303-
UnixTimestamp(Seq(Literal(sdf2.format(new Timestamp(-1000000))), Literal(fmt2))), -1000L)
313+
UnixTimestamp(Literal(sdf2.format(new Timestamp(-1000000))), Literal(fmt2)), -1000L)
304314
checkEvaluation(UnixTimestamp(
305-
Seq(Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3))),
315+
Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3)),
306316
DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24"))) / 1000L)
307-
val t1 = UnixTimestamp(Seq()).eval(InternalRow.empty).asInstanceOf[Long]
308-
val t2 = UnixTimestamp(Seq()).eval(InternalRow.empty).asInstanceOf[Long]
317+
val t1 = UnixTimestamp(
318+
CurrentTimestamp(), Literal("yyyy-MM-dd HH:mm:ss")).eval().asInstanceOf[Long]
319+
val t2 = UnixTimestamp(
320+
CurrentTimestamp(), Literal("yyyy-MM-dd HH:mm:ss")).eval().asInstanceOf[Long]
309321
assert(t2 - t1 <= 1)
322+
checkEvaluation(
323+
UnixTimestamp(Literal.create(null, DateType), Literal.create(null, StringType)), null)
324+
checkEvaluation(
325+
UnixTimestamp(Literal.create(null, DateType), Literal("yyyy-MM-dd HH:mm:ss")), null)
326+
checkEvaluation(UnixTimestamp(Literal(date1), Literal.create(null, StringType)), null)
310327
}
311328

312329
}

‎sql/core/src/main/scala/org/apache/spark/sql/functions.scala‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2096,29 +2096,29 @@ object functions {
20962096
* @group datetime_funcs
20972097
* @since 1.5.0
20982098
*/
2099-
def from_unixtime(ut: Column): Column = FromUnixTime(Seq(ut.expr))
2099+
def from_unixtime(ut: Column): Column = FromUnixTime(ut.expr, Literal("yyyy-MM-dd HH:mm:ss"))
21002100

21012101
/**
21022102
* Gets current Unix timestamp in seconds.
21032103
* @group datetime_funcs
21042104
* @since 1.5.0
21052105
*/
2106-
def from_unixtime(ut: Column, f: Column): Column = FromUnixTime(Seq(ut.expr, f.expr))
2106+
def from_unixtime(ut: Column, f: String): Column = FromUnixTime(ut.expr, Literal(f))
21072107

21082108
/**
21092109
* Gets current Unix timestamp in seconds.
21102110
* @group datetime_funcs
21112111
* @since 1.5.0
21122112
*/
2113-
def unix_timestamp(): Column = UnixTimestamp(Nil: Seq[Expression])
2113+
def unix_timestamp(): Column = UnixTimestamp(CurrentTimestamp(), Literal("yyyy-MM-dd HH:mm:ss"))
21142114

21152115
/**
21162116
* Converts time string in format yyyy-MM-dd HH:mm:ss to Unix timestamp (in seconds),
21172117
* using the default timezone and the default locale, return null if fail.
21182118
* @group datetime_funcs
21192119
* @since 1.5.0
21202120
*/
2121-
def unix_timestamp(s: Column): Column = UnixTimestamp(Seq(s.expr))
2121+
def unix_timestamp(s: Column): Column = UnixTimestamp(s.expr, Literal("yyyy-MM-dd HH:mm:ss"))
21222122

21232123
/**
21242124
* Convert time string with given pattern
@@ -2127,7 +2127,7 @@ object functions {
21272127
* @group datetime_funcs
21282128
* @since 1.5.0
21292129
*/
2130-
def unix_timestamp(s: Column, p: Column): Column = UnixTimestamp(Seq(s.expr, p.expr))
2130+
def unix_timestamp(s: Column, p: String): Column = UnixTimestamp(s.expr, Literal(p))
21312131

21322132
//////////////////////////////////////////////////////////////////////////////////////////////
21332133
// Collection functions

‎sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -208,18 +208,19 @@ class DateFunctionsSuite extends QueryTest {
208208

209209
test("from_unixtime") {
210210
val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
211-
val sdf2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS")
211+
val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
212+
val sdf2 = new SimpleDateFormat(fmt2)
212213
val fmt3 = "yy-MM-dd HH-mm-ss"
213214
val sdf3 = new SimpleDateFormat(fmt3)
214215
val df = Seq((1000, "yyyy-MM-dd HH:mm:ss.SSS"), (-1000, "yy-MM-dd HH-mm-ss")).toDF("a", "b")
215216
checkAnswer(
216217
df.select(from_unixtime(col("a"))),
217218
Seq(Row(sdf1.format(new Timestamp(1000000))), Row(sdf1.format(new Timestamp(-1000000)))))
218219
checkAnswer(
219-
df.select(from_unixtime(col("a"), col("b"))),
220-
Seq(Row(sdf2.format(new Timestamp(1000000))), Row(sdf3.format(new Timestamp(-1000000)))))
220+
df.select(from_unixtime(col("a"), fmt2)),
221+
Seq(Row(sdf2.format(new Timestamp(1000000))), Row(sdf2.format(new Timestamp(-1000000)))))
221222
checkAnswer(
222-
df.select(from_unixtime(col("a"), lit(fmt3))),
223+
df.select(from_unixtime(col("a"), fmt3)),
223224
Seq(Row(sdf3.format(new Timestamp(1000000))), Row(sdf3.format(new Timestamp(-1000000)))))
224225
}
225226

@@ -238,9 +239,9 @@ class DateFunctionsSuite extends QueryTest {
238239
Row(ts1.getTime / 1000L), Row(ts2.getTime / 1000L)))
239240
checkAnswer(df.select(unix_timestamp(col("ss"))), Seq(
240241
Row(ts1.getTime / 1000L), Row(ts2.getTime / 1000L)))
241-
checkAnswer(df.select(unix_timestamp(col("d"), lit(fmt))), Seq(
242+
checkAnswer(df.select(unix_timestamp(col("d"), fmt)), Seq(
242243
Row(date1.getTime / 1000L), Row(date2.getTime / 1000L)))
243-
checkAnswer(df.select(unix_timestamp(col("s"), lit(fmt))), Seq(
244+
checkAnswer(df.select(unix_timestamp(col("s"), fmt)), Seq(
244245
Row(ts1.getTime / 1000L), Row(ts2.getTime / 1000L)))
245246
}
246247

0 commit comments

Comments
 (0)