diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 9d43701f03056..a85be0e29a3aa 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -98,9 +98,7 @@ case class CurrentTimestamp() extends LeafExpression with CodegenFallback { override def dataType: DataType = TimestampType - override def eval(input: InternalRow): Any = { - instantToMicros(Instant.now()) - } + override def eval(input: InternalRow): Any = currentTimestamp() override def prettyName: String = "current_timestamp" } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 65a9bee5eaedd..a82471aae652d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -17,12 +17,14 @@ package org.apache.spark.sql.catalyst.util +import java.nio.charset.StandardCharsets import java.sql.{Date, Timestamp} import java.time._ import java.time.temporal.{ChronoField, ChronoUnit, IsoFields} import java.util.{Locale, TimeZone} import java.util.concurrent.TimeUnit._ +import scala.util.Try import scala.util.control.NonFatal import org.apache.spark.sql.types.Decimal @@ -218,6 +220,8 @@ object DateTimeUtils { var i = 0 var currentSegmentValue = 0 val bytes = s.trim.getBytes + val specialTimestamp = convertSpecialTimestamp(bytes, timeZoneId) + if (specialTimestamp.isDefined) return specialTimestamp var j = 0 var digitsMilli = 0 var justTime = false @@ -848,4 +852,67 @@ object DateTimeUtils { val sinceEpoch = BigDecimal(timestamp) / MICROS_PER_SECOND + offset new Decimal().set(sinceEpoch, 20, 6) } + + def currentTimestamp(): SQLTimestamp = instantToMicros(Instant.now()) + + private def today(zoneId: ZoneId): ZonedDateTime = { + Instant.now().atZone(zoneId).`with`(LocalTime.MIDNIGHT) + } + + private val specialValueRe = """(\p{Alpha}+)\p{Blank}*(.*)""".r + + /** + * Extracts special values from an input string ignoring case. + * @param input - a trimmed string + * @param zoneId - zone identifier used to get the current date. + * @return some special value in lower case or None. + */ + private def extractSpecialValue(input: String, zoneId: ZoneId): Option[String] = { + def isValid(value: String, timeZoneId: String): Boolean = { + // Special value can be without any time zone + if (timeZoneId.isEmpty) return true + // "now" must not have the time zone field + if (value.compareToIgnoreCase("now") == 0) return false + // If the time zone field presents in the input, it must be resolvable + try { + getZoneId(timeZoneId) + true + } catch { + case NonFatal(_) => false + } + } + + assert(input.trim.length == input.length) + if (input.length < 3 || !input(0).isLetter) return None + input match { + case specialValueRe(v, z) if isValid(v, z) => Some(v.toLowerCase(Locale.US)) + case _ => None + } + } + + /** + * Converts notational shorthands that are converted to ordinary timestamps. + * @param input - a trimmed string + * @param zoneId - zone identifier used to get the current date. + * @return some of microseconds since the epoch if the conversion completed + * successfully otherwise None. + */ + def convertSpecialTimestamp(input: String, zoneId: ZoneId): Option[SQLTimestamp] = { + extractSpecialValue(input, zoneId).flatMap { + case "epoch" => Some(0) + case "now" => Some(currentTimestamp()) + case "today" => Some(instantToMicros(today(zoneId).toInstant)) + case "tomorrow" => Some(instantToMicros(today(zoneId).plusDays(1).toInstant)) + case "yesterday" => Some(instantToMicros(today(zoneId).minusDays(1).toInstant)) + case _ => None + } + } + + private def convertSpecialTimestamp(bytes: Array[Byte], zoneId: ZoneId): Option[SQLTimestamp] = { + if (bytes.length > 0 && Character.isAlphabetic(bytes(0))) { + convertSpecialTimestamp(new String(bytes, StandardCharsets.UTF_8), zoneId) + } else { + None + } + } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala index b23cec64568df..8c256e1d56b9c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala @@ -22,9 +22,11 @@ import java.time._ import java.time.format.DateTimeParseException import java.time.temporal.ChronoField.MICRO_OF_SECOND import java.time.temporal.TemporalQueries -import java.util.{Locale, TimeZone} +import java.util.Locale import java.util.concurrent.TimeUnit.SECONDS +import DateTimeUtils.convertSpecialTimestamp + sealed trait TimestampFormatter extends Serializable { /** * Parses a timestamp in a string and converts it to microseconds. @@ -50,14 +52,17 @@ class Iso8601TimestampFormatter( protected lazy val formatter = getOrCreateFormatter(pattern, locale) override def parse(s: String): Long = { - val parsed = formatter.parse(s) - val parsedZoneId = parsed.query(TemporalQueries.zone()) - val timeZoneId = if (parsedZoneId == null) zoneId else parsedZoneId - val zonedDateTime = toZonedDateTime(parsed, timeZoneId) - val epochSeconds = zonedDateTime.toEpochSecond - val microsOfSecond = zonedDateTime.get(MICRO_OF_SECOND) + val specialDate = convertSpecialTimestamp(s.trim, zoneId) + specialDate.getOrElse { + val parsed = formatter.parse(s) + val parsedZoneId = parsed.query(TemporalQueries.zone()) + val timeZoneId = if (parsedZoneId == null) zoneId else parsedZoneId + val zonedDateTime = toZonedDateTime(parsed, timeZoneId) + val epochSeconds = zonedDateTime.toEpochSecond + val microsOfSecond = zonedDateTime.get(MICRO_OF_SECOND) - Math.addExact(SECONDS.toMicros(epochSeconds), microsOfSecond) + Math.addExact(SECONDS.toMicros(epochSeconds), microsOfSecond) + } } override def format(us: Long): String = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index 056337205ae7e..31fefd613f9c8 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -19,16 +19,18 @@ package org.apache.spark.sql.catalyst.util import java.sql.{Date, Timestamp} import java.text.SimpleDateFormat -import java.time.ZoneId +import java.time.{LocalDateTime, LocalTime, ZoneId} import java.util.{Locale, TimeZone} import java.util.concurrent.TimeUnit +import org.scalatest.Matchers + import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._ import org.apache.spark.sql.catalyst.util.DateTimeUtils._ import org.apache.spark.unsafe.types.UTF8String -class DateTimeUtilsSuite extends SparkFunSuite { +class DateTimeUtilsSuite extends SparkFunSuite with Matchers { val TimeZonePST = TimeZone.getTimeZone("PST") private def defaultZoneId = ZoneId.systemDefault() @@ -142,10 +144,14 @@ class DateTimeUtilsSuite extends SparkFunSuite { assert(stringToDate(UTF8String.fromString("1999 08")).isEmpty) } + private def toTimestamp(str: String, zoneId: ZoneId): Option[SQLTimestamp] = { + stringToTimestamp(UTF8String.fromString(str), zoneId) + } + test("string to timestamp") { for (tz <- ALL_TIMEZONES) { def checkStringToTimestamp(str: String, expected: Option[Long]): Unit = { - assert(stringToTimestamp(UTF8String.fromString(str), tz.toZoneId) === expected) + assert(toTimestamp(str, tz.toZoneId) === expected) } checkStringToTimestamp("1969-12-31 16:00:00", Option(date(1969, 12, 31, 16, tz = tz))) @@ -271,8 +277,8 @@ class DateTimeUtilsSuite extends SparkFunSuite { UTF8String.fromString("2015-02-29 00:00:00"), defaultZoneId).isEmpty) assert(stringToTimestamp( UTF8String.fromString("2015-04-31 00:00:00"), defaultZoneId).isEmpty) - assert(stringToTimestamp(UTF8String.fromString("2015-02-29"), defaultZoneId).isEmpty) - assert(stringToTimestamp(UTF8String.fromString("2015-04-31"), defaultZoneId).isEmpty) + assert(toTimestamp("2015-02-29", defaultZoneId).isEmpty) + assert(toTimestamp("2015-04-31", defaultZoneId).isEmpty) } test("hours") { @@ -456,8 +462,7 @@ class DateTimeUtilsSuite extends SparkFunSuite { timezone: TimeZone = DateTimeUtils.defaultTimeZone()): Unit = { val truncated = DateTimeUtils.truncTimestamp(inputTS, level, timezone) - val expectedTS = - DateTimeUtils.stringToTimestamp(UTF8String.fromString(expected), defaultZoneId) + val expectedTS = toTimestamp(expected, defaultZoneId) assert(truncated === expectedTS.get) } @@ -564,4 +569,21 @@ class DateTimeUtilsSuite extends SparkFunSuite { assert(DateTimeUtils.toMillis(-9223372036844776001L) === -9223372036844777L) assert(DateTimeUtils.toMillis(-157700927876544L) === -157700927877L) } + + test("special timestamp values") { + DateTimeTestUtils.outstandingZoneIds.foreach { zoneId => + val tolerance = TimeUnit.SECONDS.toMicros(30) + + assert(toTimestamp("Epoch", zoneId).get === 0) + val now = instantToMicros(LocalDateTime.now(zoneId).atZone(zoneId).toInstant) + toTimestamp("NOW", zoneId).get should be (now +- tolerance) + assert(toTimestamp("now UTC", zoneId) === None) + val today = instantToMicros(LocalDateTime.now(zoneId) + .`with`(LocalTime.MIDNIGHT) + .atZone(zoneId).toInstant) + toTimestamp(" Yesterday", zoneId).get should be (today - MICROS_PER_DAY +- tolerance) + toTimestamp("Today ", zoneId).get should be (today +- tolerance) + toTimestamp(" tomorrow CET ", zoneId).get should be (today + MICROS_PER_DAY +- tolerance) + } + } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala index c223639a47294..170daa6277c49 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala @@ -17,14 +17,18 @@ package org.apache.spark.sql.util -import java.time.{LocalDateTime, ZoneId, ZoneOffset} +import java.time.{LocalDateTime, LocalTime, ZoneOffset} import java.util.concurrent.TimeUnit +import org.scalatest.Matchers + import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.plans.SQLHelper import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils, TimestampFormatter} +import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, instantToMicros, MICROS_PER_DAY} +import org.apache.spark.sql.internal.SQLConf -class TimestampFormatterSuite extends SparkFunSuite with SQLHelper { +class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers { test("parsing timestamps using time zones") { val localDate = "2018-12-02T10:11:12.001234" @@ -131,4 +135,24 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper { val micros = DateTimeUtils.instantToMicros(instant) assert(TimestampFormatter(ZoneOffset.UTC).format(micros) === "-0099-01-01 00:00:00") } + + test("special timestamp values") { + DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone => + withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) { + val zoneId = getZoneId(timeZone) + val formatter = TimestampFormatter(zoneId) + val tolerance = TimeUnit.SECONDS.toMicros(30) + + assert(formatter.parse("EPOCH") === 0) + val now = instantToMicros(LocalDateTime.now(zoneId).atZone(zoneId).toInstant) + formatter.parse("now") should be (now +- tolerance) + val today = instantToMicros(LocalDateTime.now(zoneId) + .`with`(LocalTime.MIDNIGHT) + .atZone(zoneId).toInstant) + formatter.parse("yesterday CET") should be (today - MICROS_PER_DAY +- tolerance) + formatter.parse(" TODAY ") should be (today +- tolerance) + formatter.parse("Tomorrow ") should be (today + MICROS_PER_DAY +- tolerance) + } + } + } } diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql index 65e8d3280e07c..92bbe14dc1e5f 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql @@ -7,7 +7,6 @@ CREATE TABLE TIMESTAMP_TBL (d1 timestamp) USING parquet; --- [SPARK-28141] Timestamp type can not accept special values -- Test shorthand input values -- We can't just "select" the results since they aren't constants; test for -- equality instead. We can do that by running the test inside a transaction @@ -17,22 +16,24 @@ CREATE TABLE TIMESTAMP_TBL (d1 timestamp) USING parquet; -- block is entered exactly at local midnight; then 'now' and 'today' have -- the same values and the counts will come out different. --- INSERT INTO TIMESTAMP_TBL VALUES ('now'); +INSERT INTO TIMESTAMP_TBL VALUES ('now'); -- SELECT pg_sleep(0.1); -- BEGIN; --- INSERT INTO TIMESTAMP_TBL VALUES ('now'); --- INSERT INTO TIMESTAMP_TBL VALUES ('today'); --- INSERT INTO TIMESTAMP_TBL VALUES ('yesterday'); --- INSERT INTO TIMESTAMP_TBL VALUES ('tomorrow'); +INSERT INTO TIMESTAMP_TBL VALUES ('now'); +INSERT INTO TIMESTAMP_TBL VALUES ('today'); +INSERT INTO TIMESTAMP_TBL VALUES ('yesterday'); +INSERT INTO TIMESTAMP_TBL VALUES ('tomorrow'); -- time zone should be ignored by this data type --- INSERT INTO TIMESTAMP_TBL VALUES ('tomorrow EST'); --- INSERT INTO TIMESTAMP_TBL VALUES ('tomorrow zulu'); - --- SELECT count(*) AS One FROM TIMESTAMP_TBL WHERE d1 = timestamp 'today'; --- SELECT count(*) AS Three FROM TIMESTAMP_TBL WHERE d1 = timestamp 'tomorrow'; --- SELECT count(*) AS One FROM TIMESTAMP_TBL WHERE d1 = timestamp 'yesterday'; +INSERT INTO TIMESTAMP_TBL VALUES ('tomorrow EST'); +-- [SPARK-29024] Ignore case while resolving time zones +INSERT INTO TIMESTAMP_TBL VALUES ('tomorrow Zulu'); + +SELECT count(*) AS One FROM TIMESTAMP_TBL WHERE d1 = timestamp 'today'; +SELECT count(*) AS Three FROM TIMESTAMP_TBL WHERE d1 = timestamp 'tomorrow'; +SELECT count(*) AS One FROM TIMESTAMP_TBL WHERE d1 = timestamp 'yesterday'; +-- [SPARK-29025] Support seconds precision by the timestamp type -- SELECT count(*) AS One FROM TIMESTAMP_TBL WHERE d1 = timestamp(2) 'now'; -- COMMIT; @@ -48,12 +49,12 @@ CREATE TABLE TIMESTAMP_TBL (d1 timestamp) USING parquet; -- SELECT count(*) AS two FROM TIMESTAMP_TBL WHERE d1 = timestamp(2) 'now'; -- COMMIT; --- TRUNCATE TIMESTAMP_TBL; +TRUNCATE TABLE TIMESTAMP_TBL; -- Special values -- INSERT INTO TIMESTAMP_TBL VALUES ('-infinity'); -- INSERT INTO TIMESTAMP_TBL VALUES ('infinity'); --- INSERT INTO TIMESTAMP_TBL VALUES ('epoch'); +INSERT INTO TIMESTAMP_TBL VALUES ('epoch'); -- [SPARK-27923] Spark SQL insert there obsolete special values to NULL -- Obsolete special values -- INSERT INTO TIMESTAMP_TBL VALUES ('invalid'); diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out index 75d9ee8d9c797..db17ee46c6829 100644 --- a/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 18 +-- Number of queries: 30 -- !query 0 @@ -11,7 +11,7 @@ struct<> -- !query 1 -INSERT INTO TIMESTAMP_TBL VALUES ('1997-01-02') +INSERT INTO TIMESTAMP_TBL VALUES ('now') -- !query 1 schema struct<> -- !query 1 output @@ -19,7 +19,7 @@ struct<> -- !query 2 -INSERT INTO TIMESTAMP_TBL VALUES ('1997-01-02 03:04:05') +INSERT INTO TIMESTAMP_TBL VALUES ('now') -- !query 2 schema struct<> -- !query 2 output @@ -27,7 +27,7 @@ struct<> -- !query 3 -INSERT INTO TIMESTAMP_TBL VALUES ('1997-02-10 17:32:01-08') +INSERT INTO TIMESTAMP_TBL VALUES ('today') -- !query 3 schema struct<> -- !query 3 output @@ -35,7 +35,7 @@ struct<> -- !query 4 -INSERT INTO TIMESTAMP_TBL VALUES ('2001-09-22T18:19:20') +INSERT INTO TIMESTAMP_TBL VALUES ('yesterday') -- !query 4 schema struct<> -- !query 4 output @@ -43,139 +43,241 @@ struct<> -- !query 5 -SELECT '' AS `64`, d1 FROM TIMESTAMP_TBL +INSERT INTO TIMESTAMP_TBL VALUES ('tomorrow') -- !query 5 schema -struct<64:string,d1:timestamp> +struct<> -- !query 5 output + + + +-- !query 6 +INSERT INTO TIMESTAMP_TBL VALUES ('tomorrow EST') +-- !query 6 schema +struct<> +-- !query 6 output + + + +-- !query 7 +INSERT INTO TIMESTAMP_TBL VALUES ('tomorrow Zulu') +-- !query 7 schema +struct<> +-- !query 7 output + + + +-- !query 8 +SELECT count(*) AS One FROM TIMESTAMP_TBL WHERE d1 = timestamp 'today' +-- !query 8 schema +struct +-- !query 8 output +1 + + +-- !query 9 +SELECT count(*) AS Three FROM TIMESTAMP_TBL WHERE d1 = timestamp 'tomorrow' +-- !query 9 schema +struct +-- !query 9 output +3 + + +-- !query 10 +SELECT count(*) AS One FROM TIMESTAMP_TBL WHERE d1 = timestamp 'yesterday' +-- !query 10 schema +struct +-- !query 10 output +1 + + +-- !query 11 +TRUNCATE TABLE TIMESTAMP_TBL +-- !query 11 schema +struct<> +-- !query 11 output + + + +-- !query 12 +INSERT INTO TIMESTAMP_TBL VALUES ('epoch') +-- !query 12 schema +struct<> +-- !query 12 output + + + +-- !query 13 +INSERT INTO TIMESTAMP_TBL VALUES ('1997-01-02') +-- !query 13 schema +struct<> +-- !query 13 output + + + +-- !query 14 +INSERT INTO TIMESTAMP_TBL VALUES ('1997-01-02 03:04:05') +-- !query 14 schema +struct<> +-- !query 14 output + + + +-- !query 15 +INSERT INTO TIMESTAMP_TBL VALUES ('1997-02-10 17:32:01-08') +-- !query 15 schema +struct<> +-- !query 15 output + + + +-- !query 16 +INSERT INTO TIMESTAMP_TBL VALUES ('2001-09-22T18:19:20') +-- !query 16 schema +struct<> +-- !query 16 output + + + +-- !query 17 +SELECT '' AS `64`, d1 FROM TIMESTAMP_TBL +-- !query 17 schema +struct<64:string,d1:timestamp> +-- !query 17 output + 1969-12-31 16:00:00 1997-01-02 00:00:00 1997-01-02 03:04:05 1997-02-10 17:32:01 2001-09-22 18:19:20 --- !query 6 +-- !query 18 SELECT '' AS `48`, d1 FROM TIMESTAMP_TBL WHERE d1 > timestamp '1997-01-02' --- !query 6 schema +-- !query 18 schema struct<48:string,d1:timestamp> --- !query 6 output +-- !query 18 output 1997-01-02 03:04:05 1997-02-10 17:32:01 2001-09-22 18:19:20 --- !query 7 +-- !query 19 SELECT '' AS `15`, d1 FROM TIMESTAMP_TBL WHERE d1 < timestamp '1997-01-02' --- !query 7 schema +-- !query 19 schema struct<15:string,d1:timestamp> --- !query 7 output +-- !query 19 output + 1969-12-31 16:00:00 - --- !query 8 +-- !query 20 SELECT '' AS one, d1 FROM TIMESTAMP_TBL WHERE d1 = timestamp '1997-01-02' --- !query 8 schema +-- !query 20 schema struct --- !query 8 output +-- !query 20 output 1997-01-02 00:00:00 --- !query 9 +-- !query 21 SELECT '' AS `63`, d1 FROM TIMESTAMP_TBL WHERE d1 != timestamp '1997-01-02' --- !query 9 schema +-- !query 21 schema struct<63:string,d1:timestamp> --- !query 9 output +-- !query 21 output + 1969-12-31 16:00:00 1997-01-02 03:04:05 1997-02-10 17:32:01 2001-09-22 18:19:20 --- !query 10 +-- !query 22 SELECT '' AS `16`, d1 FROM TIMESTAMP_TBL WHERE d1 <= timestamp '1997-01-02' --- !query 10 schema +-- !query 22 schema struct<16:string,d1:timestamp> --- !query 10 output +-- !query 22 output + 1969-12-31 16:00:00 1997-01-02 00:00:00 --- !query 11 +-- !query 23 SELECT '' AS `49`, d1 FROM TIMESTAMP_TBL WHERE d1 >= timestamp '1997-01-02' --- !query 11 schema +-- !query 23 schema struct<49:string,d1:timestamp> --- !query 11 output +-- !query 23 output 1997-01-02 00:00:00 1997-01-02 03:04:05 1997-02-10 17:32:01 2001-09-22 18:19:20 --- !query 12 +-- !query 24 SELECT '' AS date_trunc_week, date_trunc( 'week', timestamp '2004-02-29 15:44:17.71393' ) AS week_trunc --- !query 12 schema +-- !query 24 schema struct --- !query 12 output +-- !query 24 output 2004-02-23 00:00:00 --- !query 13 +-- !query 25 SELECT '' AS `54`, d1 as `timestamp`, date_part( 'year', d1) AS `year`, date_part( 'month', d1) AS `month`, date_part( 'day', d1) AS `day`, date_part( 'hour', d1) AS `hour`, date_part( 'minute', d1) AS `minute`, date_part( 'second', d1) AS `second` FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' --- !query 13 schema +-- !query 25 schema struct<54:string,timestamp:timestamp,year:int,month:int,day:int,hour:int,minute:int,second:int> --- !query 13 output +-- !query 25 output + 1969-12-31 16:00:00 1969 12 31 16 0 0 1997-01-02 00:00:00 1997 1 2 0 0 0 1997-01-02 03:04:05 1997 1 2 3 4 5 1997-02-10 17:32:01 1997 2 10 17 32 1 2001-09-22 18:19:20 2001 9 22 18 19 20 --- !query 14 +-- !query 26 SELECT '' AS `54`, d1 as `timestamp`, date_part( 'quarter', d1) AS quarter, date_part( 'msec', d1) AS msec, date_part( 'usec', d1) AS usec FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' --- !query 14 schema +-- !query 26 schema struct<54:string,timestamp:timestamp,quarter:int,msec:decimal(8,3),usec:int> --- !query 14 output +-- !query 26 output + 1969-12-31 16:00:00 4 0 0 1997-01-02 00:00:00 1 0 0 1997-01-02 03:04:05 1 5000 5000000 1997-02-10 17:32:01 1 1000 1000000 2001-09-22 18:19:20 3 20000 20000000 --- !query 15 +-- !query 27 SELECT '' AS `54`, d1 as `timestamp`, date_part( 'isoyear', d1) AS isoyear, date_part( 'week', d1) AS week, date_part( 'dow', d1) AS dow FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' --- !query 15 schema +-- !query 27 schema struct<54:string,timestamp:timestamp,isoyear:int,week:int,dow:int> --- !query 15 output +-- !query 27 output + 1969-12-31 16:00:00 1970 1 3 1997-01-02 00:00:00 1997 1 4 1997-01-02 03:04:05 1997 1 4 1997-02-10 17:32:01 1997 7 1 2001-09-22 18:19:20 2001 38 6 --- !query 16 +-- !query 28 SELECT make_timestamp(2014,12,28,6,30,45.887) --- !query 16 schema +-- !query 28 schema struct --- !query 16 output +-- !query 28 output 2014-12-28 06:30:45.887 --- !query 17 +-- !query 29 DROP TABLE TIMESTAMP_TBL --- !query 17 schema +-- !query 29 schema struct<> --- !query 17 output +-- !query 29 output diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala index 52cf91cfade51..1094d7d23e5ea 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql +import java.sql.Timestamp import java.text.SimpleDateFormat import java.util.Locale @@ -181,4 +182,13 @@ class CsvFunctionsSuite extends QueryTest with SharedSparkSession { checkAnswer(df, Row(Row(java.sql.Timestamp.valueOf("2018-11-06 18:00:00.0")))) } } + + test("special timestamp values") { + Seq("now", "today", "epoch", "tomorrow", "yesterday").foreach { specialValue => + val input = Seq(specialValue).toDS() + val readback = input.select(from_csv($"value", lit("t timestamp"), + Map.empty[String, String].asJava)).collect() + assert(readback(0).getAs[Row](0).getAs[Timestamp](0).getTime >= 0) + } + } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala index 92a4acc130be5..c61c8109ee8e6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql +import java.sql.Timestamp import java.text.SimpleDateFormat import java.util.Locale @@ -608,4 +609,13 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession { checkAnswer(df, Row(Row(java.sql.Timestamp.valueOf("2018-11-06 18:00:00.0")))) } } + + test("special timestamp values") { + Seq("now", "today", "epoch", "tomorrow", "yesterday").foreach { specialValue => + val input = Seq(s"""{"t": "$specialValue"}""").toDS() + val readback = input.select(from_json($"value", lit("t timestamp"), + Map.empty[String, String].asJava)).collect() + assert(readback(0).getAs[Row](0).getAs[Timestamp](0).getTime >= 0) + } + } }