Skip to content

Commit b250bc7

Browse files
linhongliu-dblinhongliu-db
committed
fix tests
1 parent 633781f commit b250bc7

File tree

8 files changed

+188
-64
lines changed

8 files changed

+188
-64
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,7 @@ object DateTimeUtils {
249249
* the input string can't be parsed as timestamp, the result timestamp segments are empty.
250250
*/
251251
private def parseTimestampString(s: UTF8String): (Array[Int], Option[ZoneId], Boolean) = {
252-
if (s == null) {
252+
if (s == null || s.trimAll().numBytes() == 0) {
253253
return (Array.empty, None, false)
254254
}
255255
var tz: Option[String] = None
@@ -524,7 +524,7 @@ object DateTimeUtils {
524524
* `[+-]y*-[m]m-[d]dT*`
525525
*/
526526
def stringToDate(s: UTF8String): Option[Int] = {
527-
if (s == null) {
527+
if (s == null || s.trimAll().numBytes() == 0) {
528528
return None
529529
}
530530
val segments: Array[Int] = Array[Int](1, 1, 1)

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/AnsiCastSuiteBase.scala‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -371,7 +371,6 @@ abstract class AnsiCastSuiteBase extends CastSuiteBase {
371371
s"Cannot cast $str to TimestampType.")
372372
}
373373

374-
checkCastWithParseError("123")
375374
checkCastWithParseError("2015-03-18 123142")
376375
checkCastWithParseError("2015-03-18T123123")
377376
checkCastWithParseError("2015-03-18X")
@@ -392,8 +391,6 @@ abstract class AnsiCastSuiteBase extends CastSuiteBase {
392391
s"Cannot cast $str to DateType.")
393392
}
394393

395-
checkCastWithParseError("12345")
396-
checkCastWithParseError("12345-12-18")
397394
checkCastWithParseError("2015-13-18")
398395
checkCastWithParseError("2015-03-128")
399396
checkCastWithParseError("2015/03/18")
@@ -413,7 +410,6 @@ abstract class AnsiCastSuiteBase extends CastSuiteBase {
413410
test("SPARK-35720: cast invalid string input to timestamp without time zone") {
414411
Seq("00:00:00",
415412
"a",
416-
"123",
417413
"a2021-06-17",
418414
"2021-06-17abc",
419415
"2021-06-17 00:00:00ABC").foreach { invalidInput =>

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -535,7 +535,7 @@ class CastSuite extends CastSuiteBase {
535535
checkEvaluation(
536536
cast(cast(cast(cast(cast(cast("5", TimestampType, UTC_OPT), ByteType),
537537
DecimalType.SYSTEM_DEFAULT), LongType), StringType), ShortType),
538-
null)
538+
-128.toShort)
539539
checkEvaluation(cast(cast(cast(cast(cast(cast("5", DecimalType.SYSTEM_DEFAULT),
540540
ByteType), TimestampType), LongType), StringType), ShortType),
541541
5.toShort)
@@ -569,7 +569,6 @@ class CastSuite extends CastSuiteBase {
569569
test("SPARK-35720: cast invalid string input to timestamp without time zone") {
570570
Seq("00:00:00",
571571
"a",
572-
"123",
573572
"a2021-06-17",
574573
"2021-06-17abc",
575574
"2021-06-17 00:00:00ABC").foreach { invalidInput =>

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,9 +194,11 @@ class HashExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
194194
// before epoch
195195
checkHiveHashForDateType("1800-01-01", -62091)
196196

197+
// negative year
198+
checkHiveHashForDateType("-1212-01-01", -1162202)
199+
197200
// Invalid input: bad date string. Hive returns 0 for such cases
198201
intercept[NoSuchElementException](checkHiveHashForDateType("0-0-0", 0))
199-
intercept[NoSuchElementException](checkHiveHashForDateType("-1212-01-01", 0))
200202
intercept[NoSuchElementException](checkHiveHashForDateType("2016-99-99", 0))
201203

202204
// Invalid input: Empty string. Hive returns 0 for this case

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -142,13 +142,11 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
142142
assert(toDate("2015.03.18").isEmpty)
143143
assert(toDate("20150318").isEmpty)
144144
assert(toDate("2015-031-8").isEmpty)
145-
assert(toDate("02015-03-18").isEmpty)
146-
assert(toDate("015-03-18").isEmpty)
147-
assert(toDate("015").isEmpty)
148-
assert(toDate("02015").isEmpty)
149145
assert(toDate("1999 08 01").isEmpty)
150146
assert(toDate("1999-08 01").isEmpty)
151147
assert(toDate("1999 08").isEmpty)
148+
assert(toDate("").isEmpty)
149+
assert(toDate(" ").isEmpty)
152150
}
153151

154152
test("SPARK-35780: support full range of date string") {
@@ -268,23 +266,21 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
268266
expected = Option(date(2011, 5, 6, 7, 8, 9, 100000, zid = zid))
269267
checkStringToTimestamp("2011-05-06 07:08:09.1000", expected)
270268

271-
checkStringToTimestamp("238", None)
272-
checkStringToTimestamp("00238", None)
273269
checkStringToTimestamp("2015-03-18 123142", None)
274270
checkStringToTimestamp("2015-03-18T123123", None)
275271
checkStringToTimestamp("2015-03-18X", None)
276272
checkStringToTimestamp("2015/03/18", None)
277273
checkStringToTimestamp("2015.03.18", None)
278274
checkStringToTimestamp("20150318", None)
279275
checkStringToTimestamp("2015-031-8", None)
280-
checkStringToTimestamp("02015-01-18", None)
281-
checkStringToTimestamp("015-01-18", None)
282276
checkStringToTimestamp("2015-03-18T12:03.17-20:0", None)
283277
checkStringToTimestamp("2015-03-18T12:03.17-0:70", None)
284278
checkStringToTimestamp("2015-03-18T12:03.17-1:0:0", None)
285279
checkStringToTimestamp("1999 08 01", None)
286280
checkStringToTimestamp("1999-08 01", None)
287281
checkStringToTimestamp("1999 08", None)
282+
checkStringToTimestamp("", None)
283+
checkStringToTimestamp(" ", None)
288284

289285
// Truncating the fractional seconds
290286
expected = Option(date(2015, 3, 18, 12, 3, 17, 123456, zid = UTC))
@@ -308,10 +304,8 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
308304
checkStringToTimestamp("015-03-18 16:00:00", Option(date(15, 3, 18, 16, zid = UTC)))
309305
checkStringToTimestamp("000001", Option(date(1, 1, 1, 0, zid = UTC)))
310306
checkStringToTimestamp("-000001", Option(date(-1, 1, 1, 0, zid = UTC)))
311-
checkStringToTimestamp("99999-03-18T12:03:17",
312-
Option(date(99999, 3, 18, 12, 3, 17, zid = UTC)))
313-
checkStringToTimestamp("-99999-03-18T12:03:17",
314-
Option(date(-99999, 3, 18, 12, 3, 17, zid = UTC)))
307+
checkStringToTimestamp("238", Option(date(238, 1, 1, 0, zid = UTC)))
308+
checkStringToTimestamp("00238", Option(date(238, 1, 1, 0, zid = UTC)))
315309
}
316310

317311
test("SPARK-15379: special invalid date string") {

‎sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out‎

Lines changed: 147 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
-- Automatically generated by SQLQueryTestSuite
2-
-- Number of queries: 193
2+
-- Number of queries: 211
33

44

55
-- !query
@@ -345,7 +345,7 @@ select '1' - interval '2' second
345345
-- !query schema
346346
struct<1 - INTERVAL '2 seconds':string>
347347
-- !query output
348-
NULL
348+
0000-12-31 23:59:58
349349

350350

351351
-- !query
@@ -1584,3 +1584,148 @@ select to_timestamp_ntz('2021-06-25 10:11:12') - interval '20 15:40:32.99899999'
15841584
struct<to_timestamp_ntz(2021-06-25 10:11:12) - INTERVAL '20 15:40:32.998999' DAY TO SECOND:timestamp without time zone>
15851585
-- !query output
15861586
2021-06-04 18:30:39.001001
1587+
1588+
1589+
-- !query
1590+
set spark.sql.datetime.java8API.enabled=true
1591+
-- !query schema
1592+
struct<key:string,value:string>
1593+
-- !query output
1594+
spark.sql.datetime.java8API.enabled true
1595+
1596+
1597+
-- !query
1598+
select date'02015-03-18'
1599+
-- !query schema
1600+
struct<DATE '2015-03-18':date>
1601+
-- !query output
1602+
2015-03-18
1603+
1604+
1605+
-- !query
1606+
select date'015'
1607+
-- !query schema
1608+
struct<DATE '0015-01-01':date>
1609+
-- !query output
1610+
0015-01-01
1611+
1612+
1613+
-- !query
1614+
select date'-1-1-28'
1615+
-- !query schema
1616+
struct<DATE '0002-01-28':date>
1617+
-- !query output
1618+
0002-01-28
1619+
1620+
1621+
-- !query
1622+
select cast('5881580-7-11' as date)
1623+
-- !query schema
1624+
struct<CAST(5881580-7-11 AS DATE):date>
1625+
-- !query output
1626+
5881580-07-11
1627+
1628+
1629+
-- !query
1630+
select cast('5881580-7-12' as date)
1631+
-- !query schema
1632+
struct<CAST(5881580-7-12 AS DATE):date>
1633+
-- !query output
1634+
NULL
1635+
1636+
1637+
-- !query
1638+
select cast('-5877641-6-23' as date)
1639+
-- !query schema
1640+
struct<>
1641+
-- !query output
1642+
java.lang.ArithmeticException
1643+
integer overflow
1644+
1645+
1646+
-- !query
1647+
select cast('-5877641-6-22' as date)
1648+
-- !query schema
1649+
struct<CAST(-5877641-6-22 AS DATE):date>
1650+
-- !query output
1651+
NULL
1652+
1653+
1654+
-- !query
1655+
select timestamp'-1969-12-31 16:00:00'
1656+
-- !query schema
1657+
struct<TIMESTAMP '-1969-12-31 16:00:00':timestamp>
1658+
-- !query output
1659+
-1969-12-31 16:00:00
1660+
1661+
1662+
-- !query
1663+
select timestamp'02015-03-18 16:00:00'
1664+
-- !query schema
1665+
struct<TIMESTAMP '2015-03-18 16:00:00':timestamp>
1666+
-- !query output
1667+
2015-03-18 16:00:00
1668+
1669+
1670+
-- !query
1671+
select timestamp'015-03-18 16:00:00'
1672+
-- !query schema
1673+
struct<TIMESTAMP '0015-03-18 16:00:00':timestamp>
1674+
-- !query output
1675+
0015-03-18 16:00:00
1676+
1677+
1678+
-- !query
1679+
select timestamp'-000001'
1680+
-- !query schema
1681+
struct<TIMESTAMP '-0001-01-01 00:00:00':timestamp>
1682+
-- !query output
1683+
-0001-01-01 00:00:00
1684+
1685+
1686+
-- !query
1687+
select timestamp'99999-03-18T12:03:17'
1688+
-- !query schema
1689+
struct<TIMESTAMP '+99999-03-18 12:03:17':timestamp>
1690+
-- !query output
1691+
+99999-03-18 12:03:17
1692+
1693+
1694+
-- !query
1695+
select cast('294247-01-10T04:00:54.775807Z' as timestamp)
1696+
-- !query schema
1697+
struct<CAST(294247-01-10T04:00:54.775807Z AS TIMESTAMP):timestamp>
1698+
-- !query output
1699+
+294247-01-09 20:00:54.775807
1700+
1701+
1702+
-- !query
1703+
select cast('294247-01-10T04:00:54.775808Z' as timestamp)
1704+
-- !query schema
1705+
struct<CAST(294247-01-10T04:00:54.775808Z AS TIMESTAMP):timestamp>
1706+
-- !query output
1707+
NULL
1708+
1709+
1710+
-- !query
1711+
select cast('-290308-12-21T19:59:05.224192Z' as timestamp)
1712+
-- !query schema
1713+
struct<CAST(-290308-12-21T19:59:05.224192Z AS TIMESTAMP):timestamp>
1714+
-- !query output
1715+
-290308-12-21 12:06:07.224192
1716+
1717+
1718+
-- !query
1719+
select cast('-290308-12-21T19:59:05.224191Z' as timestamp)
1720+
-- !query schema
1721+
struct<CAST(-290308-12-21T19:59:05.224191Z AS TIMESTAMP):timestamp>
1722+
-- !query output
1723+
NULL
1724+
1725+
1726+
-- !query
1727+
set spark.sql.datetime.java8API.enabled=false
1728+
-- !query schema
1729+
struct<key:string,value:string>
1730+
-- !query output
1731+
spark.sql.datetime.java8API.enabled false

‎sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out‎

Lines changed: 4 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -454,29 +454,17 @@ struct<DATE '4714-11-23':date>
454454
-- !query
455455
SELECT date '5874897-12-31'
456456
-- !query schema
457-
struct<>
457+
struct<DATE '+5874897-12-31':date>
458458
-- !query output
459-
org.apache.spark.sql.catalyst.parser.ParseException
460-
461-
Cannot parse the DATE value: 5874897-12-31(line 1, pos 7)
462-
463-
== SQL ==
464-
SELECT date '5874897-12-31'
465-
-------^^^
459+
5874897-12-31
466460

467461

468462
-- !query
469463
SELECT date '5874898-01-01'
470464
-- !query schema
471-
struct<>
465+
struct<DATE '+5874898-01-01':date>
472466
-- !query output
473-
org.apache.spark.sql.catalyst.parser.ParseException
474-
475-
Cannot parse the DATE value: 5874898-01-01(line 1, pos 7)
476-
477-
== SQL ==
478-
SELECT date '5874898-01-01'
479-
-------^^^
467+
5874898-01-01
480468

481469

482470
-- !query

0 commit comments

Comments
 (0)