From 23c01c53185574ef50adc0e2c42b63ec22775c5d Mon Sep 17 00:00:00 2001 From: Rahul Mahadev Date: Wed, 16 Oct 2019 11:02:39 -0700 Subject: [PATCH 1/3] Fix for ArrayOutofBoundsException while converting string to timestamp --- .../org/apache/spark/sql/catalyst/util/DateTimeUtils.scala | 2 +- .../spark/sql/catalyst/util/DateTimeUtilsSuite.scala | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 088876921dcc..439b05e572c0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -286,7 +286,7 @@ object DateTimeUtils { i += 1 } } else { - if (b == ':' || b == ' ') { + if (i < 9 && (b == ':' || b == ' ')) { segments(i) = currentSegmentValue currentSegmentValue = 0 i += 1 diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index a9e3c9006a33..a8a631c9fa4f 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -456,6 +456,13 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers { } } + test("trailing characters while converting string to timestamp") { + val s = UTF8String.fromString("2019-10-31T10:59:23Z:::") + println(s) + val time = DateTimeUtils.stringToTimestamp(s, defaultZoneId) + assert(time == None) + } + test("truncTimestamp") { def testTrunc( level: Int, From 17edc79d220ba29fe768cc3a5ed5930bcecbded9 Mon Sep 17 00:00:00 2001 From: Rahul Mahadev Date: Wed, 16 Oct 2019 11:19:34 -0700 Subject: [PATCH 2/3] Removed println --- .../org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index a8a631c9fa4f..1abbca6c8cd2 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -458,7 +458,6 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers { test("trailing characters while converting string to timestamp") { val s = UTF8String.fromString("2019-10-31T10:59:23Z:::") - println(s) val time = DateTimeUtils.stringToTimestamp(s, defaultZoneId) assert(time == None) } From 5ef53b7ca639c0bff5db28b9d96ba810974b3207 Mon Sep 17 00:00:00 2001 From: Rahul Shivu Mahadev <51690557+rahulsmahadev@users.noreply.github.com> Date: Fri, 18 Oct 2019 10:50:34 -0700 Subject: [PATCH 3/3] Update sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala Co-Authored-By: Maxim Gekk --- .../org/apache/spark/sql/catalyst/util/DateTimeUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 439b05e572c0..8dd541b1e438 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -286,7 +286,7 @@ object DateTimeUtils { i += 1 } } else { - if (i < 9 && (b == ':' || b == ' ')) { + if (i < segments.length && (b == ':' || b == ' ')) { segments(i) = currentSegmentValue currentSegmentValue = 0 i += 1