From a25d8b441f81d1a8650c3f28378b687bf64e18ae Mon Sep 17 00:00:00 2001 From: Rahul Mahadev Date: Fri, 18 Oct 2019 15:50:16 -0700 Subject: [PATCH 1/2] Fixed ArrayIndexOutOfBounds when converting from string to timestamp (2.4) --- .../org/apache/spark/sql/catalyst/util/DateTimeUtils.scala | 2 +- .../apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index cc3fcb2a2a39..228613710c30 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -378,7 +378,7 @@ object DateTimeUtils { i += 1 } } else { - if (b == ':' || b == ' ') { + if (i < 9 && (b == ':' || b == ' ')) { segments(i) = currentSegmentValue currentSegmentValue = 0 i += 1 diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index abdb91619ccb..2d055c7dddac 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -581,6 +581,12 @@ class DateTimeUtilsSuite extends SparkFunSuite { } } + test("trailing characters while converting string to timestamp") { + val s = UTF8String.fromString("2019-10-31T10:59:23Z:::") + val time = DateTimeUtils.stringToTimestamp(s, DateTimeUtils.defaultTimeZone()) + assert(time == None) + } + test("truncTimestamp") { def testTrunc( level: Int, From 4376b88dfc1f278cc97fb8b162c4277bc98148ac Mon Sep 17 00:00:00 2001 From: Rahul Mahadev Date: Fri, 18 Oct 2019 15:52:49 -0700 Subject: [PATCH 2/2] addressed review comments --- .../org/apache/spark/sql/catalyst/util/DateTimeUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 228613710c30..5deb83ef5624 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -378,7 +378,7 @@ object DateTimeUtils { i += 1 } } else { - if (i < 9 && (b == ':' || b == ' ')) { + if (i < segments.length && (b == ':' || b == ' ')) { segments(i) = currentSegmentValue currentSegmentValue = 0 i += 1