From 7985de776958ddec7335a3399f2a6d772dbb18e1 Mon Sep 17 00:00:00 2001 From: 0x0FFF Date: Tue, 1 Sep 2015 06:30:22 -0700 Subject: [PATCH 1/3] [SPARK-10392] [SQL] Pyspark - Wrong DateType support on JDBC connection --- python/pyspark/sql/types.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py index 94e581a78364c..b0ac207418e71 100644 --- a/python/pyspark/sql/types.py +++ b/python/pyspark/sql/types.py @@ -168,10 +168,12 @@ def needConversion(self): return True def toInternal(self, d): - return d and d.toordinal() - self.EPOCH_ORDINAL + if d is not None: + return d.toordinal() - self.EPOCH_ORDINAL def fromInternal(self, v): - return v and datetime.date.fromordinal(v + self.EPOCH_ORDINAL) + if v is not None: + return datetime.date.fromordinal(v + self.EPOCH_ORDINAL) class TimestampType(AtomicType): From f41b125c7e0443828ff69cb5a9bffeeba268b680 Mon Sep 17 00:00:00 2001 From: 0x0FFF Date: Tue, 1 Sep 2015 12:38:09 -0700 Subject: [PATCH 2/3] [SPARK-10392] [SQL] Regression test added --- python/pyspark/sql/tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py index cd32e26c64f22..db740c3a858ae 100644 --- a/python/pyspark/sql/tests.py +++ b/python/pyspark/sql/tests.py @@ -1164,6 +1164,9 @@ def test_window_functions_without_partitionBy(self): for r, ex in zip(rs, expected): self.assertEqual(tuple(r), ex[:len(r)]) + def test_datetype_equal_zero(self): + dt = DateType() + self.assertEqual(dt.fromInternal(0), datetime.date(1970, 1, 1)) if __name__ == "__main__": unittest.main() From a7fd68156764960ebf773912d1dfcebb331cf55a Mon Sep 17 00:00:00 2001 From: 0x0FFF Date: Tue, 1 Sep 2015 13:28:54 -0700 Subject: [PATCH 3/3] [SPARK-10392] [SQL] Moving regression test to DataTypeTests class --- python/pyspark/sql/tests.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py index db740c3a858ae..6b647f3aacfa1 100644 --- a/python/pyspark/sql/tests.py +++ b/python/pyspark/sql/tests.py @@ -167,6 +167,11 @@ def test_decimal_type(self): t3 = DecimalType(8) self.assertNotEqual(t2, t3) + # regression test for SPARK-10392 + def test_datetype_equal_zero(self): + dt = DateType() + self.assertEqual(dt.fromInternal(0), datetime.date(1970, 1, 1)) + class SQLTests(ReusedPySparkTestCase): @@ -1164,9 +1169,6 @@ def test_window_functions_without_partitionBy(self): for r, ex in zip(rs, expected): self.assertEqual(tuple(r), ex[:len(r)]) - def test_datetype_equal_zero(self): - dt = DateType() - self.assertEqual(dt.fromInternal(0), datetime.date(1970, 1, 1)) if __name__ == "__main__": unittest.main()