use int for timestamp

Author: Davies Liu

python/pyspark/sql/tests.py

@@ -612,10 +612,10 @@ def test_time_with_timezone(self):
         utcnow = datetime.datetime.fromtimestamp(ts, utc)
         df = self.sqlCtx.createDataFrame([(now, utcnow)])
         now1, utcnow1 = df.first()
-        # Spark SQL does not support microsecond, the error should be
+        # Pyrolite does not support microsecond, the error should be
         # less than 1 millisecond
-        self.assertTrue(now1 - now < datetime.timedelta(0.001))
-        self.assertTrue(utcnow1 - now < datetime.timedelta(0.001))
+        self.assertTrue(now - now1 < datetime.timedelta(0.001))
+        self.assertTrue(now - utcnow1 < datetime.timedelta(0.001))
 
     def test_dropna(self):
         schema = StructType([

python/pyspark/sql/types.py

@@ -653,8 +653,6 @@ def _need_python_to_sql_conversion(dataType):
     elif isinstance(dataType, MapType):
         return _need_python_to_sql_conversion(dataType.keyType) or \
             _need_python_to_sql_conversion(dataType.valueType)
-    elif isinstance(dataType, TimestampType):
-        return True
     elif isinstance(dataType, UserDefinedType):
         return True
     elif isinstance(dataType, TimestampType):
@@ -711,25 +709,17 @@ def converter(obj):
         value_converter = _python_to_sql_converter(dataType.valueType)
         return lambda m: dict([(key_converter(k), value_converter(v)) for k, v in m.items()])
 
-    elif isinstance(dataType, TimestampType):
-
-        def to_posix_timstamp(dt):
-            if dt.tzinfo is None:
-                return time.mktime(dt.timetuple()) + dt.microsecond / 1e6
-            else:
-                return calendar.timegm(dt.utctimetuple()) + dt.microsecond / 1e6
-        return to_posix_timstamp
-
     elif isinstance(dataType, UserDefinedType):
         return lambda obj: dataType.serialize(obj)
+
     elif isinstance(dataType, TimestampType):
 
         def to_posix_timstamp(dt):
-            if dt.tzinfo is None:
-                return int(time.mktime(dt.timetuple()) * 1e7 + dt.microsecond * 10)
-            else:
-                return int(calendar.timegm(dt.utctimetuple()) * 1e7 + dt.microsecond * 10)
+            seconds = (calendar.timegm(dt.utctimetuple()) if dt.tzinfo
+                       else time.mktime(dt.timetuple()))
+            return int(seconds * 1e7 + dt.microsecond * 10)
         return to_posix_timstamp
+
     else:
         raise ValueError("Unexpected type %r" % dataType)