BUG: Timestamp cannot parse nanosecond from string

Author: sinhrks

doc/source/v0.15.0.txt

@@ -166,6 +166,8 @@ previously results in ``Exception`` or ``TypeError`` (:issue:`7812`)
 - ``DataFrame.tz_localize`` and ``DataFrame.tz_convert`` now accepts an optional ``level`` argument
   for localizing a specific level of a MultiIndex (:issue:`7846`)
 
+- ``Timestamp.__repr__`` displays ``dateutil.tz.tzoffset`` info (:issue:`7907`)
+
 .. _whatsnew_0150.dt:
 
 .dt accessor
@@ -443,7 +445,8 @@ Bug Fixes
 - Bug in ``Series.str.cat`` with an index which was filtered as to not include the first item (:issue:`7857`)
 
 
-
+- Bug in ``Timestamp`` cannot parse ``nanosecond`` from string (:issue:`7878`)
+- Bug in ``Timestamp`` with string offset and ``tz`` results incorrect (:issue:`7833`)
 
 - Bug in ``tslib.tz_convert`` and ``tslib.tz_convert_single`` may return different results (:issue:`7798`)
 - Bug in ``DatetimeIndex.intersection`` of non-overlapping timestamps with tz raises ``IndexError`` (:issue:`7880`)

pandas/src/datetime.pxd

@@ -109,7 +109,8 @@ cdef extern from "datetime/np_datetime_strings.h":
 
     int parse_iso_8601_datetime(char *str, int len, PANDAS_DATETIMEUNIT unit,
                                 NPY_CASTING casting, pandas_datetimestruct *out,
-                                npy_bool *out_local, PANDAS_DATETIMEUNIT *out_bestunit,
+                                int *out_local, int *out_tzoffset,
+                                PANDAS_DATETIMEUNIT *out_bestunit,
                                 npy_bool *out_special)
 
     int make_iso_8601_datetime(pandas_datetimestruct *dts, char *outstr, int outlen,
@@ -123,29 +124,31 @@ cdef extern from "datetime/np_datetime_strings.h":
 
 
 
-cdef inline _string_to_dts(object val, pandas_datetimestruct* dts):
+cdef inline _string_to_dts(object val, pandas_datetimestruct* dts,
+                           int* out_local, int* out_tzoffset):
     cdef int result
     cdef char *tmp
 
     if PyUnicode_Check(val):
         val = PyUnicode_AsASCIIString(val);
 
     tmp = val
-    result = _cstring_to_dts(tmp, len(val), dts)
+    result = _cstring_to_dts(tmp, len(val), dts, out_local, out_tzoffset)
 
     if result == -1:
         raise ValueError('Unable to parse %s' % str(val))
 
 cdef inline int _cstring_to_dts(char *val, int length,
-                                pandas_datetimestruct* dts):
+                                pandas_datetimestruct* dts,
+                                int* out_local, int* out_tzoffset):
     cdef:
-        npy_bool islocal, special
+        npy_bool special
         PANDAS_DATETIMEUNIT out_bestunit
         int result
 
     result = parse_iso_8601_datetime(val, length, PANDAS_FR_ns,
                                      NPY_UNSAFE_CASTING,
-                                     dts, &islocal, &out_bestunit, &special)
+                                     dts, out_local, out_tzoffset, &out_bestunit, &special)
     return result
 
 

pandas/src/datetime/np_datetime_strings.c

@@ -363,7 +363,9 @@ convert_datetimestruct_local_to_utc(pandas_datetimestruct *out_dts_utc,
  *           to be cast to the 'unit' parameter.
  *
  * 'out' gets filled with the parsed date-time.
- * 'out_local' gets set to 1 if the parsed time was in local time,
+ * 'out_local' gets whether returned value contains timezone. 0 for UTC, 1 for local time.
+ * 'out_tzoffset' gets set to timezone offset by minutes
+ *      if the parsed time was in local time,
  *      to 0 otherwise. The values 'now' and 'today' don't get counted
  *      as local, and neither do UTC +/-#### timezone offsets, because
  *      they aren't using the computer's local timezone offset.
@@ -381,7 +383,8 @@ parse_iso_8601_datetime(char *str, int len,
                     PANDAS_DATETIMEUNIT unit,
                     NPY_CASTING casting,
                     pandas_datetimestruct *out,
-                    npy_bool *out_local,
+                    int *out_local,
+                    int *out_tzoffset,
                     PANDAS_DATETIMEUNIT *out_bestunit,
                     npy_bool *out_special)
 {
@@ -778,19 +781,6 @@ parse_iso_8601_datetime(char *str, int len,
     if (sublen == 0) {
         // Unlike NumPy, treating no time zone as naive
         goto finish;
-
-/*
-        if (convert_datetimestruct_local_to_utc(out, out) < 0) {
-            goto error;
-        }
-
-        // Since neither "Z" nor a time-zone was specified, it's local
-        if (out_local != NULL) {
-            *out_local = 1;
-        }
-
-        goto finish;
-*/
     }
 
     /* UTC specifier */
@@ -816,9 +806,6 @@ parse_iso_8601_datetime(char *str, int len,
          * Since "local" means local with respect to the current
          * machine, we say this is non-local.
          */
-        if (out_local != NULL) {
-            *out_local = 0;
-        }
 
         if (*substr == '-') {
             offset_neg = 1;
@@ -872,7 +859,11 @@ parse_iso_8601_datetime(char *str, int len,
             offset_hour = -offset_hour;
             offset_minute = -offset_minute;
         }
-        add_minutes_to_datetimestruct(out, -60 * offset_hour - offset_minute);
+        if (out_local != NULL) {
+            *out_local = 1;
+            // Unlike NumPy, do not change internal value to local time
+            *out_tzoffset = 60 * offset_hour - offset_minute;
+        }
     }
 
     /* Skip trailing whitespace */

pandas/src/datetime/np_datetime_strings.h

@@ -27,7 +27,9 @@
  *           to be cast to the 'unit' parameter.
  *
  * 'out' gets filled with the parsed date-time.
- * 'out_local' gets set to 1 if the parsed time was in local time,
+ * 'out_local' gets whether returned value contains timezone. 0 for UTC, 1 for local time.
+ * 'out_tzoffset' gets set to timezone offset by minutes
+ *      if the parsed time was in local time,
  *      to 0 otherwise. The values 'now' and 'today' don't get counted
  *      as local, and neither do UTC +/-#### timezone offsets, because
  *      they aren't using the computer's local timezone offset.
@@ -45,7 +47,8 @@ parse_iso_8601_datetime(char *str, int len,
                     PANDAS_DATETIMEUNIT unit,
                     NPY_CASTING casting,
                     pandas_datetimestruct *out,
-                    npy_bool *out_local,
+                    int *out_local,
+                    int *out_tzoffset,
                     PANDAS_DATETIMEUNIT *out_bestunit,
                     npy_bool *out_special);
 

pandas/tseries/tests/test_timeseries.py

@@ -2173,10 +2173,31 @@ def test_constructor_coverage(self):
     def test_constructor_datetime64_tzformat(self):
         # GH 6572
         tm._skip_if_no_pytz()
+        import pytz
+        # ISO 8601 format results in pytz.FixedOffset
+        for freq in ['AS', 'W-SUN']:
+            idx = date_range('2013-01-01T00:00:00-05:00', '2016-01-01T23:59:59-05:00', freq=freq)
+            expected = date_range('2013-01-01T00:00:00', '2016-01-01T23:59:59',
+                                  freq=freq, tz=pytz.FixedOffset(-300))
+            tm.assert_index_equal(idx, expected)
+            # Unable to use `US/Eastern` because of DST
+            expected_i8 = date_range('2013-01-01T00:00:00', '2016-01-01T23:59:59',
+                                     freq=freq, tz='America/Lima')
+            self.assert_numpy_array_equal(idx.asi8, expected_i8.asi8)
+
+            idx = date_range('2013-01-01T00:00:00+09:00', '2016-01-01T23:59:59+09:00', freq=freq)
+            expected = date_range('2013-01-01T00:00:00', '2016-01-01T23:59:59',
+                                  freq=freq, tz=pytz.FixedOffset(540))
+            tm.assert_index_equal(idx, expected)
+            expected_i8 = date_range('2013-01-01T00:00:00', '2016-01-01T23:59:59',
+                                     freq=freq, tz='Asia/Tokyo')
+            self.assert_numpy_array_equal(idx.asi8, expected_i8.asi8)
+
         tm._skip_if_no_dateutil()
         from dateutil.tz import tzoffset
+        # Non ISO 8601 format results in dateutil.tz.tzoffset
         for freq in ['AS', 'W-SUN']:
-            idx = date_range('2013-01-01T00:00:00-05:00', '2016-01-01T23:59:59-05:00', freq=freq)
+            idx = date_range('2013/1/1 0:00:00-5:00', '2016/1/1 23:59:59-5:00', freq=freq)
             expected = date_range('2013-01-01T00:00:00', '2016-01-01T23:59:59',
                                   freq=freq, tz=tzoffset(None, -18000))
             tm.assert_index_equal(idx, expected)
@@ -2185,7 +2206,7 @@ def test_constructor_datetime64_tzformat(self):
                                      freq=freq, tz='America/Lima')
             self.assert_numpy_array_equal(idx.asi8, expected_i8.asi8)
 
-            idx = date_range('2013-01-01T00:00:00+09:00', '2016-01-01T23:59:59+09:00', freq=freq)
+            idx = date_range('2013/1/1 0:00:00+9:00', '2016/1/1 23:59:59+09:00', freq=freq)
             expected = date_range('2013-01-01T00:00:00', '2016-01-01T23:59:59',
                                   freq=freq, tz=tzoffset(None, 32400))
             tm.assert_index_equal(idx, expected)