Merge pull request #4302 from jreback/nat_convert

BUG: explicity change nan -> NaT when assigning to datelike dtypes

Author: jreback

pandas/core/common.py

@@ -42,6 +42,7 @@ class AmbiguousIndexError(PandasError, KeyError):
 _NS_DTYPE = np.dtype('M8[ns]')
 _TD_DTYPE = np.dtype('m8[ns]')
 _INT64_DTYPE = np.dtype(np.int64)
+_DATELIKE_DTYPES = set([ np.dtype(t) for t in ['M8[ns]','m8[ns]'] ])
 
 def isnull(obj):
     """Detect missing values (NaN in numeric arrays, None/NaN in object arrays)
@@ -718,6 +719,12 @@ def _infer_dtype_from_scalar(val):
     return dtype, val
 
 
+def _maybe_cast_scalar(dtype, value):
+    """ if we a scalar value and are casting to a dtype that needs nan -> NaT conversion """
+    if np.isscalar(value) and dtype in _DATELIKE_DTYPES and isnull(value):
+        return tslib.iNaT
+    return value
+
 def _maybe_promote(dtype, fill_value=np.nan):
 
     # if we passed an array here, determine the fill value by dtype
@@ -789,6 +796,7 @@ def _maybe_upcast_putmask(result, mask, other, dtype=None, change=None):
 
     if mask.any():
 
+        other = _maybe_cast_scalar(result.dtype, other)
         def changeit():
 
             # try to directly set by expanding our array to full
@@ -851,6 +859,7 @@ def _maybe_upcast_indexer(result, indexer, other, dtype=None):
         return the result and a changed flag
         """
 
+    other = _maybe_cast_scalar(result.dtype, other)
     original_dtype = result.dtype
     def changeit():
         # our type is wrong here, need to upcast

pandas/tests/test_common.py

@@ -5,7 +5,7 @@
 import nose
 import unittest
 
-from pandas import Series, DataFrame, date_range, DatetimeIndex
+from pandas import Series, DataFrame, date_range, DatetimeIndex, Timestamp
 from pandas.core.common import notnull, isnull
 import pandas.core.common as com
 import pandas.util.testing as tm
@@ -117,6 +117,24 @@ def test_datetimeindex_from_empty_datetime64_array():
         idx = DatetimeIndex(np.array([], dtype='datetime64[%s]' % unit))
         assert(len(idx) == 0)
 
+def test_nan_to_nat_conversions():
+
+    df = DataFrame(dict({
+        'A' : np.asarray(range(10),dtype='float64'),
+        'B' : Timestamp('20010101') }))
+    df.iloc[3:6,:] = np.nan
+    result = df.loc[4,'B'].value
+    assert(result == iNaT)
+
+    values = df['B'].values
+    result, changed = com._maybe_upcast_indexer(values,tuple([slice(8,9)]),np.nan)
+    assert(isnull(result[8]))
+
+    # numpy < 1.7.0 is wrong
+    from distutils.version import LooseVersion
+    if LooseVersion(np.__version__) >= '1.7.0':
+        assert(result[8] == np.datetime64('NaT'))
+
 def test_any_none():
     assert(com._any_none(1, 2, 3, None))
     assert(not com._any_none(1, 2, 3, 4))