BUG: Concatentation of TZ-aware dataframes

closes #12396
closes #18447

Author: paul-mannino

doc/source/whatsnew/v0.23.0.txt

@@ -1373,6 +1373,8 @@ Reshaping
 - Bug in :class:`Series` constructor with a ``dtype=str``, previously raised in some cases (:issue:`19853`)
 - Bug in :func:`get_dummies`, and :func:`select_dtypes`, where duplicate column names caused incorrect behavior (:issue:`20848`)
 - Bug in :func:`isna`, which cannot handle ambiguous typed lists (:issue:`20675`)
+- Bug in :func:`concat` which raises an error when concatenating TZ-aware dataframes and all-NaT dataframes (:issue:`12396`)
+- Bug in :func:`concat` which raises an error when concatenating empty TZ-aware series (:issue:`18447`)
 
 Other
 ^^^^^

pandas/core/indexes/base.py

@@ -2183,17 +2183,19 @@ def _assert_take_fillable(self, values, indices, allow_fill=True,
                               fill_value=None, na_value=np.nan):
         """ Internal method to handle NA filling of take """
         indices = _ensure_platform_int(indices)
-
         # only fill if we are passing a non-None fill_value
         if allow_fill and fill_value is not None:
             if (indices < -1).any():
                 msg = ('When allow_fill=True and fill_value is not None, '
                        'all indices must be >= -1')
                 raise ValueError(msg)
-            taken = values.take(indices)
             mask = indices == -1
-            if mask.any():
-                taken[mask] = na_value
+            if mask.all():
+                taken = np.full(indices.shape, fill_value=na_value)
+            else:
+                taken = values.take(indices)
+                if mask.any():
+                    taken[mask] = na_value
         else:
             taken = values.take(indices)
         return taken

pandas/core/internals.py

@@ -5835,8 +5835,10 @@ def get_reindexed_values(self, empty_dtype, upcasted_na):
                     if len(values) and values[0] is None:
                         fill_value = None
 
-                if getattr(self.block, 'is_datetimetz', False):
-                    pass
+                if getattr(self.block, 'is_datetimetz', False) or \
+                        is_datetimetz(empty_dtype):
+                    missing_arr = np.full(np.prod(self.shape), fill_value)
+                    return DatetimeIndex(missing_arr, dtype=empty_dtype)
                 elif getattr(self.block, 'is_categorical', False):
                     pass
                 elif getattr(self.block, 'is_sparse', False):

pandas/tests/reshape/test_concat.py

@@ -1917,6 +1917,135 @@ def test_concat_tz_series_tzlocal(self):
         tm.assert_series_equal(result, pd.Series(x + y))
         assert result.dtype == 'datetime64[ns, tzlocal()]'
 
+    def test_concat_NaT_dataframes_all_NaT_axis_0(self):
+        # GH 12396
+
+        # tz-naive
+        first = pd.DataFrame([[pd.NaT], [pd.NaT]])
+        second = pd.DataFrame([[pd.NaT]])
+
+        result = pd.concat([first, second], axis=0)
+        expected = pd.DataFrame([pd.NaT, pd.NaT, pd.NaT], index=[0, 1, 0])
+        assert_frame_equal(result, expected)
+
+        # one side timezone-aware
+        # upcasts for mixed case
+        first = pd.DataFrame(pd.Series([pd.NaT, pd.NaT]).dt.tz_localize('UTC'))
+        result = pd.concat([first, second], axis=0)
+        expected = pd.DataFrame(
+            pd.Series([pd.NaT, pd.NaT, pd.NaT]).dt.tz_localize('UTC'),
+            index=[0, 1, 0]
+        )
+        assert_frame_equal(result, expected)
+
+        # both sides timezone-aware
+        # upcasts to tz-aware
+        second = pd.DataFrame(pd.Series([pd.NaT]).dt.tz_localize('UTC'))
+        result = pd.concat([first, second], axis=0)
+        assert_frame_equal(result, expected)
+
+    def test_concat_NaT_dataframes_all_NaT_axis_1(self):
+        # GH 12396
+
+        # tz-naive
+        first = pd.DataFrame([[pd.NaT], [pd.NaT]])
+        second = pd.DataFrame([[pd.NaT]], columns=[1])
+        expected = pd.DataFrame([[pd.NaT, pd.NaT], [pd.NaT, pd.NaT]],
+                                columns=[0, 1])
+        result = pd.concat([first, second], axis=1)
+        assert_frame_equal(result, expected)
+
+        # one side timezone-aware
+        # upcasts result to tz-aware
+        first = pd.DataFrame(pd.Series([pd.NaT, pd.NaT]).dt.tz_localize('UTC'))
+        expected = pd.DataFrame(
+            {0: pd.Series([pd.NaT, pd.NaT]).dt.tz_localize('UTC'),
+             1: pd.Series([pd.NaT, pd.NaT])}
+        )
+        result = pd.concat([first, second], axis=1)
+        assert_frame_equal(result, expected)
+
+        # both sides timezone-aware
+        # upcasts result to tz-aware
+        second[1] = second[1].dt.tz_localize('UTC')
+        expected = pd.DataFrame(
+            {0: pd.Series([pd.NaT, pd.NaT]).dt.tz_localize('UTC'),
+             1: pd.Series([pd.NaT, pd.NaT]).dt.tz_localize('UTC')}
+        )
+        result = pd.concat([first, second], axis=1)
+        assert_frame_equal(result, expected)
+
+    def test_concat_NaT_dataframes_mixed_timestamps_and_NaT(self):
+        # GH 12396
+
+        # tz-naive
+        first = pd.DataFrame([[pd.NaT], [pd.NaT]])
+        second = pd.DataFrame([[pd.Timestamp('2015/01/01')],
+                               [pd.Timestamp('2016/01/01')]],
+                              index=[2, 3])
+        expected = pd.DataFrame([pd.NaT, pd.NaT,
+                                 pd.Timestamp('2015/01/01'),
+                                 pd.Timestamp('2016/01/01')])
+
+        result = pd.concat([first, second], axis=0)
+        assert_frame_equal(result, expected)
+
+        # one side timezone-aware
+        second = second[0].dt.tz_localize('UTC')
+        expected = pd.DataFrame(
+            pd.Series([pd.NaT, pd.NaT,
+                       pd.Timestamp('2015/01/01'),
+                       pd.Timestamp('2016/01/01')]).dt.tz_localize('UTC')
+        )
+        result = pd.concat([first, second], axis=0)
+        assert_frame_equal(result, expected)
+
+    def test_concat_NaT_series_dataframe_all_NaT(self):
+        # GH 12396
+
+        # tz-naive
+        first = pd.Series([pd.NaT, pd.NaT])
+        second = pd.DataFrame([[pd.Timestamp('2015/01/01')],
+                               [pd.Timestamp('2016/01/01')]],
+                              index=[2, 3])
+
+        expected = pd.DataFrame([pd.NaT, pd.NaT,
+                                 pd.Timestamp('2015/01/01'),
+                                 pd.Timestamp('2016/01/01')])
+
+        result = pd.concat([first, second])
+        assert_frame_equal(result, expected)
+
+        # one side timezone-aware
+        second[0] = second[0].dt.tz_localize('UTC')
+        result = pd.concat([first, second])
+
+        expected = pd.DataFrame(
+            pd.Series([pd.NaT, pd.NaT,
+                       pd.Timestamp('2015/01/01'),
+                       pd.Timestamp('2016/01/01')]).dt.tz_localize('UTC')
+        )
+        assert_frame_equal(result, expected)
+
+        # both sides timezone-aware
+        first = first.dt.tz_localize('UTC')
+        result = pd.concat([first, second])
+        assert_frame_equal(result, expected)
+
+        # mixed tz
+        first = pd.DataFrame([[pd.NaT], [pd.NaT]])
+        second = pd.DataFrame([[pd.Timestamp('2015/01/01', tz='UTC')],
+                               [pd.Timestamp('2016/01/01', tz='US/Eastern')]],
+                              index=[2, 3])
+
+        expected = pd.DataFrame([pd.NaT,
+                                 pd.NaT,
+                                 pd.Timestamp('2015/01/01', tz='UTC'),
+                                 pd.Timestamp('2016/01/01', tz='US/Eastern')])
+
+        result = pd.concat([first, second], axis=0)
+        assert_frame_equal(result, expected)
+
     def test_concat_period_series(self):
         x = Series(pd.PeriodIndex(['2015-11-01', '2015-12-01'], freq='D'))
         y = Series(pd.PeriodIndex(['2015-10-01', '2016-01-01'], freq='D'))
@@ -1978,6 +2107,32 @@ def test_concat_empty_series(self):
                            columns=['x', 0])
         tm.assert_frame_equal(res, exp)
 
+        # GH 18447
+        # tz-naive
+        first = Series(pd.to_datetime([], utc=False))
+        second = Series([1, 2, 3])
+        expected = DataFrame([[pd.NaT, 1], [pd.NaT, 2], [pd.NaT, 3]])
+        result = concat([first, second], axis=1)
+        assert_frame_equal(result, expected)
+
+        # timezone-aware
+        first = Series(pd.to_datetime([], utc=True))
+        second = Series([1, 2, 3])
+        expected = DataFrame(
+            {0: pd.Series([pd.NaT, pd.NaT, pd.NaT]).dt.tz_localize('UTC'),
+             1: pd.Series([1, 2, 3])}
+        )
+        result = concat([first, second], axis=1)
+        assert_frame_equal(result, expected)
+
+        # both empty
+        first = Series(pd.to_datetime([], utc=True))
+        second = Series([])
+        result = concat([first, second], axis=1)
+        assert result.size == 0
+        assert result.dtypes[0] == 'datetime64[ns, UTC]'
+        assert result.dtypes[1] == 'float64'
+
     def test_default_index(self):
         # is_series and ignore_index
         s1 = pd.Series([1, 2, 3], name='x')