BUG: Sparse concat may fill fill_value with NaN

closes #9765

Author: sinhrks <[email protected]>

Closes #12966 from sinhrks/sparse_dfconcat and squashes the following commits:

4873e06 [sinhrks] BUG: Sparse concat may fill fill_value with NaN

Author: sinhrks

doc/source/whatsnew/v0.18.1.txt

@@ -111,6 +111,7 @@ These changes conform sparse handling to return the correct types and work to ma
    s.take([1, 2, 3])
 
 - Bug in ``SparseSeries[]`` indexing with ``Ellipsis`` raises ``KeyError`` (:issue:`9467`)
+- Bug in ``SparseArray[]`` indexing with tuples are not handled properly (:issue:`12966`)
 - Bug in ``SparseSeries.loc[]`` with list-like input raises ``TypeError`` (:issue:`10560`)
 - Bug in ``SparseSeries.iloc[]`` with scalar input may raise ``IndexError`` (:issue:`10560`)
 - Bug in ``SparseSeries.loc[]``, ``.iloc[]`` with ``slice`` returns ``SparseArray``, rather than ``SparseSeries`` (:issue:`10560`)
@@ -126,6 +127,8 @@ These changes conform sparse handling to return the correct types and work to ma
 - Bug in ``SparseArray.to_dense()`` does not preserve ``dtype`` (:issue:`10648`)
 - Bug in ``SparseArray.to_dense()`` incorrectly handle ``fill_value`` (:issue:`12797`)
 - Bug in ``pd.concat()`` of ``SparseSeries`` results in dense (:issue:`10536`)
+- Bug in ``pd.concat()`` of ``SparseDataFrame`` incorrectly handle ``fill_value`` (:issue:`9765`)
+- Bug in ``pd.concat()`` of ``SparseDataFrame`` may raise ``AttributeError`` (:issue:`12174`)
 - Bug in ``SparseArray.shift()`` may raise ``NameError`` or ``TypeError`` (:issue:`12908`)
 
 .. _whatsnew_0181.api:

pandas/core/internals.py

@@ -4872,6 +4872,11 @@ def is_null(self):
         values = self.block.values
         if self.block.is_categorical:
             values_flat = values.categories
+        elif self.block.is_sparse:
+            # fill_value is not NaN and have holes
+            if not values._null_fill_value and values.sp_index.ngaps > 0:
+                return False
+            values_flat = values.ravel(order='K')
         else:
             values_flat = values.ravel(order='K')
         total_len = values_flat.shape[0]
@@ -4904,6 +4909,8 @@ def get_reindexed_values(self, empty_dtype, upcasted_na):
                     pass
                 elif getattr(self.block, 'is_categorical', False):
                     pass
+                elif getattr(self.block, 'is_sparse', False):
+                    pass
                 else:
                     missing_arr = np.empty(self.shape, dtype=empty_dtype)
                     missing_arr.fill(fill_value)

pandas/sparse/array.py

@@ -278,14 +278,18 @@ def __getitem__(self, key):
         """
         if com.is_integer(key):
             return self._get_val_at(key)
+        elif isinstance(key, tuple):
+            data_slice = self.values[key]
         else:
             if isinstance(key, SparseArray):
                 key = np.asarray(key)
+
             if hasattr(key, '__len__') and len(self) != len(key):
                 return self.take(key)
             else:
                 data_slice = self.values[key]
-            return self._constructor(data_slice)
+
+        return self._constructor(data_slice)
 
     def __getslice__(self, i, j):
         if i < 0:

pandas/sparse/series.py

@@ -115,9 +115,12 @@ def __init__(self, data=None, index=None, sparse_index=None, kind='block',
         if fastpath:
 
             # data is an ndarray, index is defined
-            data = SingleBlockManager(data, index, fastpath=True)
+
+            if not isinstance(data, SingleBlockManager):
+                data = SingleBlockManager(data, index, fastpath=True)
             if copy:
                 data = data.copy()
+
         else:
 
             if data is None:

pandas/sparse/tests/test_array.py

@@ -347,6 +347,26 @@ def test_getslice(self):
         exp = SparseArray(self.arr.values[:0])
         tm.assert_sp_array_equal(result, exp)
 
+    def test_getslice_tuple(self):
+        dense = np.array([np.nan, 0, 3, 4, 0, 5, np.nan, np.nan, 0])
+
+        sparse = SparseArray(dense)
+        res = sparse[4:, ]
+        exp = SparseArray(dense[4:, ])
+        tm.assert_sp_array_equal(res, exp)
+
+        sparse = SparseArray(dense, fill_value=0)
+        res = sparse[4:, ]
+        exp = SparseArray(dense[4:, ], fill_value=0)
+        tm.assert_sp_array_equal(res, exp)
+
+        with tm.assertRaises(IndexError):
+            sparse[4:, :]
+
+        with tm.assertRaises(IndexError):
+            # check numpy compat
+            dense[4:, :]
+
     def test_binary_operators(self):
         data1 = np.random.randn(20)
         data2 = np.random.randn(20)