Merge pull request #139 from pandas-dev/master

Sync Fork from Upstream Repo

Author: sthagen

.github/workflows/ci.yml

@@ -155,10 +155,7 @@ jobs:
       run: |
         source activate pandas-dev
 
-        pytest pandas/tests/frame/methods
-        pytest pandas/tests/frame/test_constructors.py
-        pytest pandas/tests/frame/test_*
-        pytest pandas/tests/frame/test_reductions.py
+        pytest pandas/tests/frame/
         pytest pandas/tests/reductions/
         pytest pandas/tests/generic/test_generic.py
         pytest pandas/tests/arithmetic/
@@ -170,10 +167,6 @@ jobs:
         pytest pandas/tests/series/test_*
 
         # indexing subset (temporary since other tests don't pass yet)
-        pytest pandas/tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_setitem_boolean
-        pytest pandas/tests/frame/indexing/test_where.py
-        pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_multi_index
-        pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_listlike_indexer_duplicate_columns
         pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_astype_assignment_with_dups
         pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_frame_setitem_multi_column
 
@@ -185,6 +178,12 @@ jobs:
         pytest pandas/tests/dtypes/
         pytest pandas/tests/generic/
         pytest pandas/tests/indexes/
+        pytest pandas/tests/io/test_* -m "not slow and not clipboard"
+        pytest pandas/tests/io/excel/ -m "not slow and not clipboard"
+        pytest pandas/tests/io/formats/ -m "not slow and not clipboard"
+        pytest pandas/tests/io/parser/ -m "not slow and not clipboard"
+        pytest pandas/tests/io/sas/ -m "not slow and not clipboard"
+        pytest pandas/tests/io/xml/ -m "not slow and not clipboard"
         pytest pandas/tests/libs/
         pytest pandas/tests/plotting/
         pytest pandas/tests/scalar/

pandas/core/frame.py

@@ -6464,6 +6464,57 @@ def swaplevel(self, i: Axis = -2, j: Axis = -1, axis: Axis = 0) -> DataFrame:
         Returns
         -------
         DataFrame
+
+        Examples
+        --------
+        >>> df = pd.DataFrame(
+        ...     {"Grade": ["A", "B", "A", "C"]},
+        ...     index=[
+        ...         ["Final exam", "Final exam", "Coursework", "Coursework"],
+        ...         ["History", "Geography", "History", "Geography"],
+        ...         ["January", "February", "March", "April"],
+        ...     ],
+        ... )
+        >>> df
+                                            Grade
+        Final exam  History     January      A
+                    Geography   February     B
+        Coursework  History     March        A
+                    Geography   April        C
+
+        In the following example, we will swap the levels of the indices.
+        Here, we will swap the levels column-wise, but levels can be swapped row-wise
+        in a similar manner. Note that column-wise is the default behaviour.
+        By not supplying any arguments for i and j, we swap the last and second to
+        last indices.
+
+        >>> df.swaplevel()
+                                            Grade
+        Final exam  January     History         A
+                    February    Geography       B
+        Coursework  March       History         A
+                    April       Geography       C
+
+        By supplying one argument, we can choose which index to swap the last
+        index with. We can for example swap the first index with the last one as
+        follows.
+
+        >>> df.swaplevel(0)
+                                            Grade
+        January     History     Final exam      A
+        February    Geography   Final exam      B
+        March       History     Coursework      A
+        April       Geography   Coursework      C
+
+        We can also define explicitly which indices we want to swap by supplying values
+        for both i and j. Here, we for example swap the first and second indices.
+
+        >>> df.swaplevel(0, 1)
+                                            Grade
+        History     Final exam  January         A
+        Geography   Final exam  February        B
+        History     Coursework  March           A
+        Geography   Coursework  April           C
         """
         result = self.copy()
 

pandas/core/indexers.py

@@ -235,7 +235,7 @@ def validate_indices(indices: np.ndarray, n: int) -> None:
 # Indexer Conversion
 
 
-def maybe_convert_indices(indices, n: int):
+def maybe_convert_indices(indices, n: int, verify: bool = True):
     """
     Attempt to convert indices into valid, positive indices.
 
@@ -248,6 +248,8 @@ def maybe_convert_indices(indices, n: int):
         Array of indices that we are to convert.
     n : int
         Number of elements in the array that we are indexing.
+    verify : bool, default True
+        Check that all entries are between 0 and n - 1, inclusive.
 
     Returns
     -------
@@ -273,9 +275,10 @@ def maybe_convert_indices(indices, n: int):
         indices = indices.copy()
         indices[mask] += n
 
-    mask = (indices >= n) | (indices < 0)
-    if mask.any():
-        raise IndexError("indices are out-of-bounds")
+    if verify:
+        mask = (indices >= n) | (indices < 0)
+        if mask.any():
+            raise IndexError("indices are out-of-bounds")
     return indices
 
 

pandas/core/internals/array_manager.py

@@ -861,7 +861,7 @@ def iset(self, loc: Union[int, slice, np.ndarray], value):
             # DataFrame into 1D array when loc is an integer
             if isinstance(value, np.ndarray) and value.ndim == 2:
                 assert value.shape[1] == 1
-                value = value[0, :]
+                value = value[:, 0]
 
             # TODO we receive a datetime/timedelta64 ndarray from DataFrame._iset_item
             # but we should avoid that and pass directly the proper array
@@ -1021,7 +1021,7 @@ def _reindex_indexer(
 
         return type(self)(new_arrays, new_axes, verify_integrity=False)
 
-    def take(self, indexer, axis: int = 1, verify: bool = True, convert: bool = True):
+    def take(self: T, indexer, axis: int = 1, verify: bool = True) -> T:
         """
         Take items along any axis.
         """
@@ -1034,12 +1034,7 @@ def take(self, indexer, axis: int = 1, verify: bool = True, convert: bool = True
         )
 
         n = self.shape_proper[axis]
-        if convert:
-            indexer = maybe_convert_indices(indexer, n)
-
-        if verify:
-            if ((indexer == -1) | (indexer >= n)).any():
-                raise Exception("Indices must be nonzero and less than the axis length")
+        indexer = maybe_convert_indices(indexer, n, verify=verify)
 
         new_labels = self._axes[axis].take(indexer)
         return self._reindex_indexer(
@@ -1168,10 +1163,6 @@ def axes(self):
     def index(self) -> Index:
         return self._axes[0]
 
-    @property
-    def array(self):
-        return self.arrays[0]
-
     @property
     def dtype(self):
         return self.array.dtype

pandas/core/internals/base.py

@@ -126,6 +126,13 @@ def isna(self: T, func) -> T:
 class SingleDataManager(DataManager):
     ndim = 1
 
+    @property
+    def array(self):
+        """
+        Quick access to the backing array of the Block or SingleArrayManager.
+        """
+        return self.arrays[0]  # type: ignore[attr-defined]
+
 
 def interleaved_dtype(dtypes: List[DtypeObj]) -> Optional[DtypeObj]:
     """

pandas/core/internals/managers.py

@@ -1491,23 +1491,29 @@ def _make_na_block(self, placement, fill_value=None):
         block_values.fill(fill_value)
         return new_block(block_values, placement=placement, ndim=block_values.ndim)
 
-    def take(self, indexer, axis: int = 1, verify: bool = True, convert: bool = True):
+    def take(self: T, indexer, axis: int = 1, verify: bool = True) -> T:
         """
         Take items along any axis.
+
+        indexer : np.ndarray or slice
+        axis : int, default 1
+        verify : bool, default True
+            Check that all entries are between 0 and len(self) - 1, inclusive.
+            Pass verify=False if this check has been done by the caller.
+
+        Returns
+        -------
+        BlockManager
         """
+        # We have 6 tests that get here with a slice
         indexer = (
             np.arange(indexer.start, indexer.stop, indexer.step, dtype="int64")
             if isinstance(indexer, slice)
             else np.asanyarray(indexer, dtype="int64")
         )
 
         n = self.shape[axis]
-        if convert:
-            indexer = maybe_convert_indices(indexer, n)
-
-        if verify:
-            if ((indexer == -1) | (indexer >= n)).any():
-                raise Exception("Indices must be nonzero and less than the axis length")
+        indexer = maybe_convert_indices(indexer, n, verify=verify)
 
         new_labels = self.axes[axis].take(indexer)
         return self.reindex_indexer(

pandas/tests/frame/indexing/test_indexing.py

@@ -8,6 +8,7 @@
 import pytest
 
 from pandas._libs import iNaT
+import pandas.util._test_decorators as td
 
 from pandas.core.dtypes.common import is_integer
 
@@ -534,6 +535,7 @@ def test_getitem_setitem_integer_slice_keyerrors(self):
         with pytest.raises(KeyError, match=r"^3$"):
             df2.loc[3:11] = 0
 
+    @td.skip_array_manager_invalid_test  # already covered in test_iloc_col_slice_view
     def test_fancy_getitem_slice_mixed(self, float_frame, float_string_frame):
         sliced = float_string_frame.iloc[:, -3:]
         assert sliced["D"].dtype == np.float64
@@ -592,6 +594,7 @@ def test_getitem_fancy_scalar(self, float_frame):
             for idx in f.index[::5]:
                 assert ix[idx, col] == ts[idx]
 
+    @td.skip_array_manager_invalid_test  # TODO(ArrayManager) rewrite not using .values
     def test_setitem_fancy_scalar(self, float_frame):
         f = float_frame
         expected = float_frame.copy()
@@ -631,6 +634,7 @@ def test_getitem_fancy_boolean(self, float_frame):
         expected = f.reindex(index=f.index[boolvec], columns=["C", "D"])
         tm.assert_frame_equal(result, expected)
 
+    @td.skip_array_manager_invalid_test  # TODO(ArrayManager) rewrite not using .values
     def test_setitem_fancy_boolean(self, float_frame):
         # from 2d, set with booleans
         frame = float_frame.copy()
@@ -990,21 +994,29 @@ def test_iloc_row(self):
         expected = df.loc[8:14]
         tm.assert_frame_equal(result, expected)
 
+        # list of integers
+        result = df.iloc[[1, 2, 4, 6]]
+        expected = df.reindex(df.index[[1, 2, 4, 6]])
+        tm.assert_frame_equal(result, expected)
+
+    def test_iloc_row_slice_view(self, using_array_manager):
+        df = DataFrame(np.random.randn(10, 4), index=range(0, 20, 2))
+        original = df.copy()
+
         # verify slice is view
         # setting it makes it raise/warn
+        subset = df.iloc[slice(4, 8)]
+
         msg = r"\nA value is trying to be set on a copy of a slice from a DataFrame"
         with pytest.raises(com.SettingWithCopyError, match=msg):
-            result[2] = 0.0
+            subset[2] = 0.0
 
-        exp_col = df[2].copy()
-        exp_col[4:8] = 0.0
+        exp_col = original[2].copy()
+        # TODO(ArrayManager) verify it is expected that the original didn't change
+        if not using_array_manager:
+            exp_col[4:8] = 0.0
         tm.assert_series_equal(df[2], exp_col)
 
-        # list of integers
-        result = df.iloc[[1, 2, 4, 6]]
-        expected = df.reindex(df.index[[1, 2, 4, 6]])
-        tm.assert_frame_equal(result, expected)
-
     def test_iloc_col(self):
 
         df = DataFrame(np.random.randn(4, 10), columns=range(0, 20, 2))
@@ -1022,19 +1034,32 @@ def test_iloc_col(self):
         expected = df.loc[:, 8:14]
         tm.assert_frame_equal(result, expected)
 
-        # verify slice is view
-        # and that we are setting a copy
-        msg = r"\nA value is trying to be set on a copy of a slice from a DataFrame"
-        with pytest.raises(com.SettingWithCopyError, match=msg):
-            result[8] = 0.0
-
-        assert (df[8] == 0).all()
-
         # list of integers
         result = df.iloc[:, [1, 2, 4, 6]]
         expected = df.reindex(columns=df.columns[[1, 2, 4, 6]])
         tm.assert_frame_equal(result, expected)
 
+    def test_iloc_col_slice_view(self, using_array_manager):
+        df = DataFrame(np.random.randn(4, 10), columns=range(0, 20, 2))
+        original = df.copy()
+        subset = df.iloc[:, slice(4, 8)]
+
+        if not using_array_manager:
+            # verify slice is view
+            # and that we are setting a copy
+            msg = r"\nA value is trying to be set on a copy of a slice from a DataFrame"
+            with pytest.raises(com.SettingWithCopyError, match=msg):
+                subset[8] = 0.0
+
+            assert (df[8] == 0).all()
+        else:
+            # TODO(ArrayManager) verify this is the desired behaviour
+            subset[8] = 0.0
+            # subset changed
+            assert (subset[8] == 0).all()
+            # but df itself did not change (setitem replaces full column)
+            tm.assert_frame_equal(df, original)
+
     def test_loc_duplicates(self):
         # gh-17105
 
@@ -1218,7 +1243,7 @@ def test_setitem(self, uint64_frame):
         )
 
 
-def test_object_casting_indexing_wraps_datetimelike():
+def test_object_casting_indexing_wraps_datetimelike(using_array_manager):
     # GH#31649, check the indexing methods all the way down the stack
     df = DataFrame(
         {
@@ -1240,6 +1265,10 @@ def test_object_casting_indexing_wraps_datetimelike():
     assert isinstance(ser.values[1], Timestamp)
     assert isinstance(ser.values[2], pd.Timedelta)
 
+    if using_array_manager:
+        # remainder of the test checking BlockManager internals
+        return
+
     mgr = df._mgr
     mgr._rebuild_blknos_and_blklocs()
     arr = mgr.fast_xs(0)

pandas/tests/frame/indexing/test_insert.py

@@ -72,11 +72,17 @@ def test_insert_with_columns_dups(self):
         )
         tm.assert_frame_equal(df, exp)
 
-    def test_insert_item_cache(self):
+    def test_insert_item_cache(self, using_array_manager):
         df = DataFrame(np.random.randn(4, 3))
         ser = df[0]
 
-        with tm.assert_produces_warning(PerformanceWarning):
+        if using_array_manager:
+            expected_warning = None
+        else:
+            # with BlockManager warn about high fragmentation of single dtype
+            expected_warning = PerformanceWarning
+
+        with tm.assert_produces_warning(expected_warning):
             for n in range(100):
                 df[n + 3] = df[1] * n