Backport PR #36051: BUG: frame._item_cache not cleared when Series is altered (#36072)

Co-authored-by: jbrockmendel <[email protected]>

Author: meeseeksmachine

doc/source/whatsnew/v1.1.2.rst

@@ -31,6 +31,7 @@ Bug fixes
 - Bug in :meth:`DataFrame.apply` with ``result_type="reduce"`` returning with incorrect index (:issue:`35683`)
 - Bug in :meth:`DateTimeIndex.format` and :meth:`PeriodIndex.format` with ``name=True`` setting the first item to ``"None"`` where it should bw ``""`` (:issue:`35712`)
 - Bug in :meth:`Float64Index.__contains__` incorrectly raising ``TypeError`` instead of returning ``False`` (:issue:`35788`)
+- Bug in :class:`DataFrame` indexing returning an incorrect :class:`Series` in some cases when the series has been altered and a cache not invalidated (:issue:`36051`)
 
 .. ---------------------------------------------------------------------------
 

pandas/core/generic.py

@@ -3233,6 +3233,10 @@ def _maybe_update_cacher(
                 if len(self) == len(ref):
                     # otherwise, either self or ref has swapped in new arrays
                     ref._maybe_cache_changed(cacher[0], self)
+                else:
+                    # GH#33675 we have swapped in a new array, so parent
+                    #  reference to self is now invalid
+                    ref._item_cache.pop(cacher[0], None)
 
         if verify_is_copy:
             self._check_setitem_copy(stacklevel=5, t="referant")

pandas/tests/frame/test_missing.py

@@ -135,13 +135,20 @@ def test_drop_and_dropna_caching(self):
         df2 = df.copy()
         df["A"].dropna()
         tm.assert_series_equal(df["A"], original)
-        return_value = df["A"].dropna(inplace=True)
-        tm.assert_series_equal(df["A"], expected)
+
+        ser = df["A"]
+        return_value = ser.dropna(inplace=True)
+        tm.assert_series_equal(ser, expected)
+        tm.assert_series_equal(df["A"], original)
         assert return_value is None
+
         df2["A"].drop([1])
         tm.assert_series_equal(df2["A"], original)
-        return_value = df2["A"].drop([1], inplace=True)
-        tm.assert_series_equal(df2["A"], original.drop([1]))
+
+        ser = df2["A"]
+        return_value = ser.drop([1], inplace=True)
+        tm.assert_series_equal(ser, original.drop([1]))
+        tm.assert_series_equal(df2["A"], original)
         assert return_value is None
 
     def test_dropna_corner(self, float_frame):

pandas/tests/indexing/test_chaining_and_caching.py

@@ -81,6 +81,21 @@ def test_setitem_cache_updating(self):
         tm.assert_frame_equal(out, expected)
         tm.assert_series_equal(out["A"], expected["A"])
 
+    def test_altering_series_clears_parent_cache(self):
+        # GH #33675
+        df = pd.DataFrame([[1, 2], [3, 4]], index=["a", "b"], columns=["A", "B"])
+        ser = df["A"]
+
+        assert "A" in df._item_cache
+
+        # Adding a new entry to ser swaps in a new array, so "A" needs to
+        #  be removed from df._item_cache
+        ser["c"] = 5
+        assert len(ser) == 3
+        assert "A" not in df._item_cache
+        assert df["A"] is not ser
+        assert len(df["A"]) == 2
+
 
 class TestChaining:
     def test_setitem_chained_setfault(self):