move tests and add whatsnew

Author: reidy-p

doc/source/whatsnew/v0.22.0.txt

@@ -24,7 +24,7 @@ Other Enhancements
 
 - Better support for :func:`Dataframe.style.to_excel` output with the ``xlsxwriter`` engine. (:issue:`16149`)
 - :func:`pandas.tseries.frequencies.to_offset` now accepts leading '+' signs e.g. '+1h'. (:issue:`18171`)
--
+- :func:`Series.fillna` now accepts a Series or a dict as a ``value`` (:issue:`17033`)
 
 .. _whatsnew_0220.api_breaking:
 

pandas/tests/frame/test_missing.py

@@ -10,7 +10,7 @@
 
 from pandas.compat import lrange
 from pandas import (DataFrame, Series, Timestamp,
-                    date_range)
+                    date_range, Categorical)
 import pandas as pd
 
 from pandas.util.testing import assert_series_equal, assert_frame_equal
@@ -270,6 +270,80 @@ def test_fillna(self):
                                   pd.Timestamp('2012-11-11 00:00:00+01:00')]})
         assert_frame_equal(df.fillna(method='bfill'), exp)
 
+    def test_na_actions(self):
+
+        cat = pd.Categorical([1, 2, 3, np.nan], categories=[1, 2, 3])
+        vals = ["a", "b", np.nan, "d"]
+        df = pd.DataFrame({"cats": cat, "vals": vals})
+        cat2 = pd.Categorical([1, 2, 3, 3], categories=[1, 2, 3])
+        vals2 = ["a", "b", "b", "d"]
+        df_exp_fill = pd.DataFrame({"cats": cat2, "vals": vals2})
+        cat3 = pd.Categorical([1, 2, 3], categories=[1, 2, 3])
+        vals3 = ["a", "b", np.nan]
+        df_exp_drop_cats = pd.DataFrame({"cats": cat3, "vals": vals3})
+        cat4 = pd.Categorical([1, 2], categories=[1, 2, 3])
+        vals4 = ["a", "b"]
+        df_exp_drop_all = pd.DataFrame({"cats": cat4, "vals": vals4})
+
+        # fillna
+        res = df.fillna(value={"cats": 3, "vals": "b"})
+        tm.assert_frame_equal(res, df_exp_fill)
+
+        def f():
+            df.fillna(value={"cats": 4, "vals": "c"})
+
+        pytest.raises(ValueError, f)
+
+        res = df.fillna(method='pad')
+        tm.assert_frame_equal(res, df_exp_fill)
+
+        res = df.dropna(subset=["cats"])
+        tm.assert_frame_equal(res, df_exp_drop_cats)
+
+        res = df.dropna()
+        tm.assert_frame_equal(res, df_exp_drop_all)
+
+        # make sure that fillna takes missing values into account
+        c = Categorical([np.nan, "b", np.nan], categories=["a", "b"])
+        df = pd.DataFrame({"cats": c, "vals": [1, 2, 3]})
+
+        cat_exp = Categorical(["a", "b", "a"], categories=["a", "b"])
+        df_exp = pd.DataFrame({"cats": cat_exp, "vals": [1, 2, 3]})
+
+        res = df.fillna("a")
+        tm.assert_frame_equal(res, df_exp)
+
+        # GH 14021
+        # np.nan should always be a is a valid filler
+        cat = Categorical([np.nan, 2, np.nan])
+        val = Categorical([np.nan, np.nan, np.nan])
+        df = DataFrame({"cats": cat, "vals": val})
+        res = df.fillna(df.median())
+        v_exp = [np.nan, np.nan, np.nan]
+        df_exp = pd.DataFrame({"cats": [2, 2, 2], "vals": v_exp},
+                              dtype='category')
+        tm.assert_frame_equal(res, df_exp)
+
+        result = df.cats.fillna(np.nan)
+        tm.assert_series_equal(result, df.cats)
+        result = df.vals.fillna(np.nan)
+        tm.assert_series_equal(result, df.vals)
+
+        idx = pd.DatetimeIndex(['2011-01-01 09:00', '2016-01-01 23:45',
+                                '2011-01-01 09:00', pd.NaT, pd.NaT])
+        df = DataFrame({'a': pd.Categorical(idx)})
+        tm.assert_frame_equal(df.fillna(value=pd.NaT), df)
+
+        idx = pd.PeriodIndex(['2011-01', '2011-01', '2011-01',
+                              pd.NaT, pd.NaT], freq='M')
+        df = DataFrame({'a': pd.Categorical(idx)})
+        tm.assert_frame_equal(df.fillna(value=pd.NaT), df)
+
+        idx = pd.TimedeltaIndex(['1 days', '2 days',
+                                 '1 days', pd.NaT, pd.NaT])
+        df = pd.DataFrame({'a': pd.Categorical(idx)})
+        tm.assert_frame_equal(df.fillna(value=pd.NaT), df)
+
     def test_fillna_downcast(self):
         # GH 15277
         # infer int64 from float64

pandas/tests/series/test_missing.py

@@ -12,7 +12,8 @@
 import pandas as pd
 
 from pandas import (Series, DataFrame, isna, date_range,
-                    MultiIndex, Index, Timestamp, NaT, IntervalIndex)
+                    MultiIndex, Index, Timestamp, NaT, IntervalIndex,
+                    Categorical)
 from pandas.compat import range
 from pandas._libs.tslib import iNaT
 from pandas.core.series import remove_na
@@ -363,6 +364,50 @@ def test_fillna_raise(self):
                 with pytest.raises(ValueError):
                     s.fillna(1, limit=limit, method=method)
 
+    @pytest.mark.parametrize('fill_value, expected_output', [
+        ('a', ['a', 'a', 'b', 'a', 'a']),
+        ({1: 'a', 3: 'b', 4: 'b'}, ['a', 'a', 'b', 'b', 'b']),
+        ({1: 'a'}, ['a', 'a', 'b', np.nan, np.nan]),
+        ({1: 'a', 3: 'b'}, ['a', 'a', 'b', 'b', np.nan]),
+        (Series('a'), ['a', np.nan, 'b', np.nan, np.nan]),
+        (Series('a', index=[1]), ['a', 'a', 'b', np.nan, np.nan]),
+        (Series({1: 'a', 3: 'b'}), ['a', 'a', 'b', 'b', np.nan]),
+        (Series(['a', 'b'], index=[3, 4]), ['a', np.nan, 'b', 'a', 'b'])
+    ])
+    def test_fillna_categorical(self, fill_value, expected_output):
+        # GH 17033
+        # Test fillna for a Categorical series
+        data = ['a', np.nan, 'b', np.nan, np.nan]
+        s = Series(Categorical(data, categories=['a', 'b']))
+        exp = Series(Categorical(expected_output, categories=['a', 'b']))
+        tm.assert_series_equal(s.fillna(fill_value), exp)
+
+    def test_fillna_categorical_raise(self):
+        data = ['a', np.nan, 'b', np.nan, np.nan]
+        s = Series(Categorical(data, categories=['a', 'b']))
+
+        with tm.assert_raises_regex(ValueError,
+                                    "fill value must be in categories"):
+            s.fillna('d')
+
+        with tm.assert_raises_regex(ValueError,
+                                    "fill value must be in categories"):
+            s.fillna(Series('d'))
+
+        with tm.assert_raises_regex(ValueError,
+                                    "fill value must be in categories"):
+            s.fillna({1: 'd', 3: 'a'})
+
+        with tm.assert_raises_regex(TypeError,
+                                    '"value" parameter must be a scalar or '
+                                    'dict, but you passed a "list"'):
+            s.fillna(['a', 'b'])
+
+        with tm.assert_raises_regex(TypeError,
+                                    '"value" parameter must be a scalar or '
+                                    'dict, but you passed a "tuple"'):
+            s.fillna(('a', 'b'))
+
     def test_fillna_nat(self):
         series = Series([0, 1, 2, iNaT], dtype='M8[ns]')
 

pandas/tests/test_categorical.py

@@ -4527,119 +4527,6 @@ def test_numpy_reshape(self):
             tm.assert_raises_regex(ValueError, msg, np.reshape,
                                    cat, cat.shape, order='F')
 
-    def test_na_actions(self):
-
-        cat = pd.Categorical([1, 2, 3, np.nan], categories=[1, 2, 3])
-        vals = ["a", "b", np.nan, "d"]
-        df = pd.DataFrame({"cats": cat, "vals": vals})
-        cat2 = pd.Categorical([1, 2, 3, 3], categories=[1, 2, 3])
-        vals2 = ["a", "b", "b", "d"]
-        df_exp_fill = pd.DataFrame({"cats": cat2, "vals": vals2})
-        cat3 = pd.Categorical([1, 2, 3], categories=[1, 2, 3])
-        vals3 = ["a", "b", np.nan]
-        df_exp_drop_cats = pd.DataFrame({"cats": cat3, "vals": vals3})
-        cat4 = pd.Categorical([1, 2], categories=[1, 2, 3])
-        vals4 = ["a", "b"]
-        df_exp_drop_all = pd.DataFrame({"cats": cat4, "vals": vals4})
-
-        # fillna
-        res = df.fillna(value={"cats": 3, "vals": "b"})
-        tm.assert_frame_equal(res, df_exp_fill)
-
-        def f():
-            df.fillna(value={"cats": 4, "vals": "c"})
-
-        pytest.raises(ValueError, f)
-
-        res = df.fillna(method='pad')
-        tm.assert_frame_equal(res, df_exp_fill)
-
-        res = df.dropna(subset=["cats"])
-        tm.assert_frame_equal(res, df_exp_drop_cats)
-
-        res = df.dropna()
-        tm.assert_frame_equal(res, df_exp_drop_all)
-
-        # make sure that fillna takes missing values into account
-        c = Categorical([np.nan, "b", np.nan], categories=["a", "b"])
-        df = pd.DataFrame({"cats": c, "vals": [1, 2, 3]})
-
-        cat_exp = Categorical(["a", "b", "a"], categories=["a", "b"])
-        df_exp = pd.DataFrame({"cats": cat_exp, "vals": [1, 2, 3]})
-
-        res = df.fillna("a")
-        tm.assert_frame_equal(res, df_exp)
-
-        # GH 14021
-        # np.nan should always be a is a valid filler
-        cat = Categorical([np.nan, 2, np.nan])
-        val = Categorical([np.nan, np.nan, np.nan])
-        df = DataFrame({"cats": cat, "vals": val})
-        res = df.fillna(df.median())
-        v_exp = [np.nan, np.nan, np.nan]
-        df_exp = pd.DataFrame({"cats": [2, 2, 2], "vals": v_exp},
-                              dtype='category')
-        tm.assert_frame_equal(res, df_exp)
-
-        result = df.cats.fillna(np.nan)
-        tm.assert_series_equal(result, df.cats)
-        result = df.vals.fillna(np.nan)
-        tm.assert_series_equal(result, df.vals)
-
-        idx = pd.DatetimeIndex(['2011-01-01 09:00', '2016-01-01 23:45',
-                                '2011-01-01 09:00', pd.NaT, pd.NaT])
-        df = DataFrame({'a': pd.Categorical(idx)})
-        tm.assert_frame_equal(df.fillna(value=pd.NaT), df)
-
-        idx = pd.PeriodIndex(['2011-01', '2011-01', '2011-01',
-                              pd.NaT, pd.NaT], freq='M')
-        df = DataFrame({'a': pd.Categorical(idx)})
-        tm.assert_frame_equal(df.fillna(value=pd.NaT), df)
-
-        idx = pd.TimedeltaIndex(['1 days', '2 days',
-                                 '1 days', pd.NaT, pd.NaT])
-        df = pd.DataFrame({'a': pd.Categorical(idx)})
-        tm.assert_frame_equal(df.fillna(value=pd.NaT), df)
-
-    @pytest.mark.parametrize('fill_value, expected_output', [
-        ('a', ['a', 'a', 'b', 'a', 'a']),
-        ({1: 'a', 3: 'b', 4: 'b'}, ['a', 'a', 'b', 'b', 'b']),
-        ({1: 'a'}, ['a', 'a', 'b', np.nan, np.nan]),
-        ({1: 'a', 3: 'b'}, ['a', 'a', 'b', 'b', np.nan]),
-        (pd.Series('a'), ['a', np.nan, 'b', np.nan, np.nan]),
-        (pd.Series('a', index=[1]), ['a', 'a', 'b', np.nan, np.nan]),
-        (pd.Series({1: 'a', 3: 'b'}), ['a', 'a', 'b', 'b', np.nan]),
-        (pd.Series(['a', 'b'], index=[3, 4]), ['a', np.nan, 'b', 'a', 'b'])
-    ])
-    def test_fillna_series_categorical(self, fill_value, expected_output):
-        # GH 17033
-        # Test fillna for a Categorical series
-        data = ['a', np.nan, 'b', np.nan, np.nan]
-        s = pd.Series(pd.Categorical(data, categories=['a', 'b']))
-        exp = pd.Series(pd.Categorical(expected_output, categories=['a', 'b']))
-        tm.assert_series_equal(s.fillna(fill_value), exp)
-
-    def test_fillna_series_categorical_errormsg(self):
-        data = ['a', np.nan, 'b', np.nan, np.nan]
-        s = pd.Series(pd.Categorical(data, categories=['a', 'b']))
-
-        with tm.assert_raises_regex(ValueError,
-                                    "fill value must be in categories"):
-            s.fillna('d')
-
-        with tm.assert_raises_regex(ValueError,
-                                    "fill value must be in categories"):
-            s.fillna(pd.Series('d'))
-
-        with tm.assert_raises_regex(ValueError,
-                                    "fill value must be in categories"):
-            s.fillna({1: 'd', 3: 'a'})
-
-        with tm.assert_raises_regex(TypeError,
-                                    '"value" parameter must be a scalar or '
-                                    'dict, but you passed a "list"'):
-            s.fillna(['a', 'b'])
-
     def test_astype_to_other(self):
 
         s = self.cat['value_group']