BUG: categorical unpickle to use _coerce_indexer_dtype

follow up for #13080 to use `_coerce_indexer_dtype`.

Author: sinhrks <[email protected]>

Closes #13426 from sinhrks/cat_pickle and squashes the following commits:

e8702f8 [sinhrks] BUG: categorical unpickle to use _coerce_indexer_dtype

Author: sinhrks

pandas/core/categorical.py

@@ -999,11 +999,12 @@ def __setstate__(self, state):
             raise Exception('invalid pickle state')
 
         # Provide compatibility with pre-0.15.0 Categoricals.
-        if '_codes' not in state and 'labels' in state:
-            state['_codes'] = state.pop('labels').astype(np.int8)
         if '_categories' not in state and '_levels' in state:
             state['_categories'] = self._validate_categories(state.pop(
                 '_levels'))
+        if '_codes' not in state and 'labels' in state:
+            state['_codes'] = _coerce_indexer_dtype(state.pop('labels'),
+                                                    state['_categories'])
 
         # 0.16.0 ordered change
         if '_ordered' not in state:

pandas/io/tests/data/legacy_pickle/0.14.1/0.14.1_x86_64_darwin_2.7.12.pickle

@@ -80,6 +80,7 @@ def create_data():
                     [u'one', u'two', u'one', u'two', u'one',
                      u'two', u'one', u'two']])),
         names=[u'first', u'second']))
+
     series = dict(float=Series(data[u'A']),
                   int=Series(data[u'B']),
                   mixed=Series(data[u'E']),
@@ -135,6 +136,10 @@ def create_data():
                            items=[u'A', u'B', u'A']),
                  mixed_dup=mixed_dup_panel)
 
+    cat = dict(int8=Categorical(list('abcdefg')),
+               int16=Categorical(np.arange(1000)),
+               int32=Categorical(np.arange(10000)))
+
     return dict(series=series,
                 frame=frame,
                 panel=panel,
@@ -143,7 +148,8 @@ def create_data():
                 mi=mi,
                 sp_series=dict(float=_create_sp_series(),
                                ts=_create_sp_tsseries()),
-                sp_frame=dict(float=_create_sp_frame()))
+                sp_frame=dict(float=_create_sp_frame()),
+                cat=cat)
 
 
 def create_pickle_data():

pandas/io/tests/data/legacy_pickle/0.15.0/0.15.0_x86_64_darwin_2.7.12.pickle

@@ -109,8 +109,12 @@ def compare_series_dt_tz(self, result, expected, typ, version):
             tm.assert_series_equal(result, expected)
 
     def compare_series_cat(self, result, expected, typ, version):
-        # Categorical.ordered is changed in < 0.16.0
-        if LooseVersion(version) < '0.16.0':
+        # Categorical dtype is added in 0.15.0
+        # ordered is changed in 0.16.0
+        if LooseVersion(version) < '0.15.0':
+            tm.assert_series_equal(result, expected, check_dtype=False,
+                                   check_categorical=False)
+        elif LooseVersion(version) < '0.16.0':
             tm.assert_series_equal(result, expected, check_categorical=False)
         else:
             tm.assert_series_equal(result, expected)
@@ -125,8 +129,12 @@ def compare_frame_dt_mixed_tzs(self, result, expected, typ, version):
             tm.assert_frame_equal(result, expected)
 
     def compare_frame_cat_onecol(self, result, expected, typ, version):
-        # Categorical.ordered is changed in < 0.16.0
-        if LooseVersion(version) < '0.16.0':
+        # Categorical dtype is added in 0.15.0
+        # ordered is changed in 0.16.0
+        if LooseVersion(version) < '0.15.0':
+            tm.assert_frame_equal(result, expected, check_dtype=False,
+                                  check_categorical=False)
+        elif LooseVersion(version) < '0.16.0':
             tm.assert_frame_equal(result, expected, check_categorical=False)
         else:
             tm.assert_frame_equal(result, expected)