provide proper index coercion with _shallow_copy for insert,delete,append operations

Author: jreback

pandas/core/index.py

@@ -89,7 +89,6 @@ class Index(IndexOpsMixin, PandasObject):
     _left_indexer = _algos.left_join_indexer_object
     _inner_indexer = _algos.inner_join_indexer_object
     _outer_indexer = _algos.outer_join_indexer_object
-
     _box_scalars = False
 
     _typ = 'index'
@@ -204,6 +203,17 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None, fastpath=False,
 
     @classmethod
     def _simple_new(cls, values, name=None, **kwargs):
+        """
+        we require the we have a dtype compat for the values
+        if we are passed a non-dtype compat, then coerce using the constructor
+
+        Must be careful not to recurse.
+        """
+        if not hasattr(values, 'dtype'):
+            values = np.array(values,copy=False)
+            if is_object_dtype(values):
+                values = cls(values, name=name, **kwargs).values
+
         result = object.__new__(cls)
         result._data = values
         result.name = name
@@ -341,15 +351,41 @@ def view(self, cls=None):
             result._id = self._id
         return result
 
-    def _shallow_copy(self, values=None, **kwargs):
-        """ create a new Index, don't copy the data, use the same object attributes
-            with passed in attributes taking precedence """
+    def _shallow_copy(self, values=None, infer=False, **kwargs):
+        """
+        create a new Index, don't copy the data, use the same object attributes
+        with passed in attributes taking precedence
+
+        *this is an internal non-public method*
+
+        Parameters
+        ----------
+        values : the values to create the new Index, optional
+        infer : boolean, default False
+            if True, infer the new type of the passed values
+        kwargs : updates the default attributes for this Index
+        """
         if values is None:
             values = self.values
         attributes = self._get_attributes_dict()
         attributes.update(kwargs)
+
+        if infer:
+            attributes['copy'] = False
+            return Index(values, **attributes)
+
         return self.__class__._simple_new(values,**attributes)
 
+    def _coerce_scalar_to_index(self, item):
+        """
+        we need to coerce a scalar to a compat for our index type
+
+        Parameters
+        ----------
+        item : scalar item to coerce
+        """
+        return Index([item], dtype=self.dtype, **self._get_attributes_dict())
+
     def copy(self, names=None, name=None, dtype=None, deep=False):
         """
         Make a copy of this object.  Name and dtype sets those attributes on
@@ -1132,7 +1168,9 @@ def append(self, other):
         appended : Index
         """
         to_concat, name = self._ensure_compat_append(other)
-        return Index(np.concatenate(to_concat), name=name)
+        attribs = self._get_attributes_dict()
+        attribs['name'] = name
+        return self._shallow_copy(np.concatenate(to_concat), infer=True, **attribs)
 
     @staticmethod
     def _ensure_compat_concat(indexes):
@@ -1548,16 +1586,12 @@ def sym_diff(self, other, result_name=None):
         other, result_name_update = self._convert_can_do_setop(other)
         if result_name is None:
             result_name = result_name_update
-        the_diff_sorted = sorted(set((self.difference(other)).union(other.difference(self))))
-        if isinstance(self, MultiIndex):
-            # multiindexes don't currently work well with _shallow_copy & can't be supplied a name
-            return Index(the_diff_sorted, **self._get_attributes_dict())
-        else:
-            # convert list to Index and pull values out - required for subtypes such as PeriodIndex
-            diff_array = Index(the_diff_sorted).values
-            return self._shallow_copy(diff_array, name=result_name)
-
-
+        the_diff = sorted(set((self.difference(other)).union(other.difference(self))))
+        attribs = self._get_attributes_dict()
+        attribs['name'] = result_name
+        if 'freq' in attribs:
+            attribs['freq'] = None
+        return self._shallow_copy(the_diff, infer=True, **attribs)
 
     def get_loc(self, key, method=None):
         """
@@ -2535,7 +2569,8 @@ def delete(self, loc):
         -------
         new_index : Index
         """
-        return self._shallow_copy(np.delete(self._data, loc))
+        attribs = self._get_attributes_dict()
+        return self._shallow_copy(np.delete(self._data, loc), **attribs)
 
     def insert(self, loc, item):
         """
@@ -2551,12 +2586,13 @@ def insert(self, loc, item):
         -------
         new_index : Index
         """
-        indexes=[self[:loc],
-                 Index([item]),
-                 self[loc:]]
-
-        return indexes[0].append(indexes[1]).append(indexes[2])
+        _self = np.asarray(self)
+        item = self._coerce_scalar_to_index(item).values
 
+        idx = np.concatenate(
+            (_self[:loc], item, _self[loc:]))
+        attribs = self._get_attributes_dict()
+        return self._shallow_copy(idx, infer=True, **attribs)
 
     def drop(self, labels, errors='raise'):
         """
@@ -3687,7 +3723,7 @@ class MultiIndex(Index):
         Level of sortedness (must be lexicographically sorted by that
         level)
     names : optional sequence of objects
-        Names for each of the index levels.
+        Names for each of the index levels. (name is accepted for compat)
     copy : boolean, default False
         Copy the meta-data
     verify_integrity : boolean, default True
@@ -3703,8 +3739,11 @@ class MultiIndex(Index):
     rename = Index.set_names
 
     def __new__(cls, levels=None, labels=None, sortorder=None, names=None,
-                copy=False, verify_integrity=True, _set_identity=True, **kwargs):
+                copy=False, verify_integrity=True, _set_identity=True, name=None, **kwargs):
 
+        # compat with Index
+        if name is not None:
+            names = name
         if levels is None or labels is None:
             raise TypeError("Must pass both levels and labels")
         if len(levels) != len(labels):
@@ -4013,7 +4052,12 @@ def view(self, cls=None):
         result._id = self._id
         return result
 
-    _shallow_copy = view
+    def _shallow_copy(self, values=None, infer=False, **kwargs):
+        if values is not None:
+            if 'name' in kwargs:
+                kwargs['names'] = kwargs.pop('name',None)
+            return MultiIndex.from_tuples(values, **kwargs)
+        return self.view()
 
     @cache_readonly
     def dtype(self):

pandas/tests/test_index.py

@@ -401,30 +401,33 @@ def test_insert_base(self):
         for name, idx in compat.iteritems(self.indices):
             result = idx[1:4]
 
-            if len(idx)>0:
-                #test 0th element
-                self.assertTrue(idx[0:4].equals(
-                    result.insert(0, idx[0])))
+            if not len(idx):
+                continue
+
+            #test 0th element
+            self.assertTrue(idx[0:4].equals(
+                result.insert(0, idx[0])))
 
     def test_delete_base(self):
 
         for name, idx in compat.iteritems(self.indices):
 
-            if len(idx)>0:
+            if not len(idx):
+                continue
 
-                expected = idx[1:]
-                result = idx.delete(0)
-                self.assertTrue(result.equals(expected))
-                self.assertEqual(result.name, expected.name)
+            expected = idx[1:]
+            result = idx.delete(0)
+            self.assertTrue(result.equals(expected))
+            self.assertEqual(result.name, expected.name)
 
-                expected = idx[:-1]
-                result = idx.delete(-1)
-                self.assertTrue(result.equals(expected))
-                self.assertEqual(result.name, expected.name)
+            expected = idx[:-1]
+            result = idx.delete(-1)
+            self.assertTrue(result.equals(expected))
+            self.assertEqual(result.name, expected.name)
 
-                with tm.assertRaises((IndexError, ValueError)):
-                    # either depending on numpy version
-                    result = idx.delete(len(idx))
+            with tm.assertRaises((IndexError, ValueError)):
+                # either depending on numpy version
+                result = idx.delete(len(idx))
 
 
     def test_equals_op(self):

pandas/tseries/index.py

@@ -5,7 +5,8 @@
 import numpy as np
 from pandas.core.common import (_NS_DTYPE, _INT64_DTYPE,
                                 _values_from_object, _maybe_box,
-                                ABCSeries, is_integer, is_float)
+                                ABCSeries, is_integer, is_float,
+                                is_object_dtype, is_datetime64_dtype)
 from pandas.core.index import Index, Int64Index, Float64Index
 import pandas.compat as compat
 from pandas.compat import u
@@ -494,9 +495,16 @@ def _local_timestamps(self):
 
     @classmethod
     def _simple_new(cls, values, name=None, freq=None, tz=None, **kwargs):
+        """
+        we require the we have a dtype compat for the values
+        if we are passed a non-dtype compat, then coerce using the constructor
+        """
+
         if not getattr(values,'dtype',None):
             values = np.array(values,copy=False)
-        if values.dtype != _NS_DTYPE:
+        if is_object_dtype(values):
+            return cls(values, name=name, freq=freq, tz=tz, **kwargs).values
+        elif not is_datetime64_dtype(values):
             values = com._ensure_int64(values).view(_NS_DTYPE)
 
         result = object.__new__(cls)

pandas/tseries/period.py

@@ -19,7 +19,7 @@
 import pandas.core.common as com
 from pandas.core.common import (isnull, _INT64_DTYPE, _maybe_box,
                                 _values_from_object, ABCSeries,
-                                is_integer, is_float)
+                                is_integer, is_float, is_object_dtype)
 from pandas import compat
 from pandas.lib import Timestamp, Timedelta
 import pandas.lib as lib
@@ -259,13 +259,32 @@ def _from_arraylike(cls, data, freq, tz):
 
     @classmethod
     def _simple_new(cls, values, name=None, freq=None, **kwargs):
+        if not getattr(values,'dtype',None):
+            values = np.array(values,copy=False)
+        if is_object_dtype(values):
+            return PeriodIndex(values, name=name, freq=freq, **kwargs)
+
         result = object.__new__(cls)
         result._data = values
         result.name = name
         result.freq = freq
         result._reset_identity()
         return result
 
+    def _shallow_copy(self, values=None, infer=False, **kwargs):
+        """ we always want to return a PeriodIndex """
+        return super(PeriodIndex, self)._shallow_copy(values=values, infer=False, **kwargs)
+
+    def _coerce_scalar_to_index(self, item):
+        """
+        we need to coerce a scalar to a compat for our index type
+
+        Parameters
+        ----------
+        item : scalar item to coerce
+        """
+        return PeriodIndex([item], **self._get_attributes_dict())
+
     @property
     def _na_value(self):
         return self._box_func(tslib.iNaT)