ENH: cython refactor, pushing more index logic into engines.pyx, broken unit tests, a mess still

Author: wesm

bench/bench_tseries.py

@@ -3103,6 +3103,7 @@ def _apply_standard(self, func, axis, ignore_failures=False):
                 for k, v in series_gen:
                     results[k] = func(v)
             except Exception, e:
+                raise # XXXXX
                 if hasattr(e, 'args'):
                     e.args = e.args + ('occurred at index %s' % str(k),)
                 raise

pandas/core/frame.py

@@ -3,6 +3,7 @@
 from datetime import time, datetime, date
 from datetime import timedelta
 from itertools import izip
+import weakref
 
 import numpy as np
 
@@ -57,21 +58,26 @@ class Index(np.ndarray):
     ----
     An Index instance can **only** contain hashable objects
     """
+    # _algos = {
+    #     'groupby' : _gin.groupby_index,
+    # }
+
+    # _map_indices = lib.map_indices_object
+
     # Cython methods
-    _map_indices = lib.map_indices_object
-    _is_monotonic = lib.is_monotonic_object
     _groupby = lib.groupby_object
     _arrmap = lib.arrmap_object
     _left_indexer = lib.left_join_indexer_object
     _inner_indexer = lib.inner_join_indexer_object
     _outer_indexer = lib.outer_join_indexer_object
-    _merge_indexer = lib.merge_indexer_object
     _pad = lib.pad_object
     _backfill = lib.backfill_object
 
     name = None
     asi8 = None
 
+    _engine_type = _gin.ObjectEngine
+
     def __new__(cls, data, dtype=None, copy=False, name=None):
         if isinstance(data, np.ndarray):
             if dtype is None:
@@ -164,25 +170,11 @@ def values(self):
 
     @property
     def is_monotonic(self):
-        return self._monotonicity_check[0]
-
-    @property
-    def is_unique(self):
-        is_unique = self._monotonicity_check[1]
-
-        if is_unique is None:
-            return self._engine.has_integrity
-        else:
-            return is_unique
+        return self._engine.is_monotonic
 
     @cache_readonly
-    def _monotonicity_check(self):
-        try:
-            # wrong buffer type raises ValueError
-            is_monotonic, is_unique = self._is_monotonic(self.values)
-            return is_monotonic, is_unique
-        except TypeError:
-            return False, None
+    def is_unique(self):
+        return self._engine.is_unique
 
     def is_numeric(self):
         return self.inferred_type in ['integer', 'floating']
@@ -210,20 +202,13 @@ def get_duplicates(self):
 
     _get_duplicates = get_duplicates
 
-    @property
-    def indexMap(self):
-        "{label -> location}"
-        return self._engine.get_mapping(1)
-
     def _cleanup(self):
         self._engine.clear_mapping()
 
     @cache_readonly
     def _engine(self):
-        import weakref
         # property, for now, slow to look up
-        return _gin.DictIndexEngine(weakref.ref(self),
-                                    self._map_indices)
+        return self._engine_type(weakref.ref(self))
 
     def _get_level_number(self, level):
         if not isinstance(level, int):
@@ -574,7 +559,15 @@ def get_loc(self, key):
         -------
         loc : int
         """
-        return self._engine.get_loc(key)
+        # TODO: push all of this into Cython
+        if self.is_unique:
+            return self._engine.get_loc(key)
+        elif self.is_monotonic:
+            left = self.searchsorted(key, side='left')
+            right = self.searchsorted(key, side='right')
+            return slice(left, right)
+        else:
+            return self.values == key
 
     def get_value(self, series, key):
         """
@@ -664,7 +657,7 @@ def _get_indexer_standard(self, other):
             self.is_monotonic and other.is_monotonic):
             return self._left_indexer(other, self)
         else:
-            return self._merge_indexer(other, self.indexMap)
+            return self._engine.get_indexer(other)
 
     def groupby(self, to_groupby):
         return self._groupby(self.values, to_groupby)
@@ -995,16 +988,19 @@ def copy(self, order='C'):
 class Int64Index(Index):
 
     _map_indices = lib.map_indices_int64
-    _is_monotonic = lib.is_monotonic_int64
+
+    # _is_monotonic = lib.is_monotonic_int64
+
     _groupby = lib.groupby_int64
     _arrmap = lib.arrmap_int64
     _left_indexer = lib.left_join_indexer_int64
     _inner_indexer = lib.inner_join_indexer_int64
     _outer_indexer = lib.outer_join_indexer_int64
-    _merge_indexer = lib.merge_indexer_int64
     _pad = lib.pad_int64
     _backfill = lib.backfill_int64
 
+    _engine_type = _gin.Int64Engine
+
     def __new__(cls, data, dtype=None, copy=False, name=None):
         if not isinstance(data, np.ndarray):
             if np.isscalar(data):
@@ -1172,13 +1168,12 @@ class DatetimeIndex(Int64Index):
         time on or just past end argument
     """
 
-    _is_monotonic  = _wrap_i8_function(lib.is_monotonic_int64)
+    # _is_monotonic  = _wrap_i8_function(lib.is_monotonic_int64)
+
     _inner_indexer = _join_i8_wrapper(lib.inner_join_indexer_int64)
     _outer_indexer = _join_i8_wrapper(lib.outer_join_indexer_int64)
     _left_indexer  = _join_i8_wrapper(lib.left_join_indexer_int64,
                                       with_indexers=False)
-    _merge_indexer = _join_i8_wrapper(lib.merge_indexer_int64,
-                                      with_indexers=False)
     _map_indices   = _wrap_i8_function(lib.map_indices_int64)
     _pad           = _wrap_i8_function(lib.pad_int64)
     _backfill      = _wrap_i8_function(lib.backfill_int64)
@@ -1199,6 +1194,10 @@ class DatetimeIndex(Int64Index):
     # structured array cache for datetime fields
     _sarr_cache = None
 
+    _engine_type = _gin.DatetimeEngine
+
+    offset = None
+
     def __new__(cls, data=None,
                 freq=None, start=None, end=None, periods=None,
                 dtype=None, copy=False, name=None, tz=None,
@@ -1437,7 +1436,6 @@ def asobject(self):
         """
         Unbox to an index of type object
         """
-        offset = getattr(self, 'offset', None)
         boxed_values = _dt_box_array(self.asi8, self.offset, self.tz)
         return Index(boxed_values, dtype=object)
 
@@ -1656,7 +1654,7 @@ def get_value(self, series, key):
         know what you're doing
         """
         try:
-            return super(DatetimeIndex, self).get_value(series, key)
+            return Index.get_value(self, series, key)
         except KeyError:
             try:
                 asdt, parsed, reso = datetools.parse_time_string(key)
@@ -1698,8 +1696,7 @@ def __getitem__(self, key):
         arr_idx = self.view(np.ndarray)
         if np.isscalar(key):
             val = arr_idx[key]
-            offset = getattr(self, 'offset', None)
-            return _dt_box(val, offset=offset, tz=self.tz)
+            return _dt_box(val, offset=self.offset, tz=self.tz)
         else:
             new_offset = None
             if (type(key) == slice):
@@ -1814,11 +1811,6 @@ def dtype(self):
     def is_all_dates(self):
         return True
 
-    @cache_readonly
-    def _engine(self):
-        mapping = lib.map_indices_int64
-        return _gin.DictIndexEngineDatetime(self.asi8, mapping)
-
     def equals(self, other):
         """
         Determines if two Index objects contain the same elements.
@@ -2977,7 +2969,7 @@ def get_indexer(self, target, method=None):
             indexer = self._backfill(self_index, target_index,
                                      self_index.indexMap, target.indexMap)
         else:
-            indexer = self._merge_indexer(target_index, self_index.indexMap)
+            indexer = self._engine.get_indexer(target_index)
 
         return indexer