PERF: improved performance of small multiindexes (pandas-dev#16324)

closes pandas-dev#16319

Author: jreback

asv_bench/benchmarks/indexing.py

@@ -193,9 +193,15 @@ def setup(self):
  np.arange(1000)], names=['one', 'two'])
 
  import string
- self.mistring = MultiIndex.from_product(
- [np.arange(1000),
- np.arange(20), list(string.ascii_letters)],
+
+ self.mi_large = MultiIndex.from_product(
+ [np.arange(1000), np.arange(20), list(string.ascii_letters)],
+ names=['one', 'two', 'three'])
+ self.mi_med = MultiIndex.from_product(
+ [np.arange(1000), np.arange(10), list('A')],
+ names=['one', 'two', 'three'])
+ self.mi_small = MultiIndex.from_product(
+ [np.arange(100), list('A'), list('A')],
  names=['one', 'two', 'three'])
 
  def time_series_xs_mi_ix(self):
@@ -218,8 +224,14 @@ def time_multiindex_get_indexer(self):
  (0, 16), (0, 17), (0, 18),
  (0, 19)], dtype=object))
 
+ def time_multiindex_large_get_loc(self):
+ self.mi_large.get_loc((999, 19, 'Z'))
+
+ def time_multiindex_med_get_loc(self):
+ self.mi_med.get_loc((999, 9, 'A'))
+
  def time_multiindex_string_get_loc(self):
- self.mistring.get_loc((999, 19, 'Z'))
+ self.mi_small.get_loc((99, 'A', 'A'))
 
  def time_is_monotonic(self):
  self.miint.is_monotonic

doc/source/whatsnew/v0.20.2.txt

@@ -27,7 +27,7 @@ Performance Improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
 - Performance regression fix when indexing with a list-like (:issue:`16285`)
-
+- Performance regression fix for small MultiIndexes (:issuse:`16319`)
 
 .. _whatsnew_0202.bug_fixes:
 

pandas/_libs/index.pyx

@@ -553,7 +553,34 @@ cdef inline bint _is_utc(object tz):
  return tz is UTC or isinstance(tz, _du_utc)
 
 
-cdef class MultiIndexEngine(IndexEngine):
+cdef class MultiIndexObjectEngine(ObjectEngine):
+ """
+ provide the same interface as the MultiIndexEngine
+ but use the IndexEngine for computation
+
+ This provides good performance with samller MI's
+ """
+ def get_indexer(self, values):
+ # convert a MI to an ndarray
+ if hasattr(values, 'values'):
+ values = values.values
+ return super(MultiIndexObjectEngine, self).get_indexer(values)
+
+ cpdef get_loc(self, object val):
+
+ # convert a MI to an ndarray
+ if hasattr(val, 'values'):
+ val = val.values
+ return super(MultiIndexObjectEngine, self).get_loc(val)
+
+
+cdef class MultiIndexHashEngine(ObjectEngine):
+ """
+ Use a hashing based MultiIndex impl
+ but use the IndexEngine for computation
+
+ This provides good performance with larger MI's
+ """
 
  def _call_monotonic(self, object mi):
  # defer these back to the mi iteself
@@ -584,6 +611,10 @@ cdef class MultiIndexEngine(IndexEngine):
  except TypeError:
  raise KeyError(val)
 
+ def get_indexer(self, values):
+ self._ensure_mapping_populated()
+ return self.mapping.lookup(values)
+
  cdef _make_hash_table(self, n):
  return _hash.MultiIndexHashTable(n)
 

pandas/core/dtypes/dtypes.py

@@ -88,12 +88,12 @@ def is_dtype(cls, dtype):
  """
  if hasattr(dtype, 'dtype'):
  dtype = dtype.dtype
- if isinstance(dtype, cls):
- return True
- elif isinstance(dtype, np.dtype):
+ if isinstance(dtype, np.dtype):
  return False
  elif dtype is None:
  return False
+ elif isinstance(dtype, cls):
+ return True
  try:
  return cls.construct_from_string(dtype) is not None
  except:

pandas/core/indexes/multi.py

@@ -75,7 +75,6 @@ class MultiIndex(Index):
  _levels = FrozenList()
  _labels = FrozenList()
  _comparables = ['names']
- _engine_type = libindex.MultiIndexEngine
  rename = Index.set_names
 
  def __new__(cls, levels=None, labels=None, sortorder=None, names=None,
@@ -629,7 +628,16 @@ def _get_level_number(self, level):
 
  @cache_readonly
  def _engine(self):
- return self._engine_type(lambda: self, len(self))
+
+ # choose our engine based on our size
+ # the hashing based MultiIndex for larger
+ # sizes, and the MultiIndexOjbect for smaller
+ # xref: https://github.com/pandas-dev/pandas/pull/16324
+ l = len(self)
+ if l > 10000:
+ return libindex.MultiIndexHashEngine(lambda: self, l)
+
+ return libindex.MultiIndexObjectEngine(lambda: self.values, l)
 
  @property
  def values(self):

pandas/core/util/hashing.py

@@ -5,16 +5,13 @@
 
 import numpy as np
 from pandas._libs import hashing
-from pandas._libs.lib import is_bool_array
 from pandas.core.dtypes.generic import (
  ABCMultiIndex,
  ABCIndexClass,
  ABCSeries,
  ABCDataFrame)
 from pandas.core.dtypes.common import (
- is_categorical_dtype, is_numeric_dtype,
- is_datetime64_dtype, is_timedelta64_dtype,
- is_list_like)
+ is_categorical_dtype, is_list_like)
 
 # 16 byte long hashing key
 _default_hash_key = '0123456789123456'
@@ -136,7 +133,6 @@ def hash_tuples(vals, encoding='utf8', hash_key=None):
  -------
  ndarray of hashed values array
  """
-
  is_tuple = False
  if isinstance(vals, tuple):
  vals = [vals]
@@ -231,29 +227,29 @@ def hash_array(vals, encoding='utf8', hash_key=None, categorize=True):
 
  if not hasattr(vals, 'dtype'):
  raise TypeError("must pass a ndarray-like")
+ dtype = vals.dtype
 
  if hash_key is None:
  hash_key = _default_hash_key
 
  # For categoricals, we hash the categories, then remap the codes to the
  # hash values. (This check is above the complex check so that we don't ask
  # numpy if categorical is a subdtype of complex, as it will choke.
- if is_categorical_dtype(vals.dtype):
+ if is_categorical_dtype(dtype):
  return _hash_categorical(vals, encoding, hash_key)
 
  # we'll be working with everything as 64-bit values, so handle this
  # 128-bit value early
- if np.issubdtype(vals.dtype, np.complex128):
+ elif np.issubdtype(dtype, np.complex128):
  return hash_array(vals.real) + 23 * hash_array(vals.imag)
 
  # First, turn whatever array this is into unsigned 64-bit ints, if we can
  # manage it.
- if is_bool_array(vals):
+ elif isinstance(dtype, np.bool):
  vals = vals.astype('u8')
- elif (is_datetime64_dtype(vals) or
- is_timedelta64_dtype(vals)):
+ elif issubclass(dtype.type, (np.datetime64, np.timedelta64)):
  vals = vals.view('i8').astype('u8', copy=False)
- elif (is_numeric_dtype(vals) and vals.dtype.itemsize <= 8):
+ elif issubclass(dtype.type, np.number) and dtype.itemsize <= 8:
  vals = vals.view('u{}'.format(vals.dtype.itemsize)).astype('u8')
  else:
  # With repeated values, its MUCH faster to categorize object dtypes,