Backport PR #41915: BUG: MultiIndex.difference incorrectly raising TypeError when indexes contain non-sortable entries (#42081)

Co-authored-by: jbrockmendel <jbrockmendel@gmail.com>

Author: meeseeksmachine

doc/source/whatsnew/v1.3.0.rst

@@ -1041,6 +1041,7 @@ MultiIndex
 - Bug in :meth:`MultiIndex.intersection` duplicating ``NaN`` in the result (:issue:`38623`)
 - Bug in :meth:`MultiIndex.equals` incorrectly returning ``True`` when the :class:`MultiIndex` contained ``NaN`` even when they are differently ordered (:issue:`38439`)
 - Bug in :meth:`MultiIndex.intersection` always returning an empty result when intersecting with :class:`CategoricalIndex` (:issue:`38653`)
+- Bug in :meth:`MultiIndex.difference` incorrectly raising ``TypeError`` when indexes contain non-sortable entries (:issue:`41915`)
 - Bug in :meth:`MultiIndex.reindex` raising a ``ValueError`` when used on an empty :class:`MultiIndex` and indexing only a specific level (:issue:`41170`)
 - Bug in :meth:`MultiIndex.reindex` raising ``TypeError`` when reindexing against a flat :class:`Index` (:issue:`41707`)
 

pandas/core/indexes/base.py

@@ -3091,7 +3091,7 @@ def intersection(self, other, sort=False):
  return this.intersection(other, sort=sort)
 
  result = self._intersection(other, sort=sort)
- return self._wrap_setop_result(other, result)
+ return self._wrap_intersection_result(other, result)
 
  def _intersection(self, other: Index, sort=False):
  """
@@ -3113,6 +3113,10 @@ def _intersection(self, other: Index, sort=False):
  res_values = _maybe_try_sort(res_values, sort)
  return res_values
 
+ def _wrap_intersection_result(self, other, result):
+ # We will override for MultiIndex to handle empty results
+ return self._wrap_setop_result(other, result)
+
  def _intersection_via_get_indexer(self, other: Index, sort) -> ArrayLike:
  """
  Find the intersection of two Indexes using get_indexer.

pandas/core/indexes/multi.py

@@ -3588,50 +3588,23 @@ def _maybe_match_names(self, other):
  names.append(None)
  return names
 
- def _intersection(self, other, sort=False) -> MultiIndex:
+ def _wrap_intersection_result(self, other, result):
  other, result_names = self._convert_can_do_setop(other)
- other = other.astype(object, copy=False)
 
- uniq_tuples = None # flag whether _inner_indexer was successful
- if self.is_monotonic and other.is_monotonic:
- try:
- inner_tuples = self._inner_indexer(other)[0]
- sort = False # inner_tuples is already sorted
- except TypeError:
- pass
- else:
- uniq_tuples = algos.unique(inner_tuples)
-
- if uniq_tuples is None:
- uniq_tuples = self._intersection_via_get_indexer(other, sort)
-
- if sort is None:
- uniq_tuples = sorted(uniq_tuples)
-
- if len(uniq_tuples) == 0:
+ if len(result) == 0:
  return MultiIndex(
  levels=self.levels,
  codes=[[]] * self.nlevels,
  names=result_names,
  verify_integrity=False,
  )
  else:
- return MultiIndex.from_arrays(
- zip(*uniq_tuples), sortorder=0, names=result_names
- )
+ return MultiIndex.from_arrays(zip(*result), sortorder=0, names=result_names)
 
  def _difference(self, other, sort) -> MultiIndex:
  other, result_names = self._convert_can_do_setop(other)
 
- this = self._get_unique_index()
-
- indexer = this.get_indexer(other)
- indexer = indexer.take((indexer != -1).nonzero()[0])
-
- label_diff = np.setdiff1d(np.arange(this.size), indexer, assume_unique=True)
- difference = this._values.take(label_diff)
- if sort is None:
- difference = sorted(difference)
+ difference = super()._difference(other, sort)
 
  if len(difference) == 0:
  return MultiIndex(

pandas/tests/indexes/multi/test_setops.py

@@ -216,11 +216,10 @@ def test_difference_sort_incomparable():
 
  other = MultiIndex.from_product([[3, pd.Timestamp("2000"), 4], ["c", "d"]])
  # sort=None, the default
- # MultiIndex.difference deviates here from other difference
- # implementations in not catching the TypeError
- msg = "'<' not supported between instances of 'Timestamp' and 'int'"
- with pytest.raises(TypeError, match=msg):
+ msg = "sort order is undefined for incomparable objects"
+ with tm.assert_produces_warning(RuntimeWarning, match=msg):
  result = idx.difference(other)
+ tm.assert_index_equal(result, idx)
 
  # sort=False
  result = idx.difference(other, sort=False)