BUG: frame[x].loc[y] inconsistent with frame.at[x, y] GH#22372 (#45287)

Co-authored-by: Jeff Reback <jeff@reback.net>

Author: jbrockmendel

doc/source/whatsnew/v1.5.0.rst

@@ -200,6 +200,7 @@ Interval
 
 Indexing
 ^^^^^^^^
+- Bug in :meth:`loc.__getitem__` with a list of keys causing an internal inconsistency that could lead to a disconnect between ``frame.at[x, y]`` vs ``frame[y].loc[x]`` (:issue:`22372`)
 - Bug in :meth:`DataFrame.iloc` where indexing a single row on a :class:`DataFrame` with a single ExtensionDtype column gave a copy instead of a view on the underlying data (:issue:`45241`)
 - Bug in :meth:`Series.__setitem__` with a non-integer :class:`Index` when using an integer key to set a value that cannot be set inplace where a ``ValueError`` was raised insead of casting to a common dtype (:issue:`45070`)
 - Bug when setting an integer too large for a :class:`Series` dtype failing to coerce to a common type (:issue:`26049`)

pandas/core/internals/managers.py

@@ -671,9 +671,6 @@ def reindex_indexer(
  result.axes[axis] = new_axis
  return result
 
- if consolidate:
- self._consolidate_inplace()
-
  # some axes don't allow reindexing with dups
  if not allow_dups:
  self.axes[axis]._validate_can_reindex(indexer)
@@ -1681,6 +1678,10 @@ def _consolidate_check(self) -> None:
  self._known_consolidated = True
 
  def _consolidate_inplace(self) -> None:
+ # In general, _consolidate_inplace should only be called via
+ # DataFrame._consolidate_inplace, otherwise we will fail to invalidate
+ # the DataFrame's _item_cache. The exception is for newly-created
+ # BlockManager objects not yet attached to a DataFrame.
  if not self.is_consolidated():
  self.blocks = tuple(_consolidate(self.blocks))
  self._is_consolidated = True

pandas/tests/indexing/test_at.py

@@ -48,6 +48,29 @@ def test_selection_methods_of_assigned_col():
 
 
 class TestAtSetItem:
+ def test_at_setitem_item_cache_cleared(self):
+ # GH#22372 Note the multi-step construction is necessary to trigger
+ # the original bug. pandas/issues/22372#issuecomment-413345309
+ df = DataFrame(index=[0])
+ df["x"] = 1
+ df["cost"] = 2
+
+ # accessing df["cost"] adds "cost" to the _item_cache
+ df["cost"]
+
+ # This loc[[0]] lookup used to call _consolidate_inplace at the
+ # BlockManager level, which failed to clear the _item_cache
+ df.loc[[0]]
+
+ df.at[0, "x"] = 4
+ df.at[0, "cost"] = 789
+
+ expected = DataFrame({"x": [4], "cost": 789}, index=[0])
+ tm.assert_frame_equal(df, expected)
+
+ # And in particular, check that the _item_cache has updated correctly.
+ tm.assert_series_equal(df["cost"], expected["cost"])
+
  def test_at_setitem_mixed_index_assignment(self):
  # GH#19860
  ser = Series([1, 2, 3, 4, 5], index=["a", "b", "c", 1, 2])

pandas/tests/internals/test_internals.py

@@ -722,7 +722,10 @@ def test_reindex_items(self):
  mgr = create_mgr("a: f8; b: i8; c: f8; d: i8; e: f8; f: bool; g: f8-2")
 
  reindexed = mgr.reindex_axis(["g", "c", "a", "d"], axis=0)
- assert reindexed.nblocks == 2
+ # reindex_axis does not consolidate_inplace, as that risks failing to
+ # invalidate _item_cache
+ assert not reindexed.is_consolidated()
+
  tm.assert_index_equal(reindexed.items, Index(["g", "c", "a", "d"]))
  tm.assert_almost_equal(
  mgr.iget(6).internal_values(), reindexed.iget(0).internal_values()