BUG/PERF: DataFrame.to_records inconsistent dtypes for MultiIndex (pandas-dev#47279)

DataFrame.to_records fix inconsistent dtypes with MultiIndex

Author: lukemanley

asv_bench/benchmarks/frame_methods.py

@@ -288,6 +288,26 @@ def time_values_mixed_wide(self):
  self.df_mixed_wide.values
 
 
+class ToRecords:
+ def setup(self):
+ N = 100_000
+ data = np.random.randn(N, 2)
+ mi = MultiIndex.from_arrays(
+ [
+ np.arange(N),
+ date_range("1970-01-01", periods=N, freq="ms"),
+ ]
+ )
+ self.df = DataFrame(data)
+ self.df_mi = DataFrame(data, index=mi)
+
+ def time_to_records(self):
+ self.df.to_records(index=True)
+
+ def time_to_records_multiindex(self):
+ self.df_mi.to_records(index=True)
+
+
 class Repr:
  def setup(self):
  nrows = 10000

doc/source/whatsnew/v1.5.0.rst

@@ -704,6 +704,7 @@ Performance improvements
 - Performance improvement in :meth:`.GroupBy.transform` for user-defined functions when only a single group exists (:issue:`44977`)
 - Performance improvement in :meth:`.GroupBy.apply` when grouping on a non-unique unsorted index (:issue:`46527`)
 - Performance improvement in :meth:`DataFrame.loc` and :meth:`Series.loc` for tuple-based indexing of a :class:`MultiIndex` (:issue:`45681`, :issue:`46040`, :issue:`46330`)
+- Performance improvement in :meth:`DataFrame.to_records` when the index is a :class:`MultiIndex` (:issue:`47263`)
 - Performance improvement in :attr:`MultiIndex.values` when the MultiIndex contains levels of type DatetimeIndex, TimedeltaIndex or ExtensionDtypes (:issue:`46288`)
 - Performance improvement in :func:`merge` when left and/or right are empty (:issue:`45838`)
 - Performance improvement in :meth:`DataFrame.join` when left and/or right are empty (:issue:`46015`)
@@ -765,6 +766,7 @@ Conversion
 - Bug in :func:`array` with ``FloatingDtype`` and values containing float-castable strings incorrectly raising (:issue:`45424`)
 - Bug when comparing string and datetime64ns objects causing ``OverflowError`` exception. (:issue:`45506`)
 - Bug in metaclass of generic abstract dtypes causing :meth:`DataFrame.apply` and :meth:`Series.apply` to raise for the built-in function ``type`` (:issue:`46684`)
+- Bug in :meth:`DataFrame.to_records` returning inconsistent numpy types if the index was a :class:`MultiIndex` (:issue:`47263`)
 - Bug in :meth:`DataFrame.to_dict` for ``orient="list"`` or ``orient="index"`` was not returning native types (:issue:`46751`)
 
 Strings

pandas/core/frame.py

@@ -2416,13 +2416,10 @@ def to_records(
  dtype=[('I', 'S1'), ('A', '<i8'), ('B', '<f8')])
  """
  if index:
- if isinstance(self.index, MultiIndex):
- # array of tuples to numpy cols. copy copy copy
- ix_vals = list(map(np.array, zip(*self.index._values)))
- else:
- # error: List item 0 has incompatible type "ArrayLike"; expected
- # "ndarray"
- ix_vals = [self.index.values] # type: ignore[list-item]
+ ix_vals = [
+ np.asarray(self.index.get_level_values(i))
+ for i in range(self.index.nlevels)
+ ]
 
  arrays = ix_vals + [
  np.asarray(self.iloc[:, i]) for i in range(len(self.columns))

pandas/tests/frame/methods/test_to_records.py

@@ -96,7 +96,7 @@ def test_to_records_index_name(self):
  + [np.asarray(df.iloc[:, i]) for i in range(3)],
  dtype={
  "names": ["A", "level_1", "0", "1", "2"],
- "formats": ["<U1", "<U1", "<f8", "<f8", "<f8"],
+ "formats": ["O", "O", "<f8", "<f8", "<f8"],
  },
  )
  tm.assert_numpy_array_equal(result, expected)
@@ -108,6 +108,33 @@ def test_to_records_with_unicode_index(self):
  expected = np.rec.array([("x", "y")], dtype=[("a", "O"), ("b", "O")])
  tm.assert_almost_equal(result, expected)
 
+ def test_to_records_index_dtype(self):
+ # GH 47263: consistent data types for Index and MultiIndex
+ df = DataFrame(
+ {
+ 1: date_range("2022-01-01", periods=2),
+ 2: date_range("2022-01-01", periods=2),
+ 3: date_range("2022-01-01", periods=2),
+ }
+ )
+
+ expected = np.rec.array(
+ [
+ ("2022-01-01", "2022-01-01", "2022-01-01"),
+ ("2022-01-02", "2022-01-02", "2022-01-02"),
+ ],
+ dtype=[("1", "<M8[ns]"), ("2", "<M8[ns]"), ("3", "<M8[ns]")],
+ )
+
+ result = df.to_records(index=False)
+ tm.assert_almost_equal(result, expected)
+
+ result = df.set_index(1).to_records(index=True)
+ tm.assert_almost_equal(result, expected)
+
+ result = df.set_index([1, 2]).to_records(index=True)
+ tm.assert_almost_equal(result, expected)
+
  def test_to_records_with_unicode_column_names(self):
  # xref issue: https://github.com/numpy/numpy/issues/2407
  # Issue GH#11879. to_records used to raise an exception when used