PERF/CLN: Preserve concat(keys=range) RangeIndex level in the result (#57755)

* PERF/CLN: Preserve RangeIndex level in the result * Whitespace * Whitespace * Fix test * Address review

Author: mroeschke

doc/source/whatsnew/v3.0.0.rst

@@ -204,6 +204,7 @@ Removal of prior version deprecations/changes
 - Enforced deprecation of ``axis=None`` acting the same as ``axis=0`` in the DataFrame reductions ``sum``, ``prod``, ``std``, ``var``, and ``sem``, passing ``axis=None`` will now reduce over both axes; this is particularly the case when doing e.g. ``numpy.sum(df)`` (:issue:`21597`)
 - Enforced deprecation of passing a dictionary to :meth:`SeriesGroupBy.agg` (:issue:`52268`)
 - Enforced deprecation of string ``A`` denoting frequency in :class:`YearEnd` and strings ``A-DEC``, ``A-JAN``, etc. denoting annual frequencies with various fiscal year ends (:issue:`57699`)
+- Enforced deprecation of the behavior of :func:`concat` when ``len(keys) != len(objs)`` would truncate to the shorter of the two. Now this raises a ``ValueError`` (:issue:`43485`)
 - Enforced silent-downcasting deprecation for :ref:`all relevant methods <whatsnew_220.silent_downcasting>` (:issue:`54710`)
 - In :meth:`DataFrame.stack`, the default value of ``future_stack`` is now ``True``; specifying ``False`` will raise a ``FutureWarning`` (:issue:`55448`)
 - Iterating over a :class:`.DataFrameGroupBy` or :class:`.SeriesGroupBy` will return tuples of length 1 for the groups when grouping by ``level`` a list of length 1 (:issue:`50064`)
@@ -255,6 +256,7 @@ Removal of prior version deprecations/changes
 
 Performance improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
+- :func:`concat` returns a :class:`RangeIndex` level in the :class:`MultiIndex` result when ``keys`` is a ``range`` or :class:`RangeIndex` (:issue:`57542`)
 - :meth:`RangeIndex.append` returns a :class:`RangeIndex` instead of a :class:`Index` when appending values that could continue the :class:`RangeIndex` (:issue:`57467`)
 - :meth:`Series.str.extract` returns a :class:`RangeIndex` columns instead of an :class:`Index` column when possible (:issue:`57542`)
 - Performance improvement in :class:`DataFrame` when ``data`` is a ``dict`` and ``columns`` is specified (:issue:`24368`)

pandas/core/groupby/groupby.py

@@ -1202,7 +1202,7 @@ def _concat_objects(
  else:
  # GH5610, returns a MI, with the first level being a
  # range index
- keys = list(range(len(values)))
+ keys = RangeIndex(len(values))
  result = concat(values, axis=0, keys=keys)
 
  elif not not_indexed_same:

pandas/core/reshape/concat.py

@@ -12,12 +12,10 @@
  cast,
  overload,
 )
-import warnings
 
 import numpy as np
 
 from pandas.util._decorators import cache_readonly
-from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.common import (
  is_bool,
@@ -493,32 +491,27 @@ def _clean_keys_and_objs(
  objs_list = list(com.not_none(*objs_list))
  else:
  # GH#1649
- clean_keys = []
+ key_indices = []
  clean_objs = []
  if is_iterator(keys):
  keys = list(keys)
  if len(keys) != len(objs_list):
  # GH#43485
- warnings.warn(
- "The behavior of pd.concat with len(keys) != len(objs) is "
- "deprecated. In a future version this will raise instead of "
- "truncating to the smaller of the two sequences",
- FutureWarning,
- stacklevel=find_stack_level(),
+ raise ValueError(
+ f"The length of the keys ({len(keys)}) must match "
+ f"the length of the objects to concatenate ({len(objs_list)})"
  )
- for k, v in zip(keys, objs_list):
- if v is None:
- continue
- clean_keys.append(k)
- clean_objs.append(v)
+ for i, obj in enumerate(objs_list):
+ if obj is not None:
+ key_indices.append(i)
+ clean_objs.append(obj)
  objs_list = clean_objs
 
- if isinstance(keys, MultiIndex):
- # TODO: retain levels?
- keys = type(keys).from_tuples(clean_keys, names=keys.names)
- else:
- name = getattr(keys, "name", None)
- keys = Index(clean_keys, name=name, dtype=getattr(keys, "dtype", None))
+ if not isinstance(keys, Index):
+ keys = Index(keys)
+
+ if len(key_indices) < len(keys):
+ keys = keys.take(key_indices)
 
  if len(objs_list) == 0:
  raise ValueError("All objects passed were None")

pandas/tests/groupby/methods/test_describe.py

@@ -90,20 +90,22 @@ def test_frame_describe_multikey(tsframe):
 
 def test_frame_describe_tupleindex():
  # GH 14848 - regression from 0.19.0 to 0.19.1
- df1 = DataFrame(
+ name = "k"
+ df = DataFrame(
  {
  "x": [1, 2, 3, 4, 5] * 3,
- "y": [10, 20, 30, 40, 50] * 3,
- "z": [100, 200, 300, 400, 500] * 3,
+ name: [(0, 0, 1), (0, 1, 0), (1, 0, 0)] * 5,
  }
  )
- df1["k"] = [(0, 0, 1), (0, 1, 0), (1, 0, 0)] * 5
- df2 = df1.rename(columns={"k": "key"})
- msg = "Names should be list-like for a MultiIndex"
- with pytest.raises(ValueError, match=msg):
- df1.groupby("k").describe()
- with pytest.raises(ValueError, match=msg):
- df2.groupby("key").describe()
+ result = df.groupby(name).describe()
+ expected = DataFrame(
+ [[5.0, 3.0, 1.581139, 1.0, 2.0, 3.0, 4.0, 5.0]] * 3,
+ index=Index([(0, 0, 1), (0, 1, 0), (1, 0, 0)], tupleize_cols=False, name=name),
+ columns=MultiIndex.from_arrays(
+ [["x"] * 8, ["count", "mean", "std", "min", "25%", "50%", "75%", "max"]]
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
 
 
 def test_frame_describe_unstacked_format():

pandas/tests/reshape/concat/test_concat.py

@@ -17,6 +17,7 @@
  Index,
  MultiIndex,
  PeriodIndex,
+ RangeIndex,
  Series,
  concat,
  date_range,
@@ -395,6 +396,29 @@ def test_concat_keys_with_none(self):
  expected = concat([df0, df0[:2], df0[:1], df0], keys=["b", "c", "d", "e"])
  tm.assert_frame_equal(result, expected)
 
+ @pytest.mark.parametrize("klass", [range, RangeIndex])
+ @pytest.mark.parametrize("include_none", [True, False])
+ def test_concat_preserves_rangeindex(self, klass, include_none):
+ df = DataFrame([1, 2])
+ df2 = DataFrame([3, 4])
+ data = [df, None, df2, None] if include_none else [df, df2]
+ keys_length = 4 if include_none else 2
+ result = concat(data, keys=klass(keys_length))
+ expected = DataFrame(
+ [1, 2, 3, 4],
+ index=MultiIndex(
+ levels=(
+ RangeIndex(start=0, stop=keys_length, step=keys_length / 2),
+ RangeIndex(start=0, stop=2, step=1),
+ ),
+ codes=(
+ np.array([0, 0, 1, 1], dtype=np.int8),
+ np.array([0, 1, 0, 1], dtype=np.int8),
+ ),
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
  def test_concat_bug_1719(self):
  ts1 = Series(
  np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
@@ -705,7 +729,7 @@ def test_concat_multiindex_with_empty_rangeindex():
  # GH#41234
  mi = MultiIndex.from_tuples([("B", 1), ("C", 1)])
  df1 = DataFrame([[1, 2]], columns=mi)
- df2 = DataFrame(index=[1], columns=pd.RangeIndex(0))
+ df2 = DataFrame(index=[1], columns=RangeIndex(0))
 
  result = concat([df1, df2])
  expected = DataFrame([[1, 2], [np.nan, np.nan]], columns=mi)
@@ -830,14 +854,14 @@ def test_concat_mismatched_keys_length():
  sers = [ser + n for n in range(4)]
  keys = ["A", "B", "C"]
 
- msg = r"The behavior of pd.concat with len\(keys\) != len\(objs\) is deprecated"
- with tm.assert_produces_warning(FutureWarning, match=msg):
+ msg = r"The length of the keys"
+ with pytest.raises(ValueError, match=msg):
  concat(sers, keys=keys, axis=1)
- with tm.assert_produces_warning(FutureWarning, match=msg):
+ with pytest.raises(ValueError, match=msg):
  concat(sers, keys=keys, axis=0)
- with tm.assert_produces_warning(FutureWarning, match=msg):
+ with pytest.raises(ValueError, match=msg):
  concat((x for x in sers), keys=(y for y in keys), axis=1)
- with tm.assert_produces_warning(FutureWarning, match=msg):
+ with pytest.raises(ValueError, match=msg):
  concat((x for x in sers), keys=(y for y in keys), axis=0)