BUG: DataFrame.groupby(., dropna=True, axis=0) incorrectly throws ShapeError

doc/source/whatsnew/v1.2.0.rst

@@ -221,7 +221,7 @@ Indexing
 Missing
 ^^^^^^^
 
-- Bug in :meth:`SeriesGroupBy.transform` now correctly handles missing values for `dropna=False` (:issue:`35014`)
+- Bug in :meth:`SeriesGroupBy.transform` and :meth:`DataFrameGroupBy.transform` now correctly handle missing values (:issue:`35014` and :issue:`35612`)
 -
 
 MultiIndex

pandas/core/groupby/generic.py

@@ -557,7 +557,6 @@ def _transform_general(
  result = maybe_downcast_numeric(result, self._selected_obj.dtype)
 
  result.name = self._selected_obj.name
- result.index = self._selected_obj.index
  return result
 
  def _transform_fast(self, result) -> Series:
@@ -1677,12 +1676,17 @@ def _gotitem(self, key, ndim: int, subset=None):
  exclusions=self.exclusions,
  as_index=self.as_index,
  observed=self.observed,
+ dropna=self.dropna,
  )
  elif ndim == 1:
  if subset is None:
  subset = self.obj[key]
  return SeriesGroupBy(
- subset, selection=key, grouper=self.grouper, observed=self.observed
+ subset,
+ selection=key,
+ grouper=self.grouper,
+ observed=self.observed,
+ dropna=self.dropna,
  )
 
  raise AssertionError("invalid ndim for _gotitem")

pandas/core/groupby/groupby.py

@@ -690,7 +690,8 @@ def _set_result_index_ordered(self, result):
  result.set_axis(index, axis=self.axis, inplace=True)
  result = result.sort_index(axis=self.axis)
 
- result.set_axis(self.obj._get_axis(self.axis), axis=self.axis, inplace=True)
+ obj = self.obj.dropna() if self.dropna else self.obj
+ result.set_axis(obj._get_axis(self.axis), axis=self.axis, inplace=True)
  return result
 
  def _dir_additions(self):

pandas/tests/groupby/test_groupby_dropna.py

@@ -165,12 +165,7 @@ def test_groupby_dropna_series_by(dropna, expected):
 @pytest.mark.parametrize(
  "dropna,df_expected,s_expected",
  [
- pytest.param(
- True,
- pd.DataFrame({"B": [2, 2, 1]}),
- pd.Series(data=[2, 2, 1], name="B"),
- marks=pytest.mark.xfail(raises=ValueError),
- ),
+ (True, pd.DataFrame({"B": [2, 2, 1]}), pd.Series(data=[2, 2, 1], name="B"),),
  (
  False,
  pd.DataFrame({"B": [2, 2, 1, 1]}),
@@ -179,7 +174,7 @@ def test_groupby_dropna_series_by(dropna, expected):
  ],
 )
 def test_slice_groupby_then_transform(dropna, df_expected, s_expected):
- # GH35014
+ # GH35014 & GH35612
 
  df = pd.DataFrame({"A": [0, 0, 1, None], "B": [1, 2, 3, None]})
  gb = df.groupby("A", dropna=dropna)

pandas/tests/groupby/test_grouping.py

@@ -610,33 +610,6 @@ def test_list_grouper_with_nat(self):
  expected = {pd.Timestamp("2011-01-01"): 365}
  tm.assert_dict_equal(result.groups, expected)
 
- @pytest.mark.parametrize(
- "func,expected",
- [
- (
- "transform",
- pd.Series(name=2, dtype=np.float64, index=pd.RangeIndex(0, 0, 1)),
- ),
- (
- "agg",
- pd.Series(name=2, dtype=np.float64, index=pd.Float64Index([], name=1)),
- ),
- (
- "apply",
- pd.Series(name=2, dtype=np.float64, index=pd.Float64Index([], name=1)),
- ),
- ],
- )
- def test_evaluate_with_empty_groups(self, func, expected):
- # 26208
- # test transform'ing empty groups
- # (not testing other agg fns, because they return
- # different index objects.
- df = pd.DataFrame({1: [], 2: []})
- g = df.groupby(1)
- result = getattr(g[2], func)(lambda x: x)
- tm.assert_series_equal(result, expected)
-
  def test_groupby_empty(self):
  # https://github.com/pandas-dev/pandas/issues/27190
  s = pd.Series([], name="name", dtype="float64")

pandas/tests/groupby/transform/test_transform.py

@@ -309,11 +309,11 @@ def test_transform_multiple(ts):
 def test_dispatch_transform(tsframe):
  df = tsframe[::5].reindex(tsframe.index)
 
- grouped = df.groupby(lambda x: x.month)
+ grouped = df.groupby(lambda x: x.month, dropna=False)
 
  filled = grouped.fillna(method="pad")
  fillit = lambda x: x.fillna(method="pad")
- expected = df.groupby(lambda x: x.month).transform(fillit)
+ expected = df.groupby(lambda x: x.month, dropna=False).transform(fillit)
  tm.assert_frame_equal(filled, expected)
 
 
@@ -412,10 +412,10 @@ def nsum(x):
  return np.nansum(x)
 
  results = [
- df.groupby("col1").transform(sum)["col2"],
- df.groupby("col1")["col2"].transform(sum),
- df.groupby("col1").transform(nsum)["col2"],
- df.groupby("col1")["col2"].transform(nsum),
+ df.groupby("col1", dropna=False).transform(sum)["col2"],
+ df.groupby("col1", dropna=False)["col2"].transform(sum),
+ df.groupby("col1", dropna=False).transform(nsum)["col2"],
+ df.groupby("col1", dropna=False)["col2"].transform(nsum),
  ]
  for result in results:
  tm.assert_series_equal(result, expected, check_names=False)
@@ -448,7 +448,9 @@ def test_groupby_transform_with_int():
  )
  )
  with np.errstate(all="ignore"):
- result = df.groupby("A").transform(lambda x: (x - x.mean()) / x.std())
+ result = df.groupby("A", dropna=False).transform(
+ lambda x: (x - x.mean()) / x.std()
+ )
  expected = DataFrame(
  dict(B=np.nan, C=Series([-1, 0, 1, -1, 0, 1], dtype="float64"))
  )
@@ -612,8 +614,7 @@ def test_cython_transform_series(op, args, targop):
 
  # series
  for data in [s, s_missing]:
- # print(data.head())
- expected = data.groupby(labels).transform(targop)
+ expected = data.groupby(labels, dropna=False).transform(targop)
 
  tm.assert_series_equal(expected, data.groupby(labels).transform(op, *args))
  tm.assert_series_equal(expected, getattr(data.groupby(labels), op)(*args))