BUG: avoid RuntimeError in groupby.max (#46408)

Author: jbrockmendel

pandas/_libs/groupby.pyx

@@ -1047,6 +1047,7 @@ def group_last(
  const uint8_t[:, :] mask,
  uint8_t[:, ::1] result_mask=None,
  Py_ssize_t min_count=-1,
+ bint is_datetimelike=False,
 ) -> None:
  """
  Only aggregates on axis=0
@@ -1056,7 +1057,6 @@ def group_last(
  iu_64_floating_obj_t val
  ndarray[iu_64_floating_obj_t, ndim=2] resx
  ndarray[int64_t, ndim=2] nobs
- bint runtime_error = False
  bint uses_mask = mask is not None
  bint isna_entry
 
@@ -1116,35 +1116,38 @@ def group_last(
  if uses_mask:
  isna_entry = mask[i, j]
  else:
- isna_entry = _treat_as_na(val, True)
- # TODO: Sure we always want is_datetimelike=True?
+ isna_entry = _treat_as_na(val, is_datetimelike)
 
  if not isna_entry:
  nobs[lab, j] += 1
  resx[lab, j] = val
 
  for i in range(ncounts):
  for j in range(K):
+ # TODO(cython3): the entire next block can be shared
+ # across 3 places once conditional-nogil is available
  if nobs[i, j] < min_count:
+ # if we are integer dtype, not is_datetimelike, and
+ # not uses_mask, then getting here implies that
+ # counts[i] < min_count, which means we will
+ # be cast to float64 and masked at the end
+ # of WrappedCythonOp._call_cython_op. So we can safely
+ # set a placeholder value in out[i, j].
  if uses_mask:
  result_mask[i, j] = True
  elif iu_64_floating_obj_t is int64_t:
- # TODO: only if datetimelike?
+ # Per above, this is a placeholder in
+ # non-is_datetimelike cases.
  out[i, j] = NPY_NAT
  elif iu_64_floating_obj_t is uint64_t:
- runtime_error = True
- break
+ # placeholder, see above
+ out[i, j] = 0
  else:
  out[i, j] = NAN
 
  else:
  out[i, j] = resx[i, j]
 
- if runtime_error:
- # We cannot raise directly above because that is within a nogil
- # block.
- raise RuntimeError("empty group with uint64_t")
-
 
 # TODO(cython3): GH#31710 use memorviews once cython 0.30 is released so we can
 # use `const iu_64_floating_obj_t[:, :] values`
@@ -1159,6 +1162,7 @@ def group_nth(
  uint8_t[:, ::1] result_mask=None,
  int64_t min_count=-1,
  int64_t rank=1,
+ bint is_datetimelike=False,
 ) -> None:
  """
  Only aggregates on axis=0
@@ -1168,7 +1172,6 @@ def group_nth(
  iu_64_floating_obj_t val
  ndarray[iu_64_floating_obj_t, ndim=2] resx
  ndarray[int64_t, ndim=2] nobs
- bint runtime_error = False
  bint uses_mask = mask is not None
  bint isna_entry
 
@@ -1230,8 +1233,7 @@ def group_nth(
  if uses_mask:
  isna_entry = mask[i, j]
  else:
- isna_entry = _treat_as_na(val, True)
- # TODO: Sure we always want is_datetimelike=True?
+ isna_entry = _treat_as_na(val, is_datetimelike)
 
  if not isna_entry:
  nobs[lab, j] += 1
@@ -1241,25 +1243,27 @@ def group_nth(
  for i in range(ncounts):
  for j in range(K):
  if nobs[i, j] < min_count:
+ # if we are integer dtype, not is_datetimelike, and
+ # not uses_mask, then getting here implies that
+ # counts[i] < min_count, which means we will
+ # be cast to float64 and masked at the end
+ # of WrappedCythonOp._call_cython_op. So we can safely
+ # set a placeholder value in out[i, j].
  if uses_mask:
  result_mask[i, j] = True
  out[i, j] = 0
  elif iu_64_floating_obj_t is int64_t:
- # TODO: only if datetimelike?
+ # Per above, this is a placeholder in
+ # non-is_datetimelike cases.
  out[i, j] = NPY_NAT
  elif iu_64_floating_obj_t is uint64_t:
- runtime_error = True
- break
+ # placeholder, see above
+ out[i, j] = 0
  else:
  out[i, j] = NAN
  else:
  out[i, j] = resx[i, j]
 
- if runtime_error:
- # We cannot raise directly above because that is within a nogil
- # block.
- raise RuntimeError("empty group with uint64_t")
-
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
@@ -1386,7 +1390,6 @@ cdef group_min_max(
  Py_ssize_t i, j, N, K, lab, ngroups = len(counts)
  iu_64_floating_t val, nan_val
  ndarray[iu_64_floating_t, ndim=2] group_min_or_max
- bint runtime_error = False
  int64_t[:, ::1] nobs
  bint uses_mask = mask is not None
  bint isna_entry
@@ -1403,7 +1406,6 @@ cdef group_min_max(
  group_min_or_max[:] = _get_min_or_max(<iu_64_floating_t>0, compute_max, is_datetimelike)
 
  if iu_64_floating_t is int64_t:
- # TODO: only if is_datetimelike?
  nan_val = NPY_NAT
  elif iu_64_floating_t is uint64_t:
  # NB: We do not define nan_val because there is no such thing
@@ -1442,25 +1444,30 @@ cdef group_min_max(
  for i in range(ngroups):
  for j in range(K):
  if nobs[i, j] < min_count:
+ # if we are integer dtype, not is_datetimelike, and
+ # not uses_mask, then getting here implies that
+ # counts[i] < min_count, which means we will
+ # be cast to float64 and masked at the end
+ # of WrappedCythonOp._call_cython_op. So we can safely
+ # set a placeholder value in out[i, j].
  if uses_mask:
  result_mask[i, j] = True
  # set out[i, j] to 0 to be deterministic, as
  # it was initialized with np.empty. Also ensures
  # we can downcast out if appropriate.
  out[i, j] = 0
+ elif iu_64_floating_t is int64_t:
+ # Per above, this is a placeholder in
+ # non-is_datetimelike cases.
+ out[i, j] = nan_val
  elif iu_64_floating_t is uint64_t:
- runtime_error = True
- break
+ # placeholder, see above
+ out[i, j] = 0
  else:
  out[i, j] = nan_val
  else:
  out[i, j] = group_min_or_max[i, j]
 
- if runtime_error:
- # We cannot raise directly above because that is within a nogil
- # block.
- raise RuntimeError("empty group with uint64_t")
-
 
 @cython.wraparound(False)
 @cython.boundscheck(False)

pandas/core/groupby/ops.py

@@ -519,7 +519,7 @@ def _call_cython_op(
  result = maybe_fill(np.empty(out_shape, dtype=out_dtype))
  if self.kind == "aggregate":
  counts = np.zeros(ngroups, dtype=np.int64)
- if self.how in ["min", "max", "mean"]:
+ if self.how in ["min", "max", "mean", "last", "first"]:
  func(
  out=result,
  counts=counts,
@@ -530,16 +530,6 @@ def _call_cython_op(
  result_mask=result_mask,
  is_datetimelike=is_datetimelike,
  )
- elif self.how in ["first", "last"]:
- func(
- out=result,
- counts=counts,
- values=values,
- labels=comp_ids,
- min_count=min_count,
- mask=mask,
- result_mask=result_mask,
- )
  elif self.how in ["add"]:
  # We support datetimelike
  func(

pandas/tests/resample/test_datetime_index.py

@@ -1856,15 +1856,11 @@ def test_resample_unsigned_int(any_unsigned_int_numpy_dtype):
  )
  df = df.loc[(df.index < "2000-01-02") | (df.index > "2000-01-03"), :]
 
- if any_unsigned_int_numpy_dtype == "uint64":
- with pytest.raises(RuntimeError, match="empty group with uint64_t"):
- df.resample("D").max()
- else:
- result = df.resample("D").max()
-
- expected = DataFrame(
- [1, np.nan, 0],
- columns=["x"],
- index=date_range(start="2000-01-01", end="2000-01-03 23", freq="D"),
- )
- tm.assert_frame_equal(result, expected)
+ result = df.resample("D").max()
+
+ expected = DataFrame(
+ [1, np.nan, 0],
+ columns=["x"],
+ index=date_range(start="2000-01-01", end="2000-01-03 23", freq="D"),
+ )
+ tm.assert_frame_equal(result, expected)