Merge remote-tracking branch 'upstream/master' into perf_sparse_min_max

Author: mzeitlin11

ci/azure/posix.yml

@@ -9,10 +9,16 @@ jobs:
  strategy:
  matrix:
  ${{ if eq(parameters.name, 'macOS') }}:
- py38_macos:
+ py38_macos_1:
  ENV_FILE: ci/deps/azure-macos-38.yaml
  CONDA_PY: "38"
  PATTERN: "not slow and not network"
+ PYTEST_TARGET: "pandas/tests/[a-h]*"
+ py38_macos_2:
+ ENV_FILE: ci/deps/azure-macos-38.yaml
+ CONDA_PY: "38"
+ PATTERN: "not slow and not network"
+ PYTEST_TARGET: "pandas/tests/[i-z]*"
 
  steps:
  - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'

ci/azure/windows.yml

@@ -13,28 +13,28 @@ jobs:
  CONDA_PY: "38"
  PATTERN: "not slow and not network"
  PYTEST_WORKERS: 2 # GH-42236
- PYTEST_TARGET: "pandas/tests/[a-i]*"
+ PYTEST_TARGET: "pandas/tests/[a-h]*"
 
  py38_np18_2:
  ENV_FILE: ci/deps/azure-windows-38.yaml
  CONDA_PY: "38"
  PATTERN: "not slow and not network"
  PYTEST_WORKERS: 2 # GH-42236
- PYTEST_TARGET: "pandas/tests/[j-z]*"
+ PYTEST_TARGET: "pandas/tests/[i-z]*"
 
  py39_1:
  ENV_FILE: ci/deps/azure-windows-39.yaml
  CONDA_PY: "39"
  PATTERN: "not slow and not network and not high_memory"
  PYTEST_WORKERS: 2 # GH-42236
- PYTEST_TARGET: "pandas/tests/[a-i]*"
+ PYTEST_TARGET: "pandas/tests/[a-h]*"
 
  py39_2:
  ENV_FILE: ci/deps/azure-windows-39.yaml
  CONDA_PY: "39"
  PATTERN: "not slow and not network and not high_memory"
  PYTEST_WORKERS: 2 # GH-42236
- PYTEST_TARGET: "pandas/tests/[j-z]*"
+ PYTEST_TARGET: "pandas/tests/[i-z]*"
 
  steps:
  - powershell: |

doc/source/whatsnew/index.rst

@@ -24,6 +24,7 @@ Version 1.3
 .. toctree::
  :maxdepth: 2
 
+ v1.3.4
  v1.3.3
  v1.3.2
  v1.3.1

doc/source/whatsnew/v1.3.3.rst

@@ -54,4 +54,4 @@ Bug fixes
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v1.3.2..v1.3.3|HEAD
+.. contributors:: v1.3.2..v1.3.3

doc/source/whatsnew/v1.3.4.rst

@@ -0,0 +1,45 @@
+.. _whatsnew_134:
+
+What's new in 1.3.4 (October ??, 2021)
+--------------------------------------
+
+These are the changes in pandas 1.3.4. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_134.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_134.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_134.other:
+
+Other
+~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_134.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v1.3.3..v1.3.4|HEAD

doc/source/whatsnew/v1.4.0.rst

@@ -282,6 +282,8 @@ Other Deprecations
 - Deprecated passing in a string column label into ``times`` in :meth:`DataFrame.ewm` (:issue:`43265`)
 - Deprecated the 'include_start' and 'include_end' arguments in :meth:`DataFrame.between_time`; in a future version passing 'include_start' or 'include_end' will raise (:issue:`40245`)
 - Deprecated the ``squeeze`` argument to :meth:`read_csv`, :meth:`read_table`, and :meth:`read_excel`. Users should squeeze the DataFrame afterwards with ``.squeeze("columns")`` instead. (:issue:`43242`)
+- Deprecated the ``index`` argument to :class:`SparseArray` construction (:issue:`23089`)
+-
 
 .. ---------------------------------------------------------------------------
 

pandas/_libs/groupby.pyi

@@ -2,6 +2,8 @@ from typing import Literal
 
 import numpy as np
 
+from pandas._typing import npt
+
 def group_median_float64(
  out: np.ndarray, # ndarray[float64_t, ndim=2]
  counts: np.ndarray, # ndarray[int64_t]
@@ -34,6 +36,7 @@ def group_shift_indexer(
 def group_fillna_indexer(
  out: np.ndarray, # ndarray[intp_t]
  labels: np.ndarray, # ndarray[int64_t]
+ sorted_labels: npt.NDArray[np.intp],
  mask: np.ndarray, # ndarray[uint8_t]
  direction: Literal["ffill", "bfill"],
  limit: int, # int64_t
@@ -84,11 +87,12 @@ def group_ohlc(
  min_count: int = ...,
 ) -> None: ...
 def group_quantile(
- out: np.ndarray, # ndarray[float64_t, ndim=2]
+ out: npt.NDArray[np.float64],
  values: np.ndarray, # ndarray[numeric, ndim=1]
- labels: np.ndarray, # ndarray[int64_t]
- mask: np.ndarray, # ndarray[uint8_t]
- qs: np.ndarray, # const float64_t[:]
+ labels: npt.NDArray[np.intp],
+ mask: npt.NDArray[np.uint8],
+ sort_indexer: npt.NDArray[np.intp], # const
+ qs: npt.NDArray[np.float64], # const
  interpolation: Literal["linear", "lower", "higher", "nearest", "midpoint"],
 ) -> None: ...
 def group_last(

pandas/_libs/groupby.pyx

@@ -322,6 +322,7 @@ def group_shift_indexer(int64_t[::1] out, const intp_t[::1] labels,
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def group_fillna_indexer(ndarray[intp_t] out, ndarray[intp_t] labels,
+ ndarray[intp_t] sorted_labels,
  ndarray[uint8_t] mask, str direction,
  int64_t limit, bint dropna) -> None:
  """
@@ -334,6 +335,9 @@ def group_fillna_indexer(ndarray[intp_t] out, ndarray[intp_t] labels,
  labels : np.ndarray[np.intp]
  Array containing unique label for each group, with its ordering
  matching up to the corresponding record in `values`.
+ sorted_labels : np.ndarray[np.intp]
+ obtained by `np.argsort(labels, kind="mergesort")`; reversed if
+ direction == "bfill"
  values : np.ndarray[np.uint8]
  Containing the truth value of each element.
  mask : np.ndarray[np.uint8]
@@ -349,7 +353,6 @@ def group_fillna_indexer(ndarray[intp_t] out, ndarray[intp_t] labels,
  """
  cdef:
  Py_ssize_t i, N, idx
- intp_t[:] sorted_labels
  intp_t curr_fill_idx=-1
  int64_t filled_vals = 0
 
@@ -358,11 +361,6 @@ def group_fillna_indexer(ndarray[intp_t] out, ndarray[intp_t] labels,
  # Make sure all arrays are the same size
  assert N == len(labels) == len(mask)
 
- sorted_labels = np.argsort(labels, kind='mergesort').astype(
- np.intp, copy=False)
- if direction == 'bfill':
- sorted_labels = sorted_labels[::-1]
-
  with nogil:
  for i in range(N):
  idx = sorted_labels[i]
@@ -774,6 +772,7 @@ def group_quantile(ndarray[float64_t, ndim=2] out,
  ndarray[numeric, ndim=1] values,
  ndarray[intp_t] labels,
  ndarray[uint8_t] mask,
+ const intp_t[:] sort_indexer,
  const float64_t[:] qs,
  str interpolation) -> None:
  """
@@ -787,6 +786,8 @@ def group_quantile(ndarray[float64_t, ndim=2] out,
  Array containing the values to apply the function against.
  labels : ndarray[np.intp]
  Array containing the unique group labels.
+ sort_indexer : ndarray[np.intp]
+ Indices describing sort order by values and labels.
  qs : ndarray[float64_t]
  The quantile values to search for.
  interpolation : {'linear', 'lower', 'highest', 'nearest', 'midpoint'}
@@ -800,9 +801,9 @@ def group_quantile(ndarray[float64_t, ndim=2] out,
  Py_ssize_t i, N=len(labels), ngroups, grp_sz, non_na_sz, k, nqs
  Py_ssize_t grp_start=0, idx=0
  intp_t lab
- uint8_t interp
+ InterpolationEnumType interp
  float64_t q_val, q_idx, frac, val, next_val
- ndarray[int64_t] counts, non_na_counts, sort_arr
+ int64_t[::1] counts, non_na_counts
 
  assert values.shape[0] == N
 
@@ -837,16 +838,6 @@ def group_quantile(ndarray[float64_t, ndim=2] out,
  if not mask[i]:
  non_na_counts[lab] += 1
 
- # Get an index of values sorted by labels and then values
- if labels.any():
- # Put '-1' (NaN) labels as the last group so it does not interfere
- # with the calculations.
- labels_for_lexsort = np.where(labels == -1, labels.max() + 1, labels)
- else:
- labels_for_lexsort = labels
- order = (values, labels_for_lexsort)
- sort_arr = np.lexsort(order).astype(np.int64, copy=False)
-
  with nogil:
  for i in range(ngroups):
  # Figure out how many group elements there are
@@ -864,7 +855,7 @@ def group_quantile(ndarray[float64_t, ndim=2] out,
  # Casting to int will intentionally truncate result
  idx = grp_start + <int64_t>(q_val * <float64_t>(non_na_sz - 1))
 
- val = values[sort_arr[idx]]
+ val = values[sort_indexer[idx]]
  # If requested quantile falls evenly on a particular index
  # then write that index's value out. Otherwise interpolate
  q_idx = q_val * (non_na_sz - 1)
@@ -873,7 +864,7 @@ def group_quantile(ndarray[float64_t, ndim=2] out,
  if frac == 0.0 or interp == INTERPOLATION_LOWER:
  out[i, k] = val
  else:
- next_val = values[sort_arr[idx + 1]]
+ next_val = values[sort_indexer[idx + 1]]
  if interp == INTERPOLATION_LINEAR:
  out[i, k] = val + (next_val - val) * frac
  elif interp == INTERPOLATION_HIGHER:

pandas/_libs/internals.pyx

@@ -760,14 +760,24 @@ cdef class BlockManager:
  cdef BlockManager _get_index_slice(self, slobj):
  cdef:
  SharedBlock blk, nb
+ BlockManager mgr
+ ndarray blknos, blklocs
 
  nbs = []
  for blk in self.blocks:
  nb = blk.getitem_block_index(slobj)
  nbs.append(nb)
 
  new_axes = [self.axes[0], self.axes[1]._getitem_slice(slobj)]
- return type(self)(tuple(nbs), new_axes, verify_integrity=False)
+ mgr = type(self)(tuple(nbs), new_axes, verify_integrity=False)
+
+ # We can avoid having to rebuild blklocs/blknos
+ blklocs = self._blklocs
+ blknos = self._blknos
+ if blknos is not None:
+ mgr._blknos = blknos.copy()
+ mgr._blklocs = blklocs.copy()
+ return mgr
 
  def get_slice(self, slobj: slice, axis: int = 0) -> BlockManager:
 

pandas/_libs/sparse.pyi

@@ -0,0 +1,45 @@
+from typing import (
+ Sequence,
+ TypeVar,
+)
+
+import numpy as np
+
+from pandas._typing import npt
+
+SparseIndexT = TypeVar("SparseIndexT", bound="SparseIndex")
+
+class SparseIndex:
+ length: int
+ npoints: int
+ def __init__(self): ...
+ @property
+ def ngaps(self) -> int: ...
+ @property
+ def nbytes(self) -> int: ...
+ def equals(self, other) -> bool: ...
+ def lookup(self, index: int) -> np.int32: ...
+ def lookup_array(self, indexer: npt.NDArray[np.int32]) -> npt.NDArray[np.int32]: ...
+ def to_int_index(self) -> IntIndex: ...
+ def to_block_index(self) -> BlockIndex: ...
+ def intersect(self: SparseIndexT, y_: SparseIndex) -> SparseIndexT: ...
+ def make_union(self: SparseIndexT, y_: SparseIndex) -> SparseIndexT: ...
+
+class IntIndex(SparseIndex):
+ indices: npt.NDArray[np.int32]
+ def __init__(
+ self, length: int, indices: Sequence[int], check_integrity: bool = True
+ ): ...
+
+class BlockIndex(SparseIndex):
+ nblocks: int
+ blocs: np.ndarray
+ blengths: np.ndarray
+ def __init__(self, length: int, blocs: np.ndarray, blengths: np.ndarray): ...
+
+def make_mask_object_ndarray(
+ arr: npt.NDArray[np.object_], fill_value
+) -> npt.NDArray[np.bool_]: ...
+def get_blocks(
+ indices: npt.NDArray[np.int32],
+) -> tuple[npt.NDArray[np.int32], npt.NDArray[np.int32]]: ...