Merge remote-tracking branch 'upstream/master' into masked-string-2

Author: TomAugspurger

.travis.yml

@@ -30,11 +30,9 @@ matrix:
  - python: 3.5
 
  include:
- - dist: bionic
- # 18.04
- python: 3.8.0
+ - dist: trusty
  env:
- - JOB="3.8-dev" PATTERN="(not slow and not network)"
+ - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network)"
 
  - dist: trusty
  env:
@@ -88,7 +86,7 @@ install:
 script:
  - echo "script start"
  - echo "$JOB"
- - if [ "$JOB" != "3.8-dev" ]; then source activate pandas-dev; fi
+ - source activate pandas-dev
  - ci/run_tests.sh
 
 after_script:

ci/azure/windows.yml

@@ -11,49 +11,45 @@ jobs:
  py36_np15:
  ENV_FILE: ci/deps/azure-windows-36.yaml
  CONDA_PY: "36"
+ PATTERN: "not slow and not network"
 
  py37_np141:
  ENV_FILE: ci/deps/azure-windows-37.yaml
  CONDA_PY: "37"
+ PATTERN: "not slow and not network"
 
  steps:
  - powershell: |
  Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
  Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin"
  displayName: 'Add conda to PATH'
  - script: conda update -q -n base conda
- displayName: Update conda
- - script: |
- call activate
+ displayName: 'Update conda'
+ - bash: |
  conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml
  displayName: 'Create anaconda environment'
- - script: |
- call activate pandas-dev
- call conda list
+ - bash: |
+ source activate pandas-dev
+ conda list
  ci\\incremental\\build.cmd
  displayName: 'Build'
- - script: |
- call activate pandas-dev
- pytest -m "not slow and not network" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %*
+ - bash: |
+ source activate pandas-dev
+ ci/run_tests.sh
  displayName: 'Test'
  - task: PublishTestResults@2
  inputs:
  testResultsFiles: 'test-data.xml'
  testRunTitle: 'Windows-$(CONDA_PY)'
  - powershell: |
- $junitXml = "test-data.xml"
- $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"'
- if ($matches[1] -eq 0)
- {
+ $(Get-Content "test-data.xml" | Out-String) -match 'failures="(.*?)"'
+ if ($matches[1] -eq 0) {
  Write-Host "No test failures in test-data"
- }
- else
- {
- # note that this will produce $LASTEXITCODE=1
- Write-Error "$($matches[1]) tests failed"
+ } else {
+ Write-Error "$($matches[1]) tests failed" # will produce $LASTEXITCODE=1
  }
  displayName: 'Check for test failures'
- - script: |
+ - bash: |
  source activate pandas-dev
  python ci/print_skipped.py
  displayName: 'Print skipped tests'

ci/build38.sh

@@ -0,0 +1,16 @@
+name: pandas-dev
+channels:
+ - defaults
+ - conda-forge
+dependencies:
+ - python=3.8.*
+ - cython>=0.29.13
+ - numpy
+ - python-dateutil
+ - nomkl
+ - pytz
+ # universal
+ - pytest>=5.0.0
+ - pytest-xdist>=1.29.0
+ - hypothesis>=3.58.0
+ - pip

ci/deps/travis-38.yaml

@@ -1,10 +1,5 @@
 #!/bin/bash -e
 
-if [ "$JOB" == "3.8-dev" ]; then
- /bin/bash ci/build38.sh
- exit 0
-fi
-
 # edit the locale file if needed
 if [ -n "$LOCALE_OVERRIDE" ]; then
  echo "Adding locale to the first line of pandas/__init__.py"

ci/setup_env.sh

@@ -342,6 +342,7 @@ Numeric
 - :class:`DataFrame` flex inequality comparisons methods (:meth:`DataFrame.lt`, :meth:`DataFrame.le`, :meth:`DataFrame.gt`, :meth: `DataFrame.ge`) with object-dtype and ``complex`` entries failing to raise ``TypeError`` like their :class:`Series` counterparts (:issue:`28079`)
 - Bug in :class:`DataFrame` logical operations (`&`, `|`, `^`) not matching :class:`Series` behavior by filling NA values (:issue:`28741`)
 - Bug in :meth:`DataFrame.interpolate` where specifying axis by name references variable before it is assigned (:issue:`29142`)
+- Bug in :meth:`Series.var` not computing the right value with a nullable integer dtype series not passing through ddof argument (:issue:`29128`)
 - Improved error message when using `frac` > 1 and `replace` = False (:issue:`27451`)
 - Bug in numeric indexes resulted in it being possible to instantiate an :class:`Int64Index`, :class:`UInt64Index`, or :class:`Float64Index` with an invalid dtype (e.g. datetime-like) (:issue:`29539`)
 - Bug in :class:`UInt64Index` precision loss while constructing from a list with values in the ``np.uint64`` range (:issue:`29526`)
@@ -432,6 +433,7 @@ Groupby/resample/rolling
 
 -
 - Bug in :meth:`DataFrame.groupby` with multiple groups where an ``IndexError`` would be raised if any group contained all NA values (:issue:`20519`)
+- Bug in :meth:`pandas.core.resample.Resampler.size` and :meth:`pandas.core.resample.Resampler.count` returning wrong dtype when used with an empty series or dataframe (:issue:`28427`)
 - Bug in :meth:`DataFrame.rolling` not allowing for rolling over datetimes when ``axis=1`` (:issue: `28192`)
 - Bug in :meth:`DataFrame.rolling` not allowing rolling over multi-index levels (:issue: `15584`).
 - Bug in :meth:`DataFrame.rolling` not allowing rolling on monotonic decreasing time indexes (:issue: `19248`).
@@ -451,6 +453,7 @@ Reshaping
 - Fix to ensure all int dtypes can be used in :func:`merge_asof` when using a tolerance value. Previously every non-int64 type would raise an erroneous ``MergeError`` (:issue:`28870`).
 - Better error message in :func:`get_dummies` when `columns` isn't a list-like value (:issue:`28383`)
 - Bug :meth:`Series.pct_change` where supplying an anchored frequency would throw a ValueError (:issue:`28664`)
+- Bug where :meth:`DataFrame.equals` returned True incorrectly in some cases when two DataFrames had the same columns in different orders (:issue:`28839`)
 
 Sparse
 ^^^^^^

doc/source/whatsnew/v1.0.0.rst

@@ -1,21 +1,14 @@
 import cython
 from cython import Py_ssize_t
 
-from cpython.object cimport PyObject
+from cpython.slice cimport PySlice_GetIndicesEx
 
 cdef extern from "Python.h":
  Py_ssize_t PY_SSIZE_T_MAX
 
 import numpy as np
 from numpy cimport int64_t
 
-cdef extern from "compat_helper.h":
- cdef int slice_get_indices(PyObject* s, Py_ssize_t length,
- Py_ssize_t *start, Py_ssize_t *stop,
- Py_ssize_t *step,
- Py_ssize_t *slicelength) except -1
-
-
 from pandas._libs.algos import ensure_int64
 
 
@@ -258,8 +251,8 @@ cpdef Py_ssize_t slice_len(
  if slc is None:
  raise TypeError("slc must be slice")
 
- slice_get_indices(<PyObject *>slc, objlen,
- &start, &stop, &step, &length)
+ PySlice_GetIndicesEx(slc, objlen,
+  &start, &stop, &step, &length)
 
  return length
 
@@ -278,8 +271,8 @@ cdef slice_get_indices_ex(slice slc, Py_ssize_t objlen=PY_SSIZE_T_MAX):
  if slc is None:
  raise TypeError("slc should be a slice")
 
- slice_get_indices(<PyObject *>slc, objlen,
- &start, &stop, &step, &length)
+ PySlice_GetIndicesEx(slc, objlen,
+  &start, &stop, &step, &length)
 
  return start, stop, step, length
 

pandas/_libs/internals.pyx

@@ -135,9 +135,8 @@ cdef class Reducer:
  else:
  res = self.f(chunk)
 
- if (not _is_sparse_array(res) and hasattr(res, 'values')
- and util.is_array(res.values)):
- res = res.values
+ # TODO: reason for not squeezing here?
+ res = _extract_result(res, squeeze=False)
  if i == 0:
  # On the first pass, we check the output shape to see
  # if this looks like a reduction.
@@ -402,18 +401,17 @@ cdef class SeriesGrouper(_BaseGrouper):
  return result, counts
 
 
-cdef inline _extract_result(object res):
+cdef inline _extract_result(object res, bint squeeze=True):
  """ extract the result object, it might be a 0-dim ndarray
  or a len-1 0-dim, or a scalar """
  if (not _is_sparse_array(res) and hasattr(res, 'values')
  and util.is_array(res.values)):
  res = res.values
- if not np.isscalar(res):
- if util.is_array(res):
- if res.ndim == 0:
- res = res.item()
- elif res.ndim == 1 and len(res) == 1:
- res = res[0]
+ if util.is_array(res):
+ if res.ndim == 0:
+ res = res.item()
+ elif squeeze and res.ndim == 1 and len(res) == 1:
+ res = res[0]
  return res
 
 

pandas/_libs/reduction.pyx

@@ -647,7 +647,7 @@ def _reduce(self, name, skipna=True, **kwargs):
  data[mask] = self._na_value
 
  op = getattr(nanops, "nan" + name)
- result = op(data, axis=0, skipna=skipna, mask=mask)
+ result = op(data, axis=0, skipna=skipna, mask=mask, **kwargs)
 
  # if we have a boolean op, don't coerce
  if name in ["any", "all"]: