Merge branch 'master' of https://github.com/pandas-dev/pandas into 29130

Author: phofl

.github/workflows/pre-commit.yml

@@ -0,0 +1,14 @@
+name: pre-commit
+
+on:
+ pull_request:
+ push:
+ branches: [master]
+
+jobs:
+ pre-commit:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions/setup-python@v2
+ - uses: pre-commit/action@v2.0.0

.github/workflows/stale-pr.yml

@@ -12,10 +12,10 @@ jobs:
  with:
  repo-token: ${{ secrets.GITHUB_TOKEN }}
  stale-pr-message: "This pull request is stale because it has been open for thirty days with no activity."
- skip-stale-pr-message: false
+ skip-stale-pr-message: true
  stale-pr-label: "Stale"
- exempt-pr-labels: "Needs Review,Blocked"
+ exempt-pr-labels: "Needs Review,Blocked,Needs Discussion"
  days-before-stale: 30
  days-before-close: -1
- remove-stale-when-updated: true
- debug-only: true
+ remove-stale-when-updated: false
+ debug-only: false

.pre-commit-config.yaml

@@ -1,32 +1,32 @@
 repos:
 - repo: https://github.com/python/black
- rev: 19.10b0
+ rev: 20.8b1
  hooks:
  - id: black
- language_version: python3
 - repo: https://gitlab.com/pycqa/flake8
- rev: 3.7.7
+ rev: 3.8.3
  hooks:
  - id: flake8
- language: python_venv
  additional_dependencies: [flake8-comprehensions>=3.1.0]
  - id: flake8
  name: flake8-pyx
- language: python_venv
  files: \.(pyx|pxd)$
  types:
  - file
  args: [--append-config=flake8/cython.cfg]
  - id: flake8
  name: flake8-pxd
- language: python_venv
  files: \.pxi\.in$
  types:
  - file
  args: [--append-config=flake8/cython-template.cfg]
-- repo: https://github.com/pre-commit/mirrors-isort
- rev: v4.3.21
+- repo: https://github.com/PyCQA/isort
+ rev: 5.2.2
  hooks:
  - id: isort
- language: python_venv
  exclude: ^pandas/__init__\.py$|^pandas/core/api\.py$
+- repo: https://github.com/asottile/pyupgrade
+ rev: v2.7.2
+ hooks:
+ - id: pyupgrade
+ args: [--py37-plus]

.travis.yml

@@ -42,7 +42,7 @@ matrix:
 
  - arch: arm64
  env:
- - JOB="3.7, arm64" PYTEST_WORKERS=8 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)"
+ - JOB="3.7, arm64" PYTEST_WORKERS=8 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
 
  - env:
  - JOB="3.7, locale" ENV_FILE="ci/deps/travis-37-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
@@ -58,10 +58,6 @@ matrix:
  services:
  - mysql
  - postgresql
- allow_failures:
- - arch: arm64
- env:
- - JOB="3.7, arm64" PYTEST_WORKERS=8 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)"
 
 
 before_install:

asv_bench/benchmarks/arithmetic.py

@@ -125,7 +125,7 @@ def setup(self, op):
  arr1 = np.random.randn(n_rows, int(n_cols / 2)).astype("f8")
  arr2 = np.random.randn(n_rows, int(n_cols / 2)).astype("f4")
  df = pd.concat(
- [pd.DataFrame(arr1), pd.DataFrame(arr2)], axis=1, ignore_index=True,
+ [pd.DataFrame(arr1), pd.DataFrame(arr2)], axis=1, ignore_index=True
  )
  # should already be the case, but just to be sure
  df._consolidate_inplace()

asv_bench/benchmarks/frame_methods.py

@@ -219,6 +219,46 @@ def time_to_html_mixed(self):
  self.df2.to_html()
 
 
+class ToNumpy:
+ def setup(self):
+ N = 10000
+ M = 10
+ self.df_tall = DataFrame(np.random.randn(N, M))
+ self.df_wide = DataFrame(np.random.randn(M, N))
+ self.df_mixed_tall = self.df_tall.copy()
+ self.df_mixed_tall["foo"] = "bar"
+ self.df_mixed_tall[0] = period_range("2000", periods=N)
+ self.df_mixed_tall[1] = range(N)
+ self.df_mixed_wide = self.df_wide.copy()
+ self.df_mixed_wide["foo"] = "bar"
+ self.df_mixed_wide[0] = period_range("2000", periods=M)
+ self.df_mixed_wide[1] = range(M)
+
+ def time_to_numpy_tall(self):
+ self.df_tall.to_numpy()
+
+ def time_to_numpy_wide(self):
+ self.df_wide.to_numpy()
+
+ def time_to_numpy_mixed_tall(self):
+ self.df_mixed_tall.to_numpy()
+
+ def time_to_numpy_mixed_wide(self):
+ self.df_mixed_wide.to_numpy()
+
+ def time_values_tall(self):
+ self.df_tall.values
+
+ def time_values_wide(self):
+ self.df_wide.values
+
+ def time_values_mixed_tall(self):
+ self.df_mixed_tall.values
+
+ def time_values_mixed_wide(self):
+ self.df_mixed_wide.values
+
+
 class Repr:
  def setup(self):
  nrows = 10000

asv_bench/benchmarks/strings.py

@@ -13,13 +13,20 @@ class Construction:
  param_names = ["dtype"]
 
  def setup(self, dtype):
- self.data = tm.rands_array(nchars=10 ** 5, size=10)
+ self.series_arr = tm.rands_array(nchars=10, size=10 ** 5)
+ self.frame_arr = self.series_arr.reshape((50_000, 2)).copy()
 
- def time_construction(self, dtype):
- Series(self.data, dtype=dtype)
+ def time_series_construction(self, dtype):
+ Series(self.series_arr, dtype=dtype)
 
- def peakmem_construction(self, dtype):
- Series(self.data, dtype=dtype)
+ def peakmem_series_construction(self, dtype):
+ Series(self.series_arr, dtype=dtype)
+
+ def time_frame_construction(self, dtype):
+ DataFrame(self.frame_arr, dtype=dtype)
+
+ def peakmem_frame_construction(self, dtype):
+ DataFrame(self.frame_arr, dtype=dtype)
 
 
 class Methods:

asv_bench/benchmarks/timeseries.py

@@ -263,6 +263,29 @@ def time_lookup_and_cleanup(self):
  self.ts.index._cleanup()
 
 
+class ToDatetimeFromIntsFloats:
+ def setup(self):
+ self.ts_sec = Series(range(1521080307, 1521685107), dtype="int64")
+ self.ts_sec_float = self.ts_sec.astype("float64")
+
+ self.ts_nanosec = 1_000_000 * self.ts_sec
+ self.ts_nanosec_float = self.ts_nanosec.astype("float64")
+
+ # speed of int64 and float64 paths should be comparable
+
+ def time_nanosec_int64(self):
+ to_datetime(self.ts_nanosec, unit="ns")
+
+ def time_nanosec_float64(self):
+ to_datetime(self.ts_nanosec_float, unit="ns")
+
+ def time_sec_int64(self):
+ to_datetime(self.ts_sec, unit="s")
+
+ def time_sec_float64(self):
+ to_datetime(self.ts_sec_float, unit="s")
+
+
 class ToDatetimeYYYYMMDD:
  def setup(self):
  rng = date_range(start="1/1/2000", periods=10000, freq="D")

ci/deps/travis-37-arm64.yaml

@@ -1,6 +1,5 @@
 name: pandas-dev
 channels:
- - defaults
  - conda-forge
 dependencies:
  - python=3.7.*

ci/deps/travis-37-cov.yaml

@@ -1,6 +1,5 @@
 name: pandas-dev
 channels:
- - defaults
  - conda-forge
 dependencies:
  - python=3.7.*
@@ -15,7 +14,6 @@ dependencies:
  # pandas dependencies
  - beautifulsoup4
  - botocore>=1.11
- - cython>=0.29.16
  - dask
  - fastparquet>=0.3.2
  - fsspec>=0.7.4
@@ -31,16 +29,18 @@ dependencies:
  - odfpy
  - openpyxl
  - pandas-gbq
+ - google-cloud-bigquery>=1.27.2 # GH 36436
  - psycopg2
  - pyarrow>=0.15.0
- - pymysql
+ - pymysql=0.7.11
  - pytables
  - python-snappy
+ - python-dateutil
  - pytz
  - s3fs>=0.4.0
  - scikit-learn
  - scipy
- - sqlalchemy
+ - sqlalchemy=1.3.0
  - statsmodels
  - xarray
  - xlrd
@@ -51,5 +51,4 @@ dependencies:
  - brotlipy
  - coverage
  - pandas-datareader
- - python-dateutil
  - pyxlsb