Merge branch 'main' of https://github.com/pandas-dev/pandas into object_reduction_axis_1_attempt_2

� Conflicts: �	doc/source/whatsnew/v2.0.0.rst

Author: rhshadrach

.devcontainer.json

@@ -9,8 +9,7 @@
 // You can edit these settings after create using File > Preferences > Settings > Remote.
 "settings": {
 "terminal.integrated.shell.linux": "/bin/bash",
-"python.condaPath": "/opt/conda/bin/conda",
-"python.pythonPath": "/opt/conda/bin/python",
+"python.pythonPath": "/usr/local/bin/python",
 "python.formatting.provider": "black",
 "python.linting.enabled": true,
 "python.linting.flake8Enabled": true,

.github/workflows/32-bit-linux.yml

@@ -38,7 +38,8 @@ jobs:
  /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
  . ~/virtualenvs/pandas-dev/bin/activate && \
  python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
- pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
+ python -m pip install versioneer[toml] && \
+ python -m pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
  python setup.py build_ext -q -j1 && \
  python -m pip install --no-build-isolation --no-use-pep517 -e . && \
  python -m pip list && \

.github/workflows/code-checks.yml

@@ -36,9 +36,11 @@ jobs:
 
  - name: Run pre-commit
  uses: pre-commit/action@v2.0.3
+ with:
+ extra_args: --verbose --all-files
 
- typing_and_docstring_validation:
- name: Docstring and typing validation
+ docstring_typing_pylint:
+ name: Docstring validation, typing, and pylint
  runs-on: ubuntu-latest
  defaults:
  run:
@@ -79,6 +81,10 @@ jobs:
  run: ci/code_checks.sh docstrings
  if: ${{ steps.build.outcome == 'success' && always() }}
 
+ - name: Run check of documentation notebooks
+ run: ci/code_checks.sh notebooks
+ if: ${{ steps.build.outcome == 'success' && always() }}
+
  - name: Use existing environment for type checking
  run: |
  echo $PATH >> $GITHUB_PATH
@@ -89,7 +95,7 @@ jobs:
  - name: Typing + pylint
  uses: pre-commit/action@v2.0.3
  with:
- extra_args: --hook-stage manual --all-files
+ extra_args: --verbose --hook-stage manual --all-files
  if: ${{ steps.build.outcome == 'success' && always() }}
 
  - name: Run docstring validation script tests
@@ -152,7 +158,7 @@ jobs:
  run: docker build --pull --no-cache --tag pandas-dev-env .
 
  - name: Show environment
- run: docker run -w /home/pandas pandas-dev-env mamba run -n pandas-dev python -c "import pandas as pd; print(pd.show_versions())"
+ run: docker run --rm pandas-dev-env python -c "import pandas as pd; print(pd.show_versions())"
 
  requirements-dev-text-installable:
  name: Test install requirements-dev.txt

.github/workflows/docbuild-and-upload.yml

@@ -54,11 +54,6 @@ jobs:
  - name: Build documentation zip
  run: doc/make.py zip_html
 
- - name: Build the interactive terminal
- run: |
- cd web/interactive_terminal
- jupyter lite build
-
  - name: Install ssh key
  run: |
  mkdir -m 700 -p ~/.ssh

.github/workflows/package-checks.yml

@@ -43,6 +43,7 @@ jobs:
  - name: Install required dependencies
  run: |
  python -m pip install --upgrade pip setuptools wheel python-dateutil pytz numpy cython
+ python -m pip install versioneer[toml]
  shell: bash -el {0}
 
  - name: Pip install with extra

.github/workflows/python-dev.yml

@@ -75,6 +75,7 @@ jobs:
  python -m pip install --upgrade pip setuptools wheel
  python -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
  python -m pip install git+https://github.com/nedbat/coveragepy.git
+ python -m pip install versioneer[toml]
  python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17
  python -m pip list
 

.github/workflows/sdist.yml

@@ -47,6 +47,7 @@ jobs:
  - name: Install dependencies
  run: |
  python -m pip install --upgrade pip setuptools wheel
+ python -m pip install versioneer[toml]
 
  # GH 39416
  pip install numpy

.gitpod.yml

@@ -0,0 +1,58 @@
+# Building pandas on init
+# Might delegate this later to prebuild with Q2 improvements on gitpod
+# https://www.gitpod.io/docs/config-start-tasks/#configuring-the-terminal
+# -------------------------------------------------------------------------
+
+# assuming we use dockerhub: name of the docker user, docker image, tag, e.g. https://hub.docker.com/r/pandas/pandas-gitpod/tags
+image: pythonpandas/pandas-gitpod:latest
+tasks:
+ - name: Prepare development environment
+ init: |
+ mkdir -p .vscode
+ cp gitpod/settings.json .vscode/settings.json
+ conda activate pandas-dev
+ git pull --unshallow # need to force this else the prebuild fails
+ git fetch --tags
+ python setup.py build_ext -j 4
+ python -m pip install -e . --no-build-isolation
+ echo "🛠 Completed rebuilding Pandas!! 🛠 "
+ echo "✨ Pre-build complete! You can close this terminal ✨ "
+
+# --------------------------------------------------------
+# exposing ports for liveserve
+ports:
+ - port: 5500
+ onOpen: notify
+
+# --------------------------------------------------------
+# some useful extensions to have
+vscode:
+ extensions:
+ - ms-python.python
+ - yzhang.markdown-all-in-one
+ - eamodio.gitlens
+ - lextudio.restructuredtext
+ # add or remove what you think is generally useful to most contributors
+ # avoid adding too many. they each open a pop-up window
+
+# --------------------------------------------------------
+# using prebuilds for the container
+# With this configuration the prebuild will happen on push to main
+github:
+ prebuilds:
+ # enable for main/default branch
+ main: true
+ # enable for other branches (defaults to false)
+ branches: false
+ # enable for pull requests coming from this repo (defaults to true)
+ pullRequests: false
+ # enable for pull requests coming from forks (defaults to false)
+ pullRequestsFromForks: false
+ # add a check to pull requests (defaults to true)
+ addCheck: false
+ # add a "Review in Gitpod" button as a comment to pull requests (defaults to false)
+ addComment: false
+ # add a "Review in Gitpod" button to the pull request's description (defaults to false)
+ addBadge: false
+ # add a label once the prebuild is ready to pull requests (defaults to false)
+ addLabel: false

.pre-commit-config.yaml

@@ -17,21 +17,18 @@ repos:
  entry: python scripts/run_vulture.py
  pass_filenames: true
  require_serial: false
-- repo: https://github.com/python/black
- rev: 22.10.0
- hooks:
- - id: black
 - repo: https://github.com/codespell-project/codespell
  rev: v2.2.2
  hooks:
  - id: codespell
  types_or: [python, rst, markdown]
+ additional_dependencies: [tomli]
 - repo: https://github.com/MarcoGorelli/cython-lint
- rev: v0.2.1
+ rev: v0.9.1
  hooks:
  - id: cython-lint
 - repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.3.0
+ rev: v4.4.0
  hooks:
  - id: debug-statements
  - id: end-of-file-fixer
@@ -50,26 +47,43 @@ repos:
  exclude: ^pandas/_libs/src/(klib|headers)/
  args: [--quiet, '--extensions=c,h', '--headers=h', --recursive, '--filter=-readability/casting,-runtime/int,-build/include_subdir']
 - repo: https://github.com/PyCQA/flake8
- rev: 5.0.4
+ rev: 6.0.0
  hooks:
  - id: flake8
  # Need to patch os.remove rule in pandas-dev-flaker
  exclude: ^ci/fix_wheels.py
  additional_dependencies: &flake8_dependencies
- - flake8==5.0.4
+ - flake8==6.0.0
  - flake8-bugbear==22.7.1
  - pandas-dev-flaker==0.5.0
 - repo: https://github.com/pycqa/pylint
- rev: v2.15.5
+ rev: v2.15.6
+ hooks:
+ - id: pylint
+ stages: [manual]
+- repo: https://github.com/pycqa/pylint
+ rev: v2.15.6
  hooks:
  - id: pylint
+ alias: redefined-outer-name
+ name: Redefining name from outer scope
+ files: ^pandas/
+ exclude: |
+ (?x)
+ ^pandas/tests # keep excluded
+ |/_testing/ # keep excluded
+ |^pandas/util/_test_decorators\.py # keep excluded
+ |^pandas/_version\.py # keep excluded
+ |^pandas/conftest\.py # keep excluded
+ |^pandas/core/generic\.py
+ args: [--disable=all, --enable=redefined-outer-name]
  stages: [manual]
 - repo: https://github.com/PyCQA/isort
  rev: 5.10.1
  hooks:
  - id: isort
 - repo: https://github.com/asottile/pyupgrade
- rev: v3.2.0
+ rev: v3.2.2
  hooks:
  - id: pyupgrade
  args: [--py38-plus]
@@ -94,6 +108,16 @@ repos:
  additional_dependencies: *flake8_dependencies
 - repo: local
  hooks:
+ # NOTE: we make `black` a local hook because if it's installed from
+ # PyPI (rather than from source) then it'll run twice as fast thanks to mypyc
+ - id: black
+ name: black
+ description: "Black: The uncompromising Python code formatter"
+ entry: black
+ language: python
+ require_serial: true
+ types_or: [python, pyi]
+ additional_dependencies: [black==22.10.0]
  - id: pyright
  # note: assumes python env is setup and activated
  name: pyright
@@ -201,7 +225,7 @@ repos:
  entry: python scripts/sync_flake8_versions.py
  files: ^(\.pre-commit-config\.yaml|environment\.yml)$
  pass_filenames: false
- additional_dependencies: [pyyaml]
+ additional_dependencies: [pyyaml, toml]
  - id: title-capitalization
  name: Validate correct capitalization among titles in documentation
  entry: python scripts/validate_rst_title_capitalization.py
@@ -250,6 +274,7 @@ repos:
  entry: python scripts/validate_min_versions_in_sync.py
  language: python
  files: ^(ci/deps/actions-.*-minimum_versions\.yaml|pandas/compat/_optional\.py)$
+ additional_dependencies: [tomli]
  - id: validate-errors-locations
  name: Validate errors locations
  description: Validate errors are in appropriate locations.

Dockerfile

@@ -1,42 +1,13 @@
-FROM quay.io/condaforge/mambaforge
+FROM python:3.10.8
+WORKDIR /home/pandas
 
-# if you forked pandas, you can pass in your own GitHub username to use your fork
-# i.e. gh_username=myname
-ARG gh_username=pandas-dev
-ARG pandas_home="/home/pandas"
+RUN apt-get update && apt-get -y upgrade
+RUN apt-get install -y build-essential
 
-# Avoid warnings by switching to noninteractive
-ENV DEBIAN_FRONTEND=noninteractive
+# hdf5 needed for pytables installation
+RUN apt-get install -y libhdf5-dev
 
-# Configure apt and install packages
-RUN apt-get update \
- && apt-get -y install --no-install-recommends apt-utils git tzdata dialog 2>&1 \
- #
- # Configure timezone (fix for tests which try to read from "/etc/localtime")
- && ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime \
- && dpkg-reconfigure -f noninteractive tzdata \
- #
- # cleanup
- && apt-get autoremove -y \
- && apt-get clean -y \
- && rm -rf /var/lib/apt/lists/*
-
-# Switch back to dialog for any ad-hoc use of apt-get
-ENV DEBIAN_FRONTEND=dialog
-
-# Clone pandas repo
-RUN mkdir "$pandas_home" \
- && git clone "https://github.com/$gh_username/pandas.git" "$pandas_home" \
- && cd "$pandas_home" \
- && git remote add upstream "https://github.com/pandas-dev/pandas.git" \
- && git pull upstream main
-
-# Set up environment
-RUN mamba env create -f "$pandas_home/environment.yml"
-
-# Build C extensions and pandas
-SHELL ["mamba", "run", "--no-capture-output", "-n", "pandas-dev", "/bin/bash", "-c"]
-RUN cd "$pandas_home" \
- && export \
- && python setup.py build_ext -j 4 \
- && python -m pip install --no-build-isolation -e .
+RUN python -m pip install --upgrade pip
+RUN python -m pip install --use-deprecated=legacy-resolver \
+ -r https://raw.githubusercontent.com/pandas-dev/pandas/main/requirements-dev.txt
+CMD ["/bin/bash"]