Merge branch 'main' of https://github.com/pandas-dev/pandas into object_reduction_axis_1_attempt_2

Author: rhshadrach

.github/workflows/package-checks.yml

@@ -9,12 +9,14 @@ on:
  branches:
  - main
  - 1.5.x
+ types: [ labeled, opened, synchronize, reopened ]
 
 permissions:
  contents: read
 
 jobs:
  pip:
+ if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}}
  runs-on: ubuntu-latest
  strategy:
  matrix:
@@ -38,13 +40,6 @@ jobs:
  with:
  python-version: '3.8'
 
- # Hacky patch to disable building cython extensions.
- # This job should only check that the extras successfully install.
- - name: Disable building ext_modules
- run: |
- sed -i '/ext_modules=/d' setup.py
- shell: bash -el {0}
-
  - name: Install required dependencies
  run: |
  python -m pip install --upgrade pip setuptools wheel python-dateutil pytz numpy cython

asv_bench/benchmarks/groupby.py

@@ -671,12 +671,8 @@ class String:
  ["str", "string[python]"],
  [
  "sum",
- "prod",
  "min",
  "max",
- "mean",
- "median",
- "var",
  "first",
  "last",
  "any",

asv_bench/benchmarks/series_methods.py

@@ -79,6 +79,48 @@ def time_dropna(self, dtype):
  self.s.dropna()
 
 
+class Fillna:
+
+ params = [
+ [
+ "datetime64[ns]",
+ "float64",
+ "Int64",
+ "int64[pyarrow]",
+ "string",
+ "string[pyarrow]",
+ ],
+ [None, "pad", "backfill"],
+ ]
+ param_names = ["dtype", "method"]
+
+ def setup(self, dtype, method):
+ N = 10**6
+ if dtype == "datetime64[ns]":
+ data = date_range("2000-01-01", freq="S", periods=N)
+ na_value = NaT
+ elif dtype == "float64":
+ data = np.random.randn(N)
+ na_value = np.nan
+ elif dtype in ("Int64", "int64[pyarrow]"):
+ data = np.arange(N)
+ na_value = NA
+ elif dtype in ("string", "string[pyarrow]"):
+ data = tm.rands_array(5, N)
+ na_value = NA
+ else:
+ raise NotImplementedError
+ fill_value = data[0]
+ ser = Series(data, dtype=dtype)
+ ser[::2] = na_value
+ self.ser = ser
+ self.fill_value = fill_value
+
+ def time_fillna(self, dtype, method):
+ value = self.fill_value if method is None else None
+ self.ser.fillna(value=value, method=method)
+
+
 class SearchSorted:
 
  goal_time = 0.2

doc/source/_static/css/pandas.css

@@ -5,6 +5,10 @@
  --pst-color-info: 23, 162, 184;
 }
 
+table {
+ width: auto; /* Override fit-content which breaks Styler user guide ipynb */
+}
+
 /* Main index page overview cards */
 
 .intro-card {

doc/source/development/contributing.rst

@@ -114,6 +114,7 @@ version control to allow many people to work together on the project.
 
 Some great resources for learning Git:
 
+* the `Git documentation <https://git-scm.com/doc>`_.
 * the `GitHub help pages <https://help.github.com/>`_.
 * the `NumPy documentation <https://numpy.org/doc/stable/dev/index.html>`_.
 * Matthew Brett's `Pydagogue <https://matthew-brett.github.io/pydagogue/>`_.

doc/source/user_guide/io.rst

@@ -3827,22 +3827,28 @@ format of an Excel worksheet created with the ``to_excel`` method. Excellent ex
 OpenDocument Spreadsheets
 -------------------------
 
-.. versionadded:: 0.25
-
-The :func:`~pandas.read_excel` method can also read OpenDocument spreadsheets
-using the ``odfpy`` module. The semantics and features for reading
+The io methods for `Excel files`_ also support reading and writing OpenDocument spreadsheets
+using the `odfpy <https://pypi.org/project/odfpy/>`__ module. The semantics and features for reading and writing
 OpenDocument spreadsheets match what can be done for `Excel files`_ using
 ``engine='odf'``.
 
+.. versionadded:: 0.25
+
+The :func:`~pandas.read_excel` method can read OpenDocument spreadsheets
+
 .. code-block:: python
 
  # Returns a DataFrame
  pd.read_excel("path_to_file.ods", engine="odf")
 
-.. note::
+.. versionadded:: 1.1.0
 
- Currently pandas only supports *reading* OpenDocument spreadsheets. Writing
- is not implemented.
+Similarly, the :func:`~pandas.to_excel` method can write OpenDocument spreadsheets
+
+.. code-block:: python
+
+ # Writes DataFrame to a .ods file
+ df.to_excel("path_to_file.ods", engine="odf")
 
 .. _io.xlsb:
 

doc/source/whatsnew/index.rst

@@ -24,6 +24,7 @@ Version 1.5
 .. toctree::
  :maxdepth: 2
 
+ v1.5.3
  v1.5.2
  v1.5.1
  v1.5.0

doc/source/whatsnew/v1.5.2.rst

@@ -1,6 +1,6 @@
 .. _whatsnew_152:
 
-What's new in 1.5.2 (November ??, 2022)
+What's new in 1.5.2 (November 21, 2022)
 ---------------------------------------
 
 These are the changes in pandas 1.5.2. See :ref:`release` for a full changelog
@@ -36,7 +36,6 @@ Bug fixes
 Other
 ~~~~~
 - Reverted ``color`` as an alias for ``c`` and ``size`` as an alias for ``s`` in function :meth:`DataFrame.plot.scatter` (:issue:`49732`)
--
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_152.contributors:

doc/source/whatsnew/v1.5.3.rst

@@ -0,0 +1,41 @@
+.. _whatsnew_153:
+
+What's new in 1.5.3 (December ??, 2022)
+---------------------------------------
+
+These are the changes in pandas 1.5.3. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_153.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+- Fixed performance regression in :meth:`Series.isin` when ``values`` is empty (:issue:`49839`)
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_153.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_153.other:
+
+Other
+~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_153.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v1.5.2..v1.5.3|HEAD

doc/source/whatsnew/v2.0.0.rst

@@ -33,7 +33,7 @@ sql-other, html, xml, plot, output_formatting, clipboard, compression, test]`` (
 Configuration option, ``io.nullable_backend``, to return pyarrow-backed dtypes from IO functions
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-A new global configuration, ``io.nullable_backend`` can now be used in conjunction with the parameter ``use_nullable_dtypes=True`` in :func:`read_parquet` and :func:`read_csv` (with ``engine="pyarrow"``)
+A new global configuration, ``io.nullable_backend`` can now be used in conjunction with the parameter ``use_nullable_dtypes=True`` in :func:`read_parquet`, :func:`read_orc` and :func:`read_csv` (with ``engine="pyarrow"``)
 to return pyarrow-backed dtypes when set to ``"pyarrow"`` (:issue:`48957`).
 
 .. ipython:: python
@@ -45,7 +45,7 @@ to return pyarrow-backed dtypes when set to ``"pyarrow"`` (:issue:`48957`).
  """)
  with pd.option_context("io.nullable_backend", "pyarrow"):
  df = pd.read_csv(data, use_nullable_dtypes=True, engine="pyarrow")
- df
+ df.dtypes
 
 .. _whatsnew_200.enhancements.other:
 
@@ -62,6 +62,7 @@ Other enhancements
 - Fix ``test`` optional_extra by adding missing test package ``pytest-asyncio`` (:issue:`48361`)
 - :func:`DataFrame.astype` exception message thrown improved to include column name when type conversion is not possible. (:issue:`47571`)
 - :func:`date_range` now supports a ``unit`` keyword ("s", "ms", "us", or "ns") to specify the desired resolution of the output index (:issue:`49106`)
+- :func:`timedelta_range` now supports a ``unit`` keyword ("s", "ms", "us", or "ns") to specify the desired resolution of the output index (:issue:`49824`)
 - :meth:`DataFrame.to_json` now supports a ``mode`` keyword with supported inputs 'w' and 'a'. Defaulting to 'w', 'a' can be used when lines=True and orient='records' to append record oriented json lines to an existing json file. (:issue:`35849`)
 - Added ``name`` parameter to :meth:`IntervalIndex.from_breaks`, :meth:`IntervalIndex.from_arrays` and :meth:`IntervalIndex.from_tuples` (:issue:`48911`)
 -
@@ -336,6 +337,7 @@ Other API changes
 - Passing ``dtype`` of "timedelta64[s]", "timedelta64[ms]", or "timedelta64[us]" to :class:`TimedeltaIndex`, :class:`Series`, or :class:`DataFrame` constructors will now retain that dtype instead of casting to "timedelta64[ns]"; passing a dtype with lower resolution for :class:`Series` or :class:`DataFrame` will be cast to the lowest supported resolution "timedelta64[s]" (:issue:`49014`)
 - Passing a ``np.datetime64`` object with non-nanosecond resolution to :class:`Timestamp` will retain the input resolution if it is "s", "ms", or "ns"; otherwise it will be cast to the closest supported resolution (:issue:`49008`)
 - The ``other`` argument in :meth:`DataFrame.mask` and :meth:`Series.mask` now defaults to ``no_default`` instead of ``np.nan`` consistent with :meth:`DataFrame.where` and :meth:`Series.where`. Entries will be filled with the corresponding NULL value (``np.nan`` for numpy dtypes, ``pd.NA`` for extension dtypes). (:issue:`49111`)
+- Changed behavior of :meth:`Series.quantile` and :meth:`DataFrame.quantile` with :class:`SparseDtype` to retain sparse dtype (:issue:`49583`)
 - When creating a :class:`Series` with a object-dtype :class:`Index` of datetime objects, pandas no longer silently converts the index to a :class:`DatetimeIndex` (:issue:`39307`, :issue:`23598`)
 - :meth:`Series.unique` with dtype "timedelta64[ns]" or "datetime64[ns]" now returns :class:`TimedeltaArray` or :class:`DatetimeArray` instead of ``numpy.ndarray`` (:issue:`49176`)
 - :func:`to_datetime` and :class:`DatetimeIndex` now allow sequences containing both ``datetime`` objects and numeric entries, matching :class:`Series` behavior (:issue:`49037`)
@@ -346,6 +348,8 @@ Other API changes
 - Changed behavior of :class:`Index` constructor with sequence containing at least one ``NaT`` and everything else either ``None`` or ``NaN`` to infer ``datetime64[ns]`` dtype instead of ``object``, matching :class:`Series` behavior (:issue:`49340`)
 - :func:`read_stata` with parameter ``index_col`` set to ``None`` (the default) will now set the index on the returned :class:`DataFrame` to a :class:`RangeIndex` instead of a :class:`Int64Index` (:issue:`49745`)
 - Changed behavior of :class:`Index` constructor with an object-dtype ``numpy.ndarray`` containing all-``bool`` values or all-complex values, this will now retain object dtype, consistent with the :class:`Series` behavior (:issue:`49594`)
+- Changed behavior of :meth:`DataFrame.shift` with ``axis=1``, an integer ``fill_value``, and homogeneous datetime-like dtype, this now fills new columns with integer dtypes instead of casting to datetimelike (:issue:`49842`)
+- :meth:`DataFrame.values`, :meth:`DataFrame.to_numpy`, :meth:`DataFrame.xs`, :meth:`DataFrame.reindex`, :meth:`DataFrame.fillna`, and :meth:`DataFrame.replace` no longer silently consolidate the underlying arrays; do ``df = df.copy()`` to ensure consolidation (:issue:`49356`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -584,6 +588,7 @@ Performance improvements
 - Performance improvement in :meth:`.DataFrameGroupBy.mean`, :meth:`.SeriesGroupBy.mean`, :meth:`.DataFrameGroupBy.var`, and :meth:`.SeriesGroupBy.var` for extension array dtypes (:issue:`37493`)
 - Performance improvement in :meth:`MultiIndex.isin` when ``level=None`` (:issue:`48622`, :issue:`49577`)
 - Performance improvement in :meth:`Index.union` and :meth:`MultiIndex.union` when index contains duplicates (:issue:`48900`)
+- Performance improvement in :meth:`Series.fillna` for pyarrow-backed dtypes (:issue:`49722`)
 - Performance improvement for :meth:`Series.value_counts` with nullable dtype (:issue:`48338`)
 - Performance improvement for :class:`Series` constructor passing integer numpy array with nullable dtype (:issue:`48338`)
 - Performance improvement for :class:`DatetimeIndex` constructor passing a list (:issue:`48609`)
@@ -597,12 +602,14 @@ Performance improvements
 - Performance improvement in :meth:`DataFrame.join` when joining on a subset of a :class:`MultiIndex` (:issue:`48611`)
 - Performance improvement for :meth:`MultiIndex.intersection` (:issue:`48604`)
 - Performance improvement in ``var`` for nullable dtypes (:issue:`48379`).
+- Performance improvement when iterating over a :class:`~arrays.ArrowExtensionArray` (:issue:`49825`).
 - Performance improvements to :func:`read_sas` (:issue:`47403`, :issue:`47405`, :issue:`47656`, :issue:`48502`)
 - Memory improvement in :meth:`RangeIndex.sort_values` (:issue:`48801`)
 - Performance improvement in :class:`DataFrameGroupBy` and :class:`SeriesGroupBy` when ``by`` is a categorical type and ``sort=False`` (:issue:`48976`)
 - Performance improvement in :class:`DataFrameGroupBy` and :class:`SeriesGroupBy` when ``by`` is a categorical type and ``observed=False`` (:issue:`49596`)
 - Performance improvement in :func:`read_stata` with parameter ``index_col`` set to ``None`` (the default). Now the index will be a :class:`RangeIndex` instead of :class:`Int64Index` (:issue:`49745`)
 - Performance improvement in :func:`merge` when not merging on the index - the new index will now be :class:`RangeIndex` instead of :class:`Int64Index` (:issue:`49478`)
+- Performance improvement in :meth:`DataFrame.to_dict` and :meth:`Series.to_dict` when using any non-object dtypes (:issue:`46470`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_200.bug_fixes:
@@ -652,6 +659,8 @@ Conversion
 - Bug where any :class:`ExtensionDtype` subclass with ``kind="M"`` would be interpreted as a timezone type (:issue:`34986`)
 - Bug in :class:`.arrays.ArrowExtensionArray` that would raise ``NotImplementedError`` when passed a sequence of strings or binary (:issue:`49172`)
 - Bug in :func:`to_datetime` was not respecting ``exact`` argument when ``format`` was an ISO8601 format (:issue:`12649`)
+- Bug in :meth:`TimedeltaArray.astype` raising ``TypeError`` when converting to a pyarrow duration type (:issue:`49795`)
+-
 
 Strings
 ^^^^^^^