Merge branch 'master' of https://github.com/pandas-dev/pandas into typing-missing

Author: arw2019

ci/deps/azure-38.yaml

@@ -17,4 +17,4 @@ dependencies:
  - python-dateutil
  - nomkl
  - pytz
- - tabulate==0.8.3
+ - tabulate==0.8.7

doc/source/getting_started/install.rst

@@ -283,7 +283,7 @@ pyreadstat SPSS files (.sav) reading
 pyxlsb 1.0.6 Reading for xlsb files
 qtpy Clipboard I/O
 s3fs 0.4.0 Amazon S3 access
-tabulate 0.8.3 Printing in Markdown-friendly format (see `tabulate`_)
+tabulate 0.8.7 Printing in Markdown-friendly format (see `tabulate`_)
 xarray 0.12.3 pandas-like API for N-dimensional data
 xclip Clipboard I/O on linux
 xlrd 1.2.0 Excel reading

doc/source/reference/indexing.rst

@@ -290,6 +290,7 @@ MultiIndex properties
  MultiIndex.codes
  MultiIndex.nlevels
  MultiIndex.levshape
+ MultiIndex.dtypes
 
 MultiIndex components
 ~~~~~~~~~~~~~~~~~~~~~

doc/source/whatsnew/v1.2.0.rst

@@ -308,6 +308,7 @@ Other enhancements
 - Improved error reporting for subsetting columns of a :class:`.DataFrameGroupBy` with ``axis=1`` (:issue:`37725`)
 - Implement method ``cross`` for :meth:`DataFrame.merge` and :meth:`DataFrame.join` (:issue:`5401`)
 - When :func:`read_csv/sas/json` are called with ``chuncksize``/``iterator`` they can be used in a ``with`` statement as they return context-managers (:issue:`38225`)
+- Augmented the list of named colors available for styling Excel exports, enabling all of CSS4 colors (:issue:`38247`)
 
 .. ---------------------------------------------------------------------------
 
@@ -745,6 +746,7 @@ I/O
 - :meth:`read_fwf` was inferring compression with ``compression=None`` which was not consistent with the other :meth:``read_*`` functions (:issue:`37909`)
 - :meth:`DataFrame.to_html` was ignoring ``formatters`` argument for ``ExtensionDtype`` columns (:issue:`36525`)
 - Bumped minimum xarray version to 0.12.3 to avoid reference to the removed ``Panel`` class (:issue:`27101`)
+- :meth:`DataFrame.to_csv` was re-opening file-like handles that also implement ``os.PathLike`` (:issue:`38125`)
 
 Period
 ^^^^^^

doc/source/whatsnew/v1.3.0.rst

@@ -19,8 +19,8 @@ Enhancements
 Other enhancements
 ^^^^^^^^^^^^^^^^^^
 
--
--
+- Added :meth:`MultiIndex.dtypes` (:issue:`37062`)
+- Improve error message when ``usecols`` and ``names`` do not match for :func:`read_csv` and ``engine="c"`` (:issue:`29042`)
 
 .. ---------------------------------------------------------------------------
 
@@ -37,7 +37,76 @@ These are bug fixes that might have notable behavior changes.
 
 Increased minimum versions for dependencies
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
+Some minimum supported versions of dependencies were updated.
+If installed, we now require:
+
++-----------------+-----------------+----------+---------+
+| Package | Minimum Version | Required | Changed |
++=================+=================+==========+=========+
+| numpy | 1.16.5 | X | |
++-----------------+-----------------+----------+---------+
+| pytz | 2017.3 | X | |
++-----------------+-----------------+----------+---------+
+| python-dateutil | 2.7.3 | X | |
++-----------------+-----------------+----------+---------+
+| bottleneck | 1.2.1 | | |
++-----------------+-----------------+----------+---------+
+| numexpr | 2.6.8 | | |
++-----------------+-----------------+----------+---------+
+| pytest (dev) | 5.0.1 | | |
++-----------------+-----------------+----------+---------+
+| mypy (dev) | 0.782 | | |
++-----------------+-----------------+----------+---------+
+
+For `optional libraries <https://dev.pandas.io/docs/install.html#dependencies>`_ the general recommendation is to use the latest version.
+The following table lists the lowest version per library that is currently being tested throughout the development of pandas.
+Optional libraries below the lowest tested version may still work, but are not considered supported.
+
++-----------------+-----------------+---------+
+| Package | Minimum Version | Changed |
++=================+=================+=========+
+| beautifulsoup4 | 4.6.0 | |
++-----------------+-----------------+---------+
+| fastparquet | 0.3.2 | |
++-----------------+-----------------+---------+
+| fsspec | 0.7.4 | |
++-----------------+-----------------+---------+
+| gcsfs | 0.6.0 | |
++-----------------+-----------------+---------+
+| lxml | 4.3.0 | |
++-----------------+-----------------+---------+
+| matplotlib | 2.2.3 | |
++-----------------+-----------------+---------+
+| numba | 0.46.0 | |
++-----------------+-----------------+---------+
+| openpyxl | 2.6.0 | |
++-----------------+-----------------+---------+
+| pyarrow | 0.15.0 | |
++-----------------+-----------------+---------+
+| pymysql | 0.7.11 | |
++-----------------+-----------------+---------+
+| pytables | 3.5.1 | |
++-----------------+-----------------+---------+
+| s3fs | 0.4.0 | |
++-----------------+-----------------+---------+
+| scipy | 1.2.0 | |
++-----------------+-----------------+---------+
+| sqlalchemy | 1.2.8 | |
++-----------------+-----------------+---------+
+| tabulate | 0.8.7 | X |
++-----------------+-----------------+---------+
+| xarray | 0.12.0 | |
++-----------------+-----------------+---------+
+| xlrd | 1.2.0 | |
++-----------------+-----------------+---------+
+| xlsxwriter | 1.0.2 | |
++-----------------+-----------------+---------+
+| xlwt | 1.3.0 | |
++-----------------+-----------------+---------+
+| pandas-gbq | 0.12.0 | |
++-----------------+-----------------+---------+
+
+See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more.
 
 .. _whatsnew_130.api.other:
 
@@ -86,7 +155,7 @@ Categorical
 
 Datetimelike
 ^^^^^^^^^^^^
-
+- Bug in :class:`DataFrame` and :class:`Series` constructors sometimes dropping nanoseconds from :class:`Timestamp` (resp. :class:`Timedelta`) ``data``, with ``dtype=datetime64[ns]`` (resp. ``timedelta64[ns]``) (:issue:`38032`)
 -
 -
 
@@ -127,7 +196,9 @@ Interval
 
 Indexing
 ^^^^^^^^
+- Bug in :meth:`CategoricalIndex.get_indexer` failing to raise ``InvalidIndexError`` when non-unique (:issue:`38372`)
 - Bug in inserting many new columns into a :class:`DataFrame` causing incorrect subsequent indexing behavior (:issue:`38380`)
+- Bug in :meth:`DataFrame.iloc.__setitem__` and :meth:`DataFrame.loc.__setitem__` with mixed dtypes when setting with a dictionary value (:issue:`38335`)
 -
 -
 
@@ -146,7 +217,11 @@ MultiIndex
 I/O
 ^^^
 
--
+- Bug in :func:`read_csv` interpreting ``NA`` value as comment, when ``NA`` does contain the comment string fixed for ``engine="python"`` (:issue:`34002`)
+- Bug in :func:`read_csv` raising ``IndexError`` with multiple header columns and ``index_col`` specified when file has no data rows (:issue:`38292`)
+- Bug in :func:`read_csv` not accepting ``usecols`` with different length than ``names`` for ``engine="python"`` (:issue:`16469`)
+- Bug in :func:`read_csv` raising ``TypeError`` when ``names`` and ``parse_dates`` is specified for ``engine="c"`` (:issue:`33699`)
+- Allow custom error values for parse_dates argument of :func:`read_sql`, :func:`read_sql_query` and :func:`read_sql_table` (:issue:`35185`)
 -
 
 Period

pandas/_libs/parsers.pyx

@@ -738,8 +738,8 @@ cdef class TextReader:
  elif self.names is None and nuse < passed_count:
  self.leading_cols = field_count - passed_count
  elif passed_count != field_count:
- raise ValueError('Passed header names '
- 'mismatches usecols')
+ raise ValueError('Number of passed names did not match number of '
+ 'header fields in the file')
  # oh boy, #2442, #2981
  elif self.allow_leading_cols and passed_count < field_count:
  self.leading_cols = field_count - passed_count

pandas/compat/_optional.py

@@ -24,7 +24,7 @@
  "scipy": "1.2.0",
  "sqlalchemy": "1.2.8",
  "tables": "3.5.1",
- "tabulate": "0.8.3",
+ "tabulate": "0.8.7",
  "xarray": "0.12.3",
  "xlrd": "1.2.0",
  "xlwt": "1.3.0",

pandas/core/algorithms.py

@@ -50,7 +50,7 @@
 from pandas.core.dtypes.generic import (
  ABCDatetimeArray,
  ABCExtensionArray,
- ABCIndexClass,
+ ABCIndex,
  ABCMultiIndex,
  ABCRangeIndex,
  ABCSeries,
@@ -63,6 +63,7 @@
 
 if TYPE_CHECKING:
  from pandas import Categorical, DataFrame, Index, Series
+ from pandas.core.arrays import DatetimeArray, TimedeltaArray
 
 _shared_docs: Dict[str, str] = {}
 
@@ -215,7 +216,7 @@ def _reconstruct_data(
  values = values.astype(dtype, copy=False)
 
  # we only support object dtypes bool Index
- if isinstance(original, ABCIndexClass):
+ if isinstance(original, ABCIndex):
  values = values.astype(object, copy=False)
  elif dtype is not None:
  if is_datetime64_dtype(dtype):
@@ -437,9 +438,7 @@ def isin(comps: AnyArrayLike, values: AnyArrayLike) -> np.ndarray:
  f"to isin(), you passed a [{type(values).__name__}]"
  )
 
- if not isinstance(
- values, (ABCIndexClass, ABCSeries, ABCExtensionArray, np.ndarray)
- ):
+ if not isinstance(values, (ABCIndex, ABCSeries, ABCExtensionArray, np.ndarray)):
  values = _ensure_arraylike(list(values))
  elif isinstance(values, ABCMultiIndex):
  # Avoid raising in extract_array
@@ -700,7 +699,7 @@ def factorize(
  and values.freq is not None
  ):
  codes, uniques = values.factorize(sort=sort)
- if isinstance(original, ABCIndexClass):
+ if isinstance(original, ABCIndex):
  uniques = original._shallow_copy(uniques, name=None)
  elif isinstance(original, ABCSeries):
  from pandas import Index
@@ -739,8 +738,11 @@ def factorize(
  uniques = _reconstruct_data(uniques, dtype, original)
 
  # return original tenor
- if isinstance(original, ABCIndexClass):
+ if isinstance(original, ABCIndex):
  if original.dtype.kind in ["m", "M"] and isinstance(uniques, np.ndarray):
+ original._data = cast(
+ "Union[DatetimeArray, TimedeltaArray]", original._data
+ )
  uniques = type(original._data)._simple_new(uniques, dtype=original.dtype)
  uniques = original._shallow_copy(uniques, name=None)
  elif isinstance(original, ABCSeries):

pandas/core/arrays/__init__.py

@@ -7,7 +7,7 @@
 from pandas.core.arrays.categorical import Categorical
 from pandas.core.arrays.datetimes import DatetimeArray
 from pandas.core.arrays.floating import FloatingArray
-from pandas.core.arrays.integer import IntegerArray, integer_array
+from pandas.core.arrays.integer import IntegerArray
 from pandas.core.arrays.interval import IntervalArray
 from pandas.core.arrays.masked import BaseMaskedArray
 from pandas.core.arrays.numpy_ import PandasArray, PandasDtype
@@ -26,7 +26,6 @@
  "DatetimeArray",
  "FloatingArray",
  "IntegerArray",
- "integer_array",
  "IntervalArray",
  "PandasArray",
  "PandasDtype",

pandas/core/arrays/base.py

@@ -41,7 +41,7 @@
  pandas_dtype,
 )
 from pandas.core.dtypes.dtypes import ExtensionDtype
-from pandas.core.dtypes.generic import ABCDataFrame, ABCIndexClass, ABCSeries
+from pandas.core.dtypes.generic import ABCDataFrame, ABCIndex, ABCSeries
 from pandas.core.dtypes.missing import isna
 
 from pandas.core import ops
@@ -1361,7 +1361,7 @@ def convert_values(param):
  ovalues = [param] * len(self)
  return ovalues
 
- if isinstance(other, (ABCSeries, ABCIndexClass, ABCDataFrame)):
+ if isinstance(other, (ABCSeries, ABCIndex, ABCDataFrame)):
  # rely on pandas to unbox and dispatch to us
  return NotImplemented