ERR non-ISO formats don't show position of error (#50366)

* wip * unify messages * fixup regexes * simplify * fix test * remove (search) and (match) from messages * simplify Co-authored-by: MarcoGorelli <>

Author: MarcoGorelli

doc/source/whatsnew/v2.0.0.rst

@@ -101,6 +101,7 @@ Other enhancements
 - Added :meth:`Index.infer_objects` analogous to :meth:`Series.infer_objects` (:issue:`50034`)
 - Added ``copy`` parameter to :meth:`Series.infer_objects` and :meth:`DataFrame.infer_objects`, passing ``False`` will avoid making copies for series or columns that are already non-object or where no better dtype can be inferred (:issue:`50096`)
 - :meth:`DataFrame.plot.hist` now recognizes ``xlabel`` and ``ylabel`` arguments (:issue:`49793`)
+- Improved error message in :func:`to_datetime` for non-ISO8601 formats, informing users about the position of the first error (:issue:`50361`)
 - Improved error message when trying to align :class:`DataFrame` objects (for example, in :func:`DataFrame.compare`) to clarify that "identically labelled" refers to both index and columns (:issue:`50083`)
 -
 

pandas/_libs/tslibs/parsing.pyx

@@ -263,7 +263,7 @@ def parse_datetime_string(
  datetime dt
 
  if not _does_string_look_like_datetime(date_string):
- raise ValueError(f"Given date string {date_string} not likely a datetime")
+ raise ValueError(f'Given date string "{date_string}" not likely a datetime')
 
  if does_string_look_like_time(date_string):
  # use current datetime as default, not pass _DEFAULT_DATETIME
@@ -297,7 +297,7 @@ def parse_datetime_string(
  except TypeError:
  # following may be raised from dateutil
  # TypeError: 'NoneType' object is not iterable
- raise ValueError(f"Given date string {date_string} not likely a datetime")
+ raise ValueError(f'Given date string "{date_string}" not likely a datetime')
 
  return dt
 
@@ -373,7 +373,7 @@ cdef parse_datetime_string_with_reso(
  int out_tzoffset
 
  if not _does_string_look_like_datetime(date_string):
- raise ValueError(f"Given date string {date_string} not likely a datetime")
+ raise ValueError(f'Given date string "{date_string}" not likely a datetime')
 
  parsed, reso = _parse_delimited_date(date_string, dayfirst)
  if parsed is not None:

pandas/_libs/tslibs/strptime.pyx

@@ -236,17 +236,22 @@ def array_strptime(
  if exact:
  found = format_regex.match(val)
  if not found:
- raise ValueError(f"time data '{val}' does not match "
- f"format '{fmt}' (match)")
+ raise ValueError(f"time data \"{val}\" at position {i} doesn't "
+ f"match format \"{fmt}\"")
  if len(val) != found.end():
- raise ValueError(f"unconverted data remains: {val[found.end():]}")
+ raise ValueError(
+ f"unconverted data remains at position {i}: "
+ f'"{val[found.end():]}"'
+ )
 
  # search
  else:
  found = format_regex.search(val)
  if not found:
- raise ValueError(f"time data {repr(val)} does not match format "
- f"{repr(fmt)} (search)")
+ raise ValueError(
+ f"time data \"{val}\" at position {i} doesn't match "
+ f"format \"{fmt}\""
+ )
 
  iso_year = -1
  year = 1900
@@ -396,7 +401,9 @@ def array_strptime(
 
  result_timezone[i] = tz
 
- except (ValueError, OutOfBoundsDatetime):
+ except (ValueError, OutOfBoundsDatetime) as ex:
+ if isinstance(ex, OutOfBoundsDatetime):
+ ex.args = (f"{str(ex)} present at position {i}",)
  if is_coerce:
  iresult[i] = NPY_NAT
  continue

pandas/tests/io/parser/test_parse_dates.py

@@ -1720,7 +1720,9 @@ def test_parse_multiple_delimited_dates_with_swap_warnings():
  # GH46210
  with pytest.raises(
  ValueError,
- match=r"^time data '31/05/2000' does not match format '%m/%d/%Y' \(match\)$",
+ match=(
+ r'^time data "31/05/2000" at position 1 doesn\'t match format "%m/%d/%Y"$'
+ ),
  ):
  pd.to_datetime(["01/01/2000", "31/05/2000", "31/05/2001", "01/02/2000"])
 

pandas/tests/scalar/period/test_period.py

@@ -303,7 +303,7 @@ def test_invalid_arguments(self):
  with pytest.raises(ValueError, match=msg):
  Period(month=1)
 
- msg = "Given date string -2000 not likely a datetime"
+ msg = '^Given date string "-2000" not likely a datetime$'
  with pytest.raises(ValueError, match=msg):
  Period("-2000", "A")
  msg = "day is out of range for month"

pandas/tests/tools/test_to_datetime.py

@@ -479,7 +479,12 @@ def test_to_datetime_parse_timezone_malformed(self, offset):
  fmt = "%Y-%m-%d %H:%M:%S %z"
  date = "2010-01-01 12:00:00 " + offset
 
- msg = "does not match format|unconverted data remains"
+ msg = "|".join(
+ [
+ r'^time data ".*" at position 0 doesn\'t match format ".*"$',
+ r'^unconverted data remains at position 0: ".*"$',
+ ]
+ )
  with pytest.raises(ValueError, match=msg):
  to_datetime([date], format=fmt)
 
@@ -1093,7 +1098,7 @@ def test_datetime_bool_arrays_mixed(self, cache):
  to_datetime([False, datetime.today()], cache=cache)
  with pytest.raises(
  ValueError,
- match=r"^time data 'True' does not match format '%Y%m%d' \(match\)$",
+ match=r'^time data "True" at position 1 doesn\'t match format "%Y%m%d"$',
  ):
  to_datetime(["20130101", True], cache=cache)
  tm.assert_index_equal(
@@ -1132,11 +1137,13 @@ def test_datetime_invalid_scalar(self, value, format, warning):
  res = to_datetime(value, errors="coerce", format=format)
  assert res is NaT
 
- msg = (
- "does not match format|"
- "unconverted data remains:|"
- "second must be in 0..59|"
- f"Given date string {value} not likely a datetime"
+ msg = "|".join(
+ [
+ r'^time data "a" at position 0 doesn\'t match format "%H:%M:%S"$',
+ r'^Given date string "a" not likely a datetime present at position 0$',
+ r'^unconverted data remains at position 0: "9"$',
+ r"^second must be in 0..59: 00:01:99 present at position 0$",
+ ]
  )
  with pytest.raises(ValueError, match=msg):
  with tm.assert_produces_warning(warning, match="Could not infer format"):
@@ -1157,7 +1164,7 @@ def test_datetime_outofbounds_scalar(self, value, format, warning):
  assert res is NaT
 
  if format is not None:
- msg = "does not match format|Out of bounds .* present at position 0"
+ msg = r'^time data ".*" at position 0 doesn\'t match format ".*"$'
  with pytest.raises(ValueError, match=msg):
  to_datetime(value, errors="raise", format=format)
  else:
@@ -1181,11 +1188,13 @@ def test_datetime_invalid_index(self, values, format, warning):
  res = to_datetime(values, errors="coerce", format=format)
  tm.assert_index_equal(res, DatetimeIndex([NaT] * len(values)))
 
- msg = (
- "does not match format|"
- "unconverted data remains:|"
- f"Given date string {values[0]} not likely a datetime|"
- "second must be in 0..59"
+ msg = "|".join(
+ [
+ r'^Given date string "a" not likely a datetime present at position 0$',
+ r'^time data "a" at position 0 doesn\'t match format "%H:%M:%S"$',
+ r'^unconverted data remains at position 0: "9"$',
+ r"^second must be in 0..59: 00:01:99 present at position 0$",
+ ]
  )
  with pytest.raises(ValueError, match=msg):
  with tm.assert_produces_warning(warning, match="Could not infer format"):
@@ -1805,8 +1814,8 @@ def test_dataframe_coerce(self, cache):
  df2 = DataFrame({"year": [2015, 2016], "month": [2, 20], "day": [4, 5]})
 
  msg = (
- "cannot assemble the datetimes: time data .+ does not "
- r"match format '%Y%m%d' \(match\)"
+ r'^cannot assemble the datetimes: time data ".+" at position 1 doesn\'t '
+ r'match format "%Y%m%d"$'
  )
  with pytest.raises(ValueError, match=msg):
  to_datetime(df2, cache=cache)
@@ -1882,7 +1891,10 @@ def test_dataframe_mixed(self, cache):
  def test_dataframe_float(self, cache):
  # float
  df = DataFrame({"year": [2000, 2001], "month": [1.5, 1], "day": [1, 1]})
- msg = "cannot assemble the datetimes: unconverted data remains: 1"
+ msg = (
+ r"^cannot assemble the datetimes: unconverted data remains at position "
+ r'0: "1"$'
+ )
  with pytest.raises(ValueError, match=msg):
  to_datetime(df, cache=cache)
 
@@ -2072,7 +2084,7 @@ def test_to_datetime_on_datetime64_series(self, cache):
  def test_to_datetime_with_space_in_series(self, cache):
  # GH 6428
  ser = Series(["10/18/2006", "10/18/2008", " "])
- msg = r"^time data ' ' does not match format '%m/%d/%Y' \(match\)$"
+ msg = r'^time data " " at position 2 doesn\'t match format "%m/%d/%Y"$'
  with pytest.raises(ValueError, match=msg):
  to_datetime(ser, errors="raise", cache=cache)
  result_coerce = to_datetime(ser, errors="coerce", cache=cache)
@@ -2342,7 +2354,10 @@ def test_dayfirst_warnings_invalid_input(self):
 
  with pytest.raises(
  ValueError,
- match=r"time data '03/30/2011' does not match format '%d/%m/%Y' \(match\)$",
+ match=(
+ r'^time data "03/30/2011" at position 1 doesn\'t match format '
+ r'"%d/%m/%Y"$'
+ ),
  ):
  to_datetime(arr, dayfirst=True)
 
@@ -2410,7 +2425,11 @@ def test_to_datetime_infer_datetime_format_consistent_format(
  def test_to_datetime_inconsistent_format(self, cache):
  data = ["01/01/2011 00:00:00", "01-02-2011 00:00:00", "2011-01-03T00:00:00"]
  ser = Series(np.array(data))
- with pytest.raises(ValueError, match="does not match format"):
+ msg = (
+ r'^time data "01-02-2011 00:00:00" at position 1 doesn\'t match format '
+ r'"%m/%d/%Y %H:%M:%S"$'
+ )
+ with pytest.raises(ValueError, match=msg):
  to_datetime(ser, cache=cache)
 
  def test_to_datetime_consistent_format(self, cache):
@@ -2923,17 +2942,22 @@ def test_incorrect_value_exception(self):
  to_datetime(["today", "yesterday"])
 
  @pytest.mark.parametrize(
- "format, warning", [(None, UserWarning), ("%Y-%m-%d %H:%M:%S", None)]
+ "format, warning",
+ [
+ (None, UserWarning),
+ ("%Y-%m-%d %H:%M:%S", None),
+ ("%Y-%d-%m %H:%M:%S", None),
+ ],
  )
  def test_to_datetime_out_of_bounds_with_format_arg(self, format, warning):
  # see gh-23830
  msg = (
- "Out of bounds nanosecond timestamp: 2417-10-27 00:00:00 "
- "present at position 0"
+ r"^Out of bounds nanosecond timestamp: 2417-10-10 00:00:00 "
+ r"present at position 0$"
  )
  with pytest.raises(OutOfBoundsDatetime, match=msg):
  with tm.assert_produces_warning(warning, match="Could not infer format"):
- to_datetime("2417-10-27 00:00:00", format=format)
+ to_datetime("2417-10-10 00:00:00", format=format)
 
  @pytest.mark.parametrize(
  "arg, origin, expected_str",