BUG: _guess_datetime_format_for_array doesn't guess if first element is '' or 'NaT' (#49120)

Author: MarcoGorelli

pandas/_libs/tslib.pyi

@@ -16,6 +16,7 @@ def array_with_unit_to_datetime(
  unit: str,
  errors: str = ...,
 ) -> tuple[np.ndarray, tzinfo | None]: ...
+def first_non_null(values: np.ndarray) -> int: ...
 def array_to_datetime(
  values: npt.NDArray[np.object_],
  errors: str = ...,

pandas/_libs/tslib.pyx

@@ -421,6 +421,23 @@ def array_with_unit_to_datetime(
 
  return oresult, tz
 
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def first_non_null(values: ndarray) -> int:
+ """Find position of first non-null value, return -1 if there isn't one."""
+ cdef:
+ Py_ssize_t n = len(values)
+ Py_ssize_t i
+ int result
+ for i in range(n):
+ val = values[i]
+ if checknull_with_nat_and_na(val):
+ continue
+ if isinstance(val, str) and (len(val) == 0 or val in nat_strings):
+ continue
+ return i
+ else:
+ return -1
 
 @cython.wraparound(False)
 @cython.boundscheck(False)

pandas/core/tools/datetimes.py

@@ -126,9 +126,8 @@ class FulldatetimeDict(YearMonthDayDict, total=False):
 
 def _guess_datetime_format_for_array(arr, dayfirst: bool | None = False) -> str | None:
  # Try to guess the format based on the first non-NaN element, return None if can't
- non_nan_elements = notna(arr).nonzero()[0]
- if len(non_nan_elements):
- if type(first_non_nan_element := arr[non_nan_elements[0]]) is str:
+ if (first_non_null := tslib.first_non_null(arr)) != -1:
+ if type(first_non_nan_element := arr[first_non_null]) is str:
  # GH#32264 np.str_ object
  return guess_datetime_format(first_non_nan_element, dayfirst=dayfirst)
  return None

pandas/tests/tools/test_to_datetime.py

@@ -2093,21 +2093,23 @@ def test_to_datetime_dta_tz(self, klass):
 
 
 class TestGuessDatetimeFormat:
- @td.skip_if_not_us_locale
  @pytest.mark.parametrize(
- "test_array",
+ "test_list",
  [
  [
  "2011-12-30 00:00:00.000000",
  "2011-12-30 00:00:00.000000",
  "2011-12-30 00:00:00.000000",
  ],
  [np.nan, np.nan, "2011-12-30 00:00:00.000000"],
+ ["", "2011-12-30 00:00:00.000000"],
+ ["NaT", "2011-12-30 00:00:00.000000"],
  ["2011-12-30 00:00:00.000000", "random_string"],
  ],
  )
- def test_guess_datetime_format_for_array(self, test_array):
+ def test_guess_datetime_format_for_array(self, test_list):
  expected_format = "%Y-%m-%d %H:%M:%S.%f"
+ test_array = np.array(test_list, dtype=object)
  assert tools._guess_datetime_format_for_array(test_array) == expected_format
 
  @td.skip_if_not_us_locale