PERF: Parse certain dates in Cython instead of falling back to dateutil.parse (#25922)

Author: vnlitvinov

asv_bench/benchmarks/io/csv.py

@@ -251,4 +251,22 @@ def mem_parser_chunks(self):
  pass
 
 
+class ReadCSVParseSpecialDate(StringIORewind):
+ params = (['mY', 'mdY'],)
+ params_name = ['value']
+ objects = {
+ 'mY': '01-2019\n10-2019\n02/2000\n',
+ 'mdY': '12/02/2010\n'
+ }
+
+ def setup(self, value):
+ count_elem = 10000
+ data = self.objects[value] * count_elem
+ self.StringIO_input = StringIO(data)
+
+ def time_read_special_date(self, value):
+ read_csv(self.data(self.StringIO_input), sep=',', header=None,
+ names=['Date'], parse_dates=['Date'])
+
+
 from ..pandas_vb_common import setup # noqa: F401

doc/source/whatsnew/v0.25.0.rst

@@ -250,6 +250,7 @@ Performance Improvements
 - Improved performance of :meth:`read_csv` by faster tokenizing and faster parsing of small float numbers (:issue:`25784`)
 - Improved performance of :meth:`read_csv` by faster parsing of N/A and boolean values (:issue:`25804`)
 - Improved performance of :meth:DataFrame.`to_csv` when write datetime dtype data (:issue:`25708`)
+- Improved performance of :meth:`read_csv` by much faster parsing of MM/YYYY and DD/MM/YYYY datetime formats (:issue:`25922`)
 
 .. _whatsnew_0250.bug_fixes:
 

pandas/_libs/src/headers/portable.h

@@ -8,6 +8,7 @@
 // GH-23516 - works around locale perf issues
 // from MUSL libc, MIT Licensed - see LICENSES
 #define isdigit_ascii(c) (((unsigned)(c) - '0') < 10u)
+#define getdigit_ascii(c, default) (isdigit_ascii(c) ? ((int)((c) - '0')) : default)
 #define isspace_ascii(c) (((c) == ' ') || (((unsigned)(c) - '\t') < 5))
 #define toupper_ascii(c) ((((unsigned)(c) - 'a') < 26) ? ((c) & 0x5f) : (c))
 #define tolower_ascii(c) ((((unsigned)(c) - 'A') < 26) ? ((c) | 0x20) : (c))

pandas/_libs/tslibs/parsing.pyx

@@ -6,8 +6,11 @@ import re
 import time
 from io import StringIO
 
-from cpython.datetime cimport datetime
+from libc.string cimport strchr
 
+from cpython.datetime cimport datetime, datetime_new, import_datetime
+from cpython.version cimport PY_VERSION_HEX
+import_datetime()
 
 import numpy as np
 
@@ -24,6 +27,10 @@ from pandas._config import get_option
 
 from pandas._libs.tslibs.ccalendar import MONTH_NUMBERS
 from pandas._libs.tslibs.nattype import nat_strings, NaT
+from pandas._libs.tslibs.util cimport get_c_string_buf_and_size
+
+cdef extern from "../src/headers/portable.h":
+ int getdigit_ascii(char c, int default) nogil
 
 # ----------------------------------------------------------------------
 # Constants
@@ -42,6 +49,99 @@ cdef:
  set _not_datelike_strings = {'a', 'A', 'm', 'M', 'p', 'P', 't', 'T'}
 
 # ----------------------------------------------------------------------
+cdef:
+ const char* delimiters = " /-."
+ int MAX_DAYS_IN_MONTH = 31, MAX_MONTH = 12
+
+
+cdef inline bint _is_not_delimiter(const char ch):
+ return strchr(delimiters, ch) == NULL
+
+
+cdef inline int _parse_2digit(const char* s):
+ cdef int result = 0
+ result += getdigit_ascii(s[0], -10) * 10
+ result += getdigit_ascii(s[1], -100) * 1
+ return result
+
+
+cdef inline int _parse_4digit(const char* s):
+ cdef int result = 0
+ result += getdigit_ascii(s[0], -10) * 1000
+ result += getdigit_ascii(s[1], -100) * 100
+ result += getdigit_ascii(s[2], -1000) * 10
+ result += getdigit_ascii(s[3], -10000) * 1
+ return result
+
+
+cdef inline object _parse_delimited_date(object date_string, bint dayfirst):
+ """
+ Parse special cases of dates: MM/DD/YYYY, DD/MM/YYYY, MM/YYYY.
+ At the beginning function tries to parse date in MM/DD/YYYY format, but
+ if month > 12 - in DD/MM/YYYY (`dayfirst == False`).
+ With `dayfirst == True` function makes an attempt to parse date in
+ DD/MM/YYYY, if an attemp is wrong - in DD/MM/YYYY
+
+ Note
+ ----
+ For MM/DD/YYYY, DD/MM/YYYY: delimiter can be a space or one of /-.
+ For MM/YYYY: delimiter can be a space or one of /-
+ If `date_string` can't be converted to date, then function returns
+ None, None
+
+ Parameters
+ ----------
+ date_string : str
+ dayfirst : bint
+
+ Returns:
+ --------
+ datetime, resolution
+ """
+ cdef:
+ const char* buf
+ Py_ssize_t length
+ int day = 1, month = 1, year
+ bint can_swap = 0
+
+ buf = get_c_string_buf_and_size(date_string, &length)
+ if length == 10:
+ # parsing MM?DD?YYYY and DD?MM?YYYY dates
+ if _is_not_delimiter(buf[2]) or _is_not_delimiter(buf[5]):
+ return None, None
+ month = _parse_2digit(buf)
+ day = _parse_2digit(buf + 3)
+ year = _parse_4digit(buf + 6)
+ reso = 'day'
+ can_swap = 1
+ elif length == 7:
+ # parsing MM?YYYY dates
+ if buf[2] == b'.' or _is_not_delimiter(buf[2]):
+ # we cannot reliably tell whether e.g. 10.2010 is a float
+ # or a date, thus we refuse to parse it here
+ return None, None
+ month = _parse_2digit(buf)
+ year = _parse_4digit(buf + 3)
+ reso = 'month'
+ else:
+ return None, None
+
+ if month < 0 or day < 0 or year < 1000:
+ # some part is not an integer, so
+ # date_string can't be converted to date, above format
+ return None, None
+
+ if 1 <= month <= MAX_DAYS_IN_MONTH and 1 <= day <= MAX_DAYS_IN_MONTH \
+ and (month <= MAX_MONTH or day <= MAX_MONTH):
+ if (month > MAX_MONTH or (day <= MAX_MONTH and dayfirst)) and can_swap:
+ day, month = month, day
+ if PY_VERSION_HEX >= 0x03060100:
+ # In Python <= 3.6.0 there is no range checking for invalid dates
+ # in C api, thus we call faster C version for 3.6.1 or newer
+ return datetime_new(year, month, day, 0, 0, 0, 0, None), reso
+ return datetime(year, month, day, 0, 0, 0, 0, None), reso
+
+ raise DateParseError("Invalid date specified ({}/{})".format(month, day))
 
 
 def parse_datetime_string(date_string, freq=None, dayfirst=False,
@@ -66,6 +166,10 @@ def parse_datetime_string(date_string, freq=None, dayfirst=False,
  yearfirst=yearfirst, **kwargs)
  return dt
 
+ dt, _ = _parse_delimited_date(date_string, dayfirst)
+ if dt is not None:
+ return dt
+
  try:
  dt, _, _ = _parse_dateabbr_string(date_string, _DEFAULT_DATETIME, freq)
  return dt
@@ -146,6 +250,10 @@ cdef parse_datetime_string_with_reso(date_string, freq=None, dayfirst=False,
  if not _does_string_look_like_datetime(date_string):
  raise ValueError('Given date string not likely a datetime.')
 
+ parsed, reso = _parse_delimited_date(date_string, dayfirst)
+ if parsed is not None:
+ return parsed, parsed, reso
+
  try:
  return _parse_dateabbr_string(date_string, _DEFAULT_DATETIME, freq)
  except DateParseError:
@@ -279,7 +387,7 @@ cdef inline object _parse_dateabbr_string(object date_string, object default,
  except ValueError:
  pass
 
- for pat in ['%Y-%m', '%m-%Y', '%b %Y', '%b-%Y']:
+ for pat in ['%Y-%m', '%b %Y', '%b-%Y']:
  try:
  ret = datetime.strptime(date_string, pat)
  return ret, ret, 'month'

pandas/tests/io/parser/test_parse_dates.py

@@ -8,14 +8,16 @@
 from datetime import date, datetime
 from io import StringIO
 
-from dateutil.parser import parse
+from dateutil.parser import parse as du_parse
+from hypothesis import given, settings, strategies as st
 import numpy as np
 import pytest
 import pytz
 
 from pandas._libs.tslib import Timestamp
 from pandas._libs.tslibs import parsing
-from pandas.compat import lrange
+from pandas._libs.tslibs.parsing import parse_datetime_string
+from pandas.compat import is_platform_windows, lrange
 from pandas.compat.numpy import np_array_datetime64_compat
 
 import pandas as pd
@@ -26,6 +28,15 @@
 import pandas.io.date_converters as conv
 import pandas.io.parsers as parsers
 
+# constant
+_DEFAULT_DATETIME = datetime(1, 1, 1)
+
+# Strategy for hypothesis
+if is_platform_windows():
+ date_strategy = st.datetimes(min_value=datetime(1900, 1, 1))
+else:
+ date_strategy = st.datetimes()
+
 
 def test_separator_date_conflict(all_parsers):
  # Regression test for gh-4678
@@ -439,7 +450,7 @@ def test_parse_dates_custom_euro_format(all_parsers, kwargs):
 """
  if "dayfirst" in kwargs:
  df = parser.read_csv(StringIO(data), names=["time", "Q", "NTU"],
- date_parser=lambda d: parse(d, **kwargs),
+ date_parser=lambda d: du_parse(d, **kwargs),
  header=0, index_col=0, parse_dates=True,
  na_values=["NA"])
  exp_index = Index([datetime(2010, 1, 31), datetime(2010, 2, 1),
@@ -451,7 +462,7 @@ def test_parse_dates_custom_euro_format(all_parsers, kwargs):
  msg = "got an unexpected keyword argument 'day_first'"
  with pytest.raises(TypeError, match=msg):
  parser.read_csv(StringIO(data), names=["time", "Q", "NTU"],
- date_parser=lambda d: parse(d, **kwargs),
+ date_parser=lambda d: du_parse(d, **kwargs),
  skiprows=[0], index_col=0, parse_dates=True,
  na_values=["NA"])
 
@@ -849,3 +860,82 @@ def test_parse_timezone(all_parsers):
 
  expected = DataFrame(expected_data)
  tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("date_string", [
+ "32/32/2019",
+ "02/30/2019",
+ "13/13/2019",
+ "13/2019",
+ "a3/11/2018",
+ "10/11/2o17"
+])
+def test_invalid_parse_delimited_date(all_parsers, date_string):
+ parser = all_parsers
+ expected = DataFrame({0: [date_string]}, dtype="object")
+ result = parser.read_csv(StringIO(date_string),
+ header=None, parse_dates=[0])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("date_string,dayfirst,expected", [
+ # %d/%m/%Y; month > 12 thus replacement
+ ("13/02/2019", False, datetime(2019, 2, 13)),
+ ("13/02/2019", True, datetime(2019, 2, 13)),
+ # %m/%d/%Y; day > 12 thus there will be no replacement
+ ("02/13/2019", False, datetime(2019, 2, 13)),
+ ("02/13/2019", True, datetime(2019, 2, 13)),
+ # %d/%m/%Y; dayfirst==True thus replacement
+ ("04/02/2019", True, datetime(2019, 2, 4))
+])
+def test_parse_delimited_date_swap(all_parsers, date_string,
+ dayfirst, expected):
+ parser = all_parsers
+ expected = DataFrame({0: [expected]}, dtype="datetime64[ns]")
+ result = parser.read_csv(StringIO(date_string), header=None,
+ dayfirst=dayfirst, parse_dates=[0])
+ tm.assert_frame_equal(result, expected)
+
+
+def _helper_hypothesis_delimited_date(call, date_string, **kwargs):
+ msg, result = None, None
+ try:
+ result = call(date_string, **kwargs)
+ except ValueError as er:
+ msg = str(er)
+ pass
+ return msg, result
+
+
+@given(date_strategy)
+@settings(deadline=None)
+@pytest.mark.parametrize("delimiter", list(" -./"))
+@pytest.mark.parametrize("dayfirst", [True, False])
+@pytest.mark.parametrize("date_format", [
+ "%d %m %Y",
+ "%m %d %Y",
+ "%m %Y",
+ "%Y %m %d",
+ "%y %m %d",
+ "%Y%m%d",
+ "%y%m%d",
+])
+def test_hypothesis_delimited_date(date_format, dayfirst,
+ delimiter, test_datetime):
+ if date_format == "%m %Y" and delimiter == ".":
+ pytest.skip("parse_datetime_string cannot reliably tell whether \
+ e.g. %m.%Y is a float or a date, thus we skip it")
+ result, expected = None, None
+ except_in_dateutil, except_out_dateutil = None, None
+ date_string = test_datetime.strftime(date_format.replace(' ', delimiter))
+
+ except_out_dateutil, result = _helper_hypothesis_delimited_date(
+ parse_datetime_string, date_string,
+ dayfirst=dayfirst)
+ except_in_dateutil, expected = _helper_hypothesis_delimited_date(
+ du_parse, date_string,
+ default=_DEFAULT_DATETIME,
+ dayfirst=dayfirst, yearfirst=False)
+
+ assert except_out_dateutil == except_in_dateutil
+ assert result == expected