Address comments

doc/source/whatsnew/v1.4.0.rst

@@ -207,7 +207,7 @@ Other enhancements
 - :meth:`Series.str.split` now supports a ``regex`` argument that explicitly specifies whether the pattern is a regular expression. Default is ``None`` (:issue:`43563`, :issue:`32835`, :issue:`25549`)
 - :meth:`DataFrame.dropna` now accepts a single label as ``subset`` along with array-like (:issue:`41021`)
 - Added :meth:`DataFrameGroupBy.value_counts` (:issue:`43564`)
-- :meth:`read_csv` now accepts a ``callable`` function in ``on_bad_lines`` when ``engine="python"`` for custom handling of bad lines (:issue:`5686`)
+- :func:`read_csv` now accepts a ``callable`` function in ``on_bad_lines`` when ``engine="python"`` for custom handling of bad lines (:issue:`5686`)
 - :class:`ExcelWriter` argument ``if_sheet_exists="overlay"`` option added (:issue:`40231`)
 - :meth:`read_excel` now accepts a ``decimal`` argument that allow the user to specify the decimal point when parsing string columns to numeric (:issue:`14403`)
 - :meth:`.GroupBy.mean`, :meth:`.GroupBy.std`, :meth:`.GroupBy.var`, :meth:`.GroupBy.sum` now supports `Numba <http://numba.pydata.org/>`_ execution with the ``engine`` keyword (:issue:`43731`, :issue:`44862`, :issue:`44939`)

pandas/io/parsers/python_parser.py

@@ -197,7 +197,6 @@ class MyDialect(csv.Dialect):
  skipinitialspace = self.skipinitialspace
  quoting = self.quoting
  lineterminator = "\n"
- strict = not callable(self.on_bad_lines)
 
  dia = MyDialect
 

pandas/io/parsers/readers.py

@@ -355,7 +355,7 @@
  .. deprecated:: 1.3.0
  The ``on_bad_lines`` parameter should be used instead to specify behavior upon
  encountering a bad line instead.
-on_bad_lines : str or callable, default 'error'
+on_bad_lines : {{'error', 'warn', 'skip'}} or callable, default 'error'
  Specifies what to do upon encountering a bad line (a line with too many fields).
  Allowed values are :
 
@@ -367,7 +367,10 @@
 
  - callable, function with signature ``(bad_line: list[str]) -> list[str]``
  that will process a single bad line. ``bad_line`` is a list of strings
- split by the ``sep``. Only supported when ``engine="python"``
+ split by the ``sep``. If the function returns a new list of strings
+ with more elements than expected, a ``ParserWarning`` will be emitted
+ while dropping extra elements.
+ Only supported when ``engine="python"``
 
  .. versionadded:: 1.4.0
 

pandas/tests/io/parser/test_python_parser_only.py

@@ -338,7 +338,12 @@ def readline(self):
 def test_on_bad_lines_callable(python_parser_only, bad_line_func):
  # GH 5686
  parser = python_parser_only
- bad_sio = StringIO("a,b\n1,2\n2,3,4,5,6\n3,4")
+ data = """a,b
+1,2
+2,3,4,5,6
+3,4
+"""
+ bad_sio = StringIO(data)
  result = parser.read_csv(bad_sio, on_bad_lines=bad_line_func)
  expected = DataFrame({"a": [1, 2, 3], "b": [2, 3, 4]})
  tm.assert_frame_equal(result, expected)
@@ -347,7 +352,12 @@ def test_on_bad_lines_callable(python_parser_only, bad_line_func):
 def test_on_bad_lines_callable_write_to_external_list(python_parser_only):
  # GH 5686
  parser = python_parser_only
- bad_sio = StringIO("a,b\n1,2\n2,3,4,5,6\n3,4")
+ data = """a,b
+1,2
+2,3,4,5,6
+3,4
+"""
+ bad_sio = StringIO(data)
  lst = []
 
  def bad_line_func(bad_line):
@@ -366,7 +376,13 @@ def test_on_bad_lines_callable_iterator_true(python_parser_only, bad_line_func,
  # GH 5686
  # iterator=True has a separate code path than iterator=False
  parser = python_parser_only
- bad_sio = StringIO(f"0{sep}1\nhi{sep}there\nfoo{sep}bar{sep}baz\ngood{sep}bye")
+ data = f"""
+0{sep}1
+hi{sep}there
+foo{sep}bar{sep}baz
+good{sep}bye
+"""
+ bad_sio = StringIO(data)
  result_iter = parser.read_csv(
  bad_sio, on_bad_lines=bad_line_func, chunksize=1, iterator=True, sep=sep
  )
@@ -383,7 +399,12 @@ def test_on_bad_lines_callable_iterator_true(python_parser_only, bad_line_func,
 def test_on_bad_lines_callable_dont_swallow_errors(python_parser_only):
  # GH 5686
  parser = python_parser_only
- bad_sio = StringIO("a,b\n1,2\n2,3,4,5,6\n3,4")
+ data = """a,b
+1,2
+2,3,4,5,6
+3,4
+"""
+ bad_sio = StringIO(data)
  msg = "This function is buggy."
 
  def bad_line_func(bad_line):
@@ -396,7 +417,12 @@ def bad_line_func(bad_line):
 def test_on_bad_lines_callable_not_expected_length(python_parser_only):
  # GH 5686
  parser = python_parser_only
- bad_sio = StringIO("a,b\n1,2\n2,3,4,5,6\n3,4")
+ data = """a,b
+1,2
+2,3,4,5,6
+3,4
+"""
+ bad_sio = StringIO(data)
 
  with tm.assert_produces_warning(ParserWarning, match="Length of header or names"):
  result = parser.read_csv(bad_sio, on_bad_lines=lambda x: x)