BUG: fix json with lines=True for quoted special characters

closes #14391 closes #14390

Author: Josh Owen

asv_bench/benchmarks/packers.py

@@ -547,6 +547,31 @@ def remove(self, f):
  pass
 
 
+class packers_write_json_lines(object):
+ goal_time = 0.2
+
+ def setup(self):
+ self.f = '__test__.msg'
+ self.N = 100000
+ self.C = 5
+ self.index = date_range('20000101', periods=self.N, freq='H')
+ self.df = DataFrame(dict([('float{0}'.format(i), randn(self.N)) for i in range(self.C)]), index=self.index)
+ self.remove(self.f)
+ self.df.index = np.arange(self.N)
+
+ def time_packers_write_json_lines(self):
+ self.df.to_json(self.f, orient="records", lines=True)
+
+ def teardown(self):
+ self.remove(self.f)
+
+ def remove(self, f):
+ try:
+ os.remove(self.f)
+ except:
+ pass
+
+
 class packers_write_json_T(object):
  goal_time = 0.2
 

doc/source/whatsnew/v0.19.1.txt

@@ -45,3 +45,4 @@ Bug Fixes
 
 - Bug in ``pd.concat`` where names of the ``keys`` were not propagated to the resulting ``MultiIndex`` (:issue:`14252`)
 - Bug in ``MultiIndex.set_levels`` where illegal level values were still set after raising an error (:issue:`13754`)
+- Bug in ``DataFrame.to_json`` where ``lines=True`` and a value contained a ``}`` character (:issue:`14391`)

pandas/io/json.py

@@ -607,14 +607,19 @@ def _convert_to_line_delimits(s):
  s = s[1:-1]
  num_open_brackets_seen = 0
  commas_to_replace = []
+ in_quotes = False
  for idx, char in enumerate(s): # iter through to find all
- if char == ',': # commas that should be \n
- if num_open_brackets_seen == 0:
+ if char == '"' and idx > 0 and s[idx - 1] != '\\':
+ in_quotes = ~in_quotes
+ elif char == ',': # commas that should be \n
+ if num_open_brackets_seen == 0 and not in_quotes:
  commas_to_replace.append(idx)
  elif char == '{':
- num_open_brackets_seen += 1
+ if not in_quotes:
+ num_open_brackets_seen += 1
  elif char == '}':
- num_open_brackets_seen -= 1
+ if not in_quotes:
+ num_open_brackets_seen -= 1
  s_arr = np.array(list(s)) # Turn to an array to set
  s_arr[commas_to_replace] = '\n' # all commas at once.
  s = ''.join(s_arr)

pandas/io/tests/json/test_pandas.py

@@ -962,6 +962,12 @@ def test_to_jsonl(self):
  expected = '{"a":1,"b":2}\n{"a":1,"b":2}'
  self.assertEqual(result, expected)
 
+ df = DataFrame([["foo}", "bar"], ['foo"', "bar"]], columns=['a', 'b'])
+ result = df.to_json(orient="records", lines=True)
+ expected = '{"a":"foo}","b":"bar"}\n{"a":"foo\\"","b":"bar"}'
+ self.assertEqual(result, expected)
+ assert_frame_equal(pd.read_json(result, lines=True), df)
+
  def test_latin_encoding(self):
  if compat.PY2:
  self.assertRaisesRegexp(