BUG: pd.to-timedelta handles missing data

Author: danielballan

doc/source/release.rst

@@ -771,6 +771,7 @@ Bug Fixes
  - Fix empty series not printing name in repr (:issue:`4651`)
  - Make tests create temp files in temp directory by default. (:issue:`5419`)
  - ``pd.to_timedelta`` of a scalar returns a scalar (:issue:`5410`)
+ - ``pd.to_timedelta`` accepts ``NaN`` and ``NaT``, returning ``NaT`` instead of raising (:issue:`5437`)
 
 pandas 0.12.0
 -------------

pandas/core/internals.py

@@ -1162,15 +1162,25 @@ def _try_fill(self, value):
 
  def _try_coerce_args(self, values, other):
  """ provide coercion to our input arguments
- we are going to compare vs i8, so coerce to integer
- values is always ndarra like, other may not be """
- values = values.view('i8')
+ we are going to compare vs i8, so coerce to floats
+ repring NaT with np.nan so nans propagate
+ values is always ndarray like, other may not be """
+ def masker(v):
+ mask = isnull(v)
+ v = v.view('i8').astype('float64')
+ v[mask] = np.nan
+ return v
+
+ values = masker(values)
+
  if isnull(other) or (np.isscalar(other) and other == tslib.iNaT):
- other = tslib.iNaT
+ other = np.nan
  elif isinstance(other, np.timedelta64):
  other = _coerce_scalar_to_timedelta_type(other,unit='s').item()
+ if other == tslib.iNaT:
+ other = np.nan
  else:
- other = other.view('i8')
+ other = masker(other)
 
  return values, other
 

pandas/core/ops.py

@@ -255,7 +255,7 @@ def __init__(self, left, right, name):
  self.name = name
 
  lvalues = self._convert_to_array(left, name=name)
- rvalues = self._convert_to_array(right, name=name)
+ rvalues = self._convert_to_array(right, name=name, other=lvalues)
 
  self.is_timedelta_lhs = com.is_timedelta64_dtype(left)
  self.is_datetime_lhs = com.is_datetime64_dtype(left)
@@ -317,17 +317,24 @@ def _validate(self):
  'of a series/ndarray of type datetime64[ns] '
  'or a timedelta')
 
- def _convert_to_array(self, values, name=None):
+ def _convert_to_array(self, values, name=None, other=None):
  """converts values to ndarray"""
  from pandas.tseries.timedeltas import _possibly_cast_to_timedelta
 
  coerce = 'compat' if pd._np_version_under1p7 else True
  if not is_list_like(values):
  values = np.array([values])
  inferred_type = lib.infer_dtype(values)
+
  if inferred_type in ('datetime64', 'datetime', 'date', 'time'):
+ # if we have a other of timedelta, but use pd.NaT here we
+ # we are in the wrong path
+ if other is not None and other.dtype == 'timedelta64[ns]' and all(isnull(v) for v in values):
+ values = np.empty(values.shape,dtype=other.dtype)
+ values[:] = tslib.iNaT
+
  # a datetlike
- if not (isinstance(values, (pa.Array, pd.Series)) and
+ elif not (isinstance(values, (pa.Array, pd.Series)) and
  com.is_datetime64_dtype(values)):
  values = tslib.array_to_datetime(values)
  elif isinstance(values, pd.DatetimeIndex):
@@ -354,6 +361,15 @@ def _convert_to_array(self, values, name=None):
  ', '.join([com.pprint_thing(v)
  for v in values[mask]])))
  values = _possibly_cast_to_timedelta(os, coerce=coerce)
+ elif inferred_type == 'floating':
+
+ # all nan, so ok, use the other dtype (e.g. timedelta or datetime)
+ if isnull(values).all():
+ values = np.empty(values.shape,dtype=other.dtype)
+ values[:] = tslib.iNaT
+ else:
+ raise TypeError("incompatible type [{0}] for a datetime/timedelta"
+ " operation".format(pa.array(values).dtype))
  else:
  raise TypeError("incompatible type [{0}] for a datetime/timedelta"
  " operation".format(pa.array(values).dtype))
@@ -452,6 +468,8 @@ def na_op(x, y):
 
  def wrapper(left, right, name=name):
 
+ if isinstance(right, pd.DataFrame):
+ return NotImplemented
  time_converted = _TimeOp.maybe_convert_for_time_op(left, right, name)
 
  if time_converted is None:
@@ -488,8 +506,6 @@ def wrapper(left, right, name=name):
 
  return left._constructor(wrap_results(arr), index=index,
  name=name, dtype=dtype)
- elif isinstance(right, pd.DataFrame):
- return NotImplemented
  else:
  # scalars
  if hasattr(lvalues, 'values'):

pandas/tseries/tests/test_timedeltas.py

@@ -195,6 +195,122 @@ def test_timedelta_ops(self):
  expected = to_timedelta('00:00:08')
  tm.assert_almost_equal(result, expected)
 
+ def test_to_timedelta_on_missing_values(self):
+ _skip_if_numpy_not_friendly()
+
+ # GH5438
+ timedelta_NaT = np.timedelta64('NaT')
+
+ actual = pd.to_timedelta(Series(['00:00:01', np.nan]))
+ expected = Series([np.timedelta64(1000000000, 'ns'), timedelta_NaT], dtype='<m8[ns]')
+ assert_series_equal(actual, expected)
+
+ actual = pd.to_timedelta(Series(['00:00:01', pd.NaT]))
+ assert_series_equal(actual, expected)
+
+ actual = pd.to_timedelta(np.nan)
+ self.assert_(actual == timedelta_NaT)
+
+ actual = pd.to_timedelta(pd.NaT)
+ self.assert_(actual == timedelta_NaT)
+
+ def test_timedelta_ops_with_missing_values(self):
+ _skip_if_numpy_not_friendly()
+
+ # setup
+ s1 = pd.to_timedelta(Series(['00:00:01']))
+ s2 = pd.to_timedelta(Series(['00:00:02']))
+ sn = pd.to_timedelta(Series([pd.NaT]))
+ df1 = DataFrame(['00:00:01']).apply(pd.to_timedelta)
+ df2 = DataFrame(['00:00:02']).apply(pd.to_timedelta)
+ dfn = DataFrame([pd.NaT]).apply(pd.to_timedelta)
+ scalar1 = pd.to_timedelta('00:00:01')
+ scalar2 = pd.to_timedelta('00:00:02')
+ timedelta_NaT = pd.to_timedelta('NaT')
+ NA = np.nan
+
+ actual = scalar1 + scalar1
+ self.assert_(actual == scalar2)
+ actual = scalar2 - scalar1
+ self.assert_(actual == scalar1)
+
+ actual = s1 + s1
+ assert_series_equal(actual, s2)
+ actual = s2 - s1
+ assert_series_equal(actual, s1)
+
+ actual = s1 + scalar1
+ assert_series_equal(actual, s2)
+ actual = s2 - scalar1
+ assert_series_equal(actual, s1)
+
+ actual = s1 + timedelta_NaT
+ assert_series_equal(actual, sn)
+ actual = s1 - timedelta_NaT
+ assert_series_equal(actual, sn)
+
+ actual = s1 + NA
+ assert_series_equal(actual, sn)
+ actual = s1 - NA
+ assert_series_equal(actual, sn)
+
+ actual = s1 + pd.NaT # NaT is datetime, not timedelta
+ assert_series_equal(actual, sn)
+ actual = s2 - pd.NaT
+ assert_series_equal(actual, sn)
+
+ actual = s1 + df1
+ assert_frame_equal(actual, df2)
+ actual = s2 - df1
+ assert_frame_equal(actual, df1)
+ actual = df1 + s1
+ assert_frame_equal(actual, df2)
+ actual = df2 - s1
+ assert_frame_equal(actual, df1)
+
+ actual = df1 + df1
+ assert_frame_equal(actual, df2)
+ actual = df2 - df1
+ assert_frame_equal(actual, df1)
+
+ actual = df1 + scalar1
+ assert_frame_equal(actual, df2)
+ actual = df2 - scalar1
+ assert_frame_equal(actual, df1)
+
+ actual = df1 + timedelta_NaT
+ assert_frame_equal(actual, dfn)
+ actual = df1 - timedelta_NaT
+ assert_frame_equal(actual, dfn)
+
+ actual = df1 + NA
+ assert_frame_equal(actual, dfn)
+ actual = df1 - NA
+ assert_frame_equal(actual, dfn)
+
+ actual = df1 + pd.NaT # NaT is datetime, not timedelta
+ assert_frame_equal(actual, dfn)
+ actual = df1 - pd.NaT
+ assert_frame_equal(actual, dfn)
+
+ def test_apply_to_timedelta(self):
+ _skip_if_numpy_not_friendly()
+
+ timedelta_NaT = pd.to_timedelta('NaT')
+
+ list_of_valid_strings = ['00:00:01', '00:00:02']
+ a = pd.to_timedelta(list_of_valid_strings)
+ b = Series(list_of_valid_strings).apply(pd.to_timedelta)
+ # Can't compare until apply on a Series gives the correct dtype
+ # assert_series_equal(a, b)
+
+ list_of_strings = ['00:00:01', np.nan, pd.NaT, timedelta_NaT]
+ a = pd.to_timedelta(list_of_strings)
+ b = Series(list_of_strings).apply(pd.to_timedelta)
+ # Can't compare until apply on a Series gives the correct dtype
+ # assert_series_equal(a, b)
+
+
 if __name__ == '__main__':
  nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
  exit=False)

pandas/tseries/timedeltas.py

@@ -9,7 +9,7 @@
 import pandas.tslib as tslib
 from pandas import compat, _np_version_under1p7
 from pandas.core.common import (ABCSeries, is_integer, is_timedelta64_dtype,
- _values_from_object, is_list_like)
+ _values_from_object, is_list_like, isnull)
 
 repr_timedelta = tslib.repr_timedelta64
 repr_timedelta64 = tslib.repr_timedelta64
@@ -84,6 +84,8 @@ def conv(v):
  r = conv(r)
  elif r == tslib.iNaT:
  return r
+ elif isnull(r):
+ return np.timedelta64('NaT')
  elif isinstance(r, np.timedelta64):
  r = r.astype("m8[{0}]".format(unit.lower()))
  elif is_integer(r):