BUG: DatetimeTZBlock.fillna raises TypeError

Author: sinhrks

doc/source/whatsnew/v0.17.0.txt

@@ -1157,3 +1157,4 @@ Bug Fixes
 - Bug in ``Index`` dtype may not applied properly (:issue:`11017`)
 - Bug in ``io.gbq`` when testing for minimum google api client version (:issue:`10652`)
 - Bug in ``DataFrame`` construction from nested ``dict`` with ``timedelta`` keys (:issue:`11129`)
+- Bug in ``.fillna`` against may raise ``TypeError`` when data contains datetime dtype (:issue:`7095`, :issue:`11153`)

pandas/core/dtypes.py

@@ -181,7 +181,7 @@ def construct_from_string(cls, string):
 
  def __unicode__(self):
  # format the tz
- return "datetime64[{unit}, {tz}]".format(unit=self.unit,tz=self.tz)
+ return "datetime64[{unit}, {tz}]".format(unit=self.unit, tz=self.tz)
 
  @property
  def name(self):

pandas/core/internals.py

@@ -1947,20 +1947,42 @@ def _try_fill(self, value):
  def fillna(self, value, limit=None,
  inplace=False, downcast=None):
 
- # straight putmask here
- values = self.values if inplace else self.values.copy()
  mask = isnull(self.values)
  value = self._try_fill(value)
+
  if limit is not None:
  if self.ndim > 2:
  raise NotImplementedError("number of dimensions for 'fillna' "
  "is currently limited to 2")
  mask[mask.cumsum(self.ndim-1)>limit]=False
 
- np.putmask(values, mask, value)
- return [self if inplace else
- self.make_block(values,
- fastpath=True)]
+ if mask.any():
+ try:
+ return self._fillna_mask(mask, value, inplace=inplace)
+ except TypeError:
+ pass
+ # _fillna_mask raises TypeError when it fails
+ # cannot perform inplace op because of object coercion
+ values = self.get_values(dtype=object)
+ np.putmask(values, mask, value)
+ return [self.make_block(values, fastpath=True)]
+ else:
+ return [self if inplace else self.copy()]
+
+ def _fillna_mask(self, mask, value, inplace=False):
+ if getattr(value, 'tzinfo', None) is None:
+ # Series comes to this path
+ values = self.values
+ if not inplace:
+ values = values.copy()
+ try:
+ np.putmask(values, mask, value)
+ return [self if inplace else
+ self.make_block(values, fastpath=True)]
+ except (ValueError, TypeError):
+ # scalar causes ValueError, and array causes TypeError
+ pass
+ raise TypeError
 
  def to_native_types(self, slicer=None, na_rep=None, date_format=None,
  quoting=None, **kwargs):
@@ -2033,6 +2055,29 @@ def get_values(self, dtype=None):
  .reshape(self.values.shape)
  return self.values
 
+ def _fillna_mask(self, mask, value, inplace=False):
+ # cannot perform inplace op for internal DatetimeIndex
+ my_tz = tslib.get_timezone(self.values.tz)
+ value_tz = tslib.get_timezone(getattr(value, 'tzinfo', None))
+
+ if (my_tz == value_tz or self.dtype == getattr(value, 'dtype', None)):
+ if my_tz == value_tz:
+ # hack for PY2.6 / numpy 1.7.1.
+ # Other versions can directly use self.values.putmask
+ # --------------------------------------
+ try:
+ value = value.asm8
+ except AttributeError:
+ value = tslib.Timestamp(value).asm8
+ ### ------------------------------------
+
+ try:
+ values = self.values.putmask(mask, value)
+ return [self.make_block(values, fastpath=True)]
+ except ValueError:
+ pass
+ raise TypeError
+
  def _slice(self, slicer):
  """ return a slice of my values """
  if isinstance(slicer, tuple):

pandas/tests/test_dtypes.py

@@ -137,6 +137,20 @@ def test_basic(self):
  self.assertFalse(is_datetimetz(np.dtype('float64')))
  self.assertFalse(is_datetimetz(1.0))
 
+ def test_dst(self):
+
+ dr1 = date_range('2013-01-01', periods=3, tz='US/Eastern')
+ s1 = Series(dr1, name='A')
+ self.assertTrue(is_datetimetz(s1))
+
+ dr2 = date_range('2013-08-01', periods=3, tz='US/Eastern')
+ s2 = Series(dr2, name='A')
+ self.assertTrue(is_datetimetz(s2))
+ self.assertEqual(s1.dtype, s2.dtype)
+
+
+
+
 if __name__ == '__main__':
  nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
  exit=False)

pandas/tests/test_frame.py

@@ -8740,6 +8740,34 @@ def test_fillna_dtype_conversion(self):
  result = df.fillna(v)
  assert_frame_equal(result, expected)
 
+ def test_fillna_datetime_columns(self):
+ # GH 7095
+ df = pd.DataFrame({'A': [-1, -2, np.nan],
+ 'B': date_range('20130101', periods=3),
+ 'C': ['foo', 'bar', None],
+ 'D': ['foo2', 'bar2', None]},
+ index=date_range('20130110', periods=3))
+ result = df.fillna('?')
+ expected = pd.DataFrame({'A': [-1, -2, '?'],
+ 'B': date_range('20130101', periods=3),
+ 'C': ['foo', 'bar', '?'],
+ 'D': ['foo2', 'bar2', '?']},
+ index=date_range('20130110', periods=3))
+ self.assert_frame_equal(result, expected)
+
+ df = pd.DataFrame({'A': [-1, -2, np.nan],
+ 'B': [pd.Timestamp('2013-01-01'), pd.Timestamp('2013-01-02'), pd.NaT],
+ 'C': ['foo', 'bar', None],
+ 'D': ['foo2', 'bar2', None]},
+ index=date_range('20130110', periods=3))
+ result = df.fillna('?')
+ expected = pd.DataFrame({'A': [-1, -2, '?'],
+ 'B': [pd.Timestamp('2013-01-01'), pd.Timestamp('2013-01-02'), '?'],
+ 'C': ['foo', 'bar', '?'],
+ 'D': ['foo2', 'bar2', '?']},
+ index=date_range('20130110', periods=3))
+ self.assert_frame_equal(result, expected)
+
  def test_ffill(self):
  self.tsframe['A'][:5] = nan
  self.tsframe['A'][-5:] = nan

pandas/tests/test_series.py

@@ -3937,6 +3937,89 @@ def test_datetime64_fillna(self):
  result = s.fillna(method='backfill')
  assert_series_equal(result, expected)
 
+ def test_datetime64_tz_fillna(self):
+ for tz in ['US/Eastern', 'Asia/Tokyo']:
+ # DatetimeBlock
+ s = Series([Timestamp('2011-01-01 10:00'), pd.NaT,
+ Timestamp('2011-01-03 10:00'), pd.NaT])
+ result = s.fillna(pd.Timestamp('2011-01-02 10:00'))
+ expected = Series([Timestamp('2011-01-01 10:00'), Timestamp('2011-01-02 10:00'),
+ Timestamp('2011-01-03 10:00'), Timestamp('2011-01-02 10:00')])
+ self.assert_series_equal(expected, result)
+
+ result = s.fillna(pd.Timestamp('2011-01-02 10:00', tz=tz))
+ expected = Series([Timestamp('2011-01-01 10:00'),
+ Timestamp('2011-01-02 10:00', tz=tz),
+ Timestamp('2011-01-03 10:00'),
+ Timestamp('2011-01-02 10:00', tz=tz)])
+ self.assert_series_equal(expected, result)
+
+ result = s.fillna('AAA')
+ expected = Series([Timestamp('2011-01-01 10:00'), 'AAA',
+ Timestamp('2011-01-03 10:00'), 'AAA'], dtype=object)
+ self.assert_series_equal(expected, result)
+
+ result = s.fillna({1: pd.Timestamp('2011-01-02 10:00', tz=tz),
+ 3: pd.Timestamp('2011-01-04 10:00')})
+ expected = Series([Timestamp('2011-01-01 10:00'),
+ Timestamp('2011-01-02 10:00', tz=tz),
+ Timestamp('2011-01-03 10:00'),
+ Timestamp('2011-01-04 10:00')])
+ self.assert_series_equal(expected, result)
+
+ result = s.fillna({1: pd.Timestamp('2011-01-02 10:00'),
+ 3: pd.Timestamp('2011-01-04 10:00')})
+ expected = Series([Timestamp('2011-01-01 10:00'), Timestamp('2011-01-02 10:00'),
+ Timestamp('2011-01-03 10:00'), Timestamp('2011-01-04 10:00')])
+ self.assert_series_equal(expected, result)
+
+ # DatetimeBlockTZ
+ idx = pd.DatetimeIndex(['2011-01-01 10:00', pd.NaT,
+ '2011-01-03 10:00', pd.NaT], tz=tz)
+ s = pd.Series(idx)
+ result = s.fillna(pd.Timestamp('2011-01-02 10:00'))
+ expected = Series([Timestamp('2011-01-01 10:00', tz=tz),
+ Timestamp('2011-01-02 10:00'),
+ Timestamp('2011-01-03 10:00', tz=tz),
+ Timestamp('2011-01-02 10:00')])
+ self.assert_series_equal(expected, result)
+
+ result = s.fillna(pd.Timestamp('2011-01-02 10:00', tz=tz))
+ idx = pd.DatetimeIndex(['2011-01-01 10:00', '2011-01-02 10:00',
+ '2011-01-03 10:00', '2011-01-02 10:00'],
+ tz=tz)
+ expected = Series(idx)
+ self.assert_series_equal(expected, result)
+
+ result = s.fillna(pd.Timestamp('2011-01-02 10:00', tz=tz).to_pydatetime())
+ idx = pd.DatetimeIndex(['2011-01-01 10:00', '2011-01-02 10:00',
+ '2011-01-03 10:00', '2011-01-02 10:00'],
+ tz=tz)
+ expected = Series(idx)
+ self.assert_series_equal(expected, result)
+
+ result = s.fillna('AAA')
+ expected = Series([Timestamp('2011-01-01 10:00', tz=tz), 'AAA',
+ Timestamp('2011-01-03 10:00', tz=tz), 'AAA'],
+ dtype=object)
+ self.assert_series_equal(expected, result)
+
+ result = s.fillna({1: pd.Timestamp('2011-01-02 10:00', tz=tz),
+ 3: pd.Timestamp('2011-01-04 10:00')})
+ expected = Series([Timestamp('2011-01-01 10:00', tz=tz),
+ Timestamp('2011-01-02 10:00', tz=tz),
+ Timestamp('2011-01-03 10:00', tz=tz),
+ Timestamp('2011-01-04 10:00')])
+ self.assert_series_equal(expected, result)
+
+ result = s.fillna({1: pd.Timestamp('2011-01-02 10:00', tz=tz),
+ 3: pd.Timestamp('2011-01-04 10:00', tz=tz)})
+ expected = Series([Timestamp('2011-01-01 10:00', tz=tz),
+ Timestamp('2011-01-02 10:00', tz=tz),
+ Timestamp('2011-01-03 10:00', tz=tz),
+ Timestamp('2011-01-04 10:00', tz=tz)])
+ self.assert_series_equal(expected, result)
+
  def test_fillna_int(self):
  s = Series(np.random.randint(-100, 100, 50))
  s.fillna(method='ffill', inplace=True)
@@ -5022,6 +5105,29 @@ def test_dropna_empty(self):
  # invalid axis
  self.assertRaises(ValueError, s.dropna, axis=1)
 
+
+ def test_datetime64_tz_dropna(self):
+ # DatetimeBlock
+ s = Series([Timestamp('2011-01-01 10:00'), pd.NaT,
+ Timestamp('2011-01-03 10:00'), pd.NaT])
+ result = s.dropna()
+ expected = Series([Timestamp('2011-01-01 10:00'),
+ Timestamp('2011-01-03 10:00')], index=[0, 2])
+ self.assert_series_equal(result, expected)
+
+ # DatetimeBlockTZ
+ idx = pd.DatetimeIndex(['2011-01-01 10:00', pd.NaT,
+ '2011-01-03 10:00', pd.NaT],
+ tz='Asia/Tokyo')
+ s = pd.Series(idx)
+ self.assertEqual(s.dtype, 'datetime64[ns, Asia/Tokyo]')
+ result = s.dropna()
+ expected = Series([Timestamp('2011-01-01 10:00', tz='Asia/Tokyo'),
+ Timestamp('2011-01-03 10:00', tz='Asia/Tokyo')],
+ index=[0, 2])
+ self.assertEqual(result.dtype, 'datetime64[ns, Asia/Tokyo]')
+ self.assert_series_equal(result, expected)
+
  def test_axis_alias(self):
  s = Series([1, 2, np.nan])
  assert_series_equal(s.dropna(axis='rows'), s.dropna(axis='index'))