wip

doc/source/whatsnew/v0.24.0.txt

@@ -327,6 +327,8 @@ is the case with :attr:`Period.end_time`, for example
 This has some notable changes
 
 - ``SparseArray`` is no longer a subclass of :class:`numpy.ndarray`
+- ``SparseArray.dtype`` and ``SparseSeries.dtype`` are now instances of ``SparseDtype``, rather than ``np.dtype``.
+ Access the underlying dtype with ``SparseDtype.subdtype``.
 - :meth:`numpy.asarray(sparse_array)` now returns a dense array with all the values,
  not just the non-fill-value values (:issue:`todo`)
 

pandas/core/sparse/api.py

@@ -3,3 +3,4 @@
 from pandas.core.sparse.array import SparseArray
 from pandas.core.sparse.series import SparseSeries
 from pandas.core.sparse.frame import SparseDataFrame
+from pandas.core.sparse.dtype import SparseDtype

pandas/core/sparse/array.py

@@ -22,6 +22,7 @@
  ensure_platform_int,
  is_float, is_integer,
  is_object_dtype,
+ is_array_like,
  is_integer_dtype,
  is_float_dtype,
  is_extension_array_dtype,
@@ -54,10 +55,11 @@
 
 
 def _get_fill(arr):
+ # type: (SparseArray) -> ndarray
  # coerce fill_value to arr dtype if possible
  # int64 SparseArray can have NaN as fill_value if there is no missing
  try:
- return np.asarray(arr.fill_value, dtype=arr.dtype)
+ return np.asarray(arr.fill_value, dtype=arr.dtype.subdtype)
  except ValueError:
  return np.asarray(arr.fill_value)
 
@@ -143,10 +145,8 @@ class SparseArray(PandasObject, ExtensionArray, ExtensionOpsMixin):
  __array_priority__ = 15
  _pandas_ftype = 'sparse'
 
- def __init__(self, data, sparse_index=None, fill_value=np.nan, kind='block',
+ def __init__(self, data, sparse_index=None, fill_value=None, kind='block',
  dtype=None, copy=False):
- if fill_value is None:
- fill_value = np.nan
  from pandas.core.internals import SingleBlockManager
 
  if isinstance(dtype, SparseDtype):
@@ -155,6 +155,18 @@ def __init__(self, data, sparse_index=None, fill_value=np.nan, kind='block',
  if isinstance(data, SingleBlockManager):
  data = data.internal_values()
 
+ # TODO: disentable the fill_value dtype inference from
+ # dtype inference
+ if not is_array_like(data):
+ data = np.asarray(data, dtype=dtype)
+
+ if fill_value is None:
+ fill_value_dtype = dtype or data.dtype
+ if fill_value_dtype is None:
+ fill_value = np.nan
+ fill_value = na_value_for_dtype(fill_value_dtype)
+
+
  if isinstance(data, type(self)) and sparse_index is None:
  sparse_index = data._sparse_index
  sparse_values = np.asarray(data.sp_values, dtype=dtype)
@@ -175,6 +187,9 @@ def __init__(self, data, sparse_index=None, fill_value=np.nan, kind='block',
  self.fill_value = fill_value
 
  def __array__(self, dtype=None, copy=True):
+ if self.sp_index.ngaps == 0:
+ # Compat for na dtype and int values.
+ return self.sp_values
  out = np.full(self.shape, self.fill_value, dtype=dtype)
  out[self.sp_index.to_int_index().indices] = self.sp_values
  return out
@@ -325,6 +340,14 @@ def __getitem__(self, key):
  return self._get_val_at(key)
  elif isinstance(key, tuple):
  data_slice = self.values[key]
+ elif isinstance(key, slice):
+ # special case to preserve dtypes
+ if key == slice(None):
+ return self.copy()
+ # TODO: this logic is surely elsewhere
+ # TODO: this could be more efficient
+ indices = np.arange(len(self))[key]
+ return self.take(indices, allow_fill=False)
  else:
  if isinstance(key, SparseArray):
  if is_bool_dtype(key):
@@ -417,6 +440,12 @@ def _take_without_fill(self, indices):
  if to_shift.any():
  indices[to_shift] += n
 
+ if self.sp_index.npoints == 0:
+ # edge case in take...
+ # I think just return
+ arr, sp_index, fill_value = make_sparse(indices, fill_value=self.fill_value)
+ return type(self)(arr, sparse_index=sp_index, fill_value=fill_value)
+
  sp_indexer = self.sp_index.lookup_array(indices)
  taken = self.sp_values.take(sp_indexer)
  fillable = (sp_indexer < 0)
@@ -506,6 +535,11 @@ def map(self, mapper):
  # TODO: series?
  return type(self)(sp_values, sparse_index=self.sp_index, fill_value=fill_value)
 
+ def get_values(self, fill=None):
+ """ return a dense representation """
+ # TODO: deprecate for to_dense?
+ return self.to_dense(fill=fill)
+
  def to_dense(self, fill=None):
  """
  Convert SparseArray to a NumPy array.

pandas/tests/series/test_missing.py

@@ -778,6 +778,7 @@ def test_sparse_series_fillna_limit(self):
  # TODO: what is this test doing? why are result an expected
  # the same call to fillna?
  with tm.assert_produces_warning(PerformanceWarning):
+ # TODO: release-note fillna performance warning
  result = ss.fillna(method='pad', limit=5)
  expected = ss.fillna(method='pad', limit=5)
  expected = expected.to_dense()
@@ -800,7 +801,8 @@ def test_sparse_series_pad_backfill_limit(self):
  s = s.to_sparse()
 
  result = s[:2].reindex(index, method='pad', limit=5)
- expected = s[:2].reindex(index).fillna(method='pad')
+ with tm.assert_produces_warning(PerformanceWarning):
+ expected = s[:2].reindex(index).fillna(method='pad')
  expected = expected.to_dense()
  expected[-3:] = np.nan
  expected = expected.to_sparse()

pandas/tests/series/test_subclass.py

@@ -1,8 +1,10 @@
 # coding=utf-8
 # pylint: disable-msg=E1101,W0612
+import pytest
 
 import numpy as np
 import pandas as pd
+from pandas.core.sparse.dtype import SparseDtype
 import pandas.util.testing as tm
 
 
@@ -47,29 +49,29 @@ def test_subclass_sparse_slice(self):
  s = tm.SubclassedSparseSeries([1, 2, 3, 4, 5])
  exp = tm.SubclassedSparseSeries([2, 3, 4], index=[1, 2, 3])
  tm.assert_sp_series_equal(s.loc[1:3], exp)
- assert s.loc[1:3].dtype == np.int64
+ assert s.loc[1:3].dtype == SparseDtype(np.int64)
 
  exp = tm.SubclassedSparseSeries([2, 3], index=[1, 2])
  tm.assert_sp_series_equal(s.iloc[1:3], exp)
- assert s.iloc[1:3].dtype == np.int64
+ assert s.iloc[1:3].dtype == SparseDtype(np.int64)
 
  exp = tm.SubclassedSparseSeries([2, 3], index=[1, 2])
  tm.assert_sp_series_equal(s[1:3], exp)
- assert s[1:3].dtype == np.int64
+ assert s[1:3].dtype == SparseDtype(np.int64)
 
  # float64
  s = tm.SubclassedSparseSeries([1., 2., 3., 4., 5.])
  exp = tm.SubclassedSparseSeries([2., 3., 4.], index=[1, 2, 3])
  tm.assert_sp_series_equal(s.loc[1:3], exp)
- assert s.loc[1:3].dtype == np.float64
+ assert s.loc[1:3].dtype == SparseDtype(np.float64)
 
  exp = tm.SubclassedSparseSeries([2., 3.], index=[1, 2])
  tm.assert_sp_series_equal(s.iloc[1:3], exp)
- assert s.iloc[1:3].dtype == np.float64
+ assert s.iloc[1:3].dtype == SparseDtype(np.float64)
 
  exp = tm.SubclassedSparseSeries([2., 3.], index=[1, 2])
  tm.assert_sp_series_equal(s[1:3], exp)
- assert s[1:3].dtype == np.float64
+ assert s[1:3].dtype == SparseDtype(np.float64)
 
  def test_subclass_sparse_addition(self):
  s1 = tm.SubclassedSparseSeries([1, 3, 5])
@@ -82,6 +84,7 @@ def test_subclass_sparse_addition(self):
  exp = tm.SubclassedSparseSeries([5., 7., 9.])
  tm.assert_sp_series_equal(s1 + s2, exp)
 
+ @pytest.mark.xfail(reason="XXX: SS used to reindex. Now we match Series.")
  def test_subclass_sparse_to_frame(self):
  s = tm.SubclassedSparseSeries([1, 2], index=list('abcd'), name='xxx')
  res = s.to_frame()

pandas/tests/sparse/test_array.py

@@ -8,7 +8,7 @@
 from numpy import nan
 import numpy as np
 
-from pandas.core.sparse.api import SparseArray, SparseSeries
+from pandas.core.sparse.api import SparseArray, SparseSeries, SparseDtype
 from pandas._libs.sparse import IntIndex
 from pandas.util.testing import assert_almost_equal
 import pandas.util.testing as tm
@@ -28,94 +28,97 @@ def setup_method(self, method):
 
  def test_constructor_dtype(self):
  arr = SparseArray([np.nan, 1, 2, np.nan])
- assert arr.dtype == np.float64
+ assert arr.dtype == SparseDtype(np.float64)
+ assert arr.dtype.subdtype == np.float64
  assert np.isnan(arr.fill_value)
 
  arr = SparseArray([np.nan, 1, 2, np.nan], fill_value=0)
- assert arr.dtype == np.float64
+ assert arr.dtype == SparseDtype(np.float64)
  assert arr.fill_value == 0
 
  arr = SparseArray([0, 1, 2, 4], dtype=np.float64)
- assert arr.dtype == np.float64
+ assert arr.dtype == SparseDtype(np.float64)
  assert np.isnan(arr.fill_value)
 
  arr = SparseArray([0, 1, 2, 4], dtype=np.int64)
- assert arr.dtype == np.int64
+ assert arr.dtype == SparseDtype(np.int64)
  assert arr.fill_value == 0
 
  arr = SparseArray([0, 1, 2, 4], fill_value=0, dtype=np.int64)
- assert arr.dtype == np.int64
+ assert arr.dtype == SparseDtype(np.int64)
  assert arr.fill_value == 0
 
  arr = SparseArray([0, 1, 2, 4], dtype=None)
- assert arr.dtype == np.int64
+ assert arr.dtype == SparseDtype(np.int64)
  assert arr.fill_value == 0
 
  arr = SparseArray([0, 1, 2, 4], fill_value=0, dtype=None)
- assert arr.dtype == np.int64
+ assert arr.dtype == SparseDtype(np.int64)
  assert arr.fill_value == 0
 
  def test_constructor_object_dtype(self):
  # GH 11856
  arr = SparseArray(['A', 'A', np.nan, 'B'], dtype=np.object)
- assert arr.dtype == np.object
+ assert arr.dtype == SparseDtype(np.object)
  assert np.isnan(arr.fill_value)
 
  arr = SparseArray(['A', 'A', np.nan, 'B'], dtype=np.object,
  fill_value='A')
- assert arr.dtype == np.object
+ assert arr.dtype == SparseDtype(np.object)
  assert arr.fill_value == 'A'
 
  # GH 17574
  data = [False, 0, 100.0, 0.0]
  arr = SparseArray(data, dtype=np.object, fill_value=False)
- assert arr.dtype == np.object
+ assert arr.dtype == SparseDtype(np.object)
  assert arr.fill_value is False
  arr_expected = np.array(data, dtype=np.object)
  it = (type(x) == type(y) and x == y for x, y in zip(arr, arr_expected))
  assert np.fromiter(it, dtype=np.bool).all()
 
  def test_constructor_spindex_dtype(self):
  arr = SparseArray(data=[1, 2], sparse_index=IntIndex(4, [1, 2]))
- tm.assert_sp_array_equal(arr, SparseArray([np.nan, 1, 2, np.nan]))
- assert arr.dtype == np.float64
+ # XXX: specifying sparse_index shouldn't change the inferred fill_value
+ expected = SparseArray([0, 1, 2, 0])
+ tm.assert_sp_array_equal(arr, SparseArray([0, 1, 2, 0]))
+ assert arr.dtype == SparseDtype(np.float64)
  assert np.isnan(arr.fill_value)
 
  arr = SparseArray(data=[1, 2, 3],
  sparse_index=IntIndex(4, [1, 2, 3]),
  dtype=np.int64, fill_value=0)
  exp = SparseArray([0, 1, 2, 3], dtype=np.int64, fill_value=0)
  tm.assert_sp_array_equal(arr, exp)
- assert arr.dtype == np.int64
+ assert arr.dtype == SparseDtype(np.int64)
  assert arr.fill_value == 0
 
  arr = SparseArray(data=[1, 2], sparse_index=IntIndex(4, [1, 2]),
  fill_value=0, dtype=np.int64)
  exp = SparseArray([0, 1, 2, 0], fill_value=0, dtype=np.int64)
  tm.assert_sp_array_equal(arr, exp)
- assert arr.dtype == np.int64
+ assert arr.dtype == SparseDtype(np.int64)
  assert arr.fill_value == 0
 
  arr = SparseArray(data=[1, 2, 3],
  sparse_index=IntIndex(4, [1, 2, 3]),
  dtype=None, fill_value=0)
  exp = SparseArray([0, 1, 2, 3], dtype=None)
  tm.assert_sp_array_equal(arr, exp)
- assert arr.dtype == np.int64
+ assert arr.dtype == SparseDtype(np.int64)
  assert arr.fill_value == 0
 
  # scalar input
  arr = SparseArray(data=1, sparse_index=IntIndex(1, [0]), dtype=None)
  exp = SparseArray([1], dtype=None)
  tm.assert_sp_array_equal(arr, exp)
- assert arr.dtype == np.int64
+ assert arr.dtype == SparseDtype(np.int64)
  assert arr.fill_value == 0
 
  arr = SparseArray(data=[1, 2], sparse_index=IntIndex(4, [1, 2]),
  fill_value=0, dtype=None)
  exp = SparseArray([0, 1, 2, 0], fill_value=0, dtype=None)
  tm.assert_sp_array_equal(arr, exp)
- assert arr.dtype == np.int64
+ assert arr.dtype == SparseDtype(np.int64)
  assert arr.fill_value == 0
 
  @pytest.mark.parametrize('scalar,dtype', [

pandas/util/testing.py

@@ -1587,14 +1587,16 @@ def assert_sp_series_equal(left, right, check_dtype=True, exact_indices=True,
  assert_index_equal(left.index, right.index,
  obj='{obj}.index'.format(obj=obj))
 
+ # TODO: this can just be .values I think
  assert_sp_array_equal(left.block.values, right.block.values)
 
  if check_names:
  assert_attr_equal('name', left, right)
  if check_dtype:
  assert_attr_equal('dtype', left, right)
 
- assert_numpy_array_equal(left.values, right.values)
+ assert_numpy_array_equal(np.asarray(left.values),
+ np.asarray(right.values))
 
 
 def assert_sp_frame_equal(left, right, check_dtype=True, exact_indices=True,