Allows for merging of SparseDataFrames, and fixes __array__ interface

doc/source/whatsnew/v0.23.0.txt

@@ -238,6 +238,7 @@ Other Enhancements
 
 - ``IntervalIndex.astype`` now supports conversions between subtypes when passed an ``IntervalDtype`` (:issue:`19197`)
 - :class:`IntervalIndex` and its associated constructor methods (``from_arrays``, ``from_breaks``, ``from_tuples``) have gained a ``dtype`` parameter (:issue:`19262`)
+- :func:`pandas.merge` now supports merging of :class:`SparseDataFrame` (:issue:`13665`)
 
 .. _whatsnew_0230.api_breaking:
 
@@ -555,7 +556,7 @@ Sparse
 
 - Bug in which creating a ``SparseDataFrame`` from a dense ``Series`` or an unsupported type raised an uncontrolled exception (:issue:`19374`)
 - Bug in :class:`SparseDataFrame.to_csv` causing exception (:issue:`19384`)
--
+- Bug in :class:`SparseSeries.__array__` returning only non-faills (:issue:`13665`)
 
 Reshaping
 ^^^^^^^^^
@@ -591,3 +592,4 @@ Other
 ^^^^^
 
 - Improved error message when attempting to use a Python keyword as an identifier in a ``numexpr`` backed query (:issue:`18221`)
+- Improved algorithms.take_1d handling of ``SparseArray`` (:issue:`19506`)

pandas/core/algorithms.py

@@ -1315,6 +1315,11 @@ def take_nd(arr, indexer, axis=0, out=None, fill_value=np.nan, mask_info=None,
  undefined if allow_fill == False and -1 is present in indexer.
  """
 
+ if is_sparse(arr):
+ return take_nd(arr.get_values(), indexer, axis=axis, out=out,
+ fill_value=fill_value, mask_info=mask_info,
+ allow_fill=allow_fill)
+
  # dispatch to internal type takes
  if is_categorical(arr):
  return arr.take_nd(indexer, fill_value=fill_value,

pandas/core/internals.py

@@ -3089,6 +3089,7 @@ def make_block(values, placement, klass=None, ndim=None, dtype=None,
  # GH#19265 pyarrow is passing this
  warnings.warn("fastpath argument is deprecated, will be removed "
  "in a future release.", DeprecationWarning)
+
  if klass is None:
  dtype = dtype or values.dtype
  klass = get_block_type(values, dtype)
@@ -5304,6 +5305,22 @@ def concatenate_block_managers(mgrs_indexers, axes, concat_axis, copy):
  elif is_uniform_join_units(join_units):
  b = join_units[0].block.concat_same_type(
  [ju.block for ju in join_units], placement=placement)
+ elif is_sparse_join_units(join_units):
+ values = concatenate_join_units(join_units, concat_axis, copy=copy)
+
+ if len(values.shape) == 2:
+ values = values[0]
+ else:
+ assert len(values.shape) == 1
+
+ block = join_units[0].block
+
+ if block:
+ fill_value = block.fill_value
+ else:
+ fill_value = np.nan
+ array = SparseArray(values, fill_value=fill_value)
+ b = make_block(array, klass=SparseBlock, placement=placement)
  else:
  b = make_block(
  concatenate_join_units(join_units, concat_axis, copy=copy),
@@ -5313,6 +5330,18 @@ def concatenate_block_managers(mgrs_indexers, axes, concat_axis, copy):
  return BlockManager(blocks, axes)
 
 
+def is_sparse_join_units(join_units):
+ """
+ Check if all of the join units are sparse. This leads to building
+ SparseArray over dense array representations so that we can merge
+ SparseSeries / SparseDataFrame
+
+ This is very similar to how pandas.concat works for conatting two
+ SparseDataFrame / SparseSeries
+ """
+ return all(type(ju.block) is SparseBlock for ju in join_units)
+
+
 def is_uniform_join_units(join_units):
  """
  Check if the join units consist of blocks of uniform type that can
@@ -5686,7 +5715,10 @@ def is_na(self):
  def get_reindexed_values(self, empty_dtype, upcasted_na):
  if upcasted_na is None:
  # No upcasting is necessary
- fill_value = self.block.fill_value
+
+ # You would think that you want self.block.fill_value here
+ # But in reality that will fill with a bunch of wrong values
+ fill_value = np.nan
  values = self.block.get_values()
  else:
  fill_value = upcasted_na

pandas/core/reshape/merge.py

@@ -38,6 +38,8 @@
  concatenate_block_managers)
 from pandas.util._decorators import Appender, Substitution
 
+from pandas.core.sparse.array import SparseArray
+
 from pandas.core.sorting import is_int64_overflow_possible
 import pandas.core.algorithms as algos
 import pandas.core.sorting as sorting
@@ -665,7 +667,6 @@ def _maybe_restore_index_levels(self, result):
  result.set_index(names_to_restore, inplace=True)
 
  def _maybe_add_join_keys(self, result, left_indexer, right_indexer):
-
  left_has_missing = None
  right_has_missing = None
 
@@ -731,7 +732,11 @@ def _maybe_add_join_keys(self, result, left_indexer, right_indexer):
  if mask.all():
  key_col = rvals
  else:
- key_col = Index(lvals).where(~mask, rvals)
+ # Might need to be IntIndex not Index
+ if isinstance(lvals, SparseArray):
+ key_col = Index(lvals.get_values()).where(~mask, rvals)
+ else:
+ key_col = Index(lvals).where(~mask, rvals)
 
  if result._is_label_reference(name):
  result[name] = key_col

pandas/core/sparse/array.py

@@ -38,7 +38,6 @@
 from pandas.util._decorators import Appender
 from pandas.core.indexes.base import _index_shared_docs
 
-
 _sparray_doc_kwargs = dict(klass='SparseArray')
 
 
@@ -259,6 +258,7 @@ def __array_wrap__(self, out_arr, context=None):
  ufunc, args, domain = context
  # to apply ufunc only to fill_value (to avoid recursive call)
  args = [getattr(a, 'fill_value', a) for a in args]
+
  with np.errstate(all='ignore'):
  fill_value = ufunc(self.fill_value, *args[1:])
  else:
@@ -292,9 +292,9 @@ def __setstate__(self, state):
  self._fill_value = fill_value
 
  def __len__(self):
- try:
+ if hasattr(self, 'sp_index'):
  return self.sp_index.length
- except:
+ else:
  return 0
 
  def __unicode__(self):

pandas/core/sparse/frame.py

@@ -73,6 +73,10 @@ def __init__(self, data=None, index=None, columns=None, default_kind=None,
  if columns is None:
  raise Exception("cannot pass a series w/o a name or columns")
  data = {columns[0]: data}
+ elif isinstance(data, BlockManager):
+ fill_value_size = len(set(b.fill_value for b in data.blocks))
+ if default_fill_value is None and fill_value_size == 1:
+ default_fill_value = data.blocks[0].fill_value
 
  if default_fill_value is None:
  default_fill_value = np.nan

pandas/core/sparse/series.py

@@ -175,7 +175,7 @@ def values(self):
 
  def __array__(self, result=None):
  """ the array interface, return my values """
- return self.block.values
+ return self.block.values.values
 
  def get_values(self):
  """ same as values """
@@ -271,6 +271,7 @@ def __array_wrap__(self, result, context=None):
 
  See SparseArray.__array_wrap__ for detail.
  """
+
  if isinstance(context, tuple) and len(context) == 3:
  ufunc, args, domain = context
  args = [getattr(a, 'fill_value', a) for a in args]
@@ -279,8 +280,18 @@ def __array_wrap__(self, result, context=None):
  else:
  fill_value = self.fill_value
 
+ # GH 14167
+ # Since we are returning a dense representation of
+ # SparseSeries sparse_index might not align when calling
+ # ufunc on the array. There doesn't seem to be a better way
+ # to do this unfortunately.
+ if len(result) != self.sp_index.npoints:
+ sparse_index = None
+ else:
+ sparse_index = self.sp_index
+
  return self._constructor(result, index=self.index,
- sparse_index=self.sp_index,
+ sparse_index=sparse_index,
  fill_value=fill_value,
  copy=False).__finalize__(self)
 
@@ -402,8 +413,8 @@ def abs(self):
  -------
  abs: type of caller
  """
- return self._constructor(np.abs(self.values),
-  index=self.index).__finalize__(self)
+
+ return np.abs(self)
 
  def get(self, label, default=None):
  """
@@ -544,7 +555,7 @@ def to_dense(self, sparse_only=False):
  index = self.index.take(int_index.indices)
  return Series(self.sp_values, index=index, name=self.name)
  else:
- return Series(self.values.to_dense(), index=self.index,
+ return Series(self.get_values(), index=self.index,
  name=self.name)
 
  @property

pandas/tests/reshape/merge/test_merge.py

@@ -7,6 +7,7 @@
 import numpy as np
 import random
 import re
+import itertools
 
 import pandas as pd
 from pandas.compat import lrange, lzip
@@ -18,7 +19,10 @@
  is_categorical_dtype,
  is_object_dtype,
 )
-from pandas import DataFrame, Index, MultiIndex, Series, Categorical
+from pandas import (
+ DataFrame, Index,
+ MultiIndex, Series, Categorical
+)
 import pandas.util.testing as tm
 from pandas.api.types import CategoricalDtype as CDT
 
@@ -1802,3 +1806,88 @@ def test_merge_on_indexes(self, left_df, right_df, how, sort, expected):
  how=how,
  sort=sort)
  tm.assert_frame_equal(result, expected)
+
+
+class TestMergeSparseDataFrames(object):
+ @pytest.mark.parametrize('fill_value,how', itertools.product([0, 1,
+ None,
+ np.nan],
+ ['left',
+ 'right',
+ 'outer',
+ 'inner']))
+ def test_merge_two_sparse_frames(self, fill_value, how):
+ dense_evens = pd.DataFrame({'A': list(range(0, 200, 2)),
+ 'B': np.random.randint(0, 100, size=100)})
+ dense_threes = pd.DataFrame({'A': list(range(0, 300, 3)),
+ 'B': np.random.randint(0, 100, size=100)})
+
+ sparse_evens = dense_evens.to_sparse(fill_value=fill_value)
+ sparse_threes = dense_threes.to_sparse(fill_value=fill_value)
+
+ to_merge_sparse = [sparse_evens, sparse_threes]
+
+ to_merge_dense = [dense_evens, dense_threes]
+
+ for _ in range(2):
+ sparse_merge = to_merge_sparse[0].merge(to_merge_sparse[1],
+ how=how, on='A')
+
+ dense_merge = to_merge_dense[0].merge(to_merge_dense[1],
+ how=how, on='A')
+
+ # If you merge two dense frames together it tends to default to
+ # float64 not the original dtype
+ dense_merge['B_x'] = dense_merge['B_x'].astype(dense_evens.A.dtype,
+ errors='ignore')
+ dense_merge['B_y'] = dense_merge['B_y'].astype(dense_evens.A.dtype,
+ errors='ignore')
+
+ if fill_value is None or fill_value is np.nan:
+ assert sparse_merge.default_fill_value is np.nan
+ else:
+ tm.assert_almost_equal(sparse_merge.default_fill_value,
+ fill_value)
+
+ exp = dense_merge.to_sparse(fill_value=fill_value)
+ tm.assert_sp_frame_equal(sparse_merge, exp,
+ exact_indices=False,
+ check_dtype=False)
+
+ to_merge_sparse = to_merge_sparse[::-1]
+ to_merge_dense = to_merge_dense[::-1]
+
+ @pytest.mark.parametrize('fill_value,how', itertools.product([0, 1,
+ None,
+ np.nan],
+ ['left',
+ 'right',
+ 'outer',
+ 'inner']))
+ def test_merge_dense_sparse_frames(self, fill_value, how):
+ fill_value = np.nan
+
+ dense_evens = pd.DataFrame({'A': list(range(0, 200, 2)),
+ 'B': np.random.randint(0, 100, size=100)})
+
+ dense_threes = pd.DataFrame({'A': list(range(0, 300, 3)),
+ 'B': np.random.randint(0, 100, size=100)})
+
+ sparse_evens = dense_evens.to_sparse(fill_value=fill_value)
+
+ to_merge = [sparse_evens, dense_threes]
+ to_merge_dense = [dense_evens, dense_threes]
+
+ for _ in range(2):
+ merged = to_merge[0].merge(to_merge[1], how=how, on='A')
+
+ dense_merge = to_merge_dense[0].merge(to_merge_dense[1],
+ how=how, on='A')
+
+ for column in dense_merge.columns:
+ dense_col = merged[column].to_dense()
+ tm.assert_series_equal(dense_col, dense_merge[column],
+ check_dtype=False)
+
+ to_merge = to_merge[::-1]
+ to_merge_dense = to_merge_dense[::-1]

pandas/tests/sparse/frame/test_frame.py

@@ -222,27 +222,34 @@ class Unknown:
  '"Unknown" for data argument'):
  SparseDataFrame(Unknown())
 
- def test_constructor_preserve_attr(self):
+ # Cannot use None as a fill_value cause it will overwrite as zeros
+ @pytest.mark.parametrize('fill_value', [0, 1, np.nan])
+ def test_constructor_preserve_attr(self, fill_value):
  # GH 13866
- arr = pd.SparseArray([1, 0, 3, 0], dtype=np.int64, fill_value=0)
+ arr = pd.SparseArray([1, 0, 3, 0], dtype=np.int64,
+ fill_value=fill_value)
+
  assert arr.dtype == np.int64
- assert arr.fill_value == 0
+ tm.assert_almost_equal(arr.fill_value, fill_value)
 
  df = pd.SparseDataFrame({'x': arr})
  assert df['x'].dtype == np.int64
- assert df['x'].fill_value == 0
+
+ tm.assert_almost_equal(df['x'].fill_value, fill_value)
 
  s = pd.SparseSeries(arr, name='x')
  assert s.dtype == np.int64
- assert s.fill_value == 0
+ tm.assert_almost_equal(s.fill_value, fill_value)
 
  df = pd.SparseDataFrame(s)
  assert df['x'].dtype == np.int64
- assert df['x'].fill_value == 0
+
+ tm.assert_almost_equal(df['x'].fill_value, fill_value)
 
  df = pd.SparseDataFrame({'x': s})
  assert df['x'].dtype == np.int64
- assert df['x'].fill_value == 0
+
+ tm.assert_almost_equal(df['x'].fill_value, fill_value)
 
  def test_constructor_nan_dataframe(self):
  # GH 10079

pandas/tests/sparse/series/test_series.py

@@ -626,10 +626,12 @@ def _check_inplace_op(iop, op):
  getattr(operator, op))
 
  def test_abs(self):
- s = SparseSeries([1, 2, -3], name='x')
- expected = SparseSeries([1, 2, 3], name='x')
+ s = SparseSeries([-1, -2, -3, None, np.nan], name='x')
+ expected = SparseSeries([1, 2, 3, None, np.nan], name='x')
  result = s.abs()
  tm.assert_sp_series_equal(result, expected)
+ assert result.npoints == expected.npoints
+ assert result.npoints == len(result.sp_values)
  assert result.name == 'x'
 
  result = abs(s)
@@ -641,9 +643,9 @@ def test_abs(self):
  assert result.name == 'x'
 
  s = SparseSeries([1, -2, 2, -3], fill_value=-2, name='x')
- expected = SparseSeries([1, 2, 3], sparse_index=s.sp_index,
- fill_value=2, name='x')
+ expected = SparseSeries([1, 2, 2, 3], fill_value=2, name='x')
  result = s.abs()
+
  tm.assert_sp_series_equal(result, expected)
  assert result.name == 'x'