fix v2

Author: OXPHOS

pandas/_libs/hashtable_class_helper.pxi.in

@@ -347,9 +347,10 @@ cdef class {{name}}HashTable(HashTable):
  for i in range(n):
  val = values[i]
 
- if check_null and {{null_condition}}:
- labels[i] = na_sentinel
- continue
+ if dropna:
+ if check_null and {{null_condition}}:
+ labels[i] = na_sentinel
+ continue
 
  k = kh_get_{{dtype}}(self.table, val)
 
@@ -830,13 +831,10 @@ cdef class PyObjectHashTable(HashTable):
  val = values[i]
  hash(val)
 
- if check_null and val != val:
- labels[i] = na_sentinel
- continue
-
- if dropna and val is None:
- labels[i] = na_sentinel
- continue
+ if dropna:
+ if (check_null and val != val) or val is None:
+ labels[i] = na_sentinel
+ continue
 
  k = kh_get_pymap(self.table, <PyObject*>val)
  if k != self.table.n_buckets:

pandas/core/algorithms.py

@@ -520,7 +520,7 @@ def sort_mixed(values):
 
 
 def factorize(values, sort=False, order=None, na_sentinel=-1, size_hint=None,
- dropna=False):
+ dropna=True):
  """
  Encode input values as an enumerated type or categorical variable
 

pandas/core/categorical.py

@@ -234,7 +234,8 @@ class Categorical(PandasObject):
  __array_priority__ = 1000
  _typ = 'categorical'
 
- def __init__(self, values, categories=None, ordered=False, fastpath=False):
+ def __init__(self, values, categories=None, ordered=False, fastpath=False,
+ dropna=True):
 
  self._validate_ordered(ordered)
 
@@ -281,9 +282,10 @@ def __init__(self, values, categories=None, ordered=False, fastpath=False):
 
  if categories is None:
  try:
- codes, categories = factorize(values, sort=True)
+ codes, categories = factorize(values, sort=True, dropna=dropna)
  except TypeError:
- codes, categories = factorize(values, sort=False)
+ codes, categories = factorize(values, sort=False,
+ dropna=dropna)
  if ordered:
  # raise, as we don't have a sortable data structure and so
  # the user should give us one by specifying categories
@@ -2106,7 +2108,7 @@ def _convert_to_list_like(list_like):
  return [list_like]
 
 
-def _factorize_from_iterable(values):
+def _factorize_from_iterable(values, dropna=True):
  """
  Factorize an input `values` into `categories` and `codes`. Preserves
  categorical dtype in `categories`.
@@ -2137,13 +2139,13 @@ def _factorize_from_iterable(values):
  ordered=values.ordered)
  codes = values.codes
  else:
- cat = Categorical(values, ordered=True)
+ cat = Categorical(values, ordered=True, dropna=dropna)
  categories = cat.categories
  codes = cat.codes
  return codes, categories
 
 
-def _factorize_from_iterables(iterables):
+def _factorize_from_iterables(iterables, dropna=True):
  """
  A higher-level wrapper over `_factorize_from_iterable`.
 
@@ -2165,4 +2167,5 @@ def _factorize_from_iterables(iterables):
  if len(iterables) == 0:
  # For consistency, it should return a list of 2 lists.
  return [[], []]
- return map(list, lzip(*[_factorize_from_iterable(it) for it in iterables]))
+ return map(list, lzip(*[_factorize_from_iterable(it, dropna)
+ for it in iterables]))

pandas/core/indexes/multi.py

@@ -1043,7 +1043,7 @@ def lexsort_depth(self):
  return 0
 
  @classmethod
- def from_arrays(cls, arrays, sortorder=None, names=None):
+ def from_arrays(cls, arrays, sortorder=None, names=None, dropna=False):
  """
  Convert arrays to MultiIndex
 
@@ -1083,7 +1083,7 @@ def from_arrays(cls, arrays, sortorder=None, names=None):
 
  from pandas.core.categorical import _factorize_from_iterables
 
- labels, levels = _factorize_from_iterables(arrays)
+ labels, levels = _factorize_from_iterables(arrays, dropna=dropna)
  if names is None:
  names = [getattr(arr, "name", None) for arr in arrays]
 

pandas/tests/reshape/test_pivot.py

@@ -1180,17 +1180,18 @@ def test_margin_dropna(self):
  df = pd.DataFrame({'a': [1, 2, 2, 2, 2, np.nan],
  'b': [3, 3, 4, 4, 4, 4]})
  actual = pd.crosstab(df.a, df.b, margins=True, dropna=False)
- expected = pd.DataFrame([[1, 0, 1], [1, 3, 4], [2, 4, 6]])
- expected.index = Index([1.0, 2.0, 'All'], name='a')
+ expected = pd.DataFrame([[1, 0, 1], [1, 3, 4], [0, 1, 1], [2, 4, 6]])
+ expected.index = Index([1.0, 2.0, np.nan, 'All'], name='a')
  expected.columns = Index([3, 4, 'All'], name='b')
  tm.assert_frame_equal(actual, expected)
 
  df = DataFrame({'a': [1, np.nan, np.nan, np.nan, 2, np.nan],
  'b': [3, np.nan, 4, 4, 4, 4]})
  actual = pd.crosstab(df.a, df.b, margins=True, dropna=False)
- expected = pd.DataFrame([[1, 0, 1], [0, 1, 1], [1, 4, 6]])
- expected.index = Index([1.0, 2.0, 'All'], name='a')
- expected.columns = Index([3.0, 4.0, 'All'], name='b')
+ expected = pd.DataFrame([[1, 0, 0, 1], [0, 1, 0, 1], [0, 3, 1, 4],
+ [1, 4, 0, 6]])
+ expected.index = Index([1.0, 2.0, np.nan, 'All'], name='a')
+ expected.columns = Index([3.0, 4.0, np.nan, 'All'], name='b')
  tm.assert_frame_equal(actual, expected)
 
  a = np.array(['foo', 'foo', 'foo', 'bar',
@@ -1202,21 +1203,25 @@ def test_margin_dropna(self):
 
  actual = pd.crosstab(a, [b, c], rownames=['a'],
  colnames=['b', 'c'], margins=True, dropna=False)
- m = MultiIndex.from_arrays([['one', 'one', 'two', 'two', 'All'],
- ['dull', 'shiny', 'dull', 'shiny', '']],
+ m = MultiIndex.from_arrays([[np.nan, np.nan, 'one', 'one', 'two',
+ 'two', 'All'],
+ ['dull', 'shiny', 'dull', 'shiny', 'dull',
+ 'shiny', '']],
  names=['b', 'c'])
- expected = DataFrame([[1, 0, 1, 0, 2], [2, 0, 1, 1, 5],
- [3, 0, 2, 1, 7]], columns=m)
+ expected = DataFrame([[0, 0, 1, 0, 1, 0, 2], [0, 1, 2, 0, 1, 1, 5],
+ [0, 1, 3, 0, 2, 1, 7]], columns=m)
  expected.index = Index(['bar', 'foo', 'All'], name='a')
  tm.assert_frame_equal(actual, expected)
 
  actual = pd.crosstab([a, b], c, rownames=['a', 'b'],
  colnames=['c'], margins=True, dropna=False)
- m = MultiIndex.from_arrays([['bar', 'bar', 'foo', 'foo', 'All'],
- ['one', 'two', 'one', 'two', '']],
+ m = MultiIndex.from_arrays([['bar', 'bar', 'bar', 'foo', 'foo',
+ 'foo', 'All'],
+ [np.nan, 'one', 'two', np.nan, 'one',
+ 'two', '']],
  names=['a', 'b'])
- expected = DataFrame([[1, 0, 1], [1, 0, 1], [2, 0, 2], [1, 1, 2],
- [5, 2, 7]], index=m)
+ expected = DataFrame([[0, 0, 0], [1, 0, 1], [1, 0, 1], [0, 1, 1],
+ [2, 0, 2], [1, 1, 2], [5, 2, 7]], index=m)
  expected.columns = Index(['dull', 'shiny', 'All'], name='c')
  tm.assert_frame_equal(actual, expected)