COMPAT: extension dtypes (DatetimeTZ, Categorical) are now Singleton cached objects

allows for proper is / == comparisons closes pandas-dev#13285

Author: jreback

doc/source/whatsnew/v0.18.2.txt

@@ -246,6 +246,7 @@ Bug Fixes
 
 
 - Bug in ``pd.to_datetime()`` when passing invalid datatypes (e.g. bool); will now respect the ``errors`` keyword (:issue:`13176`)
+- Bug in extension dtype creation where the created types were not is/identical (:issue:`13285`)
 
 - Bug in ``NaT`` - ``Period`` raises ``AttributeError`` (:issue:`13071`)
 - Bug in ``Period`` addition raises ``TypeError`` if ``Period`` is on right hand side (:issue:`13069`)

pandas/tests/types/test_dtypes.py

@@ -45,6 +45,16 @@ class TestCategoricalDtype(Base, tm.TestCase):
  def setUp(self):
  self.dtype = CategoricalDtype()
 
+ def test_hash_vs_equality(self):
+ # make sure that we satisfy is semantics
+ dtype = self.dtype
+ dtype2 = CategoricalDtype()
+ self.assertTrue(dtype == dtype2)
+ self.assertTrue(dtype2 == dtype)
+ self.assertTrue(dtype is dtype2)
+ self.assertTrue(dtype2 is dtype)
+ self.assertTrue(hash(dtype) == hash(dtype2))
+
  def test_equality(self):
  self.assertTrue(is_dtype_equal(self.dtype, 'category'))
  self.assertTrue(is_dtype_equal(self.dtype, CategoricalDtype()))
@@ -88,6 +98,20 @@ class TestDatetimeTZDtype(Base, tm.TestCase):
  def setUp(self):
  self.dtype = DatetimeTZDtype('ns', 'US/Eastern')
 
+ def test_hash_vs_equality(self):
+ # make sure that we satisfy is semantics
+ dtype = self.dtype
+ dtype2 = DatetimeTZDtype('ns', 'US/Eastern')
+ dtype3 = DatetimeTZDtype(dtype2)
+ self.assertTrue(dtype == dtype2)
+ self.assertTrue(dtype2 == dtype)
+ self.assertTrue(dtype3 == dtype)
+ self.assertTrue(dtype is dtype2)
+ self.assertTrue(dtype2 is dtype)
+ self.assertTrue(dtype3 is dtype)
+ self.assertTrue(hash(dtype) == hash(dtype2))
+ self.assertTrue(hash(dtype) == hash(dtype3))
+
  def test_construction(self):
  self.assertRaises(ValueError,
  lambda: DatetimeTZDtype('ms', 'US/Eastern'))

pandas/types/dtypes.py

@@ -108,6 +108,16 @@ class CategoricalDtype(ExtensionDtype):
  kind = 'O'
  str = '|O08'
  base = np.dtype('O')
+ _cache = {}
+
+ def __new__(cls):
+
+ try:
+ return cls._cache[cls.name]
+ except KeyError:
+ c = object.__new__(cls)
+ cls._cache[cls.name] = c
+ return c
 
  def __hash__(self):
  # make myself hashable
@@ -155,38 +165,58 @@ class DatetimeTZDtype(ExtensionDtype):
  base = np.dtype('M8[ns]')
  _metadata = ['unit', 'tz']
  _match = re.compile("(datetime64|M8)\[(?P<unit>.+), (?P<tz>.+)\]")
+ _cache = {}
+
+ def __new__(cls, unit=None, tz=None):
+ """ Create a new unit if needed, otherwise return from the cache
 
- def __init__(self, unit, tz=None):
- """
  Parameters
  ----------
  unit : string unit that this represents, currently must be 'ns'
  tz : string tz that this represents
  """
 
  if isinstance(unit, DatetimeTZDtype):
- self.unit, self.tz = unit.unit, unit.tz
- return
+ unit, tz = unit.unit, unit.tz
 
- if tz is None:
+ elif unit is None:
+ # we are called as an empty constructor
+ # generally for pickle compat
+ return object.__new__(cls)
+
+ elif tz is None:
 
  # we were passed a string that we can construct
  try:
- m = self._match.search(unit)
+ m = cls._match.search(unit)
  if m is not None:
- self.unit = m.groupdict()['unit']
- self.tz = m.groupdict()['tz']
- return
+ unit = m.groupdict()['unit']
+ tz = m.groupdict()['tz']
  except:
  raise ValueError("could not construct DatetimeTZDtype")
 
+ elif isinstance(unit, compat.string_types):
+
+ if unit != 'ns':
+ raise ValueError("DatetimeTZDtype only supports ns units")
+
+ unit = unit
+ tz = tz
+
+ if tz is None:
  raise ValueError("DatetimeTZDtype constructor must have a tz "
  "supplied")
 
- if unit != 'ns':
- raise ValueError("DatetimeTZDtype only supports ns units")
- self.unit = unit
- self.tz = tz
+ # set/retrieve from cache
+ key = (unit, str(tz))
+ try:
+ return cls._cache[key]
+ except KeyError:
+ u = object.__new__(cls)
+ u.unit = unit
+ u.tz = tz
+ cls._cache[key] = u
+ return u
 
  @classmethod
  def construct_from_string(cls, string):