Merge pull request #6481 from jreback/di_perf

PERF: perf improvements in DataFrame construction with a non-daily datelike index (GH6479)

Author: jreback

doc/source/release.rst

@@ -127,6 +127,8 @@ Improvements to existing features
 - ``Series.rank()`` now has a percentage rank option (:issue:`5971`)
 - ``quotechar``, ``doublequote``, and ``escapechar`` can now be specified when
  using ``DataFrame.to_csv`` (:issue:`5414`, :issue:`4528`)
+- perf improvements in DataFrame construction with certain offsets, by removing faulty caching
+ (e.g. MonthEnd,BusinessMonthEnd), (:issue:`6479`)
 
 .. _release.bug_fixes-0.14.0:
 

doc/source/v0.14.0.txt

@@ -237,6 +237,9 @@ Enhancements
 Performance
 ~~~~~~~~~~~
 
+- perf improvements in DataFrame construction with certain offsets, by removing faulty caching
+ (e.g. MonthEnd,BusinessMonthEnd), (:issue:`6479`)
+
 Experimental
 ~~~~~~~~~~~~
 

pandas/tseries/offsets.py

@@ -291,7 +291,7 @@ def _from_name(cls, suffix=None):
  return cls()
 
 
-class BusinessDay(CacheableOffset, SingleConstructorOffset):
+class BusinessDay(SingleConstructorOffset):
  """
  DateOffset subclass representing possibly n business days
  """
@@ -399,7 +399,7 @@ def apply(self, other):
  n -= 5 * k
  if n == 0 and result.weekday() > 4:
  n -= 1
- 
+
  while n != 0:
  k = n // abs(n)
  result = result + timedelta(k)
@@ -548,7 +548,7 @@ def name(self):
  return "%s-%s" % (self.rule_code, _int_to_month[self.n])
 
 
-class MonthEnd(CacheableOffset, MonthOffset):
+class MonthEnd(MonthOffset):
  """DateOffset of one month end"""
 
  def apply(self, other):
@@ -572,7 +572,7 @@ def onOffset(cls, dt):
  _prefix = 'M'
 
 
-class MonthBegin(CacheableOffset, MonthOffset):
+class MonthBegin(MonthOffset):
  """DateOffset of one month at beginning"""
 
  def apply(self, other):
@@ -591,7 +591,7 @@ def onOffset(cls, dt):
  _prefix = 'MS'
 
 
-class BusinessMonthEnd(CacheableOffset, MonthOffset):
+class BusinessMonthEnd(MonthOffset):
  """DateOffset increments between business EOM dates"""
 
  def isAnchored(self):
@@ -619,7 +619,7 @@ def apply(self, other):
  _prefix = 'BM'
 
 
-class BusinessMonthBegin(CacheableOffset, MonthOffset):
+class BusinessMonthBegin(MonthOffset):
  """DateOffset of one business month at beginning"""
 
  def apply(self, other):
@@ -654,7 +654,7 @@ def onOffset(cls, dt):
  _prefix = 'BMS'
 
 
-class Week(CacheableOffset, DateOffset):
+class Week(DateOffset):
  """
  Weekly offset
 
@@ -744,7 +744,7 @@ class WeekDay(object):
 _weekday_to_int = dict((v, k) for k, v in _int_to_weekday.items())
 
 
-class WeekOfMonth(CacheableOffset, DateOffset):
+class WeekOfMonth(DateOffset):
  """
  Describes monthly dates like "the Tuesday of the 2nd week of each month"
 
@@ -830,7 +830,7 @@ def _from_name(cls, suffix=None):
  weekday = _weekday_to_int[suffix[1:]]
  return cls(week=week, weekday=weekday)
 
-class LastWeekOfMonth(CacheableOffset, DateOffset):
+class LastWeekOfMonth(DateOffset):
  """
  Describes monthly dates in last week of month like "the last Tuesday of each month"
 
@@ -940,7 +940,7 @@ def rule_code(self):
  return '%s-%s' % (self._prefix, _int_to_month[self.startingMonth])
 
 
-class BQuarterEnd(CacheableOffset, QuarterOffset):
+class BQuarterEnd(QuarterOffset):
  """DateOffset increments between business Quarter dates
  startingMonth = 1 corresponds to dates like 1/31/2007, 4/30/2007, ...
  startingMonth = 2 corresponds to dates like 2/28/2007, 5/31/2007, ...
@@ -999,7 +999,7 @@ def onOffset(self, dt):
 
 
 # TODO: This is basically the same as BQuarterEnd
-class BQuarterBegin(CacheableOffset, QuarterOffset):
+class BQuarterBegin(QuarterOffset):
  _outputName = "BusinessQuarterBegin"
  # I suspect this is wrong for *all* of them.
  _default_startingMonth = 3
@@ -1036,7 +1036,7 @@ def apply(self, other):
  return as_timestamp(result)
 
 
-class QuarterEnd(CacheableOffset, QuarterOffset):
+class QuarterEnd(QuarterOffset):
  """DateOffset increments between business Quarter dates
  startingMonth = 1 corresponds to dates like 1/31/2007, 4/30/2007, ...
  startingMonth = 2 corresponds to dates like 2/28/2007, 5/31/2007, ...
@@ -1077,7 +1077,7 @@ def onOffset(self, dt):
  return MonthEnd().onOffset(dt) and modMonth == 0
 
 
-class QuarterBegin(CacheableOffset, QuarterOffset):
+class QuarterBegin(QuarterOffset):
  _outputName = 'QuarterBegin'
  _default_startingMonth = 3
  _from_name_startingMonth = 1
@@ -1129,7 +1129,7 @@ def rule_code(self):
  return '%s-%s' % (self._prefix, _int_to_month[self.month])
 
 
-class BYearEnd(CacheableOffset, YearOffset):
+class BYearEnd(YearOffset):
  """DateOffset increments between business EOM dates"""
  _outputName = 'BusinessYearEnd'
  _default_month = 12
@@ -1166,7 +1166,7 @@ def apply(self, other):
  return result
 
 
-class BYearBegin(CacheableOffset, YearOffset):
+class BYearBegin(YearOffset):
  """DateOffset increments between business year begin dates"""
  _outputName = 'BusinessYearBegin'
  _default_month = 1
@@ -1198,7 +1198,7 @@ def apply(self, other):
  return as_timestamp(datetime(other.year, self.month, first))
 
 
-class YearEnd(CacheableOffset, YearOffset):
+class YearEnd(YearOffset):
  """DateOffset increments between calendar year ends"""
  _default_month = 12
  _prefix = 'A'
@@ -1254,7 +1254,7 @@ def onOffset(self, dt):
  return self.month == dt.month and dt.day == days_in_month
 
 
-class YearBegin(CacheableOffset, YearOffset):
+class YearBegin(YearOffset):
  """DateOffset increments between calendar year begin dates"""
  _default_month = 1
  _prefix = 'AS'
@@ -1300,7 +1300,7 @@ def onOffset(self, dt):
  return dt.month == self.month and dt.day == 1
 
 
-class FY5253(CacheableOffset, DateOffset):
+class FY5253(DateOffset):
  """
  Describes 52-53 week fiscal year. This is also known as a 4-4-5 calendar.
 
@@ -1501,7 +1501,7 @@ def _from_name(cls, *args):
  return cls(**cls._parse_suffix(*args))
 
 
-class FY5253Quarter(CacheableOffset, DateOffset):
+class FY5253Quarter(DateOffset):
  """
  DateOffset increments between business quarter dates
  for 52-53 week fiscal year (also known as a 4-4-5 calendar).
@@ -1772,7 +1772,7 @@ def _delta_to_nanoseconds(delta):
  + delta.microseconds) * 1000
 
 
-class Day(CacheableOffset, Tick):
+class Day(Tick):
  _inc = timedelta(1)
  _prefix = 'D'
 

pandas/tseries/tests/test_offsets.py

@@ -316,7 +316,7 @@ def test_apply_large_n(self):
  rs = st + off
  xp = datetime(2011, 12, 26)
  self.assertEqual(rs, xp)
- 
+
  off = BDay() * 10
  rs = datetime(2014, 1, 5) + off # see #5890
  xp = datetime(2014, 1, 17)
@@ -2427,25 +2427,9 @@ def get_all_subclasses(cls):
  return ret
 
 class TestCaching(tm.TestCase):
- no_simple_ctr = [WeekOfMonth, FY5253,
- FY5253Quarter,
- LastWeekOfMonth]
-
- def test_should_cache_month_end(self):
- self.assertTrue(MonthEnd()._should_cache())
-
- def test_should_cache_bmonth_end(self):
- self.assertTrue(BusinessMonthEnd()._should_cache())
-
- def test_should_cache_week_month(self):
- self.assertTrue(WeekOfMonth(weekday=1, week=2)._should_cache())
 
- def test_all_cacheableoffsets(self):
- for subclass in get_all_subclasses(CacheableOffset):
- if subclass.__name__[0] == "_" \
- or subclass in TestCaching.no_simple_ctr:
- continue
- self.run_X_index_creation(subclass)
+ # as of GH 6479 (in 0.14.0), offset caching is turned off
+ # as of v0.12.0 only BusinessMonth/Quarter were actually caching
 
  def setUp(self):
  _daterange_cache.clear()
@@ -2462,20 +2446,35 @@ def run_X_index_creation(self, cls):
  DatetimeIndex(start=datetime(2013,1,31), end=datetime(2013,3,31), freq=inst1, normalize=True)
  self.assertTrue(cls() in _daterange_cache, cls)
 
+ def test_should_cache_month_end(self):
+ self.assertFalse(MonthEnd()._should_cache())
+
+ def test_should_cache_bmonth_end(self):
+ self.assertFalse(BusinessMonthEnd()._should_cache())
+
+ def test_should_cache_week_month(self):
+ self.assertFalse(WeekOfMonth(weekday=1, week=2)._should_cache())
+
+ def test_all_cacheableoffsets(self):
+ for subclass in get_all_subclasses(CacheableOffset):
+ if subclass.__name__[0] == "_" \
+ or subclass in TestCaching.no_simple_ctr:
+ continue
+ self.run_X_index_creation(subclass)
+
  def test_month_end_index_creation(self):
  DatetimeIndex(start=datetime(2013,1,31), end=datetime(2013,3,31), freq=MonthEnd(), normalize=True)
- self.assertTrue(MonthEnd() in _daterange_cache)
+ self.assertFalse(MonthEnd() in _daterange_cache)
 
  def test_bmonth_end_index_creation(self):
  DatetimeIndex(start=datetime(2013,1,31), end=datetime(2013,3,29), freq=BusinessMonthEnd(), normalize=True)
- self.assertTrue(BusinessMonthEnd() in _daterange_cache)
+ self.assertFalse(BusinessMonthEnd() in _daterange_cache)
 
  def test_week_of_month_index_creation(self):
  inst1 = WeekOfMonth(weekday=1, week=2)
  DatetimeIndex(start=datetime(2013,1,31), end=datetime(2013,3,29), freq=inst1, normalize=True)
  inst2 = WeekOfMonth(weekday=1, week=2)
- self.assertTrue(inst2 in _daterange_cache)
-
+ self.assertFalse(inst2 in _daterange_cache)
 
 class TestReprNames(tm.TestCase):
  def test_str_for_named_is_name(self):

vb_suite/frame_ctor.py

@@ -1,7 +1,15 @@
 from vbench.benchmark import Benchmark
 from datetime import datetime
+try:
+ import pandas.tseries.offsets as offsets
+except:
+ import pandas.core.datetools as offsets
 
 common_setup = """from pandas_vb_common import *
+try:
+ from pandas.tseries.offsets import *
+except:
+ from pandas.core.datetools import *
 """
 
 #----------------------------------------------------------------------
@@ -36,6 +44,21 @@
 """
 frame_ctor_nested_dict_int64 = Benchmark("DataFrame(data)", setup)
 
+# dynamically generate benchmarks for every offset
+dynamic_benchmarks = {}
+n_steps = [1, 2]
+for offset in offsets.__all__:
+ for n in n_steps:
+ setup = common_setup + """
+df = DataFrame(np.random.randn(1000,10),index=date_range('1/1/1900',periods=1000,freq={}({})))
+d = dict([ (col,df[col]) for col in df.columns ])
+""".format(offset, n)
+ key = 'frame_ctor_dtindex_{}({})'.format(offset, n)
+ dynamic_benchmarks[key] = Benchmark("DataFrame(d)", setup, name=key)
+
+# Have to stuff them in globals() so vbench detects them
+globals().update(dynamic_benchmarks)
+
 # from a mi-series
 setup = common_setup + """
 mi = MultiIndex.from_tuples([(x,y) for x in range(100) for y in range(100)])