Add ability to collapse empty root container

Author: rth

jwzthreading/jwzthreading.py

@@ -157,26 +157,73 @@ def root(self):
  else:
  return self.parent.root
 
+ def collapse_empty(self, inplace=True):
+ """ Collapse empty top level containers.
 
- def to_dict(self, include_subject=False):
+ If multiple messages reference a non existing top level message,
+ by default JWZ threading algorithm will create a en empty top level
+ container to be used as the root node.
+
+ This method removes this empty container and makes the first child
+ to be the root message. The other messages at depth == 1 then become 
+ it's children.
+
+ Parameters
+ ----------
+
+ inplace : bool, default=True
+ if True the original container is modified
+ """
+
+ if not inplace:
+ raise NotImplementedError
+
+
+ if self.message is not None:
+ # nothing to be done
+ return self
+
+ if any([el.message is None for el in self.children]):
+ raise ValueError('Children containers cannot be empty!')
+
+ # In the following, self.message is None
+
+ # make the 1st children the new root container
+ children = self.children
+
+ new_root = children[0]
+ new_root.parent = None
+
+ for idx in range(1, len(children)):
+ child = children[idx]
+ child.parent = new_root
+ new_root.children.append(child)
+
+ return new_root
+
+
+ def to_dict(self, include=[]):
  """ Convert a Container tree to a nested dict """
  if self.message is None:
  raise ValueError('Containers with None messages are not supported!')
+ raise ValueError('Containers with None messages are not supported:!\n'\
+ ' this: {}'.format(self))
 
  res = {'id': self.message.message_idx}
 
- if include_subject:
- res['subject'] = self.message.subject
+ for key in include:
+ res[key] = getattr(self.message, key)
 
  if self.parent is not None:
  if self.parent.message is not None:
  res['parent'] = self.parent.message.message_idx
  else:
- raise ValueError('Containers with None messages are not supported!')
+ raise ValueError('Containers with None messages are not supported:!\n'\
+ ' this: {}\n parent: {}'.format(self, self.parent))
  else:
  res['parent'] = None
 
- res['children'] = [el.to_dict() for el in self.children]
+ res['children'] = [el.to_dict(include=include) for el in self.children]
 
  return res
 

jwzthreading/tests/test_jwz.py

@@ -274,3 +274,11 @@ def test_thread_two_missing_parent():
  assert d[0].message == None
  assert len(d[0].children) == 2
  assert d[0].children[0].message == m1
+ assert d[0].size == 3
+
+ # check that collapsing the empty container works
+ container = d[0].collapse_empty()
+ assert container.size == 2
+ assert container.message is not None
+ assert container.message.message_id == 'First'
+ assert container.parent is None

jwzthreading/tests/test_newsgroups.py

@@ -58,7 +58,7 @@ def test_mailbox_delimiter():
  assert re.match(MAILBOX_DELIMITER, line)
 
 
-def test_fedora_June2010():
+def test_threading_fedora_June2010():
  """ Test threading on the fedora-devel mailing list data
  from June 2010"""
 
@@ -154,3 +154,43 @@ def test_fedora_June2010():
 
 
 
+def test_empty_collapsing_fedora_June2010():
+ """ Test threading on the fedora-devel mailing list data
+ from June 2010"""
+
+ try:
+ import lxml
+ except ImportError:
+ raise SkipTest
+
+ try:
+ import numpy as np
+ from numpy.testing import assert_array_equal
+ NUMPY_PRESENT = True
+ except ImportError:
+ NUMPY_PRESENT = False
+
+
+
+ msglist = parse_mailbox(os.path.join(DATA_DIR, '2010-January.txt.gz'),
+ encoding='latin1', headersonly=True)
+
+ assert len(msglist) == N_EMAILS_JUNE2010
+
+
+ threads_ref = parse_mailman_htmlthread(os.path.join(DATA_DIR,
+ '2010-January_thread.html.gz'))
+ threads_ref = sort_threads(threads_ref, key='subject', missing='Z')
+
+
+ threads = thread([Message(el, message_idx=idx) for idx, el in enumerate(msglist)],
+ group_by_subject=False)
+ # There is one single "empty root container"
+ assert sum([el.message is None for el in threads]) == 1
+
+ threads = [el.collapse_empty() for el in threads]
+
+ # The empty container was removed
+ assert sum([el.message is None for el in threads]) == 0
+
+ assert sum([el.parent is None for el in threads]) == len(threads)