Adding parser for mailman ground truth files (html)

Author: rth

jwzthreading/jwzthreading.py

@@ -112,6 +112,14 @@ def has_descendant(self, ctr):
 
  return False
 
+ @property
+ def size(self):
+ """Count the number of objects included in the container,
+ including itself"""
+
+ return 1 + sum([child.size for child in self.children])
+
+
 
 class Message(object):
  """Represents a message to be threaded.

jwzthreading/tests/test_newsgroups.py

@@ -6,12 +6,46 @@
 from __future__ import unicode_literals
 
 import os
+from unittest import SkipTest
+
 from jwzthreading import Message, thread, print_container
-from jwzthreading.utils import parse_mailman_gzfiles
+from jwzthreading.utils import (parse_mailman_gzfiles,
+ parse_mailman_htmlthread)
 
 BASE_DIR = os.path.dirname(__file__)
 DATA_DIR = os.path.join(BASE_DIR, 'data/fedora-devel-mailman') 
 
+# Expected number of emails and threads
+N_JUNE2010_THREADS = 292
+N_JUNE2010_EMAILS = 292
+
+
+def test_parse_mailman_gzfiles():
+ """ Test that we can parse mailman files """
+ msglist = parse_mailman_gzfiles(os.path.join(DATA_DIR, '2010-January.txt.gz'),
+ encoding='latin1', headersonly=True)
+
+ assert len(msglist) == N_JUNE2010_EMAILS
+
+def test_parse_mailman_htmlthread():
+ """ Test that we can parse mailman html thread """
+ try:
+ import lxml
+ except ImportError:
+ raise SkipTest
+ threads = parse_mailman_htmlthread(os.path.join(DATA_DIR,
+ '2010-January_thread.html'))
+
+ print(sum([el.size for el in threads]))
+
+ print('OK')
+ #for el in threads:
+ # print_container(el)
+
+
+
+
+
 def test_fedora():
  """ Test threading on the fedora-devel mailing list data"""
  # 2010-January https://www.redhat.com/archives/fedora-devel-list/

jwzthreading/utils.py

@@ -53,3 +53,63 @@ def parse_mailman_gzfiles(filename, encoding='utf-8', headersonly=False):
  out.append(msg_obj)
  return out
 
+
+def parse_mailman_htmlthread(filename):
+ """ Parse a gzipped files with multiple concatenaged emails
+ that can be downloaded from mailman.
+
+ Parameters
+ ----------
+ filename : str
+ path to the filename
+
+ Returns
+ -------
+
+ response : list
+ a thread list
+ """
+ from lxml import etree
+ from .jwzthreading import Container
+ parser = etree.HTMLParser()
+ with open(filename, 'rt') as fh:
+ tree = etree.parse(fh, parser)
+
+ elements = filter(lambda x: x.tag == 'ul', tree.find('body'))
+
+ tree = list(elements)[-1].getchildren() # pick last <ul> element
+
+ class DummyMessage(object):
+ subject = None
+ id = None
+
+ def create_thread(root, parent_container=None):
+ """ Parse the html nested lists to produce the threading structure"""
+ #print(dir(root))
+ if root.tag != 'li':
+ raise ValueError('Element {} was not expected'.format(root))
+
+ container = Container()
+ for child in root.getchildren():
+ if child.tag == 'strong':
+ # url with to the actual email
+ a_el = child.getchildren()[0]
+ container.message = DummyMessage()
+ container.message.subject = a_el.text
+ container.message.id = int(a_el.get('name'))
+ elif child.tag == 'em':
+ pass # email sender, ignore this line
+ elif child.tag == 'ul':
+ for nested_child in child.getchildren():
+ create_thread(nested_child, parent_container=container)
+ else:
+ raise ValueError('Unexpected element {}'.format(child))
+ if parent_container is not None:
+ parent_container.add_child(container)
+
+ return container
+
+ threads = [create_thread(el) for el in tree]
+
+ return threads
+