Updating newsgroups tests

Author: rth

jwzthreading/jwzthreading.py

@@ -162,7 +162,6 @@ def __init__(self, msg=None):
  def __repr__(self):
  return '<%s: %r>' % (self.__class__.__name__, self.message_id)
 
-
 #
 # functions
 #

jwzthreading/tests/test_newsgroups.py

@@ -16,16 +16,17 @@
 DATA_DIR = os.path.join(BASE_DIR, 'data/fedora-devel-mailman') 
 
 # Expected number of emails and threads
-N_JUNE2010_THREADS = 292
-N_JUNE2010_EMAILS = 292
+N_EMAILS_JUNE2010 = 292
+N_THREADS_JUNE2010 = 63
+
 
 
 def test_parse_mailman_gzfiles():
  """ Test that we can parse mailman files """
  msglist = parse_mailman_gzfiles(os.path.join(DATA_DIR, '2010-January.txt.gz'),
  encoding='latin1', headersonly=True)
 
- assert len(msglist) == N_JUNE2010_EMAILS
+ assert len(msglist) == N_EMAILS_JUNE2010
 
 def test_parse_mailman_htmlthread():
  """ Test that we can parse mailman html thread """
@@ -36,9 +37,12 @@ def test_parse_mailman_htmlthread():
  threads = parse_mailman_htmlthread(os.path.join(DATA_DIR,
  '2010-January_thread.html'))
 
- assert sum([el.size for el in threads]) == N_JUNE2010_EMAILS
-
+ #assert len(threads) == N_THREADS_JUNE2010
  print('OK')
+ #for el in threads:
+ # print(' - ({})'.format(el.size), el.message.subject)
+ assert sum([el.size for el in threads]) == N_EMAILS_JUNE2010
+
  #for el in threads:
  # print_container(el)
 
@@ -47,24 +51,36 @@ def test_parse_mailman_htmlthread():
 
 
 def test_fedora():
- """ Test threading on the fedora-devel mailing list data"""
- # 2010-January https://www.redhat.com/archives/fedora-devel-list/
- #import mailbox
+ """ Test threading on the fedora-devel mailing list data
+ from June 2010"""
+
+ try:
+ import lxml
+ except ImportError:
+ raise SkipTest
 
 
  msglist = parse_mailman_gzfiles(os.path.join(DATA_DIR, '2010-January.txt.gz'),
  encoding='latin1', headersonly=True)
 
- msglist_parsed = map(Message, msglist)
+ assert len(msglist) == N_EMAILS_JUNE2010
 
- subject_table = thread(msglist_parsed)
+
+ threads_ref = parse_mailman_htmlthread(os.path.join(DATA_DIR,
+ '2010-January_thread.html'))
+
+
+ subject_table = thread([Message(el) for el in msglist])
 
  subjects = subject_table.items()
  subjects = sorted(subjects)
- for idx, (_, container) in enumerate(subjects):
+ for _, container in enumerate(subjects):
  #print(idx)
  #print_container(container)
  pass
+ assert sum([el.size for _, el in subjects]) == N_EMAILS_JUNE2010
+
+ #assert len(subjects) == N_THREADS_JUNE2010
 
 
 

jwzthreading/utils.py

@@ -70,7 +70,7 @@ def parse_mailman_htmlthread(filename):
  a thread list
  """
  from lxml import etree
- from .jwzthreading import Container
+ from .jwzthreading import Container, Message
  parser = etree.HTMLParser()
  with open(filename, 'rt') as fh:
  tree = etree.parse(fh, parser)
@@ -89,14 +89,18 @@ def create_thread(root, parent_container=None):
  if root.tag != 'li':
  raise ValueError('Element {} was not expected'.format(root))
 
+ if len(root.getchildren()) == 0:
+ # this is a dummy element "<li>Possible follow-ups</li>"
+ return None
+
  container = Container()
  for child in root.getchildren():
  if child.tag == 'strong':
  # url with to the actual email
  a_el = child.getchildren()[0]
- container.message = DummyMessage()
+ container.message = Message()
  container.message.subject = a_el.text
- container.message.id = int(a_el.get('name'))
+ container.message.message_id = int(a_el.get('name'))
  elif child.tag == 'em':
  pass # email sender, ignore this line
  elif child.tag == 'ul':