# Parse a plain text newsletter and show it with nicer markdown formatting

I wrote some code to parse an e-mailed events newsletter I'm on and redisplay it with better formatting. I like the output, but I'm not thrilled with the code. It feels like there's a lot of boilerplate, and the code may be overly abstracted for the simple task it performs, so I'm seeking advice here.

# -*- coding: utf-8 -*-
"""

- Add anchors all over the place so that the links have somewhere to point
- Throw out "Cover art" section
- Add formatting, e.g., by making the title of each section a Markdown heading
- Add horizontal rules between sections
- Show the location and time info at the bottom of each event as code (start
each line with four spaces) to keep the newlines in the right place and to
make it visually distinct

"""
from __future__ import absolute_import, division, print_function, unicode_literals

import re
import abc
import codecs
from collections import OrderedDict, defaultdict
from markdown import markdown # http://packages.python.org/Markdown/

INITIAL = "Initial"
SEP_LINES = ["", "-------------------------------", ""]
TOP_ANCHOR = "TOP"

class Event(list):
"""Represents a single event from the e-mail. Processes input lines and produces output lines"""
def __init__(self, title=None, location=None):
self.lines = []
self.title = title
self.location = location

if self.title is None:
if line:
self.title = line
else:
self.lines.append(line)

def get_body_blocks(self):
"""Parses the input lines for this event, grouping them into blocks of code that are not separated by blank
lines."""
done_blocks = []
current_block = []
for line in self.lines:
if line:
# If there's a line that matches the title in the first block,
# throw it away
if done_blocks or line != self.title:
current_block.append(line)
else:
if current_block:
done_blocks.append(current_block)
current_block = []
if current_block:
done_blocks.append(current_block)

return done_blocks

def get_body_lines(self, section_name):
result = ['<a name="{}"></a>'.format(self.title),
"### {}, {} ###".format(self.title, self.location),
"**{}**".format(section_name), ""]

blocks = self.get_body_blocks()
if not blocks:
return []

last_block = blocks.pop()
for block in blocks:
result += block + [""]
result += ["    " + line for line in last_block]
return result

def get_toc_line(self):
return "    * [{0}](#{0}), {1}".format(self.title, self.location)

class Section(object):
"""We parse the newsletter into sections. This subclasses of this abstract class know how to parse the input
sections and produce appropriate output. The abstract base class exists to host docstrings"""
__metaclass__ = abc.ABCMeta

def __init__(self, name):
self.name = name
self.lines = []

self.lines.append(line)

@abc.abstractmethod
def get_top_body_lines(self):
"""These are the header lines for the sectionl given the sections' title and anchor"""
pass

@abc.abstractmethod
def get_bottom_body_lines(self):
pass

@abc.abstractmethod
def get_other_body_lines(self):
"""The meat of what we want to output for this section"""
pass

class BlankSection(Section):
"""A section that we're not going to include in the output, so very simple"""
def __init__(self, name):
super(BlankSection, self).__init__(name)

def get_top_body_lines(self):
return []

def get_bottom_body_lines(self):
return []

def get_other_body_lines(self):
return []

def get_toc_lines(self):
return []

class PrintedSection(BlankSection):
"""Base class for sections of the newsletter that we want to include in the output"""
def __init__(self, name):
super(PrintedSection, self).__init__(name)
self.toc_anchor = "_TOC_{0}".format(name)

def get_top_body_lines(self):
return ['<a name="{}"></a>'.format(self.name),
"## {} ##".format(self.name)]

def get_bottom_body_lines(self):

def get_other_body_lines(self):
return self.lines

def get_toc_lines(self):
return ['* <a name="{0}"></a>[{1}](#{1})'.format(self.toc_anchor, self.name)]

class DatedSection(PrintedSection):
"""The sections of the newsletter that report events, each on a given date"""
def __init__(self, date, eventinfo_from_toc):
self.eventinfo_from_toc = iter(list(eventinfo_from_toc))
self.events = []
self.new_event()
super(DatedSection, self).__init__(date)

m = re.match(r"\*\*\*\*\* Also on (.*) \*\*\*\*\*", line)
if m:
self.new_event()
else:

def new_event(self):
ei = self.eventinfo_from_toc.next()
title, location = re.match("(.*), ([^,]+)", ei).groups()
self.events.append(Event(title, location))

def current_event(self):
return self.events[-1]

def get_bottom_body_lines(self):
return []

def get_other_body_lines(self):
result = []
for event in self.events:
result += event.get_body_lines(self.name)
return result

def get_toc_lines(self):
return super(DatedSection, self).get_toc_lines() +\
[event.get_toc_line() for event in self.events]

class TableOfContents(BlankSection):
"""Represents the table of contents in the input data. We pretty much throw it out and build our own, so this is a
BlankSection. However, we do keep track of the text used to list each event, which will come in handy later"""
def __init__(self):
super(TableOfContents, self).__init__(TABLE_OF_CONTENTS)
self._in_date = None # As we parse line by line, remember if the are currently in a block with a date at the top
self.entries = defaultdict(list)

assert self._in_date
self.entries[self._in_date.upper()].append(item)

if "" == line:
self._in_date = None
elif line.startswith("* "):
else:
self._in_date = line

class Document(object):
"""This class represents the entire newsletter, handling parsing of the input and then subsequent output"""
def __init__(self):
self.sections = OrderedDict()
self.new_initial_section()

def current_section(self):
return self.sections.values()[-1]

def _record_section(self, section):
self.sections[section.name] = section

def new_blank_section(self, name):
self._record_section(BlankSection(name))

def new_printed_section(self, name):
self._record_section(PrintedSection(name))

def new_initial_section(self):
self.new_blank_section(INITIAL)

def new_table_of_contents(self):
self.table_of_contents = TableOfContents()
self._record_section(self.table_of_contents)

def new_dated_section(self, date):
self._record_section(DatedSection(date, self.table_of_contents.entries[date]))

def new_section(self, name):
if "COVER ART" == name:
self.new_blank_section(name)
else:
self.new_printed_section(name)

def parse(self, infile):
for line in [lv.strip().decode("utf-8") for lv in infile.readlines()]:
if line.startswith("Date:"):
self.new_table_of_contents()
continue
m = re.match("XXXXX (.*) XXXXX", line)
if m:
section_name = m.group(1)
if "," in section_name:
self.new_dated_section(section_name)
else:
self.new_section(section_name)
continue

def get_toc_lines(self):
for section in self.sections.values():
result += section.get_toc_lines()
return result

def get_output_page(self):
body_lines = []
for count, section in enumerate(self.sections.values()):
section_body_lines = section.get_top_body_lines() +\
section.get_other_body_lines() +\
section.get_bottom_body_lines()
body_lines += section_body_lines
if count > 0 and section_body_lines:
body_lines += SEP_LINES
markup = "\n".join(self.get_toc_lines() + SEP_LINES + body_lines)
return markdown(markup)

def main():
document = Document()
with open("nonsense.txt") as infile:
document.parse(infile)
html = document.get_output_page()
# http://stackoverflow.com/a/934203/2829764
with codecs.open("nonsense.html", "w", "utf-8-sig") as outfile:
outfile.write(html)

if "__main__" == __name__:
main()

• This would be easier to review if you added some description of the transformations it performs. – Janne Karila Sep 14 '15 at 5:49
• Lines should be wrapped to 79 characters; comments and docstrings to 72. – alexwlchan Sep 14 '15 at 5:54
• Your Event implementation seems odd - you subclass a list, then give it an attribute that's also a list? It doesn't behave at all like a list - you can't iterate over it, index into it or get its length, for example. – jonrsharpe Sep 14 '15 at 10:09
• @JanneKarila, I believe I have fixed the issue by adding more detail to the comment on the top of the script. Please let me know if this wasn't what you meant – kuzzooroo Sep 14 '15 at 12:05
• @jonrsharpe, you are right about subclassing list. I was doing so early in the development of the program and just forgot to remove it when I took the code in a different section – kuzzooroo Sep 14 '15 at 12:07