I've been teaching myself Python - my first programming language - for about two years now.
I recently discovered the concurrent.futures
module and wanted to do something with it. What do you think about this script?
import re
import shutil
import string
from collections import Counter
from concurrent.futures import ProcessPoolExecutor
from itertools import chain, islice, zip_longest
from urllib.request import urlopen
# Regex to use for splitting text into words, dropping everything but
# alphabetic characters.
REGEX = re.compile(r"[{}{}{}]+".format(
string.whitespace, string.digits, string.punctuation))
# http://docs.python.org/3/library/itertools.html#itertools-recipes
def grouper(iterable, n, fillvalue=None):
"Collect data into fixed-length chunks or blocks"
# grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx"
args = [iter(iterable)] * n
return zip_longest(*args, fillvalue=fillvalue)
def split_and_clean(line):
"""Returns an iterator of the words in line.
Example:
>>> list(split_and_clean("3, Four, five'"))
['four', 'five']
Args:
line: A string.
Returns:
A filter of the alphabetic words in the line.
Raises:
TypeError: Your input was of type <>. Must be a string.
"""
try:
return filter(None, re.split(REGEX, line.lower()))
except AttributeError:
input_type_str = str(type(line))[8:-2]
error_message = "Your input was of type {}. Must be a string.".format(
input_type_str)
raise TypeError(error_message)
def wc_some_lines(lines):
"""Return a Counter containing the word count of several lines.
Excludes any digital numbers or punctuation.
Example:
>>> wc_some_lines(["Line 1.", "Another line."])
Counter({'line': 2, 'another': 1})
Args:
lines: An iterable of strings.
Returns:
A collections.Counter mapping words to their word counts.
Raises:
TypeError
"""
return Counter(chain(*map(split_and_clean, lines)))
def main(num_processes=5):
"""Fetch Shakespeare's complete works and perform a concurrent word count.
"""
url = "http://www.gutenberg.org/files/100/100.txt"
filename = "shakespeare_complete_works.txt"
try:
text = open(filename)
except IOError:
with open(filename, "wb") as f:
r = urlopen(url)
shutil.copyfileobj(r, f) # write r to f
text = open(filename)
# exclude the small print at the beginning and the end
relevant_part = islice(text, 181, 124379)
# group text into blocks of 5000 lines
blocks = grouper(relevant_part, 5000, "")
# perform actual word count in concurrent processes
result = Counter()
with ProcessPoolExecutor(num_processes) as pool:
for part in pool.map(wc_some_lines, blocks):
result.update(part)
# output results
for word, count in result.most_common(20):
print("{}: {}".format(word, count))
if __name__ == "__main__":
main()