I've done a Python script to find frequent keywords from raw texts and display them along with source sentences and texts. But, having not exercised my Python for an age, I'm sure the code is full of smell. Any pointers would be warmly appreciated.

import os
import nltk
import string
import json
from datetime import datetime
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords

class Word:
    """Stores data relating to individual words."""

    def __init__(self, spelling):
        self.spelling = spelling
        self.sentences = []
        self.frequency = 0
        self.found_in = []

    def add_sentences(self, *sentences):
        [self.sentences.append(s) for s in sentences]

    def to_json(self):
        structure = {self.spelling: {"Sentences": self.sentences, "Documents": self.found_in}}
        return structure

class Hashtagger:
    """Extracts frequent words from text, and the sentences in which they are found."""

    def __init__(self, raw_text):
        self.raw_text = raw_text
        self.words = []
        self.sentences = []
        self.stopword_list = stopwords.words('english') # words to not process
        self.lemmatizer = nltk.WordNetLemmatizer() # groups inflections to single word

    def process_text(self):

    def process_words(self, text):
        stripped_text = self.strip_punctuation(text)
        tokens = nltk.Text(word_tokenize(stripped_text))
        content = [self.lemmatizer.lemmatize(t.lower()) for t in tokens if t not in self.stopword_list]
        counts = {t: content.count(t) for t in content}

        for token, count in counts.items():
            word = Word(token)
            word.frequency = count

    def deconstruct_sentences(self, text):
        raw_sentences = nltk.sent_tokenize(text)
        self.sentences = [s.strip(string.punctuation) + '.' for s in raw_sentences]

    def allocate_sentences_to(self, word):
        sentences_containing = [s for s in self.sentences if word.spelling in s.lower()]

    def allocate_sentences_to_all(self):
        [self.allocate_sentences_to(key) for key in self.words]

    def strip_punctuation(self, text):
        for ch in string.punctuation:
            text = text.replace(ch, ' ')
        return text

class ResultsAccumulator:
    """Accumulates the results of many hashtaggers."""

    def __init__(self):
        self.word_tally = []

    def add_tally(self, hashtagger, file=None):
        for h_word in hashtagger.words:
            matched = False
            for word in self.word_tally:
                if word.spelling == h_word.spelling:
                    word.frequency += h_word.frequency
                    matched = True
            if not matched:

    def write_word_json(self):
        total_json = {}
        for word in self.word_tally:
        return total_json

def main():
    path = r'test_docs/'

    results = ResultsAccumulator()

    for file in os.listdir(path):
        with open(os.path.join(path, file), encoding='utf8') as d:
            raw_text = d.read()

        hashtagger = Hashtagger(raw_text)
        results.add_tally(hashtagger, file)

    final_tally = results.write_word_json()

    datestring = datetime.strftime(datetime.now(), '%Y-%m-%d-%H-%M-%S')
    with open('word_json_' + datestring + '.txt', 'w') as f:
        json.dump(final_tally, f, indent=4,

if __name__ == '__main__':

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.