6
\$\begingroup\$

Description:

I am working on a classifier which categorizes the text based on some criteria, at present, it is a map of category and list of words if any of the words appear in the text, a category is assigned to it. The text can have multiple categories. The number of categories can be more than 100.

Problem:

There is a need to make this design as flexible as possible so that new classifier can be just plugged in.

Code:

from collections import defaultdict
import string
import re

def createIndex(my_dict):
  index = defaultdict(list);
  for category, words in my_dict.items():
    for word in words:
      index[word.lower()].append(category)
  return index


class Document(object):
  def __init__(self, category, text):
    self.category = category
    self._text = text.strip().translate(str.maketrans('', '', string.punctuation))
    self._paragraphs = []

  def parse(self):
    paragraphs = re.split('\n\n', self._text)
    for para in paragraphs:
      self._paragraphs.append(Paragraph(para))

  def paragraphs(self):
    return self._paragraphs

class Paragraph(object):
  def __init__(self, text):
    self.number = None
    self._text = text

  def words(self):
    return self._text.split()

  def text(self):
    return self._text

  def __str__(self):
    return self._text

class ClassifiedText(object):
  def __init__(self, text, categories):
    self.text = text
    self.categories = categories

  def __str__(self):
    return self.text + '->' + str(self.categories)

  def __repr__(self):
    return self.text + '->' + str(self.categories)

class UncassifiedText(object):
  def __init__(self, text):
    self.text = text
    self.weight = 0
    self.words  = []
    self.category = None

  def __str__(self):
    return self.text + '[Unclassified]'

  def __repr__(self):
    return self.text + '[Unclassified]'

class WeightedClassifiedText(object):
  def __init__(self, text, weight, category, words):
    self.text = text
    self.weight = weight
    self.words = words
    self.category = category

  def __str__(self):
    return self.text + '[' + str(self.weight) + ']'

  def __repr__(self):
    return self.text + '[' + str(self.weight) + ']' 

class CategoryClassifier(object):
  def classify(self, text):
      raise NotImplementedError('subclasses must override classifier()!')

class TimeClassifier(CategoryClassifier):
  def __init__(self):
    self.index = createIndex(wordsByCategory)
    self._key = 'time'
    self._label = 'Time'

  def classify(self, paragraph):
    count = 0
    matched_words = set()
    for word in paragraph.words():
      if 'time' in self.index[word.lower()]:
        matched_words.add(word)
        count += 1 
    if count > 0:
      return WeightedClassifiedText(
        paragraph.text(), count, 'Time', list(matched_words))
    else:
      return UncassifiedText(paragraph.text())

class MoodClassifier(CategoryClassifier):
  def __init__(self):
    self.index = createIndex(wordsByCategory)
    self._key = 'mood'
    self._label = 'Mood'

  def classify(self, paragraph):
    count = 0
    matched_words = set()
    for word in paragraph.words():
      if 'mood' in self.index[word.lower()]:
        matched_words.add(word)
        count += 1 
    if count > 0:
      return WeightedClassifiedText(
        paragraph.text(), count, 'Mood', list(matched_words))
    else:
      return UncassifiedText(paragraph.text())   

class DayClassifier(CategoryClassifier):
  def __init__(self):
    self.index = createIndex(wordsByCategory)
    self._key = 'day'
    self._label = 'Day'

  def classify(self, paragraph):
    count = 0
    matched_words = set()
    for word in paragraph.words():
      if self._key in self.index[word.lower()]:
        matched_words.add(word)
        count += 1 
    if count > 0:
      return WeightedClassifiedText(
        paragraph.text(), count, self._label, list(matched_words))
    else:
      return UncassifiedText(paragraph.text())  

class LocationClassifier(CategoryClassifier):
  def __init__(self):
    self.index = createIndex(wordsByCategory)
    self._key = 'location'
    self._label = 'Location'

  def classify(self, paragraph):
    count = 0
    matched_words = set()
    for word in paragraph.words():
      if 'location' in self.index[word.lower()]:
        matched_words.add(word)
        count += 1 
    if count > 0:
      return WeightedClassifiedText(
        paragraph.text(), count, 'Location', list(matched_words))
    else:
      return UncassifiedText(paragraph.text())  

wordsByCategory = {
  'time': ['monday', 'noon', 'morning'],
  'location': ['Here', 'Near', 'City', 'London', 'desk', 'office', 'home'],
  'mood': ['Happy', 'Excited', 'smiling', 'smiled', 'sick'],
  'day': ['sunday', 'monday', 'Friday']
}

raw_text = """
Friday brings joy and happiness. We get together
and have fun in our London office.

Monday is normally boring and makes me sick.
Everyone is busy in meetings and no fun.

She looked and smiled at me again, I am thinking
to have coffee with her.
"""

document = Document('Contract', raw_text)
document.parse()
#print(document.paragraphs[0])

#print(ManualClassifier(','.join(texts[0])).classify())

classfiers = [
  TimeClassifier(),
  MoodClassifier(),
  DayClassifier(),
  LocationClassifier()
]

for text in document.paragraphs():
  result = list(map(lambda x: x.classify(text), classfiers))
  categories = list(filter(
    lambda x: x is not None, list(map(lambda x: x.category, result))))
  print(text, categories)

It works as expected and to add a new classifier I just have to create a class and add it to the list of classifier and it works. But I feel lack of object-oriented design and I am new to Python as well so I am not sure if I am doing it the "Python" way.

Misc:

In the future I need to introduce the ML-based classifiers as well and then for a given text I need to decide on the category decided by the ML vs Manual classification. For the same reason, I have added weight in the ClassifiedText.

\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.