I have this code which looks for query words in a given text. I preprocess the give text and store it in an inverted-index (dictionary with word as key and position in text as value). Searching for a query words then becomes contant time operation.
The below code also prints the surrounding text which contains the maximum number of queries.
Please help me understand the shortcomings (coding style, naming conventions, performance issues) of the code.
# -*- coding: cp1252 -*-
from collections import defaultdict
class c_Highlight():
"""Highlight Class to find and print relevant snippet of a text"""
Grammar_Suffix = ['ing' ,'ly', 'ed', 'ious', 'ies', 'ive', 'es', 's', 'ment']
Punctuation = [',', '!', '.', '?', '"', '\n']
def __init__(self):
self.InvertedIndex = defaultdict( list )
def m_Read_File( self, f_name ):
"""
Function to Read a file, if the text is to be read from file
Args: FileName
Returns: List of words
"""
list_words = []
try:
f = open(f_name)
for line in f:
line = line.strip()
list_words.extend( line.split())
f.close()
except IOError as (errno, strerror):
print "File I/O Error({0}): {1}".format(errno, strerror)
exit(0)
return list_words
def __m_Stem( self, word ):
"""
Function to remove Suffix from the end of a word.
Input: Word
Returns: Cropped Word
"""
word = word.lower()
for suffix in c_Highlight.Grammar_Suffix:
if word.endswith(suffix):
word = word[:-len(suffix)]
break
for punctuation in c_Highlight.Punctuation:
while word.endswith(punctuation):
word = word[:-len(punctuation)]
return word #can be empty list
def m_Create_InvertedIndex( self, words ):
"""
Function to parse the input words and store them in a InvertedIndex
The Key is the word itself and Value is the position of the word in the text
Input: List of Words
Return: None
"""
idx = 0
for word in words:
word = self.__m_Stem(word)
self.InvertedIndex[word].append(idx)
idx = idx+1
def m_Search_Query( self, search_query, length ):
"""
Function to search for query words in the InvertedIndex
Input: List of query words, and the number of words in the text
Return: Integer List of length same as the number of words. Each index indicates
if the word is present in the query or not. So if List[i]==0, the word is not present
and if List[i]==2, then the 2nd word in the query is present at location i in the input text.
"""
words_present = []
idx = 1
for x in range(length):
words_present.append(0)
for word in search_query:
word = self.__m_Stem(word)
if word in self.InvertedIndex:
for word_index in self.InvertedIndex[word]:
words_present[word_index] = idx
idx = idx + 1
return words_present
def m_Find_Snippet( self, words_present, num_words, MaxWindow ):
"""
Function to find a snippet of input text with has the most number of query
words present.
Input: Integer List, length, and number of words to be present in the snippet
Return: begin, end position of the window and the count of query words present
in the snippet
"""
begin = 0
count = 0
end = 0
max_count = -1
Snippet_End = 0
Snippet_Begin = 0
num_words = len(words_present)
try:
while begin < num_words:
if words_present[begin]!=0:
count = 0
end = begin
end_done = min((MaxWindow + begin), num_words)
while end < end_done:
if words_present[end]!=0:
count+=1
end+=1
if end == num_words:
end-=1
if count > max_count:
max_count = count
Snippet_Begin = begin
Snippet_End = end
begin+=1
while Snippet_End > 0:
if words_present[Snippet_End]!=0:
break
Snippet_End-=1
except IndexError:
print "Tying to access out of bound index in method m_Find_Snippet"
return Snippet_Begin, Snippet_End, max_count
def m_Sentence_End( self, word ):
for suffix in ['.', '!', '?']:
if word.endswith(suffix):
return 1
return 0
def m_Print_Snippet( self, Snippet_Begin, Snippet_End, raw_data, words_present ):
"""
Function to generate the output text.
Input: begin and end position of snippet
Returns: Beautifies the snippet and returns the new start and end position of the snippet
"""
start = Snippet_Begin
new_start = Snippet_Begin
end = Snippet_End
num_sentence = 0
flag = 0
num_words = len(raw_data)
try:
start_done = min( 20, Snippet_Begin )
while (Snippet_Begin - start) <= start_done:
if self.m_Sentence_End( raw_data[start] ):
num_sentence += 1
new_start = start+1
if num_sentence == 2:
flag = 1
break
start -= 1
if flag == 1:
Snippet_Begin = start + 1
elif flag == 0:
if start_done == Snippet_Begin:
Snippet_Begin = start + 1
else:
Snippet_Begin = new_start
num_sentence = 0
new_end = Snippet_End
flag = 0
end_done = min( 20, num_words - Snippet_End )
while (end - Snippet_End) < end_done:
if self.m_Sentence_End( raw_data[end] ):
num_sentence+=1
new_end = end
if num_sentence == 2:
flag = 1
break
end += 1
if flag == 1:
Snippet_End = end
elif flag == 0:
if end_done == (num_words - Snippet_End):
Snippet_End = num_words - 1
else:
Snippet_End = new_end
except IndexError:
print "Tying to access out of bound index in method m_Print_Snippet"
output_text = []
query_start = 0
try:
for x in range( Snippet_Begin, Snippet_End+1 ):
if words_present[x]!=0:
if query_start == 0:
output_text.append('[[HIGHLIGHT]]'+raw_data[x])
query_start = 1
else:
output_text.append(raw_data[x])
else:
if query_start == 1:
w = output_text.pop()
output_text.append(w+'[[ENDHIGHLIGHT]]')
output_text.append(raw_data[x])
query_start = 0
else:
output_text.append(raw_data[x])
if words_present[x]!=0:
w = output_text.pop()
output_text.append(w+'[[ENDHIGHLIGHT]]')
except IndexError:
print "Trying to access out of bound index in method m_Print_Snippet"
return ' '.join(output_text)
def m_Highlight_doc( doc, query ):
"""
Args:
doc-string that is a document to be c_Highlighted
query-string that contains the search query
Return:
Text Snippet
"""
WINDOW_LENGTH = 35
text = doc.split()
query = query.split()
num_words = len(text)
Search = c_Highlight()
Search.m_Create_InvertedIndex(text)
words_present = Search.m_Search_Query(query, num_words)
begin, end, count = Search.m_Find_Snippet( words_present, num_words, WINDOW_LENGTH )
if count <= 0:
return "Query Not Found"
else:
return Search.m_Print_Snippet( begin, end, text, words_present )
