# Python Program Generating N-Gram Language Model

I am pretty new to python, and I am writing this program to randomly generate sentences based on the n-gram language. It takes me very long to run this with the large input file I have, so it is very hard for me to check my work. I guess my problem is that, when I need 2 words as the history and based on the count of words appear after the 2 words, I generate the next word. And it takes very long and hard for me to do that for some reason.

Any suggestion would be really helpful.

from __future__ import unicode_literals
from io import open
import string
import re
import operator
import math
import random
corpus = ''
begin_token = '<s>'

# read in file and preprocessing the text
sentences_all = []
translate_table = dict((ord(char), None) for char in string.punctuation)
# open file and preprocessing
with open('moviereview.txt', 'r', encoding ='iso-8859-15') as file:
for line in file:
line = line.translate(translate_table) # remove punctuations
line = '<s> ' + line.rstrip('\n') + '</s>' # add <s> and </s>
sentences_all += line
sentences_all.append(" ") # append each line with space
corpus = ''.join(sentences_all)
corpus = re.sub(' +', ' ', corpus) # replace multiple spaces with single space
return corpus

# a dictionary of sentences in corpus and their count
def N_Gram(corpus, n):
corpus = ''.join(corpus)
corpus = corpus.split(' ')
output = {}
for i in range(len(corpus)-n+1):
g = ' '.join(corpus[i:i+n])
output.setdefault(g,0)
output[g] += 1
return output

def Uni_Generation():
uni = N_Gram(corpus, 1)
print(uni)
final = unsmoothed_totalcount(uni)
print(final)
sentence_list = [] # the list of 5 sentences

for b in xrange(0,5):
sentence = '<s> '
while sentence.split()[-1] != '</s>': #last word is not </s>
sentence += return_random_selected_item(final, uni)
if len(sentence.split()) >= 15 : # if the length of sentence is more than 15
sentence += '</s>'
#print(sentence)
sentence = post_processing(sentence)
sentence_list.append(sentence)
#print(sentence_list)

def post_processing(x):
x = x.split(" ")
x.pop(0)
x[0] = x[0].capitalize() # capitalize the first word
if(x[-1] == ''):
x.pop(-1)
x.pop(-1)
elif(x[-1] == '</s>'):
x.pop(-1)
x = " ".join(x)
x = '<s> ' + x + ' . </s>'
return x

def unsmoothed_totalcount(n_gram_dict):
#get total count of words
keyList = list(n_gram_dict.keys())
final = 0
for i, x in enumerate(keyList):
if i-1 < 0:
continue
prev_word = keyList[i-1]
prev_count = n_gram_dict[prev_word] # get the previous word count
final += prev_count
return final

def find_next_word(N_Gram_dic, N_m1_Gram_dic, histo):
#find all bigram start with histo, use the count and do everything
n_count_dic = {}
uni_dic= N_Gram(corpus, 1)
final = 0
if histo.count(' ') == 0: # there is no history for it
#print("get here")
n_count_dic = uni_dic
# elif histo == '<s> <s>' or histo == '<s> <s> <s>' or histo == '<s> <s> <s> <s>' or histo == '</s> <s>':
#   n_count_dic = uni_dic
#   return find_next_word(uni_dic, uni_dic, '<s>')
else:
# keyList = N_Gram_dic.keys()
# #print(keyList)
# leng = histo.count(' ') + 1
# #print("histo is ")
# #print(histo)
# for a in keyList:
#   word = ' '.join(a.split()[:leng])
#   #print("word is " + word)
#   if(histo == word):

if histo in N_Gram_dic:
n_count_dic[histo] = N_Gram_dic[histo]
print("histo")
print(n_count_dic[histo])
#print(n_count_dic)
# for x in N_m1_Gram_dic:
#   if histo == x:
#       final = N_m1_Gram_dic[x]
final = unsmoothed_totalcount(N_m1_Gram_dic)
print("i did my best")
print(final)
return return_random_selected_item(final, n_count_dic)

def return_random_selected_item(total_count, n_count_dict):
#print("hello from the other side")
#print(total_count)

r = random.randint(1,total_count)
for x in n_count_dict:
f1 = n_count_dict[x]
if r - f1 <= 0 : # if the word choosen is not ending token
print(x)
return x.split()[-1] + ' '
if r > f1:
r = r - f1

#print(Uni_Generation())

def N_Gram_Generation(n):
if n == 1:
return Uni_Generation()

uni_gram = N_Gram(corpus, 1)
#bi_gram = N_Gram(corpus, 2)
n_Gram = N_Gram(corpus, n)
N_m1_Gram = N_Gram(corpus, n-1)
#final = unsmoothed_totalcount(N_m1_Gram)
sentence_list = [] # the list of 5 sentences

for b in xrange(0,5):
sentence = '<s> '
while len(sentence.split()) <= n:
word = find_next_word(uni_gram, uni_gram, sentence.split()[-1])
#print("word" + word)
sentence += word
#print("sentence" + sentence )
while sentence.split()[-1] != '</s>':
list = sentence.split()
his= list[-(n-1):]
histor = ' '.join(his)
print("histor ")
print(histor)
next_word = find_next_word(n_Gram, N_m1_Gram, histor)
print(next_word)
if next_word != '</s>':
sentence += next_word
if next_word == '</s>':
break
if len(sentence.split()) >= 15 : # if the length of sentence is more than 15
sentence += '</s>'
sentence = post_processing(sentence)
print(sentence)
sentence_list.append(sentence)
return sentence_list

print(N_Gram_Generation(3))


Sorry if I was not clear or anything. I am really lack of sleep right now for this assignment. The readfile and postprocessing part is all good so I will not put that here. Thanks!

• Please fix your indentation. The easiest way to post code is to paste it into the question editor, highlight it, and press Ctrl-K to mark it as a code block. – 200_success Feb 1 at 5:10
• @Ludisposed It just read file in and preprocess the corpus. I add it in if that helps. – Yuhe Zhu Feb 1 at 15:17
• Thank you, CR works best, when all relevant code is in the question. – Ludisposed Feb 1 at 15:25