Python Program Generating N-Gram Language Model

Question

I am pretty new to Python. I am writing this program to randomly generate sentences based on the n-gram language. It takes me very long to run this with the large input file I have, so it is very hard for me to check my work. I guess my problem is that, when I need two words as the history and based on the count of words appearing after the two words, I generate the next word. And it takes very long and hard for me to do that for some reason.

Any suggestion would be really helpful.

from __future__ import unicode_literals
from io import open
import string
import re
import operator
import math
import random
corpus = ''
begin_token = '<s>'

# read in file and preprocessing the text
def ReadFile():
    sentences_all = []
    translate_table = dict((ord(char), None) for char in string.punctuation)
    # open file and preprocessing   
    with open('moviereview.txt', 'r', encoding ='iso-8859-15') as file:
        for line in file:
            line = line.translate(translate_table) # remove punctuations
            line = '<s> ' + line.rstrip('\n') + '</s>' # add <s> and </s>
            sentences_all += line 
            sentences_all.append(" ") # append each line with space
        corpus = ''.join(sentences_all)
        corpus = re.sub(' +', ' ', corpus) # replace multiple spaces with single space
    return corpus


# a dictionary of sentences in corpus and their count 
def N_Gram(corpus, n):
    corpus = ''.join(corpus)
    corpus = corpus.split(' ')
    output = {}
    for i in range(len(corpus)-n+1):
        g = ' '.join(corpus[i:i+n])
        output.setdefault(g,0)
        output[g] += 1
    return output


def Uni_Generation():
    corpus = ReadFile()
    uni = N_Gram(corpus, 1)
    print(uni)
    final = unsmoothed_totalcount(uni)
    print(final)
    sentence_list = [] # the list of 5 sentences

    for b in xrange(0,5):
        sentence = '<s> '
        while sentence.split()[-1] != '</s>': #last word is not </s>
            sentence += return_random_selected_item(final, uni)
            if len(sentence.split()) >= 15 : # if the length of sentence is more than 15
                sentence += '</s>'
            #print(sentence)
        sentence = post_processing(sentence)
        sentence_list.append(sentence)
    #print(sentence_list)
     

def post_processing(x):
    x = x.split(" ")
    x.pop(0)
    x[0] = x[0].capitalize() # capitalize the first word
    if(x[-1] == ''):
        x.pop(-1)
        x.pop(-1)
    elif(x[-1] == '</s>'):
        x.pop(-1)
    x = " ".join(x)
    x = '<s> ' + x + ' . </s>'
    return x

def unsmoothed_totalcount(n_gram_dict):
    #get total count of words
    keyList = list(n_gram_dict.keys())
    final = 0
    for i, x in enumerate(keyList):
        if i-1 < 0:
            continue
        prev_word = keyList[i-1]
        prev_count = n_gram_dict[prev_word] # get the previous word count
        final += prev_count
    return final

def find_next_word(N_Gram_dic, N_m1_Gram_dic, histo):
    #find all bigram start with histo, use the count and do everything
    corpus = ReadFile()
    n_count_dic = {}
    uni_dic= N_Gram(corpus, 1)
    final = 0
    if histo.count(' ') == 0: # there is no history for it
        #print("get here")
        n_count_dic = uni_dic
    # elif histo == '<s> <s>' or histo == '<s> <s> <s>' or histo == '<s> <s> <s> <s>' or histo == '</s> <s>':
    #   n_count_dic = uni_dic
    #   return find_next_word(uni_dic, uni_dic, '<s>')
    else:
        # keyList = N_Gram_dic.keys()
        # #print(keyList)
        # leng = histo.count(' ') + 1
        # #print("histo is ")
        # #print(histo)
        # for a in keyList:
        #   word = ' '.join(a.split()[:leng])
        #   #print("word is " + word)
        #   if(histo == word):
            
        if histo in N_Gram_dic:
            n_count_dic[histo] = N_Gram_dic[histo]
            print("histo")
            print(n_count_dic[histo])
    #print(n_count_dic)
    # for x in N_m1_Gram_dic:
    #   if histo == x:
    #       final = N_m1_Gram_dic[x]
    final = unsmoothed_totalcount(N_m1_Gram_dic)
    print("i did my best")
    print(final)
    return return_random_selected_item(final, n_count_dic)

def return_random_selected_item(total_count, n_count_dict):
    #print("hello from the other side")
    #print(total_count)

    r = random.randint(1,total_count)
    for x in n_count_dict:
        f1 = n_count_dict[x]
        if r - f1 <= 0 : # if the word choosen is not ending token
            print(x)
            return x.split()[-1] + ' '
        if r > f1:
            r = r - f1

#print(Uni_Generation())

def N_Gram_Generation(n):
    corpus = ReadFile()
    if n == 1:
        return Uni_Generation()

    uni_gram = N_Gram(corpus, 1)
    #bi_gram = N_Gram(corpus, 2)
    n_Gram = N_Gram(corpus, n)
    N_m1_Gram = N_Gram(corpus, n-1)
    #final = unsmoothed_totalcount(N_m1_Gram)
    sentence_list = [] # the list of 5 sentences

    for b in xrange(0,5):
        sentence = '<s> '
        while len(sentence.split()) <= n:
                word = find_next_word(uni_gram, uni_gram, sentence.split()[-1])
                #print("word" + word)
                sentence += word
                #print("sentence" + sentence )
        while sentence.split()[-1] != '</s>':
            list = sentence.split()
            his= list[-(n-1):]
            histor = ' '.join(his)
            print("histor ")
            print(histor)
            next_word = find_next_word(n_Gram, N_m1_Gram, histor)
            print(next_word)
            if next_word != '</s>':
                sentence += next_word
            if next_word == '</s>':
                break
            if len(sentence.split()) >= 15 : # if the length of sentence is more than 15
                sentence += '</s>'
        sentence = post_processing(sentence)
        print(sentence)
        sentence_list.append(sentence)
    return sentence_list

print(N_Gram_Generation(3))

The readfile and postprocessing part is all good so I will not put that here.

Answer 1

Overview

The code layout is good, and you used meaningful names for functions and variables.

Intent

Having these separate if statements implies that they are independent of each other:

        if next_word != '</s>':
            sentence += next_word
        if next_word == '</s>':
            break

However, since they are related, the code is simpler in this case to use an if/else, which better conveys the intent:

        if next_word == '</s>':
            break
        else:
            sentence += next_word

Comments

Remove all commented-out code because it adds clutter. If you were testing out some of this code, you can store revisions using a version control system (such as Git, etc.). Then you can retrieve old versions as needed.

Lint check

pylint identified several issues.

The unused imports (operator, math) can be removed.

This unused variable can be removed:

    begin_token = '<s>'

The indentation inside this while loop is inconsistent with indentation elsewhere:

    while len(sentence.split()) <= n:
            word = find_next_word(uni_gram, uni_gram, sentence.split()[-1])
            sentence += word

Instead of indenting by 8 spaces, 4 will be consistent:

    while len(sentence.split()) <= n:
        word = find_next_word(uni_gram, uni_gram, sentence.split()[-1])
        sentence += word

There is no need for parentheses in this if/else:

if(x[-1] == ''):
    ...
elif(x[-1] == '</s>'):

Documentation

It would be good to have documentation at the top of the file to state the purpose of the code, what its expected input is and what type of output to expect.

For example, the code requires the moviereview.txt input file. There should be some description of the file contents.

Print

Many of the functions have print statements. If these are just used while debugging, then they should either be removed or you could control them with a variable.

Python 2.x

xrange is not a built-in function for Python 3.x. Consider replacing it with range since 2.x is deprecated.

Typo

choosen should be chosen.

punctuations should simply be punctuation.