0
\$\begingroup\$

I am pretty new to python, and I am writing this program to randomly generate sentences based on the n-gram language. It takes me very long to run this with the large input file I have, so it is very hard for me to check my work. I guess my problem is that, when I need 2 words as the history and based on the count of words appear after the 2 words, I generate the next word. And it takes very long and hard for me to do that for some reason.

Any suggestion would be really helpful.

from __future__ import unicode_literals
from io import open
import string
import re
import operator
import math
import random
corpus = ''
begin_token = '<s>'

# read in file and preprocessing the text
def ReadFile():
    sentences_all = []
    translate_table = dict((ord(char), None) for char in string.punctuation)
    # open file and preprocessing   
    with open('moviereview.txt', 'r', encoding ='iso-8859-15') as file:
        for line in file:
            line = line.translate(translate_table) # remove punctuations
            line = '<s> ' + line.rstrip('\n') + '</s>' # add <s> and </s>
            sentences_all += line 
            sentences_all.append(" ") # append each line with space
        corpus = ''.join(sentences_all)
        corpus = re.sub(' +', ' ', corpus) # replace multiple spaces with single space
    return corpus


# a dictionary of sentences in corpus and their count 
def N_Gram(corpus, n):
    corpus = ''.join(corpus)
    corpus = corpus.split(' ')
    output = {}
    for i in range(len(corpus)-n+1):
        g = ' '.join(corpus[i:i+n])
        output.setdefault(g,0)
        output[g] += 1
    return output


def Uni_Generation():
    corpus = ReadFile()
    uni = N_Gram(corpus, 1)
    print(uni)
    final = unsmoothed_totalcount(uni)
    print(final)
    sentence_list = [] # the list of 5 sentences

    for b in xrange(0,5):
        sentence = '<s> '
        while sentence.split()[-1] != '</s>': #last word is not </s>
            sentence += return_random_selected_item(final, uni)
            if len(sentence.split()) >= 15 : # if the length of sentence is more than 15
                sentence += '</s>'
            #print(sentence)
        sentence = post_processing(sentence)
        sentence_list.append(sentence)
    #print(sentence_list)


def post_processing(x):
    x = x.split(" ")
    x.pop(0)
    x[0] = x[0].capitalize() # capitalize the first word
    if(x[-1] == ''):
        x.pop(-1)
        x.pop(-1)
    elif(x[-1] == '</s>'):
        x.pop(-1)
    x = " ".join(x)
    x = '<s> ' + x + ' . </s>'
    return x

def unsmoothed_totalcount(n_gram_dict):
    #get total count of words
    keyList = list(n_gram_dict.keys())
    final = 0
    for i, x in enumerate(keyList):
        if i-1 < 0:
            continue
        prev_word = keyList[i-1]
        prev_count = n_gram_dict[prev_word] # get the previous word count
        final += prev_count
    return final

def find_next_word(N_Gram_dic, N_m1_Gram_dic, histo):
    #find all bigram start with histo, use the count and do everything
    corpus = ReadFile()
    n_count_dic = {}
    uni_dic= N_Gram(corpus, 1)
    final = 0
    if histo.count(' ') == 0: # there is no history for it
        #print("get here")
        n_count_dic = uni_dic
    # elif histo == '<s> <s>' or histo == '<s> <s> <s>' or histo == '<s> <s> <s> <s>' or histo == '</s> <s>':
    #   n_count_dic = uni_dic
    #   return find_next_word(uni_dic, uni_dic, '<s>')
    else:
        # keyList = N_Gram_dic.keys()
        # #print(keyList)
        # leng = histo.count(' ') + 1
        # #print("histo is ")
        # #print(histo)
        # for a in keyList:
        #   word = ' '.join(a.split()[:leng])
        #   #print("word is " + word)
        #   if(histo == word):

        if histo in N_Gram_dic:
            n_count_dic[histo] = N_Gram_dic[histo]
            print("histo")
            print(n_count_dic[histo])
    #print(n_count_dic)
    # for x in N_m1_Gram_dic:
    #   if histo == x:
    #       final = N_m1_Gram_dic[x]
    final = unsmoothed_totalcount(N_m1_Gram_dic)
    print("i did my best")
    print(final)
    return return_random_selected_item(final, n_count_dic)

def return_random_selected_item(total_count, n_count_dict):
    #print("hello from the other side")
    #print(total_count)

    r = random.randint(1,total_count)
    for x in n_count_dict:
        f1 = n_count_dict[x]
        if r - f1 <= 0 : # if the word choosen is not ending token
            print(x)
            return x.split()[-1] + ' '
        if r > f1:
            r = r - f1

#print(Uni_Generation())

def N_Gram_Generation(n):
    corpus = ReadFile()
    if n == 1:
        return Uni_Generation()

    uni_gram = N_Gram(corpus, 1)
    #bi_gram = N_Gram(corpus, 2)
    n_Gram = N_Gram(corpus, n)
    N_m1_Gram = N_Gram(corpus, n-1)
    #final = unsmoothed_totalcount(N_m1_Gram)
    sentence_list = [] # the list of 5 sentences

    for b in xrange(0,5):
        sentence = '<s> '
        while len(sentence.split()) <= n:
                word = find_next_word(uni_gram, uni_gram, sentence.split()[-1])
                #print("word" + word)
                sentence += word
                #print("sentence" + sentence )
        while sentence.split()[-1] != '</s>':
            list = sentence.split()
            his= list[-(n-1):]
            histor = ' '.join(his)
            print("histor ")
            print(histor)
            next_word = find_next_word(n_Gram, N_m1_Gram, histor)
            print(next_word)
            if next_word != '</s>':
                sentence += next_word
            if next_word == '</s>':
                break
            if len(sentence.split()) >= 15 : # if the length of sentence is more than 15
                sentence += '</s>'
        sentence = post_processing(sentence)
        print(sentence)
        sentence_list.append(sentence)
    return sentence_list

print(N_Gram_Generation(3))

Sorry if I was not clear or anything. I am really lack of sleep right now for this assignment. The readfile and postprocessing part is all good so I will not put that here. Thanks!

\$\endgroup\$
  • 3
    \$\begingroup\$ Please fix your indentation. The easiest way to post code is to paste it into the question editor, highlight it, and press Ctrl-K to mark it as a code block. \$\endgroup\$ – 200_success Feb 1 at 5:10
  • \$\begingroup\$ @Ludisposed It just read file in and preprocess the corpus. I add it in if that helps. \$\endgroup\$ – Yuhe Zhu Feb 1 at 15:17
  • \$\begingroup\$ Thank you, CR works best, when all relevant code is in the question. \$\endgroup\$ – Ludisposed Feb 1 at 15:25

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.