1
\$\begingroup\$

I am trying to write one Name Entity Recognition in Hindi.

I have primarily used NLTK of Python.

I have used HMM Module with its supervised training.

The data is annotated and saved in .pos files as required by the training module. I have included one snippet of the labelled data here.

I have also taken confusion matrix and calculated F Measure.

I am trying to classify using single sentence and files.

The result looks satisfactory. I am trying to post the code below below for your kind review and comment. Please suggest how may I write a better code.

import nltk
from nltk.corpus.reader import TaggedCorpusReader
import dill
from collections import Counter
import random
import itertools
@COULD HANDLE FILE,SENTENCE GIVEN AND FROM CONSOLE

    def NE_TAGGER():
        reader = TaggedCorpusReader('/python27/POS_FILES/POS_7/', r'.*\.pos')
        f1=reader.fileids()
        print "The Files of Corpus are:",f1
        sents=reader.tagged_sents()
        sentn=reader.sents()
        tsent1=sents[9]
        ftsentn1=" ".join(list(itertools.chain(*tsent1)))
        print "Original tagged version of the sentence is;",tsent1
        print "And its Flattened one is:",ftsentn1
        sent1=sentn[9]
        fsent1=" ".join(list(itertools.chain(*sent1)))
        print "The Sentence for tagging is:",sent1
        print "And its flattened version is:",fsent1
        ls=len(sents)
        print "Length of Corpus Is:",ls
        size1 = int(ls * 0.1)
        test_sents=sents[:size1]
        train_sents=sents[size1:]
        hmm_tagger=nltk.HiddenMarkovModelTagger.train(train_sents)
        test=hmm_tagger.test(test_sents)
        tag=hmm_tagger.tag(sent1)
        tag_sent1=" ".join(list(itertools.chain(*tag)))
        print "The Tag Is:",tag
        print "And its flattened version is:",tag_sent1
        #THE GIVEN INPUT
        given_sent="बिहार में नीतीश कुमार के नेतृत्व वाली नवगठित सरकार शुक्रवार को विधानसभा में बहुमत साबित कर लिया है".decode('utf-8')
        gsw=given_sent.split()
        tag_gs=hmm_tagger.tag(gsw)
        print "GIVEN SENT TAG:",tag_gs
        ftag_gs=" ".join(list(itertools.chain(*tag_gs)))
        print "And its flattened Version is:",ftag_gs
        ftag_inp=" ".join(list(itertools.chain(*tag_sentinp)))
        print "And its flattened Version is:",ftag_inp
        #INPUT FROM FILE
        print "WE WOULD DETECT NER FROM FILE NOW"
        file_input=open("/python27/File1","r")
        try:
            for line in file_input:
                print "The Sentence Is:",line
                linew=line.decode('utf-8').split()
                tag_linef=hmm_tagger.tag(linew)
                #print "The Tagged Value of the line Is:",tag_linef
                ftag_linef=" ".join(list(itertools.chain(*tag_linef)))
                print "And its flattened Version is:",ftag_linef
        except IndexError:
            print "OOPS"

        with open('HMMTrainPOS115.dill', 'wb') as f:
            dill.dump(hmm_tagger, f)
        with open('HMMTrainPOS115.dill', 'rb') as f:
            hmm_tagger1 = dill.load(f)

        test_tags = [tag for sent in reader.sents()
        for (word, tag) in hmm_tagger1.tag(sent)]
        gold_tags = [tag for (word, tag) in reader.tagged_words()]
        #print "+++",test_tags
        #print "%%%%",gold_tags
        ltesttag=len(test_tags)
        lgtags=len(gold_tags)
        print "Test Tag Len:",ltesttag
        print "Gold Tag Len:",lgtags
        cm=nltk.ConfusionMatrix(gold_tags, test_tags)
        print(cm.pretty_format(sort_by_count=True, show_percents=False, truncate=5))
        labels = set('NA GPE PERS DATE  ORG'.split())#THE TAG SETS AS GENERATED IN CONFUSION MATRIX
        true_positives = Counter()
        false_negatives = Counter()
        false_positives = Counter()
        for i in labels:
            for j in labels:
                if i == j:
                    true_positives[i] += cm[i,j]
                else:
                    false_negatives[i] += cm[i,j]
                    false_positives[j] += cm[i,j]
        print "TP:", sum(true_positives.values()), true_positives
        print "FN:", sum(false_negatives.values()), false_negatives
        print "FP:", sum(false_positives.values()), false_positives
        print 

        for i in sorted(labels):
            if true_positives[i] == 0:
                fscore = 0
            else:
                precision = true_positives[i] / float(true_positives[i]+false_positives[i])
                recall = true_positives[i] / float(true_positives[i]+false_negatives[i])
                fscore = 2 * (precision * recall) / float(precision + recall)
                fscore1=fscore*100
                print "TAG:",i,"FMEASURE:", fscore1

Thanking in advance.

\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.