3
\$\begingroup\$

I wrote a script to compute the polarity of sentences. The script is based on this:

The lexicon based approach assigns a sentiment tags to words in a text based on dictionaries of positive and negative words. A sentiment score is then calculated for each document as:

$$\frac{\text{number of positive words} - \text{number of negative words}}{\text{total number of words}}$$

Is the code below okay? Should I have to improve it to take into consideration the position of the word in the sentence?

fileDir = os.path.dirname(os.path.realpath('__file__'))

lexiconpos = open(os.path.join(fileDir, 'Ressources/positive.txt'), 'r', encoding='utf-8')
lexiconpos = set([line.rstrip() for line in lexiconpos])
lexiconneg = open(os.path.join(fileDir, 'Ressources/negative.txt'),'r', encoding='utf-8')
lexiconneg = set([line.rstrip() for line in lexiconneg])

path = os.path.join(fileDir, 'data/reference/freins.tsv')
with open(path, 'r', encoding='utf-8') as l_1:
    next(l_1)
    l_1 = [line.rstrip() for line in l_1]

tagger = treetaggerwrapper.TreeTagger(TAGLANG='fr')     
def process_text(text):
    '''extract lemma and lowerize then removing stopwords.'''

    text_preprocess =[]
    text_without_stopwords= []

    text = tagger.tag_text(text)
    for word in text:
        parts = word.split('\t')
        try:
            if parts[2] == '':
                text_preprocess.append(parts[0])
            else:
                text_preprocess.append(parts[2])
        except:
            print(parts)
            
            
    text_without_stopwords= [word.lower() for word in text_preprocess if word.isalnum()] #if word not in stopWords
    #return text_without_stopwords
    return text_preprocess
 

def extract_info(texte, lexique1, lexique2):

    lemme_treetagger = []
    lemme_sent = []
    len_sent = []
    for elt in texte:
        texte_treetagger = process_text(elt.lower())
        lemme_treetagger.append(" ".join(texte_treetagger))
        len_sent.append(len(" ".join(texte_treetagger)))
        
    word_pos=[]
    word_neg=[]
    word_pos_len=[]
    word_neg_len=[]
    for elt in lemme_treetagger:
        word_pos.append([word for word in lexique1 if word in elt])
        word_neg.append([word for word in lexique2 if word in elt])
    
    for lst in word_pos:
        word_pos_len.append(len(lst))
    for lst in word_neg:
        word_neg_len.append(len(lst))
    
    
    print(len_sent)
    return texte, lemme_treetagger, len_sent, word_pos, word_neg, word_pos_len, word_neg_len

texte, lemme_treetagger, len_sent, word_pos, word_neg, word_pos_len, word_neg_len = extract_info(l_1, lexiconpos, lexiconneg)



def polarity_word(texte, *args):
    
    texte, lemme_treetagger, len_sent, word_pos, word_neg, word_pos_len, word_neg_len = extract_info(texte, lexiconpos, lexiconneg)
    
    for elt in lemme_treetagger:
        print(elt)
        sub_list = list(map(operator.sub, word_pos_len, word_neg_len))
        score = list(map(truediv, sub_list, len_sent))
    
    return score
\$\endgroup\$
0

1 Answer 1

1
\$\begingroup\$

TSV parsing

You should not need to do this yourself. The csv module accepts a delimiter that you can set to '\t'.

Context management

You use with in one out of three file opens - the other two would benefit.

Inner list

This:

set([line.rstrip() for line in lexiconpos])

can be

{line.rstrip() for line in lexiconpos}

The set literal and removal of the inner list will both express this better.

\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.