I am studying the techniques of data mining and data processing. I'm doing this through data I've collected and stored in a csv file. The problem is that this filed was very large, to the point of having as astonishing 40 thousand lines of text.
Some of the algorithms in the rendering part are fast and agile, but the part of the orthographic correction of words is laborious. I am using the NLTK package nltk.corpus import forest. So when it comes time to do this step, I daresay it will not end in a timely manner.
From this, I was wondering if someone can help me with a solution where I can read a file line, do the whole process, save it to the bank and then read another line from the file. So by reading line by line and each line do the process. I think this way I can improve the performance of the algorithm.
txtCorpus = []
dtype_dic= {'status_id': str, 'status_message' : str, 'status_published':str}
for csvfile in pd.read_csv('data/MyCSV.csv',dtype=dtype_dic,encoding='utf-8',sep=',', header='infer',engine='c', chunksize=2):
txtCorpus.append(csvfile)
def status_processing(txtCorpus):
myCorpus = preprocessing.PreProcessing()
myCorpus.text = str(txtCorpus)
print "Doing the Initial Process..."
myCorpus.initial_processing()
print "Done."
print "----------------------------"
print ("StartingLexical Diversity...")
myCorpus.lexical_diversity()
print "Done"
print "----------------------------"
print "Removing Stopwords..."
myCorpus.stopwords()
print "Done"
print "----------------------------"
print "Lemmatization..."
myCorpus.lemmatization()
print "Feito"
print "----------------------------"
print "Correcting the words..."
myCorpus.spell_correct()
print "Done"
print "----------------------------"
print "Untokenizing..."
word_final = myCorpus.untokenizing()
print "Feito"
print "----------------------------"
print "Saving in DB...."
try:
db.myDB.insert(word_final, continue_on_error=True)
except pymongo.errors.DuplicateKeyError:
pass
print "Insertion in the BB Completed. End of the Pre-Processing Process "
def main():
status_processing(txtCorpus)
main()
I believe that by visualizing the code, you can better understand what I explained above. I thought about doing a for
where I read a line and passed it on def status_processing(txtCorpus):
and so, I repeated the process until the end. But I could not reach a solution.
preprocessing file:
import nltk,re, htmlentitydefs
from nltk.stem.snowball import SnowballStemmer
from nltk.stem import WordNetLemmatizer
from bs4 import BeautifulSoup
import spellcorrect
class Techniques(object):
Lemmatizing = 1
Stopwords = 2
Stemming = 3
Spellcorrect = 4
def __init__(self, Type):
self.value = Type
def __str__(self):
if self.value == Techniques.Lemmatizing:
return 'Lemmatizing'
if self.value == Techniques.Stopwords:
return 'Stopwords'
if self.value == Techniques.Stemming:
return 'Stemming'
if self.value == Techniques.Spellcorrect:
return 'Spell Correct'
def __eq__(self,y):
return self.value==y.value
class PreProcessing():
@property
def text(self):
return self.__text
@text.setter
def text(self, text):
self.__text = text
tokens = None
def initial_processing(self):
soup = BeautifulSoup(self.text,"html.parser")
self.text = soup.get_text()
#Todo Se quiser salvar os links mudar aqui
self.text = re.sub(r'(http://|https://|www.)[^"\' ]+', " ", self.text)
self.tokens = self.tokenizing(1, self.text)
pass
def lexical_diversity(self):
word_count = len(self.text)
vocab_size = len(set(self.text))
return vocab_size / word_count
def tokenizing(self, type, text):
if (type == 1):
return nltk.tokenize.word_tokenize(text)
elif (type == 2):
stok = nltk.data.load('tokenizers/punkt/portuguese.pickle')
#stok = nltk.PunktSentenceTokenizer(train)
return stok.tokenize(text)
def stopwords(self):
stopwords = nltk.corpus.stopwords.words('portuguese')
stopWords = set(stopwords)
palavroesPortugues = ['foda','caralho', 'porra', 'puta', 'merda', 'cu', 'foder', 'viado', 'cacete']
stopWords.update(palavroesPortugues)
filteredWords = []
for word in self.tokens:
if word not in stopWords:
filteredWords.append(word)
self.tokens = filteredWords
def stemming(self):
snowball = SnowballStemmer('portuguese')
stemmedWords = []
for word in self.tokens:
stemmedWords.append(snowball.stem(word))
self.tokens = stemmedWords
def lemmatization(self):
lemmatizer = WordNetLemmatizer()#'portuguese'
lemmatizedWords = []
for word in self.tokens:
lemmatizedWords.append(lemmatizer.lemmatize(word, pos='v'))
self.tokens = lemmatizedWords
def part_of_speech_tagging(self):
return 'Not implemented yet'
def padronizacaoInternetes(self):
return 'Not implementes yet'
def untokenize(self, words):
"""
Untokenizing a text undoes the tokenizing operation, restoring
punctuation and spaces to the places that people expect them to be.
Ideally, `untokenize(tokenize(text))` should be identical to `text`,
except for line breaks.
"""
text = ' '.join(words)
step1 = text.replace("`` ", '"').replace(" ''", '"').replace('. . .', '...')
step2 = step1.replace(" ( ", " (").replace(" ) ", ") ")
step3 = re.sub(r' ([.,:;?!%]+)([ \'"`])', r"\1\2", step2)
step4 = re.sub(r' ([.,:;?!%]+)$', r"\1", step3)
step5 = step4.replace(" '", "'").replace(" n't", "n't").replace(
"can not", "cannot")
step6 = step5.replace(" ` ", " '")
return step6.strip()
def untokenizing(self):
return ' '.join(self.tokens)
#return self.untokenize(self.tokens)
#return tokenize.untokenize(self.tokens)
def spell_correct(self):
correctedWords = []
spell = spellcorrect.SpellCorrect()
for word in self.tokens:
correctedWords.append(spell.correct(word))
self.tokens = correctedWords
spellcorret file:
import re, collections
from nltk.corpus import floresta
class SpellCorrect:
def words(self, text): return re.findall('[a-z]+', text.lower())
def train(features):
model = collections.defaultdict(lambda: 1)
for f in features:
model[f] += 1
return model
NWORDS = train(floresta.words()) #words(file('big.txt').read())
alphabet = 'abcdefghijklmnopqrstuvwxyz'
def edits1(self, word):
splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]
deletes = [a + b[1:] for a, b in splits if b]
transposes = [a + b[1] + b[0] + b[2:] for a, b in splits if len(b)>1]
replaces = [a + c + b[1:] for a, b in splits for c in self.alphabet if b]
inserts = [a + c + b for a, b in splits for c in self.alphabet]
return set(deletes + transposes + replaces + inserts)
def known_edits2(self, word):
return set(e2 for e1 in self.edits1(word) for e2 in self.edits1(e1) if e2 in self.NWORDS)
def known(self, words): return set(w for w in words if w in self.NWORDS)
def correct(self, word):
candidates = self.known([word]) or self.known(self.edits1(word)) or self.known_edits2(word) or [word]
return max(candidates, key=self.NWORDS.get)
status_processing()
that is taking a long time, rather than reading the CSV file? If so, I'm not sure we can help you, considering that you haven't shown us the code behindmyCorpus
. \$\endgroup\$status_processing()
The first operations it does in an acceptable time. The problem is when I invoke thespell_correct ()
. Already after I read the entire CSV file, it has to fix everything. \$\endgroup\$