I have written a file to extract text from news articles. The program works as I want it to except it takes hours for it to write the output files. I have tried different buffering options as well as adding the data I want written to an array before writing that to the file. I have almost no experience programming so I might just be doing something really dumb.

The main issue with the program is when i'm printing ngrams. d2 contains all of the items I want to print and it prints it based off of an index in a list. d2 is generated quickly, and only the writing is slow.

import os, re
from collections import Counter
from math import sqrt

path = os.getcwd() + "/20_newsgroups"
#print path
def findFiles(path):
    Files = []
    for root, dirs, files in os.walk(path):
        for f in files:
           #print (os.path.join(root , f).replace("\\","/"))
            Files.append(os.path.join(root , f).replace("\\","/"))
            
    return Files
          
def cleanData():
    data = findFiles(path)    
     
    remove = []
    
    for entry in data:
        f = open(entry,'r')
        for line in f:
            if line.lower().startswith("subject: re:") or line.lower().startswith("subject: re:"):
                #flag = 1
                remove.append(entry)
                break
            elif line.lower().startswith("subject: "):
                break
                #print line
                #for word in line.split():                    
                  #  if word == "Re:" or word == "re:":
                        #print("Found Re:")
                   #     remove.append(entry)
                   #     break
            
            
    #print len(data)
    #print len(set(data))
    #print len(remove)
    #print len(set(remove))
    data2 = list(set(data) - set(remove))
    #print len(data2)    
    return set(data2) #6694

d1 = {} 
#d1.setdefault('test',[]).append('test2') ##this allows duplicate entries.
SetofUniqueWords = set()


def readInWords():
    data = cleanData()
    for entry in data:
        flag = 1
        AfterLinesLine = False
        f = open(entry,'r')       
        for line in f:
            if line.lower().startswith("subject:"):  
                if flag ==1:
                    for word in line.lower().split("subject: ")[1].split():                                        
                        for stuff in re.sub(r'[\W_]+',' ',word).lower().split():
                            if not re.match('^ *[0-9 ]*$',stuff):
                                d1.setdefault(entry,[]).append(stuff)
                                SetofUniqueWords.add(stuff)
                            
                flag = 0 ##this is because multiple lines in a file can start with subject so now we stop after the first one                               
                
            if AfterLinesLine == True:
                for word in line.lower().split(): 
                    for stuff in re.sub(r'[\W_]+',' ',word).lower().split():##
                        if not re.match('^ *[0-9 ]*$',stuff):
                            d1.setdefault(entry,[]).append(stuff)
                            SetofUniqueWords.add(stuff)
            if line.startswith("Lines:"):
                AfterLinesLine = True
                
readInWords()
ItemList = list(SetofUniqueWords)

ObjID = {}
def SetObjectIDs():
    id = 0
    for key in d1:
        ObjID[key] = id
        id = id + 1
                                 
def printBagOWords():
    f = open('bag.csv','w')
    for key in d1:
        #f.write(str(d1[key]))
        words = d1[key]
        freqs = Counter(words)
        mag = 0
        for i in freqs:
            mag = mag + freqs[i]*freqs[i]
            
        mag = sqrt(mag)
        for item in freqs:
            index = ItemList.index(item)
            #print key.split("/")[-1],index,freqs[item]
            f.write(str(ObjID[key])+','+str(index)+','+str(freqs[item]/mag)+'\n')
            #f.write(key.split("/")[-1]+','+str(item)+','+str(freqs[item])+'\n')
    f.close()
    h = open('bag.clabel','w')
    for entry in ItemList:
        h.write(entry+'\n')
    h.close()

def printClassFilerlabel():
    f = open('newsgroup.rlabel','w')
    for key in d1:
        f.write(str(ObjID[key])+','+key.split("/")[-2]+'_'+key.split("/")[-1]+'\n')
    f.close()

def printClassFile():
    f = open('newsgroup.class','w')
    for key in d1:
        f.write(str(ObjID[key])+','+key.split("/")[-2]+'\n')
    f.close()
    

def printNGrams(n):   
    d2 = {}
    SetofNGram = set()
    for key in d1:
        Str = " "
        Str = Str.join(d1[key])
        for i in range(0,len(Str)-n+1):
            item = Str[i:i+n]
            #if not re.match('^ *[0-9 ]*$',item):
            SetofNGram.add(item)
            d2.setdefault(key,[]).append(item)
            #print Str[i:i+n]
        #print len(Str)
    IL = list(SetofNGram)
    #print len(IL)
    #count = 0
    #print "Done finding dictionaries"
    g = open('char'+ str(n) +'.csv','w')
    for key in d1: #this is intentionally d1 and not d2 to make sure the files are written in the same order and they have the same keys so doesnt matter
        #count = count + 1
        #print count        
        words = d2[key]
        freqs = Counter(words)
        mag = 0
        a = str(ObjID[key])
        for i in freqs:
            mag = mag + freqs[i]**2 #making unit vectors!! 
            
        mag = sqrt(mag)
        for item in freqs:
            index = IL.index(item)
            g.write(a+','+str(index)+','+str(freqs[item]/mag)+'\n')
    g.close()
    h = open('char'+ str(n) +'.clabel','w')
    for entry in IL:
        h.write(entry+'\n')
    h.close()


SetObjectIDs()
     
printBagOWords()     
printNGrams(3)
printNGrams(5)
printNGrams(7)
printClassFile() 
printClassFilerlabel()