Speed up Python Script to populate MongoDB

Question

How would I go about making this script much faster? Essentially it reads from a file and the slowest part is populating the words. In the words file there are over 100k words and I was looking for a way to speed it up since this script is going to be used to populate the database with over 200 languages.

import os, django
os.environ.setdefault("DJANGO_SETTINGS_MODULE","eureka.settings")
django.setup()
from django.contrib.auth.models import User
from django.contrib import admin
from wordDictionary.models import Genus, Word, Feature, Dimension, Language, Lemma, Family, TagSet, POS
import multiprocessing



# Dimensions
def dimensionPop():
    dimData = open("data/models/dimensions.txt","r")

    for x in dimData:
        a = x.split("\n")
        dimName = a[0]
        # Create Object 
        nextDim = Dimension(name=dimName)
        nextDim.save()
        #print(f":{nextDim.name}:")
    dimData.close()
    print("Dimension done")


# Features
def featurePop():
    featData = open("data/models/features.txt","r")
    for x in featData:
        line = x.split(";")
        featName = line[1]
        dimName = line[0]
        # Create Object 
        nextFeature = Feature(name=featName)
        dimObject = Dimension.objects.get(name=dimName)
        nextFeature.dimension = dimObject   

        #print(f"{nextFeature.dimension.name}")
        nextFeature.save()

    featData.close()
    print("Feature done")


# Part of Speech
def posPop():
    posData = open("data/models/POS.txt","r")
    for x in posData:
        line = x.split(";")
        posName = line[1]

        # Create Object 
        nextPOS = POS(name=posName)

        #print(f"{nextPOS.name}")
        nextPOS.save()
    posData.close()
    print("Part of Speech done")

# Genus
def genusPop():
    genusData = open("data/models/genus.txt","r")
    for x in genusData:
        genusName = x.split("\n")[0]
        # Create Object 
        nextGenus = Genus(name=genusName)

        #print(f":{nextGenus.name}:")
        nextGenus.save()

    genusData.close()
    print("Genus done")




# Family
def familyPop():

    famData = open("data/models/families.txt","r")
    for x in famData:
        FamilyName = x.split(";")[0]
        # Create Object 
        nextFamily = Family(name=FamilyName)

        #print(f":{nextFamily.name}:")

        nextFamily.save()
    famData.close()
    print("Family done")


def languagePop():

    #Populate only english for now
     nextLang = Language(name="English")
     nextLang.walsCode = "eng"
     nextLang.genus = Genus.objects.get(name="Germanic")
     nextLang.family = Family.objects.get(name="Indo-European")
     nextLang.save()

     print("Language done")


def lemmaPop():
    lemmaData = open("data/models/lemmas.txt","r",encoding="utf8")

    for x in lemmaData:
        x = x.split("\n")

        lemmaName = x[0]

        nextLemma = Lemma(name=lemmaName)


        langName = Language.objects.get(name="English")
        nextLemma.language = langName
        posName = POS.objects.get(name="Verb")
        nextLemma.pos = posName
        nextLemma.save()

    lemmaData.close()
    print("Lemma done")


findFeature={}
def readAppendix():
    fileContent =  open("data/models/features.txt","r")
    for row in fileContent:

        rowWords = row.split(";")
        dimension = rowWords[0]
        feature = rowWords[1]
        label =(rowWords[2].rstrip()).upper()
        findFeature[label]=feature # assign feature to label
    fileContent.close()
    print("\nStarting with words...")

usedTagset = {}

def wordPop():
    wordData =  open("data/langs/English.txt","r",encoding="utf8")
    it = 0
    for line in wordData:

        it += 1
        if it % 1000 :
            print(f"> {it}...")

        rowContent = line.split()
        if(len(rowContent)>=3): # checks if line is valid
            tagsetName = rowContent[-1]
            tagSetObject = None
            try:
                if usedTagset[tagsetName] == 1:
                    someTagset = TagSet.objects.get(name=tagsetName)
                    tagSetObject = someTagset
            except KeyError:
                usedTagset[tagsetName]=1
                tagSetObject = TagSet(name=tagsetName)


            rootWord = rowContent[0]
            currWordList = rowContent[1:-1] # it can be more than a single words

            currWord = ""
            for temp in currWordList:
                currWord += temp + " "
            currWord = currWord[:-1] # remove last space

            allLabels = tagsetName.split(";") # last block of words corrensponds to allLabels

            for currLabel in allLabels:
                try:
                    currFeature = findFeature[currLabel.upper()]
                    featObject = Feature.objects.get(name=currFeature)

                    tagSetObject.features.add(featObject)
                except KeyError:
                    print(f"{currLabel} label doesn't exist.")
        # print(tagSetObject.features.all())

        tagSetObject.save()

        # Defining the Word/Form


        wordObject = Word(name=currWord)

        lemmaObject = Lemma.objects.get(name=rootWord)
        wordObject.lemma  = lemmaObject
        wordObject.tagset = tagSetObject
        wordObject.language = lemmaObject.language
        # print(f"{wordObject.name} : {wordObject.lemma} : {wordObject.tagset} : {wordObject.language}")
        wordObject.save()
    wordData.close()




# *  uncomment below to populate !!in order!! *

dimensionPop()
featurePop()
genusPop()
posPop()
familyPop()
languagePop()
lemmaPop()
readAppendix()
wordPop()
#

# processes = []
# for _ in range(16):
#     p = multiprocessing.Process(target=wordPop)
#     p.start()
#     processes.append(p)

# for proc in processes:
#     process.join()

# Just in case it goes wrong

def emptyDatabase():

    Word.objects.all().delete()
    Lemma.objects.all().delete()
    TagSet.objects.all().delete()
    Language.objects.all().delete()
    Family.objects.all().delete()
    Dimension.objects.all().delete()
    Genus.objects.all().delete()
    POS.objects.all().delete()

    print("Database is empty...")


# emptyDatabase()


```

Answer 1

Just looking at wordPop since that's where you say the most time is being taken:

• I think your validity check is broken. There's an if statement to make sure the line is valid, but no else, and the code after the if uses variables defined within the conditional block.

• Use " ".join(currWordList) instead of manually concatenating a bunch of strings. That'll save you the creation of a bunch of intermediate string objects.

• You're storing values of 1 in usedTagset, but that's meaningless. What you're actually doing each time is fetching the object from the database. Sure, that's a "fast" lookup if the name field is indexed, but nowhere near as fast as just fetching the object from the dictionary. Instead, consider something like the following:

  if tagsetName not in usedTagset:
      usedTagset[tagsetName] = TagSet.objects.create(name=tagsetName)
  tagSetObject = usedTagset[tagsetName]
  

• Same thing for the findFeature chunk in the allLabels loop. If you're just doing a couple lookups, .get on an indexed column is super fast... but that's "fast" relative to database speeds. In this context, "fast" means a few milliseconds or most of a ms, most likely. That's completely intolerable when you're doing a loop of hundreds of thousands or millions of rows. Fetch the objects once, cache them in memory, and do direct lookups using a dictionary. You can initialize the cache as follows:

  usedTagset = {tag.name: tag for tag in TagSet.objects.all()}
  

  and then, as discussed above, save any newly created objects as you create them rather than re-fetch them each time. If you're not sure if something already exists in the database, use get_or_create:

  if tagsetName not in usedTagset:
      # get_or_create returns a tuple, the first element of which is the ORM object
      usedTagset[tagsetName] = TagSet.objects.get_or_create(name=tagsetName)[0]
  tagSetObject = usedTagset[tagsetName]
  

Cutting the DB and associated driver/network/ORM overhead out of the loop will likely be all you need to get reasonably performant code.

• I'm also a fan of ...objects.create. It's more explicit that constructing the object one bit at a time.

  lemma = lemma_lookup[rootWord]
  Word.objects.create(
      name=currWord,
      lemma=lemma,
      tagset=tagSetObject,
      language=lemma.language
  )
  

• And finally, not performance related, but for the love of other Python developers, please follow PEP8 naming conventions, particularly using lower-case underscore-separated names for variables; e.g., word_object instead of wordObject, and tag_set_object instead of tagSetObject.