1
\$\begingroup\$

I'm trying to compare some entries in a table to some entries built from some other tables in arcgis using python. When I ran similar code to this, the script took no longer than 15 minutes. I moved some things around in the code because I had to in order to write the type of comparison I wanted. As a result a certain list comprehension went from around 1.5k lists to 90k lists - I'm using a list of lists because I didn't know I had access to pandas and at this point I don't have time to learn pandas to do this. The script now takes over 3 hours to run and almost all of it is on this one list comprehension. In fact I haven't actually been able to get the script to move past the list comprehension since I made these changes.

Why is this taking so damn long and how do I fix this? Here is the relevant part of the "script":

import arcpy
import os
import csv
import time
import shutil


def compareLists(listOne, listTwo,):

    notInList = [item for item in listOne if item not in listTwo]
    print "Lists compared wp"
    return notInList

def compareDicts(dictOne, dictTwo):
    #should build a dictionary, no?
    #For some reason, the results for this are different when
    #dictOne is swapped with dictTwo. Why?
    dictCompare = {}
    for item in dictOne:
        areaMoreThan1Percent = ""
        lengthMoreThan1Percent = ""
        if item in dictTwo:
            if not (dictOne[item][0] < (dictTwo[item][0] * 1.01) and
            (dictOne[item][0] > 0.99)):
                areaMoreThan1Percent = "Check Area"
            if not (dictOne[item][1] < (dictTwo[item][1] * 1.01) and
            (dictOne[item][1] > 0.99)):
                lengthMoreThan1Percent = "Check Length"
            if areaMoreThan1Percent or lengthMoreThan1Percent:
                dictCompare.update({item: [areaMoreThan1Percent,
                                    lengthMoreThan1Percent] +
                                   dictOne[item]})
    print "Done with what's in {} and not {}".format(compareNameOne,
                                                    compareNameTwo)
    return dictCompare

#Making a FC to cut things down to the relevant records
Directory = r"U:\Temp Connections"

# Calculate date, time, and FGDB name
date = time.strftime('%Y %m %d  %H %M %S')
GDB_Name = date + '_NewFootprints.gdb'

# Create a new FileGDB
arcpy.CreateFileGDB_management(Directory, GDB_Name)

# Set to path of created FileGDB
GDB = os.path.join(Directory, GDB_Name)

connection_name = "Real enterprise path"
database_platform = "SQL_SERVER"
instance = "realInstance"
authentication = "correctAuthForm"
username = "RightUserName"
password = "CorrectPassword"
savePW = "SAVE_USERNAME"
database = "SolidDatabase"

# Look for folder to put Prod Connection into
if not os.path.isdir(Directory):
    os.path.makedirs(Directory)

# Look for Prod connection and create if absent
if not os.path.isfile(os.path.join(Directory, connection_name)):
    print ("Making connection file")
    arcpy.CreateDatabaseConnection_management(Directory,
                                          connection_name,
                                          database_platform,
                                          instance,
                                          authentication,
                                          username,
                                          password,
                                          savePW,
                                          database)

easyFDS = r"realFeatureDataset"
FeatureClass = r"realFeatureClass"
FullPathFC = os.path.join(Directory, connection_name, easyFDS, FeatureClass)
sr = arcpy.Describe(FullPathFC).spatialReference

# Create new FDS for fc
arcpy.CreateFeatureDataset_management(GDB,
                                      "someStuff",
                                      sr)

# Get file path of FDS in created FileGDB
newFDS = os.path.join(GDB, "someStuff")

print ("New File GDB and feature dataset have been created")

difficultFC = os.path.join(r"first half of the connection path",
                      r"second half of the connection path")
easyFC = os.path.join(r"first half of the connection path",
                             r"Second half of the other connection path"
                             )
anotherFCForTheHardOne = r"rightConnectionPathForThis"


inFeaturesList = [hardFC, anotherFCForTheHardOne]
outFeatureClass = os.path.join(newFDS,"The Right Stuff")

arcpy.Intersect_analysis(inFeaturesList, outFeatureClass)
print "Intersect ran"

#list and dict comprehensions
easyFC = os.path.join(r"first half of the connection path",
                             r"Second half of the other connection path"
                             )
easyFields = ["common","OBJECTID","SHAPE.STArea()","SHAPE.STLength()"]
#Dunno if we're using OWNER_NUMBER
easyFieldsDict = ["common","OBJECTID","SHAPE.STArea()","SHAPE.STLength()",
                         "Field0", "Field1", "Field2", "Field3",
                         "Field3", "Field4","Field5",
                         "Field6", "Field7", "Field8", "Field9"]
easyList = [[str(row[0]),str(row[4]), str(row[5]), str(row[6]),
                   str(row[7]), str(row[8]), str(row[9]),str(row[10]),
                   str(row[11]), str(row[12]),str(row[13])]
                  for row in arcpy.da.SearchCursor(easyFC,
                                                   easyFieldsDict)]
easyDict = {str(row[0]): [float(row[2]), float(row[3])] for row in
                  arcpy.da.SearchCursor(easyFC, easyDict)}
print "Done with easy List"

hardFC = outFeatureClass
hardFields = ["common","OBJECTID","Shape_Area","Shape_Length"]
hardFieldsDict = ["common","OBJECTID","Shape_Area","Shape_Length"]
hardList = [str(row[0]) for row in arcpy.da.SearchCursor(hardFC, hardFields)]
hardDict = {str(row[0]): [float(row[2]) ,float(row[3])] for row in
            arcpy.da.SearchCursor(hardFC, hardFieldsDict)}
moreData = r"Correct connection path"
moreDataFields = ["common", "OBJECTID", "Field0", "Field1", "Field2",
                  "Field3", "Field4", "Field5", "Field6", "Field7",
                  "Field8", "Field9"]

#This takes forever.
#This one bit seems to be taking forever. What did I do!?

hardListComp = [[str(row[0]), str(row[1]), str(row[2]), str(row[11]),str(row[3]),
             str(row[4]), str(row[5]), str(row[6]), str(row[7]), str(row[8]),
             str(row[9]), str(row[10])] for item in hardList for row in
            arcpy.da.SearchCursor(moreData, moreDataFields,
                                  "common = '{}'".format(item))]
print "hardListComp made"

Is this just too big for a list? As always any comments on the poor quality of my code are welcome.

hardListComp finally finished running. It took about 3.5 hours.

\$\endgroup\$
  • \$\begingroup\$ It looks like compareLists is never called. \$\endgroup\$ – Solomon Ucko Dec 18 '17 at 23:37
  • \$\begingroup\$ @SolomonUcko I haven't called the functions. I'm still trying to build hardList \$\endgroup\$ – user106363 Dec 18 '17 at 23:39
  • \$\begingroup\$ @AustinHastings hardList is being made correctly. I have checked the items in it to make sure that they're right. I have not checked the results of the hardListComp yet since it hasn't finished. \$\endgroup\$ – user106363 Dec 18 '17 at 23:43
  • \$\begingroup\$ Deleted. Now, it looks like your hardlist is every record in the hardFC, yes? \$\endgroup\$ – Austin Hastings Dec 18 '17 at 23:45
  • 2
    \$\begingroup\$ hardList is only one field for each record - the 'common' value. But if you're looping over moreData once for each entry in hardFC, then looping over the results of the query, why not do a JOIN of the two and just one loop? \$\endgroup\$ – Austin Hastings Dec 18 '17 at 23:48

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy