# List comprehension that contains a query

I'm trying to compare some entries in a table to some entries built from some other tables in arcgis using python. When I ran similar code to this, the script took no longer than 15 minutes. I moved some things around in the code because I had to in order to write the type of comparison I wanted. As a result a certain list comprehension went from around 1.5k lists to 90k lists - I'm using a list of lists because I didn't know I had access to pandas and at this point I don't have time to learn pandas to do this. The script now takes over 3 hours to run and almost all of it is on this one list comprehension. In fact I haven't actually been able to get the script to move past the list comprehension since I made these changes.

Why is this taking so damn long and how do I fix this? Here is the relevant part of the "script":

import arcpy
import os
import csv
import time
import shutil

def compareLists(listOne, listTwo,):

notInList = [item for item in listOne if item not in listTwo]
print "Lists compared wp"
return notInList

def compareDicts(dictOne, dictTwo):
#should build a dictionary, no?
#For some reason, the results for this are different when
#dictOne is swapped with dictTwo. Why?
dictCompare = {}
for item in dictOne:
areaMoreThan1Percent = ""
lengthMoreThan1Percent = ""
if item in dictTwo:
if not (dictOne[item][0] < (dictTwo[item][0] * 1.01) and
(dictOne[item][0] > 0.99)):
areaMoreThan1Percent = "Check Area"
if not (dictOne[item][1] < (dictTwo[item][1] * 1.01) and
(dictOne[item][1] > 0.99)):
lengthMoreThan1Percent = "Check Length"
if areaMoreThan1Percent or lengthMoreThan1Percent:
dictCompare.update({item: [areaMoreThan1Percent,
lengthMoreThan1Percent] +
dictOne[item]})
print "Done with what's in {} and not {}".format(compareNameOne,
compareNameTwo)
return dictCompare

#Making a FC to cut things down to the relevant records
Directory = r"U:\Temp Connections"

# Calculate date, time, and FGDB name
date = time.strftime('%Y %m %d  %H %M %S')
GDB_Name = date + '_NewFootprints.gdb'

# Create a new FileGDB
arcpy.CreateFileGDB_management(Directory, GDB_Name)

# Set to path of created FileGDB
GDB = os.path.join(Directory, GDB_Name)

connection_name = "Real enterprise path"
database_platform = "SQL_SERVER"
instance = "realInstance"
authentication = "correctAuthForm"
database = "SolidDatabase"

# Look for folder to put Prod Connection into
if not os.path.isdir(Directory):
os.path.makedirs(Directory)

# Look for Prod connection and create if absent
if not os.path.isfile(os.path.join(Directory, connection_name)):
print ("Making connection file")
arcpy.CreateDatabaseConnection_management(Directory,
connection_name,
database_platform,
instance,
authentication,
savePW,
database)

easyFDS = r"realFeatureDataset"
FeatureClass = r"realFeatureClass"
FullPathFC = os.path.join(Directory, connection_name, easyFDS, FeatureClass)
sr = arcpy.Describe(FullPathFC).spatialReference

# Create new FDS for fc
arcpy.CreateFeatureDataset_management(GDB,
"someStuff",
sr)

# Get file path of FDS in created FileGDB
newFDS = os.path.join(GDB, "someStuff")

print ("New File GDB and feature dataset have been created")

difficultFC = os.path.join(r"first half of the connection path",
r"second half of the connection path")
easyFC = os.path.join(r"first half of the connection path",
r"Second half of the other connection path"
)
anotherFCForTheHardOne = r"rightConnectionPathForThis"

inFeaturesList = [hardFC, anotherFCForTheHardOne]
outFeatureClass = os.path.join(newFDS,"The Right Stuff")

arcpy.Intersect_analysis(inFeaturesList, outFeatureClass)
print "Intersect ran"

#list and dict comprehensions
easyFC = os.path.join(r"first half of the connection path",
r"Second half of the other connection path"
)
easyFields = ["common","OBJECTID","SHAPE.STArea()","SHAPE.STLength()"]
#Dunno if we're using OWNER_NUMBER
easyFieldsDict = ["common","OBJECTID","SHAPE.STArea()","SHAPE.STLength()",
"Field0", "Field1", "Field2", "Field3",
"Field3", "Field4","Field5",
"Field6", "Field7", "Field8", "Field9"]
easyList = [[str(row[0]),str(row[4]), str(row[5]), str(row[6]),
str(row[7]), str(row[8]), str(row[9]),str(row[10]),
str(row[11]), str(row[12]),str(row[13])]
for row in arcpy.da.SearchCursor(easyFC,
easyFieldsDict)]
easyDict = {str(row[0]): [float(row[2]), float(row[3])] for row in
arcpy.da.SearchCursor(easyFC, easyDict)}
print "Done with easy List"

hardFC = outFeatureClass
hardFields = ["common","OBJECTID","Shape_Area","Shape_Length"]
hardFieldsDict = ["common","OBJECTID","Shape_Area","Shape_Length"]
hardList = [str(row[0]) for row in arcpy.da.SearchCursor(hardFC, hardFields)]
hardDict = {str(row[0]): [float(row[2]) ,float(row[3])] for row in
arcpy.da.SearchCursor(hardFC, hardFieldsDict)}
moreData = r"Correct connection path"
moreDataFields = ["common", "OBJECTID", "Field0", "Field1", "Field2",
"Field3", "Field4", "Field5", "Field6", "Field7",
"Field8", "Field9"]

#This takes forever.
#This one bit seems to be taking forever. What did I do!?

hardListComp = [[str(row[0]), str(row[1]), str(row[2]), str(row[11]),str(row[3]),
str(row[4]), str(row[5]), str(row[6]), str(row[7]), str(row[8]),
str(row[9]), str(row[10])] for item in hardList for row in
arcpy.da.SearchCursor(moreData, moreDataFields,
"common = '{}'".format(item))]

hardListComp finally finished running. It took about 3.5 hours.
• It looks like compareLists is never called. – Solomon Ucko Dec 18 '17 at 23:37
• hardList is only one field for each record - the 'common' value. But if you're looping over moreData once for each entry in hardFC, then looping over the results of the query, why not do a JOIN of the two and just one loop? – Austin Hastings Dec 18 '17 at 23:48