The following is code that I used to do a task. the code runs fine but takes over 2 minutes to get executed. And this is with only one rule (Rule_1), the rule is checked using an .exe ( written in CPP).Actually there are many CSV files that are needed to pass through the rules.
My question is will the program always use this much amount of time, because I have to implement more the 50 rule for the files, or there is any other way around?
import os import fnmatch import subprocess import xml.etree.ElementTree as ElementTree from xml.parsers.expat import ExpatError import sys from shutil import copyfileobj def locate(pattern, root="Z:/Automation/"): '''Locate all files matching supplied filename pattern in and below supplied root directory.''' for path, dirs, files in os.walk(os.path.abspath(root)): for filename in fnmatch.filter(files, pattern): yield os.path.join(path, filename) csv_path_unrefined =  for xml in locate("*.csv"): try: ElementTree.parse(xml) except (SyntaxError, ExpatError): csv_path_unrefined.append(xml) csv_path =  for paths in csv_path_unrefined: if "results" in str(paths): csv_path.append(paths) def check_rule1(path): # path = "PWLLOGGER_DEMO.csv" file = 'ConsoleApplication9.exe "' + path + '"' # print(file) details = os.popen(file).read() log_file = open("logs/Rule_1.txt") state = log_file.read() with open('results/Rule_1_log.log', 'a+') as files: files.write("\n========" + path + "========\n") files.close with open('results/Rule_1_log.log', 'a+') as output, open('logs/Rule_1.txt', 'r') as input: copyfileobj(input, output) if "failed" in state: return False else: return True rule_1_passed =  rule_1_failed =  for paths in csv_path: result_r1 = check_rule1(paths) # print(result_r1) if result_r1 == False: rule_1_failed.append(paths) #print("Rule 1 has failed for " + paths) elif result_r1 == True: rule_1_passed.append(paths) #print("Rule 1 has passed for " + paths) open('logs/Rule_1.txt', 'w').close() print(rule_1_failed) print(rule_1_passed)