The whole process mainly depends on the ProcessFile()
function:
I get all the credentials and details from a JSON file.
Initially I get data from the db: two IDs and a URL.
There is a text file for address state matching, reading, and compiling with
|
to match regex.The same is done for
zip
and zip keywords.Iterate over the row.
Get content of the URL and remove unwanted things.
Match the address regex and store the result to a variable.
Match the zip regex, see if it is available in the zip keyword and write the result to variable append the total results to outputs list. Write only when the list size is equal or greater than 1000.
After completing all the URLs, see if outputs contains data. If so, write to the file, else leave it.
Move the result to another FTP location.
#Importing modules
import requests
import re
import sys
import pyodbc
import json
import datetime
import random
import time
import ftplib
import traceback
#loading regex files
start_time_split=datetime.datetime.now()
json_data=open('server_details.json')
server_details = json.load(json_data)
json_data.close()
'''SQL SERVER CREDENTIALS AND DB NAME'''
SERVER=server_details["SERVER"]
DATABASE=server_details["DATABASE"]
UID=server_details["UID"]
PWD=server_details["PWD"]
ADDRESS=server_details["ADDRESS"]
ZIP=server_details["ZIP"]
ZIP_WORD=server_details["ZIP_WORD"]
FTP_IP=server_details["FTP_IP"]
FTP_USER=server_details["FTP_USER"]
FTP_PWD=server_details["FTP_PWD"]
FTP_LOCATION=server_details["FTP_LOCATION"]
def ProcessFile(fileReadStr):
try:
header = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0'}
with open(ADDRESS,"r") as l:
address="|".join([a.strip() for a in l])
#address=address
with open(ZIP,"r") as ll:
regex="|".join([a.strip() for a in ll])
#regex=regex
with open(ZIP_WORD) as inp:
zip_word=[a.strip() for a in inp.readlines()]
print "Started "+str(start_time_split)
filename=str(random.randint(1,100000))+"_"+str(int(time.time()))+"_us_link_classifier.txt"
with open(filename,"w") as output:
output.write("ID\tClientID\tURL\tAddress\tword\tzip\tzipmatch\n")
outputs=[]
for line in fileReadStr:
id1=line[0]
web=line[1] if line[1].strip().startswith("http") else "http://"+line[1].strip()
id2=line[2]
ad="0"
word="None"
try:
objReq = requests.get(web,timeout=30,verify=False,allow_redirects=True,headers=header)
htmlStr = objReq.content
htmlStr=re.sub("\s+"," ",htmlStr)
htmlStr=re.sub("<!--.*?-->", " ", htmlStr)
htmlStr=re.sub("<head.*?>.*?</head>"," ",htmlStr)
htmlStr=re.sub("<.*?>", " ", htmlStr)
htmlStr=re.sub("\s+", " ", htmlStr)
address_match=re.search(address,htmlStr,re.IGNORECASE)
if address_match :
ad="1"
word=address_match.group(0).strip()
address_zip=re.findall(regex,htmlStr)
zip_match =[el for sub in address_zip for el in sub if el and el.split()[0] in zip_word]
if zip_match:
zp="1"
zip_match=zip_match[0]
else:
zip_match="None"
outputs.append(str(id1)+"\t"+str(id2)+"\t"+web+"\t"+ad+"\t"+word+"\t"+zp+"\t"+zip_match)
if len(outputs)>=150:
with open(filename,"a") as output:
output.write("\n".join(outputs)+"\n")
del outputs[:]
except Exception,e:
print e
print traceback.print_exc()
with open("us_error.txt","a") as ded:
ded.write(str(id1)+"\t"+web+"\t"+str(e)+"\n")
if outputs:
with open(filename,"a") as output:
output.write("\n".join(outputs)+"\n")
del outputs[:]
infile = open(filename,'r')
ftp = ftplib.FTP(FTP_IP,FTP_USER ,FTP_PWD )
ftp.storbinary('STOR '+FTP_LOCATION+str(filename), infile)
infile.close()
ftp.quit()
except Exception,e:
print e
print traceback.print_exc()
with open("us_error_process_error.txt","a") as ded:
ded.write( str(e))
if __name__ == '__main__':
try:
access_string='DRIVER={SQL Server};SERVER='+SERVER+';DATABASE='+DATABASE+';UID='+UID+';PWD='+PWD
cnxn = pyodbc.connect(access_string,autocommit=True)
cursor = cnxn.cursor()
print sys.argv[1]+'\n'+sys.argv[2]
exec1=server_details["QUERY"]
cursor.execute(exec1,(sys.argv[1],sys.argv[2]))
rows = cursor.fetchall()
cnxn.close()
ProcessFile(rows)
end_time_split=datetime.datetime.now()
elasped_time1=end_time_split-start_time_split
print "Total time:"+str(elasped_time1)
print "done"
except Exception,e:
print e
print traceback.print_exc()
with open("us_error_starting_process_error.txt","a") as ded:
ded.write(str(e))