I'm a newbie getting into web scrapers. I've made something that works, it takes 3.2 hours to complete job and randomly have 10 lines blank each time I run this job. Help is much appreciated!
import sys
import pandas as pd
from selenium import webdriver
import time
from datetime import datetime
from bs4 import BeautifulSoup
from selenium.webdriver.support.select import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ActionChains
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from webdriver_manager.chrome import ChromeDriverManager
def getBrowser():
options = Options()
options.add_argument("--incognito")
global browser
options.add_argument("start-maximized")
s = Service('''C:\\Users\\rajes\\yogita\\drivers\\chromedriver.exe''')
browser = webdriver.Chrome('''C:\\Users\\rajes\\yogita\\drivers\\chromedriver.exe''')
return browser
def getISINUrls(browser):
url = 'http://www.nasdaqomxnordic.com/bonds/denmark/'
browser.get(url)
browser.maximize_window()
time.sleep(1)
bonds = {}
try:
getUrls(browser, bonds)
pg_down = browser.find_element(By.CSS_SELECTOR,
"#bondsSearchDKOutput > div:nth-child(1) > table > tbody > tr > td.pgDown")
browser.execute_script("arguments[0].click();", pg_down)
time.sleep(1)
while (True):
# pages = browser.find_element(By.ID, 'bondsSearchDKOutput')
getUrls(browser, bonds)
pg_down = browser.find_element(By.CSS_SELECTOR,
"#bondsSearchDKOutput > div:nth-child(1) > table > tbody > tr > td.pgDown")
browser.execute_script("arguments[0].click();", pg_down)
time.sleep(1)
except NoSuchElementException as e:
pass
return bonds
def getUrls(browser, bonds):
hrefs_in_table = browser.find_elements(By.XPATH, '//a[@href]')
count = 0
for element in hrefs_in_table:
href = element.get_attribute('href')
if 'microsite?Instrumen' in href:
bonds[element.text] = href
count += 1
def saveURLs(bond):
filename = "linkstoday.txt"
fo = open(filename, "w")
for k, v in bonds.items():
fo.write(str(v) + '\n')
fo.close()
def getSleepTime(count):
first = 1
res = 1
i = 0;
while i < count:
i += 1
temp = res
res = temp + first
first = temp
return res
def getISINData(browser2):
with open("linkstoday.txt", "r") as a_file:
denmark_drawing = []
for line in a_file:
result_found = False
count = 2
Isin_code = str()
short_name = str()
Volume_circulating = str()
Repayment_date = str()
Drawing_percent = str()
wait_time = getSleepTime(0) + 1
while not result_found and count < 5:
stripped_line = line.strip()
browser2.get(stripped_line)
browser2.maximize_window()
time.sleep(getSleepTime(count) + 1)
WebDriverWait(browser2, 1).until(
EC.element_to_be_clickable((By.CSS_SELECTOR, '#ui-id-3 > span'))).click()
time.sleep(getSleepTime(count))
Isin_code = browser2.find_element(By.CSS_SELECTOR, '#db-f-isin').text
short_name = browser2.find_element(By.CSS_SELECTOR, '#db-f-nm').text
Volume_circulating = browser2.find_element(By.CSS_SELECTOR, '#db-f-oa').text
Repayment_date = browser2.find_element(By.CSS_SELECTOR, '#db-f-drd').text
Drawing_percent = browser2.find_element(By.CSS_SELECTOR, '#db-f-dp').text
if Isin_code == " ":
count += 1
else:
result_found = True
temp_data = [Isin_code,
short_name,
Volume_circulating,
Repayment_date,
Drawing_percent]
denmark_drawing.append(temp_data)
# Writing data to dataframe
df3 = pd.DataFrame(denmark_drawing,
columns=['ISIN', 'Shortname', 'OutstandingVolume', 'Repaymentdate', 'Drawingpercent'])
df3.to_csv('Denamrkscrapedsata_20220121.csv', index=False)
if __name__ == "__main__":
browser = getBrowser()
print(f'''Call to getISINUrls start at: {datetime.now()}''')
bonds = getISINUrls(browser)
print(f'''Call to getISINUrls ends at : {datetime.now()}''')
print(f'''total records: {len(bonds)}''')
browser.close()
browser2 = getBrowser()
print(f'''Call to getISINData start at: {datetime.now()}''')
getISINData(browser2)
print(f'''Call to getISINData ends at : {datetime.now()}''')
saveURLs(bonds)
browser2.close()
sys.exit(0)
