Converting python code to async calls

Question

I'm going to start out by saying this is the 1st python program I have ever written and have no real background in the language so my code is probably pretty rough. Take it easy on me!

When I wrote this it works fine for a small number of tickers but when I have a file with 6k+ it is very slow. Now I know there are other ways I can improve performance BUT I want to try and tackle the async thing first.

I thought it was as simple as making the read_ticker_file function async and adding await in front of the yfin_options() call but obviously that didn't work.

I'm thinking possibly I need to restructure the way things are called but kind of stuck here. Hoping someone can point me in the right direction! Thanks in advance

import logging
import pyodbc
import config
import yahoo_fin as yfin
from yahoo_fin import options
from datetime import datetime, date
from selenium import webdriver


def main():
    read_ticker_file()


def init_selenium():
    driver = webdriver.Chrome(config.CHROME_DRIVER)
    return driver


def yfin_options(symbol):
    logging.basicConfig(filename='yfin.log', level=logging.INFO)
    logging.basicConfig(filename='no_options.log', level=logging.ERROR)

    try:

        # get all options dates (in epoch) from dropdown on yahoo finance options page
        dates = get_exp_dates(symbol)

        # iterate each date to get all calls and insert into sql db
        for date in dates:
            arr = yfin.options.get_calls(symbol, date)

            arr_length = len(arr.values)

            i = 0

            for x in range(0, arr_length):
                strike = str(arr.values[i][2])
                volume = str(arr.values[i][8])
                open_interest = str(arr.values[i][9])
                convert_epoch = datetime.fromtimestamp(int(date))
                try:
                    sql_insert(symbol, strike, volume, open_interest, convert_epoch)
                    i += 1
                except Exception as insert_fail:
                    print("I failed at sqlinsert {0}".format(insert_fail))
            file_name_dir = "C:\\temp\\rh\\options{0}{1}.xlsx".format(symbol, date)
            logging.info(arr.to_excel(file_name_dir))

    except Exception as e:
        bad_tickers_file_dir = config.BAD_TICKERS
        f = open(bad_tickers_file_dir, "a")
        f.write(symbol)
        f.write('\n')


def sql_insert(symbol, strike, volume, open_interest, exp_date):
    conn_string = ('Driver={SQL Server};'
                   'Server={0};'
                   'Database={1};'
                   'Trusted_Connection=yes;').format(config.SERVER, config.DATABASE)
    conn = pyodbc.connect(conn_string)
    cursor = conn.cursor()

    insert_string = """INSERT INTO dbo.options (Ticker, Strike, Volume, OpenInterest, expDate)
                    VALUES
                    (?, ?, ?, ?, ?)"""

    cursor.execute(insert_string, symbol, strike, volume, open_interest, str(exp_date))

    conn.commit()


def get_exp_dates(symbol):
    url = "https://finance.yahoo.com/quote/" + symbol + "/options?p=" + symbol
    chromedriver = init_selenium()
    chromedriver.get(url)
    # Yahoo Finance options dropdown class name (find better way to do this)
    select_dropdown = chromedriver.find_element_by_css_selector("div[class='Fl(start) Pend(18px)'] > select")
    options_list = [x for x in select_dropdown.find_elements_by_tag_name("option")]
    dates = []
    for element in options_list:
        dates.append(element.get_attribute("value"))

    return dates


def read_ticker_file():
    file1 = open(config.TICKER_FILE, 'r')
    lines = file1.readlines()

    count = 0
    # loop to read each ticker in file
    for line in lines:
        count += 1
        line = line.strip('\n')
        line = line.strip()
        yfin_options(line)


if __name__ == "__main__":
    main()

Answer 1

After doing some reading and taking the advice of FMc, I decided against async for what I was doing.

I ended up with multi-processing

pool = multiprocessing.Pool()

# input list
inputs = read_ticker_file()
# pool object with number of element
pool = multiprocessing.Pool(processes=4)

pool.map(yfin_options, inputs)

pool.close()
pool.join()

I've noticed some weird behavior but we'll see if it gets the same result more quickly when it's done running.