Yahoo finance option chain scraper

Question

This is my 1st Python program I've actually written and I have little to no background in the language. I just figured I could learn and do something that interested me at the same time.

I'm not a book learner. I can't just sit down and read a bunch of technical documents and stuff like that so I'm sure a lot of this code is unconventional at best and just plain bad at worst. I learn best by doing, so now that I got this program to work I want to learn how to make it better.

The goal of the program is to load the Yahoo finance options page for each ticker in a file that I created. Then pull all of the call options data for each expiration date and load all of that data into a SQL database (to be queried later).

I added the multi-processing to try and make it faster, but it seems to have just slowed it down. I gotta figure that part out.

import logging
import pyodbc
import config
import yahoo_fin as yfin
import asyncio
import multiprocessing
import time
from yahoo_fin import options
from datetime import datetime, date
from selenium import webdriver


def main():
    read_ticker_file()


def init_selenium():
    driver = webdriver.Chrome(config.CHROME_DRIVER)
    return driver


def yfin_options(symbol):
    logging.basicConfig(filename='yfin.log', level=logging.INFO)
    logging.basicConfig(filename='no_options.log', level=logging.ERROR)

    try:
        # get all options dates (in epoch) from dropdown on yahoo finance options page
        dates = get_exp_dates(symbol)

        # iterate each date to get all calls and insert into sql db
        for date in dates:
            arr = yfin.options.get_calls(symbol, date)

            arr_length = len(arr.values)

            i = 0

            for x in range(0, arr_length):
                strike: str = str(arr.values[i][2])
                volume = str(arr.values[i][8])
                open_interest = str(arr.values[i][9])
                convert_epoch = datetime.fromtimestamp(int(date))
                try:
                    sql_insert(symbol, strike, volume, open_interest, convert_epoch)
                    i += 1
                except Exception as insert_fail:
                    print("I failed at sqlinsert {0}".format(insert_fail))
            file_name_dir = "C:\\temp\\rh\\options{0}{1}.xlsx".format(symbol, date)
            logging.info(arr.to_excel(file_name_dir))

    except Exception as e:
        bad_tickers_file_dir = config.BAD_TICKERS
        f = open(bad_tickers_file_dir, "a")
        f.write(symbol)
        f.write('\n')


def sql_insert(symbol, strike, volume, open_interest, exp_date):
    conn_string = ('Driver={SQL Server};'
                   'Server=DESKTOP-7ONNV8L;'
                   'Database=optionsdb;'
                   'Trusted_Connection=yes;')

    conn = pyodbc.connect(conn_string)
    cursor = conn.cursor()

    insert_string = """INSERT INTO dbo.options (Ticker, Strike, Volume, OpenInterest, expDate)
                    VALUES
                    (?, ?, ?, ?, ?)"""

    cursor.execute(insert_string, symbol, strike, volume, open_interest, str(exp_date))

    conn.commit()


def get_exp_dates(symbol):
    url = "https://finance.yahoo.com/quote/" + symbol + "/options?p=" + symbol
    chromedriver = init_selenium()
    chromedriver.get(url)
    # Yahoo Finance options dropdown class name (find better way to do this)
    select_dropdown = chromedriver.find_element_by_css_selector("div[class='Fl(start) Pend(18px)'] > select")
    options_list = [x for x in select_dropdown.find_elements_by_tag_name("option")]
    dates = []
    for element in options_list:
        dates.append(element.get_attribute("value"))

    return dates


def read_ticker_file():
    file1 = open(config.TICKER_FILE, 'r')
    lines = file1.readlines()

    count = 0

    ticker_arr = []
    # loop to read each ticker in file
    for line in lines:
        count += 1
        line = line.strip('\n')
        line = line.strip()
        ticker_arr.append(line)

    return ticker_arr


if __name__ == "__main__":
    pool = multiprocessing.Pool()

    # input list
    inputs = read_ticker_file()
    # pool object with number of element
    pool = multiprocessing.Pool(processes=4)

    pool.map(yfin_options, inputs)

    pool.close()
    pool.join()

Answer 1

For get_exp_dates - and everything else here - Selenium is unneeded. The dates you seek are not based on AJAX etc., but baked right into the HTML:

<select class="Fz(s) H(25px) Bd Bdc($seperatorColor)" data-reactid="5">
<option selected="" value="1626998400" data-reactid="6">July 23, 2021</option>
<option value="1627603200" data-reactid="7">July 30, 2021</option>
<option value="1628208000" data-reactid="8">August 6, 2021</option>
<option value="1628812800" data-reactid="9">August 13, 2021</option>
<option value="1629417600" data-reactid="10">August 20, 2021</option>
<option value="1630022400" data-reactid="11">August 27, 2021</option>
<option value="1631836800" data-reactid="12">September 17, 2021</option>
<option value="1634256000" data-reactid="13">October 15, 2021</option>
<option value="1637280000" data-reactid="14">November 19, 2021</option>
<option value="1642723200" data-reactid="15">January 21, 2022</option>
<option value="1647561600" data-reactid="16">March 18, 2022</option>
<option value="1655424000" data-reactid="17">June 17, 2022</option>
<option value="1663286400" data-reactid="18">September 16, 2022</option>
<option value="1674172800" data-reactid="19">January 20, 2023</option>
<option value="1679011200" data-reactid="20">March 17, 2023</option>
<option value="1686873600" data-reactid="21">June 16, 2023</option>
</select>

So just get the HTML via Requests and parse it via BeautifulSoup.

for x in range(0, arr_length): can drop the 0, as it's default.

Since everything within arr.values[i] has a positional meaning, it's worth unpacking; something like

@dataclass
class SymbolRow:
    contract_name: str
    last_trade_date: datetime
    strike: Decimal
    last_price: Decimal
    bid: Decimal
    ask: Decimal
    change: Decimal
    change_percent: float
    volume: int
    open_interest: int
    implied_volatility: float
    
...

row = SymbolRow(*arr.values[i])

^ That version assumes that the fields are already properly deserialized from the markup into reasonable types, which they probably aren't. A constructor could do this.

"C:\\temp\\rh\\options{0}{1}.xlsx"

should not be hard-coded within your method, and should exist in either configuration or maybe a separated global variable.

f = open(bad_tickers_file_dir, "a")

should be using a with.

conn_string should not exist within your sql_insert, and indeed the sql_insert method should not be connecting; that should be done at the outer level and only once per program run.

[x for x in select_dropdown.find_elements_by_tag_name("option")]

can just be

list(select_dropdown.find_elements_by_tag_name("option"))

There's no kind of close-guarantee for your connection and cursors. Based on the code it doesn't look like their implementation of __exit__ is sensible, so just try/finally/.close().

Your logger configuration doesn't make much sense. You should probably make your own module-specific logger instance (rather than calling into the basic config stuff and using the root logger) - and have just a single logger with multiple handlers, each having a different level.

The following code demonstrates (minus your SQL stuff) a way to scrape Yahoo Finance without needing the yahoo_fin library, covering some of the above, and also demonstrating two different kinds of HTML path selector precompilation.

import enum
import locale
from datetime import datetime
from decimal import Decimal
from enum import Enum
from locale import setlocale, LC_NUMERIC
from pprint import pprint
from typing import Iterable, Optional, Dict

import pytz
import soupsieve
from soupsieve import SoupSieve
from bs4 import BeautifulSoup, SoupStrainer, Tag
from requests import Session


DATE_STRAINER = SoupStrainer(
    'select', class_='Fz(s) H(25px) Bd Bdc($seperatorColor)',
)
OPTION_STRAINER = SoupStrainer(
    'section', attrs={'data-yaft-module': 'tdv2-applet-OptionContracts'},
)
CALLS_SIEVE = soupsieve.compile('table.calls')
PUTS_SIEVE = soupsieve.compile('table.puts')


@enum.unique
class OptionKind(Enum):
    CALL = enum.auto()
    PUT = enum.auto()


class Option:
    def __init__(self, row: Dict[str, Tag], kind: OptionKind):
        self.kind = kind

        self.contract_name = row['Contract Name'].text
        self.contract_path = row['Contract Name'].a['href']

        dt, tz = row['Last Trade Date'].text.rsplit(maxsplit=1)
        # Ugh.
        tz = {
            'EDT': 'US/Eastern',
            # ...
        }.get(tz, tz)

        self.last_trade_date = datetime.strptime(
            dt, '%Y-%m-%d %I:%M%p'
        ).replace(tzinfo=pytz.timezone(tz))

        self.strike = money_or_none(row['Strike'].text)
        self.last_price = money_or_none(row['Last Price'].text)
        self.bid = money_or_none(row['Bid'].text)
        self.ask = money_or_none(row['Ask'].text)
        self.change = money_or_none(row['Change'].text)
        self.percent_change = percent_or_none(row['% Change'].text)
        self.volume = int_or_none(row['Volume'].text)
        self.open_interest = int_or_none(row['Open Interest'].text)
        self.implied_volatility = percent_or_none(row['Implied Volatility'].text)


def get_yfin(
    session: Session,
    symbol: str,
    when: Optional[int] = None,
    straddle: bool = False,
) -> str:
    params = {
        'p': symbol,
        'straddle': straddle,
    }
    if when is not None:
        params['date'] = when

    with session.get(
        f'https://finance.yahoo.com/quote/{symbol}/options',
        params=params,
        headers={'Accept': 'text/html'},
    ) as resp:
        resp.raise_for_status()
        return resp.text


def get_exp_dates(session: Session, symbol: str) -> Iterable[int]:
    doc = BeautifulSoup(
        get_yfin(session, symbol),
        features='html.parser', parse_only=DATE_STRAINER,
    )
    for option in doc.find_all('option'):
        yield int(option['value'])


def int_or_none(s: str) -> Optional[int]:
    if s == '-':
        return None
    return locale.atoi(s)


def percent_or_none(s: str) -> Optional[float]:
    if s == '-':
        return None
    return float(s.rsplit('%', 1)[0])


def money_or_none(s: str) -> Optional[Decimal]:
    if s == '-':
        return None
    return Decimal(s)


def table_to_dicts(parent: Tag, sieve: SoupSieve) -> Iterable[Dict[str, Tag]]:
    table, = sieve.select(parent, limit=1)
    heads = [th.text for th in table.thead.tr.find_all('th')]
    for tr in table.tbody.find_all('tr'):
        yield dict(zip(heads, tr.find_all('td')))


def get_options(session: Session, symbol: str, when: int) -> Iterable:
    doc = BeautifulSoup(
        get_yfin(session, symbol, when),
        features='html.parser', parse_only=OPTION_STRAINER,
    )

    for call in table_to_dicts(doc, CALLS_SIEVE):
        yield Option(call, OptionKind.CALL)
    for put in table_to_dicts(doc, PUTS_SIEVE):
        yield Option(put, OptionKind.PUT)


def main():
    setlocale(LC_NUMERIC, 'en_US.UTF8')

    with Session() as session:
        session.headers = {'User-Agent': 'Mozilla/5.0'}
        dates = tuple(get_exp_dates(session, 'msft'))
        for option in get_options(session, 'msft', dates[2]):
            pprint(option.__dict__)


if __name__ == '__main__':
    main()