Parse JSON from HTML Python

Question

I'm parsing out specific values on web pages with BeautifulSoup. However, since I'm using RegEx, my program is taking forever to run. Would love ideas on how to speed this up.

from bs4 import BeautifulSoup
import datetime
import json
from progressbar import progressbar
import pdb
import pickle
import re


class Listing():
    def __init__(self, custom_name, **entries):
        self.__dict__.update(entries)
        self.custom_name = custom_name
        self.date_accessed = datetime.datetime.today()

    def __hash__(self):
        return hash(self.custom_name)

    def __eq__(self, other):
        return self.custom_name == other.custom_name

    def __repr__(self):
        return self.custom_name


def list_to_dict(rlist):
    # QUEST: There are multiple colons in many of the entries.  I couldn't
    # figure out how to use re.split where it only split the first occurence
    # so instead I replace only the first occurence and then split that new str
    list_with_replace_str = [re.sub(":", ":REPLACE", e, 1) for e in rlist]
    temp_dict = dict(f.split(":REPLACE") for f in list_with_replace_str)
    clean_dict = {}
    for key in temp_dict.keys():
        clean_key = key.strip()
        clean_value = temp_dict[key].strip()
        clean_dict[clean_key] = clean_value
    return clean_dict


def parse_listings(listing_objs):
    def parse_financials_div(financials_soup, listing_obj):
        try:
            financials_text = financials_soup.text
            financials_list = financials_text.split("\r\n")[:-1]
            financials_dict = list_to_dict(financials_list)

            not_included = []
            for key in financials_dict:
                if "*" in financials_dict[key]:
                    not_included.append(key)

            financials_dict["notIncluded"] = not_included

            for key in financials_dict:
                try:
                    financials_dict[key] = int(
                        re.sub("[^0-9]", "", financials_dict[key]))
                except Exception:
                    continue

            return financials_dict
        except Exception as e:
            print(f"error {e}")
            pdb.set_trace()

    def parse_details_div(details_soup, listing_obj):
        try:
            details_tag_list = details_soup.contents
            details_str = " ".join([str(element)
                                    for element in details_tag_list])
            details_list = details_str.split("<dt>")[1:]
            strs_to_tags = [BeautifulSoup(detail, "html.parser")
                            for detail in details_list]
            details_text = [tag.text for tag in strs_to_tags]
            details_dict = list_to_dict(details_text)

            return details_dict
        except Exception as e:
            print(f"error {e}")
            pdb.set_trace()

    def parse_category(product_json_soup, listing_obj):
        product_json_str = product_json_soup.contents[0].replace(
            "\r", "").replace("\n", "")
        product_json_str = product_json_str.replace(
            "\'", "").replace('\\"', '').replace("\t", "")
        product_dict = json.loads(product_json_str)
        category_str = product_dict["category"]
        category_list = category_str.split(">")
        category_list = [category.strip() for category in category_list]
        listing_obj.category = {}
        listing_obj.category["parent_category"] = category_list[0]
        try:
            listing_obj.category["sub_category"] = category_list[1]
        except Exception:
            listing_obj.category["sub_category"] = "Not Present"

    def parse_address(address_json_soup, listing_obj):
        address_json_str = address_json_soup.contents[0].replace(
            "\r", "").replace("\n", "")
        address_json_str = address_json_str.replace(
            "\'", "").replace('\\"', '').replace("\t", "")
        address_dict = json.loads(address_json_str)
        listing_obj.address = address_dict["address"]

    # Parse available listing fields into a dict
    print("Parse financials and details for listings")
    for listing_obj in progressbar(listing_objs):
        try:
            index = listing_objs.index(listing_obj)
            length = len(listing_objs)
            soup = BeautifulSoup(listing_obj.response_text, "html.parser")

            # Parse category
            category_json_pattern = re.compile(r"\"@type\" : \"Product\"")
            category_json_soup = soup.find(
                "script", {"type": "application/ld+json"}, text=category_json_pattern)
            if category_json_soup:
                parse_category(category_json_soup, listing_obj)

            # Parse address
            address_json_pattern = re.compile(r"LocalBusiness")
            address_json_soup = soup.find(
                "script", {"type": "application/ld+json"}, text=address_json_pattern)
            if address_json_soup:
                parse_address(address_json_soup, listing_obj)

            # Price details
            financials_span_pattern = re.compile(r"Asking Price:")
            financials_span_soup = soup.find(
                "span", text=financials_span_pattern)

            if financials_span_soup:
                financials_soup = financials_span_soup.parent.parent.parent.parent
                financials_dict = parse_financials_div(
                    financials_soup, listing_obj)
                listing_obj.financials = financials_dict
            else:
                print(
                    f"Financials not present #{index} of {length} {listing_obj.url}")
                print(soup)

            # Listing Details
            details_soup = soup.find("dl", {"class": "listingProfile_details"})
            if details_soup:
                details_dict = parse_details_div(details_soup, listing_obj)
                listing_obj.details = details_dict
        except Exception as e:
            print(f"error {e}")


def run_listing_calculations(listing_obj):
    # All in price
    extra_costs = 0
    price = listing_obj.financials["Asking Price"]
    for item in listing_obj.financials["notIncluded"]:
        if "Real Estate" not in item:
            extra_costs += listing_obj.financials[item]
    if isinstance(price, int):
        all_in_price = listing_obj.financials["Asking Price"] + extra_costs
    else:
        all_in_price = listing_obj.financials["Asking Price"]
    listing_obj.financials["allInPrice"] = all_in_price

    # Multiple
    all_in_price = listing_obj.financials["allInPrice"]
    cashflow = listing_obj.financials["Cash Flow"]
    try:
        listing_obj.financials["Multiple"] = all_in_price / cashflow
    except Exception:
        listing_obj.financials["Multiple"] = "N/A"


def parse_listings_from_pkl():
    with open("/Users/work/Dropbox/Projects/Working Data/bizbuysell/listings20191231.pkl", "rb") as infile:
        listing_objs = pickle.load(infile)

    print("Validate listing responses")
    listing_resp_validated = []
    for listing_obj in progressbar(listing_objs):
        try:
            if "Soup test failed" not in listing_obj.response_text:
                listing_resp_validated.append(listing_obj)
        except Exception:
            continue
    parse_listings(listing_resp_validated)
    print("Perform listing calculations")
    for listing_obj in progressbar(listing_resp_validated):
        financials_present = hasattr(listing_obj, "financials")
        if financials_present:
            run_listing_calculations(listing_obj)

    pdb.set_trace()


if __name__ == "__main__":
    parse_listings_from_pkl()

Here's a link to the .pkl file needed to run this.

Here's a gist with the example HTML response and product_json_soup.

Answer 1

The most time is consumed by BeautifulSoup conversions, namely

soup = BeautifulSoup(listing_obj.response_text, "html.parser")

For proof, firstly create a .pkl file of a reasonable size for debugging:

if __name__ == "__main__":
    with open("D:\\Downloads\\listings20191231.pkl", "rb") as infile:
        listing_objs = pickle.load(infile)
    data = listing_objs[222:666]
    with open("D:\\Python\\CR\\listings20191231.pkl", "wb") as oufile:
        pickle.dump(data, oufile, pickle.HIGHEST_PROTOCOL)

Then, check and compare consumed time using following adapted code (moreover, I removed all the progressbar stuff from the rest of original code):

if __name__ == "__main__":
    import time
    import sys
    argcnt = len(sys.argv) - 1
    argtxt = 'parse_listings_from_pkl()' if argcnt == 0 else 'BeautifulSoup'
    startload = time.time()
    with open("D:\\Python\\CR\\listings20191231.pkl", "rb") as infile:
        listing_objs = pickle.load(infile)

    length = len(listing_objs)
    print( 'checking time: ', argtxt, length, 'records')
    start0 = time.time()
    if argcnt == 0:
        parse_listings_from_pkl()
    else:
        for listing_obj in listing_objs: #progressbar(listing_objs):
            soap = BeautifulSoup(listing_obj.response_text, "html.parser")

    start1 = time.time()
    print("time consumed: ", argtxt, start1 - start0)

Output shows that cca 86 % of time (100 * 32.761232137680054 / 38.00445818901062) is consumed by converting original html to BeautifulSoup format:

D:\Python\CR\234876.py

checking time:  parse_listings_from_pkl() 444 records
Validate listing responses
Parse financials and details for listings
Perform listing calculations
time consumed:  parse_listings_from_pkl() 38.00445818901062

D:\Python\CR\234876.py 1

checking time:  BeautifulSoup 444 records
time consumed:  BeautifulSoup 32.761232137680054

Although there are some optimizable parts in the rest of pure python code (and I tried them with only minor performance improvements), I found that the BeautifulSoup conversion time corresponds to original html size and there is most of gubbins of no use inside the analyzed html.

Hence, I'd try cutting the listing_obj.response_text into pieces of useful parts and convert merely those parts to <class 'bs4.BeautifulSoup'> type. Maybe Speeding up beautifulsoup or Simple HTML and XHTML parser could help extracting useful info from the original html?