1
\$\begingroup\$

Is there any way to lower the lines of this code and make it better in terms of being more Pythonic? The program collects data from truecar.com and then puts them into my database.

Any help would be highly appreciated.

Code

import mysql.connector
import requests
import math
 

# takes the car's name
requested_car_name = input('Enter car name: ')

# injects the car's name into the URL

url = 'https://www.truecar.com/abp/api/vehicles/used/listings'
payload = {
    'city': 'holtsville',
    'collapse': 'true',
    'fallback': 'true',
    'include_incentives': 'true',
    'include_targeted_incentives': 'true',
    'make_slug': requested_car_name,
    'new_or_used': 'u',
    'per_page': '30',
    'postal_code': '',
    'search_event': 'true',
    'sort[]': 'best_match',
    'sponsored': 'true',
    'state': 'ny',
    'page': '1'}


jsonData = requests.get(url, params=payload).json()
total = jsonData['total']
total_pages = math.ceil(total/30)

total_pages_input = input(
    'There are %s pages to iterate.\nEnter the number of pages to go through or type ALL: ' % total_pages)
if total_pages_input.upper() == 'ALL':
    total_pages = total_pages
else:
    total_pages = int(total_pages_input)

values = []
for page in range(1, total_pages+1):
    if page == 1:
        car_listings = jsonData['listings']
    else:
        payload.update({'page': '%s' % page})
        jsonData = requests.get(url, params=payload).json()
        car_listings = jsonData['listings']

    for listing in car_listings:
        vehicle = listing['vehicle']
        ex_color = vehicle['exterior_color']
        in_color = vehicle['interior_color']
        location = vehicle['location']
        price = vehicle['list_price']
        make = vehicle['make']
        model = vehicle['model']
        mileage = vehicle['mileage']
        style = vehicle['style']
        year = vehicle['year']
        engine = vehicle['engine']
        accidentCount = vehicle['condition_history']['accidentCount']
        ownerCount = vehicle['condition_history']['ownerCount']
        isCleanTitle = vehicle['condition_history']['titleInfo']['isCleanTitle']
        isFrameDamaged = vehicle['condition_history']['titleInfo']['isFrameDamaged']
        isLemon = vehicle['condition_history']['titleInfo']['isLemon']
        isSalvage = vehicle['condition_history']['titleInfo']['isSalvage']
        isTheftRecovered = vehicle['condition_history']['titleInfo']['isTheftRecovered']

        values.append((ex_color, in_color, location, price, make, model, mileage,
                       style, year, engine, accidentCount, ownerCount, isCleanTitle, isFrameDamaged,
                       isLemon, isSalvage, isTheftRecovered))
    print('Completed: Page %s of %s' % (page, total_pages))


# checks the connection
print('CONNECTING ...')

mydb = mysql.connector.connect(
    host="xxx",
    user="xxx",
    password="xxx",
    port='xxx',
    database='xxx'
)

print('CONNECTED')

# checking the connection is done

my_cursor = mydb.cursor(buffered=True)
# create_command = ''' create table car_information (exterior_color varchar(255), interior_color varchar(255),location varchar(255),price varchar(255),make varchar(255),model varchar(255),mileage varchar(255),
#         style varchar(255),year varchar(255),engine varchar(255),accidentCount varchar(255),ownerCount varchar(255),isCleanTitle varchar(255),isFrameDamaged varchar(255),
#         isLemon varchar(255), isSalvage varchar(255),isTheftRecovered varchar(255))'''

# my_cursor.execute(create_command)
# print('created')
insert_command = '''INSERT INTO car_information (exterior_color, interior_color,location,price,make,model,mileage,
        style,year,engine,accidentCount,ownerCount,isCleanTitle,isFrameDamaged,
        isLemon, isSalvage,isTheftRecovered) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)'''
my_cursor.executemany(insert_command, values)
mydb.commit()

print(my_cursor.rowcount, "Record Inserted")

mydb.close()
\$\endgroup\$
1
  • 1
    \$\begingroup\$ This is not scraping, since you're hitting a JSON API. \$\endgroup\$
    – Reinderien
    Apr 16, 2021 at 1:51

1 Answer 1

3
\$\begingroup\$
  • Write functions; currently you don't have any
  • requested_car_name name is deceiving (along with the Enter car name prompt), because this is actually being used for the car make parameter
  • The fetch/prompt/fetch logic to ask for the number of pages is a little awkward. Instead, just ask for a page limit up front.
  • Avoid the if page == 1 / else block, and simply loop over the pages.
  • Separate your HTTP fetch, depagination, and data-shaping methods
  • Use a Requests session to persist cookies, apply headers and ensure that connection reuse is being performed
  • Do not use a tuple for your prepared statement; use a dictionary and %(keyword)s syntax to avoid tuple misalignment errors
  • Omit your DDL (create table) from your application source

The following suggested source does successfully run, but doesn't use a database at all; that part is up to you.

from itertools import count
from typing import Any, Dict, Optional, Iterable, Tuple

import math

from requests import Session


BASE_PARAMS = {
    'city': 'denver',
    'collapse': 'true',
    'fallback': 'true',
    'include_incentives': 'true',
    'include_targeted_incentives': 'true',
    'new_or_used': 'u',
    'per_page': 30,
    'postal_code': '80210',
    'search_event': 'true',
    'sort[]': 'best_match',
    'sponsored': 'true',
    'state': 'ny',
}


def get_truecar(session: Session, make_name: str, page: int) -> Dict[str, Any]:
    with session.get(
        'https://www.truecar.com/abp/api/vehicles/used/listings',
        params={
            **BASE_PARAMS,
            'make_slug': make_name.lower(),
            'page': page,
        },
    ) as response:
        response.raise_for_status()
        return response.json()


def depaginate_truecar(
    session: Session, make_name: str, max_pages: Optional[int],
) -> Iterable[Tuple[Dict[str, Any], int, int]]:
    if max_pages is None:
        pages = count(1)
    else:
        pages = range(1, max_pages + 1)

    for page in pages:
        doc = get_truecar(session, make_name, page=page)
        total = doc.get('total')
        per_page = int(doc['per_page'])

        if total is None:
            n_pages = 1
        else:
            n_pages = math.ceil(total / per_page)
            if max_pages is not None:
                n_pages = min(max_pages, n_pages)

        yield doc, page, n_pages

        if total is None or page >= n_pages:
            break


def reshape(doc: Dict[str, Any]) -> Iterable[Dict[str, Any]]:
    for listing in doc['listings']:
        vehicle = listing['vehicle']

        yield {
            'ex_color': vehicle['exterior_color'],
            'in_color': vehicle['interior_color'],
            'location': vehicle['location'],
            'price': vehicle['list_price'],
            'make': vehicle['make'],
            'model': vehicle['model'],
            'mileage': vehicle['mileage'],
            'style': vehicle['style'],
            'year': vehicle['year'],
            'engine': vehicle['engine'],
            'accidentCount': vehicle['condition_history']['accidentCount'],
            'ownerCount': vehicle['condition_history']['ownerCount'],
            'isCleanTitle': vehicle['condition_history']['titleInfo']['isCleanTitle'],
            'isFrameDamaged': vehicle['condition_history']['titleInfo']['isFrameDamaged'],
            'isLemon': vehicle['condition_history']['titleInfo']['isLemon'],
            'isSalvage': vehicle['condition_history']['titleInfo']['isSalvage'],
            'isTheftRecovered': vehicle['condition_history']['titleInfo']['isTheftRecovered'],
        }


def main():
    car_name = input('Enter car name: ')
    max_pages = input('Enter page limit, or press enter for none: ')
    if max_pages == '':
        max_pages = None
    else:
        max_pages = int(max_pages)

    cars = []

    with Session() as session:
        session.headers = {
            'User-Agent': 'Mozilla/5.0',
            'Accept': 'application/json',
            'Referer': 'https://www.truecar.com/shop/used/?filterType=brand',
        }

        for doc, page, n_pages in depaginate_truecar(session, car_name, max_pages):
            print(f'{page}/{n_pages}')
            cars.extend(reshape(doc))


if __name__ == '__main__':
    main()
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.