0
\$\begingroup\$

Standard code review, tell me what's good, what's bad, and how to improve. Critical suggestions welcomed. This is a content aggregator using bs4 and requests. I did not use any tutorials or help.

import requests
from bs4 import BeautifulSoup as bs

topic = input('Enter the topic: ').lower()
print()
def getdata(url, headers):
    r = requests.get(url, headers=headers)
    return r.text

def linesplit():
    print('-~'*16, 'COLIN IS MOOCH', '-~'*16, '\n')
headers = {
    'User-agent':
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.19582"
}
google = getdata(f'https://www.google.com/search?q={topic}&oq={topic}&aqs=chrome..69i59j69i57j69i59j69i60l3j69i65j69i60.2666j0j7&sourceid=chrome&ie=UTF-8', headers)
soup = bs(google, 'html.parser')
links = str(soup.find_all('div', class_='TbwUpd NJjxre'))
links = links.replace('<div class="TbwUpd NJjxre"><cite class="iUh30 Zu0yb qLRx3b tjvcx">', '')
links = links.replace('<span class="dyjrff qzEoUe">', '')
links = links.replace('</span></cite>< /div>', '')
links = links.replace('<div class="TbwUpd NJjxre"><cite class="iUh30 Zu0yb tjvcx">', '')
links = links.replace('</cite></div>', '')
links = links.replace('</span>', '')
links = links.replace(' › ', '/')
links = links.split(', ')
links[-1] = links[-1].replace(']', '')
links[0] = links[0].replace('[', '')
info = []
counter = 0

for x in range(len(links)):
    try:
        htmldata = getdata(links[x], headers)
        newsoup = bs(htmldata, 'html.parser')
        website = ''
        for i in newsoup.find_all('p'):
            website = website + ' ' + i.text
        info.append(links[x])
        info.append(website)
        counter += 1
    except Exception:
        continue
try:
    for x in range(0, (counter * 2) + 2, 2):
        if info[x+1] != '':
            linesplit()
            print()
            print('From ', info[x], ':')
            print()
            print(info[x+1])
            linesplit()
except IndexError:
    pass
\$\endgroup\$
2
  • \$\begingroup\$ Why not use the API? \$\endgroup\$ – Reinderien May 6 at 21:54
  • \$\begingroup\$ Which API? I didn't know there was one. \$\endgroup\$ – user May 6 at 22:54
2
\$\begingroup\$

Your linesplit is... strange? I'm going to ignore that message and pretend it makes sense.

Whenever there's an API, prefer it. In this case Google has a Custom Search API that allows you to skip past all of the scraping insanity. You'll need to get an API key and configure an engine instance to search the whole web. The front-end of such an application, without including your second "scrape all of the paragraphs" step, looks like:

from typing import Iterable
from requests import Session


# To set a custom search engine to search the entire web, read
# https://support.google.com/programmable-search/answer/4513886
API_KEY = '...'
ENGINE_ID = '...'


def api_get(session: Session, query: str) -> Iterable[str]:
    with session.get(
        'https://customsearch.googleapis.com/customsearch/v1',
        headers={'Accept': 'application/json'},
        params={
            'key': API_KEY,
            'cx': ENGINE_ID,
            'q': query,
        }
    ) as resp:
        resp.raise_for_status()
        body = resp.json()

    for item in body['items']:
        yield item['link']


def main():
    topic = input('Enter the topic: ').lower()

    with Session() as session:
        urls = api_get(session, topic)

    print('\n'.join(urls))


if __name__ == '__main__':
    main()
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.