So I want to do some research on the impact COVID-19 is having on businesses. I've managed to generate a database with the company name and the website URLs associated with it. Now I want to scrape them all as fast as possible so I can do some analysis on it. I am new to using parallel programming and am skeptical I am connecting to each database as safely as possible.

from __future__ import division

from multiprocessing import Pool

import pymongo as pym
import requests
from bs4 import BeautifulSoup

# Set up local client
client = pym.MongoClient('mongodb://localhost:27017/')
# Connect to local DB
db = client.local_db
# Connect to Collections
My_Collection = db.MyCollection
ScrapedPagesAprilCollection = db.ScrapedPagesApril

# I don't want to scrape these
LIST_OF_DOMAINS_TO_IGNORE = ['google.com/', 'linkedin.com/', 'facebook.com/']

def parse(url):
    if any(domain in url for domain in LIST_OF_DOMAINS_TO_IGNORE):
    elif '.pdf' in url:
        # print(url)
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:74.0) Gecko/20100101 Firefox/74.0',
        page = requests.get(url, headers=headers)
        print(f'{url}: {page.status_code}')
        if page.status_code == 200:
            soup = BeautifulSoup(page.text, 'lxml')
            text = soup.get_text(separator=" ")
            info_to_store = {
                '_id': url,
                'content': text

            if 'coronavirus' in text:
                info_to_store['Impacted'] = True

            # Insert into Collection
                {'_id': url}, info_to_store, upsert=True)

        elif page.status_code != 200:
            print(f'{url}: {str(page.status_code)}')

def covid19_scrape_pages(collection, query: dict):
    Wanting to update the pages already matched

    collection : pymongo.collection.Collection
    query : dict

    A url

    # Get the cursor
    mongo_cursor = collection.find(query, no_cursor_timeout=True)
    # For company in the cursor, yield the urls
    for company in mongo_cursor:
        for url in company['URLs']:
            doc = ScrapedPagesAprilCollection.find_one({'_id': url})
            # If I haven't already scraped it, then yield the url
            if doc is None:
                yield (url)

def main():
    print('Make sure LIST_OF_DOMAINS_TO_IGNORE is updated by running',
          'blacklisted_domains.py first')
    urls_gen = covid19_scrape_pages(
        My_Collection, {})
    pool = Pool(8)
    pool.map(parse, urls_gen)

if __name__ == "__main__":  # Required logic expression
  • \$\begingroup\$ Welcome to Code Review! What kind of safety are you looking for, data integrity? \$\endgroup\$
    – Mast
    Apr 16, 2020 at 18:43
  • \$\begingroup\$ Thank you. Yes that would be great. I also want to ensure speed. The general idea is that I am yielding urls from database, A (My_Collection), scraping them and inputting them into database B (ScrapedPagesAprilCollection) \$\endgroup\$
    – Jack
    Apr 16, 2020 at 18:48


Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge that you have read and understand our privacy policy and code of conduct.