So I want to do some research on the impact COVID-19 is having on businesses. I've managed to generate a database with the company name and the website URLs associated with it. Now I want to scrape them all as fast as possible so I can do some analysis on it. I am new to using parallel programming and am skeptical I am connecting to each database as safely as possible.
from __future__ import division
from multiprocessing import Pool
import pymongo as pym
import requests
from bs4 import BeautifulSoup
# Set up local client
client = pym.MongoClient('mongodb://localhost:27017/')
# Connect to local DB
db = client.local_db
# Connect to Collections
My_Collection = db.MyCollection
ScrapedPagesAprilCollection = db.ScrapedPagesApril
# I don't want to scrape these
LIST_OF_DOMAINS_TO_IGNORE = ['google.com/', 'linkedin.com/', 'facebook.com/']
def parse(url):
if any(domain in url for domain in LIST_OF_DOMAINS_TO_IGNORE):
pass
elif '.pdf' in url:
pass
else:
# print(url)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:74.0) Gecko/20100101 Firefox/74.0',
}
page = requests.get(url, headers=headers)
print(f'{url}: {page.status_code}')
if page.status_code == 200:
soup = BeautifulSoup(page.text, 'lxml')
text = soup.get_text(separator=" ")
info_to_store = {
'_id': url,
'content': text
}
if 'coronavirus' in text:
info_to_store['Impacted'] = True
# Insert into Collection
ScrapedPagesAprilCollection.replace_one(
{'_id': url}, info_to_store, upsert=True)
elif page.status_code != 200:
print(f'{url}: {str(page.status_code)}')
pass
def covid19_scrape_pages(collection, query: dict):
"""
Wanting to update the pages already matched
Parameters
----------
collection : pymongo.collection.Collection
query : dict
Yields
-------
A url
"""
# Get the cursor
mongo_cursor = collection.find(query, no_cursor_timeout=True)
# For company in the cursor, yield the urls
for company in mongo_cursor:
for url in company['URLs']:
doc = ScrapedPagesAprilCollection.find_one({'_id': url})
# If I haven't already scraped it, then yield the url
if doc is None:
yield (url)
def main():
print('Make sure LIST_OF_DOMAINS_TO_IGNORE is updated by running',
'blacklisted_domains.py first')
urls_gen = covid19_scrape_pages(
My_Collection, {})
pool = Pool(8)
pool.map(parse, urls_gen)
pool.close()
pool.join()
if __name__ == "__main__": # Required logic expression
main()
