# Recursively scrape links from web pages and check them

I'm new to programming and especially new to object oriented programming. I have built a web scraper using functional programming and am trying to build another using OOP principles.

The overall idea for this scraper is I give it a url, it searches and retrieves all urls on that page, stores them in a set and as it checks each url, it removes it from the set. In terms of the "Check", I want to see if a page contains a set of identifiers. If it does, I'll download it. If it doesn't, I'll pass.

I've got 3 objects:

1. domainObject - should hold identifiers and init_link against domain
3. pageObject - takes a url, looks for identifiers, downloads if meets criteria

I am mostly confused by how 2 and 3 interact and would like some feedback on how to improve feeding back the link that has been checked by pageObject back into linkManager's "link to skip" pool.

# -*- coding: utf-8 -*-
import re
import time
import random
import sqlite3
import requests
from bs4 import BeautifulSoup as bs

def soup(url):
try:
return bs((requests.get(url)).text, "html.parser")
except:
alt_url = "https://" + url
return bs((requests.get(alt_url)).text, "html.parser")

class domainObject:
"""Stores "Domain level" info - init_link, "page of interest identifiers", domain name, object name"""

def __init__(self, include_exclude):

self.name = "en.wikipedia.org"
self.include_exclude = include_exclude[0].append(str(self.name + ".*"))

""" Manages a pool of visited and non-visited links as the """

def __init__(self, domObj):

self.inclusions = include_exclude[0]

print url

### collect the hrefs in the given url and add it to pool ###

for link in soup(url).find_all('a', href=True):                 # find all hrefs in url
for inclusion in self.inclusions:                           # those that match the "inlude list" (currently just the domain)

### pass each link to pageObject ###

time.sleep(random.uniform(0.1, 1.0))                        # rest for a bit

class pageObject:
""" PageObject generated for every link provided. Checks it's own link and determines if it's a product page. If it is, it's downloaded. """

def __init__(self, url):
self.url = url

def check_page_type(self, ManObj):
# Else, pass.

if __name__ == '__main__':

include_exclude = [[],[]]                                           # will later have a list of items to include and exclude

domObj = domainObject(include_exclude)

domObj = domainObject(include_exclude)