This script is my web crawlerfor small sites. Can you possibly review it for best coding practices?
Code
import bs4 as bs
import urllib.request
from urllib.parse import urlparse,urljoin
import pprint
class Page(object):
def __init__(self,base_url,url):
self.url = url
self.base_url = base_url
def soup(self):
sauce = urllib.request.urlopen(self.url).read()
return (bs.BeautifulSoup(sauce,'lxml'))
def title(self):
soup = self.soup()
return soup.title.string
def links(self):
urls = []
soup = self.soup()
href = [i.get('href') for i in soup.findAll('a') ]
links = [i for i in (list(map((lambda url : url if bool(urlparse(url).netloc) == True else urljoin (self.base_url, url)),href))) if i.startswith(self.base_url)]
return links
def map_page(self):
map = {self.url:{'title':self.title(),'links':set(self.links())}}
return map
def site_map(base_url):
map_pages = {}
links_to_map = [base_url]
def check_and_add(url):
if url not in map_pages:
[links_to_map.append(i) for i in Page(base_url,url).links()]
(map_pages.update(Page(base_url,url).map_page()))
links_to_map.remove(url)
else:
links_to_map.remove(url)
while links_to_map != []:
url = links_to_map[0]
check_and_add(url)
pprint.pprint(map_pages)