I recently started to work with python and came to idea to automate process of looking for new jobs. There is nice website and I managed to get content, because they use angular to render webpage and using just requests
gives no results, because page is rendered using javascript.
Below code is functional, but a bit slow, it take about 10-15 seconds to get rendered page. Full process include going through all job ads, get title,description,link, etc, and check local database to see whether this ad already exists. If not, new mail is sent through mailgun. Any suggestion is welcomed.
#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys
from PyQt4.QtGui import *
from PyQt4.QtCore import *
from PyQt4.QtWebKit import *
from bs4 import BeautifulSoup
import pdfkit
import requests
from db import DB
# Variables
base_url = "https://www.helloworld.rs"
links = []
def send_mail(title, body, attach):
return requests.post(
"https://api.mailgun.net/v3/MY_URL/messages",
auth=("api", "MY-API-KEY"),
files=[("attachment", open(attach))],
data={"from": 'Job Bot <my@mail>',
"to": ["another@mail"],
"subject": "New job: " + title,
"text": body})
class Render(QWebPage):
def __init__(self, url):
self.app = QApplication(sys.argv)
QWebPage.__init__(self)
self.loadFinished.connect(self._loadFinished)
self.mainFrame().load(QUrl(url))
self.app.exec_()
def _loadFinished(self, result):
self.frame = self.mainFrame()
self.app.quit()
# full url of jobs of interests
url = 'https://www.helloworld.rs/oglasi-za-posao/#/page=0&tag=&cat=&show_more=0&senioritet=1&vreme_postavljanja=&rok_konkursa=&jezik='
try:
r = Render(url)
except:
print "[!] Connection error!"
exit(0)
result = r.frame.toHtml()
formatted_result = str(result.toAscii())
bs = BeautifulSoup(formatted_result, 'lxml')
jobs = []
for job in bs.find_all('div', class_="job-item"):
_company = job.find('strong')
try:
company = _company.a.text
except:
company = _company.text
company = company.replace('\n', '')
company = company.replace('\r', '')
company = company.replace('\t', '')
title = job.h3.a.string
description = job.find('p', class_="description").contents[0]
link = base_url + job.find('a', class_="job-link")["href"]
jobs.append({"title": title, "description": description, "company": company, "link": link})
db = DB(db="DATABASE-NAME")
#get all links from db to check if new result exists
db_links = db.execute("SELECT link FROM jobs")
for u in db_links.fetchall():
links.append(u[0])
for job in jobs:
job_title = job['title'].encode("utf-8")
job_company = job['company'].encode("utf-8")
job_description = job['description'].encode("utf-8")
job_link = job['link']
if job_link not in links:
db.execute(
"INSERT INTO jobs (title,description,company,link) VALUES ('{}','{}','{}','{}','{}')".format(
job_title,
job_description,
job_company,
job_link))
mail_body = "{}\n\n{}".format(job_description, job_link)
print "[+] New ad, sending mail."
generated_pdf_location = "/tmp/" + job_title + ".pdf"
pdfkit.from_url(job_link, generated_pdf_location)
print send_mail(job_title, mail_body, generated_pdf_location)
else:
print "[-] Already exists, skipping."