I started learning Python recently and I really like it, so I decided to share one of my first projects mainly in hopes of someone telling me what I can do to make it run faster (threading/multiprocessing?).
from requests import get
from bs4 import BeautifulSoup
from time import time
from re import compile
print('***PYTHON LEAGUE OF LEGENDS USERNAME SCRAPER***')
print('This script scrapes usernames from lolprofile.net')
region = input('Enter the region for scraping(eune/euw/na/br/tr/kr/jp/lan/las/oce/ru)\n')
numStart = input('What page to start on? Min 0\n')
numEnd = input('What page to end on? Min starting page + 1\n')
size = [] #for logging
#count = -1 #for logging
def setUrl(pageNum, region):
global url
url = 'http://lolprofile.net/leaderboards/'+region+'/'+pageNum
def is_ascii(i):
return all(ord(c) < 128 for c in i)
setUrl(numStart, region)
start = time()
while int(numStart) != int(numEnd):
print(len(size))
page = get(url)
soup = BeautifulSoup(page.text, "lxml")
userName = [a.string for a in soup.findAll(href=compile('http://lolprofile.net/summoner/*'))]
with open('usernames1.txt', 'a') as file:
for i in userName:
if is_ascii(i) and (' ' in i) == False:
file.write('%s\n' % i.lower())
size.append('0')
numStart = int(numStart)
numStart += 1
setUrl(str(numStart), region)
#count += 1
#if count % 250 == 0: #every n iterations print progress
# print(len(size))
end = time()
print(len(size),'usernames scraped in a total of',end-start,'seconds')