Some background info here:
- This is a small fun project I made utilizing Steam APIs and web-scraping
- This is the first time I've ever used Python, so I'm not very familiar with the language
- I used Flask in conjunction with this code (google "Flask" if you don't know what I'm talking about)
- The purpose of the code is determine whatever Steam user is a phisher, and I determined that using my observations/research, and this is the algorithm I developed.
The code is very inefficient as it takes 3-4 minutes to complete. I'm asking if the code can be improved so that it's more efficient and it runs faster, because I want to accomplish more with this script but with the original 3-4 minute run time that may make the code horribly slow and inefficient. (NOTE: I don't want to change the function of the code, just minor or major things that can make the code run faster).
import urllib
import itertools
import urllib2
import time
from datetime import datetime
from bs4 import BeautifulSoup
from flask import Flask
import requests
import json
import xml.etree.ElementTree as ET
from xml.dom.minidom import parseString
app = Flask(__name__)
API_KEY = 'XXX'
steamIDs = []
myFriends = []
class steamUser:
def __init__(self, name, steamid, isPhisher):
self.name = name
self.steamid = steamid
self.isPhisher = isPhisher
def setPhisherStatus(self, phisher):
self.isPhisher = phisher
@app.route('/DeterminePhisher/<steamid>')
def getFriendList(steamid):
r = requests.get('http://api.steampowered.com/ISteamUser/GetFriendList/v0001/?key='+API_KEY+'&steamid='+steamid+'&relationship=all')
data = r.json()
for i in range(len(data['friendslist']['friends'])):
steamIDs.append(data['friendslist']['friends'][i]['steamid'])
return isPhisher(steamIDs)
def isPhisher(ids):
for l in chunksiter(ids, 50):
sids = ','.join(map(str, l))
r = requests.get('http://api.steampowered.com/ISteamUser/GetPlayerSummaries/v0002/?key='+API_KEY+'&steamids=' + sids)
data = r.json();
for i in range(len(data['response']['players'])):
steamFriend = data['response']['players'][i]
n = steamUser(steamFriend['personaname'], steamFriend['steamid'], False)
if((('%s' % steamFriend['communityvisibilitystate']) == '1') and (('%s' % steamFriend['personastate']) == '0')):
#checks if user's profile is private and is offline
#communityvisibilitystate=1 could also mean that the user has set his/her profile to friends only. These people are not phishers.
url = 'http://steamcommunity.com/profiles/{0}?xml=1'.format(steamFriend['steamid'])
dat = requests.get(url)
if 'profilestate' not in steamFriend:
n.setPhisherStatus(True);
if len(parseString(dat.text.encode('utf-8')).getElementsByTagName('privacyState')) > 0:
privacy = str(parseString(dat.text.encode('utf-8')).getElementsByTagName('privacyState')[0].firstChild.wholeText)
else:
privacy = ''
if (privacy == 'private'):
n.setPhisherStatus(True)
elif 'profilestate' not in steamFriend:
n.setPhisherStatus(True);
else:
#at this point the user may still be a phisher, and is trying to look like a normal steam user. If the user hasn't been reported and the account removed, we can then look at the public profile comments to determine if the user is a phisher or not.
steamprofile = BeautifulSoup(urllib.urlopen('http://steamcommunity.com/profiles/'+steamFriend['steamid']).read())
for row in steamprofile('div', {'class': 'commentthread_comment '}):
comment = row.find_all('div', 'commentthread_comment_text')[0].get_text().lower()
if ('phisher' in comment) or ('scammer' in comment):
n.setPhisherStatus(True)
myFriends.append(n);
regularusers = ''
phisherusers = ''
for j in myFriends:
if(j.isPhisher == True):
phisherusers = ((phisherusers) + ('%s is a phisher\n' % j.name) + ', ')
else:
regularusers = ((regularusers) + ('%s is a regular user' % j.name) + ', ')
return phisherusers + ' ' + regularusers
def chunksiter(l, chunks):
i,j,n = 0,0,0
rl = []
while n < len(l)/chunks:
rl.append(l[i:j+chunks])
i+=chunks
j+=j+chunks
n+=1
return iter(rl)
app.run(debug=True)
requests.get, the bottleneck is the Steam server. \$\endgroup\$