I created a project for academic purposes that works on Twitter data. It should:
- Get all friends and follows of the user
- Store them in MongoDB
- Display them in a table
I'm using django + plain HTML, but when trying on an account with only 230 contacts it took 3 minutes to load. The application is intended to be scalable to handle accounts with at least 100k contacts.
How can I improve the scalability of this program?
from twitter_auth.apiKeys import *
import tweepy
from manager.models import Contact
def get_api(user):
access_tokens = user.social_auth \
.filter(provider='twitter')[0] \
.extra_data['access_token']
auth = tweepy.OAuthHandler(
SOCIAL_AUTH_TWITTER_KEY,
SOCIAL_AUTH_TWITTER_SECRET
)
auth.set_access_token(
access_tokens['oauth_token'],
access_tokens['oauth_token_secret']
)
api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)
return api
def get_user_contacts(user):
# Setup Twitter API instance of logged User
api = get_api(user)
me = api.me()
# Set number of contacts fetched per call Max:200
count = 200
# Fetch logged User's friends into a list
friends = tweepy.Cursor(api.friends, me.screen_name, count=count).items()
# friends = api.friends(me.screen_name, count=10)
friend_list = []
for i in friends:
friend_list.append(i)
# Fetch logged User's followers into a list
followers = tweepy.Cursor(api.followers, me.screen_name, count=count).items()
# followers = api.followers(me.screen_name, count=10)
follower_list = []
for j in followers:
follower_list.append(j)
existing_contacts = Contact.objects.filter(user=user)
for contact in existing_contacts:
if contact not in friend_list and contact not in follower_list:
contact.delete()
# Start iterating friend list fetched from Twitter API
for friend in friend_list:
# Initialize Contact Object
temp_friend = Contact(
twitter_id=friend.id,
profile_image_url=friend.profile_image_url_https,
screen_name=friend.screen_name,
name=friend.name,
followers_count=friend.followers_count,
friends_count=friend.friends_count,
statuses_count=friend.statuses_count,
description=friend.description,
location=friend.location,
friendship_status=1,
protected_status=friend.protected,
user=user
)
# Check if entry is just Friend (1) or Friend & Follower (3)
if friend in follower_list:
temp_friend.friendship_status = 3
# Check if current friend already exists in DB
existing_friend = Contact.objects.filter(screen_name=friend.screen_name, user=user)
if existing_friend:
# Update Contact info
existing_friend.update(
profile_image_url=friend.profile_image_url_https,
name=friend.name,
followers_count=friend.followers_count,
friends_count=friend.friends_count,
statuses_count=friend.statuses_count,
description=friend.description,
location=friend.location,
protected_status=friend.protected,
)
# Check if existing Contact followed back
# Case: Have followed back so friendship status updates to (3)
if existing_friend in follower_list:
existing_friend.update(
friendship_status=3
)
# Case: Have not followed back so friendship status remains unchanged(1)
else:
temp_friend.save()
# Start iterating follower list fetched from Twitter API
for follower in follower_list:
# Initialize Contact Object
temp_follower = Contact(
twitter_id=follower.id,
profile_image_url=follower.profile_image_url_https,
screen_name=follower.screen_name,
name=follower.name,
followers_count=follower.followers_count,
friends_count=follower.friends_count,
statuses_count=follower.statuses_count,
description=follower.description,
location=follower.location,
friendship_status=2,
protected_status=follower.protected,
user=user
)
# Check if current follower already exists in DB
existing_follower = Contact.objects.filter(twitter_id=follower.id, user=user)
if existing_follower:
# Update Contact info
existing_follower.update(
profile_image_url=follower.profile_image_url_https,
name=follower.name,
followers_count=follower.followers_count,
friends_count=follower.friends_count,
statuses_count=follower.statuses_count,
description=follower.description,
location=follower.location,
protected_status=follower.protected,
)
# Check if user followed back existing the existing follower
# Case: Have followed back so friendship status updates to (3)
if existing_follower in friend_list:
existing_follower.update(
friendship_status=3
)
# Case: Have not followed back so friendship status remains unchanged(2)
else:
temp_follower.save()
def get_user_tweets(user, id):
api = get_api(user)
tweet_list = api.user_timeline(id=id, count=2)
return tweet_list