Twitter contact scraper with Tweepy & Django

Question

I created a project for academic purposes that works on Twitter data. It should:

• Get all friends and follows of the user
• Store them in MongoDB
• Display them in a table

I'm using django + plain HTML, but when trying on an account with only 230 contacts it took 3 minutes to load. The application is intended to be scalable to handle accounts with at least 100k contacts.

How can I improve the scalability of this program?

from twitter_auth.apiKeys import *
import tweepy
from manager.models import Contact


def get_api(user):
    access_tokens = user.social_auth \
        .filter(provider='twitter')[0] \
        .extra_data['access_token']

    auth = tweepy.OAuthHandler(
        SOCIAL_AUTH_TWITTER_KEY,
        SOCIAL_AUTH_TWITTER_SECRET
    )

    auth.set_access_token(
        access_tokens['oauth_token'],
        access_tokens['oauth_token_secret']
    )

    api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)

    return api


def get_user_contacts(user):
    # Setup Twitter API instance of logged User
    api = get_api(user)
    me = api.me()

    # Set number of contacts fetched per call Max:200
    count = 200

    # Fetch logged User's friends into a list
    friends = tweepy.Cursor(api.friends, me.screen_name, count=count).items()
    # friends = api.friends(me.screen_name, count=10)
    friend_list = []
    for i in friends:
        friend_list.append(i)

    # Fetch logged User's followers into a list
    followers = tweepy.Cursor(api.followers, me.screen_name, count=count).items()
    # followers = api.followers(me.screen_name, count=10)
    follower_list = []
    for j in followers:
        follower_list.append(j)

    existing_contacts = Contact.objects.filter(user=user)
    for contact in existing_contacts:
        if contact not in friend_list and contact not in follower_list:
            contact.delete()

    # Start iterating friend list fetched from Twitter API
    for friend in friend_list:
        # Initialize Contact Object
        temp_friend = Contact(
            twitter_id=friend.id,
            profile_image_url=friend.profile_image_url_https,
            screen_name=friend.screen_name,
            name=friend.name,
            followers_count=friend.followers_count,
            friends_count=friend.friends_count,
            statuses_count=friend.statuses_count,
            description=friend.description,
            location=friend.location,
            friendship_status=1,
            protected_status=friend.protected,
            user=user
        )
        # Check if entry is just Friend (1) or Friend & Follower (3)
        if friend in follower_list:
            temp_friend.friendship_status = 3

        # Check if current friend already exists in DB
        existing_friend = Contact.objects.filter(screen_name=friend.screen_name, user=user)

        if existing_friend:
            # Update Contact info
            existing_friend.update(
                profile_image_url=friend.profile_image_url_https,
                name=friend.name,
                followers_count=friend.followers_count,
                friends_count=friend.friends_count,
                statuses_count=friend.statuses_count,
                description=friend.description,
                location=friend.location,
                protected_status=friend.protected,
            )
            # Check if existing Contact followed back
            # Case: Have followed back so friendship status updates to (3)
            if existing_friend in follower_list:
                existing_friend.update(
                    friendship_status=3
                )
            # Case: Have not followed back so friendship status remains unchanged(1)
        else:
            temp_friend.save()

    # Start iterating follower list fetched from Twitter API
    for follower in follower_list:
        # Initialize Contact Object
        temp_follower = Contact(
            twitter_id=follower.id,
            profile_image_url=follower.profile_image_url_https,
            screen_name=follower.screen_name,
            name=follower.name,
            followers_count=follower.followers_count,
            friends_count=follower.friends_count,
            statuses_count=follower.statuses_count,
            description=follower.description,
            location=follower.location,
            friendship_status=2,
            protected_status=follower.protected,
            user=user
        )

        # Check if current follower already exists in DB
        existing_follower = Contact.objects.filter(twitter_id=follower.id, user=user)
        if existing_follower:
            # Update Contact info
            existing_follower.update(
                profile_image_url=follower.profile_image_url_https,
                name=follower.name,
                followers_count=follower.followers_count,
                friends_count=follower.friends_count,
                statuses_count=follower.statuses_count,
                description=follower.description,
                location=follower.location,
                protected_status=follower.protected,
            )
            # Check if user followed back existing the existing follower
            # Case: Have followed back so friendship status updates to (3)
            if existing_follower in friend_list:
                existing_follower.update(
                    friendship_status=3
                )
            # Case: Have not followed back so friendship status remains unchanged(2)
        else:
            temp_follower.save()


def get_user_tweets(user, id):
    api = get_api(user)

    tweet_list = api.user_timeline(id=id, count=2)

    return tweet_list

Answer 1

Line continuations

They're possible but discouraged.

access_tokens = user.social_auth \
    .filter(provider='twitter')[0] \
    .extra_data['access_token']

would be better, according to most linters, as

access_tokens = (
    user.social_auth
    .filter(provider='twitter')[0]
    .extra_data['access_token']
)

List formation

friend_list = []
for i in friends:
    friend_list.append(i)

can just be

friend_list = list(friends)

However, given your usage:

for contact in existing_contacts:
    if contact not in friend_list and contact not in follower_list:
        contact.delete()

you're better off using sets:

friend_list = set(friends)
follower_list = set(followers)
existing_contacts = set(Contact.objects.filter(user=user))
to_delete = existing_contacts - friend_list - follower_list
for contact in to_delete:
    contact.delete()