Script that saving most relevant Youtube video for each database object

Question

Due to the tightened Youtube API daily quota (that allows to use only ~ 98 search requests per project daily) I've implemented a multi-project system. For this to work you need to create a project/projects, download JSON with client secrets and put it in the credentials directory in the script's folder.

I'm aware that some lines > than 79 symbols, one line even 102 symbols. Just wasn't sure I've needed to adjust every line to PEP recommendation in this case.

import datetime
import json
import os
import random
import webbrowser
from itertools import chain
from pathlib import Path

import furl
import googleapiclient.discovery
import googleapiclient.errors
from dateutil.relativedelta import relativedelta
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from googleapiclient.errors import HttpError
from oauth2client import client
from oauth2client.file import Storage
from tqdm import tqdm

from players.models import Goalie, Skater

API_SERVICE_NAME = 'youtube'
API_VERSION = 'v3'
SCOPES = ['https://www.googleapis.com/auth/youtube.force-ssl']
BASE_YT_URL = 'https://www.youtube.com/embed/'
SEARCH_PARAMS = {
    'part': 'snippet',
    'content_type': 'video',
    'fields': 'items/id/videoId',
    'relevance_language': 'en',
    'region_code': 'us',
    'video_embeddable': 'true',
    'max_results': 1,
}
CREDENTIALS_FOLDER = 'credentials'
CLIENT_SECRETS_FILES = [
    'client_secret-1.json',
    'client_secret-2.json',
    'client_secret-3.json',
    'client_secret-4.json',
    'client_secret-5.json',
    ]
CREDENTIALS_FILES = [
    'credentials-1.json',
    'credentials-2.json',
    'credentials-3.json',
    'credentials-4.json',
    'credentials-5.json',
    ]
DAILY_LIMIT = ['dailyLimitExceeded', 'quotaExceeded']
REDIRECT_URI = 'http://127.0.0.1:8000/auth_callback/'
TIME_AGO = {
    'months': 6,
}
LINK_UPDATE_TIME = 30
ZULU_TIMEZONE = 'Z'


class Command(BaseCommand):
    """ """

    def handle(self, *args, **options):
        players = list(chain(Goalie.objects.all(), Skater.objects.all()))
        for player in tqdm(players):
            if link_old_or_missing(player):
                updated = False
                while CREDENTIALS_FILES and not updated:
                    link = get_link(player.name)
                    if link:
                        player.relevant_video = link
                        player.video_link_updated_at = datetime.datetime.now()
                        player.save(update_fields=['relevant_video', 'video_link_updated_at'])
                        updated = True
                        print(f'{player.name} link is updated!')
                    else:
                        print(f'{player.name} link could not be updated with these credentials!')
            else:
                print(f'{player.name} link is up do date!')


def link_old_or_missing(player):
    if player.relevant_video:
        return((datetime.date.today() - player.video_link_updated_at).days > LINK_UPDATE_TIME)
    else:
        return True


def get_storage(*index):
    if index:
        credentials_file = CREDENTIALS_FILES[index[0]]
    else:
        credentials_file = random.choice(CREDENTIALS_FILES)
        index = CREDENTIALS_FILES.index(credentials_file)

    credential_path = os.path.join(get_current_folder(), CREDENTIALS_FOLDER, credentials_file)
    return (Storage(credential_path), index)


def credentials_not_valid(credentials):
    return not credentials or credentials.invalid


def get_current_folder():
    return Path(__file__).parent.absolute()


def get_new_credentials(index):
    secrets_file = os.path.join(get_current_folder(), CREDENTIALS_FOLDER, CLIENT_SECRETS_FILES[index])
    flow = client.flow_from_clientsecrets(secrets_file, SCOPES, redirect_uri=REDIRECT_URI)
    auth_url = flow.step1_get_authorize_url()
    webbrowser.open(auth_url)
    url = input('Insert URL with code:\n')
    code = furl.furl(url).args.get('code', '')
    return flow.step2_exchange(code)


def get_link(full_name):
    credentials, index = get_storage()
    credentials = credentials.get()
    if credentials_not_valid(credentials):
        credentials = get_new_credentials(index)
        get_storage(index)[0].put(credentials)

    youtube = googleapiclient.discovery.build(
        API_SERVICE_NAME, API_VERSION, credentials=credentials)

    request = youtube.search().list(
        part=SEARCH_PARAMS['part'],
        fields=SEARCH_PARAMS['fields'],
        q=full_name,
        type=SEARCH_PARAMS['content_type'],
        maxResults=SEARCH_PARAMS['max_results'],
        publishedAfter=published_after(),
        videoEmbeddable=SEARCH_PARAMS['video_embeddable'],
        relevanceLanguage=SEARCH_PARAMS['relevance_language'],
        regionCode=SEARCH_PARAMS['region_code'],
    )

    try:
        response = request.execute()
        print(BASE_YT_URL + response['items'][0]['id']['videoId'])
        print(f'{CREDENTIALS_FILES[index]} used successfullly!')
        return BASE_YT_URL + response['items'][0]['id']['videoId']
    except HttpError:
        data = json.loads(HttpError.content.decode('utf-8'))
        reason = data['error']['errors'][0]['reason']
        if reason in DAILY_LIMIT:
            print(f'{CREDENTIALS_FILES[index]} quota is exceeded!')
            del CREDENTIALS_FILES[index]
            del CLIENT_SECRETS_FILES[index]
        return None


def published_after():
    date_now = datetime.datetime.now()
    return (date_now - relativedelta(**TIME_AGO)).isoformat() + ZULU_TIMEZONE

Answer 1

Don't repeat yourself

This:

CLIENT_SECRETS_FILES = [
    'client_secret-1.json',
    'client_secret-2.json',
    'client_secret-3.json',
    'client_secret-4.json',
    'client_secret-5.json',
    ]
CREDENTIALS_FILES = [
    'credentials-1.json',
    'credentials-2.json',
    'credentials-3.json',
    'credentials-4.json',
    'credentials-5.json',
    ]

can be

CLIENT_SECRET_FILES = [f'client_secret-{i}' for i in range(1, 6)]
CREDENTIAL_FILES = [f'credentials-{i}' for i in range(1, 6)]

Also note that you shouldn't double-pluralize your variable names.

Empty comments

For this:

""" """

Fill it out, or delete it.

Generator mutability

This:

    players = list(chain(Goalie.objects.all(), Skater.objects.all()))
    for player in tqdm(players):

should not use list, because you never change the contents. Use tuple instead.

Just break

This:

            updated = False
            while CREDENTIALS_FILES and not updated:
                link = get_link(player.name)
                if link:
                    player.relevant_video = link
                    player.video_link_updated_at = datetime.datetime.now()
                    player.save(update_fields=['relevant_video', 'video_link_updated_at'])
                    updated = True

should not use a flag to break out of the loop. Instead, just issue break.

Do the simpler thing first

This:

if player.relevant_video:
    return((datetime.date.today() - player.video_link_updated_at).days > LINK_UPDATE_TIME)
else:
    return True

is more legible as:

if not player.relevant_video:
    return True

diff = datetime.date.today() - player.video_link_updated_at
return diff.days > LINK_UPDATE_TIME

Positive logic

This:

def credentials_not_valid(credentials):

can lend itself to confusing logic by the caller. Try to avoid negative predicates, and instead make this a positive predicate (credentials_valid).