Walk file system checking that backup files are present

Question

Everyday I manually check that that my systems were backed up. I use the windows file system through the Explorer gui and check that the backup files were modified on the previous day or later.

This script walks the file system backup directory and compares the files modified since the start of the previous day to a list I've prepared in the workfile.txt file.

There are a number of areas to address.

1. When building the path\filename strings should I break out the 'ifs' into their own functions?
2. When walking the file system I exclude directories by hard coding them in if statements. Should I check if a set of excluded directories is in the dirs list? I tried but I couldn't get it to work that way so I reverted to individual checks.
3. Would be better to parse the dates out of the path and file names rather then hard coding the places to insert dates with strings like [yesterday]?

backproj.py

import datetime
import os
from os.path import join, getsize
from colorama import Fore, Back, Style

TODAY = datetime.date.today()
YESTERDAY = TODAY - datetime.timedelta(days=1)
# TODO: Move to config or CLI argument
working_directory = 'W:\\SQLServer_Backups_Recent'


def get_raw_names_from_file(file):
    with open(file, 'r') as f:  # TODO: Try/Catch if file doesn't exist?
        return list(f)


def process_raw_file_names(raw_names):

    for file in raw_names:
        if ('<yesterday>') in file:
            yest_file = file.replace(
                '<yesterday>', '{d.month}-{d.day}-{d.year}-'.format(
                    d=YESTERDAY) + YESTERDAY.strftime('%a'))
        else:
            yest_file = file
        if ('_' + str(YESTERDAY.year)) in yest_file:
            nostamp_file = yest_file[:(yest_file.find(
                '_' + str(YESTERDAY.year)))] + '\n'
        else:
            nostamp_file = yest_file
        if ('<jira_yesterday>') in nostamp_file:
            clean_file = file.replace(
                '<jira_yesterday>', YESTERDAY.strftime('%Y-%b-%d'))
        elif ('<jira_today>') in nostamp_file:
            clean_file = file.replace('<jira_today>',
                                      TODAY.strftime('%Y-%b-%d'))
        else:
            clean_file = nostamp_file
        base_file_names.append(clean_file)

    return base_file_names



def walk_file_system(working_directory):
    current_files = []
    yesterday = datetime.date.today() - datetime.timedelta(days=1)
    for root, dirs, files in os.walk(working_directory):
        # TODO: Move all these directories to config, determine how to
        # filter dirs list
        if 'Marked' in dirs:
            dirs.remove('Marked')
        if 'BackupNonCritical' in dirs:
            dirs.remove('BackupNonCritical')
        if 'Attachments' in dirs:
            dirs.remove('Attachments')
        if 'x_GrouplinkBackupDB_20131121' in dirs:
            dirs.remove('x_GrouplinkBackupDB_20131121')
        if 'x_JIRA-LastOldJiraBackup' in dirs:
            dirs.remove('x_JIRA-LastOldJiraBackup')
        if 'x_PTS-DB' in dirs:
            dirs.remove('x_PTS-DB')
        if 'x_IvanBackups' in dirs:
            dirs.remove('x_IvanBackups')
        if 'x_PTS-Devel' in dirs:
            dirs.remove('x_PTS-Devel')
        if 'x_DBAPPS' in dirs:
            dirs.remove('x_DBAPPS')
        if 'x_pledgerep' in dirs:
            dirs.remove('x_pledgerep')
        file_list = (join(root, name) for name in files)
        for item in file_list:
            mtime = datetime.datetime.fromtimestamp(
                int(os.stat(item).st_mtime)).strftime('%Y-%m-%d')
            if mtime >= (YESTERDAY).__str__():
                if ('_' + str(YESTERDAY.year)) in item:
                    clean_item = item[:(item.find('_' + str(YESTERDAY.year)))]
                else:
                    clean_item = item
                current_files.append(clean_item + '\n')
    return current_files


if __name__ == '__main__':

    base_file_names = []

    current_files = []

    print('\n\n' + Back.BLUE + 'Welcome to Backup Checker')

    print(Back.BLUE + 'Scanning: ' + working_directory, Style.RESET_ALL, '\n')

    raw_names = get_raw_names_from_file('workfile.txt')

    base_file_names = process_raw_file_names(raw_names)

    current_files = walk_file_system(working_directory)

    missing_files = set(base_file_names).difference(current_files)

    extra_files = set(current_files).difference(base_file_names)

    print('Lines in input file:', len(raw_names))

    print('Lines in processed base file list:', len(base_file_names))

    print(Fore.GREEN + 'File system files counted:', len(current_files),
          Style.RESET_ALL)

    print('Base files missing from File system:', len(missing_files))

    for file in missing_files:
        print('Missing:', Fore.RED + file, Style.RESET_ALL)

    for file in extra_files:
        print('Extra:', Fore.GREEN + file, Style.RESET_ALL)

workfile.txt

W:\SQLServer_Backups_Recent\Apps\CriticalBackups\<yesterday>\MPBackups\Bitbucket\Bitbucket_backup
W:\SQLServer_Backups_Recent\JIRA\XML-Backups\<jira_yesterday>--0200.zip
W:\SQLServer_Backups_Recent\AppTest\GEN\GEN_backup

Answer 1

You can definitely put the excluded directories in a constant and iterate over it:

EXCLUDE = 'Marked', 'BackupNonCritical', 'Attachments', ...

def walk_file_system(working_directory):
    ...
        for to_exclude in EXCLUDE:
            if to_exclude in dirs:
                dirs.remove(to_exclude)

As an alternative, since the if to_exclude in dirs might be costly (for a long list of sub-directories), you could convert them to sets:

EXCLUDE = {'Marked', 'BackupNonCritical', 'Attachments', ...}

def walk_file_system(working_directory):
    ...
        for to_exclude in EXCLUDE.intersection(dirs):
            dirs.remove(to_exclude)

Or, since directory names have to be unique in every directory:

def walk_file_system(working_directory):
    ...
        dirs = set(dirs).difference(EXCLUDE)

Regarding the dates, the comparisons where you first call __str__ seem very hacky. Better use the fact that datetime.date objects are comparable:

import datetime

TODAY = datetime.date.today()
YESTERDAY = TODAY - datetime.timedelta(days=1)

file_name = "foo.txt"

mod_time = datetime.datetime.fromtimestamp(os.stat(file_name).st_mtime)
if mod_time.date() > YESTERDAY:
    print "new file"
if mod_time.date() == TODAY:
    print "still a new file"

You can also use format's nice direct format specifiers for date objects:

file.replace('<yesterday>', '{:%m-%d-%Y}-'.format(YESTERDAY)

Your whole process_raw_file_names function is actually very hard to understand at the moment. But I'm currently struggling to come up with good alternatives.

You should avoid shadowing the built-in function join with os.path.join. Since you actually use that function only once and in a short line, why not do this to follow the Python Zen "Explicit is better then implicit"

import os

for name in files:
    item = os.path.join(root, name)
    mtime = datetime.datetime.fromtimestamp(os.stat(file_name).st_mtime)
    ...

This

file_list = (join(root, name) for name in files)
for item in file_list:
    mtime = datetime.datetime.fromtimestamp(
        int(os.stat(item).st_mtime)).strftime('%Y-%m-%d')
    if mtime >= (YESTERDAY).__str__():
        if ('_' + str(YESTERDAY.year)) in item:
            clean_item = item[:(item.find('_' + str(YESTERDAY.year)))]
        else:
            clean_item = item
        current_files.append(clean_item + '\n')

can now be better written as:

year_suffix = "_{:%Y}".format(YESTERDAY)
for name in files:
    item = os.path.join(root, name)
    mtime = datetime.datetime.fromtimestamp(os.stat(item).st_mtime)
    if mtime.date() >= YESTERDAY:
        index = item.find(year_suffix)
        if index != -1:
            item = item[:index]
        current_files.append(item + '\n')