I'm still pretty new at Python and working on a small database program, where at the start of the program new files are imported from another folder (approx. 2000), and older files are backed up. Subfolders of the import folder sometimes contain multiple files with the same name (I always want just one of them, but the newest version isn't always the one I want). This is an unfortunate situation that I don't have control over.
Because I don't have that much experience yet, I find it very difficult to judge if this is a "valid" way of doing it, and it would be great to have some feedback as for its readability and general usability.
The transfer-settings referred to in the script:
# settings.py
class FileTransferSettings:
def __init__(self):
self.input_folder: str = r"./input"
self.search_subdirectories: bool = True
self.output_folder: str = r"./output"
self.backup_folder: str = r"./backup"
self.report_folder: str = r"./reports"
self.file_type_filter: list | None = [".yaml", ".csv"]
self.file_name_filter: str | None = "standardized"
transfer_settings = FileTransferSettings()
The file transfer script:
# file_transfer.py
from collections import defaultdict
import os
from pathlib import Path
import shutil
import time
from logs import file_transfer_log
from settings import transfer_settings as ts
def _find_valid_files():
def generate_glob_search_patterns() -> list[str]:
file_filters = (ts.file_type_filter, ts.file_name_filter)
match file_filters:
case (None, None):
return ["*"]
case (None, ts.file_name_filter):
return [f"*{ts.file_name_filter.lower()}*"]
case (ts.file_type_filter, None):
return [f"*{file_type}" for file_type in ts.file_type_filter]
case (ts.file_type_filter, ts.file_name_filter):
return [f"*{ts.file_name_filter.lower()}*{file_type}" for file_type in ts.file_type_filter]
def find_all_matching_files(pattern_list: list[str]) -> dict[str, list[Path]]:
result = defaultdict(list)
for pattern in pattern_list:
if ts.search_subdirectories:
for path in Path(ts.input_folder).rglob(pattern):
if path.is_file():
result[path.name].append(path)
elif not ts.search_subdirectories:
for path in Path(ts.input_folder).glob(pattern):
if path.is_file():
result[path.name].append(path)
return result
def resolve_duplicate_file_names(path_object_dict: dict[str, list[Path]]) -> list[Path]:
def print_duplicate_file_information(paths_to_print: list[Path]):
print(f"\nDuplicate file names found:\n")
for i, path in enumerate(paths_to_print):
print(f"{i}:\n"
f"Full path : {path.resolve()}\n"
f"Last modified : {time.ctime(os.path.getmtime(path))}\n")
def choose_file_to_transfer(paths_to_choose_from: list[Path]) -> int:
while True:
user_choice = input("Which file do you want to transfer? Input number here: ")
if not user_choice.isnumeric():
print("Invalid input, input a number.\n")
elif int(user_choice) < 0 or int(user_choice) > len(paths_to_choose_from) - 1:
print("Invalid input, number is not available.\n")
else:
return int(user_choice)
result = []
for path_name in path_object_dict:
path_objects_with_same_name = path_object_dict[path_name]
path_index = 0
if len(path_objects_with_same_name) > 1:
print_duplicate_file_information(path_objects_with_same_name)
path_index = choose_file_to_transfer(path_objects_with_same_name)
result.append(path_objects_with_same_name[path_index])
return result
glob_patterns = generate_glob_search_patterns()
path_list = resolve_duplicate_file_names(find_all_matching_files(glob_patterns))
return path_list
def _transfer_file(path: Path):
def is_file_to_transfer_newer(file_to_transfer: Path) -> bool:
duplicate_file_in_output_folder = f"{ts.output_folder}/{file_to_transfer.name}"
if os.path.getmtime(file_to_transfer) > os.path.getmtime(duplicate_file_in_output_folder):
return True
def backup_old_file(path_to_transfer: Path):
file_to_backup = Path(f"{ts.output_folder}/{path_to_transfer.name}")
filename, extension = file_to_backup.stem, file_to_backup.suffix
version_number = 1
while True:
if os.path.exists(f"{ts.backup_folder}/{filename}_version_{version_number}{extension}"):
version_number += 1
else:
backup = file_to_backup.rename(f"{ts.output_folder}/{filename}_version_{version_number}{extension}")
shutil.move(backup, ts.backup_folder)
break
if not os.path.exists(f"{ts.output_folder}/{path.name}"):
shutil.copy2(path, ts.output_folder)
file_transfer_log.add_new(path)
else:
if is_file_to_transfer_newer(path):
backup_old_file(path)
shutil.copy2(path, ts.output_folder)
file_transfer_log.add_updated(path)
else:
file_transfer_log.add_old(path)
def transfer_files():
for file_path in _find_valid_files():
_transfer_file(file_path)
