The following code aims to identify and delete duplicate files in a folder. At first, files are compared by size, and then (if needed) by hash.
The helper function sort_duplicates determines which file of the group of duplicates to keep. I have skipped the part of the code dealing with CLI.
As a self-learner of computer science, I would very appreciate any feedback.
Code is written on Python 3.6.4
import os
import hashlib
import collections
import copy
import test_duplicates
def calculate_hash(file):
"""
file: path to a file object
Returns a hash of a provided file object
"""
md5 = hashlib.md5()
with open(file, 'rb') as fh:
for chunk in iter(lambda: fh.read(4098), b''):
md5.update(chunk)
return md5.hexdigest()
def sort_duplicates(duplicates):
"""
duplicates: sequence of groups of duplicates
returns two list of the duplicates divided by the rule described in
'find_duplicates'
"""
duplicates = copy.deepcopy(duplicates)
to_keep = []
to_delete = []
for group in duplicates:
file = [file for file in group if ('cop') not in file][0]
if file:
to_keep.append(file)
group.remove(file)
to_delete.extend(group)
else:
to_keep.append(group.pop(file))
to_delete.extend(group)
return to_keep, to_delete
def find_duplicates(directory, recursive=True,
exclude_hidden=False, delete=True):
"""returns
list of duplicates grouped in sets by common hash
list of duplicates proposed to delete
list of duplicated proposed to keep
to_keep contains an arbitrary duplicate from each group of duplicates if
path of the duplicate does not contain string 'cop'
if such duplicate does not exist to_keep collects any arbitrary
duplicate from the group
to_delete contains the rest of the duplicates
"""
duplicates = collections.defaultdict(set)
sizes_dict = collections.defaultdict(set)
# filter by size
for root, dirnames, filenames in os.walk(directory):
if exclude_hidden:
filenames = (filename for filename in filenames
if not filename.startswith('.'))
for filename in filenames:
path = os.path.join(root, filename)
sizes_dict[os.path.getsize(path)].add(path)
if not recursive:
break
groups = [values for values in sizes_dict.values()
if len(values) > 1]
for group in groups:
for item in group:
duplicates[calculate_hash(item)].add(item)
duplicates = [values for values in duplicates.values()
if len(values) > 1]
if not duplicates:
print('No duplicates has been found')
return None
print('----Found following groups of duplicates: ----------------------')
for group in duplicates:
print(' - '.join(str(duplicate) for duplicate in group), end='\n')
print()
if not delete:
return
to_keep, to_delete = sort_duplicates(duplicates)
print('----Following files from the list of duplicates will be kept:')
print(*to_keep, sep='\n', end='\n')
print()
print('----Following files from the list of duplicates will be deleted:')
print(*to_delete, sep='\n')
for path in to_delete:
os.remove(path)
print('Done')
return duplicates, to_keep, to_delete
if __name__ == '__main__':
test_duplicates.make_files('/tmp/d')
find_duplicates('/tmp/d')
Output:
----Found following groups of duplicates: ----------------------
/tmp/d/2 - /tmp/d/copied/2
/tmp/d/copied/4 - /tmp/d/4
/tmp/d/3 - /tmp/d/copied/3
/tmp/d/copied/0 - /tmp/d/0
/tmp/d/copied/1 - /tmp/d/1
----Following files from the list of duplicates will be kept:
/tmp/d/2
/tmp/d/4
/tmp/d/3
/tmp/d/0
/tmp/d/1
----Following files from the list of duplicates will be deleted:
/tmp/d/copied/2
/tmp/d/copied/4
/tmp/d/copied/3
/tmp/d/copied/0
/tmp/d/copied/1
Done
There is test_duplicates module.
import py.path
import random
from duplicates import find_duplicates
def make_files(directory):
"""
Creates dummy objects in a given directory:
10 unique file objects in the parent directory
copies of the first 5 objects in the subfolder 'copied'
changed copies (added empty string) of second 5 objects in
the subfolder copied
returns: mapping of dublicate objects
list of objects to keep
list of objects to deleted
"""
dir_path = py.path.local(directory)
copied_path = dir_path.join('copied')
copied_path.ensure(dir=True)
changed_path = dir_path.join('changed')
changed_path.ensure(dir=True)
duplicates, paths, to_delete = [], [], []
for ind in range(10):
path = dir_path.join(str(ind))
path.write(random.randrange(ind * 10 - 10, ind * 10), ensure=True)
paths.append(path)
assert path.check()
# copy first 5 files into the subfolder 'copy'
for path in paths[:5]:
path_copy = copied_path.join(path.basename)
path.copy(path_copy)
paths.append(path_copy)
to_delete.append(path_copy)
duplicates.append({str(path), str(path_copy)})
assert path_copy.check()
# copy second 5 files and add empty string to each of them:
for path in paths[5:10]:
path_changed = changed_path.join(path.basename)
path.copy(path_changed)
path_changed.write('\n', mode='a')
paths.append(path_changed)
assert path_changed.check()
to_keep = [str(path) for path in paths[:5]]
paths.extend((copied_path, changed_path))
files = sorted(list(dir_path.visit()))
assert files == sorted(paths)
return duplicates, to_keep, to_delete
def test_duplicates(tmpdir):
"""Compares received and expected results of find_duplicate function"""
exp_duplicates, exp_to_keep, exp_to_delete = make_files(tmpdir)
rec_duplicates, rec_to_keep, rec_to_delete = find_duplicates(
tmpdir, recursive=True, delete=True)
assert len(exp_duplicates) == len(rec_duplicates)
for group in rec_duplicates:
assert group in exp_duplicates
assert sorted(exp_to_keep) == sorted(rec_to_keep)
assert sorted(exp_to_delete) == sorted(rec_to_delete)
no_duplicates = tmpdir.join('copied')
assert find_duplicates(no_duplicates) == None
