Scan similar files in different directories and report differences

Question

I wrote a program in Python that scans all files in a directory, makes dictionaries with stuff like date of creation and a hash of the contents of a file. Then i scan another directory which is just a copy of the original dir, but with some files added, deleted and changed. My program reports these changes.

import os
import stat
import hashlib
import json

class DirInfo:
    def __init__(self):
        self._dirInfo = {}

    def probe(self, dirname):
        fname = os.path.join(dirname, "data.txt") 
        blocksize = 512000000 
        for file in os.listdir(dirname): 
            if file != "data.txt": 
                fileWithPath = os.path.join(dirname, file) 
                fileInfo = {} 
                fileInfo['name'] = file
                if os.path.isfile(fileWithPath): 
                    sbuf = os.fstat(os.open(fileWithPath, os.O_RDONLY))
                    fileInfo['type'] = stat.S_IFMT(sbuf.st_mode) 
                    fileInfo['mode'] = stat.S_IMODE(sbuf.st_mode) 
                    fileInfo['mtime'] = sbuf.st_mtime 
                    fileInfo['size'] = sbuf.st_size 
                    hasher = hashlib.sha256() 
                    with open(fileWithPath, 'rb') as x: 
                        for chunk in iter(lambda: x.read(blocksize), b""):
                            hasher.update(chunk)
                    fileInfo['sha256'] = hasher.hexdigest()
                if os.path.islink(fileWithPath): 
                    fileInfo['symlink'] = os.readlink(fileWithPath) 
                self._dirInfo[file] = fileInfo 
                del fileInfo 
        DirInfo.save(self._dirInfo, fname)
        pass

    def compare(self, other):
        pad = "D:\\Testmapje met random files\\data.txt"
        pad2 = "D:\\Testmapje met random files 2\\data.txt"
        dct = DirInfo.load(self, pad)
        dct2 = DirInfo.load(other, pad2)
        assert isinstance(other, DirInfo)
        for key in dct:
            if key not in dct2: 
                print (key, "is verwijderd")
        for key in dct2: 
            fileHuidig = key 
            if key not in dct: 
                print (key, "is toegevoegd")
            else:
                allKeysPerFileMeasurement1 = (dct.get(key)) 
                allKeysPerFileMeasurement2 = (dct2.get(key))
                for key in allKeysPerFileMeasurement2: 
                    Measurement1Value = (allKeysPerFileMeasurement1.get(key))
                    Measurement2Value = (allKeysPerFileMeasurement2.get(key))
                    if Measurement1Value != Measurement2Value: 
                        print("In bestand:", fileHuidig, "was", key, Measurement1Value, "nu is", key, Measurement2Value)
        return []

    def save(self, fname):
        json.dump(self, open(fname, 'w')) 
        pass

    def load(self, fname):
        return json.load(open(fname, 'r')) 

if __name__ == '__main__':
    dirname = "D:\\Testmapje met random files" 
    dirnameanders = "D:\\Testmapje met random files 2" 
    dirInfo1 = DirInfo()
    dirInfo1.probe(dirname)
    # change some files
    dirInfo2 = DirInfo()
    dirInfo2.probe(dirnameanders)
    for err in dirInfo1.compare(dirInfo2):
        print(err)

My program gives the following output:

LinkNaarTestFolder2 is verwijderd (deleted) 

In bestand: Lief dagboek.txt was mtime 1530555920.17531 nu is mtime 1530723663.3718975

In bestand: Lief dagboek.txt was size 30 nu is size 22 

In bestand: Lief dagboek.txt was sha256 c33b32ad0a34316c970febc149199ca59946e75753ddc6438f09d1204835d0e4 nu is sha256 4728379b3f491004319de08ca6161d0b47d3f7921d79d7a9b95e1ca2c458ed48 

In bestand: LinkNaarTestFolder was symlink C:\Test voor python nu is symlink C:\Users\Kevin\Documents\Battlefield 3 

lol.txt is toegevoegd (is added)

Any ways to make my code shorter?

Answer 1

There is no reason here for this to be a class. You don't even use any class features and even have to work around it being a class by doing e.g. DirInfo.save(self._dirInfo, fname) instead of being able to do self.save(fname).

I would also add more functions, for example for hashing the file content.

import os
import stat
import hashlib
import json

STAT_FILE = "data.txt"

def get_stat_info(file_name):
    sbuf = os.fstat(os.open(file_name, os.O_RDONLY))
    return {'type': stat.S_IFMT(sbuf.st_mode),
            'mode': stat.S_IMODE(sbuf.st_mode),
            'mtime': sbuf.st_mtime,
            'size': sbuf.st_size}


def hash_file(file_name):
    hasher = hashlib.sha256()
    blocksize = 512000000
    with open(file_name, 'rb') as f:
        for chunk in iter(lambda: f.read(blocksize), b""):
            hasher.update(chunk)
    return hasher.hexdigest()


def probe(dir_name):
    dir_info = {}
    for file in os.listdir(dir_name):
        if file == STAT_FILE:
            continue
        file_with_path = os.path.join(dir_name, file)
        file_info = {'name': file}
        if os.path.isfile(file_with_path):
            file_info.update(get_stat_info(file_with_path))
            file_info['sha256'] = hash_file(file_with_path)
        if os.path.islink(file_with_path):
            file_info['symlink'] = os.readlink(file_with_path)
        dir_info[file] = file_info
    return dir_info


def save(dir_info, file_name):
    with open(file_name, 'w') as f:
        json.dump(dir_info, f)


def load(file_name):
    with open(file_name, 'r') as f:
        return json.load(f)


def compare(dir_info1, dir_info2):
    files1, files2 = dir_info1.keys(), dir_info2.keys()

    for f in files1 - files2:
        print (f, "is verwijderd")
    for f in files2 - files1:
        print (f, "is toegevoegd")
    for f in files1 & files2:
        file_info1, file_info2 = dir_info1[f], dir_info2[f]
        for key in file_info1:
            if file_info1[key] != file_info2[key]:
                print("In bestand:", f, "was", key, file_info1[key], "nu is", key, file_info2[key])


if __name__ == '__main__':
    dir_name1 = "D:\\Testmapje met random files"
    dir_info1 = probe(dir_name1)
    save(dir_info1, os.path.join(dir_name1, STAT_FILE))

    dir_name2 = "D:\\Testmapje met random files 2"
    dir_info2 = probe(dir_name2)
    save(dir_info2, os.path.join(dir_name2, STAT_FILE))

    compare(dir_info1, dir_info2)

Note that the save and load functions are not actually necessary, but I left them in because you might want to look at the results. In the compare function it makes a lot more sense to directly compare the two dictionaries, instead of saving them to a file and reading them back again.

I also changed all variables to conform to Python's official style-guide, PEP8, which recommends lower_case.

Answer 2

This is too long to be a comment

• Naming

  You mix Dutch and English variable names, you should stick to one language to avoid confusion, preferably English.

  fileWithPath = os.path.join(dirname, file)
  ...
  pad = "D:\\Testmapje met random files\\data.txt"
  

• There is no need to use an expensive hashing algorithm such as SHA-256.

  Something like SHA-1 or even md5 would suffice.

Other than that, @Graipher pretty much covered everything.