6
\$\begingroup\$

I wrote a program in Python that scans all files in a directory, makes dictionaries with stuff like date of creation and a hash of the contents of a file. Then i scan another directory which is just a copy of the original dir, but with some files added, deleted and changed. My program reports these changes.

import os
import stat
import hashlib
import json

class DirInfo:
    def __init__(self):
        self._dirInfo = {}

    def probe(self, dirname):
        fname = os.path.join(dirname, "data.txt") 
        blocksize = 512000000 
        for file in os.listdir(dirname): 
            if file != "data.txt": 
                fileWithPath = os.path.join(dirname, file) 
                fileInfo = {} 
                fileInfo['name'] = file
                if os.path.isfile(fileWithPath): 
                    sbuf = os.fstat(os.open(fileWithPath, os.O_RDONLY))
                    fileInfo['type'] = stat.S_IFMT(sbuf.st_mode) 
                    fileInfo['mode'] = stat.S_IMODE(sbuf.st_mode) 
                    fileInfo['mtime'] = sbuf.st_mtime 
                    fileInfo['size'] = sbuf.st_size 
                    hasher = hashlib.sha256() 
                    with open(fileWithPath, 'rb') as x: 
                        for chunk in iter(lambda: x.read(blocksize), b""):
                            hasher.update(chunk)
                    fileInfo['sha256'] = hasher.hexdigest()
                if os.path.islink(fileWithPath): 
                    fileInfo['symlink'] = os.readlink(fileWithPath) 
                self._dirInfo[file] = fileInfo 
                del fileInfo 
        DirInfo.save(self._dirInfo, fname)
        pass

    def compare(self, other):
        pad = "D:\\Testmapje met random files\\data.txt"
        pad2 = "D:\\Testmapje met random files 2\\data.txt"
        dct = DirInfo.load(self, pad)
        dct2 = DirInfo.load(other, pad2)
        assert isinstance(other, DirInfo)
        for key in dct:
            if key not in dct2: 
                print (key, "is verwijderd")
        for key in dct2: 
            fileHuidig = key 
            if key not in dct: 
                print (key, "is toegevoegd")
            else:
                allKeysPerFileMeasurement1 = (dct.get(key)) 
                allKeysPerFileMeasurement2 = (dct2.get(key))
                for key in allKeysPerFileMeasurement2: 
                    Measurement1Value = (allKeysPerFileMeasurement1.get(key))
                    Measurement2Value = (allKeysPerFileMeasurement2.get(key))
                    if Measurement1Value != Measurement2Value: 
                        print("In bestand:", fileHuidig, "was", key, Measurement1Value, "nu is", key, Measurement2Value)
        return []

    def save(self, fname):
        json.dump(self, open(fname, 'w')) 
        pass

    def load(self, fname):
        return json.load(open(fname, 'r')) 

if __name__ == '__main__':
    dirname = "D:\\Testmapje met random files" 
    dirnameanders = "D:\\Testmapje met random files 2" 
    dirInfo1 = DirInfo()
    dirInfo1.probe(dirname)
    # change some files
    dirInfo2 = DirInfo()
    dirInfo2.probe(dirnameanders)
    for err in dirInfo1.compare(dirInfo2):
        print(err)

My program gives the following output:

LinkNaarTestFolder2 is verwijderd (deleted) 

In bestand: Lief dagboek.txt was mtime 1530555920.17531 nu is mtime 1530723663.3718975

In bestand: Lief dagboek.txt was size 30 nu is size 22 

In bestand: Lief dagboek.txt was sha256 c33b32ad0a34316c970febc149199ca59946e75753ddc6438f09d1204835d0e4 nu is sha256 4728379b3f491004319de08ca6161d0b47d3f7921d79d7a9b95e1ca2c458ed48 

In bestand: LinkNaarTestFolder was symlink C:\Test voor python nu is symlink C:\Users\Kevin\Documents\Battlefield 3 

lol.txt is toegevoegd (is added)

Any ways to make my code shorter?

\$\endgroup\$
5
\$\begingroup\$

There is no reason here for this to be a class. You don't even use any class features and even have to work around it being a class by doing e.g. DirInfo.save(self._dirInfo, fname) instead of being able to do self.save(fname).

I would also add more functions, for example for hashing the file content.

import os
import stat
import hashlib
import json

STAT_FILE = "data.txt"

def get_stat_info(file_name):
    sbuf = os.fstat(os.open(file_name, os.O_RDONLY))
    return {'type': stat.S_IFMT(sbuf.st_mode),
            'mode': stat.S_IMODE(sbuf.st_mode),
            'mtime': sbuf.st_mtime,
            'size': sbuf.st_size}


def hash_file(file_name):
    hasher = hashlib.sha256()
    blocksize = 512000000
    with open(file_name, 'rb') as f:
        for chunk in iter(lambda: f.read(blocksize), b""):
            hasher.update(chunk)
    return hasher.hexdigest()


def probe(dir_name):
    dir_info = {}
    for file in os.listdir(dir_name):
        if file == STAT_FILE:
            continue
        file_with_path = os.path.join(dir_name, file)
        file_info = {'name': file}
        if os.path.isfile(file_with_path):
            file_info.update(get_stat_info(file_with_path))
            file_info['sha256'] = hash_file(file_with_path)
        if os.path.islink(file_with_path):
            file_info['symlink'] = os.readlink(file_with_path)
        dir_info[file] = file_info
    return dir_info


def save(dir_info, file_name):
    with open(file_name, 'w') as f:
        json.dump(dir_info, f)


def load(file_name):
    with open(file_name, 'r') as f:
        return json.load(f)


def compare(dir_info1, dir_info2):
    files1, files2 = dir_info1.keys(), dir_info2.keys()

    for f in files1 - files2:
        print (f, "is verwijderd")
    for f in files2 - files1:
        print (f, "is toegevoegd")
    for f in files1 & files2:
        file_info1, file_info2 = dir_info1[f], dir_info2[f]
        for key in file_info1:
            if file_info1[key] != file_info2[key]:
                print("In bestand:", f, "was", key, file_info1[key], "nu is", key, file_info2[key])


if __name__ == '__main__':
    dir_name1 = "D:\\Testmapje met random files"
    dir_info1 = probe(dir_name1)
    save(dir_info1, os.path.join(dir_name1, STAT_FILE))

    dir_name2 = "D:\\Testmapje met random files 2"
    dir_info2 = probe(dir_name2)
    save(dir_info2, os.path.join(dir_name2, STAT_FILE))

    compare(dir_info1, dir_info2)

Note that the save and load functions are not actually necessary, but I left them in because you might want to look at the results. In the compare function it makes a lot more sense to directly compare the two dictionaries, instead of saving them to a file and reading them back again.

I also changed all variables to conform to Python's official style-guide, PEP8, which recommends lower_case.

\$\endgroup\$
  • 1
    \$\begingroup\$ I implemented most of your proposals, thanks for providing such awesome analys and feedback! \$\endgroup\$ – Woask Jul 9 '18 at 20:12
5
\$\begingroup\$

This is too long to be a comment

  • Naming

    You mix Dutch and English variable names, you should stick to one language to avoid confusion, preferably English.

    fileWithPath = os.path.join(dirname, file)
    ...
    pad = "D:\\Testmapje met random files\\data.txt"
    
  • There is no need to use an expensive hashing algorithm such as SHA-256.

    Something like SHA-1 or even md5 would suffice.

Other than that, @Graipher pretty much covered everything.

\$\endgroup\$
  • \$\begingroup\$ You're right, I should pay more attentoin to my commenting and variabele naming. The whole thing was in Dutch actually so I changed the important parts for here. Thanks for your feedback. \$\endgroup\$ – Woask Jul 9 '18 at 20:13

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.