Recursively Save Python Dictionaries to HDF5 Files Using h5py

Question

I have a bunch of custom classes for which I've implemented a method of saving files in HDF5 format using the h5py module.

A bit of background: I've accomplished this by first implementing a serialization interface that represents the data in each class as a dictionary containing specific types of data (at the moment, the representations can only contain numpy.ndarray, numpy.int64, numpy.float64, str, and other dictionary instances). The advantage of this limitation is that it puts the dictionaries in data types that are h5py defaults. I was surprised to find a dearth of code tutorials on recursively saving dictionaries to HDF5 files, so I would really appreciate feedback on my implementation.

Imports:

import numpy as np
import h5py
import os

Saving the data:

def __save_dict_to_hdf5__(cls, dic, filename):
    """
    Save a dictionary whose contents are only strings, np.float64, np.int64,
    np.ndarray, and other dictionaries following this structure
    to an HDF5 file. These are the sorts of dictionaries that are meant
    to be produced by the ReportInterface__to_dict__() method.
    """
    assert not os.path.exists(filename), 'this is a noclobber operation bud'
    with h5py.File(filename, 'w') as h5file:
        cls.__recursively_save_dict_contents_to_group__(h5file, '/', dic)

@classmethod
def __recursively_save_dict_contents_to_group__(cls, h5file, path, dic):
    """
    Take an already open HDF5 file and insert the contents of a dictionary
    at the current path location. Can call itself recursively to fill
    out HDF5 files with the contents of a dictionary.
    """
    assert type(dic) is types.DictionaryType, "must provide a dictionary"
    assert type(path) is types.StringType, "path must be a string"
    assert type(h5file) is h5py._hl.files.File, "must be an open h5py file"
    for key in dic:
        assert type(key) == types.StringType, 'dict keys must be strings to save to hdf5'
        if type(dic[key]) in (np.int64, np.float64, types.StringType):
            h5file[path + key] = dic[key]
            assert h5file[path + key].value == dic[key], 'The data representation in the HDF5 file does not match the original dict.'
        if type(dic[key]) is np.ndarray:
            h5file[path + key] = dic[key]
            assert np.array_equal(h5file[path + key].value, dic[key]), 'The data representation in the HDF5 file does not match the original dict.'
        elif type(dic[key]) is types.DictionaryType:
            cls.__recursively_save_dict_contents_to_group__(h5file, path + key + '/', dic[key])

Loading the Data:

@classmethod
def __load_dict_from_hdf5__(cls, filename):
    """
    Load a dictionary whose contents are only strings, floats, ints,
    numpy arrays, and other dictionaries following this structure
    from an HDF5 file. These dictionaries can then be used to reconstruct
    ReportInterface subclass instances using the
    ReportInterface.__from_dict__() method.
    """
    with h5py.File(filename, 'r') as h5file:
        return cls.__recursively_load_dict_contents_from_group__(h5file, '/')

@classmethod
def __recursively_load_dict_contents_from_group__(cls, h5file, path):
    """
    Load contents of an HDF5 group. If further groups are encountered,
    treat them like dicts and continue to load them recursively.
    """
    ans = {}
    for key, item in h5file[path].items():
        if type(item) is h5py._hl.dataset.Dataset:
            ans[key] = item.value
        elif type(item) is h5py._hl.group.Group:
            ans[key] = cls.__recursively_load_dict_contents_from_group__(h5file, path + key + '/')
    return ans

This passes my unit tests for saving and loading dictionaries with data intact. But I really don't know how Pythonic this is and would appreciate feedback. I tried to leave an HDF5 tag, but it doesn't exist; anyone who is more familiar with the format, and perhaps with h5py, can maybe tell me if there is a more elegant or idiomatic way to do this (I don't want to confuse the next student who will maintain this), or if I am setting myself up for any nasty surprises.

Answer 1

Here's what I tested.

I took out the classmethod stuff to make it easier to read, and simplified names a bit. I'll defer judgement on whether that stuff is needed as part of a larger package or not.

My h5py is installed with Python3, so I had to change the handling of types. Use of isinstance is, I think a preferred testing tool, but I it's not something I've focused on. Most of my code changes are in the recursive write function.

I'll let others focus on preferred naming conventions and error checking.

import numpy as np
import h5py
import os
def save_dict_to_hdf5(dic, filename):
    """
    ....
    """
    with h5py.File(filename, 'w') as h5file:
        recursively_save_dict_contents_to_group(h5file, '/', dic)

def recursively_save_dict_contents_to_group(h5file, path, dic):
    """
    ....
    """
    for key, item in dic.items():
        if isinstance(item, (np.ndarray, np.int64, np.float64, str, bytes)):
            h5file[path + key] = item
        elif isinstance(item, dict):
            recursively_save_dict_contents_to_group(h5file, path + key + '/', item)
        else:
            raise ValueError('Cannot save %s type'%type(item))

def load_dict_from_hdf5(filename):
    """
    ....
    """
    with h5py.File(filename, 'r') as h5file:
        return recursively_load_dict_contents_from_group(h5file, '/')

def recursively_load_dict_contents_from_group(h5file, path):
    """
    ....
    """
    ans = {}
    for key, item in h5file[path].items():
        if isinstance(item, h5py._hl.dataset.Dataset):
            ans[key] = item.value
        elif isinstance(item, h5py._hl.group.Group):
            ans[key] = recursively_load_dict_contents_from_group(h5file, path + key + '/')
    return ans

if __name__ == '__main__':

    data = {'x': 'astring',
            'y': np.arange(10),
            'd': {'z': np.ones((2,3)),
                  'b': b'bytestring'}}
    print(data)
    filename = 'test.h5'
    save_dict_to_hdf5(data, filename)
    dd = load_dict_from_hdf5(filename)
    print(dd)
    # should test for bad type

with results:

0858:~/mypy$ python3.4 cr120802.py 
{'x': 'astring', 'd': {'b': b'bytestring', 'z': array([[ 1.,  1.,  1.],
       [ 1.,  1.,  1.]])}, 'y': array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])}
{'x': 'astring', 'd': {'b': b'bytestring', 'z': array([[ 1.,  1.,  1.],
       [ 1.,  1.,  1.]])}, 'y': array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])}

Answer 2

Don't use assert to test for possible issues with the values passed to your functions. That's where you raise ValueErrors. The main purpose of assert statements is to note things that should never happen. Things that actually require reprogramming to fix so that you can remove the asserts when you're ready to ship the code.

Manually raising exceptions is common to do when an expected circumstance has arisen that makes continuing the code impossible. For example when someone passes an invalid parameter or a file you cannot open.

One key reason is that you can run Python in optimise mode (using the flag -o) and then all assert statements will be ignored, meaning you've lose all your tests. But to the broader point, that indicates how asserts are intended for debugging, not writing real code.

Answer 3

Correct several bugs in the above answers, now it can save list, float...

def save_dict_to_hdf5(dic, filename):

    with h5py.File(filename, 'w') as h5file:
        recursively_save_dict_contents_to_group(h5file, '/', dic)

def load_dict_from_hdf5(filename):

    with h5py.File(filename, 'r') as h5file:
        return recursively_load_dict_contents_from_group(h5file, '/')



def recursively_save_dict_contents_to_group( h5file, path, dic):

    # argument type checking
    if not isinstance(dic, dict):
        raise ValueError("must provide a dictionary")        

    if not isinstance(path, str):
        raise ValueError("path must be a string")
    if not isinstance(h5file, h5py._hl.files.File):
        raise ValueError("must be an open h5py file")
    # save items to the hdf5 file
    for key, item in dic.items():
        #print(key,item)
        key = str(key)
        if isinstance(item, list):
            item = np.array(item)
            #print(item)
        if not isinstance(key, str):
            raise ValueError("dict keys must be strings to save to hdf5")
        # save strings, numpy.int64, and numpy.float64 types
        if isinstance(item, (np.int64, np.float64, str, np.float, float, np.float32,int)):
            #print( 'here' )
            h5file[path + key] = item
            if not h5file[path + key].value == item:
                raise ValueError('The data representation in the HDF5 file does not match the original dict.')
        # save numpy arrays
        elif isinstance(item, np.ndarray):            
            try:
                h5file[path + key] = item
            except:
                item = np.array(item).astype('|S9')
                h5file[path + key] = item
            if not np.array_equal(h5file[path + key].value, item):
                raise ValueError('The data representation in the HDF5 file does not match the original dict.')
        # save dictionaries
        elif isinstance(item, dict):
            recursively_save_dict_contents_to_group(h5file, path + key + '/', item)
        # other types cannot be saved and will result in an error
        else:
            #print(item)
            raise ValueError('Cannot save %s type.' % type(item))

def recursively_load_dict_contents_from_group( h5file, path): 

    ans = {}
    for key, item in h5file[path].items():
        if isinstance(item, h5py._hl.dataset.Dataset):
            ans[key] = item.value
        elif isinstance(item, h5py._hl.group.Group):
            ans[key] = recursively_load_dict_contents_from_group(h5file, path + key + '/')
    return ans            

Answer 4

I changed the assertions to isinstance checks which raise ValueError()s. I've also added in the class declaration at the top (which I didn't include in my original question) to help clarify that these are class methods that are supposed to be inheritable (I don't want them floating around in my module's namespace, and they should really only be used by subclasses of this interface in my implementation):

import numpy as np
import h5py
import os

class ReportInterface(object):

    # ...more details about this class...

    @classmethod
    def __save_dict_to_hdf5__(cls, dic, filename):
        """..."""
        if os.path.exists(filename):
            raise ValueError('File %s exists, will not overwrite.' % filename)
        with h5py.File(filename, 'w') as h5file:
            cls.__recursively_save_dict_contents_to_group__(h5file, '/', dic)

    @classmethod
    def __recursively_save_dict_contents_to_group__(cls, h5file, path, dic):
        """..."""
        # argument type checking
        if not isinstance(dic, dict):
            raise ValueError("must provide a dictionary")
        if not isinstance(path, str):
            raise ValueError("path must be a string")
        if not isinstance(h5file, h5py._hl.files.File):
            raise ValueError("must be an open h5py file")
        # save items to the hdf5 file
        for key, item in dic.items():
            if not isinstance(key, str):
                raise ValueError("dict keys must be strings to save to hdf5")
            # save strings, numpy.int64, and numpy.float64 types
            if isinstance(item, (np.int64, np.float64, str)):
                h5file[path + key] = item
                if not h5file[path + key].value == item:
                    raise ValueError('The data representation in the HDF5 file does not match the original dict.')
            # save numpy arrays
            elif isinstance(item, np.ndarray):
                h5file[path + key] = item
                if not np.array_equal(h5file[path + key].value, item):
                    raise ValueError('The data representation in the HDF5 file does not match the original dict.')
            # save dictionaries
            elif isinstance(item, dict):
                cls.__recursively_save_dict_contents_to_group__(h5file, path + key + '/', item)
            # other types cannot be saved and will result in an error
            else:
                raise ValueError('Cannot save %s type.' % type(item))

    @classmethod
    def __load_dict_from_hdf5__(cls, filename):
        """..."""
        with h5py.File(filename, 'r') as h5file:
            return cls.__recursively_load_dict_contents_from_group__(h5file, '/')

    @classmethod
    def __recursively_load_dict_contents_from_group__(cls, h5file, path):
        """..."""
        ans = {}
        for key, item in h5file[path].items():
            if isinstance(item, h5py._hl.dataset.Dataset):
                ans[key] = item.value
            elif isinstance(item, h5py._hl.group.Group):
                ans[key] = cls.__recursively_load_dict_contents_from_group__(h5file, path + key + '/')
        return ans

# a test
if __name__ == "__main__":

    ex = {
        'name': 'stefan',
        'age':  np.int64(24),
        'fav_numbers': np.array([2,4,4.3]),
        'fav_tensors': {
            'levi_civita3d': np.array([
                [[0,0,0],[0,0,1],[0,-1,0]],
                [[0,0,-1],[0,0,0],[1,0,0]],
                [[0,1,0],[-1,0,0],[0,0,0]]
            ]),
            'kronecker2d': np.identity(3)
        }
    }
    print ex
    ReportInterface.__save_dict_to_hdf5__(ex, 'foo.hdf5')
    loaded = ReportInterface.__load_dict_from_hdf5__('foo.hdf5')
    print loaded
    np.testing.assert_equal(loaded, ex)
    print 'check passed!'

which passes its own test:

>>> python hdf5test.py
{'age': 24, 'fav_numbers': array([ 2. ,  4. ,  4.3]), 'name': 'stefan', 'fav_tensors': {'levi_civita3d': array([[[ 0,  0,  0],
        [ 0,  0,  1],
        [ 0, -1,  0]],

    [[ 0,  0, -1],
        [ 0,  0,  0],
        [ 1,  0,  0]],

    [[ 0,  1,  0],
        [-1,  0,  0],
        [ 0,  0,  0]]]), 'kronecker2d': array([[ 1.,  0.,  0.],
    [ 0.,  1.,  0.],
    [ 0.,  0.,  1.]])}}
{u'age': 24, u'fav_numbers': array([ 2. ,  4. ,  4.3]), u'name': 'stefan', u'fav_tensors': {u'levi_civita3d': array([[[ 0,  0,  0],
        [ 0,  0,  1],
        [ 0, -1,  0]],

    [[ 0,  0, -1],
        [ 0,  0,  0],
        [ 1,  0,  0]],

    [[ 0,  1,  0],
        [-1,  0,  0],
        [ 0,  0,  0]]]), u'kronecker2d': array([[ 1.,  0.,  0.],
    [ 0.,  1.,  0.],
    [ 0.,  0.,  1.]])}}
check passed!