# Recursively Save Python Dictionaries to HDF5 Files Using h5py

I have a bunch of custom classes for which I've implemented a method of saving files in HDF5 format using the h5py module.

A bit of background: I've accomplished this by first implementing a serialization interface that represents the data in each class as a dictionary containing specific types of data (at the moment, the representations can only contain numpy.ndarray, numpy.int64, numpy.float64, str, and other dictionary instances). The advantage of this limitation is that it puts the dictionaries in data types that are h5py defaults. I was surprised to find a dearth of code tutorials on recursively saving dictionaries to HDF5 files, so I would really appreciate feedback on my implementation.

Imports:

import numpy as np
import h5py
import os


Saving the data:

def __save_dict_to_hdf5__(cls, dic, filename):
"""
Save a dictionary whose contents are only strings, np.float64, np.int64,
np.ndarray, and other dictionaries following this structure
to an HDF5 file. These are the sorts of dictionaries that are meant
to be produced by the ReportInterface__to_dict__() method.
"""
assert not os.path.exists(filename), 'this is a noclobber operation bud'
with h5py.File(filename, 'w') as h5file:
cls.__recursively_save_dict_contents_to_group__(h5file, '/', dic)

@classmethod
def __recursively_save_dict_contents_to_group__(cls, h5file, path, dic):
"""
Take an already open HDF5 file and insert the contents of a dictionary
at the current path location. Can call itself recursively to fill
out HDF5 files with the contents of a dictionary.
"""
assert type(dic) is types.DictionaryType, "must provide a dictionary"
assert type(path) is types.StringType, "path must be a string"
assert type(h5file) is h5py._hl.files.File, "must be an open h5py file"
for key in dic:
assert type(key) == types.StringType, 'dict keys must be strings to save to hdf5'
if type(dic[key]) in (np.int64, np.float64, types.StringType):
h5file[path + key] = dic[key]
assert h5file[path + key].value == dic[key], 'The data representation in the HDF5 file does not match the original dict.'
if type(dic[key]) is np.ndarray:
h5file[path + key] = dic[key]
assert np.array_equal(h5file[path + key].value, dic[key]), 'The data representation in the HDF5 file does not match the original dict.'
elif type(dic[key]) is types.DictionaryType:
cls.__recursively_save_dict_contents_to_group__(h5file, path + key + '/', dic[key])


@classmethod
"""
Load a dictionary whose contents are only strings, floats, ints,
numpy arrays, and other dictionaries following this structure
from an HDF5 file. These dictionaries can then be used to reconstruct
ReportInterface subclass instances using the
ReportInterface.__from_dict__() method.
"""
with h5py.File(filename, 'r') as h5file:

@classmethod
"""
Load contents of an HDF5 group. If further groups are encountered,
treat them like dicts and continue to load them recursively.
"""
ans = {}
for key, item in h5file[path].items():
if type(item) is h5py._hl.dataset.Dataset:
ans[key] = item.value
elif type(item) is h5py._hl.group.Group:
ans[key] = cls.__recursively_load_dict_contents_from_group__(h5file, path + key + '/')
return ans


This passes my unit tests for saving and loading dictionaries with data intact. But I really don't know how Pythonic this is and would appreciate feedback. I tried to leave an HDF5 tag, but it doesn't exist; anyone who is more familiar with the format, and perhaps with h5py, can maybe tell me if there is a more elegant or idiomatic way to do this (I don't want to confuse the next student who will maintain this), or if I am setting myself up for any nasty surprises.

• StackOverflow has much higher participation of numpy coders, and a small number of h5py users. I've answered questions on how to store/fetch arrays, but haven't paid any attention to the mapping between dictionaries and groups. Feb 26, 2016 at 23:17
• The h5py part seems to work fine; I don't see any special issues. I wonder about the classmethod stuff. What is the class? Why not just define a set of functions? Feb 27, 2016 at 1:32
• I know, I couldn't find any either, which surprised me. I'm calling it a classmethod because I need to call __recursively_load_dict_contents_from_group__ recursively, and I don't want to write the name of the class, ReportInterface, explicitly in the code; this way I have one less thing to break if I change the name ReportInterface. But perhaps this is a silly concern. Feb 27, 2016 at 15:53
• What you may and may not do after receiving answers. I've rolled back Rev 4 → 3.
– user34073
Feb 27, 2016 at 19:02
• That makes sense, thank you; will post fixed code in an answer. Feb 27, 2016 at 19:06

Here's what I tested.

I took out the classmethod stuff to make it easier to read, and simplified names a bit. I'll defer judgement on whether that stuff is needed as part of a larger package or not.

My h5py is installed with Python3, so I had to change the handling of types. Use of isinstance is, I think a preferred testing tool, but I it's not something I've focused on. Most of my code changes are in the recursive write function.

I'll let others focus on preferred naming conventions and error checking.

import numpy as np
import h5py
import os
def save_dict_to_hdf5(dic, filename):
"""
....
"""
with h5py.File(filename, 'w') as h5file:
recursively_save_dict_contents_to_group(h5file, '/', dic)

def recursively_save_dict_contents_to_group(h5file, path, dic):
"""
....
"""
for key, item in dic.items():
if isinstance(item, (np.ndarray, np.int64, np.float64, str, bytes)):
h5file[path + key] = item
elif isinstance(item, dict):
recursively_save_dict_contents_to_group(h5file, path + key + '/', item)
else:
raise ValueError('Cannot save %s type'%type(item))

"""
....
"""
with h5py.File(filename, 'r') as h5file:

"""
....
"""
ans = {}
for key, item in h5file[path].items():
if isinstance(item, h5py._hl.dataset.Dataset):
ans[key] = item.value
elif isinstance(item, h5py._hl.group.Group):
ans[key] = recursively_load_dict_contents_from_group(h5file, path + key + '/')
return ans

if __name__ == '__main__':

data = {'x': 'astring',
'y': np.arange(10),
'd': {'z': np.ones((2,3)),
'b': b'bytestring'}}
print(data)
filename = 'test.h5'
save_dict_to_hdf5(data, filename)
print(dd)
# should test for bad type


with results:

0858:~/mypy\$ python3.4 cr120802.py
{'x': 'astring', 'd': {'b': b'bytestring', 'z': array([[ 1.,  1.,  1.],
[ 1.,  1.,  1.]])}, 'y': array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])}
{'x': 'astring', 'd': {'b': b'bytestring', 'z': array([[ 1.,  1.,  1.],
[ 1.,  1.,  1.]])}, 'y': array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])}

• Thanks! I get the same results as you in Python 2.7.10. I edited your example to include a recursive equality test, np.testing.assert_equal(data, dd), which still doesn't check for modified type. My approach only works for a few types and isn't meant to work for any others; you'd have to be very thorough and use hdf5 attributes, or something like that, to perfectly preserve type. (P.S. upvoted your response, it'll show when my rep goes up.) Feb 27, 2016 at 18:09
• I tried getting rid of the @classmethod declarations, but if I am declaring these functions inside a class definition, Python 2.7.10 gets confused about the namespace and says "global name <...> is not defined" when I try calling any of these methods from one another. So your approach is good in the main body, but at least for 2.7.10, it looks like you need the @classmethod approach to put these within a class and have them work properly. Feb 27, 2016 at 18:43

Don't use assert to test for possible issues with the values passed to your functions. That's where you raise ValueErrors. The main purpose of assert statements is to note things that should never happen. Things that actually require reprogramming to fix so that you can remove the asserts when you're ready to ship the code.

Manually raising exceptions is common to do when an expected circumstance has arisen that makes continuing the code impossible. For example when someone passes an invalid parameter or a file you cannot open.

One key reason is that you can run Python in optimise mode (using the flag -o) and then all assert statements will be ignored, meaning you've lose all your tests. But to the broader point, that indicates how asserts are intended for debugging, not writing real code.

• Thanks for the feedback. I'll change the assert statements to raise ValueErrors in cases where user input is bad vs. debug tests. You didn't mention the actual save/load implementation; does that part look okay? Feb 23, 2016 at 19:57

Correct several bugs in the above answers, now it can save list, float...

def save_dict_to_hdf5(dic, filename):

with h5py.File(filename, 'w') as h5file:
recursively_save_dict_contents_to_group(h5file, '/', dic)

with h5py.File(filename, 'r') as h5file:

def recursively_save_dict_contents_to_group( h5file, path, dic):

# argument type checking
if not isinstance(dic, dict):
raise ValueError("must provide a dictionary")

if not isinstance(path, str):
raise ValueError("path must be a string")
if not isinstance(h5file, h5py._hl.files.File):
raise ValueError("must be an open h5py file")
# save items to the hdf5 file
for key, item in dic.items():
#print(key,item)
key = str(key)
if isinstance(item, list):
item = np.array(item)
#print(item)
if not isinstance(key, str):
raise ValueError("dict keys must be strings to save to hdf5")
# save strings, numpy.int64, and numpy.float64 types
if isinstance(item, (np.int64, np.float64, str, np.float, float, np.float32,int)):
#print( 'here' )
h5file[path + key] = item
if not h5file[path + key].value == item:
raise ValueError('The data representation in the HDF5 file does not match the original dict.')
# save numpy arrays
elif isinstance(item, np.ndarray):
try:
h5file[path + key] = item
except:
item = np.array(item).astype('|S9')
h5file[path + key] = item
if not np.array_equal(h5file[path + key].value, item):
raise ValueError('The data representation in the HDF5 file does not match the original dict.')
# save dictionaries
elif isinstance(item, dict):
recursively_save_dict_contents_to_group(h5file, path + key + '/', item)
# other types cannot be saved and will result in an error
else:
#print(item)
raise ValueError('Cannot save %s type.' % type(item))

ans = {}
for key, item in h5file[path].items():
if isinstance(item, h5py._hl.dataset.Dataset):
ans[key] = item.value
elif isinstance(item, h5py._hl.group.Group):
ans[key] = recursively_load_dict_contents_from_group(h5file, path + key + '/')
return ans


I changed the assertions to isinstance checks which raise ValueError()s. I've also added in the class declaration at the top (which I didn't include in my original question) to help clarify that these are class methods that are supposed to be inheritable (I don't want them floating around in my module's namespace, and they should really only be used by subclasses of this interface in my implementation):

import numpy as np
import h5py
import os

class ReportInterface(object):

@classmethod
def __save_dict_to_hdf5__(cls, dic, filename):
"""..."""
if os.path.exists(filename):
raise ValueError('File %s exists, will not overwrite.' % filename)
with h5py.File(filename, 'w') as h5file:
cls.__recursively_save_dict_contents_to_group__(h5file, '/', dic)

@classmethod
def __recursively_save_dict_contents_to_group__(cls, h5file, path, dic):
"""..."""
# argument type checking
if not isinstance(dic, dict):
raise ValueError("must provide a dictionary")
if not isinstance(path, str):
raise ValueError("path must be a string")
if not isinstance(h5file, h5py._hl.files.File):
raise ValueError("must be an open h5py file")
# save items to the hdf5 file
for key, item in dic.items():
if not isinstance(key, str):
raise ValueError("dict keys must be strings to save to hdf5")
# save strings, numpy.int64, and numpy.float64 types
if isinstance(item, (np.int64, np.float64, str)):
h5file[path + key] = item
if not h5file[path + key].value == item:
raise ValueError('The data representation in the HDF5 file does not match the original dict.')
# save numpy arrays
elif isinstance(item, np.ndarray):
h5file[path + key] = item
if not np.array_equal(h5file[path + key].value, item):
raise ValueError('The data representation in the HDF5 file does not match the original dict.')
# save dictionaries
elif isinstance(item, dict):
cls.__recursively_save_dict_contents_to_group__(h5file, path + key + '/', item)
# other types cannot be saved and will result in an error
else:
raise ValueError('Cannot save %s type.' % type(item))

@classmethod
"""..."""
with h5py.File(filename, 'r') as h5file:

@classmethod
"""..."""
ans = {}
for key, item in h5file[path].items():
if isinstance(item, h5py._hl.dataset.Dataset):
ans[key] = item.value
elif isinstance(item, h5py._hl.group.Group):
ans[key] = cls.__recursively_load_dict_contents_from_group__(h5file, path + key + '/')
return ans

# a test
if __name__ == "__main__":

ex = {
'name': 'stefan',
'age':  np.int64(24),
'fav_numbers': np.array([2,4,4.3]),
'fav_tensors': {
'levi_civita3d': np.array([
[[0,0,0],[0,0,1],[0,-1,0]],
[[0,0,-1],[0,0,0],[1,0,0]],
[[0,1,0],[-1,0,0],[0,0,0]]
]),
'kronecker2d': np.identity(3)
}
}
print ex
ReportInterface.__save_dict_to_hdf5__(ex, 'foo.hdf5')
print 'check passed!'


which passes its own test:

>>> python hdf5test.py
{'age': 24, 'fav_numbers': array([ 2. ,  4. ,  4.3]), 'name': 'stefan', 'fav_tensors': {'levi_civita3d': array([[[ 0,  0,  0],
[ 0,  0,  1],
[ 0, -1,  0]],

[[ 0,  0, -1],
[ 0,  0,  0],
[ 1,  0,  0]],

[[ 0,  1,  0],
[-1,  0,  0],
[ 0,  0,  0]]]), 'kronecker2d': array([[ 1.,  0.,  0.],
[ 0.,  1.,  0.],
[ 0.,  0.,  1.]])}}
{u'age': 24, u'fav_numbers': array([ 2. ,  4. ,  4.3]), u'name': 'stefan', u'fav_tensors': {u'levi_civita3d': array([[[ 0,  0,  0],
[ 0,  0,  1],
[ 0, -1,  0]],

[[ 0,  0, -1],
[ 0,  0,  0],
[ 1,  0,  0]],

[[ 0,  1,  0],
[-1,  0,  0],
[ 0,  0,  0]]]), u'kronecker2d': array([[ 1.,  0.,  0.],
[ 0.,  1.,  0.],
[ 0.,  0.,  1.]])}}
check passed!

• This will not work, if you want to save np.nan to a file, since nan comparison is always negative. if not h5file[cur_path].value == item and not np.isnan(item) Nov 9, 2016 at 13:15