Return to Question

EDIT

I've replaced assertions with isinstance checks that raise ValueError()s when they fail. I didn't get rid of the classmethod decorators because they were necessary for running the code within a my ReportInterface class, which is hopefully clearer now that I have included the class declaration.

The code has some redundant comments because it will probably be viewed and forked by other physicists with even less python experience than me; if this is interfering with people's reading it, I can edit the comments out.

Imports:

Class definition, with save/load functions

class ReportInterface(object): # ...more class details... @classmethod def __save_dict_to_hdf5__(cls, dic, filename):   """   Save a dictionary whose contents are only strings, np.float64, np.int64,   np.ndarray, and other dictionaries following this structure   to an HDF5 file. These are the sorts of dictionaries that are meant   to be produced by the ReportInterface__to_dict__() method. The saved dictionary can then be loaded using __load_dict_to_hdf5__(), and the contents of the loaded dictionary will be the same as those of the original.   """   if os.path.exists(filename): raise ValueError('File %s exists, will not overwrite.' % filename)  with h5py.File(filename, 'w') as h5file:   cls.__recursively_save_dict_contents_to_group__(h5file, '/', dic)  @classmethod  def __recursively_save_dict_contents_to_group__(cls, h5file, path, dic):   """   Take an already open HDF5 file and insert the contents of a dictionary   at the current path location. Can call itself recursively to fill   out HDF5 files with the contents of a dictionary.   """   # argument type checking if not isinstance(dic, dict):  raise ValueError("must provide a dictionary")   if not isinstance(path, str):  raise ValueError("path must be a string")   if not isinstance(h5file, h5py._hl.files.File): raise ValueError("must be an open h5py file")   # save items to the hdf5 file for key, item in dic.items():   if not isinstance(key, str):  raise ValueError("dict keys must be strings to save to hdf5")  # save strings, numpy.int64, and numpy.float64 types   if isinstance(item, (np.int64, np.float64, str)):   h5file[path + key] = item   if not h5file[path + key].value == item:  raise ValueError('The data representation in the HDF5 file does not match the original dict.') # save numpy arrays   elif isinstance(item, np.ndarray):   h5file[path + key] = item   if not np.array_equal(h5file[path + key].value, item): raise ValueError('The data representation in the HDF5 file does not match the original dict.')   # save dictionaries elif isinstance(item, dict):  cls.__recursively_save_dict_contents_to_group__(h5file, path + key + '/', item)  # other types cannot be saved and will result in an error else: raise ValueError('Cannot save %s type.' % type(item))    @classmethod  def __load_dict_from_hdf5__(cls, filename):   """   Load a dictionary whose contents are only strings, floats, ints,   numpy arrays, and other dictionaries following this structure   from an HDF5 file. These dictionaries can then be used to reconstruct   ReportInterface subclass instances using the   ReportInterface.__from_dict__() method.   """   with h5py.File(filename, 'r') as h5file:   return cls.__recursively_load_dict_contents_from_group__(h5file, '/')  @classmethod  def __recursively_load_dict_contents_from_group__(cls, h5file, path):   """   Load contents of an HDF5 group. If further groups are encountered,   treat them like dicts and continue to load them recursively.   """   ans = {}   for key, item in h5file[path].items():   if isinstance(item, h5py._hl.dataset.Dataset):   ans[key] = item.value   elif isinstance(item, h5py._hl.group.Group):   ans[key] = cls.__recursively_load_dict_contents_from_group__(h5file, path + key + '/')   return ans 

Unit Test

print 'Testing HDF5 file saving capabilities.' ex = { 'name': 'stefan', 'age': np.int64(24), 'fav_numbers': np.array([2,4,4.3]), 'fav_tensors': { 'levi_civita3d': np.array([ [[0,0,0],[0,0,1],[0,-1,0]], [[0,0,-1],[0,0,0],[1,0,0]], [[0,1,0],[-1,0,0],[0,0,0]] ]), 'kronecker2d': np.identity(3) } } ReportInterface.__save_dict_to_hdf5__(ex, 'foo.hdf5') loaded = ReportInterface.__load_dict_from_hdf5__('foo.hdf5') np.testing.assert_equal(loaded, ex) 

Imports:

Saving the data:

def __save_dict_to_hdf5__(cls, dic, filename): """ Save a dictionary whose contents are only strings, np.float64, np.int64, np.ndarray, and other dictionaries following this structure to an HDF5 file. These are the sorts of dictionaries that are meant to be produced by the ReportInterface__to_dict__() method. """ assert not os.path.exists(filename), 'this is a noclobber operation bud'  with h5py.File(filename, 'w') as h5file: cls.__recursively_save_dict_contents_to_group__(h5file, '/', dic) @classmethod def __recursively_save_dict_contents_to_group__(cls, h5file, path, dic): """ Take an already open HDF5 file and insert the contents of a dictionary at the current path location. Can call itself recursively to fill out HDF5 files with the contents of a dictionary. """ assert type(dic) is types.DictionaryType, "must provide a dictionary" assert type(path) is types.StringType, "path must be a string" assert type(h5file) is h5py._hl.files.File, "must be an open h5py file" for key in dic: assert type(key) == types.StringType, 'dict keys must be strings to save to hdf5' if type(dic[key]) in (np.int64, np.float64, types.StringType): h5file[path + key] = dic[key] assert h5file[path + key].value == dic[key], 'The data representation in the HDF5 file does not match the original dict.' if type(dic[key]) is np.ndarray: h5file[path + key] = dic[key] assert np.array_equal(h5file[path + key].value, dic[key]), 'The data representation in the HDF5 file does not match the original dict.' elif type(dic[key]) is types.DictionaryType:  cls.__recursively_save_dict_contents_to_group__(h5file, path + key + '/', dic[key]) 

Loading the Data:

@classmethod def __load_dict_from_hdf5__(cls, filename): """ Load a dictionary whose contents are only strings, floats, ints, numpy arrays, and other dictionaries following this structure from an HDF5 file. These dictionaries can then be used to reconstruct ReportInterface subclass instances using the ReportInterface.__from_dict__() method. """ with h5py.File(filename, 'r') as h5file: return cls.__recursively_load_dict_contents_from_group__(h5file, '/') @classmethod def __recursively_load_dict_contents_from_group__(cls, h5file, path): """ Load contents of an HDF5 group. If further groups are encountered, treat them like dicts and continue to load them recursively. """ ans = {} for key, item in h5file[path].items(): if type(item) is h5py._hl.dataset.Dataset: ans[key] = item.value elif type(item) is h5py._hl.group.Group: ans[key] = cls.__recursively_load_dict_contents_from_group__(h5file, path + key + '/') return ans 

Imports:

Saving the data:

def __save_dict_to_hdf5__(cls, dic, filename): """ Save a dictionary whose contents are only strings, np.float64, np.int64, np.ndarray, and other dictionaries following this structure to an HDF5 file. These are the sorts of dictionaries that are meant to be produced by the ReportInterface__to_dict__() method. """ assert not os.path.exists(filename), 'this is a noclobber operation bud' with h5py.File(filename, 'w') as h5file: cls.__recursively_save_dict_contents_to_group__(h5file, '/', dic) @classmethod def __recursively_save_dict_contents_to_group__(cls, h5file, path, dic): """ Take an already open HDF5 file and insert the contents of a dictionary at the current path location. Can call itself recursively to fill out HDF5 files with the contents of a dictionary. """ assert type(dic) is types.DictionaryType, "must provide a dictionary" assert type(path) is types.StringType, "path must be a string" assert type(h5file) is h5py._hl.files.File, "must be an open h5py file" for key in dic: assert type(key) == types.StringType, 'dict keys must be strings to save to hdf5' if type(dic[key]) in (np.int64, np.float64, types.StringType): h5file[path + key] = dic[key] assert h5file[path + key].value == dic[key], 'The data representation in the HDF5 file does not match the original dict.' if type(dic[key]) is np.ndarray: h5file[path + key] = dic[key] assert np.array_equal(h5file[path + key].value, dic[key]), 'The data representation in the HDF5 file does not match the original dict.' elif type(dic[key]) is types.DictionaryType: cls.__recursively_save_dict_contents_to_group__(h5file, path + key + '/', dic[key]) 

Loading the Data:

@classmethod def __load_dict_from_hdf5__(cls, filename): """ Load a dictionary whose contents are only strings, floats, ints, numpy arrays, and other dictionaries following this structure from an HDF5 file. These dictionaries can then be used to reconstruct ReportInterface subclass instances using the ReportInterface.__from_dict__() method. """ with h5py.File(filename, 'r') as h5file: return cls.__recursively_load_dict_contents_from_group__(h5file, '/') @classmethod def __recursively_load_dict_contents_from_group__(cls, h5file, path): """ Load contents of an HDF5 group. If further groups are encountered, treat them like dicts and continue to load them recursively. """ ans = {} for key, item in h5file[path].items(): if type(item) is h5py._hl.dataset.Dataset: ans[key] = item.value elif type(item) is h5py._hl.group.Group: ans[key] = cls.__recursively_load_dict_contents_from_group__(h5file, path + key + '/') return ans 

EDIT

I've replaced assertions with isinstance checks that raise ValueError()s when they fail. I didn't get rid of the classmethod decorators because they were necessary for running the code within a my ReportInterface class, which is hopefully clearer now that I have included the class declaration.

The code has some redundant comments because it will probably be viewed and forked by other physicists with even less python experience than me; if this is interfering with people's reading it, I can edit the comments out.

Imports:

Class definition, with save/load functions

class ReportInterface(object): # ...more class details... @classmethod def __save_dict_to_hdf5__(cls, dic, filename):   """   Save a dictionary whose contents are only strings, np.float64, np.int64,   np.ndarray, and other dictionaries following this structure   to an HDF5 file. These are the sorts of dictionaries that are meant   to be produced by the ReportInterface__to_dict__() method. The saved dictionary can then be loaded using __load_dict_to_hdf5__(), and the contents of the loaded dictionary will be the same as those of the original.   """  if os.path.exists(filename):   raise ValueError('File %s exists, will not overwrite.' % filename)   with h5py.File(filename, 'w') as h5file:   cls.__recursively_save_dict_contents_to_group__(h5file, '/', dic)  @classmethod  def __recursively_save_dict_contents_to_group__(cls, h5file, path, dic):   """   Take an already open HDF5 file and insert the contents of a dictionary   at the current path location. Can call itself recursively to fill   out HDF5 files with the contents of a dictionary.   """   # argument type checking if not isinstance(dic, dict):    raise ValueError("must provide a dictionary")  if not isinstance(path, str):    raise ValueError("path must be a string")  if not isinstance(h5file, h5py._hl.files.File):  raise ValueError("must be an open h5py file")   # save items to the hdf5 file for key, item in dic.items():  if not isinstance(key, str):    raise ValueError("dict keys must be strings to save to hdf5")   # save strings, numpy.int64, and numpy.float64 types if isinstance(item, (np.int64, np.float64, str)):   h5file[path + key] = item   if not h5file[path + key].value == item:  raise ValueError('The data representation in the HDF5 file does not match the original dict.')  # save numpy arrays  elif isinstance(item, np.ndarray):   h5file[path + key] = item   if not np.array_equal(h5file[path + key].value, item):  raise ValueError('The data representation in the HDF5 file does not match the original dict.')   # save dictionaries elif isinstance(item, dict):   cls.__recursively_save_dict_contents_to_group__(h5file, path + key + '/', item)  # other types cannot be saved and will result in an error else: raise ValueError('Cannot save %s type.' % type(item)) @classmethod  def __load_dict_from_hdf5__(cls, filename):   """   Load a dictionary whose contents are only strings, floats, ints,   numpy arrays, and other dictionaries following this structure   from an HDF5 file. These dictionaries can then be used to reconstruct   ReportInterface subclass instances using the   ReportInterface.__from_dict__() method.   """   with h5py.File(filename, 'r') as h5file:   return cls.__recursively_load_dict_contents_from_group__(h5file, '/')  @classmethod  def __recursively_load_dict_contents_from_group__(cls, h5file, path):   """   Load contents of an HDF5 group. If further groups are encountered,   treat them like dicts and continue to load them recursively.   """   ans = {}   for key, item in h5file[path].items():   if isinstance(item, h5py._hl.dataset.Dataset):   ans[key] = item.value   elif isinstance(item, h5py._hl.group.Group):   ans[key] = cls.__recursively_load_dict_contents_from_group__(h5file, path + key + '/')   return ans 

Unit Test

print 'Testing HDF5 file saving capabilities.' ex = { 'name': 'stefan', 'age': np.int64(24), 'fav_numbers': np.array([2,4,4.3]), 'fav_tensors': { 'levi_civita3d': np.array([ [[0,0,0],[0,0,1],[0,-1,0]], [[0,0,-1],[0,0,0],[1,0,0]], [[0,1,0],[-1,0,0],[0,0,0]] ]), 'kronecker2d': np.identity(3) } } ReportInterface.__save_dict_to_hdf5__(ex, 'foo.hdf5') loaded = ReportInterface.__load_dict_from_hdf5__('foo.hdf5') np.testing.assert_equal(loaded, ex)