Skip to content

HDF5 error with netcdf4 1.7.3 and h5py on linux with pypi #1438

@shumpohl

Description

@shumpohl

Our CI broke today-ish with the error below with python 3.12 and 3.13. We do not test with 3.14.

I cannot reproduce the error on windows. The error only appears when h5py is installed and imported.

Limiting the version to "netcdf4<1.7.3" is our current workaround.

A minimal reproducer that leverages xarray is

uv pip install xarray netcdf4==1.7.3 h5py==3.15.0
import xarray as xr import h5py ds = xr.Dataset({"rootvar": ("x", [100, 200])}) ds.to_netcdf('./test_out.nc', engine='netcdf4', format="NETCDF4")
CI error message
dt = <xarray.DataTree 'first'> Group: / │ Attributes: │ elicit_md: b'{"gate":{"gate_1":{"data":0.1,"time":176... uuid: 0d796dfb-60ef-42bc-9a7c-0f8aa5ca4e99 uuid_root: ded3e81e-260f-47c5-b860-a0e233cc93fd filepath = '/tmp/pytest-of-root/pytest-0/test_simple_experiment_with_xr0/2025-10-15 10_19_22_63 first.nc' mode = 'w', encoding = {'/foo': {}}, unlimited_dims = {}, format = 'NETCDF4' engine = 'netcdf4', group = None, write_inherited_coords = False, compute = True invalid_netcdf = False, auto_complex = None def _datatree_to_netcdf( dt: DataTree, filepath: str | PathLike | io.IOBase | None = None, mode: NetcdfWriteModes = "w", encoding: Mapping[str, Any] | None = None, unlimited_dims: Mapping | None = None, format: T_DataTreeNetcdfTypes | None = None, engine: T_DataTreeNetcdfEngine | None = None, group: str | None = None, write_inherited_coords: bool = False, compute: bool = True, invalid_netcdf: bool = False, auto_complex: bool | None = None, ) -> None | memoryview | Delayed: """Implementation of `DataTree.to_netcdf`.""" if format not in [None, *get_args(T_DataTreeNetcdfTypes)]: raise ValueError("DataTree.to_netcdf only supports the NETCDF4 format") if engine not in [None, *get_args(T_DataTreeNetcdfEngine)]: raise ValueError( "DataTree.to_netcdf only supports the netcdf4 and h5netcdf engines" ) normalized_path = _normalize_path(filepath) if engine is None: engine = get_default_netcdf_write_engine( path_or_file=normalized_path, format="NETCDF4", # required for supporting groups ) # type: ignore[assignment] if group is not None: raise NotImplementedError( "specifying a root group for the tree has not been implemented" ) if encoding is None: encoding = {} # In the future, we may want to expand this check to insure all the provided encoding # options are valid. For now, this simply checks that all provided encoding keys are # groups in the datatree. if set(encoding) - set(dt.groups): raise ValueError( f"unexpected encoding group name(s) provided: {set(encoding) - set(dt.groups)}" ) if normalized_path is None: if not compute: raise NotImplementedError( "to_netcdf() with compute=False is not yet implemented when " "returning a memoryview" ) target = BytesIOProxy() else: target = normalized_path # type: ignore[assignment] if unlimited_dims is None: unlimited_dims = {} scheduler = get_dask_scheduler() have_chunks = any( v.chunks is not None for node in dt.subtree for v in node.variables.values() ) autoclose = have_chunks and scheduler in ["distributed", "multiprocessing"] root_store = get_writable_netcdf_store( target, engine, # type: ignore[arg-type] mode=mode, format=format, autoclose=autoclose, invalid_netcdf=invalid_netcdf, auto_complex=auto_complex, ) writer = ArrayWriter() # TODO: allow this work (setting up the file for writing array data) # to be parallelized with dask try: for node in dt.subtree: at_root = node is dt dataset = node.to_dataset(inherit=write_inherited_coords or at_root) node_store = ( root_store if at_root else root_store.get_child_store(node.path) ) > dump_to_store( dataset, node_store, writer, encoding=encoding.get(node.path), unlimited_dims=unlimited_dims.get(node.path), ) $VENV/lib/python3.12/site-packages/xarray/backends/writers.py:899: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ $VENV/lib/python3.12/site-packages/xarray/backends/writers.py:491: in dump_to_store store.store(variables, attrs, check_encoding, writer, unlimited_dims=unlimited_dims) $VENV/lib/python3.12/site-packages/xarray/backends/common.py:533: in store self.set_variables( $VENV/lib/python3.12/site-packages/xarray/backends/common.py:575: in set_variables writer.add(source, target) $VENV/lib/python3.12/site-packages/xarray/backends/common.py:403: in add target[...] = source ^^^^^^^^^^^ $VENV/lib/python3.12/site-packages/xarray/backends/netCDF4_.py:95: in __setitem__ data[key] = value ^^^^^^^^^ src/netCDF4/_netCDF4.pyx:5645: in netCDF4._netCDF4.Variable.__setitem__ ??? src/netCDF4/_netCDF4.pyx:5932: in netCDF4._netCDF4.Variable._put ??? _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ > ??? E RuntimeError: NetCDF: HDF error src/netCDF4/_netCDF4.pyx:2160: RuntimeError 

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions