- Notifications
You must be signed in to change notification settings - Fork 268
Open
Description
Our CI broke today-ish with the error below with python 3.12 and 3.13. We do not test with 3.14.
I cannot reproduce the error on windows. The error only appears when h5py is installed and imported.
Limiting the version to "netcdf4<1.7.3" is our current workaround.
A minimal reproducer that leverages xarray is
uv pip install xarray netcdf4==1.7.3 h5py==3.15.0import xarray as xr import h5py ds = xr.Dataset({"rootvar": ("x", [100, 200])}) ds.to_netcdf('./test_out.nc', engine='netcdf4', format="NETCDF4")CI error message
dt = <xarray.DataTree 'first'> Group: / │ Attributes: │ elicit_md: b'{"gate":{"gate_1":{"data":0.1,"time":176... uuid: 0d796dfb-60ef-42bc-9a7c-0f8aa5ca4e99 uuid_root: ded3e81e-260f-47c5-b860-a0e233cc93fd filepath = '/tmp/pytest-of-root/pytest-0/test_simple_experiment_with_xr0/2025-10-15 10_19_22_63 first.nc' mode = 'w', encoding = {'/foo': {}}, unlimited_dims = {}, format = 'NETCDF4' engine = 'netcdf4', group = None, write_inherited_coords = False, compute = True invalid_netcdf = False, auto_complex = None def _datatree_to_netcdf( dt: DataTree, filepath: str | PathLike | io.IOBase | None = None, mode: NetcdfWriteModes = "w", encoding: Mapping[str, Any] | None = None, unlimited_dims: Mapping | None = None, format: T_DataTreeNetcdfTypes | None = None, engine: T_DataTreeNetcdfEngine | None = None, group: str | None = None, write_inherited_coords: bool = False, compute: bool = True, invalid_netcdf: bool = False, auto_complex: bool | None = None, ) -> None | memoryview | Delayed: """Implementation of `DataTree.to_netcdf`.""" if format not in [None, *get_args(T_DataTreeNetcdfTypes)]: raise ValueError("DataTree.to_netcdf only supports the NETCDF4 format") if engine not in [None, *get_args(T_DataTreeNetcdfEngine)]: raise ValueError( "DataTree.to_netcdf only supports the netcdf4 and h5netcdf engines" ) normalized_path = _normalize_path(filepath) if engine is None: engine = get_default_netcdf_write_engine( path_or_file=normalized_path, format="NETCDF4", # required for supporting groups ) # type: ignore[assignment] if group is not None: raise NotImplementedError( "specifying a root group for the tree has not been implemented" ) if encoding is None: encoding = {} # In the future, we may want to expand this check to insure all the provided encoding # options are valid. For now, this simply checks that all provided encoding keys are # groups in the datatree. if set(encoding) - set(dt.groups): raise ValueError( f"unexpected encoding group name(s) provided: {set(encoding) - set(dt.groups)}" ) if normalized_path is None: if not compute: raise NotImplementedError( "to_netcdf() with compute=False is not yet implemented when " "returning a memoryview" ) target = BytesIOProxy() else: target = normalized_path # type: ignore[assignment] if unlimited_dims is None: unlimited_dims = {} scheduler = get_dask_scheduler() have_chunks = any( v.chunks is not None for node in dt.subtree for v in node.variables.values() ) autoclose = have_chunks and scheduler in ["distributed", "multiprocessing"] root_store = get_writable_netcdf_store( target, engine, # type: ignore[arg-type] mode=mode, format=format, autoclose=autoclose, invalid_netcdf=invalid_netcdf, auto_complex=auto_complex, ) writer = ArrayWriter() # TODO: allow this work (setting up the file for writing array data) # to be parallelized with dask try: for node in dt.subtree: at_root = node is dt dataset = node.to_dataset(inherit=write_inherited_coords or at_root) node_store = ( root_store if at_root else root_store.get_child_store(node.path) ) > dump_to_store( dataset, node_store, writer, encoding=encoding.get(node.path), unlimited_dims=unlimited_dims.get(node.path), ) $VENV/lib/python3.12/site-packages/xarray/backends/writers.py:899: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ $VENV/lib/python3.12/site-packages/xarray/backends/writers.py:491: in dump_to_store store.store(variables, attrs, check_encoding, writer, unlimited_dims=unlimited_dims) $VENV/lib/python3.12/site-packages/xarray/backends/common.py:533: in store self.set_variables( $VENV/lib/python3.12/site-packages/xarray/backends/common.py:575: in set_variables writer.add(source, target) $VENV/lib/python3.12/site-packages/xarray/backends/common.py:403: in add target[...] = source ^^^^^^^^^^^ $VENV/lib/python3.12/site-packages/xarray/backends/netCDF4_.py:95: in __setitem__ data[key] = value ^^^^^^^^^ src/netCDF4/_netCDF4.pyx:5645: in netCDF4._netCDF4.Variable.__setitem__ ??? src/netCDF4/_netCDF4.pyx:5932: in netCDF4._netCDF4.Variable._put ??? _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ > ??? E RuntimeError: NetCDF: HDF error src/netCDF4/_netCDF4.pyx:2160: RuntimeError NickGeneva and adamjstewart
Metadata
Metadata
Assignees
Labels
No labels