pytests failing on ubuntu/x86 #68
Description
Running pytests on Ubuntu/x86 on the main branch gives me a bunch of unexpected failures related to NetCDF/HDF/h5netcdf. Running those same tests on macOS/arm64 gives me no problems.
What seems to be happening is that when reading a BytesIO object, the h5netcdf backend must be used because netcdf4 can't read those. But on x86, the NWM output netCDFs can only be read with netcdf4. This is not a problem on arm64
Running an example in the docs folder
ForcingProcessor has awoken. Let's do this.
NWM file names:
/home/qylee/Documents/forcingprocessor/docs/examples/troute-restart_example/nwm.t00z.analysis_assim.channel_rt.tm00.conus.nc
Entering data extraction...
Traceback (most recent call last):
File "/home/qylee/Documents/forcingprocessor/.venv/lib/python3.11/site-packages/xarray/backends/file_manager.py", line 219, in _acquire_with_cache_info
file = self._cache[self._key]
~~~~~~~~~~~^^^^^^^^^^^
File "/home/qylee/Documents/forcingprocessor/.venv/lib/python3.11/site-packages/xarray/backends/lru_cache.py", line 56, in __getitem__
value = self._cache[key]
~~~~~~~~~~~^^^^^
KeyError: [<class 'h5netcdf.core.File'>, ('/home/qylee/Documents/forcingprocessor/docs/examples/troute-restart_example/nwm.t00z.analysis_assim.channel_rt.tm00.conus.nc',), 'r', (('decode_vlen_strings', True), ('driver', None), ('format', 'NETCDF4'), ('invalid_netcdf', None), ('phony_dims', 'access')), '93c7b998-0d27-472f-8cf5-8178f9200ae0']
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/qylee/Documents/forcingprocessor/docs/examples/troute-restart_example/../../../src/forcingprocessor/processor.py", line 1329, in <module>
prep_ngen_data(conf)
File "/home/qylee/Documents/forcingprocessor/docs/examples/troute-restart_example/../../../src/forcingprocessor/processor.py", line 1088, in prep_ngen_data
with xr.open_dataset(file_obj, engine="h5netcdf") as nwm_ds:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/qylee/Documents/forcingprocessor/.venv/lib/python3.11/site-packages/xarray/backends/api.py", line 607, in open_dataset
backend_ds = backend.open_dataset(
^^^^^^^^^^^^^^^^^^^^^
File "/home/qylee/Documents/forcingprocessor/.venv/lib/python3.11/site-packages/xarray/backends/h5netcdf_.py", line 540, in open_dataset
store = H5NetCDFStore.open(
^^^^^^^^^^^^^^^^^^^
File "/home/qylee/Documents/forcingprocessor/.venv/lib/python3.11/site-packages/xarray/backends/h5netcdf_.py", line 242, in open
return cls(
^^^^
File "/home/qylee/Documents/forcingprocessor/.venv/lib/python3.11/site-packages/xarray/backends/h5netcdf_.py", line 152, in __init__
self._filename = find_root_and_group(self.ds)[0].filename
^^^^^^^
File "/home/qylee/Documents/forcingprocessor/.venv/lib/python3.11/site-packages/xarray/backends/h5netcdf_.py", line 260, in ds
return self._acquire()
^^^^^^^^^^^^^^^
File "/home/qylee/Documents/forcingprocessor/.venv/lib/python3.11/site-packages/xarray/backends/h5netcdf_.py", line 252, in _acquire
with self._manager.acquire_context(needs_lock) as root:
File "/home/qylee/.local/share/uv/python/cpython-3.11.13-linux-x86_64-gnu/lib/python3.11/contextlib.py", line 137, in __enter__
return next(self.gen)
^^^^^^^^^^^^^^
File "/home/qylee/Documents/forcingprocessor/.venv/lib/python3.11/site-packages/xarray/backends/file_manager.py", line 207, in acquire_context
file, cached = self._acquire_with_cache_info(needs_lock)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/qylee/Documents/forcingprocessor/.venv/lib/python3.11/site-packages/xarray/backends/file_manager.py", line 225, in _acquire_with_cache_info
file = self._opener(*self._args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/qylee/Documents/forcingprocessor/.venv/lib/python3.11/site-packages/h5netcdf/core.py", line 1962, in __init__
super().__init__(self, self._h5path)
File "/home/qylee/Documents/forcingprocessor/.venv/lib/python3.11/site-packages/h5netcdf/core.py", line 1174, in __init__
if _unlabeled_dimension_mix(v) == "unlabeled":
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/qylee/Documents/forcingprocessor/.venv/lib/python3.11/site-packages/h5netcdf/core.py", line 914, in _unlabeled_dimension_mix
dimset = {len(j) for j in dimlist}
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/qylee/Documents/forcingprocessor/.venv/lib/python3.11/site-packages/h5netcdf/core.py", line 914, in <setcomp>
dimset = {len(j) for j in dimlist}
^^^^^^
File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
File "/home/qylee/Documents/forcingprocessor/.venv/lib/python3.11/site-packages/h5py/_hl/dims.py", line 60, in __len__
return h5ds.get_num_scales(self._id, self._dimension)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
File "h5py/h5ds.pyx", line 71, in h5py.h5ds.get_num_scales
File "h5py/defs.pyx", line 4282, in h5py.defs.H5DSget_num_scales
RuntimeError: Unspecified error in H5DSget_num_scales (return value <0)
Running a pytest
(forcingprocessor) qylee@cahaba:~/Documents/forcingprocessor$ pytest -q --tb=short --disable-warnings tests/test_plotter.py
F [100%]
=========================================================================== FAILURES ============================================================================
______________________________________________________________________ test_forcings_plot _______________________________________________________________________
.venv/lib/python3.11/site-packages/xarray/backends/file_manager.py:219: in _acquire_with_cache_info
file = self._cache[self._key]
^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.11/site-packages/xarray/backends/lru_cache.py:56: in __getitem__
value = self._cache[key]
^^^^^^^^^^^^^^^^
E KeyError: [<class 'netCDF4._netCDF4.Dataset'>, ('/home/qylee/Documents/forcingprocessor/tests/data/nwm_forcings/nwm.t00z.short_range.forcing.f001.conus.nc',), 'r', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False)), '9c40233c-3dcd-4fb6-8c51-14b5ffe16053']
During handling of the above exception, another exception occurred:
tests/test_plotter.py:95: in test_forcings_plot
nwm_data = get_nwm_data_array(nwm_dir,geopackage)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
src/forcingprocessor/plot_forcings.py:147: in get_nwm_data_array
ds = xr.open_dataset(jfile_path)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.11/site-packages/xarray/backends/api.py:607: in open_dataset
backend_ds = backend.open_dataset(
.venv/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:771: in open_dataset
store = NetCDF4DataStore.open(
.venv/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:529: in open
return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:429: in __init__
self.format = self.ds.data_model
^^^^^^^
.venv/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:538: in ds
return self._acquire()
^^^^^^^^^^^^^^^
.venv/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:532: in _acquire
with self._manager.acquire_context(needs_lock) as root:
../../.local/share/uv/python/cpython-3.11.13-linux-x86_64-gnu/lib/python3.11/contextlib.py:137: in __enter__
return next(self.gen)
^^^^^^^^^^^^^^
.venv/lib/python3.11/site-packages/xarray/backends/file_manager.py:207: in acquire_context
file, cached = self._acquire_with_cache_info(needs_lock)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.11/site-packages/xarray/backends/file_manager.py:225: in _acquire_with_cache_info
file = self._opener(*self._args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
src/netCDF4/_netCDF4.pyx:2521: in netCDF4._netCDF4.Dataset.__init__
???
src/netCDF4/_netCDF4.pyx:2158: in netCDF4._netCDF4._ensure_nc_success
???
E OSError: [Errno -101] NetCDF: HDF error: '/home/qylee/Documents/forcingprocessor/tests/data/nwm_forcings/nwm.t00z.short_range.forcing.f001.conus.nc'