From 08ce59aba5f1a2f400576bf2e56ba3dd28bad3a8 Mon Sep 17 00:00:00 2001 From: Mike Kryjak Date: Wed, 4 Mar 2026 12:36:30 +0000 Subject: [PATCH 1/5] Add file engine global variable option --- xbout/load.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/xbout/load.py b/xbout/load.py index 6a8effd2..2cf2c450 100644 --- a/xbout/load.py +++ b/xbout/load.py @@ -18,6 +18,11 @@ _is_dir, ) +# Override file reading engine. +# Use: xbout.load.file_engine = "netcdf4" or "h5netcdf" +file_engine = None + + _BOUT_GEOMETRY_VARS = [ "ixseps1", "ixseps2", @@ -584,7 +589,9 @@ def _check_dataset_type(datapath): filepaths, filetype = _expand_filepaths(datapath) - ds = xr.open_dataset(filepaths[0], engine=filetype) + ds = xr.open_dataset( + filepaths[0], engine=file_engine if file_engine is not None else filetype + ) ds.close() if "metadata:keep_yboundaries" in ds.attrs: # (i) @@ -653,7 +660,7 @@ def _auto_open_mfboutdataset( concat_dim=concat_dims, combine="nested", preprocess=_preprocess, - engine=filetype, + engine=file_engine if file_engine is not None else filetype, chunks=chunks, # Only data variables in which the dimension already # appears are concatenated. @@ -790,7 +797,7 @@ def get_nonnegative_scalar(ds, key, default=1, info=True): print(f"{key} not found, setting to {default}") if default < 0: raise ValueError( - f"Default for {key} is {val}," f" but negative values are not valid" + f"Default for {key} is {val}, but negative values are not valid" ) return default @@ -1117,7 +1124,11 @@ def _open_grid(datapath, chunks, keep_xboundaries, keep_yboundaries, mxg=2, **kw if _is_path(datapath): gridfilepath = Path(datapath) grid = xr.open_dataset( - gridfilepath, engine=_check_filetype(gridfilepath), **kwargs + gridfilepath, + engine=file_engine + if file_engine is not None + else _check_filetype(gridfilepath), + **kwargs, ) else: grid = datapath From 1886ed8343f3815c42a457c72b85bf0c4b66baaa Mon Sep 17 00:00:00 2001 From: Mike Kryjak Date: Wed, 4 Mar 2026 13:00:15 +0000 Subject: [PATCH 2/5] Add h5py/netcdf error message --- xbout/load.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/xbout/load.py b/xbout/load.py index 2cf2c450..ab3e5a0d 100644 --- a/xbout/load.py +++ b/xbout/load.py @@ -589,10 +589,31 @@ def _check_dataset_type(datapath): filepaths, filetype = _expand_filepaths(datapath) - ds = xr.open_dataset( - filepaths[0], engine=file_engine if file_engine is not None else filetype - ) - ds.close() + try: + ds = xr.open_dataset( + filepaths[0], engine=file_engine if file_engine is not None else filetype + ) + ds.close() + except RuntimeError as e: + if "H5DSget_num_scales" in str(e): + msg = ( + "\n\nFailed to open dataset due to an HDF5 compatibility error between\n" + "h5py and h5netcdf, likely because both were installed with pip.\n" + "See: https://github.com/boutproject/xBOUT/issues/329\n" + "Also see: https://github.com/HDFGroup/hdf5/issues/6268\n\n" + "There are three possible fixes:\n" + " 1. Install both from source against a single shared HDF5:\n" + " sudo apt install libhdf5-dev libnetcdf-dev\n" + " pip install --no-binary netCDF4,h5py netCDF4 h5py\n\n" + " 2. Install both from your distribution package manager,\n" + " e.g. apt, conda or Spack\n\n" + " 3. Switch to the netcdf4 engine:\n" + " import xbout\n" + " xbout.load.file_engine = 'netcdf4'\n\n" + f"Original error: {e}" + ) + raise RuntimeError(msg) from e + raise if "metadata:keep_yboundaries" in ds.attrs: # (i) return "reload" From f23b1087a1940d17a58822c719b092f603d6cd94 Mon Sep 17 00:00:00 2001 From: mikekryjak Date: Wed, 4 Mar 2026 13:00:15 +0000 Subject: [PATCH 3/5] Apply black formatting --- xbout/load.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/xbout/load.py b/xbout/load.py index ab3e5a0d..2798c696 100644 --- a/xbout/load.py +++ b/xbout/load.py @@ -1146,9 +1146,11 @@ def _open_grid(datapath, chunks, keep_xboundaries, keep_yboundaries, mxg=2, **kw gridfilepath = Path(datapath) grid = xr.open_dataset( gridfilepath, - engine=file_engine - if file_engine is not None - else _check_filetype(gridfilepath), + engine=( + file_engine + if file_engine is not None + else _check_filetype(gridfilepath) + ), **kwargs, ) else: From 5836d3d70d1b00c8c62d2ff87887176ecc11413c Mon Sep 17 00:00:00 2001 From: mikekryjak <62797494+mikekryjak@users.noreply.github.com> Date: Fri, 6 Mar 2026 09:33:37 +0000 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: David Bold --- xbout/load.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/xbout/load.py b/xbout/load.py index 2798c696..85a3d0ab 100644 --- a/xbout/load.py +++ b/xbout/load.py @@ -591,7 +591,7 @@ def _check_dataset_type(datapath): try: ds = xr.open_dataset( - filepaths[0], engine=file_engine if file_engine is not None else filetype + filepaths[0], engine=file_engine or filetype ) ds.close() except RuntimeError as e: @@ -606,11 +606,12 @@ def _check_dataset_type(datapath): " sudo apt install libhdf5-dev libnetcdf-dev\n" " pip install --no-binary netCDF4,h5py netCDF4 h5py\n\n" " 2. Install both from your distribution package manager,\n" - " e.g. apt, conda or Spack\n\n" + " like apt or dnf or install with conda or Spack\n\n" " 3. Switch to the netcdf4 engine:\n" " import xbout\n" - " xbout.load.file_engine = 'netcdf4'\n\n" - f"Original error: {e}" + " xbout.load.file_engine = 'netcdf4'\n" + " netcdf4 may however cause segfaults on file closure\n\n" + f"Original error:\n\t{e}" ) raise RuntimeError(msg) from e raise @@ -681,7 +682,7 @@ def _auto_open_mfboutdataset( concat_dim=concat_dims, combine="nested", preprocess=_preprocess, - engine=file_engine if file_engine is not None else filetype, + engine=file_engine or filetype, chunks=chunks, # Only data variables in which the dimension already # appears are concatenated. @@ -1148,8 +1149,7 @@ def _open_grid(datapath, chunks, keep_xboundaries, keep_yboundaries, mxg=2, **kw gridfilepath, engine=( file_engine - if file_engine is not None - else _check_filetype(gridfilepath) + or _check_filetype(gridfilepath) ), **kwargs, ) From 5a929059147d1300512e3cb1e19f28d2de0fe3ad Mon Sep 17 00:00:00 2001 From: mikekryjak Date: Fri, 6 Mar 2026 09:33:57 +0000 Subject: [PATCH 5/5] Apply black formatting --- xbout/load.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/xbout/load.py b/xbout/load.py index 85a3d0ab..4f565f8b 100644 --- a/xbout/load.py +++ b/xbout/load.py @@ -590,9 +590,7 @@ def _check_dataset_type(datapath): filepaths, filetype = _expand_filepaths(datapath) try: - ds = xr.open_dataset( - filepaths[0], engine=file_engine or filetype - ) + ds = xr.open_dataset(filepaths[0], engine=file_engine or filetype) ds.close() except RuntimeError as e: if "H5DSget_num_scales" in str(e): @@ -1147,10 +1145,7 @@ def _open_grid(datapath, chunks, keep_xboundaries, keep_yboundaries, mxg=2, **kw gridfilepath = Path(datapath) grid = xr.open_dataset( gridfilepath, - engine=( - file_engine - or _check_filetype(gridfilepath) - ), + engine=(file_engine or _check_filetype(gridfilepath)), **kwargs, ) else: