From d348e038b4d1cc0e61c823fd87b5230608ffff8b Mon Sep 17 00:00:00 2001 From: David Bold Date: Fri, 19 Sep 2025 10:08:35 +0200 Subject: [PATCH 1/9] Switch to h5netcdf as default engine This works around https://github.com/pydata/xarray/issues/9779 --- pyproject.toml | 2 +- requirements.txt | 2 +- xbout/tests/conftest.py | 2 +- xbout/tests/test_grid.py | 4 ++-- xbout/utils.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fc73eb3a..1cfabb12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dependencies = [ "natsort>=5.5.0", "matplotlib>=3.1.1,!=3.3.0,!=3.3.1,!=3.3.2", "animatplot-ng>=0.4.2", - "netcdf4>=1.4.0", + "h5netcdf", "Pillow>=6.1.0", ] diff --git a/requirements.txt b/requirements.txt index 1f8dbfff..73d9aefe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ dask[array]>=2.10.0 natsort>=5.5.0 matplotlib>=3.1.1,!=3.3.0,!=3.3.1,!=3.3.2 animatplot>=0.4.2 -netcdf4>=1.4.0 +h5netcdf Pillow>=6.1.0 # Not required for Python>=3.8, but included because conda does not support diff --git a/xbout/tests/conftest.py b/xbout/tests/conftest.py index 99b3b016..1867f312 100644 --- a/xbout/tests/conftest.py +++ b/xbout/tests/conftest.py @@ -165,7 +165,7 @@ def create_example_grid_file_fci(tmp_path_factory): # Save filepath = save_dir.joinpath("fci.nc") - grid.to_netcdf(filepath, engine="netcdf4") + grid.to_netcdf(filepath, engine="h5netcdf") return filepath diff --git a/xbout/tests/test_grid.py b/xbout/tests/test_grid.py index 007733b4..87ff4a98 100644 --- a/xbout/tests/test_grid.py +++ b/xbout/tests/test_grid.py @@ -27,7 +27,7 @@ def create_example_grid_file(tmp_path_factory): # Save filepath = save_dir.joinpath("grid.nc") - grid.to_netcdf(filepath, engine="netcdf4") + grid.to_netcdf(filepath, engine="h5netcdf") return filepath @@ -52,7 +52,7 @@ def test_open_grid_extra_dims(self, create_example_grid_file, tmp_path_factory): dodgy_grid_directory = tmp_path_factory.mktemp("dodgy_grid") dodgy_grid_path = dodgy_grid_directory.joinpath("dodgy_grid.nc") - merge([example_grid, new_var]).to_netcdf(dodgy_grid_path, engine="netcdf4") + merge([example_grid, new_var]).to_netcdf(dodgy_grid_path, engine="h5netcdf") with pytest.warns( UserWarning, match="drop all variables containing " "the dimensions 'w'" diff --git a/xbout/utils.py b/xbout/utils.py index 66e6439d..b4a00f31 100644 --- a/xbout/utils.py +++ b/xbout/utils.py @@ -31,7 +31,7 @@ def _add_attrs_to_var(ds, varname, copy=False): def _check_filetype(path): if path.suffix == ".nc": - filetype = "netcdf4" + filetype = "h5netcdf" elif path.suffix == ".h5netcdf": filetype = "h5netcdf" elif path.suffix == ".bp": From ee9526fb988da3f27e76cd242c148b7313d96858 Mon Sep 17 00:00:00 2001 From: dschwoerer Date: Fri, 19 Sep 2025 09:27:22 +0000 Subject: [PATCH 2/9] Apply black formatting --- xbout/xarraybackend.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xbout/xarraybackend.py b/xbout/xarraybackend.py index 38f4d65c..6a9644c5 100644 --- a/xbout/xarraybackend.py +++ b/xbout/xarraybackend.py @@ -1,6 +1,6 @@ """License: - Distributed under the OSI-approved Apache License, Version 2.0. See - accompanying file Copyright.txt for details. +Distributed under the OSI-approved Apache License, Version 2.0. See +accompanying file Copyright.txt for details. """ from __future__ import annotations From 5416d9096a44a780ba3189739a801782c0da3a78 Mon Sep 17 00:00:00 2001 From: David Bold Date: Mon, 22 Sep 2025 18:57:26 +0200 Subject: [PATCH 3/9] Update test for switch to h5netcdf --- xbout/tests/test_load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xbout/tests/test_load.py b/xbout/tests/test_load.py index 20ddcae0..6b7ad323 100644 --- a/xbout/tests/test_load.py +++ b/xbout/tests/test_load.py @@ -43,7 +43,7 @@ def test_check_extensions(tmp_path): example_nc_file.write_text("content_nc") filetype = _check_filetype(example_nc_file) - assert filetype == "netcdf4" + assert filetype == "h5netcdf" example_hdf5_file = files_dir.joinpath("example.h5netcdf") example_hdf5_file.write_text("content_hdf5") From 6e261ff28c247719ea1a2a617ab7236ec43d740f Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 27 Nov 2025 13:41:43 +0100 Subject: [PATCH 4/9] CI: Update minimum versions for tests --- .github/workflows/master.yml | 10 ++++------ .github/workflows/pythonpackage.yml | 8 ++++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 4da18ff8..60fc1134 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -20,18 +20,17 @@ jobs: if: always() strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.12", "3.13", "3.14", "3.x"] fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - sudo apt-get update && sudo apt-get install libhdf5-dev libnetcdf-dev python -m pip install --upgrade pip - name: Install package run: | @@ -48,7 +47,7 @@ jobs: if: always() strategy: matrix: - python-version: ["3.8"] + python-version: ["3.12"] fail-fast: false steps: @@ -59,9 +58,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - sudo apt-get update && sudo apt-get install libhdf5-dev libnetcdf-dev python -m pip install --upgrade pip - pip install xarray~=2023.1.0 pandas~=1.4.0 + pip install xarray~=2023.12.0 pandas~=2.2.0 - name: Install package run: | pip install -e .[tests] diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 823eafcf..9fed7c3e 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -20,11 +20,11 @@ jobs: if: always() strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.x"] + python-version: ["3.12", "3.13", "3.14", "3.x"] fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -47,7 +47,7 @@ jobs: if: always() strategy: matrix: - python-version: ["3.8"] + python-version: ["3.12"] fail-fast: false steps: @@ -59,7 +59,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install xarray~=2023.1.0 pandas~=1.4.0 + pip install xarray~=2023.12.0 pandas~=2.2.0 - name: Install package run: | pip install -e .[tests] From 82657994e940d7dbd32055ae04986a7e231885aa Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 27 Nov 2025 13:54:07 +0100 Subject: [PATCH 5/9] CI: Switch to xarray with fix for race condition --- .github/workflows/master.yml | 2 ++ .github/workflows/pythonpackage.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 4da18ff8..e68e886b 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -36,6 +36,7 @@ jobs: - name: Install package run: | pip install -e .[calc,tests] + pip install git+https://github.com/dschwoerer/xarray@netcdf4-locking-closing - name: Test with pytest run: | pytest -vv --long --cov @@ -65,6 +66,7 @@ jobs: - name: Install package run: | pip install -e .[tests] + pip install git+https://github.com/dschwoerer/xarray@netcdf4-locking-closing - name: Test with pytest run: | pytest -vv --long --cov diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 823eafcf..c4c0fab4 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -35,6 +35,7 @@ jobs: - name: Install package run: | pip install -e .[calc,tests] + pip install git+https://github.com/dschwoerer/xarray@netcdf4-locking-closing - name: Test with pytest run: | pytest -vv --long --cov @@ -63,6 +64,7 @@ jobs: - name: Install package run: | pip install -e .[tests] + pip install git+https://github.com/dschwoerer/xarray@netcdf4-locking-closing - name: Test with pytest run: | pytest -vv --long --cov From 7cdc45ef44160e98519d17f1b7b1fb02a29a5f32 Mon Sep 17 00:00:00 2001 From: David Bold Date: Fri, 28 Nov 2025 14:07:15 +0100 Subject: [PATCH 6/9] CI: Install h5netcdf from package manger --- .github/workflows/master.yml | 2 ++ .github/workflows/pythonpackage.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index a42ae96a..0c5848e3 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -32,6 +32,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + sudo apt-get -y install python3-h5netcdf - name: Install package run: | pip install -e .[calc,tests] @@ -60,6 +61,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + sudo apt-get -y install python3-h5netcdf pip install xarray~=2023.12.0 pandas~=2.2.0 - name: Install package run: | diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 26a4ccc1..6573204c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -32,6 +32,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + sudo apt-get -y install python3-h5netcdf - name: Install package run: | pip install -e .[calc,tests] @@ -60,6 +61,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + sudo apt-get -y install python3-h5netcdf pip install xarray~=2023.12.0 pandas~=2.2.0 - name: Install package run: | From 71594c67ee73b2d9e7e293c6b3fcac8bfdc8e57d Mon Sep 17 00:00:00 2001 From: David Bold Date: Fri, 28 Nov 2025 14:17:18 +0100 Subject: [PATCH 7/9] CI: Force h5netcdf from source --- .github/workflows/pythonpackage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 6573204c..c74f9af6 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -32,10 +32,10 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - sudo apt-get -y install python3-h5netcdf + sudo apt-get -y install libhdf5-dev - name: Install package run: | - pip install -e .[calc,tests] + pip install -e .[calc,tests] --no-binary h5netcdf pip install git+https://github.com/dschwoerer/xarray@netcdf4-locking-closing - name: Test with pytest run: | From dc511144dfbae4fc01412a12753ea5eda56b7e45 Mon Sep 17 00:00:00 2001 From: David Bold Date: Mon, 1 Dec 2025 09:06:13 +0100 Subject: [PATCH 8/9] CI: install h5py from souce --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index c74f9af6..13a0dc9e 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -35,7 +35,7 @@ jobs: sudo apt-get -y install libhdf5-dev - name: Install package run: | - pip install -e .[calc,tests] --no-binary h5netcdf + pip install -e .[calc,tests] --no-binary h5netcdf --no-binary h5py pip install git+https://github.com/dschwoerer/xarray@netcdf4-locking-closing - name: Test with pytest run: | From 0a3e2c34e91c51e653515e75d84f3ac4d4f1568a Mon Sep 17 00:00:00 2001 From: David Bold Date: Mon, 1 Dec 2025 13:28:01 +0100 Subject: [PATCH 9/9] CI: Always install h5py from source --- .github/workflows/master.yml | 8 ++++---- .github/workflows/pythonpackage.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 0c5848e3..33019e4a 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -32,10 +32,10 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - sudo apt-get -y install python3-h5netcdf + sudo apt-get -y install libhdf5-dev - name: Install package run: | - pip install -e .[calc,tests] + pip install -e .[calc,tests] --no-binary h5netcdf --no-binary h5py pip install git+https://github.com/dschwoerer/xarray@netcdf4-locking-closing - name: Test with pytest run: | @@ -61,11 +61,11 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - sudo apt-get -y install python3-h5netcdf + sudo apt-get -y install libhdf5-dev pip install xarray~=2023.12.0 pandas~=2.2.0 - name: Install package run: | - pip install -e .[tests] + pip install -e .[tests] --no-binary h5netcdf --no-binary h5py pip install git+https://github.com/dschwoerer/xarray@netcdf4-locking-closing - name: Test with pytest run: | diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 13a0dc9e..f0f9392c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -61,11 +61,11 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - sudo apt-get -y install python3-h5netcdf + sudo apt-get -y install libhdf5-dev pip install xarray~=2023.12.0 pandas~=2.2.0 - name: Install package run: | - pip install -e .[tests] + pip install -e .[tests] --no-binary h5netcdf --no-binary h5py pip install git+https://github.com/dschwoerer/xarray@netcdf4-locking-closing - name: Test with pytest run: |