diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index 7eee536a..45a3b893 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -8,7 +8,7 @@ name: "Bump Patch Version" on: push: branches: - - master + - main paths-ignore: - .cruft.json - .editorconfig diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2fa651d2..67879a7c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,9 +1,9 @@ -name: RavenPy +name: Testing on: push: branches: - - master + - main pull_request: env: @@ -192,7 +192,7 @@ jobs: ~/.cache/raven-testdata key: ${{ hashFiles('src/ravenpy/testing/registry.txt') }}-${{ env.RAVEN_TESTDATA_BRANCH }}-conda-${{ matrix.os }} - - name: Prefetch RavenPy test data + - name: Prefetch RavenPy testing data run: | python -c "import ravenpy.testing.utils as rtu; rtu.populate_testing_data()" diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml new file mode 100644 index 00000000..ead97b00 --- /dev/null +++ b/.github/workflows/notebooks.yml @@ -0,0 +1,85 @@ +name: Notebooks + +on: + push: + branches: + - main + pull_request: + schedule: + - cron: '0 9 * * 1' + +env: + RAVEN_TESTDATA_BRANCH: v2025.6.12 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} + +permissions: + contents: read + +jobs: + notebooks: + name: Test Notebooks (Anaconda, ${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ "ubuntu-latest", "macos-latest" ] + python-version: [ "3.13" ] # pymetalink not yet supported in Python 3.14 + defaults: + run: + shell: bash -l {0} + steps: + - name: Harden Runner + uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2 + with: + disable-sudo: true + egress-policy: audit + + - name: Checkout Repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }} + uses: mamba-org/setup-micromamba@add3a49764cedee8ee24e82dfde87f5bc2914462 # v2.0.7 + with: + cache-downloads: true + cache-environment: true + cache-environment-key: environment-${{ matrix.python-version }}-${{ matrix.os }}-${{ github.head_ref }} + environment-file: environment-dev.yml + create-args: >- + python=${{ matrix.python-version }} + + - name: Install RavenPy + run: | + python -m pip install --no-deps --editable . + + - name: List installed packages + run: | + micromamba list + python -m pip check || true + + - name: Cache test data (macOS) + if: matrix.os == 'macos-latest' + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: | + ~/Library/Caches/raven-testdata + key: ${{ hashFiles('src/ravenpy/testing/registry.txt') }}-${{ env.RAVEN_TESTDATA_BRANCH }}-conda-${{ matrix.os }} + - name: Cache test data (Ubuntu) + if: matrix.os == 'ubuntu-latest' + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: | + ~/.cache/raven-testdata + key: ${{ hashFiles('src/ravenpy/testing/registry.txt') }}-${{ env.RAVEN_TESTDATA_BRANCH }}-conda-${{ matrix.os }} + + - name: Prefetch RavenPy testing data + run: | + python -c "import ravenpy.testing.utils as rtu; rtu.populate_testing_data()" + + - name: Test RavenPy + run: | + make test-notebooks diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 650ec271..8c5a1f51 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -13,7 +13,7 @@ on: - cron: '41 8 * * 4' push: branches: - - master + - main # Read-all permission is not technically needed for this workflow. permissions: diff --git a/.readthedocs.yml b/.readthedocs.yml index cc7338d3..7b442385 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -33,5 +33,4 @@ python: - method: pip path: . extra_requirements: - - dev - docs diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 92ab2a76..df568f4b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,19 +2,26 @@ Changelog ========= -.. - `Unreleased `_ (latest) - ------------------------------------------------------------- +`Unreleased `_ (latest) +------------------------------------------------------------- - Contributors: +Contributors: Trevor James Smith (:user:`Zeitsperre`). + +Changes +^^^^^^^ +* Dependency updates. (PR #584): + * Updated required `xskillscore` (``>= 0.0.29``) and `climpred` (``>= 2.6.0``). + * Removed version pins on `intake` and `intake-xarray`. + * Minimum required `birdy` is now ``>= 0.9.1``. - Changes - ^^^^^^^ - * No change. +Fixes +^^^^^ +* Updated notebooks to address several deprecation warnings stemming from `dask`, `numpy`, and `xarray`. (PR #584) - Fixes - ^^^^^ - * No change. +Internal changes +^^^^^^^^^^^^^^^^ +* Added a Makefile recipe and a GitHub Workflow to run tests against the notebooks using ``pytest --nbval`` on changes as well as on a weekly schedule. (PR #584) +* Fixed a bug in several workflows that was impeding triggers when Pull Requests are merged to `main`. (PR #584) .. _changes_0.20.0: @@ -42,15 +49,15 @@ Fixes Internal changes ^^^^^^^^^^^^^^^^ * Updated the cookiecutter template to the latest version (PR #548): - * Updated the Contributor Covenant Agreement to v3.0. - * Added a `CITATION.cff` file. - * Removed `black`, `blackdoc`, and `isort`, as well as their configurations. - * Updated `pyproject.toml` to be `PEP 639 `_-compliant. + * Updated the Contributor Covenant Agreement to v3.0. + * Added a `CITATION.cff` file. + * Removed `black`, `blackdoc`, and `isort`, as well as their configurations. + * Updated `pyproject.toml` to be `PEP 639 `_-compliant. * Pinned `pydantic` below v2.12 due to breaking changes in their API. (PR #548) - * Unpinned `pydantic` as newer 2.12 patch releases appear to have addressed regressions. (PR #559) + * Unpinned `pydantic` as newer 2.12 patch releases appear to have addressed regressions. (PR #559) * Pinned `pydap` >=3.5.6 and `h5netcdf` >=1.5.0 to ensure modern versions with better `xarray` support are installed by default. (PR #559) * Updated the cookiecutter template to the latest version (PR #569): - * Added a workflow for automatically accepting and merging periodic updates from Dependabot affecting CI dependencies. + * Added a workflow for automatically accepting and merging periodic updates from Dependabot affecting CI dependencies. * Added a `pytest` fixture to perform a teardown of changes performed within the installed `ravenpy` source location. (PR #572) .. _changes_0.19.1: diff --git a/Makefile b/Makefile index 2de269be..b1858630 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ import os, webbrowser, sys from urllib.request import pathname2url -webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) +webbrowser.open(sys.argv[1]) endef export BROWSER_PYSCRIPT @@ -54,6 +54,8 @@ clean-test: ## remove test and coverage artifacts rm -fr htmlcov/ rm -fr .pytest_cache +## Testing targets: + lint/flake8: ## check style with flake8 python -m ruff check src/ravenpy tests python -m flake8 --config=.flake8 src/ravenpy tests @@ -73,7 +75,14 @@ coverage: ## check code coverage quickly with the default Python python -m coverage html $(BROWSER) htmlcov/index.html -autodoc: clean-docs ## create sphinx-apidoc files: +NOTEBOOKS := $(shell find $(CURDIR)/docs/notebooks -name '*.ipynb') + +test-notebooks: ## test all notebooks under docs/notebooks + python -m pytest --nbval --numprocesses=logical --maxprocesses=8 --dist=loadscope $(NOTEBOOKS) + +## Sphinx targets: + +autodoc: clean-docs ## create sphinx-apidoc files sphinx-apidoc -o docs/apidoc --private --module-first src/ravenpy autodoc-custom-index: clean-docs ## create sphinx-apidoc files but with special index handling for indices and indicators @@ -100,6 +109,8 @@ endif servedocs: docs ## compile the docs watching for changes watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . +## Development targets: + dist: clean ## builds source and wheel package python -m flit build ls -l dist diff --git a/docs/conf.py b/docs/conf.py index 9e146692..183b6a38 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -101,10 +101,6 @@ def rebuild_readme(): "notebooks/paper/*.ipynb", ] -# nbsphinx_execute = "auto" -# nbsphinx_timeout = 1 -# nbsphinx_allow_errors = True - extlinks = { "issue": ("https://github.com/CSHS-CWRA/RavenPy/issues/%s", "GH/%s"), "pull": ("https://github.com/CSHS-CWRA/RavenPy/pull/%s", "PR/%s"), @@ -121,33 +117,18 @@ def rebuild_readme(): # To avoid having to install these and burst memory limit on ReadTheDocs. autodoc_mock_imports = [ - "affine", - "cftime", - "cf_xarray", - "click", - "climpred", - "clisops", - "fiona", - "gdal", - "h5netcdf", - "netCDF4", - "osgeo", - "geopandas", - "haversine", - "holoviews", - "hvplot", - "lxml", - "owslib", - "pandas", - "pyproj", - "rasterio", - "rioxarray", - "scipy", - "shapely", - "spotpy", - "statsmodels", - "xarray", - "xclim", + "affine", + "fiona", + "geopandas", + "holoviews", + "hvplot", + "lxml", + "osgeo", + "netCDF4", + "pyproj", + "rasterio", + "rioxarray", + "shapely" ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/notebooks/02_Extract_geographical_watershed_properties.ipynb b/docs/notebooks/02_Extract_geographical_watershed_properties.ipynb index 9d02b8c2..b8e68304 100644 --- a/docs/notebooks/02_Extract_geographical_watershed_properties.ipynb +++ b/docs/notebooks/02_Extract_geographical_watershed_properties.ipynb @@ -417,7 +417,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.13.11" } }, "nbformat": 4, diff --git a/docs/notebooks/08_Getting_and_bias_correcting_CMIP6_data.ipynb b/docs/notebooks/08_Getting_and_bias_correcting_CMIP6_data.ipynb index 00c57f69..069cbe10 100644 --- a/docs/notebooks/08_Getting_and_bias_correcting_CMIP6_data.ipynb +++ b/docs/notebooks/08_Getting_and_bias_correcting_CMIP6_data.ipynb @@ -365,7 +365,7 @@ "\n", " out[exp] = {}\n", " for variable in [\"tasmin\", \"tasmax\", \"pr\"]:\n", - " print(exp, variable)\n", + " print(f\"Now processing: {exp} x {variable}\")\n", " query = dict(\n", " experiment_id=exp,\n", " table_id=\"day\",\n", @@ -558,7 +558,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "tags": [] + "tags": [ + "nbval-skip" + ] }, "outputs": [], "source": [ @@ -569,7 +571,8 @@ " corrected_ref_precip.to_dataset(name=\"pr\"),\n", " corrected_ref_tasmax.to_dataset(name=\"tasmax\"),\n", " corrected_ref_tasmin.to_dataset(name=\"tasmin\"),\n", - " ]\n", + " ],\n", + " compat=\"no_conflicts\"\n", ")\n", "\n", "# Write to temporary folder.\n", @@ -581,8 +584,9 @@ " [\n", " corrected_fut_precip.to_dataset(name=\"pr\"),\n", " corrected_fut_tasmax.to_dataset(name=\"tasmax\"),\n", - " corrected_fut_tasmin.to_dataset(name=\"tasmin\"),\n", - " ]\n", + " corrected_fut_tasmin.to_dataset(name=\"tasmin\")\n", + " ],\n", + " compat=\"no_conflicts\"\n", ")\n", "\n", "fn_fut = tmp / \"future_dataset.nc\"\n", @@ -593,7 +597,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "tags": [] + "tags": [ + "nbval-skip" + ] }, "outputs": [], "source": [ @@ -605,7 +611,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "tags": [] + "tags": [ + "nbval-skip" + ] }, "outputs": [], "source": [ @@ -625,7 +633,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.13.12" } }, "nbformat": 4, diff --git a/docs/notebooks/11_Climatological_ESP_forecasting.ipynb b/docs/notebooks/11_Climatological_ESP_forecasting.ipynb index e5df4ca7..485b8b30 100644 --- a/docs/notebooks/11_Climatological_ESP_forecasting.ipynb +++ b/docs/notebooks/11_Climatological_ESP_forecasting.ipynb @@ -181,6 +181,9 @@ "# Adjust the streamflow to convert missing data from -1.2345 format to NaN. Set all negative values to NaN.\n", "q_obs = q_obs.where(q_obs > 0, np.nan)\n", "\n", + "# Drop non-numerical variable\n", + "q_sims = q_sims.drop_vars(\"basin_fullname\")\n", + "\n", "# Compute the Continuous Ranked Probability Score using xskillscore\n", "xs.crps_ensemble(q_obs, q_sims, dim=\"time\").q_sim.values[0]" ] @@ -287,7 +290,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.9" + "version": "3.13.12" } }, "nbformat": 4, diff --git a/docs/notebooks/paper/Perform_a_climate_change_impact_study_on_a_watershed.ipynb b/docs/notebooks/paper/Perform_a_climate_change_impact_study_on_a_watershed.ipynb index ca1489e8..341ed9de 100644 --- a/docs/notebooks/paper/Perform_a_climate_change_impact_study_on_a_watershed.ipynb +++ b/docs/notebooks/paper/Perform_a_climate_change_impact_study_on_a_watershed.ipynb @@ -454,7 +454,7 @@ "\n", " out[exp] = {}\n", " for variable in [\"tasmin\", \"tasmax\", \"pr\"]:\n", - " print(exp, variable)\n", + " print(f\"Now processing: {exp} x {variable}\")\n", " query = dict(\n", " experiment_id=exp,\n", " table_id=\"day\",\n", @@ -768,12 +768,6 @@ "low = (0.01, -15.0, 10.0, 0.0, 1.0, 0.0)\n", "high = (2.5, 10.0, 700.0, 7.0, 30.0, 1.0)\n", "\n", - "# Random seed.\n", - "# We will provide one for consistency purposes, but operationally this should not be provided.\n", - "# FIXME: This will change in numpy v2.0, so we will need to update this code then.\n", - "random_seed = 42\n", - "np.random.seed(random_seed)\n", - "\n", "# Build the optimizer object\n", "spot_setup = SpotSetup(\n", " config=model_config,\n", @@ -1026,7 +1020,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.9" + "version": "3.13.11" } }, "nbformat": 4, diff --git a/environment-dev.yml b/environment-dev.yml index a3ab9a00..270af7b5 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -5,32 +5,22 @@ dependencies: - python >=3.10,<3.15 - raven-hydro >=0.5.0,<1.0 # - libgcc # for mixing raven-hydro from PyPI with conda environments - - affine >=2.4.0 - cftime >=1.4.1 - cf_xarray >=0.9.3 - click >=8.2.0 - - climpred >=2.4.0 + - climpred >=2.6.0 - dask >=2024.8.1 - - fiona >=1.9.0 - - gdal >=3.1 - - geopandas >=1.0 - h5netcdf >=1.3.0 - haversine >=2.8.0 - matplotlib-base >=3.6.0 - - netcdf4 >=1.7.2 - numpy >=1.25.0 - owslib >=0.29.1 - pandas >=2.2.0 - pint >=0.24.4 - pydantic >=2.11 - pydap >=3.5.6 - - pymetalink >=6.5.2 - pymbolic >=2024.2 - - pyproj >=3.3.0 - - rasterio - - rioxarray - scipy >=1.11.0 - - shapely >=2.0.0 - spotpy >=1.6.1 - statsmodels >=0.14.2 - typing_extensions @@ -38,8 +28,8 @@ dependencies: - xclim >=0.57.0 - xesmf - xsdba >=0.4.0 - - xskillscore - - zarr >=2.13,<3.0 # FIXME: zarr v3 does not support FSMap like before: https://github.com/zarr-developers/zarr-python/issues/2706 + - xskillscore >=0.0.29 + - zarr # Dev tools and testing - pip >=25.2 - bump-my-version >=1.2.3 @@ -50,10 +40,7 @@ dependencies: - flake8 >=7.3.0 - flake8-rst-docstrings >=0.4.0 - flit >=3.11.0,<4.0 - - holoviews - - hvplot - mypy >=1.18.2 - - nbval - numpydoc >=1.9.0 - pooch >=1.8.0 - pre-commit >=3.5.0 @@ -62,7 +49,35 @@ dependencies: - pytest-cov >=5.0.0 - pytest-xdist >=3.2.0 - ruff >=0.13.3 - - setuptools >=71.0 - tox >=4.30.3 - vulture >=2.14 - watchdog >=4.0.0 + # GIS support + - affine >=2.4.0 + - fiona >=1.9.0 + - gdal >=3.1 + - geopandas >=1.0 + - lxml + - netcdf4 >=1.7.2 + - pyproj >=3.3.0 + - rasterio + - rioxarray + - setuptools >=71.0 + - shapely >=2.0.0 + # Notebook support + - birdy >=0.9.1 + - clisops >=0.16.1 + - gcsfs + - holoviews + - hvplot + - intake >=2.0 + - intake-esm >=2023.07.07 + - intake-xarray >=2.0 + - iprogress + - ipyleaflet + - nbval + - pymetalink >=6.5.3 + - s3fs + - salib +# Needed for notebooks/Managing_Jupyter_Environments.ipynb + - seaborn diff --git a/environment-docs.yml b/environment-docs.yml index 4e2f7827..108d1749 100644 --- a/environment-docs.yml +++ b/environment-docs.yml @@ -4,46 +4,40 @@ channels: dependencies: - python >=3.13,<3.14 - raven-hydro >=0.5.0,<1.0 - - autodoc-pydantic >=2.1.0 - - birdy - - cairosvg >=2.6.0 - - cartopy >=0.23.0 + - cftime >=1.4.1 + - cf-xarray >=0.9.3 - click >=8.2.0 - - clisops >=0.15.0 - - gcsfs - - gdal >=3.1 + - climpred >=2.6.0 + - dask >=2024.8.1 - h5netcdf >=1.5.0 + - haversine >=2.8.0 + - matplotlib >=3.6.0 + - numpy >=1.25.0 + - owslib >=0.29.1 + - pandas >=2.2.0 + - pint >=0.24.4 + - pydantic >=2.11 + - pydap >=3.5.6 + - pymbolic >=2024.2 + - scipy >=1.11.0 + - spotpy >=1.6.1 + - statsmodels >=0.14.2 + - typing-extensions + - xarray >=2023.11.0,!=2024.10.0 + - xclim >=0.57.0 + - xsdba >=0.4.0 + - xskillscore >=0.0.29 # Needed for notebooks/HydroShare_integration.ipynb # See: https://github.com/CSHS-CWRA/RavenPy/pull/326 # - "hsclient", - - intake <2.0 - - intake-esm >=2023.07.07 - - intake-xarray <2.0 - - ipykernel - - ipyleaflet - - ipython >=8.5.0 - - jupyter-cache - - jupyter_client - - jupytext - - matplotlib-base >=3.6.0 + # Docs support + - autodoc-pydantic >=2.1.0 + - matplotlib-base >=3.7.0 - myst-nb - - nbsphinx >=0.9.5 - - netCDF4 >=1.7.2 - - numpy >=1.25.0 - - notebook - - pandas >=2.2 - - pydantic >=2.11 - - pymetalink >=6.5.2 - - s3fs - - salib - - seaborn + - nbsphinx >=0.9.8 - sphinx >=8.2.0 - sphinx-click - sphinx-codeautolink >=0.16.2 - sphinx-copybutton - sphinx-rtd-theme >=1.0 - sphinxcontrib-svg2pdfconverter >=1.2.3 - - typing_extensions - - xarray >=2023.11.0,!=2024.10.0 -# - xesmf # mocked - - zarr >=2.13,<3.0 # FIXME: zarr v3 does not support FSMap like before: https://github.com/zarr-developers/zarr-python/issues/2706 diff --git a/pyproject.toml b/pyproject.toml index d7d98bea..b057b04c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ dependencies = [ # cf-xarray is differently named on conda-forge "cf-xarray >=0.9.3", "click >=8.2.0", - "climpred >=2.4.0", + "climpred >=2.6.0", "dask >=2024.8.1", "h5netcdf >=1.5.0", "haversine >=2.8.0", @@ -60,7 +60,7 @@ dependencies = [ "xarray >=2023.11.0,!=2024.10.0", "xclim >=0.57.0", "xsdba >=0.4.0", - "xskillscore" + "xskillscore >=0.0.29" ] [project.optional-dependencies] @@ -94,7 +94,7 @@ dev = [ docs = [ # Requires python >=3.11 "autodoc-pydantic >=2.1.0", - "birdhouse-birdy", + "birdhouse-birdy >=0.9.1", "cairosvg >=2.6.0", "cartopy >=0.23.0", "clisops >=0.16.1", @@ -102,20 +102,20 @@ docs = [ # Needed for notebooks/HydroShare_integration.ipynb # See: https://github.com/CSHS-CWRA/RavenPy/pull/326 # "hsclient", - "intake <2.0", + "intake >=2.0", "intake-esm >=2023.07.07", - "intake-xarray <2.0", + "intake-xarray >=2.0", "ipykernel", "ipyleaflet", - "ipython >=8.5.0", + "ipython >=8.7.0", "ipywidgets", "jupyter-cache", "jupyter_client", "jupytext", - "matplotlib >=3.6.0", + "matplotlib >=3.7.0", "myst_nb", "nbsphinx >=0.9.5", - "pymetalink >=6.5.2", + "pymetalink >=6.5.3", "s3fs", "salib", "sphinx >=8.2.0", @@ -125,7 +125,7 @@ docs = [ "sphinx-rtd-theme >=1.0", "sphinxcontrib-svg2pdfconverter >=1.2.3", "xesmf", - "zarr >=2.10.0,<3.0" # FIXME: zarr v3 does not support FSMap like before: https://github.com/zarr-developers/zarr-python/issues/2706 + "zarr" ] gis = [ "affine >=2.4.0",