From 070e4d212e7523a7835eb4f9fa3008256ae6cf7c Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 13 Aug 2026 17:58:07 -0500 Subject: [PATCH 1/5] Add pre-commit auto-update workflow Add pre-commit auto-update workflow by copying changes from equivalent zppy-interfaces PR: https://github.com/E3SM-Project/zppy-interfaces/pull/57/changes --- .../workflows/pre_commit_update_workflow.yml | 93 +++++++ .pre-commit-config.yaml | 2 + conda/dev.yml | 5 +- scripts/sync_pre_commit_versions.py | 229 ++++++++++++++++++ 4 files changed, 328 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/pre_commit_update_workflow.yml create mode 100644 scripts/sync_pre_commit_versions.py diff --git a/.github/workflows/pre_commit_update_workflow.yml b/.github/workflows/pre_commit_update_workflow.yml new file mode 100644 index 00000000..da7a2a72 --- /dev/null +++ b/.github/workflows/pre_commit_update_workflow.yml @@ -0,0 +1,93 @@ +name: Pre-commit auto-update + +on: + schedule: + # Cron syntax: + # 1. Entry: Minute when the process will be started [0-59] + # 2. Entry: Hour when the process will be started [0-23] + # 3. Entry: Day of the month when the process will be started [1-28/29/30/31] + # 4. Entry: Month of the year when the process will be started [1-12] + # 5. Entry: Weekday when the process will be started [0-6] [0 is Sunday] + - cron: '0 8 1 * *' + # Allow manual triggering of the workflow + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + issues: write + +env: + UP_TO_DATE: false + PYTHON_VERSION: "3.14" + REVIEWERS: "xylar,forsyth2" +jobs: + auto-update: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Set up Conda Environment + uses: mamba-org/setup-micromamba@v3 + with: + environment-name: pre_commit_dev + init-shell: bash + condarc: | + channel_priority: strict + channels: + - conda-forge + create-args: >- + python=${{ env.PYTHON_VERSION }} + + - name: Install pre-commit, pre-commit-update and gh + run: | + eval "$(micromamba shell hook --shell bash)" + micromamba activate pre_commit_dev + # permissions issue with gh 2.76.0 + micromamba install -y --override-channels -c conda-forge \ + pre-commit pyyaml "gh !=2.76.0" + python -m pip install pre-commit-update + gh --version + + - name: Apply and commit updates + run: | + eval "$(micromamba shell hook --shell bash)" + micromamba activate pre_commit_dev + git clone https://github.com/E3SM-Project/zppy.git update-pre-commit-deps + cd update-pre-commit-deps + # Configure git using GitHub Actions credentials. + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + git checkout -b update-pre-commit-deps + # pre-commit-update doesn't try to include non-release versions + pre-commit-update + # Propagate the new versions to conda/dev.yml and the `qa` extra in + # pyproject.toml + python scripts/sync_pre_commit_versions.py + git add . + # The second command will fail if no changes were present, so we ignore it + git commit -m "Update pre-commit dependencies" || ( echo "UP_TO_DATE=true" >> "$GITHUB_ENV") + + - name: Push Changes + if: ${{ env.UP_TO_DATE == 'false' }} + uses: ad-m/github-push-action@v1.1.0 + with: + branch: update-pre-commit-deps + directory: update-pre-commit-deps + github_token: ${{ secrets.GITHUB_TOKEN }} + force: true + env: + GH_TOKEN: ${{ github.token }} + + - name: Make PR and add reviewers and labels + if: ${{ env.UP_TO_DATE == 'false' }} + run: | + cd update-pre-commit-deps + gh pr create \ + --title "Update pre-commit and its dependencies" \ + --body "This PR was auto-generated to update pre-commit and its dependencies." \ + --head update-pre-commit-deps \ + --reviewer ${{ env.REVIEWERS }} \ + --label DevOps + env: + GH_TOKEN: ${{ github.token }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 70daad33..b281c363 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,7 @@ exclude: "docs|node_modules|migrations|.git|.tox" fail_fast: true +# (run `python scripts/sync_pre_commit_versions.py` to sync them) repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 @@ -37,6 +38,7 @@ repos: hooks: - id: mypy args: ["--config=pyproject.toml"] + additional_dependencies: [types-PyYAML] # https://pre-commit.ci/#configuration ci: diff --git a/conda/dev.yml b/conda/dev.yml index 410ee11c..f9b2a462 100644 --- a/conda/dev.yml +++ b/conda/dev.yml @@ -37,7 +37,10 @@ dependencies: - docutils>=0.16 # Removed <0.17 constraint # Quality Assurance Tools # ======================= - # If versions are updated, also update 'rev' in `.pre-commit-config.yaml` + # Run `pre-commit autoupdate` to get the latest pinned versions of 'rev' in + # `.pre-commit-config.yaml`, then run + # `python scripts/sync_pre_commit_versions.py` to update the pinned versions + # here and in the `qa` extra of pyproject.toml. - black ==25.1.0 - flake8 ==7.3.0 - isort ==6.0.1 diff --git a/scripts/sync_pre_commit_versions.py b/scripts/sync_pre_commit_versions.py new file mode 100644 index 00000000..07f59052 --- /dev/null +++ b/scripts/sync_pre_commit_versions.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python +"""Sync pinned QA tool versions from ``.pre-commit-config.yaml``. + +``.pre-commit-config.yaml`` is the source of truth for the versions of the +quality-assurance tools (black, flake8, isort, mypy, ...). The same versions +are pinned in ``conda/dev.yml`` and in the ``qa`` extra of ``pyproject.toml``, +so they have to be updated whenever ``pre-commit autoupdate`` (or the +``pre-commit-update`` GitHub workflow) bumps a ``rev``. + +Run this script after updating ``.pre-commit-config.yaml``:: + + python scripts/sync_pre_commit_versions.py + +Use ``--check`` to verify that everything is already in sync without modifying +any files (exits with status 1 if it is not). +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path +from typing import Callable, Optional + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent + +PRE_COMMIT_CONFIG = REPO_ROOT / ".pre-commit-config.yaml" +DEV_YML = REPO_ROOT / "conda" / "dev.yml" +PYPROJECT_TOML = REPO_ROOT / "pyproject.toml" + +# Some pre-commit repos are mirrors of the package they run, e.g. +# https://github.com/pre-commit/mirrors-mypy provides `mypy`. +MIRROR_PREFIXES = ("mirrors-", "mirror-") + +# A pinned dependency in `conda/dev.yml`, e.g. ` - black ==25.1.0` or +# ` - tbump=6.9.0`. Ranges such as `- numpy >=2.0,<3.0` are not matched. +CONDA_PIN = re.compile( + r"^(?P\s*-\s+)" + r"(?P[A-Za-z0-9._-]+)" + r"(?P\s*)(?P==|=)(?P\s*)" + r"(?P[^\s#]+)" + r"(?P.*)$" +) + +# A pinned dependency in a `pyproject.toml` requirement list, e.g. +# ` "black==25.1.0",`. +PYPI_PIN = re.compile( + r"^(?P\s*\")" + r"(?P[A-Za-z0-9._-]+)" + r"(?P\s*)(?P==)(?P\s*)" + r"(?P[^\"\s]+)" + r"(?P\".*)$" +) + +# The `qa = [...]` list in `pyproject.toml`. +PYPROJECT_QA = re.compile(r"^qa\s*=\s*\[\s*$") + + +def normalize(name: str) -> str: + """Normalize a package name the way PEP 503 does.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def package_names(repo_url: str, hook_ids: list[str]) -> set[str]: + """Guess the package names provided by a pre-commit repo. + + The hook ids are usually the package name (``black``, ``isort``, ...), and + so is the last component of the repo URL, once any ``mirrors-`` prefix has + been stripped. + """ + repo_name = repo_url.rstrip("/").rsplit("/", 1)[-1] + if repo_name.endswith(".git"): + repo_name = repo_name[: -len(".git")] + for prefix in MIRROR_PREFIXES: + if repo_name.startswith(prefix): + repo_name = repo_name[len(prefix) :] + return {normalize(name) for name in [repo_name, *hook_ids]} + + +def collect_versions(config_path: Path) -> dict[str, str]: + """Map normalized package names to versions from ``.pre-commit-config.yaml``. + + Both the ``rev`` of each repo and any ``additional_dependencies`` pinned + with ``==`` are collected. + """ + with open(config_path) as f: + config = yaml.safe_load(f) + + versions: dict[str, str] = {} + + def add(name: str, version: str, source: str) -> None: + name = normalize(name) + previous = versions.get(name) + if previous is not None and previous != version: + print( + f"Warning: conflicting versions for {name} in {config_path.name}: " + f"{previous} and {version} (from {source}); keeping {previous}" + ) + return + versions[name] = version + + for repo in config.get("repos", []): + repo_url = repo.get("repo", "") + if repo_url in ("local", "meta"): + continue + rev = repo.get("rev") + hooks = repo.get("hooks", []) or [] + hook_ids = [hook["id"] for hook in hooks if "id" in hook] + if rev is not None: + # `rev` is a git tag, which is often prefixed with a `v`. + version = str(rev).lstrip("v") + for name in package_names(repo_url, hook_ids): + add(name, version, f"rev of {repo_url}") + for hook in hooks: + for dependency in hook.get("additional_dependencies", []) or []: + if "==" not in dependency: + continue + name, _, version = dependency.partition("==") + add( + name.strip(), + version.strip(), + f"additional_dependencies of {hook.get('id')}", + ) + + return versions + + +def sync_lines( + lines: list[str], + versions: dict[str, str], + pattern: re.Pattern[str], + path: Path, + line_range: Optional[tuple[int, int]] = None, +) -> tuple[list[str], list[str]]: + """Update pinned versions in ``lines``, returning the new lines and a log.""" + start, end = line_range if line_range is not None else (0, len(lines)) + updated = list(lines) + changes: list[str] = [] + + for index in range(start, end): + content = lines[index].rstrip("\r\n") + line_ending = lines[index][len(content) :] + match = pattern.match(content) + if match is None: + continue + name = normalize(match.group("name")) + version = versions.get(name) + if version is None or version == match.group("version"): + continue + updated[index] = ( + "{prefix}{name}{pre_op}{op}{post_op}{version}{suffix}".format( + **{**match.groupdict(), "version": version} + ) + + line_ending + ) + changes.append( + f"{path.relative_to(REPO_ROOT)}:{index + 1}: " + f"{match.group('name')} {match.group('version')} -> {version}" + ) + + return updated, changes + + +def find_qa_block(lines: list[str]) -> tuple[int, int]: + """Find the line range of the ``qa = [...]`` list in ``pyproject.toml``.""" + for index, line in enumerate(lines): + if PYPROJECT_QA.match(line): + for end in range(index + 1, len(lines)): + if lines[end].startswith("]"): + return index + 1, end + raise ValueError("unterminated `qa = [` list in pyproject.toml") + raise ValueError("no `qa = [` list found in pyproject.toml") + + +def sync_file( + path: Path, + versions: dict[str, str], + pattern: re.Pattern[str], + check: bool, + find_range: Optional[Callable[[list[str]], tuple[int, int]]] = None, +) -> list[str]: + """Sync one file, writing it back unless ``check`` is set.""" + lines = path.read_text().splitlines(keepends=True) + line_range = find_range(lines) if find_range is not None else None + updated, changes = sync_lines(lines, versions, pattern, path, line_range) + if changes and not check: + path.write_text("".join(updated)) + return changes + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="report out-of-sync versions without modifying any files", + ) + args = parser.parse_args() + + versions = collect_versions(PRE_COMMIT_CONFIG) + + changes = sync_file(DEV_YML, versions, CONDA_PIN, args.check) + changes += sync_file( + PYPROJECT_TOML, versions, PYPI_PIN, args.check, find_range=find_qa_block + ) + + if not changes: + print(f"All versions are in sync with {PRE_COMMIT_CONFIG.name}.") + return 0 + + for change in changes: + print(change) + + if args.check: + print( + f"\nRun `python {Path(__file__).relative_to(REPO_ROOT)}` " + "to apply these updates." + ) + return 1 + + print(f"\nUpdated {len(changes)} pinned version(s).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From e1a60792790da44efa18aaafa6f3c97f563f1253 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 13 Aug 2026 19:23:43 -0500 Subject: [PATCH 2/5] Sync QA tool versions into pyproject.toml and conda/dev.yml Add a `qa` extra (pinned to the same black/flake8/isort/mypy/etc. versions as `.pre-commit-config.yaml`) and a `dev` extra to `pyproject.toml`, plus the missing `flake8-isort` and `types-PyYAML` pins in `conda/dev.yml`. Also add `[tool.pre-commit-update.yaml]` so `pre-commit-update` doesn't reformat the whole config on its next run. Brings zppy in line with the version-pinning pattern already used in zppy-interfaces, so `scripts/sync_pre_commit_versions.py --check` passes here too. Generated by Claude (Anthropic). --- conda/dev.yml | 2 ++ pyproject.toml | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/conda/dev.yml b/conda/dev.yml index f9b2a462..921bec27 100644 --- a/conda/dev.yml +++ b/conda/dev.yml @@ -43,9 +43,11 @@ dependencies: # here and in the `qa` extra of pyproject.toml. - black ==25.1.0 - flake8 ==7.3.0 + - flake8-isort ==6.1.1 - isort ==6.0.1 - mypy ==1.18.2 - pre-commit ==4.3.0 + - types-PyYAML >=6.0.0 - tbump >=6.9.0 # Developer Tools # ======================= diff --git a/pyproject.toml b/pyproject.toml index 0412b05b..5993e3e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,24 @@ Repository = "https://github.com/E3SM-Project/zppy.git" [project.scripts] zppy = "zppy.__main__:main" +# make sure these always match conda/dev.yml and .pre-commit-config.yaml +# (run `python scripts/sync_pre_commit_versions.py` to sync them) +[project.optional-dependencies] +qa = [ + "black==25.1.0", + "flake8==7.3.0", + "flake8-isort==6.1.1", + "isort==6.0.1", + "mypy==1.18.2", + "pre-commit==4.3.0", + "types-PyYAML>=6.0.0", +] + +dev = [ + "tbump==6.9.0", + "ipykernel", +] + [tool.setuptools] include-package-data = true @@ -59,6 +77,15 @@ zppy = [ "**/*.jinja", ] +# `pre-commit-update` rewrites .pre-commit-config.yaml with ruamel.yaml. Without +# this, it defaults to `indent(sequence=4)`, which puts the `-` of each list item +# in column 0 and reformats the whole file. These settings keep its output in +# the conventional ` - item` style, so an update is a diff of just the revs. +[tool.pre-commit-update.yaml] +mapping = 2 +sequence = 4 +offset = 2 + [tool.black] line-length = 88 target-version = ['py311'] From a0c5ba076f6a35d7103ee99310a81a872a23d11f Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 13 Aug 2026 19:34:26 -0500 Subject: [PATCH 3/5] Bump flake8-isort to 6.1.2 to fix isort 6.x conflict flake8-isort==6.1.1 pins isort<6, which conflicts with the isort==6.0.1 pin introduced in the previous commit and broke the conda env solve in CI. 6.1.2 raises the cap to isort<7, which is compatible. Change made in .pre-commit-config.yaml and propagated to conda/dev.yml and pyproject.toml via `python scripts/sync_pre_commit_versions.py`. Claude-generated, verified against the flake8-isort 6.1.1/6.1.2 PyPI metadata. --- .pre-commit-config.yaml | 2 +- conda/dev.yml | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b281c363..3b2d1082 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,7 +30,7 @@ repos: hooks: - id: flake8 args: ["--config=.flake8.cfg"] - additional_dependencies: [flake8-isort==6.1.1] + additional_dependencies: [flake8-isort==6.1.2] # Can run individually with `pre-commit run mypy --all-files` - repo: https://github.com/pre-commit/mirrors-mypy diff --git a/conda/dev.yml b/conda/dev.yml index 921bec27..bfbb6f9a 100644 --- a/conda/dev.yml +++ b/conda/dev.yml @@ -43,7 +43,7 @@ dependencies: # here and in the `qa` extra of pyproject.toml. - black ==25.1.0 - flake8 ==7.3.0 - - flake8-isort ==6.1.1 + - flake8-isort ==6.1.2 - isort ==6.0.1 - mypy ==1.18.2 - pre-commit ==4.3.0 diff --git a/pyproject.toml b/pyproject.toml index 5993e3e2..aa70824c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ zppy = "zppy.__main__:main" qa = [ "black==25.1.0", "flake8==7.3.0", - "flake8-isort==6.1.1", + "flake8-isort==6.1.2", "isort==6.0.1", "mypy==1.18.2", "pre-commit==4.3.0", From 2b3a28b8da750632026f8140b5291aa19454cbdb Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 13 Aug 2026 19:56:39 -0500 Subject: [PATCH 4/5] Address Copilot comments --- .github/workflows/pre_commit_update_workflow.yml | 1 - pyproject.toml | 2 +- scripts/sync_pre_commit_versions.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pre_commit_update_workflow.yml b/.github/workflows/pre_commit_update_workflow.yml index da7a2a72..42e782b0 100644 --- a/.github/workflows/pre_commit_update_workflow.yml +++ b/.github/workflows/pre_commit_update_workflow.yml @@ -15,7 +15,6 @@ on: permissions: contents: write pull-requests: write - issues: write env: UP_TO_DATE: false diff --git a/pyproject.toml b/pyproject.toml index aa70824c..9ddc807a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ Repository = "https://github.com/E3SM-Project/zppy.git" [project.scripts] zppy = "zppy.__main__:main" -# make sure these always match conda/dev.yml and .pre-commit-config.yaml +# make sure these always match pinned versions in conda/dev.yml and .pre-commit-config.yaml # (run `python scripts/sync_pre_commit_versions.py` to sync them) [project.optional-dependencies] qa = [ diff --git a/scripts/sync_pre_commit_versions.py b/scripts/sync_pre_commit_versions.py index 07f59052..27c4748b 100644 --- a/scripts/sync_pre_commit_versions.py +++ b/scripts/sync_pre_commit_versions.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Sync pinned QA tool versions from ``.pre-commit-config.yaml``. ``.pre-commit-config.yaml`` is the source of truth for the versions of the From 0afa8d01f50c479110bb904ee6a062ce0b365eaf Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 14 Aug 2026 11:36:18 -0500 Subject: [PATCH 5/5] Address review comments --- .github/workflows/pre_commit_update_workflow.yml | 2 +- conda/dev.yml | 1 + pyproject.toml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre_commit_update_workflow.yml b/.github/workflows/pre_commit_update_workflow.yml index 42e782b0..b20f86c7 100644 --- a/.github/workflows/pre_commit_update_workflow.yml +++ b/.github/workflows/pre_commit_update_workflow.yml @@ -19,7 +19,7 @@ permissions: env: UP_TO_DATE: false PYTHON_VERSION: "3.14" - REVIEWERS: "xylar,forsyth2" + REVIEWERS: "forsyth2" jobs: auto-update: runs-on: ubuntu-latest diff --git a/conda/dev.yml b/conda/dev.yml index bfbb6f9a..1a97099a 100644 --- a/conda/dev.yml +++ b/conda/dev.yml @@ -47,6 +47,7 @@ dependencies: - isort ==6.0.1 - mypy ==1.18.2 - pre-commit ==4.3.0 + - pyyaml - types-PyYAML >=6.0.0 - tbump >=6.9.0 # Developer Tools diff --git a/pyproject.toml b/pyproject.toml index 9ddc807a..ecbbe4e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ qa = [ "isort==6.0.1", "mypy==1.18.2", "pre-commit==4.3.0", + "pyyaml", "types-PyYAML>=6.0.0", ]