diff --git a/.github/workflows/pre_commit_update_workflow.yml b/.github/workflows/pre_commit_update_workflow.yml new file mode 100644 index 00000000..b20f86c7 --- /dev/null +++ b/.github/workflows/pre_commit_update_workflow.yml @@ -0,0 +1,92 @@ +name: Pre-commit auto-update + +on: + schedule: + # Cron syntax: + # 1. Entry: Minute when the process will be started [0-59] + # 2. Entry: Hour when the process will be started [0-23] + # 3. Entry: Day of the month when the process will be started [1-28/29/30/31] + # 4. Entry: Month of the year when the process will be started [1-12] + # 5. Entry: Weekday when the process will be started [0-6] [0 is Sunday] + - cron: '0 8 1 * *' + # Allow manual triggering of the workflow + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +env: + UP_TO_DATE: false + PYTHON_VERSION: "3.14" + REVIEWERS: "forsyth2" +jobs: + auto-update: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Set up Conda Environment + uses: mamba-org/setup-micromamba@v3 + with: + environment-name: pre_commit_dev + init-shell: bash + condarc: | + channel_priority: strict + channels: + - conda-forge + create-args: >- + python=${{ env.PYTHON_VERSION }} + + - name: Install pre-commit, pre-commit-update and gh + run: | + eval "$(micromamba shell hook --shell bash)" + micromamba activate pre_commit_dev + # permissions issue with gh 2.76.0 + micromamba install -y --override-channels -c conda-forge \ + pre-commit pyyaml "gh !=2.76.0" + python -m pip install pre-commit-update + gh --version + + - name: Apply and commit updates + run: | + eval "$(micromamba shell hook --shell bash)" + micromamba activate pre_commit_dev + git clone https://github.com/E3SM-Project/zppy.git update-pre-commit-deps + cd update-pre-commit-deps + # Configure git using GitHub Actions credentials. + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + git checkout -b update-pre-commit-deps + # pre-commit-update doesn't try to include non-release versions + pre-commit-update + # Propagate the new versions to conda/dev.yml and the `qa` extra in + # pyproject.toml + python scripts/sync_pre_commit_versions.py + git add . + # The second command will fail if no changes were present, so we ignore it + git commit -m "Update pre-commit dependencies" || ( echo "UP_TO_DATE=true" >> "$GITHUB_ENV") + + - name: Push Changes + if: ${{ env.UP_TO_DATE == 'false' }} + uses: ad-m/github-push-action@v1.1.0 + with: + branch: update-pre-commit-deps + directory: update-pre-commit-deps + github_token: ${{ secrets.GITHUB_TOKEN }} + force: true + env: + GH_TOKEN: ${{ github.token }} + + - name: Make PR and add reviewers and labels + if: ${{ env.UP_TO_DATE == 'false' }} + run: | + cd update-pre-commit-deps + gh pr create \ + --title "Update pre-commit and its dependencies" \ + --body "This PR was auto-generated to update pre-commit and its dependencies." \ + --head update-pre-commit-deps \ + --reviewer ${{ env.REVIEWERS }} \ + --label DevOps + env: + GH_TOKEN: ${{ github.token }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 70daad33..3b2d1082 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,7 @@ exclude: "docs|node_modules|migrations|.git|.tox" fail_fast: true +# (run `python scripts/sync_pre_commit_versions.py` to sync them) repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 @@ -29,7 +30,7 @@ repos: hooks: - id: flake8 args: ["--config=.flake8.cfg"] - additional_dependencies: [flake8-isort==6.1.1] + additional_dependencies: [flake8-isort==6.1.2] # Can run individually with `pre-commit run mypy --all-files` - repo: https://github.com/pre-commit/mirrors-mypy @@ -37,6 +38,7 @@ repos: hooks: - id: mypy args: ["--config=pyproject.toml"] + additional_dependencies: [types-PyYAML] # https://pre-commit.ci/#configuration ci: diff --git a/conda/dev.yml b/conda/dev.yml index 410ee11c..1a97099a 100644 --- a/conda/dev.yml +++ b/conda/dev.yml @@ -37,12 +37,18 @@ dependencies: - docutils>=0.16 # Removed <0.17 constraint # Quality Assurance Tools # ======================= - # If versions are updated, also update 'rev' in `.pre-commit-config.yaml` + # Run `pre-commit autoupdate` to get the latest pinned versions of 'rev' in + # `.pre-commit-config.yaml`, then run + # `python scripts/sync_pre_commit_versions.py` to update the pinned versions + # here and in the `qa` extra of pyproject.toml. - black ==25.1.0 - flake8 ==7.3.0 + - flake8-isort ==6.1.2 - isort ==6.0.1 - mypy ==1.18.2 - pre-commit ==4.3.0 + - pyyaml + - types-PyYAML >=6.0.0 - tbump >=6.9.0 # Developer Tools # ======================= diff --git a/pyproject.toml b/pyproject.toml index 0412b05b..ecbbe4e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,25 @@ Repository = "https://github.com/E3SM-Project/zppy.git" [project.scripts] zppy = "zppy.__main__:main" +# make sure these always match pinned versions in conda/dev.yml and .pre-commit-config.yaml +# (run `python scripts/sync_pre_commit_versions.py` to sync them) +[project.optional-dependencies] +qa = [ + "black==25.1.0", + "flake8==7.3.0", + "flake8-isort==6.1.2", + "isort==6.0.1", + "mypy==1.18.2", + "pre-commit==4.3.0", + "pyyaml", + "types-PyYAML>=6.0.0", +] + +dev = [ + "tbump==6.9.0", + "ipykernel", +] + [tool.setuptools] include-package-data = true @@ -59,6 +78,15 @@ zppy = [ "**/*.jinja", ] +# `pre-commit-update` rewrites .pre-commit-config.yaml with ruamel.yaml. Without +# this, it defaults to `indent(sequence=4)`, which puts the `-` of each list item +# in column 0 and reformats the whole file. These settings keep its output in +# the conventional ` - item` style, so an update is a diff of just the revs. +[tool.pre-commit-update.yaml] +mapping = 2 +sequence = 4 +offset = 2 + [tool.black] line-length = 88 target-version = ['py311'] diff --git a/scripts/sync_pre_commit_versions.py b/scripts/sync_pre_commit_versions.py new file mode 100644 index 00000000..27c4748b --- /dev/null +++ b/scripts/sync_pre_commit_versions.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +"""Sync pinned QA tool versions from ``.pre-commit-config.yaml``. + +``.pre-commit-config.yaml`` is the source of truth for the versions of the +quality-assurance tools (black, flake8, isort, mypy, ...). The same versions +are pinned in ``conda/dev.yml`` and in the ``qa`` extra of ``pyproject.toml``, +so they have to be updated whenever ``pre-commit autoupdate`` (or the +``pre-commit-update`` GitHub workflow) bumps a ``rev``. + +Run this script after updating ``.pre-commit-config.yaml``:: + + python scripts/sync_pre_commit_versions.py + +Use ``--check`` to verify that everything is already in sync without modifying +any files (exits with status 1 if it is not). +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path +from typing import Callable, Optional + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent + +PRE_COMMIT_CONFIG = REPO_ROOT / ".pre-commit-config.yaml" +DEV_YML = REPO_ROOT / "conda" / "dev.yml" +PYPROJECT_TOML = REPO_ROOT / "pyproject.toml" + +# Some pre-commit repos are mirrors of the package they run, e.g. +# https://github.com/pre-commit/mirrors-mypy provides `mypy`. +MIRROR_PREFIXES = ("mirrors-", "mirror-") + +# A pinned dependency in `conda/dev.yml`, e.g. ` - black ==25.1.0` or +# ` - tbump=6.9.0`. Ranges such as `- numpy >=2.0,<3.0` are not matched. +CONDA_PIN = re.compile( + r"^(?P\s*-\s+)" + r"(?P[A-Za-z0-9._-]+)" + r"(?P\s*)(?P==|=)(?P\s*)" + r"(?P[^\s#]+)" + r"(?P.*)$" +) + +# A pinned dependency in a `pyproject.toml` requirement list, e.g. +# ` "black==25.1.0",`. +PYPI_PIN = re.compile( + r"^(?P\s*\")" + r"(?P[A-Za-z0-9._-]+)" + r"(?P\s*)(?P==)(?P\s*)" + r"(?P[^\"\s]+)" + r"(?P\".*)$" +) + +# The `qa = [...]` list in `pyproject.toml`. +PYPROJECT_QA = re.compile(r"^qa\s*=\s*\[\s*$") + + +def normalize(name: str) -> str: + """Normalize a package name the way PEP 503 does.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def package_names(repo_url: str, hook_ids: list[str]) -> set[str]: + """Guess the package names provided by a pre-commit repo. + + The hook ids are usually the package name (``black``, ``isort``, ...), and + so is the last component of the repo URL, once any ``mirrors-`` prefix has + been stripped. + """ + repo_name = repo_url.rstrip("/").rsplit("/", 1)[-1] + if repo_name.endswith(".git"): + repo_name = repo_name[: -len(".git")] + for prefix in MIRROR_PREFIXES: + if repo_name.startswith(prefix): + repo_name = repo_name[len(prefix) :] + return {normalize(name) for name in [repo_name, *hook_ids]} + + +def collect_versions(config_path: Path) -> dict[str, str]: + """Map normalized package names to versions from ``.pre-commit-config.yaml``. + + Both the ``rev`` of each repo and any ``additional_dependencies`` pinned + with ``==`` are collected. + """ + with open(config_path) as f: + config = yaml.safe_load(f) + + versions: dict[str, str] = {} + + def add(name: str, version: str, source: str) -> None: + name = normalize(name) + previous = versions.get(name) + if previous is not None and previous != version: + print( + f"Warning: conflicting versions for {name} in {config_path.name}: " + f"{previous} and {version} (from {source}); keeping {previous}" + ) + return + versions[name] = version + + for repo in config.get("repos", []): + repo_url = repo.get("repo", "") + if repo_url in ("local", "meta"): + continue + rev = repo.get("rev") + hooks = repo.get("hooks", []) or [] + hook_ids = [hook["id"] for hook in hooks if "id" in hook] + if rev is not None: + # `rev` is a git tag, which is often prefixed with a `v`. + version = str(rev).lstrip("v") + for name in package_names(repo_url, hook_ids): + add(name, version, f"rev of {repo_url}") + for hook in hooks: + for dependency in hook.get("additional_dependencies", []) or []: + if "==" not in dependency: + continue + name, _, version = dependency.partition("==") + add( + name.strip(), + version.strip(), + f"additional_dependencies of {hook.get('id')}", + ) + + return versions + + +def sync_lines( + lines: list[str], + versions: dict[str, str], + pattern: re.Pattern[str], + path: Path, + line_range: Optional[tuple[int, int]] = None, +) -> tuple[list[str], list[str]]: + """Update pinned versions in ``lines``, returning the new lines and a log.""" + start, end = line_range if line_range is not None else (0, len(lines)) + updated = list(lines) + changes: list[str] = [] + + for index in range(start, end): + content = lines[index].rstrip("\r\n") + line_ending = lines[index][len(content) :] + match = pattern.match(content) + if match is None: + continue + name = normalize(match.group("name")) + version = versions.get(name) + if version is None or version == match.group("version"): + continue + updated[index] = ( + "{prefix}{name}{pre_op}{op}{post_op}{version}{suffix}".format( + **{**match.groupdict(), "version": version} + ) + + line_ending + ) + changes.append( + f"{path.relative_to(REPO_ROOT)}:{index + 1}: " + f"{match.group('name')} {match.group('version')} -> {version}" + ) + + return updated, changes + + +def find_qa_block(lines: list[str]) -> tuple[int, int]: + """Find the line range of the ``qa = [...]`` list in ``pyproject.toml``.""" + for index, line in enumerate(lines): + if PYPROJECT_QA.match(line): + for end in range(index + 1, len(lines)): + if lines[end].startswith("]"): + return index + 1, end + raise ValueError("unterminated `qa = [` list in pyproject.toml") + raise ValueError("no `qa = [` list found in pyproject.toml") + + +def sync_file( + path: Path, + versions: dict[str, str], + pattern: re.Pattern[str], + check: bool, + find_range: Optional[Callable[[list[str]], tuple[int, int]]] = None, +) -> list[str]: + """Sync one file, writing it back unless ``check`` is set.""" + lines = path.read_text().splitlines(keepends=True) + line_range = find_range(lines) if find_range is not None else None + updated, changes = sync_lines(lines, versions, pattern, path, line_range) + if changes and not check: + path.write_text("".join(updated)) + return changes + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="report out-of-sync versions without modifying any files", + ) + args = parser.parse_args() + + versions = collect_versions(PRE_COMMIT_CONFIG) + + changes = sync_file(DEV_YML, versions, CONDA_PIN, args.check) + changes += sync_file( + PYPROJECT_TOML, versions, PYPI_PIN, args.check, find_range=find_qa_block + ) + + if not changes: + print(f"All versions are in sync with {PRE_COMMIT_CONFIG.name}.") + return 0 + + for change in changes: + print(change) + + if args.check: + print( + f"\nRun `python {Path(__file__).relative_to(REPO_ROOT)}` " + "to apply these updates." + ) + return 1 + + print(f"\nUpdated {len(changes)} pinned version(s).") + return 0 + + +if __name__ == "__main__": + sys.exit(main())