diff --git a/setup.cfg b/.flake8 similarity index 50% rename from setup.cfg rename to .flake8 index 5a75b3ef..6cb928d9 100644 --- a/setup.cfg +++ b/.flake8 @@ -22,33 +22,3 @@ exclude = .pytest_cache, *__init__.py, venv, - -[isort] -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -use_parentheses=True -line_length=88 - -[pycodestyle] -max-line-length = 119 -exclude = - .tox - .git - */migrations/* - */static/CACHE/* - docs - node_modules - .idea - .mypy_cache - .pytest_cache - *__init__.py - venv - -[mypy] -python_version = 3.14 -check_untyped_defs = True -ignore_missing_imports = True -warn_unused_ignores = True -warn_redundant_casts = True -warn_unused_configs = True diff --git a/.github/workflows/pre_commit_update_workflow.yml b/.github/workflows/pre_commit_update_workflow.yml new file mode 100644 index 00000000..e1687940 --- /dev/null +++ b/.github/workflows/pre_commit_update_workflow.yml @@ -0,0 +1,95 @@ +name: Pre-commit auto-update + +on: + schedule: + # Cron syntax: + # 1. Entry: Minute when the process will be started [0-59] + # 2. Entry: Hour when the process will be started [0-23] + # 3. Entry: Day of the month when the process will be started [1-28/29/30/31] + # 4. Entry: Month of the year when the process will be started [1-12] + # 5. Entry: Weekday when the process will be started [0-6] [0 is Sunday] + - cron: '0 8 1 * *' + # Allow manual triggering of the workflow + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +env: + UP_TO_DATE: false + PYTHON_VERSION: "3.14" + REVIEWERS: "forsyth2" +jobs: + auto-update: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Set up Conda Environment + uses: mamba-org/setup-micromamba@v3 + with: + environment-name: pre_commit_dev + init-shell: bash + condarc: | + channel_priority: strict + channels: + - conda-forge + create-args: >- + python=${{ env.PYTHON_VERSION }} + + - name: Install pre-commit, pre-commit-update and gh + run: | + eval "$(micromamba shell hook --shell bash)" + micromamba activate pre_commit_dev + # permissions issue with gh 2.76.0 + micromamba install -y --override-channels -c conda-forge \ + pre-commit pyyaml "gh !=2.76.0" + python -m pip install pre-commit-update + gh --version + + - name: Apply and commit updates + run: | + eval "$(micromamba shell hook --shell bash)" + micromamba activate pre_commit_dev + git clone https://github.com/E3SM-Project/zstash.git update-pre-commit-deps + cd update-pre-commit-deps + # Configure git using GitHub Actions credentials. + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + git checkout -b update-pre-commit-deps + # pre-commit-update doesn't try to include non-release versions + pre-commit-update + # Propagate the new versions to conda/dev.yml and the `qa` extra in + # pyproject.toml + python scripts/sync_pre_commit_versions.py + git add . + if git diff --cached --quiet; then + echo "UP_TO_DATE=true" >> "$GITHUB_ENV" + else + git commit -m "Update pre-commit dependencies" + fi + + - name: Push Changes + if: ${{ env.UP_TO_DATE == 'false' }} + uses: ad-m/github-push-action@v1.1.0 + with: + branch: update-pre-commit-deps + directory: update-pre-commit-deps + github_token: ${{ secrets.GITHUB_TOKEN }} + force: true + env: + GH_TOKEN: ${{ github.token }} + + - name: Make PR and add reviewers and labels + if: ${{ env.UP_TO_DATE == 'false' }} + run: | + cd update-pre-commit-deps + gh pr create \ + --title "Update pre-commit and its dependencies" \ + --body "This PR was auto-generated to update pre-commit and its dependencies." \ + --head update-pre-commit-deps \ + --reviewer ${{ env.REVIEWERS }} \ + --label DevOps + env: + GH_TOKEN: ${{ github.token }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bf0d71b3..1d3a6bc1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,8 @@ exclude: "docs|node_modules|migrations|.git|.tox" -default_stages: [commit] +default_stages: [pre-commit] fail_fast: true +# (run `python scripts/sync_pre_commit_versions.py` to sync them) repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 @@ -27,15 +28,16 @@ repos: rev: 7.3.0 hooks: - id: flake8 - args: ["--config=setup.cfg"] - additional_dependencies: [flake8-isort] + args: ["--config=.flake8"] + additional_dependencies: [flake8-isort==6.1.2] exclude: analysis_data_preprocess - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.18.2 hooks: - id: mypy - args: ["--config=setup.cfg", "--install-types", "--non-interactive"] + args: ["--config=pyproject.toml", "--install-types", "--non-interactive"] + additional_dependencies: [types-PyYAML] exclude: test* # https://pre-commit.ci/#configuration diff --git a/.vscode/settings.json b/.vscode/settings.json index ecfde195..0f65f8f7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,7 +9,7 @@ }, "python.formatting.provider": "black", "python.linting.flake8Enabled": true, - "python.linting.flake8Args": ["--config=setup.cfg"], + "python.linting.flake8Args": ["--config=.flake8"], "python.linting.mypyEnabled": true, "python.pythonPath": "/opt/miniconda3/envs/zstash_dev/bin/python" } diff --git a/conda/dev.yml b/conda/dev.yml index 492918c1..291dfbb0 100644 --- a/conda/dev.yml +++ b/conda/dev.yml @@ -12,12 +12,17 @@ dependencies: - globus-sdk >=3.15.0,<4.0 # Developer Tools # ================= - # If versions are updated, also update 'rev' in `.pre-commit-config.yaml` + # Run `pre-commit autoupdate` to get the latest pinned versions of 'rev' in + # `.pre-commit-config.yaml`, then run + # `python scripts/sync_pre_commit_versions.py` to update the pinned versions + # here and in the `qa` extra of pyproject.toml. - black ==25.1.0 - flake8 ==7.3.0 + - flake8-isort ==6.1.2 - isort ==6.0.1 - mypy ==1.18.2 - pre-commit ==4.3.0 + - pyyaml - tbump >=6.9.0 # Testing # ======================= diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..5134c334 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,95 @@ +[build-system] +requires = [ + "setuptools>=64", +] +build-backend = "setuptools.build_meta" + +[project] +name = "zstash" +version = "1.6.0" +description = "Long term HPSS archiving software for E3SM" +readme = "README.md" +requires-python = ">=3.11,<3.15" +authors = [ + { name = "Ryan Forsyth", email = "forsyth2@llnl.gov" }, + { name = "Chris Golaz", email = "golaz1@llnl.gov" }, + { name = "Zeshawn Shaheen", email = "shaheen2@llnl.gov" }, +] + +[project.scripts] +zstash = "zstash.main:main" + +# make sure these always match pinned versions in conda/dev.yml and .pre-commit-config.yaml +# (run `python scripts/sync_pre_commit_versions.py` to sync them) +[project.optional-dependencies] +qa = [ + "black==25.1.0", + "flake8==7.3.0", + "flake8-isort==6.1.2", + "isort==6.0.1", + "mypy==1.18.2", + "pre-commit==4.3.0", + "pyyaml", +] + +dev = [ + "tbump>=6.9.0", +] + +[tool.setuptools.packages.find] +include = ["zstash", "zstash.*"] + +# `pre-commit-update` rewrites .pre-commit-config.yaml with ruamel.yaml. Without +# this, it defaults to `indent(sequence=4)`, which puts the `-` of each list item +# in column 0 and reformats the whole file. These settings keep its output in +# the conventional ` - item` style, so an update is a diff of just the revs. +[tool.pre-commit-update.yaml] +mapping = 2 +sequence = 4 +offset = 2 + +[tool.black] +line-length = 88 +target-version = ['py311'] +include = '\.pyi?$' +exclude = ''' +/( + \.eggs + | \.git + | \.mypy_cache + | _build + | conda + | docs + )/ +''' + +[tool.isort] +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +line_length = 88 + +[tool.pycodestyle] +max-line-length = 119 +exclude = [ + ".tox", + ".git", + "*/migrations/*", + "*/static/CACHE/*", + "docs", + "node_modules", + ".idea", + ".mypy_cache", + ".pytest_cache", + "*__init__.py", + "venv", +] + +[tool.mypy] +python_version = "3.14" +check_untyped_defs = true +ignore_missing_imports = true +warn_unused_ignores = true +warn_redundant_casts = true +warn_unused_configs = true diff --git a/scripts/sync_pre_commit_versions.py b/scripts/sync_pre_commit_versions.py new file mode 100644 index 00000000..07f59052 --- /dev/null +++ b/scripts/sync_pre_commit_versions.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python +"""Sync pinned QA tool versions from ``.pre-commit-config.yaml``. + +``.pre-commit-config.yaml`` is the source of truth for the versions of the +quality-assurance tools (black, flake8, isort, mypy, ...). The same versions +are pinned in ``conda/dev.yml`` and in the ``qa`` extra of ``pyproject.toml``, +so they have to be updated whenever ``pre-commit autoupdate`` (or the +``pre-commit-update`` GitHub workflow) bumps a ``rev``. + +Run this script after updating ``.pre-commit-config.yaml``:: + + python scripts/sync_pre_commit_versions.py + +Use ``--check`` to verify that everything is already in sync without modifying +any files (exits with status 1 if it is not). +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path +from typing import Callable, Optional + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent + +PRE_COMMIT_CONFIG = REPO_ROOT / ".pre-commit-config.yaml" +DEV_YML = REPO_ROOT / "conda" / "dev.yml" +PYPROJECT_TOML = REPO_ROOT / "pyproject.toml" + +# Some pre-commit repos are mirrors of the package they run, e.g. +# https://github.com/pre-commit/mirrors-mypy provides `mypy`. +MIRROR_PREFIXES = ("mirrors-", "mirror-") + +# A pinned dependency in `conda/dev.yml`, e.g. ` - black ==25.1.0` or +# ` - tbump=6.9.0`. Ranges such as `- numpy >=2.0,<3.0` are not matched. +CONDA_PIN = re.compile( + r"^(?P\s*-\s+)" + r"(?P[A-Za-z0-9._-]+)" + r"(?P\s*)(?P==|=)(?P\s*)" + r"(?P[^\s#]+)" + r"(?P.*)$" +) + +# A pinned dependency in a `pyproject.toml` requirement list, e.g. +# ` "black==25.1.0",`. +PYPI_PIN = re.compile( + r"^(?P\s*\")" + r"(?P[A-Za-z0-9._-]+)" + r"(?P\s*)(?P==)(?P\s*)" + r"(?P[^\"\s]+)" + r"(?P\".*)$" +) + +# The `qa = [...]` list in `pyproject.toml`. +PYPROJECT_QA = re.compile(r"^qa\s*=\s*\[\s*$") + + +def normalize(name: str) -> str: + """Normalize a package name the way PEP 503 does.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def package_names(repo_url: str, hook_ids: list[str]) -> set[str]: + """Guess the package names provided by a pre-commit repo. + + The hook ids are usually the package name (``black``, ``isort``, ...), and + so is the last component of the repo URL, once any ``mirrors-`` prefix has + been stripped. + """ + repo_name = repo_url.rstrip("/").rsplit("/", 1)[-1] + if repo_name.endswith(".git"): + repo_name = repo_name[: -len(".git")] + for prefix in MIRROR_PREFIXES: + if repo_name.startswith(prefix): + repo_name = repo_name[len(prefix) :] + return {normalize(name) for name in [repo_name, *hook_ids]} + + +def collect_versions(config_path: Path) -> dict[str, str]: + """Map normalized package names to versions from ``.pre-commit-config.yaml``. + + Both the ``rev`` of each repo and any ``additional_dependencies`` pinned + with ``==`` are collected. + """ + with open(config_path) as f: + config = yaml.safe_load(f) + + versions: dict[str, str] = {} + + def add(name: str, version: str, source: str) -> None: + name = normalize(name) + previous = versions.get(name) + if previous is not None and previous != version: + print( + f"Warning: conflicting versions for {name} in {config_path.name}: " + f"{previous} and {version} (from {source}); keeping {previous}" + ) + return + versions[name] = version + + for repo in config.get("repos", []): + repo_url = repo.get("repo", "") + if repo_url in ("local", "meta"): + continue + rev = repo.get("rev") + hooks = repo.get("hooks", []) or [] + hook_ids = [hook["id"] for hook in hooks if "id" in hook] + if rev is not None: + # `rev` is a git tag, which is often prefixed with a `v`. + version = str(rev).lstrip("v") + for name in package_names(repo_url, hook_ids): + add(name, version, f"rev of {repo_url}") + for hook in hooks: + for dependency in hook.get("additional_dependencies", []) or []: + if "==" not in dependency: + continue + name, _, version = dependency.partition("==") + add( + name.strip(), + version.strip(), + f"additional_dependencies of {hook.get('id')}", + ) + + return versions + + +def sync_lines( + lines: list[str], + versions: dict[str, str], + pattern: re.Pattern[str], + path: Path, + line_range: Optional[tuple[int, int]] = None, +) -> tuple[list[str], list[str]]: + """Update pinned versions in ``lines``, returning the new lines and a log.""" + start, end = line_range if line_range is not None else (0, len(lines)) + updated = list(lines) + changes: list[str] = [] + + for index in range(start, end): + content = lines[index].rstrip("\r\n") + line_ending = lines[index][len(content) :] + match = pattern.match(content) + if match is None: + continue + name = normalize(match.group("name")) + version = versions.get(name) + if version is None or version == match.group("version"): + continue + updated[index] = ( + "{prefix}{name}{pre_op}{op}{post_op}{version}{suffix}".format( + **{**match.groupdict(), "version": version} + ) + + line_ending + ) + changes.append( + f"{path.relative_to(REPO_ROOT)}:{index + 1}: " + f"{match.group('name')} {match.group('version')} -> {version}" + ) + + return updated, changes + + +def find_qa_block(lines: list[str]) -> tuple[int, int]: + """Find the line range of the ``qa = [...]`` list in ``pyproject.toml``.""" + for index, line in enumerate(lines): + if PYPROJECT_QA.match(line): + for end in range(index + 1, len(lines)): + if lines[end].startswith("]"): + return index + 1, end + raise ValueError("unterminated `qa = [` list in pyproject.toml") + raise ValueError("no `qa = [` list found in pyproject.toml") + + +def sync_file( + path: Path, + versions: dict[str, str], + pattern: re.Pattern[str], + check: bool, + find_range: Optional[Callable[[list[str]], tuple[int, int]]] = None, +) -> list[str]: + """Sync one file, writing it back unless ``check`` is set.""" + lines = path.read_text().splitlines(keepends=True) + line_range = find_range(lines) if find_range is not None else None + updated, changes = sync_lines(lines, versions, pattern, path, line_range) + if changes and not check: + path.write_text("".join(updated)) + return changes + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="report out-of-sync versions without modifying any files", + ) + args = parser.parse_args() + + versions = collect_versions(PRE_COMMIT_CONFIG) + + changes = sync_file(DEV_YML, versions, CONDA_PIN, args.check) + changes += sync_file( + PYPROJECT_TOML, versions, PYPI_PIN, args.check, find_range=find_qa_block + ) + + if not changes: + print(f"All versions are in sync with {PRE_COMMIT_CONFIG.name}.") + return 0 + + for change in changes: + print(change) + + if args.check: + print( + f"\nRun `python {Path(__file__).relative_to(REPO_ROOT)}` " + "to apply these updates." + ) + return 1 + + print(f"\nUpdated {len(changes)} pinned version(s).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/setup.py b/setup.py deleted file mode 100644 index 3847453e..00000000 --- a/setup.py +++ /dev/null @@ -1,12 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="zstash", - version="1.6.0", - author="Ryan Forsyth, Chris Golaz, Zeshawn Shaheen", - author_email="forsyth2@llnl.gov, golaz1@llnl.gov, shaheen2@llnl.gov", - description="Long term HPSS archiving software for E3SM", - packages=find_packages(include=["zstash", "zstash.*"]), - python_requires=">=3.11,<3.15", - entry_points={"console_scripts": ["zstash=zstash.main:main"]}, -) diff --git a/tbump.toml b/tbump.toml index d1c9cb8f..bf682ab5 100644 --- a/tbump.toml +++ b/tbump.toml @@ -23,7 +23,7 @@ tag_template = "v{new_version}" # section containing the path of the file, relative to the # tbump.toml location. [[file]] -src = "setup.py" +src = "pyproject.toml" [[file]] src = "zstash/__init__.py" diff --git a/zstash/parallel.py b/zstash/parallel.py index 53beb53d..65f10694 100644 --- a/zstash/parallel.py +++ b/zstash/parallel.py @@ -56,7 +56,7 @@ def wait_turn( workers_curr_tar: str, indef_wait: bool = True, *args, - **kwargs + **kwargs, ): """ While a worker's current tar isn't the one @@ -85,7 +85,7 @@ def done_dequeuing_output_for_tar( worker, workers_curr_tar: str, *args, - **kwargs + **kwargs, ): """ A worker has finished printing the output for workers_curr_tar @@ -120,7 +120,7 @@ def __init__( # TODO: failure_queue has type `multiprocessing.Queue[FilesRow]` failure_queue, *args, - **kwargs + **kwargs, ): """ print_monitor is used to determine if it's this worker's turn to print.