Add pre-commit auto-update workflow and migrate zstash from setup.py/setup.cfg to pyproject.toml

setup.cfg → .flake8

@@ -22,33 +22,3 @@ exclude =
     .pytest_cache,
     *__init__.py,
     venv,
-
-[isort]
-multi_line_output=3
-include_trailing_comma=True
-force_grid_wrap=0
-use_parentheses=True
-line_length=88
-
-[pycodestyle]
-max-line-length = 119
-exclude =
-    .tox
-    .git
-    */migrations/*
-    */static/CACHE/*
-    docs
-    node_modules
-    .idea
-    .mypy_cache
-    .pytest_cache
-    *__init__.py
-    venv
-
-[mypy]
-python_version = 3.14
-check_untyped_defs = True
-ignore_missing_imports = True
-warn_unused_ignores = True
-warn_redundant_casts = True
-warn_unused_configs = True

.github/workflows/pre_commit_update_workflow.yml

@@ -0,0 +1,95 @@
+name: Pre-commit auto-update
+
+on:
+  schedule:
+    # Cron syntax:
+    # 1. Entry: Minute when the process will be started [0-59]
+    # 2. Entry: Hour when the process will be started [0-23]
+    # 3. Entry: Day of the month when the process will be started [1-28/29/30/31]
+    # 4. Entry: Month of the year when the process will be started [1-12]
+    # 5. Entry: Weekday when the process will be started [0-6] [0 is Sunday]
+    - cron: '0 8 1 * *'
+  # Allow manual triggering of the workflow
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+env:
+  UP_TO_DATE: false
+  PYTHON_VERSION: "3.14"
+  REVIEWERS: "forsyth2"
+jobs:
+  auto-update:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Set up Conda Environment
+        uses: mamba-org/setup-micromamba@v3
+        with:
+          environment-name: pre_commit_dev
+          init-shell: bash
+          condarc: |
+            channel_priority: strict
+            channels:
+                - conda-forge
+          create-args: >-
+            python=${{ env.PYTHON_VERSION }}
+
+      - name: Install pre-commit, pre-commit-update and gh
+        run: |
+          eval "$(micromamba shell hook --shell bash)"
+          micromamba activate pre_commit_dev
+          # permissions issue with gh 2.76.0
+          micromamba install -y --override-channels -c conda-forge \
+            pre-commit pyyaml "gh !=2.76.0"
+          python -m pip install pre-commit-update
+          gh --version
+
+      - name: Apply and commit updates
+        run: |
+          eval "$(micromamba shell hook --shell bash)"
+          micromamba activate pre_commit_dev
+          git clone https://github.com/E3SM-Project/zstash.git update-pre-commit-deps
+          cd update-pre-commit-deps
+          # Configure git using GitHub Actions credentials.
+          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git checkout -b update-pre-commit-deps
+          # pre-commit-update doesn't try to include non-release versions
+          pre-commit-update
+          # Propagate the new versions to conda/dev.yml and the `qa` extra in
+          # pyproject.toml
+          python scripts/sync_pre_commit_versions.py
+          git add .
+          if git diff --cached --quiet; then
+            echo "UP_TO_DATE=true" >> "$GITHUB_ENV"
+          else
+            git commit -m "Update pre-commit dependencies"
+          fi
+
+      - name: Push Changes
+        if: ${{ env.UP_TO_DATE == 'false' }}
+        uses: ad-m/github-push-action@v1.1.0
+        with:
+          branch: update-pre-commit-deps
+          directory: update-pre-commit-deps
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          force: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Make PR and add reviewers and labels
+        if: ${{ env.UP_TO_DATE == 'false' }}
+        run: |
+          cd update-pre-commit-deps
+          gh pr create \
+          --title "Update pre-commit and its dependencies" \
+          --body "This PR was auto-generated to update pre-commit and its dependencies." \
+          --head update-pre-commit-deps \
+          --reviewer ${{ env.REVIEWERS }} \
+          --label DevOps
+        env:
+          GH_TOKEN: ${{ github.token }}

.pre-commit-config.yaml

@@ -1,7 +1,8 @@
 exclude: "docs|node_modules|migrations|.git|.tox"
-default_stages: [commit]
+default_stages: [pre-commit]
 fail_fast: true
 
+# (run `python scripts/sync_pre_commit_versions.py` to sync them)
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
@@ -27,15 +28,16 @@ repos:
     rev: 7.3.0
     hooks:
       - id: flake8
-        args: ["--config=setup.cfg"]
-        additional_dependencies: [flake8-isort]
+        args: ["--config=.flake8"]
+        additional_dependencies: [flake8-isort==6.1.2]
         exclude: analysis_data_preprocess
 
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.18.2
     hooks:
       - id: mypy
-        args: ["--config=setup.cfg", "--install-types", "--non-interactive"]
+        args: ["--config=pyproject.toml", "--install-types", "--non-interactive"]
+        additional_dependencies: [types-PyYAML]
         exclude: test*
 
 # https://pre-commit.ci/#configuration

.vscode/settings.json

@@ -9,7 +9,7 @@
   },
   "python.formatting.provider": "black",
   "python.linting.flake8Enabled": true,
-  "python.linting.flake8Args": ["--config=setup.cfg"],
+  "python.linting.flake8Args": ["--config=.flake8"],
   "python.linting.mypyEnabled": true,
   "python.pythonPath": "/opt/miniconda3/envs/zstash_dev/bin/python"
 }

conda/dev.yml

@@ -12,12 +12,17 @@ dependencies:
   - globus-sdk >=3.15.0,<4.0
   # Developer Tools
   # =================
-  # If versions are updated, also update 'rev' in `.pre-commit-config.yaml`
+  # Run `pre-commit autoupdate` to get the latest pinned versions of 'rev' in
+  # `.pre-commit-config.yaml`, then run
+  # `python scripts/sync_pre_commit_versions.py` to update the pinned versions
+  # here and in the `qa` extra of pyproject.toml.
   - black ==25.1.0
   - flake8 ==7.3.0
+  - flake8-isort ==6.1.2
   - isort ==6.0.1
   - mypy ==1.18.2
   - pre-commit ==4.3.0
+  - pyyaml
   - tbump >=6.9.0
   # Testing
   # =======================

pyproject.toml

@@ -0,0 +1,95 @@
+[build-system]
+requires = [
+  "setuptools>=64",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "zstash"
+version = "1.6.0"
+description = "Long term HPSS archiving software for E3SM"
+readme = "README.md"
+requires-python = ">=3.11,<3.15"
+authors = [
+  { name = "Ryan Forsyth", email = "forsyth2@llnl.gov" },
+  { name = "Chris Golaz", email = "golaz1@llnl.gov" },
+  { name = "Zeshawn Shaheen", email = "shaheen2@llnl.gov" },
+]
+
+[project.scripts]
+zstash = "zstash.main:main"
+
+# make sure these always match pinned versions in conda/dev.yml and .pre-commit-config.yaml
+# (run `python scripts/sync_pre_commit_versions.py` to sync them)
+[project.optional-dependencies]
+qa = [
+  "black==25.1.0",
+  "flake8==7.3.0",
+  "flake8-isort==6.1.2",
+  "isort==6.0.1",
+  "mypy==1.18.2",
+  "pre-commit==4.3.0",
+  "pyyaml",
+]
+
+dev = [
+  "tbump>=6.9.0",
+]
+
+[tool.setuptools.packages.find]
+include = ["zstash", "zstash.*"]
+
+# `pre-commit-update` rewrites .pre-commit-config.yaml with ruamel.yaml.  Without
+# this, it defaults to `indent(sequence=4)`, which puts the `-` of each list item
+# in column 0 and reformats the whole file.  These settings keep its output in
+# the conventional `  - item` style, so an update is a diff of just the revs.
+[tool.pre-commit-update.yaml]
+mapping = 2
+sequence = 4
+offset = 2
+
+[tool.black]
+line-length = 88
+target-version = ['py311']
+include = '\.pyi?$'
+exclude = '''
+/(
+      \.eggs
+    | \.git
+    | \.mypy_cache
+    | _build
+    | conda
+    | docs
+ )/
+'''
+
+[tool.isort]
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+use_parentheses = true
+line_length = 88
+
+[tool.pycodestyle]
+max-line-length = 119
+exclude = [
+  ".tox",
+  ".git",
+  "*/migrations/*",
+  "*/static/CACHE/*",
+  "docs",
+  "node_modules",
+  ".idea",
+  ".mypy_cache",
+  ".pytest_cache",
+  "*__init__.py",
+  "venv",
+]
+
+[tool.mypy]
+python_version = "3.14"
+check_untyped_defs = true
+ignore_missing_imports = true
+warn_unused_ignores = true
+warn_redundant_casts = true
+warn_unused_configs = true