Add global ruff linter; unify python/ onto repo-wide ruff

.github/workflows/check.yml

@@ -40,17 +40,14 @@ jobs:
         with:
           version: "0.8.9"
 
-      - name: Run Go lint checks (does not include formatting checks)
+      - name: Run Go and Python lint checks (does not include formatting checks)
+        # `task lint` runs golangci-lint and `ruff check` (lint-python). ruff is
+        # provided by uvx, installed above; formatting is checked by `task fmt` below.
         run: go tool -modfile=tools/task/go.mod task lint
 
-      - name: Run ruff (Python linter and formatter)
-        uses: astral-sh/ruff-action@0ce1b0bf8b818ef400413f810f8a11cdbda0034b # v4.0.0
-        with:
-          version: "0.15.17"
-          args: "format --check"
-
-      - name: "task fmt: Python and Go formatting"
-        # Python formatting is already checked above, but this also checks Go and YAML formatting
+      - name: "task fmt: Python, Go and YAML formatting"
+        # `task fmt` runs ruff format (Python) plus Go and YAML formatters; the diff
+        # check fails the build if anything was left unformatted.
         run: |
           go tool -modfile=tools/task/go.mod task fmt
           git diff --exit-code

Taskfile.yml

@@ -4,6 +4,9 @@ vars:
   # Absolute path so tasks with `dir:` (lint-go-tools, lint-go-codegen) can use it.
   GO_TOOL: go tool -modfile={{.ROOT_DIR}}/tools/go.mod
   EXE_EXT: '{{if eq OS "windows"}}.exe{{end}}'
+  # Pinned ruff (linter + formatter). Keep in sync with the version in
+  # .github/workflows/check.yml and python/Taskfile.yml.
+  RUFF: uvx ruff@0.15.17
   TEST_PACKAGES: ./acceptance/internal ./libs/... ./internal/... ./cmd/... ./bundle/... ./experimental/ssh/... .
   ACCEPTANCE_TEST_FILTER: ""
   # Single brace-expansion glob covering every //go:embed target in the repo,
@@ -75,14 +78,29 @@ tasks:
   # speed; `./task all` uses the full variants.
 
   lint:
-    desc: Lint all Go files (root + tools + codegen modules)
+    desc: Lint all Go files (root + tools + codegen modules) and Python scripts
     cmds:
       - task: lint-go
+      - task: lint-python
 
   lint-q:
-    desc: Lint changed Go files in root module (diff vs main, with --fix)
+    desc: Lint changed Go files in root module (diff vs main, with --fix) and Python scripts
     cmds:
       - task: lint-q-go-root
+      - task: lint-python
+
+  lint-python:
+    desc: Lint first-party Python scripts with ruff
+    sources:
+      - "**/*.py"
+      - ruff.toml
+    cmds:
+      # Lints the whole repo; fixtures and generated outputs are skipped via
+      # `lint.exclude` / `extend-exclude` in ruff.toml. --no-cache: some excluded
+      # acceptance fixtures carry their own pyproject.toml/ruff.toml (config
+      # roots), so without it ruff writes a .ruff_cache into those checked-in
+      # dirs, polluting the goldens and breaking the acceptance tests.
+      - "{{.RUFF}} check --no-cache ."
 
   # `golangci-lint run` typechecks, so it stops at go.mod boundaries. We have
   # one task per Go module; `lint-go` composes them to cover the whole repo.
@@ -166,14 +184,19 @@ tasks:
     deps: ['fmt-python', 'fmt-q-go', 'fmt-yaml']
 
   fmt-python:
-    desc: Format Python files
+    desc: Lint-fix and format Python files
     sources:
       - "**/*.py"
+      - ruff.toml
     cmds:
-      # Pinned to match the version used by the `ruff format --check` step in
+      # Auto-fix the lint issues that ruff can fix (whole repo, minus lint.exclude);
+      # `|| true` so unfixable findings don't abort fmt (lint-python reports them).
+      # --no-cache: see lint-python (avoids polluting acceptance fixture goldens).
+      - "{{.RUFF}} check --no-cache --fix . || true"
+      # Pinned (via the RUFF var) to match the `ruff format --check` step in
       # .github/workflows/check.yml — newer ruff versions reformat files that
       # CI considers already formatted.
-      - "uvx ruff@0.15.17 format -n"
+      - "{{.RUFF}} format -n"
 
   # `golangci-lint fmt` walks the filesystem and doesn't typecheck, so it
   # formats files across all nested modules (tools/, bundle/internal/tf/codegen/)

acceptance/bin/add_repl.py

@@ -5,8 +5,8 @@
 If entry already exists, it'll add suffix in _<number> format.
 """
 
-import os
 import argparse
+import os
 from pathlib import Path
 
 ACC_REPLS = Path(os.environ["TEST_TMP_DIR"]) / "ACC_REPLS"

acceptance/bin/assert_exists.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
-import os, sys
+import os
+import sys
 
 errors = 0
 

acceptance/bin/assert_not_exists.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
-import os, sys
+import os
+import sys
 
 errors = 0
 

acceptance/bin/benchmark.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
 import argparse
-import subprocess
-import time
+import json
+import os
 import statistics
+import subprocess
 import sys
-import os
-import json
+import time
 
 try:
     import resource
@@ -19,7 +19,7 @@ def run_benchmark(command, warmup, runs):
 
     for i in range(runs):
         # double fork to reset max statistics like ru_maxrss
-        cp = subprocess.run([sys.executable, sys.argv[0], "--once"] + command, stdout=subprocess.PIPE)
+        cp = subprocess.run([sys.executable, sys.argv[0], "--once", *command], stdout=subprocess.PIPE)
         if cp.returncode != 0:
             sys.exit(cp.returncode)
 
@@ -54,7 +54,7 @@ def run_benchmark(command, warmup, runs):
 
 
 def run_once(command):
-    if len(command) == 1 and " " in command[0] or ">" in command[0]:
+    if (len(command) == 1 and " " in command[0]) or ">" in command[0]:
         shell = True
         command = command[0]
     else:

acceptance/bin/browser.py

@@ -6,8 +6,8 @@
 Usage: browser.py <url>
 """
 
-import urllib.request
 import sys
+import urllib.request
 
 if len(sys.argv) < 2:
     sys.stderr.write("Usage: browser.py <url>\n")

acceptance/bin/callserver.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
-import sys
-import os
 import json
+import os
+import sys
 import urllib.request
 from urllib.parse import urlencode
 

acceptance/bin/diff.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
 """This script implements "diff -r -U2 dir1 dir2" but applies replacements first"""
 
-import sys
-import os
 import difflib
 import json
+import os
 import re
+import sys
 from pathlib import Path
 
 

acceptance/bin/edit_resource.py

@@ -5,17 +5,15 @@
 Fetches a given resource by id from the backend, executes Python code read from stdin and updates the resource.
 """
 
-import sys
-import os
-import subprocess
 import argparse
 import json
+import os
 import pprint
+import sys
 
 sys.path.insert(0, os.path.dirname(__file__))
 import util
-from util import run_json, run
-
+from util import run, run_json
 
 CLI = os.environ["CLI"]
 

acceptance/bin/envsubst.py

@@ -14,9 +14,9 @@
   echo 'Hello $USER from $HOME' | python envsubst.py
 """
 
-import sys
 import os
 import re
+import sys
 
 
 def substitute_variables(text):

acceptance/bin/extract_command_exec_id.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 import json
 import re
-import sys
 from pathlib import Path
 
 

acceptance/bin/find.py

@@ -6,11 +6,10 @@
 If --expect N is provided, the number of matches must be N or error is printed.
 """
 
-import sys
+import argparse
 import os
 import re
-import argparse
-
+import sys
 
 parser = argparse.ArgumentParser()
 parser.add_argument("regex")
@@ -20,7 +19,7 @@
 regex = re.compile(args.regex)
 result = []
 
-for root, dirs, files in os.walk("."):
+for root, _dirs, files in os.walk("."):
     for filename in files:
         path = os.path.join(root, filename).replace("\\", "/")
         path = path.removeprefix("./")

acceptance/bin/gen_config.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import argparse
-import json
 import copy
+import json
 
 JOB_TEMPLATE_BASE = {
     "description": "This job contain multiple tasks that are required to produce the weekly shark sightings report.",

acceptance/bin/kill_port.py

@@ -4,10 +4,10 @@
 Usage: kill_port.py <port1> [port2] [port3] ...
 """
 
-import sys
-import subprocess
 import platform
 import re
+import subprocess
+import sys
 
 
 def kill_processes_on_port(port):
@@ -58,7 +58,7 @@ def kill_processes_on_port(port):
                                     capture_output=True,
                                 )
 
-    except Exception as e:
+    except Exception:
         # Silently continue - port cleanup is best effort
         pass
 

acceptance/bin/print_requests.py

@@ -78,13 +78,12 @@
 ['GET', 'DELETE', 'GET']
 """
 
+import argparse
+import json
 import os
 import sys
-import json
-import argparse
 from pathlib import Path
 
-
 # I've originally tried ADD_PREFIX to be empty, so you can just do "print_requests.py /jobs"
 # However, that causes test to fail on Windows CI because "/jobs" becomes "C:/Program Files/Git/jobs"
 # This behaviour can be disabled with MSYS_NO_PATHCONV=1 but that causes other failures, so we require extra slash here.
@@ -181,7 +180,7 @@ def filter_requests(requests, path_filters, include_get, should_sort, unique=Fal
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument("path_filters", nargs="*", help=f"Path substring filters")
+    parser.add_argument("path_filters", nargs="*", help="Path substring filters")
     parser.add_argument("-v", "--verbose", action="store_true", help="Enable diagnostic messages")
     parser.add_argument("--get", action="store_true", help="Include GET requests (excluded by default)")
     parser.add_argument("--keep", action="store_true", help="Keep out.requests.json file after processing")

acceptance/bin/print_state.py

@@ -6,9 +6,9 @@
 the goal is to record all states that are available.
 """
 
-import os
-import glob
 import argparse
+import glob
+import os
 
 
 def print_file(filename):

acceptance/bin/read_id.py

@@ -8,14 +8,14 @@
 Usage: <group> <name> [attr...]
 """
 
-import sys
-import json
 import argparse
+import json
+import sys
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent))
-from print_state import get_state_file
 from add_repl import add_repl
+from print_state import get_state_file
 
 
 def get_id_terraform(filename, name):

acceptance/bin/read_state.py

@@ -5,9 +5,9 @@
 Usage: <group> <name> [attr...]
 """
 
-import sys
-import os
 import json
+import os
+import sys
 
 
 def print_resource_terraform(group, name, *attrs):

acceptance/bin/replace_ids.py

@@ -3,14 +3,14 @@
 Read state and add all resource IDs to ACC_REPLS.
 """
 
-import sys
-import json
 import argparse
+import json
+import sys
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent))
-from print_state import get_state_file
 from add_repl import add_repl
+from print_state import get_state_file
 
 
 def iter_ids_terraform(filename):

acceptance/bin/setmtime.py

@@ -4,9 +4,9 @@
 Usage: setmtime.py <timestamp> <filenames>
 """
 
-import sys
-import os
 import datetime
+import os
+import sys
 
 timestamp = sys.argv[1]
 ts, ns = timestamp.split(".")

acceptance/bin/sort_lines.py

@@ -4,10 +4,10 @@
 With --repl, applies TEST_TMP_DIR/repls.json replacements as sort key for stable output across different environments.
 """
 
-import sys
-import os
 import json
+import os
 import re
+import sys
 from pathlib import Path
 
 use_repl = "--repl" in sys.argv[1:]