From ac3b3ecba45451be2ae3dd55f3eb33ec5f070d78 Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Sun, 27 Sep 2026 07:49:28 -0700 Subject: [PATCH 1/2] ci: enforce running-process cache budget --- .github/workflows/cache-budget.yml | 34 ++ .github/workflows/ci-all-features.yml | 7 +- ci/cache_budget.py | 312 ++++++++++++++++++ tests/test_cache_budget.py | 451 ++++++++++++++++++++++++++ 4 files changed, 802 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/cache-budget.yml create mode 100644 ci/cache_budget.py create mode 100644 tests/test_cache_budget.py diff --git a/.github/workflows/cache-budget.yml b/.github/workflows/cache-budget.yml new file mode 100644 index 00000000..8374abd6 --- /dev/null +++ b/.github/workflows/cache-budget.yml @@ -0,0 +1,34 @@ +name: Cache budget (main) + +on: + push: + branches: [main] + +permissions: + contents: read + +jobs: + enforce-cache-budget: + name: Retire disabled all-features caches and check repository budget + runs-on: ubuntu-latest + timeout-minutes: 240 + permissions: + actions: write + contents: read + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - uses: astral-sh/setup-uv@v5 + + - name: Retire the disabled families and enforce the live budget + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_REF: ${{ github.ref }} + GITHUB_SHA: ${{ github.sha }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_TOKEN: ${{ github.token }} + run: uv run --no-project ci/cache_budget.py diff --git a/.github/workflows/ci-all-features.yml b/.github/workflows/ci-all-features.yml index b8869eee..34c696dd 100644 --- a/.github/workflows/ci-all-features.yml +++ b/.github/workflows/ci-all-features.yml @@ -105,9 +105,12 @@ jobs: - name: Rust build cache uses: Swatinem/rust-cache@v2 with: - save-if: ${{ github.ref == 'refs/heads/main' }} + # This optional cross-platform lane adds >2 GB of target archives + # across its three OS shapes. Restore remains useful when a cache + # exists, but durable writes are disabled so it cannot grow main's + # repository cache budget (#1216). + save-if: false key: ${{ matrix.label }}-all-features - cache-on-failure: true - name: Sync venv (frozen, no project install) run: uv sync --frozen --group dev --no-install-project --python 3.11 diff --git a/ci/cache_budget.py b/ci/cache_budget.py new file mode 100644 index 00000000..3cd0da11 --- /dev/null +++ b/ci/cache_budget.py @@ -0,0 +1,312 @@ +"""Retire the disabled all-features Rust target cache and enforce budget.""" + +from __future__ import annotations + +import json +import os +import sys +import time +import urllib.error +import urllib.request +from collections.abc import Callable, Mapping +from typing import Any + +API_ROOT = "https://api.github.com" +MAIN_REF = "refs/heads/main" +BUDGET_LIMIT_BYTES = 9_500_000_000 +POLL_ATTEMPTS = 6 +POLL_INTERVAL_SECONDS = 10 +WRITER_BARRIER_ATTEMPTS = 356 +WRITER_BARRIER_INTERVAL_SECONDS = 30 +WRITER_QUIET_SAMPLES = 2 +WRITER_QUIET_INTERVAL_SECONDS = 120 +ACTIVE_RUN_STATUSES = ("queued", "in_progress", "waiting", "requested", "pending") +KNOWN_TERMINAL_RUN_STATUSES = { + "completed", +} +# Display names of workflows that can write a durable Actions cache on main. +# Reusable workflows are included because GitHub exposes them in the Runs API +# separately on some event paths; unknown active statuses fail closed below. +CACHE_WRITER_WORKFLOWS = { + "All-Features Tests (Windows/macOS)", + "Auto Release", + "Build Template", + "CI Preflight (Linux)", + "CI Preflight (macOS)", + "CI Preflight (Windows)", + "CI Preflight (Reusable)", + "CI Preflight Rust-only (Reusable)", + "Coverage", + "Independent spawn", + "Linux x86 Build", + "Native output shutdown", + "Linux ARM Build", + "macOS ARM Build", + "macOS x86 Build", + "Security audit", + "security-fuzz", + "Windows ARM Build", + "Windows x86 Build", +} +CACHE_WRITER_REUSABLE_WORKFLOWS = { + "_build.yml", + "ci-preflight-rust.yml", + "ci-preflight.yml", +} +ALL_FEATURES_CACHE_PREFIXES = ( + "v0-rust-windows-x86-all-features-", + "v0-rust-macos-x86-all-features-", + "v0-rust-macos-arm-all-features-", +) + + +def is_retired_cache(cache: Mapping[str, Any]) -> bool: + """Select only the three disabled cache families on main.""" + ref = cache.get("ref") + key = cache.get("key") + return ( + ref == MAIN_REF + and isinstance(key, str) + and key.startswith(ALL_FEATURES_CACHE_PREFIXES) + ) + + +def listed_bytes(caches: list[Mapping[str, Any]]) -> int: + return sum(int(cache.get("size_in_bytes", 0)) for cache in caches) + + +def budget_bytes(endpoint_bytes: int, caches: list[Mapping[str, Any]]) -> int: + """Use the conservative maximum while the usage endpoint/list may lag.""" + return max(endpoint_bytes, listed_bytes(caches)) + + +def assert_main_sha(api: GitHubCacheAPI, expected_sha: str) -> None: + current_sha = api.main_sha() + if current_sha != expected_sha: + raise RuntimeError( + "main advanced during cache-budget workflow: " + f"expected {expected_sha}, found {current_sha}" + ) + + +class GitHubCacheAPI: + def __init__( + self, + repository: str, + token: str, + *, + opener: Callable[..., Any] = urllib.request.urlopen, + ) -> None: + self.repository = repository + self.token = token + self.opener = opener + + def _request(self, path: str, *, method: str = "GET") -> Any: + request = urllib.request.Request( + f"{API_ROOT}/repos/{self.repository}{path}", + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.token}", + "X-GitHub-Api-Version": "2022-11-28", + }, + method=method, + ) + try: + with self.opener(request, timeout=30) as response: + body = response.read() + except urllib.error.HTTPError as error: + if method == "DELETE" and error.code == 404: + return None + raise + return json.loads(body) if body else None + + def list_caches(self) -> list[Mapping[str, Any]]: + caches: list[Mapping[str, Any]] = [] + page = 1 + while True: + payload = self._request(f"/actions/caches?per_page=100&page={page}") + page_caches = payload.get("actions_caches", []) + caches.extend(page_caches) + if len(page_caches) < 100: + return caches + page += 1 + + def usage_bytes(self) -> int: + payload = self._request("/actions/cache/usage") + return int(payload["active_caches_size_in_bytes"]) + + def main_sha(self) -> str: + payload = self._request("/git/ref/heads/main") + return str(payload["object"]["sha"]) + + def active_main_writer_runs( + self, *, exclude_run_id: int + ) -> list[Mapping[str, Any]]: + runs: list[Mapping[str, Any]] = [] + for status in ACTIVE_RUN_STATUSES: + page = 1 + while True: + payload = self._request( + "/actions/runs?branch=main&per_page=100" + f"&status={status}&page={page}" + ) + page_runs = payload.get("workflow_runs", []) + for run in page_runs: + run_status = run.get("status") + if ( + run_status + not in set(ACTIVE_RUN_STATUSES) | KNOWN_TERMINAL_RUN_STATUSES + ): + raise RuntimeError( + f"unknown workflow run status from GitHub API: {run_status!r}" + ) + if ( + int(run.get("id", -1)) != exclude_run_id + and run.get("head_branch") == "main" + and run.get("name") in CACHE_WRITER_WORKFLOWS + and run_status in ACTIVE_RUN_STATUSES + ): + runs.append(run) + if len(page_runs) < 100: + break + page += 1 + return runs + + def delete_cache(self, cache_id: int) -> None: + self._request(f"/actions/caches/{cache_id}", method="DELETE") + + +def wait_for_main_writers( + api: GitHubCacheAPI, + *, + expected_sha: str, + own_run_id: int, + sleep: Callable[[float], None] = time.sleep, + attempts: int = WRITER_BARRIER_ATTEMPTS, + interval: int = WRITER_BARRIER_INTERVAL_SECONDS, + quiet_samples: int = WRITER_QUIET_SAMPLES, + quiet_interval: int = WRITER_QUIET_INTERVAL_SECONDS, +) -> None: + """Wait for main cache writers to finish and for their run records to settle.""" + consecutive_quiet = 0 + for attempt in range(1, attempts + 1): + assert_main_sha(api, expected_sha) + + writers = api.active_main_writer_runs(exclude_run_id=own_run_id) + if writers: + consecutive_quiet = 0 + names = sorted({str(run.get("name", "unknown")) for run in writers}) + print(f"waiting for active main cache writers: {', '.join(names)}") + if attempt < attempts: + sleep(interval) + continue + + consecutive_quiet += 1 + if consecutive_quiet >= quiet_samples: + # Ensure no newly-dispatched push/scheduled writer appeared just + # after the previous empty snapshot. + return + if attempt < attempts: + sleep(quiet_interval) + + raise RuntimeError( + f"main cache writers did not settle after {attempts} barrier snapshots" + ) + + +def enforce_budget( + api: GitHubCacheAPI, + *, + expected_sha: str | None = None, + sleep: Callable[[float], None] = time.sleep, + attempts: int = POLL_ATTEMPTS, + interval: int = POLL_INTERVAL_SECONDS, +) -> int: + if expected_sha is not None: + assert_main_sha(api, expected_sha) + caches = api.list_caches() + candidates = [cache for cache in caches if is_retired_cache(cache)] + for cache in candidates: + if expected_sha is not None: + assert_main_sha(api, expected_sha) + cache_id = int(cache["id"]) + print( + "retiring disabled all-features cache " + f"id={cache_id} ref={cache['ref']} size={cache['size_in_bytes']} " + f"key={cache['key']}" + ) + api.delete_cache(cache_id) + + previous_snapshot: tuple[int, int] | None = None + for attempt in range(1, attempts + 1): + if expected_sha is not None: + assert_main_sha(api, expected_sha) + endpoint_bytes = api.usage_bytes() + current_caches = api.list_caches() + listed_size = listed_bytes(current_caches) + effective_size = budget_bytes(endpoint_bytes, current_caches) + remaining = [cache for cache in current_caches if is_retired_cache(cache)] + print( + "cache budget: " + f"max(endpoint={endpoint_bytes}, listed={listed_size})=" + f"{effective_size} bytes; limit={BUDGET_LIMIT_BYTES}; " + f"retired={len(candidates)} entries/" + f"{sum(int(c['size_in_bytes']) for c in candidates)} bytes; " + f"remaining_disabled={len(remaining)}" + ) + + snapshot = (endpoint_bytes, listed_size) + if not remaining and endpoint_bytes == listed_size: + if snapshot == previous_snapshot: + if effective_size > BUDGET_LIMIT_BYTES: + raise RuntimeError( + f"repository cache budget exceeded: {effective_size} > " + f"{BUDGET_LIMIT_BYTES} bytes" + ) + if expected_sha is not None: + assert_main_sha(api, expected_sha) + return effective_size + previous_snapshot = snapshot + else: + previous_snapshot = None + + if attempt < attempts: + sleep(interval) + + raise RuntimeError( + "cache inventory did not converge below budget within " f"{attempts} snapshots" + ) + + +def main() -> int: + if os.environ.get("GITHUB_REF") != MAIN_REF: + print("cache budget cleanup is restricted to refs/heads/main", file=sys.stderr) + return 2 + repository = os.environ.get("GITHUB_REPOSITORY") + token = os.environ.get("GITHUB_TOKEN") + expected_sha = os.environ.get("GITHUB_SHA") + run_id = os.environ.get("GITHUB_RUN_ID") + if not repository or not token or not expected_sha or not run_id: + print( + "GITHUB_REPOSITORY, GITHUB_TOKEN, GITHUB_SHA, and GITHUB_RUN_ID are required", + file=sys.stderr, + ) + return 2 + + api = GitHubCacheAPI(repository, token) + try: + wait_for_main_writers( + api, + expected_sha=expected_sha, + own_run_id=int(run_id), + ) + size = enforce_budget(api, expected_sha=expected_sha) + except (RuntimeError, urllib.error.URLError, ValueError, KeyError) as error: + print(f"cache budget enforcement failed: {error}", file=sys.stderr) + return 1 + print(f"cache budget green: {size} <= {BUDGET_LIMIT_BYTES} bytes") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_cache_budget.py b/tests/test_cache_budget.py new file mode 100644 index 00000000..652e6b0b --- /dev/null +++ b/tests/test_cache_budget.py @@ -0,0 +1,451 @@ +"""Tests for the main-only all-features cache retirement policy.""" + +import io +import json +import re +import unittest +from collections.abc import Mapping +from pathlib import Path +from typing import Any +from unittest.mock import patch +from urllib.error import HTTPError + +from ci import cache_budget + + +def entry( + cache_id: int, + key: str, + *, + ref: str = "refs/heads/main", + size: int = 100, +) -> Mapping[str, Any]: + return { + "id": cache_id, + "key": key, + "ref": ref, + "size_in_bytes": size, + } + + +class CacheBudgetPolicyTests(unittest.TestCase): + def test_selects_only_exact_disabled_families_on_main(self) -> None: + candidates = [ + entry( + 1, + "v0-rust-windows-x86-all-features-all-features-Windows-x64-digest", + ), + entry( + 2, + "v0-rust-macos-x86-all-features-all-features-Darwin-x64-digest", + ), + entry( + 3, + "v0-rust-macos-arm-all-features-all-features-Darwin-arm64-digest", + ), + entry(4, "v0-rust-linux-x86-all-features-all-features-Linux-x64-digest"), + entry(5, "v0-rust-windows-x86-shared-preflight-Windows-x64-digest"), + entry( + 6, + "v0-rust-windows-x86-all-features-all-features-Windows-x64-digest", + ref="refs/pull/1216/merge", + ), + ] + + self.assertEqual( + [ + int(cache["id"]) + for cache in candidates + if cache_budget.is_retired_cache(cache) + ], + [1, 2, 3], + ) + + def test_budget_uses_max_of_usage_endpoint_and_listing(self) -> None: + caches = [entry(1, "v0-rust-current", size=200)] + + self.assertEqual(cache_budget.budget_bytes(300, caches), 300) + self.assertEqual(cache_budget.budget_bytes(100, caches), 200) + + def test_budget_limit_is_conservative_decimal_target(self) -> None: + self.assertEqual(cache_budget.BUDGET_LIMIT_BYTES, 9_500_000_000) + + def test_restore_only_all_features_cache_step(self) -> None: + workflow = ( + Path(__file__).resolve().parents[1] + / ".github" + / "workflows" + / "ci-all-features.yml" + ) + lines = workflow.read_text(encoding="utf-8").splitlines() + cache_step_start = next( + index + for index, line in enumerate(lines) + if "uses: Swatinem/rust-cache@v2" in line + and any( + "name: Rust build cache" in prior + for prior in lines[max(0, index - 2) : index] + ) + ) + step = "\n".join(lines[cache_step_start : cache_step_start + 10]) + self.assertIn("save-if: false", step) + self.assertNotIn("cache-on-failure: true", step) + + def test_enforcer_retires_only_the_disabled_main_families(self) -> None: + stale = [ + entry( + 1, + "v0-rust-windows-x86-all-features-all-features-Windows-x64-old", + size=800, + ), + entry( + 2, + "v0-rust-macos-x86-all-features-all-features-Darwin-x64-old", + size=600, + ), + entry( + 3, + "v0-rust-macos-arm-all-features-all-features-Darwin-arm64-old", + size=600, + ), + entry(4, "v0-rust-windows-x86-shared-preflight-Windows-x64", size=200), + ] + + class FakeAPI: + def __init__(self) -> None: + self.caches = list(stale) + self.deleted: list[int] = [] + + def list_caches(self) -> list[Mapping[str, Any]]: + return list(self.caches) + + def usage_bytes(self) -> int: + return cache_budget.listed_bytes(self.caches) + + def delete_cache(self, cache_id: int) -> None: + self.deleted.append(cache_id) + self.caches = [ + cache for cache in self.caches if int(cache["id"]) != cache_id + ] + + api = FakeAPI() + budget = cache_budget.enforce_budget( + api, sleep=lambda _seconds: None, attempts=2 + ) + + self.assertEqual(api.deleted, [1, 2, 3]) + self.assertEqual([int(cache["id"]) for cache in api.caches], [4]) + self.assertEqual(budget, 200) + + def test_enforcer_fails_when_other_families_keep_budget_over_limit(self) -> None: + class FakeAPI: + def __init__(self) -> None: + self.caches = [ + entry( + 1, + "v0-rust-macos-arm-all-features-all-features-Darwin-arm64-old", + size=100, + ), + entry( + 2, + "v0-rust-windows-x86-shared-preflight-Windows-x64", + size=cache_budget.BUDGET_LIMIT_BYTES + 1, + ), + ] + + def list_caches(self) -> list[Mapping[str, Any]]: + return list(self.caches) + + def usage_bytes(self) -> int: + return cache_budget.listed_bytes(self.caches) + + def delete_cache(self, cache_id: int) -> None: + self.caches = [ + cache for cache in self.caches if int(cache["id"]) != cache_id + ] + + with self.assertRaisesRegex(RuntimeError, "repository cache budget exceeded"): + cache_budget.enforce_budget( + FakeAPI(), sleep=lambda _seconds: None, attempts=2 + ) + + def test_enforcer_waits_for_usage_and_listing_to_converge(self) -> None: + class FakeAPI: + def __init__(self) -> None: + self.list_calls = 0 + + def list_caches(self) -> list[Mapping[str, Any]]: + self.list_calls += 1 + if self.list_calls == 1: + return [] + size = ( + cache_budget.BUDGET_LIMIT_BYTES + 1 + if self.list_calls < 4 + else cache_budget.BUDGET_LIMIT_BYTES - 1 + ) + return [entry(9, "v0-rust-current", size=size)] + + def usage_bytes(self) -> int: + return cache_budget.BUDGET_LIMIT_BYTES - 1 + + def delete_cache(self, cache_id: int) -> None: + raise AssertionError(f"unexpected deletion: {cache_id}") + + waits: list[float] = [] + result = cache_budget.enforce_budget(FakeAPI(), sleep=waits.append, attempts=6) + + self.assertEqual(result, cache_budget.BUDGET_LIMIT_BYTES - 1) + self.assertEqual(waits, [10, 10, 10]) + + def test_budget_workflow_is_main_push_and_permission_scoped(self) -> None: + workflow = ( + Path(__file__).resolve().parents[1] + / ".github" + / "workflows" + / "cache-budget.yml" + ) + text = workflow.read_text(encoding="utf-8") + self.assertIn("push:\n branches: [main]", text) + self.assertIn( + "permissions:\n contents: read\n\njobs:\n enforce-cache-budget:", + text, + ) + self.assertIn("permissions:\n actions: write\n contents: read", text) + self.assertIn("run: uv run --no-project ci/cache_budget.py", text) + self.assertIn("GITHUB_REF: ${{ github.ref }}", text) + self.assertIn("GITHUB_SHA: ${{ github.sha }}", text) + self.assertIn("GITHUB_RUN_ID: ${{ github.run_id }}", text) + self.assertNotIn("pull_request:", text) + + def test_budget_workflow_knows_the_all_features_writer_name(self) -> None: + self.assertIn( + "All-Features Tests (Windows/macOS)", + cache_budget.CACHE_WRITER_WORKFLOWS, + ) + self.assertNotIn("Cache budget (main)", cache_budget.CACHE_WRITER_WORKFLOWS) + + def test_writer_barrier_leaves_runner_timeout_headroom(self) -> None: + workflow = ( + Path(__file__).resolve().parents[1] + / ".github" + / "workflows" + / "cache-budget.yml" + ) + text = workflow.read_text(encoding="utf-8") + timeout = re.search(r"^ timeout-minutes:\s*(\d+)$", text, re.MULTILINE) + self.assertIsNotNone(timeout) + job_timeout_seconds = int(timeout.group(1)) * 60 + max_barrier_seconds = ( + (cache_budget.WRITER_BARRIER_ATTEMPTS - 1) + * cache_budget.WRITER_BARRIER_INTERVAL_SECONDS + + cache_budget.WRITER_QUIET_INTERVAL_SECONDS + ) + self.assertLess(max_barrier_seconds, job_timeout_seconds - 60 * 60) + + def test_known_writer_inventory_covers_every_cache_writer_workflow(self) -> None: + workflows = Path(__file__).resolve().parents[1] / ".github" / "workflows" + discovered: set[str] = set() + for workflow in workflows.glob("*.yml"): + text = workflow.read_text(encoding="utf-8") + direct_writer = any( + marker in text + for marker in ( + "uses: Swatinem/rust-cache@", + "uses: actions/cache/save@", + "uses: actions/cache@", + ) + ) + reusable_writer = any( + f"uses: ./.github/workflows/{template}" in text + for template in cache_budget.CACHE_WRITER_REUSABLE_WORKFLOWS + ) + if not direct_writer and not reusable_writer: + continue + match = re.search(r"^name:\s*(.+)$", text, re.MULTILINE) + self.assertIsNotNone(match, f"cache writer needs workflow name: {workflow}") + discovered.add(match.group(1).strip().strip("'\"")) + + self.assertEqual(discovered, cache_budget.CACHE_WRITER_WORKFLOWS) + for template in cache_budget.CACHE_WRITER_REUSABLE_WORKFLOWS: + text = (workflows / template).read_text(encoding="utf-8") + self.assertTrue( + any( + marker in text + for marker in ( + "uses: Swatinem/rust-cache@", + "uses: actions/cache/save@", + "uses: actions/cache@", + ) + ), + f"listed reusable cache writer has no cache writer step: {template}", + ) + + def test_writer_barrier_waits_for_active_writer_then_quiet_window(self) -> None: + writer = {"id": 7, "name": "Coverage", "status": "in_progress"} + + class FakeAPI: + def __init__(self) -> None: + self.active_calls = 0 + self.sha_calls = 0 + + def main_sha(self) -> str: + self.sha_calls += 1 + return "abc123" + + def active_main_writer_runs( + self, *, exclude_run_id: int + ) -> list[Mapping[str, Any]]: + self.assert_run_id = exclude_run_id + self.active_calls += 1 + return [writer] if self.active_calls == 1 else [] + + api = FakeAPI() + waits: list[float] = [] + cache_budget.wait_for_main_writers( + api, + expected_sha="abc123", + own_run_id=99, + sleep=waits.append, + attempts=4, + ) + + self.assertEqual(api.sha_calls, 3) + self.assertEqual(api.active_calls, 3) + self.assertEqual(api.assert_run_id, 99) + self.assertEqual( + waits, + [ + cache_budget.WRITER_BARRIER_INTERVAL_SECONDS, + cache_budget.WRITER_QUIET_INTERVAL_SECONDS, + ], + ) + + def test_writer_barrier_fails_closed_if_main_advanced(self) -> None: + class FakeAPI: + def main_sha(self) -> str: + return "newer-sha" + + def active_main_writer_runs( + self, *, exclude_run_id: int + ) -> list[Mapping[str, Any]]: + raise AssertionError("must check the main SHA before querying runs") + + with self.assertRaisesRegex(RuntimeError, "main advanced"): + cache_budget.wait_for_main_writers( + FakeAPI(), expected_sha="old-sha", own_run_id=99, sleep=lambda _: None + ) + + def test_budget_rechecks_main_sha_before_each_delete(self) -> None: + candidate = entry( + 1, + "v0-rust-macos-arm-all-features-all-features-Darwin-arm64-old", + size=100, + ) + + class FakeAPI: + def __init__(self) -> None: + self.sha_calls = 0 + self.deleted: list[int] = [] + + def main_sha(self) -> str: + self.sha_calls += 1 + return "expected" if self.sha_calls == 1 else "advanced" + + def list_caches(self) -> list[Mapping[str, Any]]: + return [candidate] + + def delete_cache(self, cache_id: int) -> None: + self.deleted.append(cache_id) + + api = FakeAPI() + with self.assertRaisesRegex(RuntimeError, "main advanced"): + cache_budget.enforce_budget(api, expected_sha="expected") + + self.assertEqual(api.deleted, []) + + def test_api_writer_inventory_fails_closed_on_unknown_status(self) -> None: + class Response: + def __enter__(self) -> "Response": + return self + + def __exit__(self, *_args: object) -> None: + return None + + def read(self) -> bytes: + return json.dumps( + { + "workflow_runs": [ + { + "id": 1, + "name": "Coverage", + "head_branch": "main", + "status": "mysterious", + } + ] + } + ).encode() + + api = cache_budget.GitHubCacheAPI( + "o/r", "token", opener=lambda *_args, **_kwargs: Response() + ) + with self.assertRaisesRegex(RuntimeError, "unknown workflow run status"): + api.active_main_writer_runs(exclude_run_id=99) + + def test_main_waits_for_writers_before_listing_or_deleting(self) -> None: + calls: list[str] = [] + + class FakeAPI: + def __init__(self, *_args: object) -> None: + pass + + with ( + patch.dict( + "os.environ", + { + "GITHUB_REF": "refs/heads/main", + "GITHUB_REPOSITORY": "o/r", + "GITHUB_TOKEN": "token", + "GITHUB_SHA": "abc123", + "GITHUB_RUN_ID": "99", + }, + clear=True, + ), + patch.object(cache_budget, "GitHubCacheAPI", FakeAPI), + patch.object( + cache_budget, + "wait_for_main_writers", + side_effect=lambda *_args, **_kwargs: calls.append("barrier"), + ), + patch.object( + cache_budget, + "enforce_budget", + side_effect=lambda _api, **kwargs: ( + calls.append("enforce") + if kwargs == {"expected_sha": "abc123"} + else self.fail(f"unexpected enforcement args: {kwargs}") + ) + or 0, + ), + ): + self.assertEqual(cache_budget.main(), 0) + + self.assertEqual(calls[0], "barrier") + self.assertEqual(calls, ["barrier", "enforce"]) + + def test_delete_404_is_idempotent_for_overlapping_main_runs(self) -> None: + def not_found(_request: Any, *, timeout: int) -> Any: + del timeout + raise HTTPError( + "https://api.github.com/repos/o/r/actions/caches/9", + 404, + "not found", + hdrs=None, + fp=io.BytesIO(), + ) + + api = cache_budget.GitHubCacheAPI("o/r", "token", opener=not_found) + self.assertIsNone(api.delete_cache(9)) + + +if __name__ == "__main__": + unittest.main() From fd192d2add66abd899508a35e230e90f7c386f42 Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Sun, 27 Sep 2026 08:31:45 -0700 Subject: [PATCH 2/2] ci: retire disabled caches before main producers --- ci/cache_budget.py | 24 +++++++++++++++----- tests/test_cache_budget.py | 46 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/ci/cache_budget.py b/ci/cache_budget.py index 3cd0da11..73a90ab1 100644 --- a/ci/cache_budget.py +++ b/ci/cache_budget.py @@ -214,14 +214,12 @@ def wait_for_main_writers( ) -def enforce_budget( +def retire_disabled_caches( api: GitHubCacheAPI, *, expected_sha: str | None = None, - sleep: Callable[[float], None] = time.sleep, - attempts: int = POLL_ATTEMPTS, - interval: int = POLL_INTERVAL_SECONDS, -) -> int: +) -> list[Mapping[str, Any]]: + """Retire only disabled all-features cache families on main.""" if expected_sha is not None: assert_main_sha(api, expected_sha) caches = api.list_caches() @@ -236,6 +234,18 @@ def enforce_budget( f"key={cache['key']}" ) api.delete_cache(cache_id) + return candidates + + +def enforce_budget( + api: GitHubCacheAPI, + *, + expected_sha: str | None = None, + sleep: Callable[[float], None] = time.sleep, + attempts: int = POLL_ATTEMPTS, + interval: int = POLL_INTERVAL_SECONDS, +) -> int: + candidates = retire_disabled_caches(api, expected_sha=expected_sha) previous_snapshot: tuple[int, int] | None = None for attempt in range(1, attempts + 1): @@ -295,6 +305,10 @@ def main() -> int: api = GitHubCacheAPI(repository, token) try: + # Free this known-dead space before other main workflows reach their + # post-job cache uploads. A later writer may recreate a key, so the + # post-barrier enforcement below repeats this exact-family cleanup. + retire_disabled_caches(api, expected_sha=expected_sha) wait_for_main_writers( api, expected_sha=expected_sha, diff --git a/tests/test_cache_budget.py b/tests/test_cache_budget.py index 652e6b0b..01848b0b 100644 --- a/tests/test_cache_budget.py +++ b/tests/test_cache_budget.py @@ -137,6 +137,40 @@ def delete_cache(self, cache_id: int) -> None: self.assertEqual([int(cache["id"]) for cache in api.caches], [4]) self.assertEqual(budget, 200) + def test_repeated_retirement_catches_a_cache_recreated_by_an_active_writer( + self, + ) -> None: + class FakeAPI: + def __init__(self) -> None: + self.caches: list[Mapping[str, Any]] = [ + entry(1, "v0-rust-macos-arm-all-features-all-features-old") + ] + self.deleted: list[int] = [] + + def list_caches(self) -> list[Mapping[str, Any]]: + return list(self.caches) + + def delete_cache(self, cache_id: int) -> None: + self.deleted.append(cache_id) + self.caches = [ + cache for cache in self.caches if int(cache["id"]) != cache_id + ] + + api = FakeAPI() + first_phase = cache_budget.retire_disabled_caches(api) + self.assertEqual([int(cache["id"]) for cache in first_phase], [1]) + + # A scheduled run that started before the barrier can repopulate the + # same retired family after the early pass; final enforcement catches it. + api.caches.append( + entry(2, "v0-rust-macos-arm-all-features-all-features-recreated") + ) + second_phase = cache_budget.retire_disabled_caches(api) + + self.assertEqual([int(cache["id"]) for cache in second_phase], [2]) + self.assertEqual(api.deleted, [1, 2]) + self.assertEqual(api.caches, []) + def test_enforcer_fails_when_other_families_keep_budget_over_limit(self) -> None: class FakeAPI: def __init__(self) -> None: @@ -411,6 +445,15 @@ def __init__(self, *_args: object) -> None: clear=True, ), patch.object(cache_budget, "GitHubCacheAPI", FakeAPI), + patch.object( + cache_budget, + "retire_disabled_caches", + side_effect=lambda _api, **kwargs: ( + calls.append("preclean") + if kwargs == {"expected_sha": "abc123"} + else self.fail(f"unexpected preclean args: {kwargs}") + ), + ), patch.object( cache_budget, "wait_for_main_writers", @@ -429,8 +472,7 @@ def __init__(self, *_args: object) -> None: ): self.assertEqual(cache_budget.main(), 0) - self.assertEqual(calls[0], "barrier") - self.assertEqual(calls, ["barrier", "enforce"]) + self.assertEqual(calls, ["preclean", "barrier", "enforce"]) def test_delete_404_is_idempotent_for_overlapping_main_runs(self) -> None: def not_found(_request: Any, *, timeout: int) -> Any: