diff --git a/scripts/check-oci-refs.py b/scripts/check-oci-refs.py index 0f16d7ae..385f0168 100644 --- a/scripts/check-oci-refs.py +++ b/scripts/check-oci-refs.py @@ -21,9 +21,17 @@ import os import re import sys +import time import urllib.request import urllib.error +# HTTP statuses that indicate transient infrastructure failures or rate limits, +# where retrying makes sense. 401 and non-rate-limit 403 are excluded as auth/permission +# failures are non-transient. +TRANSIENT_HTTP_STATUSES = {429, 500, 502, 503, 504} +MAX_RETRIES = 3 +RETRY_BACKOFF_SECONDS = 2 + # ── Check 1: no ublue-os refs ──────────────────────────────────────────────── # The org migration from ublue-os to projectbluefin is complete. # ghcr.io/ublue-os/ must not appear in workflow files or docs. @@ -111,8 +119,10 @@ def collect_tag_refs(root=None): return refs -def tag_exists_in_ghcr(image: str, tag: str) -> bool: - """Return True if image:tag exists in GHCR under projectbluefin.""" +def tag_exists_in_ghcr(image: str, tag: str): + """Return True/False if image:tag existence in GHCR is known, or None if + the GHCR packages API could not be reached after retries (transient + outage/rate limit) -- callers must treat None as "unknown, don't fail".""" token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN", "") page = 1 while True: @@ -126,13 +136,40 @@ def tag_exists_in_ghcr(image: str, tag: str) -> bool: req.add_header("X-GitHub-Api-Version", "2022-11-28") if token: req.add_header("Authorization", f"Bearer {token}") - try: - with urllib.request.urlopen(req) as resp: - versions = json.loads(resp.read()) - except urllib.error.HTTPError as e: - if e.code == 404: - return False # image doesn't exist at all - raise + + versions = None + last_error = None + for attempt in range(MAX_RETRIES + 1): + try: + with urllib.request.urlopen(req) as resp: + versions = json.loads(resp.read()) + last_error = None + break + except urllib.error.HTTPError as e: + if e.code == 404: + return False # image doesn't exist at all + is_rate_limit_403 = False + if e.code == 403 and e.headers is not None: + # Treat 403 as transient only if rate-limit headers confirm it + rem = e.headers.get("x-ratelimit-remaining") + retry_after = e.headers.get("retry-after") + if (rem is not None and rem.strip() == "0") or retry_after is not None: + is_rate_limit_403 = True + + if e.code not in TRANSIENT_HTTP_STATUSES and not is_rate_limit_403: + raise + last_error = e + if attempt < MAX_RETRIES: + time.sleep(RETRY_BACKOFF_SECONDS * (2 ** attempt)) + + if last_error is not None: + print( + f" \u26a0 GHCR packages API returned HTTP {last_error.code} for " + f"{image} after {MAX_RETRIES + 1} attempts -- skipping existence " + "check for this ref (transient error, not a ref regression)." + ) + return None + if not versions: return False for version in versions: @@ -164,9 +201,16 @@ def main(root=None): return 0 missing = [] + skipped = [] for key, locations in sorted(refs.items()): image, tag = key.rsplit(":", 1) exists = tag_exists_in_ghcr(image, tag) + if exists is None: + # Transient GHCR API error after retries — don't fail the build + # over a registry hiccup, but don't silently claim it's fine. + print(f" ⚠ ghcr.io/projectbluefin/{key} (skipped, GHCR unreachable)") + skipped.append(key) + continue status = "✅" if exists else "❌" print(f" {status} ghcr.io/projectbluefin/{key}") if not exists: @@ -174,6 +218,13 @@ def main(root=None): print(f" referenced at: {loc}") missing.append(key) + if skipped: + print( + "\nWARNING: GHCR packages API was unreachable (transient error) for " + f"{len(skipped)} ref(s); their existence could not be verified this run:\n" + + "\n".join(f" ghcr.io/projectbluefin/{s}" for s in skipped) + ) + if missing: print( "\nERROR: The following image:tag refs in docs do not exist in GHCR:\n" @@ -184,7 +235,13 @@ def main(root=None): ) return 1 - print(f"\n✓ All {len(refs)} image:tag refs validated against GHCR.") + if skipped: + print( + f"\n✓ Validated {len(refs) - len(skipped)} of {len(refs)} image:tag refs against GHCR " + f"({len(skipped)} skipped due to transient errors)." + ) + else: + print(f"\n✓ All {len(refs)} image:tag refs validated against GHCR.") return 0 diff --git a/tests/test_check_oci_refs.py b/tests/test_check_oci_refs.py index 0fc8d383..686b5f6f 100644 --- a/tests/test_check_oci_refs.py +++ b/tests/test_check_oci_refs.py @@ -216,16 +216,72 @@ def test_returns_false_on_404(self): ): assert tag_exists_in_ghcr("nonexistent-image", "latest") is False - def test_re_raises_non_404_http_error(self): + def test_re_raises_non_transient_http_error(self): with patch( "urllib.request.urlopen", side_effect=urllib.error.HTTPError( - url="", code=500, msg="Server Error", hdrs=None, fp=None + url="", code=422, msg="Unprocessable Entity", hdrs=None, fp=None ), ): with pytest.raises(urllib.error.HTTPError): tag_exists_in_ghcr("bluefin", "stable") + def test_401_raises_http_error(self): + """401 is an authentication error and must raise immediately rather than retry.""" + with patch( + "urllib.request.urlopen", + side_effect=urllib.error.HTTPError( + url="", code=401, msg="Unauthorized", hdrs=None, fp=None + ), + ) as mock_urlopen: + with pytest.raises(urllib.error.HTTPError): + tag_exists_in_ghcr("bluefin", "stable") + assert mock_urlopen.call_count == 1 + + def test_transient_5xx_retries_then_returns_none(self): + with patch( + "urllib.request.urlopen", + side_effect=urllib.error.HTTPError( + url="", code=500, msg="Server Error", hdrs=None, fp=None + ), + ) as mock_urlopen, patch("time.sleep") as mock_sleep: + assert tag_exists_in_ghcr("bluefin", "stable") is None + # MAX_RETRIES=3 retries + the initial attempt = 4 calls total. + assert mock_urlopen.call_count == 4 + assert mock_sleep.call_count == 3 + + def test_transient_403_rate_limit_recovers_on_retry(self): + mock_resp = MagicMock() + mock_resp.read.return_value = b"[]" + mock_resp.__enter__ = lambda s: s + mock_resp.__exit__ = MagicMock(return_value=False) + hdrs = {"x-ratelimit-remaining": "0"} + with patch( + "urllib.request.urlopen", + side_effect=[ + urllib.error.HTTPError( + url="", code=403, msg="Forbidden", hdrs=hdrs, fp=None + ), + mock_resp, + ], + ) as mock_urlopen, patch("time.sleep") as mock_sleep: + assert tag_exists_in_ghcr("bluefin", "stable") is False + assert mock_urlopen.call_count == 2 + assert mock_sleep.call_count == 1 + + def test_403_without_rate_limit_header_raises_http_error(self): + """403 without rate limit headers indicates missing scope/permission and must raise immediately.""" + hdrs = {"x-ratelimit-remaining": "4676"} + with patch( + "urllib.request.urlopen", + side_effect=urllib.error.HTTPError( + url="", code=403, msg="Forbidden", hdrs=hdrs, fp=None + ), + ) as mock_urlopen: + with pytest.raises(urllib.error.HTTPError): + tag_exists_in_ghcr("bluefin", "stable") + assert mock_urlopen.call_count == 1 + def test_returns_false_on_empty_versions_list(self): mock_resp = MagicMock() mock_resp.read.return_value = b"[]"