fix(ci): retry transient GHCR errors in check-oci-refs.py

scripts/check-oci-refs.py

@@ -21,9 +21,16 @@
 import os
 import re
 import sys
+import time
 import urllib.request
 import urllib.error
 
+# HTTP statuses that indicate transient infrastructure failures or rate limits,
+# where retrying makes sense. 401 is excluded as authentication failure is non-transient.
+TRANSIENT_HTTP_STATUSES = {403, 429, 500, 502, 503, 504}
+MAX_RETRIES = 3
+RETRY_BACKOFF_SECONDS = 2
+
 # ── Check 1: no ublue-os refs ────────────────────────────────────────────────
 # The org migration from ublue-os to projectbluefin is complete.
 # ghcr.io/ublue-os/ must not appear in workflow files or docs.
@@ -111,8 +118,10 @@ def collect_tag_refs(root=None):
     return refs
 
 
-def tag_exists_in_ghcr(image: str, tag: str) -> bool:
-    """Return True if image:tag exists in GHCR under projectbluefin."""
+def tag_exists_in_ghcr(image: str, tag: str):
+    """Return True/False if image:tag existence in GHCR is known, or None if
+    the GHCR packages API could not be reached after retries (transient
+    outage/rate limit) -- callers must treat None as "unknown, don't fail"."""
     token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN", "")
     page = 1
     while True:
@@ -126,13 +135,32 @@ def tag_exists_in_ghcr(image: str, tag: str) -> bool:
         req.add_header("X-GitHub-Api-Version", "2022-11-28")
         if token:
             req.add_header("Authorization", f"Bearer {token}")
-        try:
-            with urllib.request.urlopen(req) as resp:
-                versions = json.loads(resp.read())
-        except urllib.error.HTTPError as e:
-            if e.code == 404:
-                return False  # image doesn't exist at all
-            raise
+
+        versions = None
+        last_error = None
+        for attempt in range(MAX_RETRIES + 1):
+            try:
+                with urllib.request.urlopen(req) as resp:
+                    versions = json.loads(resp.read())
+                last_error = None
+                break
+            except urllib.error.HTTPError as e:
+                if e.code == 404:
+                    return False  # image doesn't exist at all
+                if e.code not in TRANSIENT_HTTP_STATUSES:
+                    raise
+                last_error = e
+                if attempt < MAX_RETRIES:
+                    time.sleep(RETRY_BACKOFF_SECONDS * (2 ** attempt))
+
+        if last_error is not None:
+            print(
+                f"  \u26a0 GHCR packages API returned HTTP {last_error.code} for "
+                f"{image} after {MAX_RETRIES + 1} attempts -- skipping existence "
+                "check for this ref (transient error, not a ref regression)."
+            )
+            return None
+
         if not versions:
             return False
         for version in versions:
@@ -164,16 +192,30 @@ def main(root=None):
         return 0
 
     missing = []
+    skipped = []
     for key, locations in sorted(refs.items()):
         image, tag = key.rsplit(":", 1)
         exists = tag_exists_in_ghcr(image, tag)
+        if exists is None:
+            # Transient GHCR API error after retries — don't fail the build
+            # over a registry hiccup, but don't silently claim it's fine.
+            print(f"  ⚠ ghcr.io/projectbluefin/{key} (skipped, GHCR unreachable)")
+            skipped.append(key)
+            continue
         status = "✅" if exists else "❌"
         print(f"  {status} ghcr.io/projectbluefin/{key}")
         if not exists:
             for loc in locations:
                 print(f"       referenced at: {loc}")
             missing.append(key)
 
+    if skipped:
+        print(
+            "\nWARNING: GHCR packages API was unreachable (transient error) for "
+            f"{len(skipped)} ref(s); their existence could not be verified this run:\n"
+            + "\n".join(f"  ghcr.io/projectbluefin/{s}" for s in skipped)
+        )
+
     if missing:
         print(
             "\nERROR: The following image:tag refs in docs do not exist in GHCR:\n"

tests/test_check_oci_refs.py

@@ -216,16 +216,58 @@ def test_returns_false_on_404(self):
         ):
             assert tag_exists_in_ghcr("nonexistent-image", "latest") is False
 
-    def test_re_raises_non_404_http_error(self):
+    def test_re_raises_non_transient_http_error(self):
         with patch(
             "urllib.request.urlopen",
             side_effect=urllib.error.HTTPError(
-                url="", code=500, msg="Server Error", hdrs=None, fp=None
+                url="", code=422, msg="Unprocessable Entity", hdrs=None, fp=None
             ),
         ):
             with pytest.raises(urllib.error.HTTPError):
                 tag_exists_in_ghcr("bluefin", "stable")
 
+    def test_401_raises_http_error(self):
+        """401 is an authentication error and must raise immediately rather than retry."""
+        with patch(
+            "urllib.request.urlopen",
+            side_effect=urllib.error.HTTPError(
+                url="", code=401, msg="Unauthorized", hdrs=None, fp=None
+            ),
+        ) as mock_urlopen:
+            with pytest.raises(urllib.error.HTTPError):
+                tag_exists_in_ghcr("bluefin", "stable")
+            assert mock_urlopen.call_count == 1
+
+    def test_transient_5xx_retries_then_returns_none(self):
+        with patch(
+            "urllib.request.urlopen",
+            side_effect=urllib.error.HTTPError(
+                url="", code=500, msg="Server Error", hdrs=None, fp=None
+            ),
+        ) as mock_urlopen, patch("time.sleep") as mock_sleep:
+            assert tag_exists_in_ghcr("bluefin", "stable") is None
+            # MAX_RETRIES=3 retries + the initial attempt = 4 calls total.
+            assert mock_urlopen.call_count == 4
+            assert mock_sleep.call_count == 3
+
+    def test_transient_403_recovers_on_retry(self):
+        mock_resp = MagicMock()
+        mock_resp.read.return_value = b"[]"
+        mock_resp.__enter__ = lambda s: s
+        mock_resp.__exit__ = MagicMock(return_value=False)
+        with patch(
+            "urllib.request.urlopen",
+            side_effect=[
+                urllib.error.HTTPError(
+                    url="", code=403, msg="Forbidden", hdrs=None, fp=None
+                ),
+                mock_resp,
+            ],
+        ) as mock_urlopen, patch("time.sleep") as mock_sleep:
+            assert tag_exists_in_ghcr("bluefin", "stable") is False
+            assert mock_urlopen.call_count == 2
+            assert mock_sleep.call_count == 1
+
     def test_returns_false_on_empty_versions_list(self):
         mock_resp = MagicMock()
         mock_resp.read.return_value = b"[]"