Fail CI on a committed test.only, and keep a failed E2E's screenshots

.github/workflows/production-deploy-verify.yml

@@ -750,10 +750,11 @@ jobs:
   e2e-prod:
     name: Playwright E2E (production)
     runs-on: ubuntu-latest
-    # A backstop above the test step's own cap, not the primary bound: a job
-    # timeout cancels the test step, and a cancelled step is not a verdict the
-    # rollback job acts on, whereas the step cap marks it failed.
-    timeout-minutes: 30
+    # A backstop above the steps' own caps (collect 3, tests 20, upload 2), with
+    # ten minutes for the setup steps, not the primary bound: a job timeout
+    # cancels the test step, and a cancelled step is not a verdict the rollback
+    # job acts on, whereas the step cap marks it failed.
+    timeout-minutes: 35
     needs: [production-smoke-tests]
     # The calling job's gate, repeated as on the deploy.
     if: github.ref == 'refs/heads/main' && github.event_name == 'push'
@@ -784,6 +785,16 @@ jobs:
       # fails here rather than in the tests, where it would roll production back.
       - name: Report the runner's Chrome
         run: node e2e/report-chrome.js
+      # A suite that cannot be collected (a committed test.only under
+      # forbidOnly, a spec that throws at module scope) says nothing about
+      # production, so it fails here, before the step the rollback reads. The
+      # same environment as that step, so it loads the modules the same way.
+      - name: Collect the E2E tests (production)
+        timeout-minutes: 3
+        run: npm run test:e2e:prod -w e2e -- --list
+        env:
+          CI: "true"
+          RADAR_API_KEY: ${{ secrets.RADAR_API_KEY }}
       # Bounded at the step rather than relying on the job cap: a step timeout
       # marks the step failed, which still trips the rollback job, whereas a
       # job timeout cancels it and would not. A production wedged badly enough
@@ -798,6 +809,21 @@ jobs:
         env:
           CI: "true"
           RADAR_API_KEY: ${{ secrets.RADAR_API_KEY }}
+      # Screenshots alone: artifacts are public, and GitHub masks secrets only in
+      # logs, while a trace and error-context.md's call log both list request
+      # headers, keys included. Bounded and allowed to fail, because an upload
+      # stalling into the job's cap would cancel the job before it outputs the
+      # verdict the rollback reads.
+      - name: Keep the failed tests' screenshots
+        if: failure()
+        timeout-minutes: 2
+        continue-on-error: true
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: e2e-production-screenshots-${{ github.run_attempt }}
+          path: e2e/test-results/**/*.png
+          retention-days: 7
+          if-no-files-found: ignore
 
   # A job of its own, so that production's root key is never handed to a job
   # that has just run npm's install scripts. test_deploy_workflows.py holds

.github/workflows/staging-deploy-verify.yml

@@ -318,9 +318,9 @@ jobs:
   e2e:
     name: Playwright E2E (staging)
     runs-on: ubuntu-latest
-    # Only the setup steps sit outside the test step's own cap. See the
-    # production job for why the job cap is a backstop rather than the primary
-    # bound.
+    # A backstop above the steps' own caps (tests 20, upload 2), with room for
+    # the setup steps. See the production job for why the job cap is a
+    # backstop rather than the primary bound.
     timeout-minutes: 30
     needs: [smoke-tests]
     # The browser is the runner image's Google Chrome, not one installed here;
@@ -355,3 +355,15 @@ jobs:
           # login page as the app. Empty secrets render empty and send nothing.
           CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }}
           CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }}
+      # Screenshots alone, bounded and allowed to fail; see the step of the same
+      # name in production-deploy-verify.yml.
+      - name: Keep the failed tests' screenshots
+        if: failure()
+        timeout-minutes: 2
+        continue-on-error: true
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: e2e-staging-screenshots-${{ github.run_attempt }}
+          path: e2e/test-results/**/*.png
+          retention-days: 7
+          if-no-files-found: ignore

backend/tests/test_deploy_workflows.py

@@ -526,3 +526,80 @@ def test_a_following_runs_staging_does_not_wait_for_this_runs_production():
     assert "github.event_name == 'pull_request' &&" in pull_request
     assert "github.run_id" in push and "github.run_id" not in pull_request
     assert _job("ci.yml", "staging")["concurrency"]["group"] != _job("ci.yml", "production")["concurrency"]["group"]
+
+
+# ── What a failed E2E leaves behind ──────────────────────────────────────────
+# This repository's artifacts are public and GitHub masks secrets only in logs.
+# A Playwright trace records every request's headers and error-context.md holds
+# a call log that lists them, keys included, so from test-results only the
+# screenshots are ever published.
+
+
+def _runs_playwright(job: dict) -> bool:
+    return any(
+        re.search(r"\btest:e2e\b|\bplaywright test\b", str(step.get("run", ""))) for step in job.get("steps", [])
+    )
+
+
+def _test_result_uploads() -> list:
+    # Every upload from a job that runs Playwright, whatever its path, since a
+    # broad one (`.`, the workspace) sweeps test-results in with it. Elsewhere,
+    # anything under e2e/ or named for Playwright: its report embeds the same
+    # call logs.
+    return [
+        pytest.param(step, id=f"{path.name}:{name}")
+        for path in _workflow_files()
+        for name, job in _workflow(path.name)["jobs"].items()
+        for step in job.get("steps", [])
+        if str(step.get("uses", "")).startswith("actions/upload-artifact")
+        and (
+            _runs_playwright(job)
+            or re.search(r"\be2e\b|test-results|playwright", str(step.get("with", {}).get("path", "")), re.IGNORECASE)
+        )
+    ]
+
+
+def test_both_e2e_jobs_keep_their_screenshots():
+    assert {param.id for param in _test_result_uploads()} == {
+        "staging-deploy-verify.yml:e2e",
+        f"{PRODUCTION}:e2e-prod",
+    }
+
+
+@pytest.mark.parametrize("upload", _test_result_uploads())
+def test_a_failed_e2e_publishes_its_screenshots_alone(upload):
+    assert upload["with"]["path"] == "e2e/test-results/**/*.png"
+    assert not upload["with"].get("include-hidden-files", False)
+
+
+@pytest.mark.parametrize("upload", _test_result_uploads())
+def test_collecting_them_cannot_cost_the_verdict(upload):
+    # e2e-prod's output decides the rollback. An upload that stalls into the
+    # job's cap would cancel the job before it publishes one.
+    assert upload["if"] == "failure()"
+    assert upload["continue-on-error"] is True
+    assert upload["timeout-minutes"] <= 2
+
+
+@pytest.mark.parametrize(("workflow", "job"), [("staging-deploy-verify.yml", "e2e"), (PRODUCTION, "e2e-prod")])
+def test_an_e2e_jobs_cap_stays_a_backstop(workflow, job):
+    # The steps' own caps, with at least five minutes for the setup steps
+    # (checkout, npm ci, the Chrome report), fit under the job's. A job timeout
+    # cancels the test step, and a cancelled step is no verdict, so a production
+    # wedged enough to hang the suite would never be rolled back.
+    steps = _job(workflow, job)["steps"]
+    assert _runs_playwright(_job(workflow, job))
+    capped = sum(int(step.get("timeout-minutes", 0)) for step in steps)
+    assert capped + 5 <= _job(workflow, job)["timeout-minutes"]
+
+
+def test_a_suite_that_cannot_load_cannot_roll_production_back():
+    # A committed test.only under forbidOnly, or a spec that throws at module
+    # scope, fails the collect step, and the step the rollback reads never runs.
+    steps = _job(PRODUCTION, "e2e-prod")["steps"]
+    collect = [step.get("name") for step in steps].index("Collect the E2E tests (production)")
+    run = [step.get("id") for step in steps].index("e2e")
+    assert collect < run
+    assert steps[collect]["run"] == steps[run]["run"] + " -- --list"
+    assert steps[collect]["env"] == steps[run]["env"]
+    assert steps[collect]["timeout-minutes"] <= 3

e2e/playwright.config.ts

@@ -139,6 +139,9 @@ export default defineConfig({
   timeout: 30_000,
   expect: { timeout: 10_000 },
   fullyParallel: true,
+  // A committed test.only would quietly shrink the suite that decides whether
+  // production rolls back, so in CI it fails the run instead.
+  forbidOnly: !!process.env.CI,
   retries: process.env.CI ? 2 : 0,
   reporter: process.env.CI ? "github" : "list",
   use: {
@@ -147,7 +150,9 @@ export default defineConfig({
     // names its host explicitly, so this only resolves a relative URL.
     baseURL: hosts.map,
     extraHTTPHeaders: accessHeaders,
-    trace: "on-first-retry",
+    // Never in CI, whose artifacts are public: a trace records every request's
+    // headers, keys included.
+    trace: process.env.CI ? "off" : "on-first-retry",
     screenshot: "only-on-failure",
     headless: true,
   },