CI: retry wedged codex agent legs once; canary stops reporting them as "stalled"

.github/workflows/_retry-probe.yml

@@ -0,0 +1,44 @@
+name: Retry Probe (temporary)
+
+# TEMPORARY end-to-end test fixture for agent-retry.yml (#489). Removed in the
+# follow-up PR once the retry chain has been observed working on main.
+
+on:
+  workflow_dispatch:
+    inputs:
+      mode:
+        type: choice
+        default: recover
+        options:
+          - recover
+          - timeout-both
+        description: recover = attempt 1 times out, attempt 2 succeeds; timeout-both = every attempt times out
+
+jobs:
+  leg:
+    runs-on: ubuntu-latest
+    timeout-minutes: 1
+    steps:
+    - name: Time out on attempt 1 (and on every attempt in timeout-both mode)
+      env:
+        MODE: ${{ inputs.mode }}
+        ATTEMPT: ${{ github.run_attempt }}
+      run: |
+        echo "mode=${MODE} attempt=${ATTEMPT}"
+        if [ "$ATTEMPT" = "1" ] || [ "$MODE" = "timeout-both" ]; then
+          echo "sleeping past timeout-minutes on purpose"
+          sleep 300
+        fi
+        echo "attempt ${ATTEMPT} succeeded"
+
+  finalize:
+    needs: leg
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+    - name: Fail the run unless the leg succeeded (mirrors monitor.yml finalize)
+      env:
+        LEG_RESULT: ${{ needs.leg.result }}
+      run: |
+        echo "leg result: ${LEG_RESULT}"
+        [ "$LEG_RESULT" = "success" ] || exit 1

.github/workflows/agent-retry.yml

@@ -0,0 +1,59 @@
+name: Agent Retry
+
+# One automatic re-run of a codex-agent run whose leg was killed by timeout-minutes
+# or a dead runner. Rationale, probe results and guard rails: PR body of #489.
+
+on:
+  workflow_run:
+    workflows:
+      - Monitor
+      - Daily Project Assessment
+      - Product Analysis
+      - Retry Probe (temporary)   # TEMPORARY, removed with _retry-probe.yml
+    types: [completed]
+
+permissions:
+  actions: write
+  checks: read   # job annotations, to tell a `timeout-minutes` kill from a deliberate cancel
+
+jobs:
+  retry:
+    if: >-
+      github.event.workflow_run.run_attempt == 1 &&
+      (github.event.workflow_run.conclusion == 'failure' ||
+       github.event.workflow_run.conclusion == 'cancelled')
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      RUN_ID: ${{ github.event.workflow_run.id }}
+      RUN_NAME: ${{ github.event.workflow_run.name }}
+      RUN_CONCLUSION: ${{ github.event.workflow_run.conclusion }}
+      RUN_URL: ${{ github.event.workflow_run.html_url }}
+    steps:
+    - name: Re-run failed and cancelled jobs once
+      run: |
+        set -euo pipefail
+        echo "${RUN_NAME} run ${RUN_ID} ended ${RUN_CONCLUSION} on attempt 1 — ${RUN_URL}"
+        JOBS=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/jobs")
+        echo "jobs that did not succeed:"
+        echo "$JOBS" | jq -r '.jobs[] | select(.conclusion != "success" and .conclusion != "skipped") | "  \(.name): \(.conclusion)"'
+        # Only a timeout-minutes kill is retried; a manual or concurrency-queue
+        # cancel stays cancelled (re-running would fight the person/queue).
+        for job_id in $(echo "$JOBS" | jq -r '.jobs[] | select(.conclusion == "cancelled") | .id'); do
+          if ! gh api "repos/${GITHUB_REPOSITORY}/check-runs/${job_id}/annotations" \
+                 --jq '.[].message' | grep -q 'exceeded the maximum execution time'; then
+            echo "job ${job_id} was cancelled but not by timeout — deliberate cancel, not retrying"
+            exit 0
+          fi
+        done
+        if gh api -X POST "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/rerun-failed-jobs" \
+             > "$RUNNER_TEMP/rerun.json" 2>&1; then
+          echo "re-run of failed jobs requested for ${RUN_ID} (attempt 2)"
+          exit 0
+        fi
+        echo "rerun-failed-jobs was refused:"
+        cat "$RUNNER_TEMP/rerun.json"
+        echo "falling back to a full re-run of ${RUN_ID}"
+        gh api -X POST "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/rerun"
+        echo "full re-run requested for ${RUN_ID} (attempt 2)"

.github/workflows/monitor.yml

@@ -624,10 +624,10 @@ jobs:
           echo "::error::Evidence pipeline degraded for: ${SKIPPED_BOTS} — see the evidence-pipeline-degraded issue. Any other bot's agent still ran normally."
           FAILED="true"
         fi
-        # AGENT_RESULT is "skipped" (not "failure") when clean_bots was `[]` —
-        # that must NOT fail this job, only an actual matrix-leg failure should.
-        if [ "$AGENT_RESULT" = "failure" ]; then
-          echo "::error::One or more bot agent invocations failed this run (see the 'agent' job's matrix legs)."
+        # "skipped" (clean_bots was `[]`) must NOT fail this job. "cancelled" is
+        # what a leg killed by timeout-minutes reports: bot never analysed → fail.
+        if [ "$AGENT_RESULT" = "failure" ] || [ "$AGENT_RESULT" = "cancelled" ]; then
+          echo "::error::One or more bot agent invocations failed or timed out this run (agent result: ${AGENT_RESULT} — see the 'agent' job's matrix legs)."
           FAILED="true"
         fi
         if [ "$FAILED" = "true" ]; then

.github/workflows/workflow-canary.yml

@@ -22,6 +22,7 @@ on:
 
 permissions:
   actions: read
+  checks: read   # job annotations, to tell a `timeout-minutes` kill from a plain cancel
 
 jobs:
   canary:
@@ -65,15 +66,35 @@ jobs:
         declare -a ROWS=()
         AGENT_BAD=0
         echo "=== workflow canary classification $(date -u +%Y-%m-%dT%H:%M:%SZ) ==="
-        printf '%-28s | %-24s | %-8s | %-11s | %-14s | %-20s | %s\n' \
-          'workflow' 'display' 'state' 'run_id' 'conclusion' 'created_at' 'threshold'
+        printf '%-28s | %-24s | %-8s | %-11s | %-14s | %-20s | %-7s | %s\n' \
+          'workflow' 'display' 'state' 'run_id' 'conclusion' 'created_at' 'attempt' 'threshold'
         while IFS='|' read -r file display thr_sec thr_human; do
           RUNS=$(gh run list --workflow "$file" --repo "$GITHUB_REPOSITORY" --limit 100 \
-            --json databaseId,event,status,conclusion,createdAt,url)
-          LATEST=$(echo "$RUNS" | jq -c '[.[] | select(.status == "completed" and .conclusion != "cancelled")][0] // empty')
-          SCHED_LATEST=$(echo "$RUNS" | jq -c '[.[] | select(.event == "schedule" and .conclusion != "cancelled")][0] // empty')
+            --json databaseId,event,status,conclusion,createdAt,url,attempt)
+          # A run "cancelled" by a timeout-minutes kill counts as failed (see #489);
+          # any other cancelled run is skipped for the newest non-cancelled one.
+          LATEST=$(echo "$RUNS" | jq -c '[.[] | select(.status == "completed")][0] // empty')
+          TIMED_OUT="false"
+          if [ "$(echo "$LATEST" | jq -r '.conclusion // empty')" = "cancelled" ]; then
+            CANCELLED_ID=$(echo "$LATEST" | jq -r '.databaseId')
+            for job_id in $(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${CANCELLED_ID}/jobs" \
+                              --jq '.jobs[] | select(.conclusion == "cancelled") | .id'); do
+              if gh api "repos/${GITHUB_REPOSITORY}/check-runs/${job_id}/annotations" \
+                   --jq '.[].message' | grep -q 'exceeded the maximum execution time'; then
+                TIMED_OUT="true"
+                break
+              fi
+            done
+            if [ "$TIMED_OUT" = "false" ]; then
+              LATEST=$(echo "$RUNS" | jq -c '[.[] | select(.status == "completed" and .conclusion != "cancelled")][0] // empty')
+            fi
+          fi
+          # Any scheduled run, whatever its conclusion, proves the cron fired.
+          SCHED_LATEST=$(echo "$RUNS" | jq -c '[.[] | select(.event == "schedule")][0] // empty')
           FAILED="false"
-          if [ -n "$LATEST" ]; then
+          if [ "$TIMED_OUT" = "true" ]; then
+            FAILED="true"
+          elif [ -n "$LATEST" ]; then
             case "$(echo "$LATEST" | jq -r '.conclusion')" in
               failure|timed_out|startup_failure) FAILED="true" ;;
             esac
@@ -97,6 +118,13 @@ jobs:
           fi
           RUN_ID=$(echo "$LATEST" | jq -r '.databaseId // empty' 2>/dev/null || true)
           CONCLUSION=$(echo "$LATEST" | jq -r '.conclusion // empty' 2>/dev/null || true)
+          if [ "$TIMED_OUT" = "true" ]; then
+            CONCLUSION="timed_out"
+          fi
+          ATTEMPT=$(echo "$LATEST" | jq -r '.attempt // empty' 2>/dev/null || true)
+          if [ "${ATTEMPT:-1}" -gt 1 ]; then
+            echo "::notice::${display}: latest run ${RUN_ID} needed attempt ${ATTEMPT} (agent-retry.yml) — a wedge happened, retry masked it"
+          fi
           CREATED_AT=$(echo "$LATEST" | jq -r '.createdAt // empty' 2>/dev/null || true)
           URL=$(echo "$LATEST" | jq -r '.url // empty' 2>/dev/null || true)
           SCHED_CREATED=$(echo "$SCHED_LATEST" | jq -r '.createdAt // empty' 2>/dev/null || true)
@@ -117,8 +145,8 @@ jobs:
             esac
           fi
           ROWS+=("$file|$display|$STATE|$RUN_ID|$CONCLUSION|$CREATED_AT|$URL|$SCHED_CREATED|$thr_human|$SUMMARY")
-          printf '%-28s | %-24s | %-8s | %-11s | %-14s | %-20s | %s\n' \
-            "$file" "$display" "$STATE" "$RUN_ID" "$CONCLUSION" "$CREATED_AT" "$thr_human"
+          printf '%-28s | %-24s | %-8s | %-11s | %-14s | %-20s | %-7s | %s\n' \
+            "$file" "$display" "$STATE" "$RUN_ID" "$CONCLUSION" "$CREATED_AT" "${ATTEMPT:-}" "$thr_human"
         done <<'EOF'
         monitor.yml|Monitor|19800|5h30m
         project.yml|Daily Project Assessment|93600|26h