From 83498195239158aa69984e46a42a502260fb07d3 Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Mon, 14 Sep 2026 11:02:06 +0100 Subject: [PATCH 1/2] CI: retry wedged codex agent legs once; canary stops reporting them as "stalled" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five Monitor slots in five days (2026-09-07..11) lost one codex agent leg to a wedge. Since PR #471 the leg dies at timeout-minutes, GitHub records that job — and the whole run — as "cancelled", and workflow-canary.yml dropped cancelled runs from its "last scheduled run" lookup. The gap it measured then doubled to ~8h and PagerDuty got two "Monitor: stalled — no run in 5h30m" pushes per wedge, although Monitor was scheduling fine. agent-retry.yml (new): on workflow_run completion of Monitor / Daily Project Assessment / Product Analysis with conclusion failure or cancelled and run_attempt == 1, call rerun-failed-jobs. Only the dead leg and its dependants re-run on a fresh runner; gather's evidence artifact is reused (no second VPN session, no duplicate baseline sample). Cancelled jobs without the timeout annotation (manual / concurrency-queue cancel) are not retried. Verified on a throwaway run: rerun-failed-jobs does include a job killed by timeout-minutes (attempt 2 re-ran it). workflow-canary.yml: any scheduled run now counts for the stall check; a cancelled latest run whose job carries the "exceeded the maximum execution time" annotation is classified failed (conclusion reported as timed_out), other cancels still fall back to the newest non-cancelled run. Table gains an attempt column and a ::notice when the latest run needed a retry, so the wedge rate stays visible. monitor.yml finalize: needs.agent.result == "cancelled" (what a timed-out leg reports) fails the run like "failure". Verified the run conclusion stays "cancelled" regardless, which is why the canary change above is needed rather than optional. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01V4AMHh8hMfZh53Tnj3TSSn --- .github/workflows/agent-retry.yml | 58 +++++++++++++++++++++++++++ .github/workflows/monitor.yml | 8 ++-- .github/workflows/workflow-canary.yml | 44 ++++++++++++++++---- 3 files changed, 98 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/agent-retry.yml diff --git a/.github/workflows/agent-retry.yml b/.github/workflows/agent-retry.yml new file mode 100644 index 00000000..935341e2 --- /dev/null +++ b/.github/workflows/agent-retry.yml @@ -0,0 +1,58 @@ +name: Agent Retry + +# One automatic re-run of a codex-agent run whose leg was killed by timeout-minutes +# or a dead runner. Rationale, probe results and guard rails: PR body of #489. + +on: + workflow_run: + workflows: + - Monitor + - Daily Project Assessment + - Product Analysis + types: [completed] + +permissions: + actions: write + checks: read # job annotations, to tell a `timeout-minutes` kill from a deliberate cancel + +jobs: + retry: + if: >- + github.event.workflow_run.run_attempt == 1 && + (github.event.workflow_run.conclusion == 'failure' || + github.event.workflow_run.conclusion == 'cancelled') + runs-on: ubuntu-latest + timeout-minutes: 5 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_NAME: ${{ github.event.workflow_run.name }} + RUN_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + RUN_URL: ${{ github.event.workflow_run.html_url }} + steps: + - name: Re-run failed and cancelled jobs once + run: | + set -euo pipefail + echo "${RUN_NAME} run ${RUN_ID} ended ${RUN_CONCLUSION} on attempt 1 — ${RUN_URL}" + JOBS=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/jobs") + echo "jobs that did not succeed:" + echo "$JOBS" | jq -r '.jobs[] | select(.conclusion != "success" and .conclusion != "skipped") | " \(.name): \(.conclusion)"' + # Only a timeout-minutes kill is retried; a manual or concurrency-queue + # cancel stays cancelled (re-running would fight the person/queue). + for job_id in $(echo "$JOBS" | jq -r '.jobs[] | select(.conclusion == "cancelled") | .id'); do + if ! gh api "repos/${GITHUB_REPOSITORY}/check-runs/${job_id}/annotations" \ + --jq '.[].message' | grep -q 'exceeded the maximum execution time'; then + echo "job ${job_id} was cancelled but not by timeout — deliberate cancel, not retrying" + exit 0 + fi + done + if gh api -X POST "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/rerun-failed-jobs" \ + > "$RUNNER_TEMP/rerun.json" 2>&1; then + echo "re-run of failed jobs requested for ${RUN_ID} (attempt 2)" + exit 0 + fi + echo "rerun-failed-jobs was refused:" + cat "$RUNNER_TEMP/rerun.json" + echo "falling back to a full re-run of ${RUN_ID}" + gh api -X POST "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/rerun" + echo "full re-run requested for ${RUN_ID} (attempt 2)" diff --git a/.github/workflows/monitor.yml b/.github/workflows/monitor.yml index 707a296e..8fef136e 100644 --- a/.github/workflows/monitor.yml +++ b/.github/workflows/monitor.yml @@ -624,10 +624,10 @@ jobs: echo "::error::Evidence pipeline degraded for: ${SKIPPED_BOTS} — see the evidence-pipeline-degraded issue. Any other bot's agent still ran normally." FAILED="true" fi - # AGENT_RESULT is "skipped" (not "failure") when clean_bots was `[]` — - # that must NOT fail this job, only an actual matrix-leg failure should. - if [ "$AGENT_RESULT" = "failure" ]; then - echo "::error::One or more bot agent invocations failed this run (see the 'agent' job's matrix legs)." + # "skipped" (clean_bots was `[]`) must NOT fail this job. "cancelled" is + # what a leg killed by timeout-minutes reports: bot never analysed → fail. + if [ "$AGENT_RESULT" = "failure" ] || [ "$AGENT_RESULT" = "cancelled" ]; then + echo "::error::One or more bot agent invocations failed or timed out this run (agent result: ${AGENT_RESULT} — see the 'agent' job's matrix legs)." FAILED="true" fi if [ "$FAILED" = "true" ]; then diff --git a/.github/workflows/workflow-canary.yml b/.github/workflows/workflow-canary.yml index b637b6b5..75605901 100644 --- a/.github/workflows/workflow-canary.yml +++ b/.github/workflows/workflow-canary.yml @@ -22,6 +22,7 @@ on: permissions: actions: read + checks: read # job annotations, to tell a `timeout-minutes` kill from a plain cancel jobs: canary: @@ -65,15 +66,35 @@ jobs: declare -a ROWS=() AGENT_BAD=0 echo "=== workflow canary classification $(date -u +%Y-%m-%dT%H:%M:%SZ) ===" - printf '%-28s | %-24s | %-8s | %-11s | %-14s | %-20s | %s\n' \ - 'workflow' 'display' 'state' 'run_id' 'conclusion' 'created_at' 'threshold' + printf '%-28s | %-24s | %-8s | %-11s | %-14s | %-20s | %-7s | %s\n' \ + 'workflow' 'display' 'state' 'run_id' 'conclusion' 'created_at' 'attempt' 'threshold' while IFS='|' read -r file display thr_sec thr_human; do RUNS=$(gh run list --workflow "$file" --repo "$GITHUB_REPOSITORY" --limit 100 \ - --json databaseId,event,status,conclusion,createdAt,url) - LATEST=$(echo "$RUNS" | jq -c '[.[] | select(.status == "completed" and .conclusion != "cancelled")][0] // empty') - SCHED_LATEST=$(echo "$RUNS" | jq -c '[.[] | select(.event == "schedule" and .conclusion != "cancelled")][0] // empty') + --json databaseId,event,status,conclusion,createdAt,url,attempt) + # A run "cancelled" by a timeout-minutes kill counts as failed (see #489); + # any other cancelled run is skipped for the newest non-cancelled one. + LATEST=$(echo "$RUNS" | jq -c '[.[] | select(.status == "completed")][0] // empty') + TIMED_OUT="false" + if [ "$(echo "$LATEST" | jq -r '.conclusion // empty')" = "cancelled" ]; then + CANCELLED_ID=$(echo "$LATEST" | jq -r '.databaseId') + for job_id in $(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${CANCELLED_ID}/jobs" \ + --jq '.jobs[] | select(.conclusion == "cancelled") | .id'); do + if gh api "repos/${GITHUB_REPOSITORY}/check-runs/${job_id}/annotations" \ + --jq '.[].message' | grep -q 'exceeded the maximum execution time'; then + TIMED_OUT="true" + break + fi + done + if [ "$TIMED_OUT" = "false" ]; then + LATEST=$(echo "$RUNS" | jq -c '[.[] | select(.status == "completed" and .conclusion != "cancelled")][0] // empty') + fi + fi + # Any scheduled run, whatever its conclusion, proves the cron fired. + SCHED_LATEST=$(echo "$RUNS" | jq -c '[.[] | select(.event == "schedule")][0] // empty') FAILED="false" - if [ -n "$LATEST" ]; then + if [ "$TIMED_OUT" = "true" ]; then + FAILED="true" + elif [ -n "$LATEST" ]; then case "$(echo "$LATEST" | jq -r '.conclusion')" in failure|timed_out|startup_failure) FAILED="true" ;; esac @@ -97,6 +118,13 @@ jobs: fi RUN_ID=$(echo "$LATEST" | jq -r '.databaseId // empty' 2>/dev/null || true) CONCLUSION=$(echo "$LATEST" | jq -r '.conclusion // empty' 2>/dev/null || true) + if [ "$TIMED_OUT" = "true" ]; then + CONCLUSION="timed_out" + fi + ATTEMPT=$(echo "$LATEST" | jq -r '.attempt // empty' 2>/dev/null || true) + if [ "${ATTEMPT:-1}" -gt 1 ]; then + echo "::notice::${display}: latest run ${RUN_ID} needed attempt ${ATTEMPT} (agent-retry.yml) — a wedge happened, retry masked it" + fi CREATED_AT=$(echo "$LATEST" | jq -r '.createdAt // empty' 2>/dev/null || true) URL=$(echo "$LATEST" | jq -r '.url // empty' 2>/dev/null || true) SCHED_CREATED=$(echo "$SCHED_LATEST" | jq -r '.createdAt // empty' 2>/dev/null || true) @@ -117,8 +145,8 @@ jobs: esac fi ROWS+=("$file|$display|$STATE|$RUN_ID|$CONCLUSION|$CREATED_AT|$URL|$SCHED_CREATED|$thr_human|$SUMMARY") - printf '%-28s | %-24s | %-8s | %-11s | %-14s | %-20s | %s\n' \ - "$file" "$display" "$STATE" "$RUN_ID" "$CONCLUSION" "$CREATED_AT" "$thr_human" + printf '%-28s | %-24s | %-8s | %-11s | %-14s | %-20s | %-7s | %s\n' \ + "$file" "$display" "$STATE" "$RUN_ID" "$CONCLUSION" "$CREATED_AT" "${ATTEMPT:-}" "$thr_human" done <<'EOF' monitor.yml|Monitor|19800|5h30m project.yml|Daily Project Assessment|93600|26h From 2f853b89cd309f4772718b61c1924883ff754142 Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Mon, 14 Sep 2026 11:11:59 +0100 Subject: [PATCH 2/2] test: temporary Retry Probe workflow to prove agent-retry.yml end to end on main workflow_run only listens to workflows on the default branch, so the retry chain cannot be exercised from a PR branch. This fixture (attempt 1 sleeps past a 1-min timeout, attempt 2 succeeds or also times out depending on input) is dispatched after merge and removed in the follow-up PR. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01V4AMHh8hMfZh53Tnj3TSSn --- .github/workflows/_retry-probe.yml | 44 ++++++++++++++++++++++++++++++ .github/workflows/agent-retry.yml | 1 + 2 files changed, 45 insertions(+) create mode 100644 .github/workflows/_retry-probe.yml diff --git a/.github/workflows/_retry-probe.yml b/.github/workflows/_retry-probe.yml new file mode 100644 index 00000000..136970c2 --- /dev/null +++ b/.github/workflows/_retry-probe.yml @@ -0,0 +1,44 @@ +name: Retry Probe (temporary) + +# TEMPORARY end-to-end test fixture for agent-retry.yml (#489). Removed in the +# follow-up PR once the retry chain has been observed working on main. + +on: + workflow_dispatch: + inputs: + mode: + type: choice + default: recover + options: + - recover + - timeout-both + description: recover = attempt 1 times out, attempt 2 succeeds; timeout-both = every attempt times out + +jobs: + leg: + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: Time out on attempt 1 (and on every attempt in timeout-both mode) + env: + MODE: ${{ inputs.mode }} + ATTEMPT: ${{ github.run_attempt }} + run: | + echo "mode=${MODE} attempt=${ATTEMPT}" + if [ "$ATTEMPT" = "1" ] || [ "$MODE" = "timeout-both" ]; then + echo "sleeping past timeout-minutes on purpose" + sleep 300 + fi + echo "attempt ${ATTEMPT} succeeded" + + finalize: + needs: leg + if: always() + runs-on: ubuntu-latest + steps: + - name: Fail the run unless the leg succeeded (mirrors monitor.yml finalize) + env: + LEG_RESULT: ${{ needs.leg.result }} + run: | + echo "leg result: ${LEG_RESULT}" + [ "$LEG_RESULT" = "success" ] || exit 1 diff --git a/.github/workflows/agent-retry.yml b/.github/workflows/agent-retry.yml index 935341e2..5f537910 100644 --- a/.github/workflows/agent-retry.yml +++ b/.github/workflows/agent-retry.yml @@ -9,6 +9,7 @@ on: - Monitor - Daily Project Assessment - Product Analysis + - Retry Probe (temporary) # TEMPORARY, removed with _retry-probe.yml types: [completed] permissions: