From a300d82f87dee91d09784fbef60ec9f4fe93a71f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 06:40:31 +0900 Subject: [PATCH] fix(noema): upload the sidecar stderr and preflight report when the verdict phase fails A failed noema-review run left artifacts=0, so a 3122 s walk across six ready routes ending in HTTP 502 (run 33981136873) was diagnosable only from the caller's one-line summary. Ship the sanitized sidecar stderr and the preflight report on failure, using the same pinned upload-artifact and the same file Strix already publishes in strix-reports. Refs #1935, #1939 Co-Authored-By: Claude Fable 5.1 --- .github/workflows/noema-review.yml | 11 +++++++ CHANGELOG.md | 4 +++ ...st_noema_orchestrator_workflow_contract.py | 31 +++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index 21ea967201..f8ab55c896 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -688,6 +688,17 @@ jobs: echo "::notice::Noema model phase produced no publishable envelope; publication is skipped." fi + - name: Upload contextual-orchestrator sidecar evidence on failure + if: failure() && env.PR_NUMBER != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: noema-sidecar-evidence + path: | + strix_runs/contextual-orchestrator-sidecar.stderr.log + strix_runs/contextual-orchestrator-preflight.json + if-no-files-found: ignore + retention-days: 5 + - name: Refresh repository-scoped Noema GitHub App token for publication if: env.PR_NUMBER != '' && steps.noema_prepare.outputs.prepared == 'true' && steps.noema_credential.outputs.source == 'github-app' id: noema_github_app_publication_token diff --git a/CHANGELOG.md b/CHANGELOG.md index 46d599a320..2e646ecf5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Noema review ships sidecar evidence on failure + +- `noema-review.yml` now uploads `strix_runs/contextual-orchestrator-sidecar.stderr.log` and `strix_runs/contextual-orchestrator-preflight.json` as the `noema-sidecar-evidence` artifact when the verdict phase fails (`if: failure()`, the same pinned `actions/upload-artifact` Strix uses, `if-no-files-found: ignore`, 5-day retention). Until now a failed Noema run left `artifacts=0` -- run `33981136873` spent 3122 s walking six ready routes twice each and ended in HTTP 502 with no per-route trace anywhere but the sidecar's stderr -- so the only diagnosis available was the caller's one-line summary. The stderr file is the sanitizer's bounded allowlist output (`sanitize_contextual_orchestrator_sidecar_stream.py`), the same file Strix already publishes in `strix-reports`; per-attempt route outcomes still need an allowlisted structured line from the orchestrator to appear in it. Refs #1935, #1939. + ### Review sidecar catalog interleaves credential accounts - `build_zdr_prioritized_catalog` now fills each free/ZDR tier round-robin across independently credentialed accounts instead of in provider-name order. The sidecar exports `ORCHESTRATOR_CATALOG_ACCOUNT_CAP=8` with `ORCHESTRATOR_CATALOG_LIMIT=12`, and the sorted fill took 8 `nvidia_nim` routes and 4 `nvidia_nim_sub` routes before any `openrouter` route was reached, so a review that admitted 62 free routes across three accounts served a NVIDIA-only catalog (`noema-review` run 33969842312: `free_pool_admitted_routes` 62, `free_selected_count` 12, runtime preflight `ready_count` 2 of 12) and the failover loop had no other account to leave a stalled NVIDIA endpoint for -- the `noema-review` 502 class tracked in contextual-orchestrator#1045. Tier order (free before priced, ZDR before non-ZDR), the account cap, the limit, and the discovery-order independence contract are unchanged; the same input now yields 4 + 4 + 4. Contrasts with #1476, which hardens `_routable_discovered_models` against a pin that regresses the OpenRouter `evidence_only` flag: on the current pin (`2e414d15`, includes contextual-orchestrator#949) OpenRouter rows already reach the catalog builder, and the selection was what dropped them. diff --git a/tests/test_noema_orchestrator_workflow_contract.py b/tests/test_noema_orchestrator_workflow_contract.py index 937cf6fe97..628fa3cbc1 100644 --- a/tests/test_noema_orchestrator_workflow_contract.py +++ b/tests/test_noema_orchestrator_workflow_contract.py @@ -485,3 +485,34 @@ def test_noema_review_job_has_no_job_level_timeout() -> None: encoding="utf-8" ) ), "the two-hour-per-model allowance this bound relies on must still be documented" + + +def test_noema_review_uploads_sidecar_evidence_on_failure() -> None: + """A failed verdict phase ships the sanitized sidecar stderr and preflight report. + + Before this step a failed Noema run left ``artifacts=0`` (run 33981136873: + 3122 s, then HTTP 502, no per-route trace in the job log). The stderr file + is the sidecar sanitizer's bounded allowlist output -- the same file Strix + already publishes in ``strix-reports`` -- so shipping it on failure adds + diagnosis without adding exposure (#1935 follow-up). + """ + workflow = workflow_text("noema-review.yml") + name = "Upload contextual-orchestrator sidecar evidence on failure" + step = workflow_step(workflow, name) + assert "if: failure() && env.PR_NUMBER != ''" in step + strix_pin = re.search( + r"actions/upload-artifact@([0-9a-f]{40})", workflow_text("strix.yml") + ).group(1) + assert f"actions/upload-artifact@{strix_pin}" in step + assert "name: noema-sidecar-evidence" in step + assert "strix_runs/contextual-orchestrator-sidecar.stderr.log" in step + assert "strix_runs/contextual-orchestrator-preflight.json" in step + assert "if-no-files-found: ignore" in step + assert "retention-days: 5" in step + prepare = workflow.index(" - name: Prepare Noema model verdict\n") + upload = workflow.index(f" - name: {name}\n") + refresh = workflow.index( + " - name: Refresh repository-scoped Noema GitHub App token for publication\n" + ) + assert prepare < upload < refresh + assert workflow.count("actions/upload-artifact@") == 1