diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index 21ea967201..f8ab55c896 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -688,6 +688,17 @@ jobs: echo "::notice::Noema model phase produced no publishable envelope; publication is skipped." fi + - name: Upload contextual-orchestrator sidecar evidence on failure + if: failure() && env.PR_NUMBER != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: noema-sidecar-evidence + path: | + strix_runs/contextual-orchestrator-sidecar.stderr.log + strix_runs/contextual-orchestrator-preflight.json + if-no-files-found: ignore + retention-days: 5 + - name: Refresh repository-scoped Noema GitHub App token for publication if: env.PR_NUMBER != '' && steps.noema_prepare.outputs.prepared == 'true' && steps.noema_credential.outputs.source == 'github-app' id: noema_github_app_publication_token diff --git a/CHANGELOG.md b/CHANGELOG.md index 15d9e6e188..55a2a2f211 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +### Noema review ships sidecar evidence on failure + +- `noema-review.yml` now uploads `strix_runs/contextual-orchestrator-sidecar.stderr.log` and `strix_runs/contextual-orchestrator-preflight.json` as the `noema-sidecar-evidence` artifact when the verdict phase fails (`if: failure()`, the same pinned `actions/upload-artifact` Strix uses, `if-no-files-found: ignore`, 5-day retention). Until now a failed Noema run left `artifacts=0` -- run `33981136873` spent 3122 s walking six ready routes twice each and ended in HTTP 502 with no per-route trace anywhere but the sidecar's stderr -- so the only diagnosis available was the caller's one-line summary. The stderr file is the sanitizer's bounded allowlist output (`sanitize_contextual_orchestrator_sidecar_stream.py`), the same file Strix already publishes in `strix-reports`; per-attempt route outcomes still need an allowlisted structured line from the orchestrator to appear in it. Refs #1935, #1939. ### Sidecar sanitizer admits orchestrator route and circuit events - `scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py` now passes the orchestrator's own `provider_attempt`, `provider_attempt_failed` (cut before the free-text `error_message=`), `provider_backoff`, `provider_exhausted`, `provider_rejected_permanent`, `provider_no_retry_budget` and `circuit_failure|opened|reset|cleared` lines (whose `failures`/`reset_seconds` are floats at runtime, `2.0`/`30.0`), matched field by field against bounded identifier and number charsets, with either Python's default `LEVEL:name:` prefix or the sidecar formatter's `asctime LEVEL name` prefix (the timestamp is kept so per-route durations can be read as differences). Until now every one of these lines was folded into `omitted_unstructured_lines`, so the `provider_exhausted` WARNING that already fires today after a route's retry budget is spent never reached an artifact, and a 3122 s walk across six ready routes (run `33981136873`) had no per-route trace. Companion to #1943 (sidecar DEBUG logging) and #1944 (Noema uploads the file on failure). Refs #1935, #1939. diff --git a/tests/test_noema_orchestrator_workflow_contract.py b/tests/test_noema_orchestrator_workflow_contract.py index 937cf6fe97..628fa3cbc1 100644 --- a/tests/test_noema_orchestrator_workflow_contract.py +++ b/tests/test_noema_orchestrator_workflow_contract.py @@ -485,3 +485,34 @@ def test_noema_review_job_has_no_job_level_timeout() -> None: encoding="utf-8" ) ), "the two-hour-per-model allowance this bound relies on must still be documented" + + +def test_noema_review_uploads_sidecar_evidence_on_failure() -> None: + """A failed verdict phase ships the sanitized sidecar stderr and preflight report. + + Before this step a failed Noema run left ``artifacts=0`` (run 33981136873: + 3122 s, then HTTP 502, no per-route trace in the job log). The stderr file + is the sidecar sanitizer's bounded allowlist output -- the same file Strix + already publishes in ``strix-reports`` -- so shipping it on failure adds + diagnosis without adding exposure (#1935 follow-up). + """ + workflow = workflow_text("noema-review.yml") + name = "Upload contextual-orchestrator sidecar evidence on failure" + step = workflow_step(workflow, name) + assert "if: failure() && env.PR_NUMBER != ''" in step + strix_pin = re.search( + r"actions/upload-artifact@([0-9a-f]{40})", workflow_text("strix.yml") + ).group(1) + assert f"actions/upload-artifact@{strix_pin}" in step + assert "name: noema-sidecar-evidence" in step + assert "strix_runs/contextual-orchestrator-sidecar.stderr.log" in step + assert "strix_runs/contextual-orchestrator-preflight.json" in step + assert "if-no-files-found: ignore" in step + assert "retention-days: 5" in step + prepare = workflow.index(" - name: Prepare Noema model verdict\n") + upload = workflow.index(f" - name: {name}\n") + refresh = workflow.index( + " - name: Refresh repository-scoped Noema GitHub App token for publication\n" + ) + assert prepare < upload < refresh + assert workflow.count("actions/upload-artifact@") == 1