diff --git a/CHANGELOG.md b/CHANGELOG.md index fa8778d695..7755e7f1ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,15 @@ Semantic Versioning where the repository publishes a release. ## [Unreleased] - Keep the required OpenCode bootstrap's Pingora policy step unconditional within its pull-request-only workflow, so the static bootstrap contract does - not depend on event payload fields. (Ported from #1414, not yet merged, so - this PR's own trusted-source cycle no longer needs #1414 to merge first.) + not depend on event payload fields. (Ported from #1414, not yet merged, to + unblock this PR's own `exact-head-path-policy` check.) +- Bump the vendored `contextual-orchestrator` review-sidecar pin from + `b2164511` (103 commits stale) to current `main` `5f2753a`, so the + gateway's model-discovery/ZDR/pool-selection fixes landed since the old pin + reach `opencode-review`/`noema-review`. The stale pin's discovery logic was + failing the sidecar's own preflight with a gateway 502 before any review + could post, which is why `opencode-review` and `noema-review` were failing + closed on most `contextual-orchestrator` PRs and several `.github` PRs. - Skip trusted base Python lock materialization for exact-head reviews with no Python source or dependency-manifest changes, while preserving the fail-closed wheel-only path when Python coverage is relevant. diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index 3e40886e5f..f1f5883505 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -22,7 +22,7 @@ all five, and auto-optimize routing by cost. 1. **Vendoring, pinned**: `scripts/ci/contextual_orchestrator_review_sidecar.sh` clones `ContextualWisdomLab/contextual-orchestrator` at an exact SHA - (`b21645116b352967e50fc497b87eb745b9cc8c61` today) into `RUNNER_TEMP`. The + (`5f2753ace756ddd81049a5221d55e8977572a416` today) into `RUNNER_TEMP`. The source's `requirements.lock` is installed with `--require-hashes` and `--no-deps`, so dependency resolution cannot silently move the reviewed runtime. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 1072e8b397..e40d09682f 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -396,6 +396,41 @@ flowchart LR or typed provider result. A green event handler that skips the LLM call is not acceptance evidence. +## 2026-08-30 sidecar pin staleness recurrence + +- Same class of defect as the 2026-08-29 entry above recurred within one day: + `scripts/ci/contextual_orchestrator_review_sidecar.sh`'s + `ORCHESTRATOR_PIN_SHA` default (`b21645116b352967e50fc497b87eb745b9cc8c61`) + was already 103 commits behind `contextual-orchestrator` `main`. Observed + directly in hosted `noema-review` job logs (`.github` PR #1421, + `ContextualWisdomLab/contextual-orchestrator` PR #857 and others): the + vendored sidecar's own preflight against the stale pin fails closed with + `gateway preflight returned HTTP 502` (and, on a differently-shaped request, + `request_failed status=413 code=request_too_large`) before the model pool + can run, so `opencode-agent`/Noema never post a verdict and the required + `opencode-review`/`noema-review` checks fail on unrelated PRs across both + repos. Confirmed via `contextual-orchestrator` main history that + `5f2753ace756ddd81049a5221d55e8977572a416` is the current `main` HEAD and + passes its own Tests/Security/Fuzz gates. +- This PR bumps the pin to `5f2753ace756ddd81049a5221d55e8977572a416` in the + three places the contract tests pin it: the sidecar script default, + `tests/test_contextual_orchestrator_review_sidecar_contract.py`'s + `ORCH_PIN_SHA`, and `docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md`'s + "today" reference. `requirements.lock` needs no separate sync — the sidecar + installs it fresh from the freshly-checked-out pinned commit, not from a + copy embedded in this repo. +- Acceptance remains open the same way the 2026-08-29 entry describes: this + fixes the reproduced local preflight failure and all static contract tests + pass, but only a fresh post-merge hosted `noema-review`/`opencode-review` + run against the new pin is proof the live gateway path actually completes + and posts a verdict. Given this is the second staleness incident in as many + days, the underlying gap is process, not just this one value: nothing + currently keeps this pin near `contextual-orchestrator` `main` on an + ongoing basis. A scheduled or CI-triggered pin-freshness check (e.g., fail + a nightly job once the pin falls more than N commits or M days behind a + green `contextual-orchestrator` main) would close that gap; not implemented + in this PR, left for a follow-up. + ## 5. 실행 루프와 고객의 다음 행동 각 hourly pass는 아래 순서를 유지한다. diff --git a/scripts/ci/contextual_orchestrator_review_launcher.py b/scripts/ci/contextual_orchestrator_review_launcher.py index abb9af3b21..aa61ea1c80 100644 --- a/scripts/ci/contextual_orchestrator_review_launcher.py +++ b/scripts/ci/contextual_orchestrator_review_launcher.py @@ -65,6 +65,19 @@ def _has_text_output(model: object) -> bool: return not modalities or "text" in {str(modality).casefold() for modality in modalities} +def _routable_discovered_models(discovered: list[object] | None) -> list[object]: + """Drop evidence-only discovery rows before any live-serving selection. + + Evidence-only rows (e.g. the OpenRouter catalog) exist solely to supply + ZDR evidence for other providers' models; contextual_orchestrator's own + ``agent_from_discovered()`` refuses to turn one into a serving agent. + Filtering here keeps that same invariant in this sidecar's selection path, + which builds its catalog independently rather than calling + ``agent_from_discovered()`` directly. + """ + return [model for model in (discovered or []) if not getattr(model, "evidence_only", False)] + + def _route_identity(model: object) -> tuple[str, str]: """Return the provider/model identity used to bind price evidence.""" @@ -385,10 +398,11 @@ def main(argv: list[str] | None = None) -> int: discovered, _ = discover_all_models() except Exception as exc: # pragma: no cover - provider/networking failure is runtime-only raise SystemExit(f"review sidecar discovery failed: {exc}") from exc - free_models = list(free_discovered_models(discovered)) if discovered else [] + routable_discovered = _routable_discovered_models(discovered) + free_models = list(free_discovered_models(routable_discovered)) if routable_discovered else [] free_route_identities = frozenset(_route_identity(model) for model in free_models) selected_models = [] - for model in discovered or []: + for model in routable_discovered: model_id = getattr(model, "model_id", "") if not is_general_chat_agent_model_id(model_id) or not _has_text_output(model): continue diff --git a/scripts/ci/contextual_orchestrator_review_sidecar.sh b/scripts/ci/contextual_orchestrator_review_sidecar.sh index bca9d5c00e..1fb2b5aea3 100755 --- a/scripts/ci/contextual_orchestrator_review_sidecar.sh +++ b/scripts/ci/contextual_orchestrator_review_sidecar.sh @@ -14,7 +14,7 @@ # (fail-closed zero-cost) pool. set -euo pipefail -ORCHESTRATOR_PIN_SHA="${ORCHESTRATOR_PIN_SHA:-b21645116b352967e50fc497b87eb745b9cc8c61}" +ORCHESTRATOR_PIN_SHA="${ORCHESTRATOR_PIN_SHA:-5f2753ace756ddd81049a5221d55e8977572a416}" ORCHESTRATOR_GIT_URL="${ORCHESTRATOR_GIT_URL:-https://github.com/ContextualWisdomLab/contextual-orchestrator.git}" # The Strix gate and Noema SSRF guard accept this one process-local origin. # Keep it fixed so an environment override cannot create an unvalidated sidecar. diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index baa1a8df76..a0fbac1f4a 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -52,6 +52,36 @@ def _openai_text(content: str) -> dict[str, object]: return {"choices": [{"message": {"content": content}}]} +def test_routable_discovered_models_excludes_evidence_only_rows() -> None: + """Evidence-only rows (e.g. OpenRouter) must never enter live selection.""" + namespace = _load_launcher() + routable = namespace.get("_routable_discovered_models") + assert callable(routable), "launcher must expose an evidence-only discovery filter" + + evidence_only_model = SimpleNamespace( + id="openrouter_evidence_only", + provider_name="openrouter", + model_id="some/model", + evidence_only=True, + ) + live_model = SimpleNamespace( + id="nvidia_ready", + provider_name="nvidia_nim", + model_id="ready/free", + evidence_only=False, + ) + no_flag_model = SimpleNamespace( + id="bytez_untagged", provider_name="bytez", model_id="untagged/free" + ) + + assert routable([evidence_only_model, live_model, no_flag_model]) == [ + live_model, + no_flag_model, + ] + assert routable(None) == [] + assert routable([]) == [] + + def test_preflight_mirrors_runtime_request_and_keeps_only_compatible_routes() -> None: """Reject provider errors/malformed replies before the sidecar becomes ready.""" namespace = _load_launcher() diff --git a/tests/test_contextual_orchestrator_review_sidecar_contract.py b/tests/test_contextual_orchestrator_review_sidecar_contract.py index a1c0746f03..3ddf8952d2 100644 --- a/tests/test_contextual_orchestrator_review_sidecar_contract.py +++ b/tests/test_contextual_orchestrator_review_sidecar_contract.py @@ -40,7 +40,7 @@ ) GATEWAY_MODEL = "contextual-orchestrator/orchestrator/free" -ORCH_PIN_SHA = "b21645116b352967e50fc497b87eb745b9cc8c61" +ORCH_PIN_SHA = "5f2753ace756ddd81049a5221d55e8977572a416" def _read(path: Path) -> str: @@ -281,7 +281,9 @@ def test_launcher_uses_orchestrator_discovery_and_governed_pools() -> None: text = _read(LAUNCHER) assert "from contextual_orchestrator.chat_capability import is_general_chat_agent_model_id" in text assert "from contextual_orchestrator.model_discovery import discover_all_models, free_discovered_models" in text - assert "free_discovered_models(discovered)" in text + assert "routable_discovered = _routable_discovered_models(discovered)" in text + assert "free_discovered_models(routable_discovered)" in text + assert 'getattr(model, "evidence_only", False)' in text assert 'getattr(model, "output_modalities", None)' in text assert 'isinstance(modalities, str)' in text assert '"text" in {str(modality).casefold() for modality in modalities}' in text