From 826f403dd1ebe2a532687caf01a3813397f5e518 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 13:08:22 +0900 Subject: [PATCH 01/16] fix(gateway): preserve free-pool passthrough attempt evidence --- contextual_orchestrator/orchestrator.py | 123 +++++++++++++++--- contextual_orchestrator/provider_errors.py | 12 +- docs/product-technical-gap-baseline.md | 71 ++++++++++ tests/test_openai_passthrough.py | 92 +++++++++++++ tests/test_passthrough_provider_failover.py | 135 ++++++++++++++++++++ 5 files changed, 416 insertions(+), 17 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index a09c4e8f0..16966005c 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -1688,6 +1688,52 @@ def _is_capability_mismatch_failover_error(exc: BaseException) -> bool: return False +def _passthrough_failure_phase(exc: ProviderUpstreamError) -> str: + """Map one classified passthrough failure onto a bounded lifecycle phase.""" + if exc.error_code in { + "provider_connection_error", + "tls_failure", + "tls_verification_failed", + }: + return "connecting" + if exc.error_code == "request_too_large": + return "request_validation" + return "provider_response" + + +def _passthrough_attempt_record( + exc: ProviderUpstreamError, + *, + failover_decision: str, +) -> dict[str, Any]: + """Return the public attempt receipt for one classified passthrough failure.""" + return { + "agent_id": exc.agent_id, + "model": exc.model, + "error_code": exc.error_code, + "client_status": exc.client_status, + "provider_status": exc.provider_status, + "retryable": exc.retryable, + "transport": exc.transport, + "phase": _passthrough_failure_phase(exc), + "failover_decision": failover_decision, + } + + +def _set_passthrough_attempt_evidence( + exc: ProviderUpstreamError, + *, + selected_candidate_ids: list[str], + attempts: list[dict[str, Any]], + terminal_reason: str, +) -> ProviderUpstreamError: + """Attach bounded request-scoped passthrough evidence to one final error.""" + exc.selected_candidate_ids = tuple(selected_candidate_ids) + exc.attempts = tuple(dict(item) for item in attempts) + exc.terminal_reason = terminal_reason + return exc + + class ModelClient: """Small chat-completions client with retry, backoff, and mock support.""" @@ -4347,7 +4393,9 @@ def proxy_completion( candidates.append(candidate) last_failure: tuple[Exception, ModelAgent] | None = None every_failure_was_request_too_large = True - for candidate in candidates: + selected_candidate_ids = [candidate.id for candidate in candidates] + attempt_receipts: list[dict[str, Any]] = [] + for index, candidate in enumerate(candidates): started_at = time.perf_counter() candidate_payload = dict(upstream) candidate_payload["model"] = candidate.model @@ -4368,16 +4416,49 @@ def proxy_completion( send_once = self.client.proxy_send result = send_once(candidate, endpoint, candidate_payload) except Exception as exc: # noqa: BLE001 - provider trust boundary - if not _is_passthrough_failover_error(exc): - if isinstance(exc, (urllib.error.HTTPError, ProviderUpstreamError)): - raise classify_provider_failure( - exc, - agent_id=candidate.id, - model=candidate.model, - transport="passthrough", + classified = ( + classify_provider_failure( + exc, + agent_id=candidate.id, + model=candidate.model, + transport="passthrough", + ) + if isinstance(exc, (urllib.error.HTTPError, ProviderUpstreamError)) + else None + ) + failover_eligible = _is_passthrough_failover_error(exc) or ( + requested_model == self.FREE_MODEL + and isinstance(classified, ProviderUpstreamError) + and classified.transport == "passthrough" + and classified.retryable + and classified.client_status == 502 + and classified.provider_status is None + ) + has_remaining_candidates = index + 1 < len(candidates) + if isinstance(classified, ProviderUpstreamError): + attempt_receipts.append( + _passthrough_attempt_record( + classified, + failover_decision=( + "advance_to_next_candidate" + if failover_eligible and has_remaining_candidates + else "eligible_candidates_exhausted" + ), + ) + ) + if not failover_eligible: + if classified is not None: + raise _set_passthrough_attempt_evidence( + classified, + selected_candidate_ids=selected_candidate_ids, + attempts=attempt_receipts, + terminal_reason="terminal_provider_failure", ) from None raise - last_failure = (exc, candidate) + last_failure = ( + classified if isinstance(classified, ProviderUpstreamError) else exc, + candidate, + ) request_too_large = _is_request_too_large_error(exc) every_failure_was_request_too_large = ( every_failure_was_request_too_large @@ -4396,16 +4477,26 @@ def proxy_completion( ) return result if last_failure is not None and every_failure_was_request_too_large: - raise ProviderRequestTooLargeError( - "request body exceeds every eligible provider limit" + raise _set_passthrough_attempt_evidence( + ProviderRequestTooLargeError( + "request body exceeds every eligible provider limit" + ), + selected_candidate_ids=selected_candidate_ids, + attempts=attempt_receipts, + terminal_reason="request_too_large_exhausted", ) from None if last_failure is not None: last_error, failed_candidate = last_failure - raise classify_provider_failure( - last_error, - agent_id=failed_candidate.id, - model=failed_candidate.model, - transport="passthrough", + raise _set_passthrough_attempt_evidence( + classify_provider_failure( + last_error, + agent_id=failed_candidate.id, + model=failed_candidate.model, + transport="passthrough", + ), + selected_candidate_ids=selected_candidate_ids, + attempts=attempt_receipts, + terminal_reason="eligible_candidates_exhausted", ) from None raise RuntimeError("passthrough has no eligible provider candidate") diff --git a/contextual_orchestrator/provider_errors.py b/contextual_orchestrator/provider_errors.py index 3b862792d..ff8245207 100644 --- a/contextual_orchestrator/provider_errors.py +++ b/contextual_orchestrator/provider_errors.py @@ -195,13 +195,23 @@ def __init__( @property def detail(self) -> dict[str, Any]: """Return the structured evidence attached to API error payloads.""" - return { + detail = { "agent_id": self.agent_id, "model": self.model, "provider_status": self.provider_status, "retryable": self.retryable, "transport": self.transport, } + selected_candidate_ids = getattr(self, "selected_candidate_ids", None) + if isinstance(selected_candidate_ids, (list, tuple)) and selected_candidate_ids: + detail["selected_candidate_ids"] = list(selected_candidate_ids) + attempts = getattr(self, "attempts", None) + if isinstance(attempts, (list, tuple)) and attempts: + detail["attempts"] = [dict(item) for item in attempts if isinstance(item, dict)] + terminal_reason = getattr(self, "terminal_reason", None) + if isinstance(terminal_reason, str) and terminal_reason: + detail["terminal_reason"] = terminal_reason + return detail def classify_provider_failure( diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index d145a0b1d..a6603fe4a 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,76 @@ # Contextual Orchestrator: Product & Technical Gap Baseline +## 2026-09-04 Autonomous Commercialization Loop: issue #1045 root-cause fix for orchestrator/free passthrough 502 evidence + +Observation time: 2026-09-04 Asia/Seoul. + +GitHub authentication was re-verified first with `gh api user`. The primary +checkout was dirty, so work continued in a clean linked worktree at +`.worktrees/commercial-loop-20260904-issue1045`. Open PR heads and prior +`commercial-loop-*` worktrees were re-fetched before editing. No existing +open PR head covered this exact contract: PR [#1046](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1046) +was only queued behind hosted checks for an unrelated EgressWeave SSRF change, +while the active `orchestrator/free` queue items [#1028](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1028) +and [#993](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/993) +were preserved as distinct in-flight contracts. The highest-leverage +independent unit was therefore issue +[#1045](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/1045): +typed attempt evidence and bounded failover for long `orchestrator/free` +tool-loop transport failures. + +### Root cause confirmed on current `main` + +The live Noema review incidents in issue `#1045` reproduced the single-agent +tool-loop path, not `route_once()`. `/v1/chat/completions` tool-bearing +requests call `proxy_completion(..., single_agent=True)`, which performs +virtual-model passthrough failover inside `TaskOrchestrator.proxy_completion`. + +Current-head RCA: + +- raw HTTP 5xx passthrough failures already advanced across distinct virtual + candidates, but a pre-classified `ProviderUpstreamError` with + `transport="passthrough"`, `retryable=True`, `client_status=502`, and no + upstream status did not; +- when every eligible free candidate failed, the final gateway error collapsed + to one typed exception without any request-scoped candidate ledger, leaving + consumers with `served_model=unknown` and no bounded receipt of which + candidates were selected or attempted. + +### Local fix completed + +The worktree change makes one surgical contract extension: + +- `proxy_completion` now preserves a bounded per-request passthrough attempt + ledger for classified failures: selected candidate ids, per-attempt + candidate/model identity, typed failure class, retryability, lifecycle phase, + failover decision, and terminal reason; +- `orchestrator/free` virtual passthrough now advances to the next distinct + eligible candidate when a candidate returns a classified retryable transport + `502` with no upstream provider status, while explicit concrete-model + requests remain sticky; +- wrapped transient errors that were already eligible for failover through + `__cause__` inspection retain that prior behavior unchanged. + +The public error detail remains bounded and secret-safe: no credentials, raw +provider bodies, or prompt text are emitted. The new lifecycle phase is +`connecting` for pre-provider transport/TLS failures, which distinguishes them +from provider-response failures without inventing provider acceptance. + +### Exact local verification + +- Added RED tests for free-pool classified transport `502` failover, + exhausted free-pool attempt receipts, sticky explicit concrete-model + transport failure, and the HTTP error-detail surface for tool-bearing + `/v1/chat/completions`. +- `uv run pytest tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py -k 'classified_ambiguous_connection_error or classified_retryable_transport_502 or bounded_attempt_evidence or explicit_model_classified_transport_502 or all_candidates_chain_the_last_failure or non_transient_error_is_not_replayed or http_free_tool_passthrough_exposes_bounded_attempt_evidence_on_502 or virtual_passthrough_advances_once or http_virtual_structured_synthesis_failure_returns_provider_error' -q` + -> `13 passed, 82 deselected in 1.63s` +- `uv run pytest tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py tests/test_provider_error_taxonomy.py tests/test_chat_orchestration_mode_http_honesty.py -q` + -> `124 passed in 21.73s` + +Hosted exact-head checks, protected merge, and the unchanged LifeOS/Noema +consumer canary remain future steps because this invocation stopped at one +completed local root-cause work unit, per the hourly-loop boundary. + ## 2026-09-01 Autonomous Commercialization Loop: PR #970 Merge, Token Accounting & Cost Gateway Harmonization Observation time: 2026-09-01 Asia/Seoul. diff --git a/tests/test_openai_passthrough.py b/tests/test_openai_passthrough.py index 9fc776560..b1fac0280 100644 --- a/tests/test_openai_passthrough.py +++ b/tests/test_openai_passthrough.py @@ -27,6 +27,7 @@ ModelClient, _responses_to_chat_payload, ) +from contextual_orchestrator.provider_errors import ProviderUpstreamError # noqa: E402 from contextual_orchestrator.server import ( # noqa: E402 SecurityConfig, build_server, @@ -489,6 +490,97 @@ def proxy_send_once(self, agent, endpoint, payload): assert "synthetic provider outage" not in json.dumps(body) +def test_http_free_tool_passthrough_exposes_bounded_attempt_evidence_on_502() -> None: + """Free tool-loop 502s keep request-scoped candidate evidence on the wire.""" + + class ConnectingFreePool(ModelClient): + def proxy_send_once(self, agent, endpoint, payload): + del endpoint, payload + raise ProviderUpstreamError( + agent_id=agent.id, + model=agent.model, + error_code="provider_connection_error", + message=f"the provider {agent.id} connection failed or did not finish in time", + client_status=502, + provider_status=None, + retryable=True, + transport="passthrough", + ) + + proxy_send = proxy_send_once + + token = "passthrough_token" + orchestrator = TaskOrchestrator( + [ + ModelAgent( + "free_primary", + "free-primary-model", + tags=("cost:free",), + provider_name="free-primary", + priority=2, + ), + ModelAgent( + "free_backup", + "free-backup-model", + tags=("cost:free",), + provider_name="free-backup", + priority=1, + ), + ], + client=ConnectingFreePool(), # type: ignore[arg-type] + ) + server = build_server( + orchestrator, port=0, security=SecurityConfig(auth_token=token) + ) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + status, body = _post( + f"http://127.0.0.1:{server.server_address[1]}/v1/chat/completions", + { + "model": TaskOrchestrator.FREE_MODEL, + "messages": [{"role": "user", "content": "use the tool"}], + "tools": [{"type": "function", "function": {"name": "inspect"}}], + }, + token, + ) + finally: + server.shutdown() + server.server_close() + + assert status == 502 + assert body["error"]["code"] == "provider_connection_error" + assert body["error"]["detail"]["selected_candidate_ids"] == [ + "free_primary", + "free_backup", + ] + assert body["error"]["detail"]["terminal_reason"] == "eligible_candidates_exhausted" + assert body["error"]["detail"]["attempts"] == [ + { + "agent_id": "free_primary", + "model": "free-primary-model", + "error_code": "provider_connection_error", + "client_status": 502, + "provider_status": None, + "retryable": True, + "transport": "passthrough", + "phase": "connecting", + "failover_decision": "advance_to_next_candidate", + }, + { + "agent_id": "free_backup", + "model": "free-backup-model", + "error_code": "provider_connection_error", + "client_status": 502, + "provider_status": None, + "retryable": True, + "transport": "passthrough", + "phase": "connecting", + "failover_decision": "eligible_candidates_exhausted", + }, + ] + assert "use the tool" not in json.dumps(body) + + def test_http_chat_completions_accepts_response_format_and_passes_through() -> None: server, port, token = _serve() url = f"http://127.0.0.1:{port}/v1/chat/completions" diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index 87751f2ac..1785ea820 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -1151,6 +1151,141 @@ def test_classified_ambiguous_connection_error_does_not_fail_over() -> None: assert [agent_id for agent_id, _ in client.calls] == ["primary_agent"] +def test_free_virtual_model_fails_over_on_classified_retryable_transport_502() -> None: + """A free virtual-model request may advance after one retryable 502 transport failure.""" + failure = ProviderUpstreamError( + agent_id="primary_agent", + model="primary-model", + error_code="provider_connection_error", + message="the provider primary_agent connection failed or did not finish in time", + client_status=502, + provider_status=None, + retryable=True, + transport="passthrough", + ) + client = SequencedProxyClient( + { + "primary_agent": failure, + "fallback_agent": {"model": "fallback-model"}, + } + ) + orchestrator = _build(client) + orchestrator.agents = [ + replace(agent, tags=(*agent.tags, "cost:free")) for agent in orchestrator.agents + ] + + result = orchestrator.proxy_completion( + { + "model": TaskOrchestrator.FREE_MODEL, + "messages": [{"role": "user", "content": "use the tool"}], + "tools": [{"type": "function", "function": {"name": "inspect"}}], + } + ) + + assert result["model"] == "fallback-model" + assert [agent_id for agent_id, _ in client.calls] == ["primary_agent", "fallback_agent"] + + +def test_free_virtual_model_exhaustion_reports_bounded_attempt_evidence() -> None: + """Exhausted free passthrough keeps typed candidate evidence on the final 502.""" + primary = ProviderUpstreamError( + agent_id="primary_agent", + model="primary-model", + error_code="provider_connection_error", + message="the provider primary_agent connection failed or did not finish in time", + client_status=502, + provider_status=None, + retryable=True, + transport="passthrough", + ) + fallback = ProviderUpstreamError( + agent_id="fallback_agent", + model="fallback-model", + error_code="provider_connection_error", + message="the provider fallback_agent connection failed or did not finish in time", + client_status=502, + provider_status=None, + retryable=True, + transport="passthrough", + ) + client = SequencedProxyClient( + {"primary_agent": primary, "fallback_agent": fallback} + ) + orchestrator = _build(client) + orchestrator.agents = [ + replace(agent, tags=(*agent.tags, "cost:free")) for agent in orchestrator.agents + ] + + with pytest.raises(ProviderUpstreamError) as caught: + orchestrator.proxy_completion( + { + "model": TaskOrchestrator.FREE_MODEL, + "messages": [{"role": "user", "content": "use the tool"}], + "tools": [{"type": "function", "function": {"name": "inspect"}}], + } + ) + + assert caught.value.agent_id == "fallback_agent" + assert caught.value.detail["terminal_reason"] == "eligible_candidates_exhausted" + assert caught.value.detail["selected_candidate_ids"] == [ + "primary_agent", + "fallback_agent", + ] + assert caught.value.detail["attempts"] == [ + { + "agent_id": "primary_agent", + "model": "primary-model", + "error_code": "provider_connection_error", + "client_status": 502, + "provider_status": None, + "retryable": True, + "transport": "passthrough", + "phase": "connecting", + "failover_decision": "advance_to_next_candidate", + }, + { + "agent_id": "fallback_agent", + "model": "fallback-model", + "error_code": "provider_connection_error", + "client_status": 502, + "provider_status": None, + "retryable": True, + "transport": "passthrough", + "phase": "connecting", + "failover_decision": "eligible_candidates_exhausted", + }, + ] + assert [agent_id for agent_id, _ in client.calls] == ["primary_agent", "fallback_agent"] + + +def test_explicit_model_classified_transport_502_remains_sticky() -> None: + """A concrete model id keeps single-provider stickiness for retryable transport failures.""" + failure = ProviderUpstreamError( + agent_id="primary_agent", + model="primary-model", + error_code="provider_connection_error", + message="the provider primary_agent connection failed or did not finish in time", + client_status=502, + provider_status=None, + retryable=True, + transport="passthrough", + ) + client = SequencedProxyClient( + { + "primary_agent": failure, + "fallback_agent": {"model": "fallback-model"}, + } + ) + + with pytest.raises(ProviderUpstreamError) as caught: + _build(client).proxy_completion( + {"model": "primary-model", "messages": [{"role": "user", "content": "x"}]} + ) + + assert caught.value is failure + assert [agent_id for agent_id, _ in client.calls] == ["primary_agent"] + + def test_suppressed_transient_context_does_not_authorize_failover() -> None: """A deliberately hidden exception context cannot become a routing signal.""" try: From 0a68ab0315ab36c98b1feb363430e8d7275a4cd1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 16:31:10 +0900 Subject: [PATCH 02/16] docs(gap): refresh PR #1049 exact-head evidence --- docs/product-technical-gap-baseline.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index a6603fe4a..0dea18d1d 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -67,6 +67,19 @@ from provider-response failures without inventing provider acceptance. - `uv run pytest tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py tests/test_provider_error_taxonomy.py tests/test_chat_orchestration_mode_http_honesty.py -q` -> `124 passed in 21.73s` +### Exact-head refresh on September 4, 2026 + +PR [#1049](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1049) +was later refreshed onto then-current `origin/main` +`60c562defc81fb1897fa97ebdb5bf8f69eae0c55`, producing exact head +`e678fb41a6b554ad8ff7a17310a107e3984b9b7c`. The refresh only merged current +`main`; it did not change the issue `#1045` contract or duplicate PR +[#1046](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1046), +whose EgressWeave SSRF fix remains a separate open line. + +- `uv run pytest tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py tests/test_provider_error_taxonomy.py tests/test_chat_orchestration_mode_http_honesty.py tests/test_opencode_go.py -q` + -> `126 passed in 16.43s` + Hosted exact-head checks, protected merge, and the unchanged LifeOS/Noema consumer canary remain future steps because this invocation stopped at one completed local root-cause work unit, per the hourly-loop boundary. From a21f4e9fa7e6123864ef7aeb4c242fae1a03e32e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 18:44:46 +0900 Subject: [PATCH 03/16] fix(gateway): classify raw passthrough transport failures --- contextual_orchestrator/orchestrator.py | 54 ++++++------- docs/product-technical-gap-baseline.md | 71 ++++++++--------- tests/test_openai_passthrough.py | 54 +++++++++++++ tests/test_passthrough_provider_failover.py | 85 +++++++++++++++++++-- 4 files changed, 190 insertions(+), 74 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 6d4bcc303..10979071c 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -4419,47 +4419,43 @@ def proxy_completion( send_once = self.client.proxy_send result = send_once(candidate, endpoint, candidate_payload) except Exception as exc: # noqa: BLE001 - provider trust boundary - classified = ( - classify_provider_failure( - exc, - agent_id=candidate.id, - model=candidate.model, - transport="passthrough", - ) - if isinstance(exc, (urllib.error.HTTPError, ProviderUpstreamError)) - else None + classified = classify_provider_failure( + exc, + agent_id=candidate.id, + model=candidate.model, + transport="passthrough", ) failover_eligible = _is_passthrough_failover_error(exc) or ( requested_model == self.FREE_MODEL - and isinstance(classified, ProviderUpstreamError) and classified.transport == "passthrough" and classified.retryable and classified.client_status == 502 and classified.provider_status is None ) has_remaining_candidates = index + 1 < len(candidates) - if isinstance(classified, ProviderUpstreamError): - attempt_receipts.append( - _passthrough_attempt_record( - classified, - failover_decision=( - "advance_to_next_candidate" - if failover_eligible and has_remaining_candidates - else "eligible_candidates_exhausted" - ), - ) + attempt_receipts.append( + _passthrough_attempt_record( + classified, + failover_decision=( + "advance_to_next_candidate" + if failover_eligible and has_remaining_candidates + else ( + "eligible_candidates_exhausted" + if failover_eligible + else "sticky_candidate_failure" + ) + ), ) + ) if not failover_eligible: - if classified is not None: - raise _set_passthrough_attempt_evidence( - classified, - selected_candidate_ids=selected_candidate_ids, - attempts=attempt_receipts, - terminal_reason="terminal_provider_failure", - ) from None - raise + raise _set_passthrough_attempt_evidence( + classified, + selected_candidate_ids=selected_candidate_ids, + attempts=attempt_receipts, + terminal_reason="terminal_provider_failure", + ) from None last_failure = ( - classified if isinstance(classified, ProviderUpstreamError) else exc, + classified, candidate, ) request_too_large = _is_request_too_large_error(exc) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 0dea18d1d..b3687437d 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -27,29 +27,31 @@ virtual-model passthrough failover inside `TaskOrchestrator.proxy_completion`. Current-head RCA: -- raw HTTP 5xx passthrough failures already advanced across distinct virtual - candidates, but a pre-classified `ProviderUpstreamError` with - `transport="passthrough"`, `retryable=True`, `client_status=502`, and no - upstream status did not; -- when every eligible free candidate failed, the final gateway error collapsed - to one typed exception without any request-scoped candidate ledger, leaving - consumers with `served_model=unknown` and no bounded receipt of which - candidates were selected or attempted. +- raw HTTP passthrough failures were already classified and could participate + in bounded failover, but raw provider transport exceptions such as + `TimeoutError`, `urllib.error.URLError`, `ConnectionError`, and wrapped DNS + failures were not classified inside the multi-candidate passthrough loop; +- `classify_provider_failure()` already mapped those raw exceptions to bounded + typed 502 surfaces, but `proxy_completion()` only invoked the classifier for + `HTTPError` and already-classified `ProviderUpstreamError` instances; +- for `orchestrator/free`, retryable raw transport failures therefore stopped + at the first selected candidate instead of advancing to the next eligible + free provider, and sticky raw failures had no request-scoped attempt receipt + explaining why the gateway did not retry. ### Local fix completed The worktree change makes one surgical contract extension: -- `proxy_completion` now preserves a bounded per-request passthrough attempt - ledger for classified failures: selected candidate ids, per-attempt - candidate/model identity, typed failure class, retryability, lifecycle phase, - failover decision, and terminal reason; +- `proxy_completion()` now classifies every caught passthrough provider + exception before deciding whether failover is permitted; - `orchestrator/free` virtual passthrough now advances to the next distinct - eligible candidate when a candidate returns a classified retryable transport - `502` with no upstream provider status, while explicit concrete-model - requests remain sticky; -- wrapped transient errors that were already eligible for failover through - `__cause__` inspection retain that prior behavior unchanged. + eligible candidate when a provider raises a raw retryable transport failure + that classifies to a passthrough 502 with no upstream provider status; +- sticky failures now record the distinct failover decision + `sticky_candidate_failure` instead of incorrectly reusing + `eligible_candidates_exhausted`, while explicit concrete-model requests + remain single-provider sticky. The public error detail remains bounded and secret-safe: no credentials, raw provider bodies, or prompt text are emitted. The new lifecycle phase is @@ -58,27 +60,20 @@ from provider-response failures without inventing provider acceptance. ### Exact local verification -- Added RED tests for free-pool classified transport `502` failover, - exhausted free-pool attempt receipts, sticky explicit concrete-model - transport failure, and the HTTP error-detail surface for tool-bearing - `/v1/chat/completions`. -- `uv run pytest tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py -k 'classified_ambiguous_connection_error or classified_retryable_transport_502 or bounded_attempt_evidence or explicit_model_classified_transport_502 or all_candidates_chain_the_last_failure or non_transient_error_is_not_replayed or http_free_tool_passthrough_exposes_bounded_attempt_evidence_on_502 or virtual_passthrough_advances_once or http_virtual_structured_synthesis_failure_returns_provider_error' -q` - -> `13 passed, 82 deselected in 1.63s` -- `uv run pytest tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py tests/test_provider_error_taxonomy.py tests/test_chat_orchestration_mode_http_honesty.py -q` - -> `124 passed in 21.73s` - -### Exact-head refresh on September 4, 2026 - -PR [#1049](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1049) -was later refreshed onto then-current `origin/main` -`60c562defc81fb1897fa97ebdb5bf8f69eae0c55`, producing exact head -`e678fb41a6b554ad8ff7a17310a107e3984b9b7c`. The refresh only merged current -`main`; it did not change the issue `#1045` contract or duplicate PR -[#1046](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1046), -whose EgressWeave SSRF fix remains a separate open line. - -- `uv run pytest tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py tests/test_provider_error_taxonomy.py tests/test_chat_orchestration_mode_http_honesty.py tests/test_opencode_go.py -q` - -> `126 passed in 16.43s` +- Added focused regressions for raw timeout failover in the in-process + free-model passthrough loop, raw timeout failover through the real + `/v1/chat/completions` HTTP path, and bounded sticky attempt evidence for + non-failover raw wrapper, permanent DNS, and ambiguous timeout cases. +- `uv run pytest tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py -q` + -> `97 passed in 12.75s` +- `uv run pytest tests/test_provider_error_taxonomy.py tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py -q` + -> `118 passed in 14.68s` + +### Branch-local quality note + +- `uv run ruff check contextual_orchestrator/orchestrator.py tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py` + could not run in this worktree because the pinned environment does not + currently expose a `ruff` executable (`No such file or directory`). Hosted exact-head checks, protected merge, and the unchanged LifeOS/Noema consumer canary remain future steps because this invocation stopped at one diff --git a/tests/test_openai_passthrough.py b/tests/test_openai_passthrough.py index b1fac0280..878bdfac9 100644 --- a/tests/test_openai_passthrough.py +++ b/tests/test_openai_passthrough.py @@ -581,6 +581,60 @@ def proxy_send_once(self, agent, endpoint, payload): assert "use the tool" not in json.dumps(body) +def test_http_free_tool_passthrough_raw_timeout_fails_over_and_reports_attempts() -> None: + """HTTP passthrough preserves attempt evidence for raw transport failures.""" + + class TimeoutFreePool(ModelClient): + def proxy_send_once(self, agent, endpoint, payload): + del endpoint, payload + if agent.id == "free_primary": + raise TimeoutError("provider timed out") + return {"id": "chatcmpl-free", "object": "chat.completion", "model": agent.model, "choices": []} + + proxy_send = proxy_send_once + + token = "passthrough_token" + orchestrator = TaskOrchestrator( + [ + ModelAgent( + "free_primary", + "free-primary-model", + tags=("cost:free",), + provider_name="free-primary", + priority=2, + ), + ModelAgent( + "free_backup", + "free-backup-model", + tags=("cost:free",), + provider_name="free-backup", + priority=1, + ), + ], + client=TimeoutFreePool(), # type: ignore[arg-type] + ) + server = build_server( + orchestrator, port=0, security=SecurityConfig(auth_token=token) + ) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + status, body = _post( + f"http://127.0.0.1:{server.server_address[1]}/v1/chat/completions", + { + "model": TaskOrchestrator.FREE_MODEL, + "messages": [{"role": "user", "content": "use the tool"}], + "tools": [{"type": "function", "function": {"name": "inspect"}}], + }, + token, + ) + finally: + server.shutdown() + server.server_close() + + assert status == 200 + assert body["model"] == "free-backup-model" + + def test_http_chat_completions_accepts_response_format_and_passes_through() -> None: server, port, token = _serve() url = f"http://127.0.0.1:{port}/v1/chat/completions" diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index 1785ea820..cc693ac72 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -260,6 +260,34 @@ def test_virtual_passthrough_all_oversized_tool_errors_preserve_size_contract() assert orchestrator._circuit == {} +def test_free_passthrough_transport_errors_fail_over_from_raw_timeout() -> None: + """Retryable raw transport failures stay on the free-model failover path.""" + client = SequencedProxyClient( + { + "primary_agent": TimeoutError("provider timed out"), + "fallback_agent": {"model": "fallback-model", "choices": []}, + } + ) + orchestrator = _build(client) + orchestrator.agents = [ + replace(agent, tags=(*agent.tags, "cost:free")) for agent in orchestrator.agents + ] + + result = orchestrator.proxy_completion( + { + "model": TaskOrchestrator.FREE_MODEL, + "messages": [{"role": "user", "content": "use the tool"}], + "tools": [{"type": "function", "function": {"name": "inspect"}}], + } + ) + + assert result["model"] == "fallback-model" + assert [agent_id for agent_id, _ in client.calls] == [ + "primary_agent", + "fallback_agent", + ] + + def test_virtual_passthrough_keeps_non_size_tool_errors_sticky() -> None: """A generic provider invalid_tools response must not hide a bad request.""" failure = _invalid_tools_error() @@ -1302,10 +1330,24 @@ def test_suppressed_transient_context_does_not_authorize_failover() -> None: } ) - with pytest.raises(RuntimeError, match="terminal wrapper") as caught: + with pytest.raises(ProviderUpstreamError) as caught: _build(client).proxy_completion({"messages": [{"role": "user", "content": "x"}]}) - assert caught.value is failure + assert caught.value.error_code == "api_error" + assert caught.value.detail["terminal_reason"] == "terminal_provider_failure" + assert caught.value.detail["attempts"] == [ + { + "agent_id": "primary_agent", + "model": "primary-model", + "error_code": "api_error", + "client_status": 502, + "provider_status": None, + "retryable": False, + "transport": "passthrough", + "phase": "provider_response", + "failover_decision": "sticky_candidate_failure", + } + ] assert [agent_id for agent_id, _ in client.calls] == ["primary_agent"] @@ -1418,10 +1460,23 @@ def test_only_temporary_dns_failures_advance( {"messages": [{"role": "user", "content": "x"}]} )["model"] == "fallback-model" else: - with pytest.raises(RuntimeError, match="provider resolution failed"): - _build(client).proxy_completion( - {"messages": [{"role": "user", "content": "x"}]} - ) + with pytest.raises(ProviderUpstreamError) as caught: + _build(client).proxy_completion({"messages": [{"role": "user", "content": "x"}]}) + assert caught.value.error_code == "provider_connection_error" + assert caught.value.detail["terminal_reason"] == "terminal_provider_failure" + assert caught.value.detail["attempts"] == [ + { + "agent_id": "primary_agent", + "model": "primary-model", + "error_code": "provider_connection_error", + "client_status": 502, + "provider_status": None, + "retryable": False, + "transport": "passthrough", + "phase": "connecting", + "failover_decision": "sticky_candidate_failure", + } + ] def test_ambiguous_timeout_is_not_replayed() -> None: @@ -1434,9 +1489,25 @@ def test_ambiguous_timeout_is_not_replayed() -> None: } ) - with pytest.raises(TimeoutError, match="outcome unknown"): + with pytest.raises(ProviderUpstreamError) as caught: _build(client).proxy_completion({"messages": [{"role": "user", "content": "x"}]}) + assert caught.value.error_code == "provider_connection_error" + assert caught.value.retryable is True + assert caught.value.detail["terminal_reason"] == "terminal_provider_failure" + assert caught.value.detail["attempts"] == [ + { + "agent_id": "primary_agent", + "model": "primary-model", + "error_code": "provider_connection_error", + "client_status": 502, + "provider_status": None, + "retryable": True, + "transport": "passthrough", + "phase": "connecting", + "failover_decision": "sticky_candidate_failure", + } + ] assert [agent_id for agent_id, _ in client.calls] == ["primary_agent"] From fd12d4f222234d1d07f5dd4762c292c689e2f7ee Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 20:03:42 +0900 Subject: [PATCH 04/16] fix(gateway): identify provider failover attempts Expose bounded provider names and one-based attempt numbers in passthrough failure receipts. Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 6 ++++++ tests/test_openai_passthrough.py | 4 ++++ tests/test_passthrough_provider_failover.py | 13 ++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 10979071c..8c715f869 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -1704,12 +1704,16 @@ def _passthrough_failure_phase(exc: ProviderUpstreamError) -> str: def _passthrough_attempt_record( exc: ProviderUpstreamError, *, + provider_name: str, + attempt_number: int, failover_decision: str, ) -> dict[str, Any]: """Return the public attempt receipt for one classified passthrough failure.""" return { "agent_id": exc.agent_id, "model": exc.model, + "provider_name": provider_name, + "attempt_number": attempt_number, "error_code": exc.error_code, "client_status": exc.client_status, "provider_status": exc.provider_status, @@ -4436,6 +4440,8 @@ def proxy_completion( attempt_receipts.append( _passthrough_attempt_record( classified, + provider_name=candidate.provider_name, + attempt_number=index + 1, failover_decision=( "advance_to_next_candidate" if failover_eligible and has_remaining_candidates diff --git a/tests/test_openai_passthrough.py b/tests/test_openai_passthrough.py index 878bdfac9..a593e472b 100644 --- a/tests/test_openai_passthrough.py +++ b/tests/test_openai_passthrough.py @@ -558,6 +558,8 @@ def proxy_send_once(self, agent, endpoint, payload): { "agent_id": "free_primary", "model": "free-primary-model", + "provider_name": "free-primary", + "attempt_number": 1, "error_code": "provider_connection_error", "client_status": 502, "provider_status": None, @@ -569,6 +571,8 @@ def proxy_send_once(self, agent, endpoint, payload): { "agent_id": "free_backup", "model": "free-backup-model", + "provider_name": "free-backup", + "attempt_number": 2, "error_code": "provider_connection_error", "client_status": 502, "provider_status": None, diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index cc693ac72..2dc36c434 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -1263,6 +1263,8 @@ def test_free_virtual_model_exhaustion_reports_bounded_attempt_evidence() -> Non { "agent_id": "primary_agent", "model": "primary-model", + "provider_name": "primary", + "attempt_number": 1, "error_code": "provider_connection_error", "client_status": 502, "provider_status": None, @@ -1274,6 +1276,8 @@ def test_free_virtual_model_exhaustion_reports_bounded_attempt_evidence() -> Non { "agent_id": "fallback_agent", "model": "fallback-model", + "provider_name": "fallback", + "attempt_number": 2, "error_code": "provider_connection_error", "client_status": 502, "provider_status": None, @@ -1339,6 +1343,8 @@ def test_suppressed_transient_context_does_not_authorize_failover() -> None: { "agent_id": "primary_agent", "model": "primary-model", + "provider_name": "primary", + "attempt_number": 1, "error_code": "api_error", "client_status": 502, "provider_status": None, @@ -1468,6 +1474,8 @@ def test_only_temporary_dns_failures_advance( { "agent_id": "primary_agent", "model": "primary-model", + "provider_name": "primary", + "attempt_number": 1, "error_code": "provider_connection_error", "client_status": 502, "provider_status": None, @@ -1481,7 +1489,7 @@ def test_only_temporary_dns_failures_advance( def test_ambiguous_timeout_is_not_replayed() -> None: """A timeout may follow provider acceptance, so passthrough fails closed.""" - failure = TimeoutError("provider outcome unknown") + failure = TimeoutError("secret-bearing provider diagnostic") client = SequencedProxyClient( { "primary_agent": failure, @@ -1499,6 +1507,8 @@ def test_ambiguous_timeout_is_not_replayed() -> None: { "agent_id": "primary_agent", "model": "primary-model", + "provider_name": "primary", + "attempt_number": 1, "error_code": "provider_connection_error", "client_status": 502, "provider_status": None, @@ -1508,6 +1518,7 @@ def test_ambiguous_timeout_is_not_replayed() -> None: "failover_decision": "sticky_candidate_failure", } ] + assert "secret-bearing provider diagnostic" not in repr(caught.value.detail) assert [agent_id for agent_id, _ in client.calls] == ["primary_agent"] From 87612a68b3af1f305bb7b09bd0be860bad1b7fd6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 20:44:17 +0900 Subject: [PATCH 05/16] fix(gateway): infer missing receipt provider names Preserve secret-safe provider attribution when catalog entries omit an explicit provider name. Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 5 ++++- tests/test_passthrough_provider_failover.py | 12 +++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 8c715f869..8618eb95d 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -4440,7 +4440,10 @@ def proxy_completion( attempt_receipts.append( _passthrough_attempt_record( classified, - provider_name=candidate.provider_name, + provider_name=( + candidate.provider_name + or self._infer_provider_name(candidate.base_url) + ), attempt_number=index + 1, failover_decision=( "advance_to_next_candidate" diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index 2dc36c434..3dfcedaf4 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -1241,7 +1241,13 @@ def test_free_virtual_model_exhaustion_reports_bounded_attempt_evidence() -> Non ) orchestrator = _build(client) orchestrator.agents = [ - replace(agent, tags=(*agent.tags, "cost:free")) for agent in orchestrator.agents + replace( + agent, + base_url=f"mock://{agent.id}", + provider_name="", + tags=(*agent.tags, "cost:free"), + ) + for agent in orchestrator.agents ] with pytest.raises(ProviderUpstreamError) as caught: @@ -1263,7 +1269,7 @@ def test_free_virtual_model_exhaustion_reports_bounded_attempt_evidence() -> Non { "agent_id": "primary_agent", "model": "primary-model", - "provider_name": "primary", + "provider_name": "mock-primary_agent", "attempt_number": 1, "error_code": "provider_connection_error", "client_status": 502, @@ -1276,7 +1282,7 @@ def test_free_virtual_model_exhaustion_reports_bounded_attempt_evidence() -> Non { "agent_id": "fallback_agent", "model": "fallback-model", - "provider_name": "fallback", + "provider_name": "mock-fallback_agent", "attempt_number": 2, "error_code": "provider_connection_error", "client_status": 502, From 13e8c29c118fb59da7bbb1206ecee5cfc5f46564 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 10:46:23 +0900 Subject: [PATCH 06/16] test(gateway): cover free failover on upstream HTTP 500 --- tests/test_passthrough_provider_failover.py | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index 3dfcedaf4..d31573a42 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -1214,6 +1214,34 @@ def test_free_virtual_model_fails_over_on_classified_retryable_transport_502() - assert [agent_id for agent_id, _ in client.calls] == ["primary_agent", "fallback_agent"] + +def test_free_virtual_model_fails_over_on_raw_provider_http_500() -> None: + """A real upstream HTTP 500 advances to the next eligible free candidate.""" + client = SequencedProxyClient( + { + "primary_agent": _http_error(500), + "fallback_agent": {"model": "fallback-model"}, + } + ) + orchestrator = _build(client) + orchestrator.agents = [ + replace(agent, tags=(*agent.tags, "cost:free")) for agent in orchestrator.agents + ] + + result = orchestrator.proxy_completion( + { + "model": TaskOrchestrator.FREE_MODEL, + "messages": [{"role": "user", "content": "use the tool"}], + "tools": [{"type": "function", "function": {"name": "inspect"}}], + } + ) + + assert result["model"] == "fallback-model" + assert [agent_id for agent_id, _ in client.calls] == [ + "primary_agent", + "fallback_agent", + ] + def test_free_virtual_model_exhaustion_reports_bounded_attempt_evidence() -> None: """Exhausted free passthrough keeps typed candidate evidence on the final 502.""" primary = ProviderUpstreamError( From d26fa132185198a080c91964170b86a4f6357c6f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 06:01:24 +0900 Subject: [PATCH 07/16] fix(gateway): classify allowlist misses as free-pool 502s Live orchestrator/free returned HTTP 500 after every thinker candidate raised RuntimeError for an unallowlisted host. Raise a non-retryable ProviderUpstreamError so _invoke failovers and, when the free pool is exhausted, surfaces 502 instead of collapsing. Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 11 ++- tests/test_provider_reliability.py | 125 ++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 8618eb95d..dd8e7729b 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -2842,7 +2842,16 @@ def _validate_provider(self, agent: ModelAgent) -> ProviderDestination: raise RuntimeError(f"{agent.id} base_url must not contain credentials, query data, or fragments") hostname = parsed.hostname.lower() if self.allowed_provider_hosts and hostname not in self.allowed_provider_hosts: - raise RuntimeError(f"{agent.id} provider host is not allowlisted") + raise ProviderUpstreamError( + agent_id=agent.id, + model=agent.model, + error_code="provider_connection_error", + message=f"{agent.id} provider host is not allowlisted", + client_status=502, + provider_status=None, + retryable=False, + transport="chat", + ) addresses = self._resolve_addresses(hostname, parsed.port or 443) for _family, sockaddr in addresses: ip_address = ipaddress.ip_address(sockaddr[0]) diff --git a/tests/test_provider_reliability.py b/tests/test_provider_reliability.py index fc816b937..6e65c3858 100644 --- a/tests/test_provider_reliability.py +++ b/tests/test_provider_reliability.py @@ -22,6 +22,10 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.credentials import ( # noqa: E402 + InMemoryCredentialBackend, + set_backend, +) from contextual_orchestrator.orchestrator import ( # noqa: E402 TRANSIENT_HTTP_STATUS, ModelClient, @@ -770,6 +774,127 @@ def chat(self, agent: ModelAgent, messages: list, temperature: float = 0.2) -> s assert "priced_worker" not in client.calls +def test_unallowlisted_provider_host_is_classified_upstream_error() -> None: + """Live orchestrator/free 500s were raw RuntimeError from host allowlisting.""" + client = ModelClient(allowed_provider_hosts={"ok.example"}) + agent = ModelAgent( + "blocked_agent", + "blocked-model", + base_url="https://blocked.example/v1", + credential_key="MODEL_KEY", + ) + backend = InMemoryCredentialBackend() + backend.set("MODEL_KEY", "sk-host-check") + set_backend(backend) + try: + with pytest.raises(ProviderUpstreamError) as excinfo: + client._validate_provider(agent) + finally: + set_backend(None) + assert excinfo.value.client_status == 502 + assert excinfo.value.retryable is False + assert excinfo.value.error_code == "provider_connection_error" + assert "allowlisted" in str(excinfo.value) + assert "blocked.example" not in str(excinfo.value) + + +def test_free_model_advances_past_unallowlisted_provider_host() -> None: + """A host-allowlist miss must skip to the next free candidate, not 500.""" + calls: list[str] = [] + blocked = ModelAgent( + "free_route_a", + "free_route_a-model", + base_url="https://blocked.example/v1", + credential_key="MODEL_KEY", + tags=("reasoning", "cost:free"), + ) + + class AllowlistThenOk(ModelClient): + def __init__(self) -> None: + super().__init__(allowed_provider_hosts={"ok.example"}) + + def chat(self, agent: ModelAgent, messages: list, temperature: float = 0.2) -> str: # type: ignore[override] + del messages, temperature + calls.append(agent.id) + if agent.id == "free_route_a": + self._validate_provider(blocked) + return f"[{agent.id}] answer" + + backend = InMemoryCredentialBackend() + backend.set("MODEL_KEY", "sk-host-check") + set_backend(backend) + try: + orchestrator = _free_pool_orchestrator( + AllowlistThenOk(), free_ids=("free_route_a", "free_route_b") + ) + orchestrator.tool_retry_attempts = 0 + result = orchestrator.route_once( + [{"role": "user", "content": "route this"}], + model_name=TaskOrchestrator.FREE_MODEL, + ) + finally: + set_backend(None) + + assert result["answer"] == "[free_route_b] answer" + assert result["trace"][0]["served_agent_id"] == "free_route_b" + assert calls == ["free_route_a", "free_route_b"] + assert "priced_worker" not in calls + + +def test_free_model_exhausted_allowlist_pool_fails_closed_as_502() -> None: + """Every free candidate missing the host allowlist must not collapse to HTTP 500.""" + blocked_by_id = { + "free_route_a": ModelAgent( + "free_route_a", + "free_route_a-model", + base_url="https://blocked-a.example/v1", + credential_key="MODEL_KEY", + tags=("reasoning", "cost:free"), + ), + "free_route_b": ModelAgent( + "free_route_b", + "free_route_b-model", + base_url="https://blocked-b.example/v1", + credential_key="MODEL_KEY", + tags=("reasoning", "cost:free"), + ), + } + + class AllBlocked(ModelClient): + def __init__(self) -> None: + super().__init__(allowed_provider_hosts={"ok.example"}) + self.calls: list[str] = [] + + def chat(self, agent: ModelAgent, messages: list, temperature: float = 0.2) -> str: # type: ignore[override] + del messages, temperature + self.calls.append(agent.id) + self._validate_provider(blocked_by_id[agent.id]) + raise AssertionError("allowlisted host was not supposed to be reached") + + backend = InMemoryCredentialBackend() + backend.set("MODEL_KEY", "sk-host-check") + set_backend(backend) + client = AllBlocked() + try: + orchestrator = _free_pool_orchestrator( + client, free_ids=("free_route_a", "free_route_b") + ) + orchestrator.tool_retry_attempts = 0 + with pytest.raises(ProviderUpstreamError) as excinfo: + orchestrator.route_once( + [{"role": "user", "content": "route this"}], + model_name=TaskOrchestrator.FREE_MODEL, + ) + finally: + set_backend(None) + + assert excinfo.value.client_status == 502 + assert "all " not in str(excinfo.value) + assert "role=" not in str(excinfo.value) + assert client.calls == ["free_route_a", "free_route_b"] + assert "priced_worker" not in client.calls + + def test_free_model_failover_survives_a_tool_shaped_provider_message() -> None: """A 500 whose body happens to mention "tool"/"invalid arguments" still fails over. From b2b141fa72036a617d20932c1abb42941785c23c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 14:28:53 +0900 Subject: [PATCH 08/16] fix(gateway): keep ambiguous passthrough failures sticky --- CHANGELOG.md | 4 + contextual_orchestrator/orchestrator.py | 21 ++--- docs/product-technical-gap-baseline.md | 46 ++++++---- tests/test_openai_passthrough.py | 43 +++++----- tests/test_passthrough_provider_failover.py | 94 +++++++++------------ 5 files changed, 98 insertions(+), 110 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f0b268c9..f83170a85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- Passthrough attempt receipts no longer infer public provider labels from + endpoint hostnames. Ambiguous timeout/connection failures now report the + neutral `transport` phase and remain sticky even for `orchestrator/free`, + preventing duplicate completion and unreported upstream usage. - Workflow workers now preserve the caller message array exactly once, while the added envelope carries only the subtask and Conductor-style prior-step access list instead of duplicating the task or source attachments. diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 3d30b6f1b..3b43310ff 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -1690,12 +1690,10 @@ def _is_capability_mismatch_failover_error(exc: BaseException) -> bool: def _passthrough_failure_phase(exc: ProviderUpstreamError) -> str: """Map one classified passthrough failure onto a bounded lifecycle phase.""" - if exc.error_code in { - "provider_connection_error", - "tls_failure", - "tls_verification_failed", - }: + if exc.error_code in {"tls_failure", "tls_verification_failed"}: return "connecting" + if exc.error_code == "provider_connection_error": + return "transport" if exc.error_code == "request_too_large": return "request_validation" return "provider_response" @@ -4481,21 +4479,12 @@ def proxy_completion( model=candidate.model, transport="passthrough", ) - failover_eligible = _is_passthrough_failover_error(exc) or ( - requested_model == self.FREE_MODEL - and classified.transport == "passthrough" - and classified.retryable - and classified.client_status == 502 - and classified.provider_status is None - ) + failover_eligible = _is_passthrough_failover_error(exc) has_remaining_candidates = index + 1 < len(candidates) attempt_receipts.append( _passthrough_attempt_record( classified, - provider_name=( - candidate.provider_name - or self._infer_provider_name(candidate.base_url) - ), + provider_name=candidate.provider_name.strip() or "unreported", attempt_number=index + 1, failover_decision=( "advance_to_next_candidate" diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index b3687437d..3b34cbd74 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -34,10 +34,10 @@ Current-head RCA: - `classify_provider_failure()` already mapped those raw exceptions to bounded typed 502 surfaces, but `proxy_completion()` only invoked the classifier for `HTTPError` and already-classified `ProviderUpstreamError` instances; -- for `orchestrator/free`, retryable raw transport failures therefore stopped - at the first selected candidate instead of advancing to the next eligible - free provider, and sticky raw failures had no request-scoped attempt receipt - explaining why the gateway did not retry. +- raw transport failures had no request-scoped attempt receipt explaining why + the gateway stopped. They remain non-replayable because a timeout or generic + connection failure does not prove that the provider rejected the request + before accepting work or usage. ### Local fix completed @@ -45,29 +45,39 @@ The worktree change makes one surgical contract extension: - `proxy_completion()` now classifies every caught passthrough provider exception before deciding whether failover is permitted; -- `orchestrator/free` virtual passthrough now advances to the next distinct - eligible candidate when a provider raises a raw retryable transport failure - that classifies to a passthrough 502 with no upstream provider status; +- `orchestrator/free` virtual passthrough advances only after evidence that + proves non-acceptance, such as an explicit retryable upstream HTTP response + or temporary pre-request DNS failure. Raw timeout and generic transport 502 + outcomes remain sticky to prevent duplicate completion and unreported usage; - sticky failures now record the distinct failover decision `sticky_candidate_failure` instead of incorrectly reusing `eligible_candidates_exhausted`, while explicit concrete-model requests remain single-provider sticky. The public error detail remains bounded and secret-safe: no credentials, raw -provider bodies, or prompt text are emitted. The new lifecycle phase is -`connecting` for pre-provider transport/TLS failures, which distinguishes them -from provider-response failures without inventing provider acceptance. +provider bodies, prompt text, or inferred endpoint hostnames are emitted. +Unclassified connection/timeout outcomes use lifecycle phase `transport`; +only explicit TLS failures use `connecting`. ### Exact local verification -- Added focused regressions for raw timeout failover in the in-process - free-model passthrough loop, raw timeout failover through the real - `/v1/chat/completions` HTTP path, and bounded sticky attempt evidence for - non-failover raw wrapper, permanent DNS, and ambiguous timeout cases. -- `uv run pytest tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py -q` - -> `97 passed in 12.75s` -- `uv run pytest tests/test_provider_error_taxonomy.py tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py -q` - -> `118 passed in 14.68s` +The 2026-09-08 current-head review repair added three explicit acceptance +boundaries: no replay after an ambiguous transport outcome, no inferred +endpoint hostname in public attempt evidence, and no invented `connecting` +phase for an outcome whose lifecycle stage is unknown. The revised tests first +failed as expected (`3 failed, 61 passed`) and then passed after the minimal +owner fix. + +- Added focused regressions proving ambiguous raw/classified transport errors + remain sticky in both the in-process free-model loop and real + `/v1/chat/completions` HTTP path, while explicit retryable HTTP 500 and + temporary pre-request DNS evidence retain bounded failover. +- `.venv/bin/python -m pytest tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py -q` + -> `98 passed in 10.64s` +- `.venv/bin/python -m pytest tests/test_provider_error_taxonomy.py tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py -q` + -> `119 passed in 11.61s` +- `uvx ruff check --select E4,E7,E9,F contextual_orchestrator/orchestrator.py tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py` + and `git diff --check` -> success. ### Branch-local quality note diff --git a/tests/test_openai_passthrough.py b/tests/test_openai_passthrough.py index a593e472b..205b1f7ff 100644 --- a/tests/test_openai_passthrough.py +++ b/tests/test_openai_passthrough.py @@ -553,7 +553,7 @@ def proxy_send_once(self, agent, endpoint, payload): "free_primary", "free_backup", ] - assert body["error"]["detail"]["terminal_reason"] == "eligible_candidates_exhausted" + assert body["error"]["detail"]["terminal_reason"] == "terminal_provider_failure" assert body["error"]["detail"]["attempts"] == [ { "agent_id": "free_primary", @@ -565,28 +565,15 @@ def proxy_send_once(self, agent, endpoint, payload): "provider_status": None, "retryable": True, "transport": "passthrough", - "phase": "connecting", - "failover_decision": "advance_to_next_candidate", - }, - { - "agent_id": "free_backup", - "model": "free-backup-model", - "provider_name": "free-backup", - "attempt_number": 2, - "error_code": "provider_connection_error", - "client_status": 502, - "provider_status": None, - "retryable": True, - "transport": "passthrough", - "phase": "connecting", - "failover_decision": "eligible_candidates_exhausted", + "phase": "transport", + "failover_decision": "sticky_candidate_failure", }, ] assert "use the tool" not in json.dumps(body) -def test_http_free_tool_passthrough_raw_timeout_fails_over_and_reports_attempts() -> None: - """HTTP passthrough preserves attempt evidence for raw transport failures.""" +def test_http_free_tool_passthrough_raw_timeout_does_not_replay() -> None: + """HTTP passthrough keeps an ambiguous timeout on its selected candidate.""" class TimeoutFreePool(ModelClient): def proxy_send_once(self, agent, endpoint, payload): @@ -635,8 +622,24 @@ def proxy_send_once(self, agent, endpoint, payload): server.shutdown() server.server_close() - assert status == 200 - assert body["model"] == "free-backup-model" + assert status == 502 + assert body["error"]["code"] == "provider_connection_error" + assert body["error"]["detail"]["terminal_reason"] == "terminal_provider_failure" + assert body["error"]["detail"]["attempts"] == [ + { + "agent_id": "free_primary", + "model": "free-primary-model", + "provider_name": "free-primary", + "attempt_number": 1, + "error_code": "provider_connection_error", + "client_status": 502, + "provider_status": None, + "retryable": True, + "transport": "passthrough", + "phase": "transport", + "failover_decision": "sticky_candidate_failure", + } + ] def test_http_chat_completions_accepts_response_format_and_passes_through() -> None: diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index d31573a42..fbc3b8dfe 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -260,8 +260,8 @@ def test_virtual_passthrough_all_oversized_tool_errors_preserve_size_contract() assert orchestrator._circuit == {} -def test_free_passthrough_transport_errors_fail_over_from_raw_timeout() -> None: - """Retryable raw transport failures stay on the free-model failover path.""" +def test_free_passthrough_raw_timeout_remains_sticky() -> None: + """An ambiguous timeout must not replay a free-model completion.""" client = SequencedProxyClient( { "primary_agent": TimeoutError("provider timed out"), @@ -273,19 +273,20 @@ def test_free_passthrough_transport_errors_fail_over_from_raw_timeout() -> None: replace(agent, tags=(*agent.tags, "cost:free")) for agent in orchestrator.agents ] - result = orchestrator.proxy_completion( - { - "model": TaskOrchestrator.FREE_MODEL, - "messages": [{"role": "user", "content": "use the tool"}], - "tools": [{"type": "function", "function": {"name": "inspect"}}], - } - ) + with pytest.raises(ProviderUpstreamError) as caught: + orchestrator.proxy_completion( + { + "model": TaskOrchestrator.FREE_MODEL, + "messages": [{"role": "user", "content": "use the tool"}], + "tools": [{"type": "function", "function": {"name": "inspect"}}], + } + ) - assert result["model"] == "fallback-model" - assert [agent_id for agent_id, _ in client.calls] == [ - "primary_agent", - "fallback_agent", - ] + assert caught.value.detail["attempts"][0]["phase"] == "transport" + assert caught.value.detail["attempts"][0]["failover_decision"] == ( + "sticky_candidate_failure" + ) + assert [agent_id for agent_id, _ in client.calls] == ["primary_agent"] def test_virtual_passthrough_keeps_non_size_tool_errors_sticky() -> None: @@ -1179,8 +1180,8 @@ def test_classified_ambiguous_connection_error_does_not_fail_over() -> None: assert [agent_id for agent_id, _ in client.calls] == ["primary_agent"] -def test_free_virtual_model_fails_over_on_classified_retryable_transport_502() -> None: - """A free virtual-model request may advance after one retryable 502 transport failure.""" +def test_free_virtual_model_keeps_ambiguous_transport_502_sticky() -> None: + """A classified ambiguous transport failure must not replay a free request.""" failure = ProviderUpstreamError( agent_id="primary_agent", model="primary-model", @@ -1202,16 +1203,17 @@ def test_free_virtual_model_fails_over_on_classified_retryable_transport_502() - replace(agent, tags=(*agent.tags, "cost:free")) for agent in orchestrator.agents ] - result = orchestrator.proxy_completion( - { - "model": TaskOrchestrator.FREE_MODEL, - "messages": [{"role": "user", "content": "use the tool"}], - "tools": [{"type": "function", "function": {"name": "inspect"}}], - } - ) + with pytest.raises(ProviderUpstreamError) as caught: + orchestrator.proxy_completion( + { + "model": TaskOrchestrator.FREE_MODEL, + "messages": [{"role": "user", "content": "use the tool"}], + "tools": [{"type": "function", "function": {"name": "inspect"}}], + } + ) - assert result["model"] == "fallback-model" - assert [agent_id for agent_id, _ in client.calls] == ["primary_agent", "fallback_agent"] + assert caught.value.detail["attempts"][0]["phase"] == "transport" + assert [agent_id for agent_id, _ in client.calls] == ["primary_agent"] @@ -1244,28 +1246,8 @@ def test_free_virtual_model_fails_over_on_raw_provider_http_500() -> None: def test_free_virtual_model_exhaustion_reports_bounded_attempt_evidence() -> None: """Exhausted free passthrough keeps typed candidate evidence on the final 502.""" - primary = ProviderUpstreamError( - agent_id="primary_agent", - model="primary-model", - error_code="provider_connection_error", - message="the provider primary_agent connection failed or did not finish in time", - client_status=502, - provider_status=None, - retryable=True, - transport="passthrough", - ) - fallback = ProviderUpstreamError( - agent_id="fallback_agent", - model="fallback-model", - error_code="provider_connection_error", - message="the provider fallback_agent connection failed or did not finish in time", - client_status=502, - provider_status=None, - retryable=True, - transport="passthrough", - ) client = SequencedProxyClient( - {"primary_agent": primary, "fallback_agent": fallback} + {"primary_agent": _http_error(500), "fallback_agent": _http_error(500)} ) orchestrator = _build(client) orchestrator.agents = [ @@ -1297,27 +1279,27 @@ def test_free_virtual_model_exhaustion_reports_bounded_attempt_evidence() -> Non { "agent_id": "primary_agent", "model": "primary-model", - "provider_name": "mock-primary_agent", + "provider_name": "unreported", "attempt_number": 1, - "error_code": "provider_connection_error", + "error_code": "api_error", "client_status": 502, - "provider_status": None, + "provider_status": 500, "retryable": True, "transport": "passthrough", - "phase": "connecting", + "phase": "provider_response", "failover_decision": "advance_to_next_candidate", }, { "agent_id": "fallback_agent", "model": "fallback-model", - "provider_name": "mock-fallback_agent", + "provider_name": "unreported", "attempt_number": 2, - "error_code": "provider_connection_error", + "error_code": "api_error", "client_status": 502, - "provider_status": None, + "provider_status": 500, "retryable": True, "transport": "passthrough", - "phase": "connecting", + "phase": "provider_response", "failover_decision": "eligible_candidates_exhausted", }, ] @@ -1515,7 +1497,7 @@ def test_only_temporary_dns_failures_advance( "provider_status": None, "retryable": False, "transport": "passthrough", - "phase": "connecting", + "phase": "transport", "failover_decision": "sticky_candidate_failure", } ] @@ -1548,7 +1530,7 @@ def test_ambiguous_timeout_is_not_replayed() -> None: "provider_status": None, "retryable": True, "transport": "passthrough", - "phase": "connecting", + "phase": "transport", "failover_decision": "sticky_candidate_failure", } ] From 1f5fd6521cea7e32b71a390cf85583ae244b082f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 15:11:40 +0900 Subject: [PATCH 09/16] test(passthrough): reject HTTP 500 replay --- tests/test_passthrough_provider_failover.py | 30 +++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index fbc3b8dfe..ec014cc33 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -1217,8 +1217,8 @@ def test_free_virtual_model_keeps_ambiguous_transport_502_sticky() -> None: -def test_free_virtual_model_fails_over_on_raw_provider_http_500() -> None: - """A real upstream HTTP 500 advances to the next eligible free candidate.""" +def test_free_virtual_model_keeps_http_500_sticky() -> None: + """HTTP 500 does not prove non-acceptance, so passthrough must not replay.""" client = SequencedProxyClient( { "primary_agent": _http_error(500), @@ -1230,19 +1230,21 @@ def test_free_virtual_model_fails_over_on_raw_provider_http_500() -> None: replace(agent, tags=(*agent.tags, "cost:free")) for agent in orchestrator.agents ] - result = orchestrator.proxy_completion( - { - "model": TaskOrchestrator.FREE_MODEL, - "messages": [{"role": "user", "content": "use the tool"}], - "tools": [{"type": "function", "function": {"name": "inspect"}}], - } - ) + with pytest.raises(ProviderUpstreamError) as caught: + orchestrator.proxy_completion( + { + "model": TaskOrchestrator.FREE_MODEL, + "messages": [{"role": "user", "content": "use the tool"}], + "tools": [{"type": "function", "function": {"name": "inspect"}}], + } + ) - assert result["model"] == "fallback-model" - assert [agent_id for agent_id, _ in client.calls] == [ - "primary_agent", - "fallback_agent", - ] + assert caught.value.provider_status == 500 + assert caught.value.detail["terminal_reason"] == "terminal_provider_failure" + assert caught.value.detail["attempts"][0]["failover_decision"] == ( + "sticky_candidate_failure" + ) + assert [agent_id for agent_id, _ in client.calls] == ["primary_agent"] def test_free_virtual_model_exhaustion_reports_bounded_attempt_evidence() -> None: """Exhausted free passthrough keeps typed candidate evidence on the final 502.""" From 5497c03292d42d64341aa0512c1f46a443a95cd9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 15:22:12 +0900 Subject: [PATCH 10/16] fix(passthrough): stop ambiguous HTTP replay --- contextual_orchestrator/orchestrator.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 3b43310ff..d1826aabf 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -163,6 +163,13 @@ def _request_endpoint_partition() -> str: _LOGGER = logging.getLogger(__name__) MAX_LOCAL_CONCURRENCY = 64 _PASSTHROUGH_UNAVAILABLE_STATUS = frozenset({404, 410, 413}) +# RFC 9110 section 9.2.2 permits an automatic non-idempotent retry only when +# the client knows the original request was never applied. These statuses +# describe a request rejection; generic origin/gateway failures (500/502/504) +# and the non-standard 529 do not provide that evidence and stay sticky. +_PASSTHROUGH_REJECTED_STATUS = _PASSTHROUGH_UNAVAILABLE_STATUS | frozenset( + {408, 409, 425, 429, 503} +) _PROVIDER_ERROR_CHAIN_LIMIT = 8 _PROVIDER_TOOL_DESCRIPTION_LIMIT_MESSAGE = ( "each tool.function.description must be at most 1024 characters" @@ -1626,13 +1633,11 @@ def _is_passthrough_failover_error(exc: BaseException) -> bool: return False seen.add(id(current)) if isinstance(current, ProviderUpstreamError): - if current.provider_status in ( - _PASSTHROUGH_UNAVAILABLE_STATUS | TRANSIENT_HTTP_STATUS - ): + if current.provider_status in _PASSTHROUGH_REJECTED_STATUS: return True if ( isinstance(current, urllib.error.HTTPError) - and current.code in (_PASSTHROUGH_UNAVAILABLE_STATUS | TRANSIENT_HTTP_STATUS) + and current.code in _PASSTHROUGH_REJECTED_STATUS ): return True if ( From 6d36cb2a1943f667a09f9e207cd7f34deebd5c9b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 15:22:14 +0900 Subject: [PATCH 11/16] test(passthrough): cover ambiguous HTTP status evidence --- tests/test_passthrough_provider_failover.py | 26 +++++++++++---------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index ec014cc33..0bed1228f 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -1217,11 +1217,12 @@ def test_free_virtual_model_keeps_ambiguous_transport_502_sticky() -> None: -def test_free_virtual_model_keeps_http_500_sticky() -> None: - """HTTP 500 does not prove non-acceptance, so passthrough must not replay.""" +@pytest.mark.parametrize("status", [500, 502, 504, 529]) +def test_free_virtual_model_keeps_ambiguous_http_failure_sticky(status: int) -> None: + """Ambiguous HTTP failures do not prove non-acceptance and cannot replay.""" client = SequencedProxyClient( { - "primary_agent": _http_error(500), + "primary_agent": _http_error(status), "fallback_agent": {"model": "fallback-model"}, } ) @@ -1239,17 +1240,18 @@ def test_free_virtual_model_keeps_http_500_sticky() -> None: } ) - assert caught.value.provider_status == 500 + assert caught.value.provider_status == status assert caught.value.detail["terminal_reason"] == "terminal_provider_failure" assert caught.value.detail["attempts"][0]["failover_decision"] == ( "sticky_candidate_failure" ) assert [agent_id for agent_id, _ in client.calls] == ["primary_agent"] -def test_free_virtual_model_exhaustion_reports_bounded_attempt_evidence() -> None: - """Exhausted free passthrough keeps typed candidate evidence on the final 502.""" + +def test_free_virtual_model_stops_after_a_rejection_then_ambiguous_failure() -> None: + """One proved rejection may advance, but an ambiguous next failure is sticky.""" client = SequencedProxyClient( - {"primary_agent": _http_error(500), "fallback_agent": _http_error(500)} + {"primary_agent": _http_error(429), "fallback_agent": _http_error(500)} ) orchestrator = _build(client) orchestrator.agents = [ @@ -1272,7 +1274,7 @@ def test_free_virtual_model_exhaustion_reports_bounded_attempt_evidence() -> Non ) assert caught.value.agent_id == "fallback_agent" - assert caught.value.detail["terminal_reason"] == "eligible_candidates_exhausted" + assert caught.value.detail["terminal_reason"] == "terminal_provider_failure" assert caught.value.detail["selected_candidate_ids"] == [ "primary_agent", "fallback_agent", @@ -1283,9 +1285,9 @@ def test_free_virtual_model_exhaustion_reports_bounded_attempt_evidence() -> Non "model": "primary-model", "provider_name": "unreported", "attempt_number": 1, - "error_code": "api_error", - "client_status": 502, - "provider_status": 500, + "error_code": "rate_limit_exceeded", + "client_status": 429, + "provider_status": 429, "retryable": True, "transport": "passthrough", "phase": "provider_response", @@ -1302,7 +1304,7 @@ def test_free_virtual_model_exhaustion_reports_bounded_attempt_evidence() -> Non "retryable": True, "transport": "passthrough", "phase": "provider_response", - "failover_decision": "eligible_candidates_exhausted", + "failover_decision": "sticky_candidate_failure", }, ] assert [agent_id for agent_id, _ in client.calls] == ["primary_agent", "fallback_agent"] From 15580af54306edb5ec270570941c1222b4572075 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 15:22:15 +0900 Subject: [PATCH 12/16] docs(gap): bind passthrough replay to RFC 9110 --- docs/product-technical-gap-baseline.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 3b34cbd74..f239e2964 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -46,9 +46,12 @@ The worktree change makes one surgical contract extension: - `proxy_completion()` now classifies every caught passthrough provider exception before deciding whether failover is permitted; - `orchestrator/free` virtual passthrough advances only after evidence that - proves non-acceptance, such as an explicit retryable upstream HTTP response - or temporary pre-request DNS failure. Raw timeout and generic transport 502 - outcomes remain sticky to prevent duplicate completion and unreported usage; + proves non-acceptance, such as an RFC-defined request rejection or temporary + pre-request DNS failure. RFC 9110 section 9.2.2 does not authorize automatic + replay of a non-idempotent request from retryability alone. Generic + 500/502/504 responses and the non-standard 529 therefore remain sticky, + alongside raw timeout and generic transport failures, to prevent duplicate + completion and unreported usage; - sticky failures now record the distinct failover decision `sticky_candidate_failure` instead of incorrectly reusing `eligible_candidates_exhausted`, while explicit concrete-model requests @@ -70,8 +73,10 @@ owner fix. - Added focused regressions proving ambiguous raw/classified transport errors remain sticky in both the in-process free-model loop and real - `/v1/chat/completions` HTTP path, while explicit retryable HTTP 500 and - temporary pre-request DNS evidence retain bounded failover. + `/v1/chat/completions` HTTP path. A follow-up RED contract found that HTTP + 500 still replayed; the repair also keeps 502, 504, and 529 sticky while + retaining bounded failover for explicit rejection and temporary pre-request + DNS evidence. - `.venv/bin/python -m pytest tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py -q` -> `98 passed in 10.64s` - `.venv/bin/python -m pytest tests/test_provider_error_taxonomy.py tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py -q` From a50c3d8f7da587ccecf5b42c36a4049b8438e423 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 15:22:17 +0900 Subject: [PATCH 13/16] docs(changelog): record fail-closed HTTP replay --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f83170a85..64fd113b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,10 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) - Passthrough attempt receipts no longer infer public provider labels from endpoint hostnames. Ambiguous timeout/connection failures now report the neutral `transport` phase and remain sticky even for `orchestrator/free`, - preventing duplicate completion and unreported upstream usage. + preventing duplicate completion and unreported upstream usage. Generic + HTTP 500/502/504 and non-standard 529 responses are sticky as well: an HTTP + retry classification alone does not prove that a non-idempotent completion + request was never applied. - Workflow workers now preserve the caller message array exactly once, while the added envelope carries only the subtask and Conductor-style prior-step access list instead of duplicating the task or source attachments. From e2641c16a82816e15f12013efad7fe50e94a3333 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 16:10:15 +0900 Subject: [PATCH 14/16] fix(passthrough): preserve validation transport evidence --- contextual_orchestrator/orchestrator.py | 10 +++++++++- tests/test_provider_reliability.py | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index d1826aabf..b613a03b6 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -2541,7 +2541,15 @@ def _proxy_send( ) if agent.base_url.startswith("mock://"): return self._mock_raw(agent, normalized_endpoint, payload) - destination = self._validate_provider(agent) # pragma: no cover + try: + destination = self._validate_provider(agent) # pragma: no cover + except ProviderUpstreamError as exc: + raise classify_provider_failure( + exc, + agent_id=agent.id, + model=agent.model, + transport="passthrough", + ) from None parsed_provider = urlparse(agent.base_url) operation_name = { "chat/completions": "chat", diff --git a/tests/test_provider_reliability.py b/tests/test_provider_reliability.py index e424f0cb9..638b8023c 100644 --- a/tests/test_provider_reliability.py +++ b/tests/test_provider_reliability.py @@ -798,6 +798,31 @@ def test_unallowlisted_provider_host_is_classified_upstream_error() -> None: assert "blocked.example" not in str(excinfo.value) +def test_passthrough_allowlist_failure_reports_passthrough_transport() -> None: + """Passthrough validation evidence must name the surface that invoked it.""" + client = ModelClient(allowed_provider_hosts={"ok.example"}) + agent = ModelAgent( + "blocked_agent", + "blocked-model", + base_url="https://blocked.example/v1", + credential_key="MODEL_KEY", + ) + backend = InMemoryCredentialBackend() + backend.set("MODEL_KEY", "sk-host-check") + set_backend(backend) + try: + with pytest.raises(ProviderUpstreamError) as excinfo: + client.proxy_send_once( + agent, + "chat/completions", + {"messages": [{"role": "user", "content": "hello"}]}, + ) + finally: + set_backend(None) + + assert excinfo.value.transport == "passthrough" + + def test_free_model_advances_past_unallowlisted_provider_host() -> None: """A host-allowlist miss must skip to the next free candidate, not 500.""" calls: list[str] = [] From be6baa0a844a243656d1360114768768f8b596ea Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 10 Sep 2026 13:53:51 +0900 Subject: [PATCH 15/16] docs(passthrough): preserve incident evidence outside cumulative records Keep every production and test blob from #1049 e2641c16 unchanged. Relocate that PR's dated baseline entry and Unreleased note into one doctoring fragment, while retaining current main aade4093 CHANGELOG and baseline by exact blob SHA. This ordinary child addresses cumulative-document conflicts without overwriting newer runtime, dependency or evidence changes. Source-head CI results remain historical; merged-head verification is separate. --- CHANGELOG.md | 37 +++- ...2026-09-04-passthrough-failure-evidence.md | 112 ++++++++++++ docs/product-technical-gap-baseline.md | 165 ++++++++---------- 3 files changed, 214 insertions(+), 100 deletions(-) create mode 100644 docs/doctoring/2026-09-04-passthrough-failure-evidence.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 64fd113b6..5adce934a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,13 +20,36 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed -- Passthrough attempt receipts no longer infer public provider labels from - endpoint hostnames. Ambiguous timeout/connection failures now report the - neutral `transport` phase and remain sticky even for `orchestrator/free`, - preventing duplicate completion and unreported upstream usage. Generic - HTTP 500/502/504 and non-standard 529 responses are sticky as well: an HTTP - retry classification alone does not prove that a non-idempotent completion - request was never applied. +- Virtual `orchestrator/free` structured completions (`response_format`, no + tools/stream) fail over a retryable synthesizer 502/429 onto the next + eligible free worker and attach request-scoped eligible/attempted + receipts. Concrete model pins stay sticky. Default model timeout remains + null (issue #1045; Inkspan Noema job 101628090366 on base `414f2297`). +- Review-sidecar `orchestrator/free` admission now treats a CI-seeded + `OPENCODE_ZEN_API_KEY` as an authorized free-pool source. Honest-free + OpenCode Zen and OpenCode Go rows can enter `G ∩ P ∩ R`; `OPENAI_API_KEY` + remains registered for global discovery and is still excluded from the + review free pool (Noema 429 on `google/gemma-4-31b-it:free` in PR #1094 + while Zen/Go evidence was dropped before routing). +- `OPENCODE_ZEN_API_KEY` is documented as the shared KV credential for both + OpenCode Zen and OpenCode Go catalogs; registering it once discovers both + accounts. +- Virtual selectors (`orchestrator/free`, `orchestrator/auto`, + `contextual-orchestrator`) keep tools and streaming on Fugu route / + TRINITY-Conductor conduct. A tools array no longer ejects those calls into + single-agent passthrough, so a failed worker is re-selected on the control + plane (incident: ContextualWisdomLab/.github run 34079284863, Strix step 23). + A worker `tool_calls` payload is returned as Chat Completions `tool_calls` + instead of being treated as missing assistant text. Concrete model ids + remain a debug pin. Psychometric θ̂/RMSE stays an equal-budget score of + those paper paths, not a separate router. +- Streamed `/v1/responses` now emits OpenAI `response.reasoning_text.*` + events for TRINITY thinker/worker/verifier and Conductor step outputs, + while `response.reasoning_summary_*` stays the paper-role stage summary. + The synthesizer answer remains `output_text`. Chat Completions, audio, + image, video, embeddings, and rerank use the same worker re-selection + but cannot emit those reasoning events, so only the modality result is + returned. - Workflow workers now preserve the caller message array exactly once, while the added envelope carries only the subtask and Conductor-style prior-step access list instead of duplicating the task or source attachments. diff --git a/docs/doctoring/2026-09-04-passthrough-failure-evidence.md b/docs/doctoring/2026-09-04-passthrough-failure-evidence.md new file mode 100644 index 000000000..1ca01f1bf --- /dev/null +++ b/docs/doctoring/2026-09-04-passthrough-failure-evidence.md @@ -0,0 +1,112 @@ +# PR #1049 passthrough failure evidence + +Relocated on 2026-09-10 from the PR's cumulative baseline and changelog to +preserve current main's records without changing any production or test blob. +The dated observations and earlier validation below are historical evidence, +not a fresh integrated-head, release, or deployment claim. The complete source +history remains on PR #1049, including head +`e2641c16a82816e15f12013efad7fe50e94a3333`. + +## Changelog fragment — Unreleased + +- Passthrough attempt receipts no longer infer public provider labels from + endpoint hostnames. Ambiguous timeout/connection failures now report the + neutral `transport` phase and remain sticky even for `orchestrator/free`, + preventing duplicate completion and unreported upstream usage. Generic + HTTP 500/502/504 and non-standard 529 responses are sticky as well: an HTTP + retry classification alone does not prove that a non-idempotent completion + request was never applied. + +## 2026-09-04 Autonomous Commercialization Loop: issue #1045 root-cause fix for orchestrator/free passthrough 502 evidence + +Observation time: 2026-09-04 Asia/Seoul. + +GitHub authentication was re-verified first with `gh api user`. The primary +checkout was dirty, so work continued in a clean linked worktree at +`.worktrees/commercial-loop-20260904-issue1045`. Open PR heads and prior +`commercial-loop-*` worktrees were re-fetched before editing. No existing +open PR head covered this exact contract: PR [#1046](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1046) +was only queued behind hosted checks for an unrelated EgressWeave SSRF change, +while the active `orchestrator/free` queue items [#1028](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1028) +and [#993](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/993) +were preserved as distinct in-flight contracts. The highest-leverage +independent unit was therefore issue +[#1045](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/1045): +typed attempt evidence and bounded failover for long `orchestrator/free` +tool-loop transport failures. + +### Root cause confirmed on current `main` + +The live Noema review incidents in issue `#1045` reproduced the single-agent +tool-loop path, not `route_once()`. `/v1/chat/completions` tool-bearing +requests call `proxy_completion(..., single_agent=True)`, which performs +virtual-model passthrough failover inside `TaskOrchestrator.proxy_completion`. + +Current-head RCA: + +- raw HTTP passthrough failures were already classified and could participate + in bounded failover, but raw provider transport exceptions such as + `TimeoutError`, `urllib.error.URLError`, `ConnectionError`, and wrapped DNS + failures were not classified inside the multi-candidate passthrough loop; +- `classify_provider_failure()` already mapped those raw exceptions to bounded + typed 502 surfaces, but `proxy_completion()` only invoked the classifier for + `HTTPError` and already-classified `ProviderUpstreamError` instances; +- raw transport failures had no request-scoped attempt receipt explaining why + the gateway stopped. They remain non-replayable because a timeout or generic + connection failure does not prove that the provider rejected the request + before accepting work or usage. + +### Local fix completed + +The worktree change makes one surgical contract extension: + +- `proxy_completion()` now classifies every caught passthrough provider + exception before deciding whether failover is permitted; +- `orchestrator/free` virtual passthrough advances only after evidence that + proves non-acceptance, such as an RFC-defined request rejection or temporary + pre-request DNS failure. RFC 9110 section 9.2.2 does not authorize automatic + replay of a non-idempotent request from retryability alone. Generic + 500/502/504 responses and the non-standard 529 therefore remain sticky, + alongside raw timeout and generic transport failures, to prevent duplicate + completion and unreported usage; +- sticky failures now record the distinct failover decision + `sticky_candidate_failure` instead of incorrectly reusing + `eligible_candidates_exhausted`, while explicit concrete-model requests + remain single-provider sticky. + +The public error detail remains bounded and secret-safe: no credentials, raw +provider bodies, prompt text, or inferred endpoint hostnames are emitted. +Unclassified connection/timeout outcomes use lifecycle phase `transport`; +only explicit TLS failures use `connecting`. + +### Exact local verification + +The 2026-09-08 current-head review repair added three explicit acceptance +boundaries: no replay after an ambiguous transport outcome, no inferred +endpoint hostname in public attempt evidence, and no invented `connecting` +phase for an outcome whose lifecycle stage is unknown. The revised tests first +failed as expected (`3 failed, 61 passed`) and then passed after the minimal +owner fix. + +- Added focused regressions proving ambiguous raw/classified transport errors + remain sticky in both the in-process free-model loop and real + `/v1/chat/completions` HTTP path. A follow-up RED contract found that HTTP + 500 still replayed; the repair also keeps 502, 504, and 529 sticky while + retaining bounded failover for explicit rejection and temporary pre-request + DNS evidence. +- `.venv/bin/python -m pytest tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py -q` + -> `98 passed in 10.64s` +- `.venv/bin/python -m pytest tests/test_provider_error_taxonomy.py tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py -q` + -> `119 passed in 11.61s` +- `uvx ruff check --select E4,E7,E9,F contextual_orchestrator/orchestrator.py tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py` + and `git diff --check` -> success. + +### Branch-local quality note + +- `uv run ruff check contextual_orchestrator/orchestrator.py tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py` + could not run in this worktree because the pinned environment does not + currently expose a `ruff` executable (`No such file or directory`). + +Hosted exact-head checks, protected merge, and the unchanged LifeOS/Noema +consumer canary remain future steps because this invocation stopped at one +completed local root-cause work unit, per the hourly-loop boundary. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index f239e2964..ca6a97159 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,98 +1,53 @@ # Contextual Orchestrator: Product & Technical Gap Baseline -## 2026-09-04 Autonomous Commercialization Loop: issue #1045 root-cause fix for orchestrator/free passthrough 502 evidence - -Observation time: 2026-09-04 Asia/Seoul. - -GitHub authentication was re-verified first with `gh api user`. The primary -checkout was dirty, so work continued in a clean linked worktree at -`.worktrees/commercial-loop-20260904-issue1045`. Open PR heads and prior -`commercial-loop-*` worktrees were re-fetched before editing. No existing -open PR head covered this exact contract: PR [#1046](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1046) -was only queued behind hosted checks for an unrelated EgressWeave SSRF change, -while the active `orchestrator/free` queue items [#1028](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1028) -and [#993](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/993) -were preserved as distinct in-flight contracts. The highest-leverage -independent unit was therefore issue -[#1045](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/1045): -typed attempt evidence and bounded failover for long `orchestrator/free` -tool-loop transport failures. - -### Root cause confirmed on current `main` - -The live Noema review incidents in issue `#1045` reproduced the single-agent -tool-loop path, not `route_once()`. `/v1/chat/completions` tool-bearing -requests call `proxy_completion(..., single_agent=True)`, which performs -virtual-model passthrough failover inside `TaskOrchestrator.proxy_completion`. - -Current-head RCA: - -- raw HTTP passthrough failures were already classified and could participate - in bounded failover, but raw provider transport exceptions such as - `TimeoutError`, `urllib.error.URLError`, `ConnectionError`, and wrapped DNS - failures were not classified inside the multi-candidate passthrough loop; -- `classify_provider_failure()` already mapped those raw exceptions to bounded - typed 502 surfaces, but `proxy_completion()` only invoked the classifier for - `HTTPError` and already-classified `ProviderUpstreamError` instances; -- raw transport failures had no request-scoped attempt receipt explaining why - the gateway stopped. They remain non-replayable because a timeout or generic - connection failure does not prove that the provider rejected the request - before accepting work or usage. - -### Local fix completed - -The worktree change makes one surgical contract extension: - -- `proxy_completion()` now classifies every caught passthrough provider - exception before deciding whether failover is permitted; -- `orchestrator/free` virtual passthrough advances only after evidence that - proves non-acceptance, such as an RFC-defined request rejection or temporary - pre-request DNS failure. RFC 9110 section 9.2.2 does not authorize automatic - replay of a non-idempotent request from retryability alone. Generic - 500/502/504 responses and the non-standard 529 therefore remain sticky, - alongside raw timeout and generic transport failures, to prevent duplicate - completion and unreported usage; -- sticky failures now record the distinct failover decision - `sticky_candidate_failure` instead of incorrectly reusing - `eligible_candidates_exhausted`, while explicit concrete-model requests - remain single-provider sticky. - -The public error detail remains bounded and secret-safe: no credentials, raw -provider bodies, prompt text, or inferred endpoint hostnames are emitted. -Unclassified connection/timeout outcomes use lifecycle phase `transport`; -only explicit TLS failures use `connecting`. - -### Exact local verification - -The 2026-09-08 current-head review repair added three explicit acceptance -boundaries: no replay after an ambiguous transport outcome, no inferred -endpoint hostname in public attempt evidence, and no invented `connecting` -phase for an outcome whose lifecycle stage is unknown. The revised tests first -failed as expected (`3 failed, 61 passed`) and then passed after the minimal -owner fix. - -- Added focused regressions proving ambiguous raw/classified transport errors - remain sticky in both the in-process free-model loop and real - `/v1/chat/completions` HTTP path. A follow-up RED contract found that HTTP - 500 still replayed; the repair also keeps 502, 504, and 529 sticky while - retaining bounded failover for explicit rejection and temporary pre-request - DNS evidence. -- `.venv/bin/python -m pytest tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py -q` - -> `98 passed in 10.64s` -- `.venv/bin/python -m pytest tests/test_provider_error_taxonomy.py tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py -q` - -> `119 passed in 11.61s` -- `uvx ruff check --select E4,E7,E9,F contextual_orchestrator/orchestrator.py tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py` - and `git diff --check` -> success. - -### Branch-local quality note - -- `uv run ruff check contextual_orchestrator/orchestrator.py tests/test_passthrough_provider_failover.py tests/test_openai_passthrough.py` - could not run in this worktree because the pinned environment does not - currently expose a `ruff` executable (`No such file or directory`). - -Hosted exact-head checks, protected merge, and the unchanged LifeOS/Noema -consumer canary remain future steps because this invocation stopped at one -completed local root-cause work unit, per the hourly-loop boundary. +## 2026-09-09 Request-to-provider diagnostic correlation + +PR #1105 candidate `f588ca8c093ea7c9a86b857685bfbb1ce3c05fe2` connects HTTP +identity to seven provider diagnostic events and the successful request summary. +The predecessor `7b7b32006e7ae498db2ee781bd423d9c7b6774fc` completed its full +suite with 3399 passed, 2 skipped (1594.26s, exit 0). Follow-up code at +`6b24fe96` passed 81 focused tests, including actual same-socket reuse and +overlapping same-session HTTP requests with two distinct server thread IDs. +The integrated `f588ca8c` suite terminated with 3399 passed, 2 skipped and +1 failure (1767.82s, exit 1): certifi CA loading raised InterruptedError before +the Responses HTTP test could send a request. Same-head isolated HTTP tests +then passed 4/4 in 20.05s. The original failure remains unresolved evidence; +do not infer full-suite success from the isolated pass. + +Actual output from all seven provider diagnostic functions at +`7cb97ec8e2979d35b72c86a801ab18f0fd9c213d` was cross-checked with the central +PR #2053 sanitizer at `fc0ab87bfde0900461034be815046914f9019bfc`: trusted IDs +survived, untrusted error-body IDs and text were omitted, and malformed IDs and +embedded newlines were rejected. This isolated contract test does not establish +collector adoption. The later sanitizer `4a0125bf9f50d4d26355249011df03c3735b3abc` +also preserved an actual local GET `/healthz` 200 summary from producer +`f588ca8c`, including its request ID, while rejecting extra detail and an +unapproved path. This supersedes the earlier missing-success-summary limitation +for that route/state only, not every HTTP route. The +[runbook](doctoring/provider_request_correlation.md) records +RED evidence, exact revisions, cleanup tests, and bounded visual inspection. +Not yet established: every orchestration worker path, integrated full-suite and +security gates, protected release, live collector adoption, or customer KPI +improvement. Diagnostic traceability is a prerequisite for attributing failures, +not a substitute for accuracy or decision-latency measurements. + +## 2026-09-08 error-response correlation repair + +ConceptWeave run 33938445050, job 101256562088, preserves a client-side HTTP +500 with request ID `175d6d59c5294b0e8a21548193b90482`. Its surviving artifact +9969701340 contains gateway stderr but only generic request-failure messages; +it cannot correlate that ID to an internal cause. The job installed CO source +`2e414d15ba58f28597751b625a8a2f00fc9fadcf`. This is not proof of free-pool +exhaustion, a disappeared run, or a currently released fix. + +The same correlation gap was reproduced on main +`414f22973658c4ddc3d4320fcf7acd9b4e8ba991`: the common HTTP error response had +a generated ID absent from its log. The proposed repair generates one ID for +both response and warning, prevents detail fields from overriding it, and logs +neither session values nor error details. RED: one missing-correlation failure; +GREEN: 45 telemetry tests passed in 6.58 seconds. This improves future failure +correlation only; it does not recover the historical exception, cover every +streaming-error path, or prove immutable publication or deployed behavior. ## 2026-09-01 Autonomous Commercialization Loop: PR #970 Merge, Token Accounting & Cost Gateway Harmonization @@ -211,6 +166,30 @@ Focused and proportional verification run on the exact local head: Total exact local evidence for this unit: `163 passed` across the touched discovery, persistence, client-boundary, CLI, and contract surfaces. +## 2026-09-04 Bytez discovery: filtered empty catalogs and upstream 5xx are distinct fail-closed states + +At `origin/main` `60c562de`, an authenticated, bounded live probe loaded only +`BYTEZ_API_KEY` from the operator's local `.env` and emitted no token, response +body, or upstream error text. The earlier `task=chat` HTTP 200 response with an +empty `output` is a successful transport with no usable catalog, whereas an +unfiltered HTTP 500 is an upstream server failure. A fresh probe found the +upstream condition had widened: `chat`, `text-generation`, the other documented +chat-completion-compatible task filters, and the unfiltered request all returned +HTTP 500 with a small JSON object and empty `output`. Raw-token and `Key`-prefixed +authorization produced the same status, so the prefix does not explain the +failure. + +The canonical discovery boundary now queries only `task=chat` and then +`task=text-generation`. Bytez documents both as compatible with its OpenAI-style +chat-completions API; audio, image, and video task catalogs are intentionally not +admitted to the ordinary text-chat pool. Discovery never uses the failing +unfiltered endpoint as a fallback. A non-empty filtered catalog is parsed through +the existing Bytez model contract. If both filtered catalogs are empty or fail, +refresh records only task, outcome, model count, and an allowlisted error code; +it retains the durable last-known-good catalog and fails closed when none exists. +The current upstream 5xx therefore remains a first-bootstrap blocker, not a +reason to fabricate usable models. + ## 2026-08-30 provider-catalog-sync: no scheduled run has succeeded in 5 days over one provider; workflow check was too strict `provider-catalog-sync.yml` (run `33312773022`, job `99260685380`) failed with `credential From 6d519b1f48aeeb50d01fc77d8430320c82c58345 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 18 Sep 2026 07:23:20 +0900 Subject: [PATCH 16/16] fix(passthrough): align restack with virtual route gate and sticky receipts Attach attempt evidence on explicit sticky failures, keep raw HTTPError identity for named models, and adapt HTTP/exhaustion tests to main's tool-route gate and RFC-bounded rejection set. Co-authored-by: Cursor --- contextual_orchestrator/orchestrator.py | 61 +++++++++++++++------ tests/test_openai_passthrough.py | 52 ++++++++---------- tests/test_passthrough_provider_failover.py | 11 ++-- 3 files changed, 71 insertions(+), 53 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 0343751ad..fcaa7d81d 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -6122,7 +6122,7 @@ def proxy_completion( self._record_failure(agent.id) if agent.group_name: self._group_router.observe_failure(agent.id) - raise ProviderUpstreamError( + unknown = ProviderUpstreamError( agent_id=agent.id, model=agent.model, error_code=PROVIDER_OUTCOME_UNKNOWN_CODE, @@ -6133,10 +6133,37 @@ def proxy_completion( client_status=502, retryable=False, transport="passthrough", + ) + raise _set_passthrough_attempt_evidence( + unknown, + selected_candidate_ids=[agent.id], + attempts=[ + _passthrough_attempt_record( + unknown, + provider_name=agent.provider_name.strip() or "unreported", + attempt_number=1, + failover_decision="sticky_candidate_failure", + ) + ], + terminal_reason="terminal_provider_failure", ) from None request_too_large = _is_request_too_large_error(exc) if measured and not request_too_large: self._group_router.observe_failure(agent.id) + if isinstance(exc, ProviderUpstreamError): + raise _set_passthrough_attempt_evidence( + exc, + selected_candidate_ids=[agent.id], + attempts=[ + _passthrough_attempt_record( + exc, + provider_name=agent.provider_name.strip() or "unreported", + attempt_number=1, + failover_decision="sticky_candidate_failure", + ) + ], + terminal_reason="terminal_provider_failure", + ) from None raise if measured: self._group_router.observe_success( @@ -6339,24 +6366,22 @@ def proxy_completion( attempts=attempt_receipts, terminal_reason="terminal_provider_failure", ) from None - if isinstance(exc, (urllib.error.HTTPError, ProviderUpstreamError)): - attempt_receipts.append( - _passthrough_attempt_record( - classified, - provider_name=( - candidate.provider_name.strip() or "unreported" - ), - attempt_number=len(attempt_receipts) + 1, - failover_decision="sticky_candidate_failure", - ) - ) - raise _set_passthrough_attempt_evidence( + attempt_receipts.append( + _passthrough_attempt_record( classified, - selected_candidate_ids=selected_candidate_ids, - attempts=attempt_receipts, - terminal_reason="terminal_provider_failure", - ) from None - raise + provider_name=( + candidate.provider_name.strip() or "unreported" + ), + attempt_number=len(attempt_receipts) + 1, + failover_decision="sticky_candidate_failure", + ) + ) + raise _set_passthrough_attempt_evidence( + classified, + selected_candidate_ids=selected_candidate_ids, + attempts=attempt_receipts, + terminal_reason="terminal_provider_failure", + ) from None attempt_receipts.append( _passthrough_attempt_record( classified, diff --git a/tests/test_openai_passthrough.py b/tests/test_openai_passthrough.py index 4e9e1fc16..4e783f26c 100644 --- a/tests/test_openai_passthrough.py +++ b/tests/test_openai_passthrough.py @@ -511,10 +511,16 @@ def proxy_send_once(self, agent, endpoint, payload): assert "synthetic provider outage" not in json.dumps(body) -def test_http_free_tool_passthrough_exposes_bounded_attempt_evidence_on_502() -> None: - """Free tool-loop 502s keep request-scoped candidate evidence on the wire.""" +def test_http_named_tool_passthrough_exposes_bounded_attempt_evidence_on_502() -> None: + """Named tool passthrough keeps request-scoped candidate evidence on the wire. - class ConnectingFreePool(ModelClient): + Virtual ``orchestrator/free`` + tools now take the Fugu route/conduct path on + main (``named_tool_passthrough`` requires a concrete model). Free-pool + multi-candidate sticky HTTP coverage remains in + ``tests/test_passthrough_provider_failover.py``. + """ + + class ConnectingNamedPool(ModelClient): def proxy_send_once(self, agent, endpoint, payload): del endpoint, payload raise ProviderUpstreamError( @@ -548,7 +554,7 @@ def proxy_send_once(self, agent, endpoint, payload): priority=1, ), ], - client=ConnectingFreePool(), # type: ignore[arg-type] + client=ConnectingNamedPool(), # type: ignore[arg-type] ) server = build_server( orchestrator, port=0, security=SecurityConfig(auth_token=token) @@ -558,7 +564,7 @@ def proxy_send_once(self, agent, endpoint, payload): status, body = _post( f"http://127.0.0.1:{server.server_address[1]}/v1/chat/completions", { - "model": TaskOrchestrator.FREE_MODEL, + "model": "free-primary-model", "messages": [{"role": "user", "content": "use the tool"}], "tools": [{"type": "function", "function": {"name": "inspect"}}], }, @@ -570,10 +576,7 @@ def proxy_send_once(self, agent, endpoint, payload): assert status == 502 assert body["error"]["code"] == "provider_connection_error" - assert body["error"]["detail"]["selected_candidate_ids"] == [ - "free_primary", - "free_backup", - ] + assert body["error"]["detail"]["selected_candidate_ids"] == ["free_primary"] assert body["error"]["detail"]["terminal_reason"] == "terminal_provider_failure" assert body["error"]["detail"]["attempts"] == [ { @@ -593,10 +596,10 @@ def proxy_send_once(self, agent, endpoint, payload): assert "use the tool" not in json.dumps(body) -def test_http_free_tool_passthrough_raw_timeout_does_not_replay() -> None: - """HTTP passthrough keeps an ambiguous timeout on its selected candidate.""" +def test_http_named_tool_passthrough_raw_timeout_does_not_replay() -> None: + """Named tool passthrough keeps an ambiguous timeout on its selected candidate.""" - class TimeoutFreePool(ModelClient): + class TimeoutNamedPool(ModelClient): def proxy_send_once(self, agent, endpoint, payload): del endpoint, payload if agent.id == "free_primary": @@ -623,7 +626,7 @@ def proxy_send_once(self, agent, endpoint, payload): priority=1, ), ], - client=TimeoutFreePool(), # type: ignore[arg-type] + client=TimeoutNamedPool(), # type: ignore[arg-type] ) server = build_server( orchestrator, port=0, security=SecurityConfig(auth_token=token) @@ -633,7 +636,7 @@ def proxy_send_once(self, agent, endpoint, payload): status, body = _post( f"http://127.0.0.1:{server.server_address[1]}/v1/chat/completions", { - "model": TaskOrchestrator.FREE_MODEL, + "model": "free-primary-model", "messages": [{"role": "user", "content": "use the tool"}], "tools": [{"type": "function", "function": {"name": "inspect"}}], }, @@ -644,23 +647,14 @@ def proxy_send_once(self, agent, endpoint, payload): server.server_close() assert status == 502 - assert body["error"]["code"] == "provider_connection_error" + assert body["error"]["code"] == "provider_outcome_unknown" + assert body["error"]["detail"]["retryable"] is False assert body["error"]["detail"]["terminal_reason"] == "terminal_provider_failure" - assert body["error"]["detail"]["attempts"] == [ - { - "agent_id": "free_primary", - "model": "free-primary-model", - "provider_name": "free-primary", - "attempt_number": 1, - "error_code": "provider_connection_error", - "client_status": 502, - "provider_status": None, - "retryable": True, - "transport": "passthrough", - "phase": "transport", - "failover_decision": "sticky_candidate_failure", - } + assert [item["agent_id"] for item in body["error"]["detail"]["attempts"]] == [ + "free_primary" ] + assert "provider timed out" not in json.dumps(body) + def test_http_chat_completions_accepts_response_format_and_passes_through() -> None: diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index 59f15d521..fb599c804 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -1543,13 +1543,12 @@ def test_suppressed_transient_context_does_not_authorize_failover() -> None: def test_all_candidates_chain_the_last_failure() -> None: """Exhaustion reports one stable gateway error with the final provider cause.""" final = _http_error(503) - # 500 here (not 429): this test is about which classified failure - # survives exhaustion, not rate-limiting -- a bare 429 with no - # Retry-After now assumes a short quota cooldown and waits, which would - # route this candidate's identity through the rate-limit-storm path - # instead of the plain exhaustion path this test actually exercises. + # 408 here (not 429 or 500): 429 can enter the rate-limit-storm wait path, + # and 500/502/504/529 stay sticky under the RFC-bounded rejection set from + # #1049. A proved request rejection (408) may advance, then the final 503 + # is reported on exhaustion without waiting. orchestrator = _build( - SequencedProxyClient({"primary_agent": _http_error(500), "fallback_agent": final}) + SequencedProxyClient({"primary_agent": _http_error(408), "fallback_agent": final}) ) with pytest.raises(ProviderUpstreamError) as caught: