From 54aa214060a20ccea2b22a24e1c82cef08f2d958 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:36:06 +0900 Subject: [PATCH 1/6] feat(review): add gated external gateway bootstrap port Keep provider-sidecar provisioning as default. External mode admits only a protected registered release adapter; the registry stays empty until the canonical CO contract is released. Test inference capability admission and secret-safe evidence with owner test doubles. Signed-off-by: Seongho Bae Commit-Message-Assisted-by: Codex (OpenAI Codex) --- .../orchestrator-free-sidecar/action.yml | 17 ++ .../external-review-gateway-admission.md | 67 ++++++ .../contextual_orchestrator_review_sidecar.sh | 8 + scripts/ci/external_review_gateway.py | 177 +++++++++++++++ tests/test_external_review_gateway.py | 212 ++++++++++++++++++ 5 files changed, 481 insertions(+) create mode 100644 docs/doctoring/external-review-gateway-admission.md create mode 100644 scripts/ci/external_review_gateway.py create mode 100644 tests/test_external_review_gateway.py diff --git a/.github/actions/orchestrator-free-sidecar/action.yml b/.github/actions/orchestrator-free-sidecar/action.yml index edddfe1bc3..7f07abfc84 100644 --- a/.github/actions/orchestrator-free-sidecar/action.yml +++ b/.github/actions/orchestrator-free-sidecar/action.yml @@ -1,6 +1,19 @@ name: Orchestrator free sidecar description: Provision the immutable contextual-orchestrator orchestrator/free gateway for a model-backed workflow. inputs: + gateway_mode: + description: Bootstrap mode; external remains blocked until a released CO adapter is registered. + required: false + default: "sidecar" + gateway_base_url: + description: Deployment-authorized HTTPS gateway origin; never sourced from PR content. + required: false + gateway_token_file: + description: Private runner-owned inference token file; never a raw credential input. + required: false + gateway_contract_revision: + description: Immutable released CO adapter revision registered by protected owner source. + required: false require_zdr: description: Require an attested Zero Data Retention route for private or internal content. required: false @@ -26,6 +39,10 @@ runs: - name: Provision contextual-orchestrator orchestrator/free shell: bash --noprofile --norc -e -o pipefail {0} env: + CONTEXTUAL_ORCHESTRATOR_GATEWAY_MODE: ${{ inputs.gateway_mode }} + CONTEXTUAL_ORCHESTRATOR_BASE_URL: ${{ inputs.gateway_base_url }} + CONTEXTUAL_ORCHESTRATOR_TOKEN_FILE: ${{ inputs.gateway_token_file }} + CONTEXTUAL_ORCHESTRATOR_GATEWAY_CONTRACT_REVISION: ${{ inputs.gateway_contract_revision }} CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: ${{ inputs.require_zdr }} ORCHESTRATOR_CATALOG_LIMIT: ${{ inputs.catalog_limit }} ORCHESTRATOR_CATALOG_ACCOUNT_CAP: ${{ inputs.catalog_account_cap }} diff --git a/docs/doctoring/external-review-gateway-admission.md b/docs/doctoring/external-review-gateway-admission.md new file mode 100644 index 0000000000..243550d5b3 --- /dev/null +++ b/docs/doctoring/external-review-gateway-admission.md @@ -0,0 +1,67 @@ +# External review gateway admission + +Status: proposed, disabled by default, with no registered released adapter. +Baseline: protected `.github` main `dd0b96feded94f66ecf59b25a5a9b58cfc8b4f69`. + +## Problem and scope + +The existing review bootstrap always starts a loopback CO instance using provider +credentials. An inference-token-only external deployment cannot use that path. +The CO inference APIs already expose authenticated model discovery and chat; +administrator `/readyz` access is neither necessary nor appropriate. + +CO PR #1084 proposes the consolidated request/evidence contract but is not a +released dependency. Its fixture, source and proposed branch are not imported +here. No production workflow opts into this change, and no provider request +or production credential is used in its tests. + +## Chosen boundary + +The existing composite action adds `gateway_mode`, default `sidecar`. Explicit +`external` branches before provider-secret bootstrap and admits only a protected +source-registered immutable contract adapter. The registry is empty, so every +real external invocation currently fails with `released_contract_unavailable` +before token access, network calls, checkout of CO, or readiness exports. +An environment value containing a full commit hash cannot authorize adoption. + +The owner probe port describes inference `/v1/models` discovery and capability +checks through `/v1/chat/completions`. It requires the exact `orchestrator/free` +alias plus JSON object/schema and tool-call evidence; a failed or missing +capability prevents partial readiness. The port accepts only an explicit HTTPS +origin and an owned, mode-0600, regular token file. It never exports a raw bearer. +Successful test-double observations produce only bounded capability evidence. + +TLS verification, redirect rejection, trusted origin authorization, secure token +opening and full response validation are obligations of the future released +adapter. They are not implemented HTTP transport in this delta. No claim of a +working external gateway or verified TLS follows from these port tests. + +## Alternatives and next owner work + +- An early return that exports a supplied URL/token path would silently bypass + all capability and private-data policy checks; rejected. +- Copying CO #1084's fixture or provider discovery into `.github` would create an + unreleased dependency or duplicate owner logic; rejected. +- A new gateway service is unnecessary: use CO's existing inference contract. + +After CO publishes its reviewed immutable contract, a separate protected change +must implement/register the released adapter and verify live external HTTPS, +token-only authentication, exact free-pool capability, and sanitized failure +evidence. It must retain failure classification without paid fallback and must +not impose a model-duration timeout. No code may infer provider retention from +a successful response: ZDR evidence is configured policy only. + +Every private review request must retain `zdr_only`, not just preflight. This +bootstrap exports that requirement for the future caller integration; it cannot +enforce it on an unrelated client's later HTTP requests. Strix's fixed-loopback +allowlist and Noema's private-origin exception need separate reviewed integration +before deployment. Default sidecar provisioning and those gates are unchanged. + +## Verification + +The baseline external-mode test failed because the legacy path demanded provider +secrets, and the port did not exist. Tests now exercise private-file/origin +admission, missing free inventory, each failed capability, unregistered revision, +safe output, and exceptions using owner test doubles. Synthetic data is confined +to unit tests. Full lifecycle evidence still requires protected review, release, +caller adoption, and a live exact-head review. diff --git a/scripts/ci/contextual_orchestrator_review_sidecar.sh b/scripts/ci/contextual_orchestrator_review_sidecar.sh index 38d9551a32..4a8bdc58eb 100755 --- a/scripts/ci/contextual_orchestrator_review_sidecar.sh +++ b/scripts/ci/contextual_orchestrator_review_sidecar.sh @@ -49,6 +49,14 @@ log() { printf '[contextual-orchestrator-sidecar] %s\n' "$*"; } fail() { log "error: $*" >&2; exit 1; } +case "${CONTEXTUAL_ORCHESTRATOR_GATEWAY_MODE:-sidecar}" in + sidecar) ;; + external) + exec "$sidecar_python" "$ORG_REPO_ROOT/scripts/ci/external_review_gateway.py" + ;; + *) fail "unsupported gateway mode" ;; +esac + # Require at least one of the five provider secrets so we never boot an empty # (or mock) pool. Missing individual secrets are allowed — discovery skips the # unregistered provider — matching the review gateway contract. diff --git a/scripts/ci/external_review_gateway.py b/scripts/ci/external_review_gateway.py new file mode 100644 index 0000000000..52630100b9 --- /dev/null +++ b/scripts/ci/external_review_gateway.py @@ -0,0 +1,177 @@ +"""Fail-closed external review admission awaiting a released CO adapter. + +This owner port does not implement or copy the proposed CO request contract. +Only a protected source change may register a reviewed immutable adapter. +""" + +from __future__ import annotations + +import json +import os +import stat +import tempfile +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Protocol +from urllib.parse import urlsplit + + +class GatewayAdmissionError(RuntimeError): + """Carry only an owner-defined, secret-free admission category.""" + + +@dataclass +class ExternalGatewayConfig: + """Bootstrap inputs from the trusted workflow, never PR-controlled values.""" + + base_url: str + token_file: Path + require_zdr: bool + + def validate(self) -> None: + """Require an HTTPS origin and private, owned, regular token file.""" + try: + parsed_url = urlsplit(self.base_url) + port_number = parsed_url.port + if ( + any( + ord(character) <= 32 or ord(character) == 127 + for character in self.base_url + ) + or parsed_url.scheme != "https" + or not parsed_url.hostname + or parsed_url.username is not None + or parsed_url.password is not None + or parsed_url.path not in {"", "/"} + or parsed_url.query + or parsed_url.fragment + or (port_number is not None and not 1 <= port_number <= 65535) + or type(self.require_zdr) is not bool + ): + raise ValueError + if any(ord(character) < 32 for character in str(self.token_file)): + raise ValueError + token_stat = self.token_file.lstat() + if ( + not stat.S_ISREG(token_stat.st_mode) + or stat.S_IMODE(token_stat.st_mode) != 0o600 + or token_stat.st_uid != os.geteuid() + or not 1 <= token_stat.st_size <= 8192 + ): + raise ValueError + except (OSError, ValueError): + raise GatewayAdmissionError("invalid_gateway_configuration") from None + + +class InferenceProbePort(Protocol): + """Released-adapter boundary; no provider credentials or admin readiness.""" + + def list_models(self) -> list[str]: + """Validate authenticated GET /v1/models and return exact model ids.""" + ... + + def probe_capability( + self, capability_name: str, *, model_name: str, require_zdr: bool + ) -> bool: + """Validate a released capability contract using POST /v1/chat/completions. + + The adapter must verify TLS, reject redirects, reopen the private token + without following symlinks, and validate response semantics. It must + not call /readyz or accept an HTTP 200 alone as capability evidence. + """ + ... + + +# CO #1084 is proposed, not a released adapter. Never populate this mapping +# from environment, a PR checkout, downloaded source, or an unverified SHA. +RELEASED_GATEWAY_ADAPTERS: dict[ + str, Callable[[ExternalGatewayConfig], InferenceProbePort] +] = {} + + +def verify_external_gateway( + gateway_config: ExternalGatewayConfig, probe_port: InferenceProbePort +) -> dict: + """Require every inference capability without partial readiness or fallback.""" + gateway_config.validate() + try: + if "orchestrator/free" not in probe_port.list_models(): + raise GatewayAdmissionError("free_pool_unavailable") + capability_results = {} + for capability_name in ("json_object", "json_schema", "tool_call"): + if ( + probe_port.probe_capability( + capability_name, + model_name="orchestrator/free", + require_zdr=gateway_config.require_zdr, + ) + is not True + ): + raise GatewayAdmissionError("capability_unavailable") + capability_results[capability_name] = "passed" + except GatewayAdmissionError: + raise + except Exception: # noqa: BLE001 - never expose transport, token or body details + raise GatewayAdmissionError("inference_probe_failed") from None + return { + "model": "orchestrator/free", + "capabilities": capability_results, + "private_requests_require_zdr": gateway_config.require_zdr, + "policy_evidence": "configured_gateway_policy_only", + } + + +def main() -> int: + """Admit only a source-registered released adapter and publish safe outputs.""" + adapter_revision = os.environ.get( + "CONTEXTUAL_ORCHESTRATOR_GATEWAY_CONTRACT_REVISION", "" + ) + adapter_factory = RELEASED_GATEWAY_ADAPTERS.get(adapter_revision) + if adapter_factory is None: + print("::error::released_contract_unavailable") + return 1 + try: + zdr_value = os.environ.get("CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR", "false") + if zdr_value not in {"true", "false"}: + raise GatewayAdmissionError("invalid_gateway_configuration") + gateway_config = ExternalGatewayConfig( + os.environ.get("CONTEXTUAL_ORCHESTRATOR_BASE_URL", ""), + Path(os.environ.get("CONTEXTUAL_ORCHESTRATOR_TOKEN_FILE", "")), + zdr_value == "true", + ) + gateway_config.validate() + evidence = verify_external_gateway( + gateway_config, adapter_factory(gateway_config) + ) + runner_temp = os.environ["RUNNER_TEMP"] + if not Path(runner_temp).is_absolute() or any( + ord(character) < 32 for character in runner_temp + ): + raise GatewayAdmissionError("invalid_output_location") + evidence_directory = Path( + tempfile.mkdtemp(prefix="external-review-", dir=runner_temp) + ) + evidence_path = evidence_directory / "preflight-evidence.json" + evidence_path.write_text( + json.dumps({**evidence, "contract_revision": adapter_revision}) + "\n" + ) + evidence_path.chmod(0o600) + with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as environment_file: + environment_file.write( + f"CONTEXTUAL_ORCHESTRATOR_BASE_URL={gateway_config.base_url.rstrip('/')}\n" + f"CONTEXTUAL_ORCHESTRATOR_TOKEN_FILE={gateway_config.token_file}\n" + f"CONTEXTUAL_ORCHESTRATOR_PREFLIGHT_EVIDENCE={evidence_path}\n" + f"CONTEXTUAL_ORCHESTRATOR_PRIVATE_REQUESTS_REQUIRE_ZDR={zdr_value}\n" + ) + except Exception: # noqa: BLE001 - bootstrap failures never reveal raw input + print("::error::external_gateway_admission_failed") + return 1 + print( + "External gateway inference preflight passed; private requests must retain ZDR policy." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_external_review_gateway.py b/tests/test_external_review_gateway.py new file mode 100644 index 0000000000..7b9626d172 --- /dev/null +++ b/tests/test_external_review_gateway.py @@ -0,0 +1,212 @@ +"""External bootstrap admission stays closed until a released adapter exists.""" + +import importlib +import json +import os +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +def gateway_module(): + """Load the owner port without importing a proposed CO implementation.""" + return importlib.import_module("scripts.ci.external_review_gateway") + + +def gateway_configuration(tmp_path): + """Create a private test credential file, never a provider credential.""" + token_file = tmp_path / "gateway.token" + token_file.write_text("unit-test-gateway-credential") + token_file.chmod(0o600) + return gateway_module().ExternalGatewayConfig( + base_url="https://gateway.example.invalid", + token_file=token_file, + require_zdr=True, + ) + + +def test_unreleased_external_mode_fails_before_provider_secret_bootstrap(tmp_path): + """Opt-in must not fall back to local provider bootstrap or export readiness.""" + output_file = tmp_path / "github-env" + command_result = subprocess.run( + ["bash", "scripts/ci/contextual_orchestrator_review_sidecar.sh"], + env={ + "PATH": os.environ["PATH"], + "RUNNER_TEMP": str(tmp_path), + "GITHUB_ENV": str(output_file), + "CONTEXTUAL_ORCHESTRATOR_GATEWAY_MODE": "external", + }, + capture_output=True, + text=True, + check=False, + ) + assert command_result.returncode == 1 + assert ( + "released_contract_unavailable" in command_result.stdout + command_result.stderr + ) + assert "provider secrets" not in command_result.stdout + command_result.stderr + assert not output_file.exists() + + +@pytest.mark.parametrize( + "base_url", + [ + "http://gateway.invalid", + "https://user@gateway.invalid", + "https://gateway.invalid/path", + "https://gateway.invalid?token=value", + "https://gateway.invalid/#fragment", + "https://gateway.invalid\n", + ], +) +def test_external_origin_rejects_ambiguous_or_insecure_configuration( + tmp_path, base_url +): + """No HTTP, credentials, path, query, fragment or control-byte origin is valid.""" + gateway_config = gateway_configuration(tmp_path) + gateway_config.base_url = base_url + with pytest.raises(gateway_module().GatewayAdmissionError): + gateway_config.validate() + + +def test_external_credentials_must_be_private_regular_owned_files(tmp_path): + """A world-readable or symlink credential cannot reach a probe adapter.""" + gateway_config = gateway_configuration(tmp_path) + gateway_config.token_file.chmod(0o644) + with pytest.raises(gateway_module().GatewayAdmissionError): + gateway_config.validate() + gateway_config.token_file.chmod(0o600) + token_link = tmp_path / "token-link" + token_link.symlink_to(gateway_config.token_file) + gateway_config.token_file = token_link + with pytest.raises(gateway_module().GatewayAdmissionError): + gateway_config.validate() + + +@pytest.mark.parametrize( + "missing_capability", ["inventory", "json_object", "json_schema", "tool_call"] +) +def test_probe_failure_never_exports_partial_readiness(tmp_path, missing_capability): + """Every required inference capability is part of one fail-closed result.""" + gateway_config = gateway_configuration(tmp_path) + probe_calls = [] + + def probe_capability(capability_name, *, model_name, require_zdr): + probe_calls.append((capability_name, model_name, require_zdr)) + return capability_name != missing_capability + + probe_port = SimpleNamespace( + list_models=lambda: ( + [] if missing_capability == "inventory" else ["orchestrator/free"] + ), + probe_capability=probe_capability, + ) + with pytest.raises(gateway_module().GatewayAdmissionError): + gateway_module().verify_external_gateway(gateway_config, probe_port) + assert all( + model_name == "orchestrator/free" and require_zdr is True + for _, model_name, require_zdr in probe_calls + ) + + +def test_probe_success_reports_only_safe_inference_evidence(tmp_path): + """A port test double cannot inject raw payload or credential evidence.""" + probe_port = SimpleNamespace( + list_models=lambda: ["orchestrator/free"], + probe_capability=lambda *args, **kwargs: True, + ) + evidence = gateway_module().verify_external_gateway( + gateway_configuration(tmp_path), probe_port + ) + assert evidence["model"] == "orchestrator/free" + assert evidence["private_requests_require_zdr"] is True + assert evidence["capabilities"] == { + "json_object": "passed", + "json_schema": "passed", + "tool_call": "passed", + } + assert "credential" not in json.dumps(evidence) + assert gateway_module().RELEASED_GATEWAY_ADAPTERS == {} + + +def test_bootstrap_action_defaults_to_existing_sidecar(): + """Only an explicit mode input admits the future external bootstrap.""" + action_source = Path( + ".github/actions/orchestrator-free-sidecar/action.yml" + ).read_text() + assert 'default: "sidecar"' in action_source + assert ( + "CONTEXTUAL_ORCHESTRATOR_GATEWAY_MODE: ${{ inputs.gateway_mode }}" + in action_source + ) + assert "inputs.gateway_token_file" in action_source + + +def test_unverified_revision_never_constructs_an_adapter(monkeypatch, capsys): + """An arbitrary full SHA is not release authorization.""" + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_GATEWAY_CONTRACT_REVISION", "a" * 40) + assert gateway_module().main() == 1 + assert "released_contract_unavailable" in capsys.readouterr().out + + +def test_registered_port_test_double_exports_only_file_paths( + tmp_path, monkeypatch, capsys +): + """Exercise future publication with an in-memory owner test double only.""" + module = gateway_module() + gateway_config = gateway_configuration(tmp_path) + output_file = tmp_path / "github-env" + probe_port = SimpleNamespace( + list_models=lambda: ["orchestrator/free"], + probe_capability=lambda *args, **kwargs: True, + ) + monkeypatch.setattr( + module, + "RELEASED_GATEWAY_ADAPTERS", + {"test_double_revision": lambda config: probe_port}, + ) + for variable_name, variable_value in { + "CONTEXTUAL_ORCHESTRATOR_GATEWAY_CONTRACT_REVISION": "test_double_revision", + "CONTEXTUAL_ORCHESTRATOR_BASE_URL": gateway_config.base_url, + "CONTEXTUAL_ORCHESTRATOR_TOKEN_FILE": str(gateway_config.token_file), + "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR": "true", + "RUNNER_TEMP": str(tmp_path), + "GITHUB_ENV": str(output_file), + }.items(): + monkeypatch.setenv(variable_name, variable_value) + assert module.main() == 0 + output_text = output_file.read_text() + capsys.readouterr().out + assert "unit-test-gateway-credential" not in output_text + assert "CONTEXTUAL_ORCHESTRATOR_PRIVATE_REQUESTS_REQUIRE_ZDR=true" in output_text + assert "CONTEXTUAL_ORCHESTRATOR_TOKEN=" not in output_text + evidence_files = list(tmp_path.glob("external-review-*/preflight-evidence.json")) + assert len(evidence_files) == 1 + assert "unit-test-gateway-credential" not in evidence_files[0].read_text() + + +def test_adapter_exception_is_sanitized_without_readiness( + tmp_path, monkeypatch, capsys +): + """Raw adapter failures cannot become output, evidence, or a fallback.""" + module = gateway_module() + gateway_config = gateway_configuration(tmp_path) + + def failing_adapter(config): + raise RuntimeError("private-upstream-body-with-secret") + + monkeypatch.setattr( + module, "RELEASED_GATEWAY_ADAPTERS", {"test_double_revision": failing_adapter} + ) + monkeypatch.setenv( + "CONTEXTUAL_ORCHESTRATOR_GATEWAY_CONTRACT_REVISION", "test_double_revision" + ) + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_BASE_URL", gateway_config.base_url) + monkeypatch.setenv( + "CONTEXTUAL_ORCHESTRATOR_TOKEN_FILE", str(gateway_config.token_file) + ) + assert module.main() == 1 + output_text = capsys.readouterr().out + assert "private-upstream" not in output_text + assert "external_gateway_admission_failed" in output_text From 2a8bcaa623eb1daa798890601a666a4e580d7c63 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:41:43 +0900 Subject: [PATCH 2/6] fix(review): reject malformed gateway inventory at admission Signed-off-by: Seongho Bae Commit-Message-Assisted-by: Codex (OpenAI Codex) --- scripts/ci/external_review_gateway.py | 7 ++++++- tests/test_external_review_gateway.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/scripts/ci/external_review_gateway.py b/scripts/ci/external_review_gateway.py index 52630100b9..b6b17ae842 100644 --- a/scripts/ci/external_review_gateway.py +++ b/scripts/ci/external_review_gateway.py @@ -96,7 +96,12 @@ def verify_external_gateway( """Require every inference capability without partial readiness or fallback.""" gateway_config.validate() try: - if "orchestrator/free" not in probe_port.list_models(): + model_inventory = probe_port.list_models() + if ( + not isinstance(model_inventory, list) + or not all(isinstance(model_name, str) for model_name in model_inventory) + or "orchestrator/free" not in model_inventory + ): raise GatewayAdmissionError("free_pool_unavailable") capability_results = {} for capability_name in ("json_object", "json_schema", "tool_call"): diff --git a/tests/test_external_review_gateway.py b/tests/test_external_review_gateway.py index 7b9626d172..fd1568a5e4 100644 --- a/tests/test_external_review_gateway.py +++ b/tests/test_external_review_gateway.py @@ -131,6 +131,22 @@ def test_probe_success_reports_only_safe_inference_evidence(tmp_path): assert gateway_module().RELEASED_GATEWAY_ADAPTERS == {} +@pytest.mark.parametrize( + "model_inventory", + ["orchestrator/free", {"orchestrator/free": True}, ["orchestrator/free", None]], +) +def test_malformed_inventory_cannot_satisfy_admission(tmp_path, model_inventory): + """Do not interpret substring or dictionary membership as a model list.""" + probe_port = SimpleNamespace( + list_models=lambda: model_inventory, + probe_capability=lambda *args, **kwargs: True, + ) + with pytest.raises(gateway_module().GatewayAdmissionError): + gateway_module().verify_external_gateway( + gateway_configuration(tmp_path), probe_port + ) + + def test_bootstrap_action_defaults_to_existing_sidecar(): """Only an explicit mode input admits the future external bootstrap.""" action_source = Path( From 78fbecde96f9b4c825be8a06115665a36b826b6c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 21:46:11 +0900 Subject: [PATCH 3/6] fix(review): require absolute gateway token file references Signed-off-by: Seongho Bae Commit-Message-Assisted-by: Codex (OpenAI Codex) --- docs/doctoring/external-review-gateway-admission.md | 3 ++- scripts/ci/external_review_gateway.py | 4 +++- tests/test_external_review_gateway.py | 9 +++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/doctoring/external-review-gateway-admission.md b/docs/doctoring/external-review-gateway-admission.md index 243550d5b3..b9d23d3ea9 100644 --- a/docs/doctoring/external-review-gateway-admission.md +++ b/docs/doctoring/external-review-gateway-admission.md @@ -28,7 +28,8 @@ The owner probe port describes inference `/v1/models` discovery and capability checks through `/v1/chat/completions`. It requires the exact `orchestrator/free` alias plus JSON object/schema and tool-call evidence; a failed or missing capability prevents partial readiness. The port accepts only an explicit HTTPS -origin and an owned, mode-0600, regular token file. It never exports a raw bearer. +origin and an absolute path to an owned, mode-0600, regular token file. It never +resolves a symlink to repair an input or exports a raw bearer. Successful test-double observations produce only bounded capability evidence. TLS verification, redirect rejection, trusted origin authorization, secure token diff --git a/scripts/ci/external_review_gateway.py b/scripts/ci/external_review_gateway.py index b6b17ae842..b6ba566eb1 100644 --- a/scripts/ci/external_review_gateway.py +++ b/scripts/ci/external_review_gateway.py @@ -50,7 +50,9 @@ def validate(self) -> None: or type(self.require_zdr) is not bool ): raise ValueError - if any(ord(character) < 32 for character in str(self.token_file)): + if not self.token_file.is_absolute() or any( + ord(character) < 32 for character in str(self.token_file) + ): raise ValueError token_stat = self.token_file.lstat() if ( diff --git a/tests/test_external_review_gateway.py b/tests/test_external_review_gateway.py index fd1568a5e4..a56915ee03 100644 --- a/tests/test_external_review_gateway.py +++ b/tests/test_external_review_gateway.py @@ -85,6 +85,15 @@ def test_external_credentials_must_be_private_regular_owned_files(tmp_path): gateway_config.validate() +def test_relative_token_reference_cannot_cross_step_boundaries(tmp_path, monkeypatch): + """Later steps may change working directory; token paths must be absolute.""" + gateway_config = gateway_configuration(tmp_path) + gateway_config.token_file = Path("gateway.token") + monkeypatch.chdir(tmp_path) + with pytest.raises(gateway_module().GatewayAdmissionError): + gateway_config.validate() + + @pytest.mark.parametrize( "missing_capability", ["inventory", "json_object", "json_schema", "tool_call"] ) From 45aa644f190d36e30873ddb896a37d33e1f327f5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:32:13 +0900 Subject: [PATCH 4/6] fix(review): preserve bounded external probe failures Replace boolean discovery and capability results with validated typed receipts. Preserve the failed probe and closed category in main output while projecting mutable exception evidence through the same allowlist. Keep the released-adapter registry empty and retain the no-fallback boundary. Signed-off-by: Seongho Bae Commit-Message-Assisted-by: Codex (OpenAI Codex desktop) --- .../external-review-gateway-admission.md | 21 ++ scripts/ci/external_review_gateway.py | 158 ++++++++++--- tests/test_external_review_gateway.py | 215 +++++++++++++++++- 3 files changed, 357 insertions(+), 37 deletions(-) diff --git a/docs/doctoring/external-review-gateway-admission.md b/docs/doctoring/external-review-gateway-admission.md index b9d23d3ea9..1ffb7cf96e 100644 --- a/docs/doctoring/external-review-gateway-admission.md +++ b/docs/doctoring/external-review-gateway-admission.md @@ -32,6 +32,22 @@ origin and an absolute path to an owned, mode-0600, regular token file. It never resolves a symlink to repair an input or exports a raw bearer. Successful test-double observations produce only bounded capability evidence. +Discovery and each capability return an owner `ProbeReceipt`, not a boolean or +raw inventory. A discovery success attests that the adapter validated the exact +free alias; an absent free pool is `policy_unavailable`. The closed failure +categories are `authentication_failed`, `transport_failed`, `invalid_response`, +`policy_unavailable` and `capability_unavailable`. Evidence contains only the +fixed probe name, an integer HTTP status (or null), pass/fail result and category. +The caller validates every receipt before use; success requires status 200 plus +the adapter's semantic validation. Invalid fields and unexpected exceptions +become `invalid_response` at the active probe, with no raw error text. + +Main preserves the bounded failed stage/category/status in its error annotation +and never publishes partial readiness. Successful evidence records only +`requested_model=orchestrator/free`, not an upstream model identifier. This port +vocabulary follows the proposed CO evidence semantics; it is not a released +transport implementation or authorization to register an adapter. + TLS verification, redirect rejection, trusted origin authorization, secure token opening and full response validation are obligations of the future released adapter. They are not implemented HTTP transport in this delta. No claim of a @@ -66,3 +82,8 @@ admission, missing free inventory, each failed capability, unregistered revision safe output, and exceptions using owner test doubles. Synthetic data is confined to unit tests. Full lifecycle evidence still requires protected review, release, caller adoption, and a live exact-head review. + +The typed-receipt regression was RED on the boolean port (33 failures). Tests +cover every failure category at all four probes, malformed receipt fields, +legacy boolean/raw response results and sanitized main output. The registry +remains empty; only in-memory test doubles exercise receipt publication. diff --git a/scripts/ci/external_review_gateway.py b/scripts/ci/external_review_gateway.py index b6ba566eb1..d9c52db871 100644 --- a/scripts/ci/external_review_gateway.py +++ b/scripts/ci/external_review_gateway.py @@ -12,13 +12,101 @@ import tempfile from collections.abc import Callable from dataclasses import dataclass +from enum import Enum from pathlib import Path from typing import Protocol from urllib.parse import urlsplit +class ProbeErrorCategory(str, Enum): + """Closed failure vocabulary accepted from a future released adapter.""" + + AUTHENTICATION_FAILED = "authentication_failed" + TRANSPORT_FAILED = "transport_failed" + INVALID_RESPONSE = "invalid_response" + POLICY_UNAVAILABLE = "policy_unavailable" + CAPABILITY_UNAVAILABLE = "capability_unavailable" + + +PROBE_NAMES = ("discovery", "json_object", "json_schema", "tool_call") + + class GatewayAdmissionError(RuntimeError): - """Carry only an owner-defined, secret-free admission category.""" + """Carry only validated, bounded admission evidence, never exception text.""" + + def __init__( + self, + category: str, + probe_name: str = "bootstrap", + http_status: int | None = None, + ): + allowed_categories = {item.value for item in ProbeErrorCategory} | { + "invalid_gateway_configuration", + "invalid_output_location", + "external_gateway_admission_failed", + } + safe_category = ( + category + if type(category) is str and category in allowed_categories + else "invalid_response" + ) + safe_probe = ( + probe_name + if type(probe_name) is str and probe_name in (*PROBE_NAMES, "bootstrap") + else "bootstrap" + ) + safe_status = ( + http_status + if type(http_status) is int and 100 <= http_status <= 599 + else None + ) + self.evidence = { + "probe_name": safe_probe, + "http_status": safe_status, + "result": "failed", + "error_category": safe_category, + } + super().__init__(safe_category) + + +@dataclass(frozen=True) +class ProbeReceipt: + """Semantic result; discovery success includes the exact free-model alias.""" + + probe_name: str + http_status: int | None + error_category: ProbeErrorCategory | None = None + + def safe_evidence(self, expected_probe: str) -> dict: + """Reject malformed adapter evidence before it reaches logs or gates.""" + if ( + type(self.probe_name) is not str + or self.probe_name != expected_probe + or ( + self.http_status is not None + and ( + type(self.http_status) is not int + or not 100 <= self.http_status <= 599 + ) + ) + or ( + self.error_category is not None + and type(self.error_category) is not ProbeErrorCategory + ) + or (self.error_category is None and self.http_status != 200) + ): + raise GatewayAdmissionError("invalid_response", expected_probe) + category = ( + self.error_category.value if self.error_category is not None else None + ) + if category is not None: + raise GatewayAdmissionError(category, expected_probe, self.http_status) + return { + "probe_name": expected_probe, + "http_status": self.http_status, + "result": "passed", + "error_category": None, + } @dataclass @@ -69,13 +157,17 @@ def validate(self) -> None: class InferenceProbePort(Protocol): """Released-adapter boundary; no provider credentials or admin readiness.""" - def list_models(self) -> list[str]: - """Validate authenticated GET /v1/models and return exact model ids.""" + def list_models(self) -> ProbeReceipt: + """Validate authenticated discovery and exact orchestrator/free presence. + + Return discovery policy_unavailable when the free pool is absent. + Never return upstream model identifiers or raw response data. + """ ... def probe_capability( self, capability_name: str, *, model_name: str, require_zdr: bool - ) -> bool: + ) -> ProbeReceipt: """Validate a released capability contract using POST /v1/chat/completions. The adapter must verify TLS, reject redirects, reopen the private token @@ -97,33 +189,27 @@ def verify_external_gateway( ) -> dict: """Require every inference capability without partial readiness or fallback.""" gateway_config.validate() - try: - model_inventory = probe_port.list_models() - if ( - not isinstance(model_inventory, list) - or not all(isinstance(model_name, str) for model_name in model_inventory) - or "orchestrator/free" not in model_inventory - ): - raise GatewayAdmissionError("free_pool_unavailable") - capability_results = {} - for capability_name in ("json_object", "json_schema", "tool_call"): - if ( - probe_port.probe_capability( - capability_name, + probe_evidence = [] + for probe_name in PROBE_NAMES: + try: + receipt = ( + probe_port.list_models() + if probe_name == "discovery" + else probe_port.probe_capability( + probe_name, model_name="orchestrator/free", require_zdr=gateway_config.require_zdr, ) - is not True - ): - raise GatewayAdmissionError("capability_unavailable") - capability_results[capability_name] = "passed" - except GatewayAdmissionError: - raise - except Exception: # noqa: BLE001 - never expose transport, token or body details - raise GatewayAdmissionError("inference_probe_failed") from None + ) + except Exception: # noqa: BLE001 - typed failures must use receipts + raise GatewayAdmissionError("invalid_response", probe_name) from None + if type(receipt) is not ProbeReceipt: + raise GatewayAdmissionError("invalid_response", probe_name) + probe_evidence.append(receipt.safe_evidence(probe_name)) return { - "model": "orchestrator/free", - "capabilities": capability_results, + "requested_model": "orchestrator/free", + "capabilities": {name: "passed" for name in PROBE_NAMES[1:]}, + "probes": probe_evidence, "private_requests_require_zdr": gateway_config.require_zdr, "policy_evidence": "configured_gateway_policy_only", } @@ -171,8 +257,24 @@ def main() -> int: f"CONTEXTUAL_ORCHESTRATOR_PREFLIGHT_EVIDENCE={evidence_path}\n" f"CONTEXTUAL_ORCHESTRATOR_PRIVATE_REQUESTS_REQUIRE_ZDR={zdr_value}\n" ) + except GatewayAdmissionError as admission_error: + source = ( + admission_error.evidence if type(admission_error.evidence) is dict else {} + ) + safe_error = GatewayAdmissionError( + source.get("error_category", "invalid_response"), + source.get("probe_name", "bootstrap"), + source.get("http_status"), + ) + print("::error::" + json.dumps(safe_error.evidence)) + return 1 except Exception: # noqa: BLE001 - bootstrap failures never reveal raw input - print("::error::external_gateway_admission_failed") + print( + "::error::" + + json.dumps( + GatewayAdmissionError("external_gateway_admission_failed").evidence + ) + ) return 1 print( "External gateway inference preflight passed; private requests must retain ZDR policy." diff --git a/tests/test_external_review_gateway.py b/tests/test_external_review_gateway.py index a56915ee03..c3fc5e0511 100644 --- a/tests/test_external_review_gateway.py +++ b/tests/test_external_review_gateway.py @@ -104,11 +104,21 @@ def test_probe_failure_never_exports_partial_readiness(tmp_path, missing_capabil def probe_capability(capability_name, *, model_name, require_zdr): probe_calls.append((capability_name, model_name, require_zdr)) - return capability_name != missing_capability + return gateway_module().ProbeReceipt( + capability_name, + 200, + gateway_module().ProbeErrorCategory.CAPABILITY_UNAVAILABLE + if capability_name == missing_capability + else None, + ) probe_port = SimpleNamespace( - list_models=lambda: ( - [] if missing_capability == "inventory" else ["orchestrator/free"] + list_models=lambda: gateway_module().ProbeReceipt( + "discovery", + 200, + gateway_module().ProbeErrorCategory.POLICY_UNAVAILABLE + if missing_capability == "inventory" + else None, ), probe_capability=probe_capability, ) @@ -123,13 +133,15 @@ def probe_capability(capability_name, *, model_name, require_zdr): def test_probe_success_reports_only_safe_inference_evidence(tmp_path): """A port test double cannot inject raw payload or credential evidence.""" probe_port = SimpleNamespace( - list_models=lambda: ["orchestrator/free"], - probe_capability=lambda *args, **kwargs: True, + list_models=lambda: gateway_module().ProbeReceipt("discovery", 200), + probe_capability=lambda name, **kwargs: gateway_module().ProbeReceipt( + name, 200 + ), ) evidence = gateway_module().verify_external_gateway( gateway_configuration(tmp_path), probe_port ) - assert evidence["model"] == "orchestrator/free" + assert evidence["requested_model"] == "orchestrator/free" assert evidence["private_requests_require_zdr"] is True assert evidence["capabilities"] == { "json_object": "passed", @@ -148,7 +160,9 @@ def test_malformed_inventory_cannot_satisfy_admission(tmp_path, model_inventory) """Do not interpret substring or dictionary membership as a model list.""" probe_port = SimpleNamespace( list_models=lambda: model_inventory, - probe_capability=lambda *args, **kwargs: True, + probe_capability=lambda name, **kwargs: gateway_module().ProbeReceipt( + name, 200 + ), ) with pytest.raises(gateway_module().GatewayAdmissionError): gateway_module().verify_external_gateway( @@ -184,8 +198,10 @@ def test_registered_port_test_double_exports_only_file_paths( gateway_config = gateway_configuration(tmp_path) output_file = tmp_path / "github-env" probe_port = SimpleNamespace( - list_models=lambda: ["orchestrator/free"], - probe_capability=lambda *args, **kwargs: True, + list_models=lambda: gateway_module().ProbeReceipt("discovery", 200), + probe_capability=lambda name, **kwargs: gateway_module().ProbeReceipt( + name, 200 + ), ) monkeypatch.setattr( module, @@ -235,3 +251,184 @@ def failing_adapter(config): output_text = capsys.readouterr().out assert "private-upstream" not in output_text assert "external_gateway_admission_failed" in output_text + + +@pytest.mark.parametrize( + "probe_name", ["discovery", "json_object", "json_schema", "tool_call"] +) +@pytest.mark.parametrize( + "category_name", + [ + "authentication_failed", + "transport_failed", + "invalid_response", + "policy_unavailable", + "capability_unavailable", + ], +) +def test_typed_failure_preserves_stage_and_category( + tmp_path, probe_name, category_name +): + """Adapter failures retain only bounded stage/category/status evidence.""" + module = gateway_module() + category = module.ProbeErrorCategory(category_name) + + def receipt(name): + return ( + module.ProbeReceipt(name, None, category) + if name == probe_name + else module.ProbeReceipt(name, 200) + ) + + port = SimpleNamespace( + list_models=lambda: receipt("discovery"), + probe_capability=lambda name, **kwargs: receipt(name), + ) + with pytest.raises(module.GatewayAdmissionError) as caught: + module.verify_external_gateway(gateway_configuration(tmp_path), port) + assert caught.value.evidence == { + "probe_name": probe_name, + "http_status": None, + "result": "failed", + "error_category": category_name, + } + + +@pytest.mark.parametrize("bad_receipt", [True, {"error_category": "secret"}, "secret"]) +def test_untyped_receipt_fails_without_raw_details(tmp_path, bad_receipt): + """Legacy booleans and raw response mappings cannot satisfy admission.""" + module = gateway_module() + port = SimpleNamespace(list_models=lambda: bad_receipt) + with pytest.raises(module.GatewayAdmissionError) as caught: + module.verify_external_gateway(gateway_configuration(tmp_path), port) + assert caught.value.evidence["error_category"] == "invalid_response" + assert "secret" not in str(caught.value) + + +@pytest.mark.parametrize( + "field,value", + [ + ("probe_name", "secret"), + ("http_status", True), + ("http_status", 600), + ("http_status", "secret"), + ("error_category", "secret"), + ("http_status", None), + ], +) +def test_malformed_receipt_fields_are_sanitized(tmp_path, field, value): + """Dataclass construction alone is not trust-boundary validation.""" + module = gateway_module() + values = {"probe_name": "discovery", "http_status": 200, "error_category": None} + values[field] = value + port = SimpleNamespace(list_models=lambda: module.ProbeReceipt(**values)) + with pytest.raises(module.GatewayAdmissionError) as caught: + module.verify_external_gateway(gateway_configuration(tmp_path), port) + assert caught.value.evidence == { + "probe_name": "discovery", + "http_status": None, + "result": "failed", + "error_category": "invalid_response", + } + + +@pytest.mark.parametrize( + "failure_kind", + [ + "authentication_failed", + "transport_failed", + "invalid_response", + "policy_unavailable", + "capability_unavailable", + "raw_exception", + "malformed", + ], +) +@pytest.mark.parametrize("failing_stage", ["discovery", "json_schema"]) +def test_main_preserves_safe_failure_and_never_exports_readiness( + tmp_path, monkeypatch, capsys, failure_kind, failing_stage +): + """Main retains validated probe failure details without exception text.""" + module = gateway_module() + config = gateway_configuration(tmp_path) + + def probe(name): + if name != failing_stage: + return module.ProbeReceipt(name, 200) + if failure_kind == "raw_exception": + raise RuntimeError("private-upstream-body-with-secret") + if failure_kind == "malformed": + return module.ProbeReceipt("secret", "secret", "secret") + return module.ProbeReceipt(name, None, module.ProbeErrorCategory(failure_kind)) + + monkeypatch.setattr( + module, + "RELEASED_GATEWAY_ADAPTERS", + { + "test": lambda config: SimpleNamespace( + list_models=lambda: probe("discovery"), + probe_capability=lambda name, **kwargs: probe(name), + ) + }, + ) + for key, value in { + "CONTEXTUAL_ORCHESTRATOR_GATEWAY_CONTRACT_REVISION": "test", + "CONTEXTUAL_ORCHESTRATOR_BASE_URL": config.base_url, + "CONTEXTUAL_ORCHESTRATOR_TOKEN_FILE": str(config.token_file), + "RUNNER_TEMP": str(tmp_path), + "GITHUB_ENV": str(tmp_path / "github-env"), + }.items(): + monkeypatch.setenv(key, value) + assert module.main() == 1 + output = capsys.readouterr().out + assert "secret" not in output + assert f'"probe_name": "{failing_stage}"' in output + expected_category = ( + "invalid_response" + if failure_kind in {"raw_exception", "malformed"} + else failure_kind + ) + assert f'"error_category": "{expected_category}"' in output + assert not (tmp_path / "github-env").exists() + assert not list(tmp_path.glob("external-review-*")) + + +@pytest.mark.parametrize( + "tampered_evidence", + [ + { + "probe_name": "discovery", + "http_status": 401, + "result": "secret", + "error_category": "authentication_failed", + "raw_body": "secret", + }, + {"probe_name": "secret", "http_status": "secret", "error_category": "secret"}, + "secret", + ], +) +def test_main_projects_factory_admission_errors( + tmp_path, monkeypatch, capsys, tampered_evidence +): + """Mutable exception evidence cannot add raw fields at final serialization.""" + module = gateway_module() + config = gateway_configuration(tmp_path) + + def factory(config): + error = module.GatewayAdmissionError("authentication_failed") + error.evidence = tampered_evidence + raise error + + monkeypatch.setattr(module, "RELEASED_GATEWAY_ADAPTERS", {"test": factory}) + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_GATEWAY_CONTRACT_REVISION", "test") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_BASE_URL", config.base_url) + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN_FILE", str(config.token_file)) + output_file = tmp_path / "github-env" + monkeypatch.setenv("GITHUB_ENV", str(output_file)) + assert module.main() == 1 + output = capsys.readouterr().out + assert "secret" not in output + evidence = json.loads(output.removeprefix("::error::")) + assert set(evidence) == {"probe_name", "http_status", "result", "error_category"} + assert evidence["result"] == "failed" + assert not output_file.exists() From 2c51e474a323c7c7942eb0aa612bb2569f74d9c7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:34:04 +0900 Subject: [PATCH 5/6] refactor(review): clarify typed evidence field names Name the internal error category, source evidence and probe receipt explicitly and document the exception constructor. No admission behavior changes. Signed-off-by: Seongho Bae Commit-Message-Assisted-by: Codex (OpenAI Codex desktop) --- scripts/ci/external_review_gateway.py | 29 +++++++++++++++------------ 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/scripts/ci/external_review_gateway.py b/scripts/ci/external_review_gateway.py index d9c52db871..4004839f36 100644 --- a/scripts/ci/external_review_gateway.py +++ b/scripts/ci/external_review_gateway.py @@ -36,18 +36,19 @@ class GatewayAdmissionError(RuntimeError): def __init__( self, - category: str, + error_category: str, probe_name: str = "bootstrap", http_status: int | None = None, ): + """Project error details onto the closed, secret-free evidence fields.""" allowed_categories = {item.value for item in ProbeErrorCategory} | { "invalid_gateway_configuration", "invalid_output_location", "external_gateway_admission_failed", } safe_category = ( - category - if type(category) is str and category in allowed_categories + error_category + if type(error_category) is str and error_category in allowed_categories else "invalid_response" ) safe_probe = ( @@ -96,11 +97,13 @@ def safe_evidence(self, expected_probe: str) -> dict: or (self.error_category is None and self.http_status != 200) ): raise GatewayAdmissionError("invalid_response", expected_probe) - category = ( + error_category = ( self.error_category.value if self.error_category is not None else None ) - if category is not None: - raise GatewayAdmissionError(category, expected_probe, self.http_status) + if error_category is not None: + raise GatewayAdmissionError( + error_category, expected_probe, self.http_status + ) return { "probe_name": expected_probe, "http_status": self.http_status, @@ -192,7 +195,7 @@ def verify_external_gateway( probe_evidence = [] for probe_name in PROBE_NAMES: try: - receipt = ( + probe_receipt = ( probe_port.list_models() if probe_name == "discovery" else probe_port.probe_capability( @@ -203,9 +206,9 @@ def verify_external_gateway( ) except Exception: # noqa: BLE001 - typed failures must use receipts raise GatewayAdmissionError("invalid_response", probe_name) from None - if type(receipt) is not ProbeReceipt: + if type(probe_receipt) is not ProbeReceipt: raise GatewayAdmissionError("invalid_response", probe_name) - probe_evidence.append(receipt.safe_evidence(probe_name)) + probe_evidence.append(probe_receipt.safe_evidence(probe_name)) return { "requested_model": "orchestrator/free", "capabilities": {name: "passed" for name in PROBE_NAMES[1:]}, @@ -258,13 +261,13 @@ def main() -> int: f"CONTEXTUAL_ORCHESTRATOR_PRIVATE_REQUESTS_REQUIRE_ZDR={zdr_value}\n" ) except GatewayAdmissionError as admission_error: - source = ( + source_evidence = ( admission_error.evidence if type(admission_error.evidence) is dict else {} ) safe_error = GatewayAdmissionError( - source.get("error_category", "invalid_response"), - source.get("probe_name", "bootstrap"), - source.get("http_status"), + source_evidence.get("error_category", "invalid_response"), + source_evidence.get("probe_name", "bootstrap"), + source_evidence.get("http_status"), ) print("::error::" + json.dumps(safe_error.evidence)) return 1 From 65765e875040d78a8ab1b8ff564590137fd8675c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:35:14 +0900 Subject: [PATCH 6/6] fix(review): sanitize absent admission error evidence Read only the exact owner exception dictionary before final evidence projection. Missing evidence and unsupported exception subclasses now fail closed without accessing hostile attributes or exposing chained exception text. Add reproductions for both cases. Signed-off-by: Seongho Bae Commit-Message-Assisted-by: Codex (OpenAI Codex desktop) --- scripts/ci/external_review_gateway.py | 6 +++++- tests/test_external_review_gateway.py | 18 +++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/scripts/ci/external_review_gateway.py b/scripts/ci/external_review_gateway.py index 4004839f36..af163f41cf 100644 --- a/scripts/ci/external_review_gateway.py +++ b/scripts/ci/external_review_gateway.py @@ -262,8 +262,12 @@ def main() -> int: ) except GatewayAdmissionError as admission_error: source_evidence = ( - admission_error.evidence if type(admission_error.evidence) is dict else {} + vars(admission_error).get("evidence") + if type(admission_error) is GatewayAdmissionError + else None ) + if type(source_evidence) is not dict: + source_evidence = {} safe_error = GatewayAdmissionError( source_evidence.get("error_category", "invalid_response"), source_evidence.get("probe_name", "bootstrap"), diff --git a/tests/test_external_review_gateway.py b/tests/test_external_review_gateway.py index c3fc5e0511..198062307d 100644 --- a/tests/test_external_review_gateway.py +++ b/tests/test_external_review_gateway.py @@ -405,6 +405,8 @@ def probe(name): }, {"probe_name": "secret", "http_status": "secret", "error_category": "secret"}, "secret", + "missing", + "inaccessible", ], ) def test_main_projects_factory_admission_errors( @@ -416,7 +418,21 @@ def test_main_projects_factory_admission_errors( def factory(config): error = module.GatewayAdmissionError("authentication_failed") - error.evidence = tampered_evidence + if tampered_evidence == "missing": + del error.evidence + error.args = ("secret",) + elif tampered_evidence == "inaccessible": + + class InaccessibleAdmissionError(module.GatewayAdmissionError): + def __getattribute__(self, field_name): + if field_name == "evidence": + raise RuntimeError("secret") + return super().__getattribute__(field_name) + + error = InaccessibleAdmissionError("authentication_failed") + error.args = ("secret",) + else: + error.evidence = tampered_evidence raise error monkeypatch.setattr(module, "RELEASED_GATEWAY_ADAPTERS", {"test": factory})