From 5aab180b306e29e80d5201cf1a31122d45ccce17 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 09:34:59 +0900 Subject: [PATCH 01/12] feat(discovery): add bounded OpenRouter free canary --- CHANGELOG.md | 6 + README.md | 17 +++ contextual_orchestrator/__main__.py | 33 +++++ contextual_orchestrator/openrouter_canary.py | 147 +++++++++++++++++++ docs/product-technical-gap-baseline.md | 9 +- tests/test_openrouter_free_canary.py | 117 +++++++++++++++ 6 files changed, 326 insertions(+), 3 deletions(-) create mode 100644 contextual_orchestrator/openrouter_canary.py create mode 100644 tests/test_openrouter_free_canary.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b38e0770..79ff0df1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ## [0.2.0] - Unreleased +### Added + +- Add a dry-run-default OpenRouter free-model canary. Its opt-in live mode + requires explicit request, output-token, timeout, evidence-path, and retention + bounds and selects only freshly discovered, completely zero-priced chat rows. + ### Fixed - OpenRouter discovery no longer marks the entire credential account diff --git a/README.md b/README.md index 60ffff831..a254caa37 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,23 @@ Model-based conduct verification requires `fast-mlsirm` in the same runtime and The agent pool is manageable at runtime: `POST`/`PATCH`/`DELETE` on `/api/v1/agent_pools/default/worker_agents[/{id}]` add, govern, and remove model-group members. Pass `--agents-db PATH` (or `CONTEXTUAL_ORCHESTRATOR_AGENTS_DB`) to persist those changes to a stdlib sqlite file — stored changes overlay the seed agents file at startup, and removals write disabled tombstones so they survive restarts; without it the pool is in-memory as before. Beyond the local MLX/llama.cpp discovery above, `python -m contextual_orchestrator discover-models [--agents-db PATH]` discovers models from remote providers (OpenAI, OpenRouter, NVIDIA NIM ×2 keys, Bytez, and an allowlisted OpenAI-compatible gateway) for any subset of their KV-registered credentials, and can persist them into the same `--agents-db` sqlite file, added disabled by default. See [docs/kv-credentials.md](docs/kv-credentials.md#multi-provider-auto-discovery) for the credential-name table and cost-based auto-selection. +`python -m contextual_orchestrator openrouter-free-canary` is a dry-run-only +catalog check by default. It selects no pinned model: the lexically first +current OpenRouter chat row with explicit zero prompt and completion prices, +USD currency, and no incomparable unit price is reported without issuing a +completion. Live mode is deliberately verbose and unscheduled: + +```bash +python -m contextual_orchestrator openrouter-free-canary --live \ + --max-requests 1 --max-output-tokens 8 --timeout-seconds 10 \ + --evidence-output ./openrouter-canary.json --retention-days 7 +``` + +Every live cap and the evidence path/retention choice is mandatory. The command +uses only the KV-registered `OPENROUTER_API_KEY`, disables retries, makes one +fixed prompt request, and writes atomic JSON evidence containing neither the +prompt, response, nor credential. No workflow invokes live mode automatically. + Seed the credential into the KV once at bootstrap: ```bash diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index ffd82536d..ea3e89526 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -6,6 +6,7 @@ import json import logging import os +from pathlib import Path import sys from dataclasses import replace @@ -35,6 +36,7 @@ TaskOrchestrator, load_agents, ) +from .openrouter_canary import OpenRouterCanaryLimits, run_openrouter_free_canary from .privacy_policy_analysis import ( analyze_discovered_privacy_policies, ) @@ -490,6 +492,37 @@ def main(argv: list[str] | None = None) -> None: if arguments and arguments[0] == "discover-models": _discover_models_command(arguments[1:]) return + if arguments and arguments[0] == "openrouter-free-canary": + parser = argparse.ArgumentParser( + prog="python -m contextual_orchestrator openrouter-free-canary" + ) + parser.add_argument("--live", action="store_true") + parser.add_argument("--max-requests", type=_positive_int) + parser.add_argument("--max-output-tokens", type=_positive_int) + parser.add_argument("--timeout-seconds", type=_positive_int) + parser.add_argument("--evidence-output") + parser.add_argument("--retention-days", type=_positive_int) + args = parser.parse_args(arguments[1:]) + supplied = ( + args.max_requests, + args.max_output_tokens, + args.timeout_seconds, + args.retention_days, + ) + if args.live and ( + any(value is None for value in supplied) or not args.evidence_output + ): + parser.error( + "--live requires positive request/output-token/timeout/retention caps and --evidence-output" + ) + limits = OpenRouterCanaryLimits(*supplied) if args.live else None + result = run_openrouter_free_canary( + live=args.live, + limits=limits, + evidence_output=Path(args.evidence_output) if args.evidence_output else None, + ) + print(json.dumps(result, sort_keys=True)) + return if arguments and arguments[0] == "check-fast-mlsirm": _check_fast_mlsirm_command() return diff --git a/contextual_orchestrator/openrouter_canary.py b/contextual_orchestrator/openrouter_canary.py new file mode 100644 index 000000000..6873a86c0 --- /dev/null +++ b/contextual_orchestrator/openrouter_canary.py @@ -0,0 +1,147 @@ +"""Bounded, opt-in evidence for one currently free OpenRouter model.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +import json +import os +from pathlib import Path +import tempfile +import time +from typing import Any, Callable + +from .credentials import get_credential +from .model_discovery import ( + DiscoveredModel, + PROVIDER_MODEL_SOURCES, + agent_from_discovered, + discover_provider_models, + is_discovered_chat_candidate, +) +from .orchestrator import ModelClient + + +class OpenRouterCanaryError(RuntimeError): + """Raised before transport when the canary contract is incomplete.""" + + +@dataclass(frozen=True) +class OpenRouterCanaryLimits: + """Explicit operator caps for the optional live request.""" + + max_requests: int + max_output_tokens: int + timeout_seconds: int + retention_days: int + + def validate(self) -> None: + """Reject absent, boolean, or non-positive cap values.""" + for name, value in asdict(self).items(): + if type(value) is not int or value < 1: + raise OpenRouterCanaryError(f"{name} must be a positive integer") + + +def _eligible(models: list[DiscoveredModel]) -> list[DiscoveredModel]: + """Return price-complete, text-chat OpenRouter rows in stable order.""" + return sorted( + ( + model + for model in models + if model.provider_name == "openrouter" + and model.prompt_price_per_1k == 0.0 + and model.completion_price_per_1k == 0.0 + and not model.unit_prices + and model.currency_code == "USD" + and not model.evidence_only + and model.spend_admitted + and is_discovered_chat_candidate(model) + ), + key=lambda model: model.model_id, + ) + + +def _write_evidence(path: Path, evidence: dict[str, Any]) -> None: + """Atomically publish one secret- and prompt-free JSON evidence document.""" + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", dir=path.parent + ) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + json.dump(evidence, stream, sort_keys=True) + stream.write("\n") + os.replace(temporary_name, path) + except BaseException: + try: + os.unlink(temporary_name) + except FileNotFoundError: + pass + raise + + +def run_openrouter_free_canary( + *, + live: bool, + limits: OpenRouterCanaryLimits | None = None, + evidence_output: Path | None = None, + discover: Callable[..., list[DiscoveredModel]] = discover_provider_models, + client_factory: Callable[..., ModelClient] = ModelClient, + now: Callable[[], float] = time.time, +) -> dict[str, Any]: + """Discover a current free candidate and optionally issue one bounded request.""" + source = next( + item for item in PROVIDER_MODEL_SOURCES if item.provider_name == "openrouter" + ) + if not get_credential(source.credential_name): + raise OpenRouterCanaryError( + "OPENROUTER_API_KEY is unavailable in the KV registry" + ) + if live: + if limits is None or evidence_output is None: + raise OpenRouterCanaryError( + "live mode requires all caps and an evidence output path" + ) + limits.validate() + discovered_at = int(now()) + candidates = _eligible( + discover(source, timeout=limits.timeout_seconds if limits else 10) + ) + if not candidates: + raise OpenRouterCanaryError( + "current discovery has no unambiguous zero-price chat candidate" + ) + selected = candidates[0] + evidence: dict[str, Any] = { + "schema_version": 1, + "mode": "live" if live else "dry_run", + "provider": "openrouter", + "model_id": selected.model_id, + "discovered_at": discovered_at, + "price_evidence": { + "prompt_price_per_1k": 0.0, + "completion_price_per_1k": 0.0, + "currency_code": "USD", + }, + "request_count": 0, + } + if live: + assert limits is not None and evidence_output is not None + client = client_factory( + timeout=limits.timeout_seconds, + max_output_tokens=limits.max_output_tokens, + max_retries=0, + temperature=0.0, + ) + client.chat( + agent_from_discovered(selected), [{"role": "user", "content": "Reply OK."}] + ) + evidence.update( + { + "request_count": 1, + "limits": asdict(limits), + "expires_at": discovered_at + limits.retention_days * 86400, + "outcome": "completed", + } + ) + _write_evidence(evidence_output, evidence) + return evidence diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index cbad42f7f..8af9f18ee 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2102,9 +2102,12 @@ Buyer-visible gaps now prioritized: keyboard/native-form REST editor; DB membership is normalized and legacy JSON membership migrates without data loss. Authenticated deployed-browser runtime evidence remains a release/UAT gate rather than an implementation gap. -3. Free-model tests are deterministic catalog-contract tests. Add an opt-in, - spend-capped live OpenRouter canary selected from current zero-price metadata; - never pin a transient free model identifier in production or CI. +3. **Implemented in the current product-gap branch:** the OpenRouter free-model + canary is dry-run-only by default and selects a current chat row only when + prompt and completion prices are explicitly zero and comparable. Live mode + is unscheduled and requires positive request, output-token, timeout, evidence + retention, and output-path choices; it disables retries, pins no model id, + and persists neither credentials, prompts, nor responses. 4. Multi-instance routing observations remain process-local. Add a time-windowed durable observation model with calibrated decay before horizontal scaling. 5. Protected main, not a feature-stack merge, remains the release boundary; do diff --git a/tests/test_openrouter_free_canary.py b/tests/test_openrouter_free_canary.py new file mode 100644 index 000000000..c8e62c08e --- /dev/null +++ b/tests/test_openrouter_free_canary.py @@ -0,0 +1,117 @@ +"""Contracts for the bounded OpenRouter free-model canary.""" + +from pathlib import Path +import pytest + +from contextual_orchestrator.credentials import InMemoryCredentialBackend, set_backend +from contextual_orchestrator.model_discovery import DiscoveredModel +from contextual_orchestrator.openrouter_canary import ( + OpenRouterCanaryError, + OpenRouterCanaryLimits, + run_openrouter_free_canary, +) +from contextual_orchestrator import __main__ as cli + + +def _model(model_id: str, *, prompt=0.0, completion=0.0) -> DiscoveredModel: + return DiscoveredModel( + "openrouter", + model_id, + "OPENROUTER_API_KEY", + "https://openrouter.ai/api/v1", + "Bearer", + capabilities=("chat",), + prompt_price_per_1k=prompt, + completion_price_per_1k=completion, + is_free=True, + ) + + +def test_dry_run_selects_current_zero_price_without_transport() -> None: + backend = InMemoryCredentialBackend() + backend.set("OPENROUTER_API_KEY", "secret") + set_backend(backend) + try: + result = run_openrouter_free_canary( + live=False, + discover=lambda *_a, **_k: [ + _model("unknown", prompt=None), + _model("z-free"), + _model("a-free"), + ], + client_factory=lambda **_k: pytest.fail("transport"), + now=lambda: 123, + ) + finally: + set_backend(None) + assert result["model_id"] == "a-free" and result["request_count"] == 0 + assert "secret" not in str(result) + + +def test_live_request_is_capped_and_writes_prompt_free_evidence(tmp_path: Path) -> None: + backend = InMemoryCredentialBackend() + backend.set("OPENROUTER_API_KEY", "secret") + set_backend(backend) + seen = {} + + class Client: + def __init__(self, **kwargs): + seen["limits"] = kwargs + + def chat(self, agent, messages): + seen["messages"] = messages + return "OK" + + output = tmp_path / "evidence.json" + try: + result = run_openrouter_free_canary( + live=True, + limits=OpenRouterCanaryLimits(1, 8, 3, 7), + evidence_output=output, + discover=lambda *_a, **_k: [_model("current-free")], + client_factory=Client, + now=lambda: 100, + ) + finally: + set_backend(None) + assert seen["limits"] == { + "timeout": 3, + "max_output_tokens": 8, + "max_retries": 0, + "temperature": 0.0, + } + assert result["request_count"] == 1 and result["expires_at"] == 604900 + assert "Reply OK" not in output.read_text() and "secret" not in output.read_text() + + +def test_canary_fails_closed_on_missing_credential_or_price() -> None: + set_backend(InMemoryCredentialBackend()) + try: + with pytest.raises(OpenRouterCanaryError, match="KV registry"): + run_openrouter_free_canary(live=False, discover=lambda *_a, **_k: []) + backend = InMemoryCredentialBackend() + backend.set("OPENROUTER_API_KEY", "secret") + set_backend(backend) + with pytest.raises(OpenRouterCanaryError, match="zero-price"): + run_openrouter_free_canary( + live=False, + discover=lambda *_a, **_k: [_model("ambiguous", completion=None)], + ) + finally: + set_backend(None) + + +def test_cli_defaults_to_dry_run_and_live_requires_every_bound( + monkeypatch, capsys +) -> None: + seen = {} + monkeypatch.setattr( + cli, + "run_openrouter_free_canary", + lambda **kwargs: seen.update(kwargs) or {"mode": "dry_run"}, + ) + cli.main(["openrouter-free-canary"]) + assert seen["live"] is False and seen["limits"] is None + assert '"mode": "dry_run"' in capsys.readouterr().out + with pytest.raises(SystemExit): + cli.main(["openrouter-free-canary", "--live", "--max-requests", "1"]) From f20d3d99f642c0fe2db4385a62ba0fc6fb268b8e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 09:40:50 +0900 Subject: [PATCH 02/12] fix(canary): enforce free evidence lifecycle Signed-off-by: Seongho Bae --- README.md | 5 +- contextual_orchestrator/openrouter_canary.py | 30 ++++++++++++ tests/test_openrouter_free_canary.py | 50 +++++++++++++++++++- 3 files changed, 82 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a254caa37..ba2c19ab3 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,10 @@ python -m contextual_orchestrator openrouter-free-canary --live \ Every live cap and the evidence path/retention choice is mandatory. The command uses only the KV-registered `OPENROUTER_API_KEY`, disables retries, makes one fixed prompt request, and writes atomic JSON evidence containing neither the -prompt, response, nor credential. No workflow invokes live mode automatically. +prompt, response, nor credential. Before transport it removes expired evidence +at the same output path and proves that a mode-`0600` atomic write is possible; +retention cleanup therefore runs whenever the canary is invoked. No workflow +invokes live mode automatically. Seed the credential into the KV once at bootstrap: diff --git a/contextual_orchestrator/openrouter_canary.py b/contextual_orchestrator/openrouter_canary.py index 6873a86c0..0bfbf38d2 100644 --- a/contextual_orchestrator/openrouter_canary.py +++ b/contextual_orchestrator/openrouter_canary.py @@ -50,6 +50,7 @@ def _eligible(models: list[DiscoveredModel]) -> list[DiscoveredModel]: if model.provider_name == "openrouter" and model.prompt_price_per_1k == 0.0 and model.completion_price_per_1k == 0.0 + and model.is_free is True and not model.unit_prices and model.currency_code == "USD" and not model.evidence_only @@ -79,6 +80,29 @@ def _write_evidence(path: Path, evidence: dict[str, Any]) -> None: raise +def _prepare_evidence_path(path: Path, current_time: int) -> None: + """Remove expired prior evidence and prove a secure atomic write is possible.""" + try: + prior = json.loads(path.read_text(encoding="utf-8")) + except (FileNotFoundError, OSError, UnicodeError, json.JSONDecodeError): + prior = None + if ( + isinstance(prior, dict) + and prior.get("provider") == "openrouter" + and type(prior.get("expires_at")) is int + and prior["expires_at"] <= current_time + ): + path.unlink(missing_ok=True) + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.preflight.", dir=path.parent + ) + os.close(descriptor) + os.unlink(temporary_name) + if path.exists() and not path.is_file(): + raise OpenRouterCanaryError("evidence output must be a regular file path") + + def run_openrouter_free_canary( *, live: bool, @@ -103,6 +127,12 @@ def run_openrouter_free_canary( ) limits.validate() discovered_at = int(now()) + if live: + assert evidence_output is not None + try: + _prepare_evidence_path(evidence_output, discovered_at) + except OSError as exc: + raise OpenRouterCanaryError("evidence output is not writable") from exc candidates = _eligible( discover(source, timeout=limits.timeout_seconds if limits else 10) ) diff --git a/tests/test_openrouter_free_canary.py b/tests/test_openrouter_free_canary.py index c8e62c08e..60ff8d049 100644 --- a/tests/test_openrouter_free_canary.py +++ b/tests/test_openrouter_free_canary.py @@ -13,7 +13,7 @@ from contextual_orchestrator import __main__ as cli -def _model(model_id: str, *, prompt=0.0, completion=0.0) -> DiscoveredModel: +def _model(model_id: str, *, prompt=0.0, completion=0.0, is_free=True) -> DiscoveredModel: return DiscoveredModel( "openrouter", model_id, @@ -23,7 +23,7 @@ def _model(model_id: str, *, prompt=0.0, completion=0.0) -> DiscoveredModel: capabilities=("chat",), prompt_price_per_1k=prompt, completion_price_per_1k=completion, - is_free=True, + is_free=is_free, ) @@ -97,8 +97,54 @@ def test_canary_fails_closed_on_missing_credential_or_price() -> None: live=False, discover=lambda *_a, **_k: [_model("ambiguous", completion=None)], ) + with pytest.raises(OpenRouterCanaryError, match="zero-price"): + run_openrouter_free_canary( + live=False, + discover=lambda *_a, **_k: [_model("paid-verdict", is_free=False)], + ) + finally: + set_backend(None) + + +def test_live_preflights_output_and_removes_expired_evidence(tmp_path: Path) -> None: + backend = InMemoryCredentialBackend() + backend.set("OPENROUTER_API_KEY", "secret") + set_backend(backend) + expired = tmp_path / "expired.json" + expired.write_text( + '{"provider":"openrouter","expires_at":99}', encoding="utf-8" + ) + seen = {} + + class Client: + def __init__(self, **_kwargs): + seen["expired_before_transport"] = not expired.exists() + + def chat(self, _agent, _messages): + return "OK" + + try: + run_openrouter_free_canary( + live=True, + limits=OpenRouterCanaryLimits(1, 8, 3, 7), + evidence_output=expired, + discover=lambda *_a, **_k: [_model("current-free")], + client_factory=Client, + now=lambda: 100, + ) + with pytest.raises(OpenRouterCanaryError, match="regular file"): + run_openrouter_free_canary( + live=True, + limits=OpenRouterCanaryLimits(1, 8, 3, 7), + evidence_output=tmp_path, + discover=lambda *_a, **_k: pytest.fail("discovery transport"), + client_factory=lambda **_k: pytest.fail("completion transport"), + now=lambda: 100, + ) finally: set_backend(None) + assert seen["expired_before_transport"] is True + assert expired.stat().st_mode & 0o777 == 0o600 def test_cli_defaults_to_dry_run_and_live_requires_every_bound( From aa9f172575da6758f52e813aa15d93a63917f82d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 09:43:44 +0900 Subject: [PATCH 03/12] fix(canary): reconcile routable free candidates Signed-off-by: Seongho Bae --- contextual_orchestrator/openrouter_canary.py | 7 ++++- tests/test_openrouter_free_canary.py | 30 +++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/contextual_orchestrator/openrouter_canary.py b/contextual_orchestrator/openrouter_canary.py index 0bfbf38d2..74d8e72f0 100644 --- a/contextual_orchestrator/openrouter_canary.py +++ b/contextual_orchestrator/openrouter_canary.py @@ -14,6 +14,8 @@ from .model_discovery import ( DiscoveredModel, PROVIDER_MODEL_SOURCES, + _deduplicate_discovered_models, + _requires_non_text_input, agent_from_discovered, discover_provider_models, is_discovered_chat_candidate, @@ -56,6 +58,7 @@ def _eligible(models: list[DiscoveredModel]) -> list[DiscoveredModel]: and not model.evidence_only and model.spend_admitted and is_discovered_chat_candidate(model) + and not _requires_non_text_input(model) ), key=lambda model: model.model_id, ) @@ -134,7 +137,9 @@ def run_openrouter_free_canary( except OSError as exc: raise OpenRouterCanaryError("evidence output is not writable") from exc candidates = _eligible( - discover(source, timeout=limits.timeout_seconds if limits else 10) + _deduplicate_discovered_models( + discover(source, timeout=limits.timeout_seconds if limits else 10) + ) ) if not candidates: raise OpenRouterCanaryError( diff --git a/tests/test_openrouter_free_canary.py b/tests/test_openrouter_free_canary.py index 60ff8d049..f13d857e5 100644 --- a/tests/test_openrouter_free_canary.py +++ b/tests/test_openrouter_free_canary.py @@ -13,7 +13,14 @@ from contextual_orchestrator import __main__ as cli -def _model(model_id: str, *, prompt=0.0, completion=0.0, is_free=True) -> DiscoveredModel: +def _model( + model_id: str, + *, + prompt=0.0, + completion=0.0, + is_free=True, + input_modalities=(), +) -> DiscoveredModel: return DiscoveredModel( "openrouter", model_id, @@ -24,6 +31,7 @@ def _model(model_id: str, *, prompt=0.0, completion=0.0, is_free=True) -> Discov prompt_price_per_1k=prompt, completion_price_per_1k=completion, is_free=is_free, + input_modalities=input_modalities, ) @@ -106,6 +114,26 @@ def test_canary_fails_closed_on_missing_credential_or_price() -> None: set_backend(None) +def test_canary_reconciles_duplicate_prices_and_skips_non_text_input() -> None: + backend = InMemoryCredentialBackend() + backend.set("OPENROUTER_API_KEY", "secret") + set_backend(backend) + try: + result = run_openrouter_free_canary( + live=False, + discover=lambda *_a, **_k: [ + _model("a-conflict"), + _model("a-conflict", prompt=0.01), + _model("b-image", input_modalities=("image",)), + _model("c-text", input_modalities=("text",)), + ], + client_factory=lambda **_k: pytest.fail("completion transport"), + ) + finally: + set_backend(None) + assert result["model_id"] == "c-text" + + def test_live_preflights_output_and_removes_expired_evidence(tmp_path: Path) -> None: backend = InMemoryCredentialBackend() backend.set("OPENROUTER_API_KEY", "secret") From d60367fc3f60a65322fffa40b8b2b8ade4ce12bf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 09:45:19 +0900 Subject: [PATCH 04/12] fix(canary): reject special evidence paths Signed-off-by: Seongho Bae --- contextual_orchestrator/openrouter_canary.py | 9 ++++++-- tests/test_openrouter_free_canary.py | 23 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/contextual_orchestrator/openrouter_canary.py b/contextual_orchestrator/openrouter_canary.py index 74d8e72f0..dc02c0a58 100644 --- a/contextual_orchestrator/openrouter_canary.py +++ b/contextual_orchestrator/openrouter_canary.py @@ -6,6 +6,7 @@ import json import os from pathlib import Path +import stat import tempfile import time from typing import Any, Callable @@ -85,6 +86,12 @@ def _write_evidence(path: Path, evidence: dict[str, Any]) -> None: def _prepare_evidence_path(path: Path, current_time: int) -> None: """Remove expired prior evidence and prove a secure atomic write is possible.""" + try: + mode = path.lstat().st_mode + except FileNotFoundError: + mode = None + if mode is not None and not stat.S_ISREG(mode): + raise OpenRouterCanaryError("evidence output must be a regular file path") try: prior = json.loads(path.read_text(encoding="utf-8")) except (FileNotFoundError, OSError, UnicodeError, json.JSONDecodeError): @@ -102,8 +109,6 @@ def _prepare_evidence_path(path: Path, current_time: int) -> None: ) os.close(descriptor) os.unlink(temporary_name) - if path.exists() and not path.is_file(): - raise OpenRouterCanaryError("evidence output must be a regular file path") def run_openrouter_free_canary( diff --git a/tests/test_openrouter_free_canary.py b/tests/test_openrouter_free_canary.py index f13d857e5..1d2efc9e1 100644 --- a/tests/test_openrouter_free_canary.py +++ b/tests/test_openrouter_free_canary.py @@ -175,6 +175,29 @@ def chat(self, _agent, _messages): assert expired.stat().st_mode & 0o777 == 0o600 +def test_live_rejects_fifo_before_discovery_or_completion_transport(tmp_path: Path) -> None: + backend = InMemoryCredentialBackend() + backend.set("OPENROUTER_API_KEY", "secret") + set_backend(backend) + fifo = tmp_path / "evidence.fifo" + fifo_path = str(fifo) + import os + + os.mkfifo(fifo_path) + try: + with pytest.raises(OpenRouterCanaryError, match="regular file"): + run_openrouter_free_canary( + live=True, + limits=OpenRouterCanaryLimits(1, 8, 3, 7), + evidence_output=fifo, + discover=lambda *_a, **_k: pytest.fail("discovery transport"), + client_factory=lambda **_k: pytest.fail("completion transport"), + now=lambda: 100, + ) + finally: + set_backend(None) + + def test_cli_defaults_to_dry_run_and_live_requires_every_bound( monkeypatch, capsys ) -> None: From 65fe3b41b0dbeb3b1a3fe84c2b3055e29db5224a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 09:42:32 +0900 Subject: [PATCH 05/12] fix(canary): harden retention and CLI boundaries --- README.md | 9 ++- contextual_orchestrator/__main__.py | 64 +++++++++++++++-- contextual_orchestrator/openrouter_canary.py | 26 +++++++ tests/test_openrouter_free_canary.py | 76 ++++++++++++++++++-- 4 files changed, 162 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index ba2c19ab3..7e08bb031 100644 --- a/README.md +++ b/README.md @@ -150,8 +150,13 @@ uses only the KV-registered `OPENROUTER_API_KEY`, disables retries, makes one fixed prompt request, and writes atomic JSON evidence containing neither the prompt, response, nor credential. Before transport it removes expired evidence at the same output path and proves that a mode-`0600` atomic write is possible; -retention cleanup therefore runs whenever the canary is invoked. No workflow -invokes live mode automatically. +retention cleanup therefore runs whenever the canary is invoked. The file also +records `expires_at`; run +`python -m contextual_orchestrator openrouter-free-canary +--prune-expired-evidence ./openrouter-canary.json` from the operator's chosen +retention lifecycle. Cleanup reads only that local file, removes it at or after +its deadline, and never resolves a credential, discovers a model, or calls a +provider. This repository deliberately adds no scheduler. Seed the credential into the KV once at bootstrap: diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index ea3e89526..57f832f91 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -36,7 +36,12 @@ TaskOrchestrator, load_agents, ) -from .openrouter_canary import OpenRouterCanaryLimits, run_openrouter_free_canary +from .openrouter_canary import ( + OpenRouterCanaryError, + OpenRouterCanaryLimits, + prune_expired_openrouter_canary_evidence, + run_openrouter_free_canary, +) from .privacy_policy_analysis import ( analyze_discovered_privacy_policies, ) @@ -502,13 +507,45 @@ def main(argv: list[str] | None = None) -> None: parser.add_argument("--timeout-seconds", type=_positive_int) parser.add_argument("--evidence-output") parser.add_argument("--retention-days", type=_positive_int) + parser.add_argument( + "--prune-expired-evidence", + help="Remove this evidence file if its recorded retention deadline has passed; never contacts OpenRouter.", + ) args = parser.parse_args(arguments[1:]) + if args.prune_expired_evidence: + if args.live or any( + value is not None + for value in ( + args.max_requests, + args.max_output_tokens, + args.timeout_seconds, + args.evidence_output, + args.retention_days, + ) + ): + parser.error("--prune-expired-evidence cannot be combined with canary options") + try: + removed = prune_expired_openrouter_canary_evidence( + Path(args.prune_expired_evidence) + ) + except OpenRouterCanaryError as exc: + print( + json.dumps({"error": {"code": exc.code, "message": str(exc)}}), + file=sys.stderr, + ) + raise SystemExit(1) from None + print(json.dumps({"expired_evidence_removed": removed}, sort_keys=True)) + return supplied = ( args.max_requests, args.max_output_tokens, args.timeout_seconds, args.retention_days, ) + if not args.live and ( + any(value is not None for value in supplied) or args.evidence_output + ): + parser.error("live caps and --evidence-output require --live") if args.live and ( any(value is None for value in supplied) or not args.evidence_output ): @@ -516,18 +553,31 @@ def main(argv: list[str] | None = None) -> None: "--live requires positive request/output-token/timeout/retention caps and --evidence-output" ) limits = OpenRouterCanaryLimits(*supplied) if args.live else None - result = run_openrouter_free_canary( - live=args.live, - limits=limits, - evidence_output=Path(args.evidence_output) if args.evidence_output else None, - ) + try: + result = run_openrouter_free_canary( + live=args.live, + limits=limits, + evidence_output=Path(args.evidence_output) if args.evidence_output else None, + ) + except OpenRouterCanaryError as exc: + print( + json.dumps({"error": {"code": exc.code, "message": str(exc)}}), + file=sys.stderr, + ) + raise SystemExit(1) from None print(json.dumps(result, sort_keys=True)) return if arguments and arguments[0] == "check-fast-mlsirm": _check_fast_mlsirm_command() return - parser = argparse.ArgumentParser(description="Route or conduct chat requests across model agents.") + parser = argparse.ArgumentParser( + description="Route or conduct chat requests across model agents.", + epilog=( + "Commands: register-credential, discover-models, " + "openrouter-free-canary, check-fast-mlsirm" + ), + ) parser.add_argument("prompt", nargs="?", help="User prompt for CLI mode.") parser.add_argument("--agents", default="examples/agents.mock.json", help="Agent config JSON.") parser.add_argument("--state-db", default=os.environ.get("CONTEXTUAL_ORCHESTRATOR_STATE_DB", "") or None, diff --git a/contextual_orchestrator/openrouter_canary.py b/contextual_orchestrator/openrouter_canary.py index dc02c0a58..fd0b5cbab 100644 --- a/contextual_orchestrator/openrouter_canary.py +++ b/contextual_orchestrator/openrouter_canary.py @@ -27,6 +27,8 @@ class OpenRouterCanaryError(RuntimeError): """Raised before transport when the canary contract is incomplete.""" + code = "openrouter_canary_failed" + @dataclass(frozen=True) class OpenRouterCanaryLimits: @@ -51,7 +53,9 @@ def _eligible(models: list[DiscoveredModel]) -> list[DiscoveredModel]: model for model in models if model.provider_name == "openrouter" + and type(model.prompt_price_per_1k) is float and model.prompt_price_per_1k == 0.0 + and type(model.completion_price_per_1k) is float and model.completion_price_per_1k == 0.0 and model.is_free is True and not model.unit_prices @@ -111,6 +115,28 @@ def _prepare_evidence_path(path: Path, current_time: int) -> None: os.unlink(temporary_name) +def prune_expired_openrouter_canary_evidence( + path: Path, *, now: Callable[[], float] = time.time +) -> bool: + """Remove one expired evidence file without contacting a provider.""" + try: + document = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return False + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise OpenRouterCanaryError("canary evidence could not be inspected") from exc + expires_at = document.get("expires_at") if isinstance(document, dict) else None + if type(expires_at) is not int: + raise OpenRouterCanaryError("canary evidence has no valid expiry") + if int(now()) < expires_at: + return False + try: + path.unlink() + except FileNotFoundError: + return False + return True + + def run_openrouter_free_canary( *, live: bool, diff --git a/tests/test_openrouter_free_canary.py b/tests/test_openrouter_free_canary.py index 1d2efc9e1..f3c0704a0 100644 --- a/tests/test_openrouter_free_canary.py +++ b/tests/test_openrouter_free_canary.py @@ -8,6 +8,7 @@ from contextual_orchestrator.openrouter_canary import ( OpenRouterCanaryError, OpenRouterCanaryLimits, + prune_expired_openrouter_canary_evidence, run_openrouter_free_canary, ) from contextual_orchestrator import __main__ as cli @@ -44,6 +45,7 @@ def test_dry_run_selects_current_zero_price_without_transport() -> None: live=False, discover=lambda *_a, **_k: [ _model("unknown", prompt=None), + _model("paid-unit", is_free=False), _model("z-free"), _model("a-free"), ], @@ -92,6 +94,41 @@ def chat(self, agent, messages): assert "Reply OK" not in output.read_text() and "secret" not in output.read_text() +def test_live_fails_before_transport_when_evidence_path_is_unwritable( + tmp_path: Path, monkeypatch +) -> None: + backend = InMemoryCredentialBackend() + backend.set("OPENROUTER_API_KEY", "secret") + set_backend(backend) + monkeypatch.setattr( + "contextual_orchestrator.openrouter_canary.tempfile.mkstemp", + lambda **_kwargs: (_ for _ in ()).throw(PermissionError("denied")), + ) + try: + with pytest.raises(OpenRouterCanaryError, match="not writable"): + run_openrouter_free_canary( + live=True, + limits=OpenRouterCanaryLimits(1, 8, 3, 7), + evidence_output=tmp_path / "evidence.json", + discover=lambda *_a, **_k: pytest.fail("discovery after bad output"), + client_factory=lambda **_k: pytest.fail("transport"), + ) + finally: + set_backend(None) + + +def test_expired_evidence_cleanup_never_contacts_provider(tmp_path: Path) -> None: + output = tmp_path / "evidence.json" + output.write_text('{"expires_at": 100}\n') + assert prune_expired_openrouter_canary_evidence(output, now=lambda: 100) is True + assert not output.exists() + assert prune_expired_openrouter_canary_evidence(output, now=lambda: 101) is False + + output.write_text('{"expires_at": 200}\n') + assert prune_expired_openrouter_canary_evidence(output, now=lambda: 199) is False + assert output.exists() + + def test_canary_fails_closed_on_missing_credential_or_price() -> None: set_backend(InMemoryCredentialBackend()) try: @@ -139,9 +176,7 @@ def test_live_preflights_output_and_removes_expired_evidence(tmp_path: Path) -> backend.set("OPENROUTER_API_KEY", "secret") set_backend(backend) expired = tmp_path / "expired.json" - expired.write_text( - '{"provider":"openrouter","expires_at":99}', encoding="utf-8" - ) + expired.write_text('{"provider":"openrouter","expires_at":99}', encoding="utf-8") seen = {} class Client: @@ -175,7 +210,9 @@ def chat(self, _agent, _messages): assert expired.stat().st_mode & 0o777 == 0o600 -def test_live_rejects_fifo_before_discovery_or_completion_transport(tmp_path: Path) -> None: +def test_live_rejects_fifo_before_discovery_or_completion_transport( + tmp_path: Path, +) -> None: backend = InMemoryCredentialBackend() backend.set("OPENROUTER_API_KEY", "secret") set_backend(backend) @@ -212,3 +249,34 @@ def test_cli_defaults_to_dry_run_and_live_requires_every_bound( assert '"mode": "dry_run"' in capsys.readouterr().out with pytest.raises(SystemExit): cli.main(["openrouter-free-canary", "--live", "--max-requests", "1"]) + + +def test_cli_cleanup_and_contract_failures_have_controlled_exit( + monkeypatch, capsys +) -> None: + monkeypatch.setattr( + cli, "prune_expired_openrouter_canary_evidence", lambda _path: True + ) + cli.main(["openrouter-free-canary", "--prune-expired-evidence", "evidence.json"]) + assert '"expired_evidence_removed": true' in capsys.readouterr().out + + monkeypatch.setattr( + cli, + "run_openrouter_free_canary", + lambda **_kwargs: (_ for _ in ()).throw(OpenRouterCanaryError("safe failure")), + ) + with pytest.raises(SystemExit) as failure: + cli.main(["openrouter-free-canary"]) + captured = capsys.readouterr() + assert failure.value.code == 1 + assert captured.err == ( + '{"error": {"code": "openrouter_canary_failed", "message": "safe failure"}}\n' + ) + assert "Traceback" not in captured.err + + +def test_root_help_lists_canary_command(capsys) -> None: + with pytest.raises(SystemExit) as help_exit: + cli.main(["--help"]) + assert help_exit.value.code == 0 + assert "openrouter-free-canary" in capsys.readouterr().out From 499c657a69441b69ddfb8ef30eed26e79e086446 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 09:55:31 +0900 Subject: [PATCH 06/12] fix(canary): persist and validate live attempts --- README.md | 14 ++-- contextual_orchestrator/openrouter_canary.py | 78 ++++++++++++++------ tests/test_openrouter_free_canary.py | 66 +++++++++++++++-- 3 files changed, 124 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 7e08bb031..8f1efd8e2 100644 --- a/README.md +++ b/README.md @@ -148,15 +148,19 @@ python -m contextual_orchestrator openrouter-free-canary --live \ Every live cap and the evidence path/retention choice is mandatory. The command uses only the KV-registered `OPENROUTER_API_KEY`, disables retries, makes one fixed prompt request, and writes atomic JSON evidence containing neither the -prompt, response, nor credential. Before transport it removes expired evidence -at the same output path and proves that a mode-`0600` atomic write is possible; +prompt, response, nor credential. It records the attempt as `pending` before +transport, then atomically records the validated `OK` outcome. Before transport +it removes expired evidence at the same output path and proves that a +mode-`0600` atomic write is possible; unrelated, malformed, or unexpired files +are never overwritten; retention cleanup therefore runs whenever the canary is invoked. The file also records `expires_at`; run `python -m contextual_orchestrator openrouter-free-canary --prune-expired-evidence ./openrouter-canary.json` from the operator's chosen -retention lifecycle. Cleanup reads only that local file, removes it at or after -its deadline, and never resolves a credential, discovers a model, or calls a -provider. This repository deliberately adds no scheduler. +retention lifecycle. Cleanup reads only that local file, verifies the canary +schema/provider/mode identity, removes it at or after its deadline, and never +resolves a credential, discovers a model, or calls a provider. This repository +deliberately adds no scheduler. Seed the credential into the KV once at bootstrap: diff --git a/contextual_orchestrator/openrouter_canary.py b/contextual_orchestrator/openrouter_canary.py index fd0b5cbab..d2ebfc76c 100644 --- a/contextual_orchestrator/openrouter_canary.py +++ b/contextual_orchestrator/openrouter_canary.py @@ -88,6 +88,16 @@ def _write_evidence(path: Path, evidence: dict[str, Any]) -> None: raise +def _is_live_canary_evidence(document: object) -> bool: + """Return whether a document carries this canary's deletion identity.""" + return ( + isinstance(document, dict) + and document.get("schema_version") == 1 + and document.get("provider") == "openrouter" + and document.get("mode") == "live" + ) + + def _prepare_evidence_path(path: Path, current_time: int) -> None: """Remove expired prior evidence and prove a secure atomic write is possible.""" try: @@ -96,17 +106,19 @@ def _prepare_evidence_path(path: Path, current_time: int) -> None: mode = None if mode is not None and not stat.S_ISREG(mode): raise OpenRouterCanaryError("evidence output must be a regular file path") - try: - prior = json.loads(path.read_text(encoding="utf-8")) - except (FileNotFoundError, OSError, UnicodeError, json.JSONDecodeError): - prior = None - if ( - isinstance(prior, dict) - and prior.get("provider") == "openrouter" - and type(prior.get("expires_at")) is int - and prior["expires_at"] <= current_time - ): - path.unlink(missing_ok=True) + if mode is not None: + try: + prior = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise OpenRouterCanaryError("existing evidence could not be inspected") from exc + if not _is_live_canary_evidence(prior): + raise OpenRouterCanaryError("evidence output contains unrelated data") + expires_at = prior.get("expires_at") + if type(expires_at) is not int: + raise OpenRouterCanaryError("existing canary evidence has no valid expiry") + if expires_at > current_time: + raise OpenRouterCanaryError("unexpired canary evidence already exists") + path.unlink() path.parent.mkdir(parents=True, exist_ok=True) descriptor, temporary_name = tempfile.mkstemp( prefix=f".{path.name}.preflight.", dir=path.parent @@ -120,11 +132,17 @@ def prune_expired_openrouter_canary_evidence( ) -> bool: """Remove one expired evidence file without contacting a provider.""" try: - document = json.loads(path.read_text(encoding="utf-8")) + mode = path.lstat().st_mode except FileNotFoundError: return False + if not stat.S_ISREG(mode): + raise OpenRouterCanaryError("canary evidence must be a regular file") + try: + document = json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeError, json.JSONDecodeError) as exc: raise OpenRouterCanaryError("canary evidence could not be inspected") from exc + if not _is_live_canary_evidence(document): + raise OpenRouterCanaryError("file is not OpenRouter canary evidence") expires_at = document.get("expires_at") if isinstance(document, dict) else None if type(expires_at) is not int: raise OpenRouterCanaryError("canary evidence has no valid expiry") @@ -160,11 +178,10 @@ def run_openrouter_free_canary( "live mode requires all caps and an evidence output path" ) limits.validate() - discovered_at = int(now()) if live: assert evidence_output is not None try: - _prepare_evidence_path(evidence_output, discovered_at) + _prepare_evidence_path(evidence_output, int(now())) except OSError as exc: raise OpenRouterCanaryError("evidence output is not writable") from exc candidates = _eligible( @@ -177,6 +194,7 @@ def run_openrouter_free_canary( "current discovery has no unambiguous zero-price chat candidate" ) selected = candidates[0] + discovered_at = int(now()) evidence: dict[str, Any] = { "schema_version": 1, "mode": "live" if live else "dry_run", @@ -192,22 +210,34 @@ def run_openrouter_free_canary( } if live: assert limits is not None and evidence_output is not None - client = client_factory( - timeout=limits.timeout_seconds, - max_output_tokens=limits.max_output_tokens, - max_retries=0, - temperature=0.0, - ) - client.chat( - agent_from_discovered(selected), [{"role": "user", "content": "Reply OK."}] - ) evidence.update( { "request_count": 1, "limits": asdict(limits), "expires_at": discovered_at + limits.retention_days * 86400, - "outcome": "completed", + "outcome": "pending", } ) _write_evidence(evidence_output, evidence) + client = client_factory( + timeout=limits.timeout_seconds, + max_output_tokens=limits.max_output_tokens, + max_retries=0, + temperature=0.0, + ) + try: + response = client.chat( + agent_from_discovered(selected), + [{"role": "user", "content": "Reply OK."}], + ) + except Exception as exc: + evidence["outcome"] = "failed" + _write_evidence(evidence_output, evidence) + raise OpenRouterCanaryError("OpenRouter canary request failed") from exc + if not isinstance(response, str) or response.strip() != "OK": + evidence["outcome"] = "invalid_response" + _write_evidence(evidence_output, evidence) + raise OpenRouterCanaryError("OpenRouter canary returned an invalid response") + evidence["outcome"] = "completed" + _write_evidence(evidence_output, evidence) return evidence diff --git a/tests/test_openrouter_free_canary.py b/tests/test_openrouter_free_canary.py index f3c0704a0..f2d67e00b 100644 --- a/tests/test_openrouter_free_canary.py +++ b/tests/test_openrouter_free_canary.py @@ -1,5 +1,6 @@ """Contracts for the bounded OpenRouter free-model canary.""" +import json from pathlib import Path import pytest @@ -119,16 +120,34 @@ def test_live_fails_before_transport_when_evidence_path_is_unwritable( def test_expired_evidence_cleanup_never_contacts_provider(tmp_path: Path) -> None: output = tmp_path / "evidence.json" - output.write_text('{"expires_at": 100}\n') + output.write_text( + '{"schema_version":1,"provider":"openrouter","mode":"live","expires_at":100}\n' + ) assert prune_expired_openrouter_canary_evidence(output, now=lambda: 100) is True assert not output.exists() assert prune_expired_openrouter_canary_evidence(output, now=lambda: 101) is False - output.write_text('{"expires_at": 200}\n') + output.write_text( + '{"schema_version":1,"provider":"openrouter","mode":"live","expires_at":200}\n' + ) assert prune_expired_openrouter_canary_evidence(output, now=lambda: 199) is False assert output.exists() +def test_cleanup_rejects_unrelated_json_and_special_paths(tmp_path: Path) -> None: + unrelated = tmp_path / "unrelated.json" + unrelated.write_text('{"expires_at": 1}\n') + with pytest.raises(OpenRouterCanaryError, match="not OpenRouter"): + prune_expired_openrouter_canary_evidence(unrelated, now=lambda: 2) + assert unrelated.exists() + + symlink = tmp_path / "evidence-link.json" + symlink.symlink_to(unrelated) + with pytest.raises(OpenRouterCanaryError, match="regular file"): + prune_expired_openrouter_canary_evidence(symlink, now=lambda: 2) + assert unrelated.exists() + + def test_canary_fails_closed_on_missing_credential_or_price() -> None: set_backend(InMemoryCredentialBackend()) try: @@ -176,12 +195,17 @@ def test_live_preflights_output_and_removes_expired_evidence(tmp_path: Path) -> backend.set("OPENROUTER_API_KEY", "secret") set_backend(backend) expired = tmp_path / "expired.json" - expired.write_text('{"provider":"openrouter","expires_at":99}', encoding="utf-8") + expired.write_text( + '{"schema_version":1,"provider":"openrouter","mode":"live","expires_at":99}', + encoding="utf-8", + ) seen = {} class Client: def __init__(self, **_kwargs): - seen["expired_before_transport"] = not expired.exists() + seen["outcome_before_transport"] = json.loads( + expired.read_text(encoding="utf-8") + )["outcome"] def chat(self, _agent, _messages): return "OK" @@ -206,10 +230,42 @@ def chat(self, _agent, _messages): ) finally: set_backend(None) - assert seen["expired_before_transport"] is True + assert seen["outcome_before_transport"] == "pending" assert expired.stat().st_mode & 0o777 == 0o600 +def test_live_persists_attempt_and_validates_response(tmp_path: Path) -> None: + backend = InMemoryCredentialBackend() + backend.set("OPENROUTER_API_KEY", "secret") + set_backend(backend) + output = tmp_path / "evidence.json" + + class Client: + def __init__(self, **_kwargs): + pass + + def chat(self, _agent, _messages): + document = output.read_text(encoding="utf-8") + assert '"outcome": "pending"' in document + assert '"request_count": 1' in document + return "not ok" + + try: + with pytest.raises(OpenRouterCanaryError, match="invalid response"): + run_openrouter_free_canary( + live=True, + limits=OpenRouterCanaryLimits(1, 8, 3, 7), + evidence_output=output, + discover=lambda *_a, **_k: [_model("current-free")], + client_factory=Client, + now=iter((100, 200)).__next__, + ) + finally: + set_backend(None) + assert '"outcome": "invalid_response"' in output.read_text(encoding="utf-8") + assert '"discovered_at": 200' in output.read_text(encoding="utf-8") + + def test_live_rejects_fifo_before_discovery_or_completion_transport( tmp_path: Path, ) -> None: From 0ec2ade29bb5caf52d1c769a8c07cab0ccdd3ccb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 10:06:33 +0900 Subject: [PATCH 07/12] fix(canary): serialize evidence lifecycle --- contextual_orchestrator/openrouter_canary.py | 55 +++++++++++++++++++- docs/product-technical-gap-baseline.md | 16 ++++-- tests/test_openrouter_free_canary.py | 45 ++++++++++++++++ 3 files changed, 111 insertions(+), 5 deletions(-) diff --git a/contextual_orchestrator/openrouter_canary.py b/contextual_orchestrator/openrouter_canary.py index d2ebfc76c..f259adef3 100644 --- a/contextual_orchestrator/openrouter_canary.py +++ b/contextual_orchestrator/openrouter_canary.py @@ -2,6 +2,7 @@ from __future__ import annotations +from contextlib import contextmanager from dataclasses import asdict, dataclass import json import os @@ -9,7 +10,7 @@ import stat import tempfile import time -from typing import Any, Callable +from typing import Any, Callable, Iterator from .credentials import get_credential from .model_discovery import ( @@ -21,6 +22,7 @@ discover_provider_models, is_discovered_chat_candidate, ) +from .nim_evidence import NimEvidenceError, _publication_lock from .orchestrator import ModelClient @@ -30,6 +32,16 @@ class OpenRouterCanaryError(RuntimeError): code = "openrouter_canary_failed" +@contextmanager +def _evidence_lock(path: Path) -> Iterator[None]: + """Serialize operations for one evidence path using the shared safe lock.""" + try: + with _publication_lock(path): + yield + except NimEvidenceError as exc: + raise OpenRouterCanaryError("canary evidence lock is unavailable") from exc + + @dataclass(frozen=True) class OpenRouterCanaryLimits: """Explicit operator caps for the optional live request.""" @@ -131,6 +143,14 @@ def prune_expired_openrouter_canary_evidence( path: Path, *, now: Callable[[], float] = time.time ) -> bool: """Remove one expired evidence file without contacting a provider.""" + with _evidence_lock(path): + return _prune_expired_openrouter_canary_evidence(path, now=now) + + +def _prune_expired_openrouter_canary_evidence( + path: Path, *, now: Callable[[], float] +) -> bool: + """Inspect and remove one expired evidence file while its path is locked.""" try: mode = path.lstat().st_mode except FileNotFoundError: @@ -178,6 +198,39 @@ def run_openrouter_free_canary( "live mode requires all caps and an evidence output path" ) limits.validate() + assert evidence_output is not None + with _evidence_lock(evidence_output): + return _run_openrouter_free_canary_locked( + live=live, + source=source, + limits=limits, + evidence_output=evidence_output, + discover=discover, + client_factory=client_factory, + now=now, + ) + return _run_openrouter_free_canary_locked( + live=live, + source=source, + limits=limits, + evidence_output=evidence_output, + discover=discover, + client_factory=client_factory, + now=now, + ) + + +def _run_openrouter_free_canary_locked( + *, + live: bool, + source: Any, + limits: OpenRouterCanaryLimits | None, + evidence_output: Path | None, + discover: Callable[..., list[DiscoveredModel]], + client_factory: Callable[..., ModelClient], + now: Callable[[], float], +) -> dict[str, Any]: + """Run discovery and optional transport under the live evidence-path lock.""" if live: assert evidence_output is not None try: diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 8af9f18ee..050652605 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1956,10 +1956,15 @@ National Institute of Standards and Technology. (2024). *Artificial intelligence risk management framework: Generative artificial intelligence profile* (NIST AI 600-1). https://doi.org/10.6028/NIST.AI.600-1 +National Institute of Standards and Technology. (2020, updated 2023). +*Security and privacy controls for information systems and organizations* +(NIST SP 800-53 Rev. 5, release 5.1.1). https://doi.org/10.6028/NIST.SP.800-53r5 + These sources support the current product shape, OpenAI-compatible wire -honesty, deep-versus-shallow orchestration allocation, cache safety, and -generative-AI risk evidence. PDFs are attached only when redistribution is -permitted; otherwise the canonical citation and link are retained. +honesty, deep-versus-shallow orchestration allocation, cache safety, +audit-evidence protection and retention, and generative-AI risk evidence. PDFs +are attached only when redistribution is permitted; otherwise the canonical +citation and link are retained. ## 9. Design and ecosystem record @@ -2107,7 +2112,10 @@ Buyer-visible gaps now prioritized: prompt and completion prices are explicitly zero and comparable. Live mode is unscheduled and requires positive request, output-token, timeout, evidence retention, and output-path choices; it disables retries, pins no model id, - and persists neither credentials, prompts, nor responses. + and persists neither credentials, prompts, nor responses. Its atomic, + path-serialized attempt ledger and operator-chosen retention implement the + audit-record protection and retention intent of NIST SP 800-53 Rev. 5.1 + controls AU-9 and AU-11. 4. Multi-instance routing observations remain process-local. Add a time-windowed durable observation model with calibrated decay before horizontal scaling. 5. Protected main, not a feature-stack merge, remains the release boundary; do diff --git a/tests/test_openrouter_free_canary.py b/tests/test_openrouter_free_canary.py index f2d67e00b..d5a1898c2 100644 --- a/tests/test_openrouter_free_canary.py +++ b/tests/test_openrouter_free_canary.py @@ -1,7 +1,9 @@ """Contracts for the bounded OpenRouter free-model canary.""" +from concurrent.futures import ThreadPoolExecutor import json from pathlib import Path +import threading import pytest from contextual_orchestrator.credentials import InMemoryCredentialBackend, set_backend @@ -266,6 +268,49 @@ def chat(self, _agent, _messages): assert '"discovered_at": 200' in output.read_text(encoding="utf-8") +def test_live_serializes_one_evidence_path_across_invocations(tmp_path: Path) -> None: + backend = InMemoryCredentialBackend() + backend.set("OPENROUTER_API_KEY", "secret") + set_backend(backend) + output = tmp_path / "evidence.json" + entered = threading.Event() + release = threading.Event() + calls = [] + + class Client: + def __init__(self, **_kwargs): + pass + + def chat(self, _agent, _messages): + calls.append("transport") + entered.set() + assert release.wait(timeout=2) + return "OK" + + def invoke(): + return run_openrouter_free_canary( + live=True, + limits=OpenRouterCanaryLimits(1, 8, 3, 7), + evidence_output=output, + discover=lambda *_a, **_k: [_model("current-free")], + client_factory=Client, + now=lambda: 100, + ) + + try: + with ThreadPoolExecutor(max_workers=2) as pool: + first = pool.submit(invoke) + assert entered.wait(timeout=2) + second = pool.submit(invoke) + release.set() + assert first.result()["outcome"] == "completed" + with pytest.raises(OpenRouterCanaryError, match="already exists"): + second.result() + finally: + set_backend(None) + assert calls == ["transport"] + + def test_live_rejects_fifo_before_discovery_or_completion_transport( tmp_path: Path, ) -> None: From 55ccf97a3b2040a6ec38f9585a468e81b1f27257 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 10:11:53 +0900 Subject: [PATCH 08/12] fix(canary): support new evidence parents --- contextual_orchestrator/openrouter_canary.py | 6 ++++ tests/test_openrouter_free_canary.py | 31 ++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/contextual_orchestrator/openrouter_canary.py b/contextual_orchestrator/openrouter_canary.py index f259adef3..2a0ea9676 100644 --- a/contextual_orchestrator/openrouter_canary.py +++ b/contextual_orchestrator/openrouter_canary.py @@ -143,6 +143,8 @@ def prune_expired_openrouter_canary_evidence( path: Path, *, now: Callable[[], float] = time.time ) -> bool: """Remove one expired evidence file without contacting a provider.""" + if not path.parent.exists(): + return False with _evidence_lock(path): return _prune_expired_openrouter_canary_evidence(path, now=now) @@ -199,6 +201,10 @@ def run_openrouter_free_canary( ) limits.validate() assert evidence_output is not None + try: + evidence_output.parent.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise OpenRouterCanaryError("evidence output is not writable") from exc with _evidence_lock(evidence_output): return _run_openrouter_free_canary_locked( live=live, diff --git a/tests/test_openrouter_free_canary.py b/tests/test_openrouter_free_canary.py index d5a1898c2..37a7070d5 100644 --- a/tests/test_openrouter_free_canary.py +++ b/tests/test_openrouter_free_canary.py @@ -143,6 +143,10 @@ def test_cleanup_rejects_unrelated_json_and_special_paths(tmp_path: Path) -> Non prune_expired_openrouter_canary_evidence(unrelated, now=lambda: 2) assert unrelated.exists() + missing = tmp_path / "missing" / "evidence.json" + assert prune_expired_openrouter_canary_evidence(missing, now=lambda: 2) is False + assert not missing.parent.exists() + symlink = tmp_path / "evidence-link.json" symlink.symlink_to(unrelated) with pytest.raises(OpenRouterCanaryError, match="regular file"): @@ -311,6 +315,33 @@ def invoke(): assert calls == ["transport"] +def test_live_creates_nested_evidence_parent_before_locking(tmp_path: Path) -> None: + backend = InMemoryCredentialBackend() + backend.set("OPENROUTER_API_KEY", "secret") + set_backend(backend) + output = tmp_path / "new" / "nested" / "evidence.json" + + class Client: + def __init__(self, **_kwargs): + pass + + def chat(self, _agent, _messages): + return "OK" + + try: + result = run_openrouter_free_canary( + live=True, + limits=OpenRouterCanaryLimits(1, 8, 3, 7), + evidence_output=output, + discover=lambda *_a, **_k: [_model("current-free")], + client_factory=Client, + now=lambda: 100, + ) + finally: + set_backend(None) + assert result["outcome"] == "completed" and output.is_file() + + def test_live_rejects_fifo_before_discovery_or_completion_transport( tmp_path: Path, ) -> None: From c4b48d0721617f631a4524759695eedebdf5f6de Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 10:16:30 +0900 Subject: [PATCH 09/12] fix(canary): surface inaccessible evidence parents --- contextual_orchestrator/openrouter_canary.py | 10 +++++++++- tests/test_openrouter_free_canary.py | 7 +++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/openrouter_canary.py b/contextual_orchestrator/openrouter_canary.py index 2a0ea9676..c6c531eea 100644 --- a/contextual_orchestrator/openrouter_canary.py +++ b/contextual_orchestrator/openrouter_canary.py @@ -143,8 +143,16 @@ def prune_expired_openrouter_canary_evidence( path: Path, *, now: Callable[[], float] = time.time ) -> bool: """Remove one expired evidence file without contacting a provider.""" - if not path.parent.exists(): + try: + parent_mode = path.parent.stat().st_mode + except FileNotFoundError: return False + except OSError as exc: + raise OpenRouterCanaryError( + "canary evidence directory could not be inspected" + ) from exc + if not stat.S_ISDIR(parent_mode): + raise OpenRouterCanaryError("canary evidence parent must be a directory") with _evidence_lock(path): return _prune_expired_openrouter_canary_evidence(path, now=now) diff --git a/tests/test_openrouter_free_canary.py b/tests/test_openrouter_free_canary.py index 37a7070d5..6c4748819 100644 --- a/tests/test_openrouter_free_canary.py +++ b/tests/test_openrouter_free_canary.py @@ -147,6 +147,13 @@ def test_cleanup_rejects_unrelated_json_and_special_paths(tmp_path: Path) -> Non assert prune_expired_openrouter_canary_evidence(missing, now=lambda: 2) is False assert not missing.parent.exists() + non_directory = tmp_path / "not-a-directory" + non_directory.write_text("occupied", encoding="utf-8") + with pytest.raises(OpenRouterCanaryError, match="parent must be a directory"): + prune_expired_openrouter_canary_evidence( + non_directory / "evidence.json", now=lambda: 2 + ) + symlink = tmp_path / "evidence-link.json" symlink.symlink_to(unrelated) with pytest.raises(OpenRouterCanaryError, match="regular file"): From b6a2cb3a0f099bbffffbe651ce39547b3e742920 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 21:52:49 +0000 Subject: [PATCH 10/12] fix(canary): remove duplicate import, correct bootstrap credential CodeRabbit found two real issues on this branch's merge with main: - contextual_orchestrator/__main__.py imported the same four openrouter_canary symbols twice (once from this branch's own history, once left over after main's independent extraction into _openrouter_free_canary_command). Removed the duplicate. - README's canary bootstrap example registered OPENAI_API_KEY immediately after stating the canary only uses OPENROUTER_API_KEY, so following the docs verbatim produces a credential-missing failure. Corrected to OPENROUTER_API_KEY. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01BV96rXhqoR3tYZ9AeAVur4 --- README.md | 2 +- contextual_orchestrator/__main__.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/README.md b/README.md index 01a8f3bf6..1e4c11f6e 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ deliberately adds no scheduler. Seed the credential into the KV once at bootstrap: ```bash -echo "$OPENAI_API_KEY" | python -m contextual_orchestrator register-credential --name OPENAI_API_KEY --value-stdin +echo "$OPENROUTER_API_KEY" | python -m contextual_orchestrator register-credential --name OPENROUTER_API_KEY --value-stdin ``` For a persistent KV-backed server token, seed a credential such as diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index b8b64d6a8..9c6150bb0 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -45,12 +45,6 @@ load_agents, redact_text, ) -from .openrouter_canary import ( - OpenRouterCanaryError, - OpenRouterCanaryLimits, - prune_expired_openrouter_canary_evidence, - run_openrouter_free_canary, -) from .privacy_policy_analysis import ( analyze_discovered_privacy_policies, ) From 980476355c30df0f2343b1aa130532ee046280af Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 10:44:23 +0000 Subject: [PATCH 11/12] fix(tests): eliminate CI-only embedding batch race by waiting for terminal status Ported from contextual-orchestrator#1025 (unrelated to this PR's own docs-only diff, needed to unblock the required "Full unit and contract suite" check). See that PR for the RED/GREEN reproduction via an artificial-delay injection into ProviderEmbeddingBatchBackend._run_job. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01BV96rXhqoR3tYZ9AeAVur4 --- CHANGELOG.md | 12 ++++++++++++ tests/test_provider_embedding_batch_backend.py | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7000d9aa8..81f487227 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,18 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- Fixed a CI-only race in + `tests/test_provider_embedding_batch_backend.py`: two tests called + `CostRoutingCoordinator.complete_embeddings_batch()` against a provider + (async, `ThreadPoolExecutor`-backed) embedding agent without a + `wait_timeout`, then asserted on the returned document's `total_tokens` + immediately. Under light load the background job usually finished before + the immediate poll; under CI's heavier concurrent load it sometimes had + not, and the poll returned a non-terminal document lacking + `total_tokens` (`KeyError`). Both tests now pass `wait_timeout=1`, + matching the pattern already used by sibling tests in the same file for + this exact provider-backend-plus-immediate-check shape. Ported from + #1025; unrelated to this PR's own docs-only diff. - Workflow workers now preserve the caller message array exactly once, while the added envelope carries only the subtask and Conductor-style prior-step access list instead of duplicating the task or source attachments. diff --git a/tests/test_provider_embedding_batch_backend.py b/tests/test_provider_embedding_batch_backend.py index 0eb661fbb..a99004f1e 100644 --- a/tests/test_provider_embedding_batch_backend.py +++ b/tests/test_provider_embedding_batch_backend.py @@ -64,7 +64,7 @@ def test_unknown_tokenizer_uses_authoritative_provider_usage() -> None: embedding_token_counter=UnavailableEmbeddingTokenCounter(), ) - document = coordinator.complete_embeddings_batch(["synthetic input"]) + document = coordinator.complete_embeddings_batch(["synthetic input"], wait_timeout=1) assert document["status"] == "completed" assert document["total_tokens"] == len("synthetic input".encode("utf-8")) @@ -124,7 +124,7 @@ def test_unknown_tokenizer_byte_bound_never_becomes_recorded_usage(text) -> None embedding_token_counter=UnavailableEmbeddingTokenCounter(), ) - document = coordinator.complete_embeddings_batch([text]) + document = coordinator.complete_embeddings_batch([text], wait_timeout=1) assert document["total_tokens"] == len(text.encode("utf-8")) From e527eafb59e56a339972ce9e7e066583692b46a5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 16:22:54 +0900 Subject: [PATCH 12/12] docs(discovery): isolate OpenRouter canary delta --- CHANGELOG.md | 12 ------------ README.md | 2 +- docs/product-technical-gap-baseline.md | 2 +- tests/test_provider_embedding_batch_backend.py | 4 ++-- 4 files changed, 4 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e626321b..7187e194b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,18 +26,6 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed -- Fixed a CI-only race in - `tests/test_provider_embedding_batch_backend.py`: two tests called - `CostRoutingCoordinator.complete_embeddings_batch()` against a provider - (async, `ThreadPoolExecutor`-backed) embedding agent without a - `wait_timeout`, then asserted on the returned document's `total_tokens` - immediately. Under light load the background job usually finished before - the immediate poll; under CI's heavier concurrent load it sometimes had - not, and the poll returned a non-terminal document lacking - `total_tokens` (`KeyError`). Both tests now pass `wait_timeout=1`, - matching the pattern already used by sibling tests in the same file for - this exact provider-backend-plus-immediate-check shape. Ported from - #1025; unrelated to this PR's own docs-only diff. - Workflow workers now preserve the caller message array exactly once, while the added envelope carries only the subtask and Conductor-style prior-step access list instead of duplicating the task or source attachments. diff --git a/README.md b/README.md index 1e4c11f6e..f97979228 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ schema/provider/mode identity, removes it at or after its deadline, and never resolves a credential, discovers a model, or calls a provider. This repository deliberately adds no scheduler. -Seed the credential into the KV once at bootstrap: +Seed the OpenRouter credential into the KV once at bootstrap: ```bash echo "$OPENROUTER_API_KEY" | python -m contextual_orchestrator register-credential --name OPENROUTER_API_KEY --value-stdin diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index e22f56034..5c45b2e7d 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2385,7 +2385,7 @@ Buyer-visible gaps now prioritized: keyboard/native-form REST editor; DB membership is normalized and legacy JSON membership migrates without data loss. Authenticated deployed-browser runtime evidence remains a release/UAT gate rather than an implementation gap. -3. **Implemented in the current product-gap branch:** the OpenRouter free-model +3. **Implemented on protected `main`:** the OpenRouter free-model canary is dry-run-only by default and selects a current chat row only when prompt and completion prices are explicitly zero and comparable. Live mode is unscheduled and requires positive request, output-token, timeout, evidence diff --git a/tests/test_provider_embedding_batch_backend.py b/tests/test_provider_embedding_batch_backend.py index a99004f1e..0eb661fbb 100644 --- a/tests/test_provider_embedding_batch_backend.py +++ b/tests/test_provider_embedding_batch_backend.py @@ -64,7 +64,7 @@ def test_unknown_tokenizer_uses_authoritative_provider_usage() -> None: embedding_token_counter=UnavailableEmbeddingTokenCounter(), ) - document = coordinator.complete_embeddings_batch(["synthetic input"], wait_timeout=1) + document = coordinator.complete_embeddings_batch(["synthetic input"]) assert document["status"] == "completed" assert document["total_tokens"] == len("synthetic input".encode("utf-8")) @@ -124,7 +124,7 @@ def test_unknown_tokenizer_byte_bound_never_becomes_recorded_usage(text) -> None embedding_token_counter=UnavailableEmbeddingTokenCounter(), ) - document = coordinator.complete_embeddings_batch([text], wait_timeout=1) + document = coordinator.complete_embeddings_batch([text]) assert document["total_tokens"] == len(text.encode("utf-8"))