From 8737f5845069b1732c1d285048a7b957ec704093 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:17:51 +0900 Subject: [PATCH 01/87] fix(server): correlate error responses with safe request logs --- contextual_orchestrator/server.py | 7 +++++-- docs/product-technical-gap-baseline.md | 18 ++++++++++++++++++ tests/test_telemetry.py | 13 +++++++++++-- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index eb6a77519..112e047b7 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -8193,8 +8193,11 @@ def _send_error( message: str, detail: dict[str, Any] | None = None, ) -> None: - _LOGGER.warning("request_failed status=%s code=%s", status, code) - self._send(_error_payload(code, message, {"request_id": uuid.uuid4().hex, **(detail or {})}), status) + request_id = uuid.uuid4().hex + _LOGGER.warning( + "request_failed status=%s code=%s request_id=%s", status, code, request_id + ) + self._send(_error_payload(code, message, {**(detail or {}), "request_id": request_id}), status) def _write_response(self, writer: Callable[[], None]) -> bool: """Run a response-writing callback, swallowing a dead-peer disconnect. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index d145a0b1d..e2be75366 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,23 @@ # Contextual Orchestrator: Product & Technical Gap Baseline +## 2026-09-08 error-response correlation repair + +ConceptWeave run 33938445050, job 101256562088, preserves a client-side HTTP +500 with request ID `175d6d59c5294b0e8a21548193b90482`. Its surviving artifact +9969701340 contains gateway stderr but only generic request-failure messages; +it cannot correlate that ID to an internal cause. The job installed CO source +`2e414d15ba58f28597751b625a8a2f00fc9fadcf`. This is not proof of free-pool +exhaustion, a disappeared run, or a currently released fix. + +The same correlation gap was reproduced on main +`414f22973658c4ddc3d4320fcf7acd9b4e8ba991`: the common HTTP error response had +a generated ID absent from its log. The proposed repair generates one ID for +both response and warning, prevents detail fields from overriding it, and logs +neither session values nor error details. RED: one missing-correlation failure; +GREEN: 45 telemetry tests passed in 6.58 seconds. This improves future failure +correlation only; it does not recover the historical exception, cover every +streaming-error path, or prove immutable publication or deployed behavior. + ## 2026-09-01 Autonomous Commercialization Loop: PR #970 Merge, Token Accounting & Cost Gateway Harmonization Observation time: 2026-09-01 Asia/Seoul. diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 542cf2a23..454cdabe0 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -384,16 +384,25 @@ def test_http_error_log_excludes_raw_session_id(monkeypatch, caplog): server = build_server(SimpleNamespace(agents=[], candidates=[]), port=0) handler = server.RequestHandlerClass.__new__(server.RequestHandlerClass) handler.path = "/v1/chat/completions" - monkeypatch.setattr(handler, "_send", lambda *_args, **_kwargs: None) + captured_send = MagicMock() + monkeypatch.setattr(handler, "_send", captured_send) token = set_session_id("session-secret") try: with caplog.at_level("WARNING"): - handler._send_error(401, "unauthorized", "not authorized") + handler._send_error( + 401, "unauthorized", "not authorized", + {"request_id": "untrusted\nlog-injection", "reason": "private-detail"}, + ) finally: reset_session_id(token) server.server_close() assert "request_failed" in caplog.text + response_payload = captured_send.call_args.args[0] + assert response_payload["error"]["detail"]["request_id"] in caplog.text + assert response_payload["error"]["detail"]["reason"] == "private-detail" + assert "untrusted" not in caplog.text + assert "private-detail" not in caplog.text assert "session-secret" not in caplog.text From 217b21f481edc94689e09681e0cb9d6454a5f817 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:28:35 +0900 Subject: [PATCH 02/87] test(server): assert exact safe error correlation identity --- tests/test_telemetry.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 454cdabe0..897532329 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -399,7 +399,18 @@ def test_http_error_log_excludes_raw_session_id(monkeypatch, caplog): assert "request_failed" in caplog.text response_payload = captured_send.call_args.args[0] - assert response_payload["error"]["detail"]["request_id"] in caplog.text + response_request_id = response_payload["error"]["detail"]["request_id"] + warning_messages = [ + record.getMessage() for record in caplog.records + if record.name == "contextual_orchestrator.server" + and record.getMessage().startswith("request_failed ") + ] + assert warning_messages == [ + f"request_failed status=401 code=unauthorized request_id={response_request_id}" + ] + assert response_request_id != "untrusted\nlog-injection" + assert len(response_request_id) == 32 + assert all(character in "0123456789abcdef" for character in response_request_id) assert response_payload["error"]["detail"]["reason"] == "private-detail" assert "untrusted" not in caplog.text assert "private-detail" not in caplog.text From 2cb6db5d1d4c041ae0d4a4193c88a233731163c7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:38:02 +0900 Subject: [PATCH 03/87] test(server): verify error correlation over real HTTP --- tests/test_telemetry.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 897532329..17448e17f 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -417,6 +417,40 @@ def test_http_error_log_excludes_raw_session_id(monkeypatch, caplog): assert "session-secret" not in caplog.text +def test_http_error_ids_correlate_over_real_connections(caplog): + """Separate HTTP errors carry distinct IDs matching their server warnings.""" + import http.client + import threading + + server = build_server(SimpleNamespace(agents=[], candidates=[]), port=0) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + request_ids = [] + try: + with caplog.at_level("WARNING", logger="contextual_orchestrator.server"): + for _request_index in range(2): + connection = http.client.HTTPConnection(*server.server_address, timeout=5) + try: + connection.request("GET", "/v1/models") + response = connection.getresponse() + assert response.status == 401 + payload = json.loads(response.read()) + request_id = payload["error"]["detail"]["request_id"] + request_ids.append(request_id) + expected_message = ( + f"request_failed status=401 code={payload['error']['code']} " + f"request_id={request_id}" + ) + assert expected_message in [record.getMessage() for record in caplog.records] + finally: + connection.close() + finally: + server.shutdown() + server_thread.join(timeout=5) + server.server_close() + assert len(set(request_ids)) == 2 + + def test_http_diagnostics_exclude_raw_path_and_swallow_client_disconnect(monkeypatch, caplog): """Client cancellation cannot create a second error or leak path identifiers.""" server = build_server(SimpleNamespace(agents=[], candidates=[]), port=0) From d7bba88f3d711883a37effe49ab4f503c4fb8e01 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 12:01:40 +0900 Subject: [PATCH 04/87] fix(persistence): roll back failed state replacements --- contextual_orchestrator/orchestrator.py | 3 +-- tests/test_persistence.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index fa85ba533..62f9a6d3c 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -3725,14 +3725,13 @@ def save(self, kind: str, key: str | None, payload: dict[str, Any], *, durable: def _save_sync(self, kind: str, key: str | None, payload: dict[str, Any]) -> None: blob = json.dumps(payload, ensure_ascii=False) - with self._lock: + with self._lock, self._conn: if kind in self._KEYED: self._conn.execute(self._DELETE_KEYED_SQL, (kind, key)) self._conn.execute(self._INSERT_SQL, (kind, key, blob)) if kind in self._STREAM_LIMITS: limit = self._STREAM_LIMITS[kind] self._conn.execute(self._PRUNE_STREAM_SQL, (kind, kind, limit)) - self._conn.commit() def _drain_stream_queue(self) -> None: while True: diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 923c026df..e4baac801 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -121,6 +121,31 @@ def test_store_upserts_keyed_records_and_appends_streams() -> None: store.close() +def test_failed_keyed_save_preserves_previous_committed_record() -> None: + """A failed replacement must not leak its deletion into the next commit.""" + with tempfile.TemporaryDirectory() as directory: + store = _StateStore(os.path.join(directory, "state.db")) + try: + store.save("workflow_run", "run_existing", {"version": 1}) + store._conn.execute( + "CREATE TRIGGER reject_replacement BEFORE INSERT ON orchestration_records " + "WHEN NEW.payload = '{\"version\": 2}' " + "BEGIN SELECT RAISE(FAIL, 'injected write failure'); END" + ) + store._conn.commit() + try: + store.save("workflow_run", "run_existing", {"version": 2}) + except sqlite3.IntegrityError: + pass + else: + raise AssertionError("the injected write failure did not occur") + store.save("workflow_run", "run_other", {"version": 3}) + assert store.load("workflow_run") == [{"version": 1}, {"version": 3}] + assert not store._conn.in_transaction + finally: + store.close() + + def test_store_treats_kind_key_and_limit_as_sql_parameters() -> None: with tempfile.TemporaryDirectory() as directory: store = _StateStore(os.path.join(directory, "s.db")) From 1beb72c125d3611978464e971997f187a33f238f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 12:02:34 +0900 Subject: [PATCH 05/87] docs(persistence): record state-write rollback reproduction --- docs/doctoring/autonomous_kpi_runbook.md | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/doctoring/autonomous_kpi_runbook.md b/docs/doctoring/autonomous_kpi_runbook.md index 70d937e1b..b7cef1aff 100644 --- a/docs/doctoring/autonomous_kpi_runbook.md +++ b/docs/doctoring/autonomous_kpi_runbook.md @@ -84,6 +84,33 @@ full visual-inspection requirement complete from this receipt. ## Break release cycles without copying implementation +## State-write atomicity prerequisite (2026-09-09) + +At CO `ab8a7caa6c00a49eede17a03d0897865cfdce9f5`, `run()` calls +`complete()` before saving its workflow record. Its duration includes generation +and cannot represent accepted-request-to-decision latency. Analytics stream +`save()` normally enqueues, whereas `durable=True` reaches the synchronous +commit path. Neither queue acceptance nor final workflow storage is a substitute +for a pre-invocation decision acknowledgement. + +Tracing that common synchronous path exposed an atomicity defect: a keyed +replacement deletes the old row before inserting the new one. An insertion +failure left the deletion in an open transaction, and a subsequent unrelated +save committed it. A real SQLite trigger injecting an insertion failure reproduced +the loss: the previous version disappeared. The new regression failed before +the fix (1 failed, 18 deselected, 6.17s). + +Code commit `d7bba88f3d711883a37effe49ab4f503c4fb8e01` uses the existing +connection's transaction context inside the existing lock. Success commits; +failure rolls back before another writer acquires the lock. No new dependency, +schema, numerical implementation, or production routing default was introduced. +The same project-local Python environment ran `python -m pytest +tests/test_persistence.py -q` from the isolated worktree: 19 passed in 16.38s. +This establishes the tested SQLite failure case, not customer KPI improvement, +full-suite success, protected merge, or production deployment. Next implement +decision timing before provider invocation with explicit failure denominators; +do not relabel existing response-generation timings. + Minimum contract → owner RED test → owner implementation → exact-SHA/digest isolated real integration → protected immutable release → consumer adoption. From aa674187b0341c7852f85c27fb696aec21f1a799 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 12:02:47 +0900 Subject: [PATCH 06/87] docs: separate atomicity and release-cycle guidance --- docs/doctoring/autonomous_kpi_runbook.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/doctoring/autonomous_kpi_runbook.md b/docs/doctoring/autonomous_kpi_runbook.md index b7cef1aff..27cc08b3f 100644 --- a/docs/doctoring/autonomous_kpi_runbook.md +++ b/docs/doctoring/autonomous_kpi_runbook.md @@ -82,8 +82,6 @@ UI acceptance. Viewport dimensions, durable image export, responsive sizes, other locales, and error/loading states remain unverified. Do not mark the full visual-inspection requirement complete from this receipt. -## Break release cycles without copying implementation - ## State-write atomicity prerequisite (2026-09-09) At CO `ab8a7caa6c00a49eede17a03d0897865cfdce9f5`, `run()` calls @@ -111,6 +109,8 @@ full-suite success, protected merge, or production deployment. Next implement decision timing before provider invocation with explicit failure denominators; do not relabel existing response-generation timings. +## Break release cycles without copying implementation + Minimum contract → owner RED test → owner implementation → exact-SHA/digest isolated real integration → protected immutable release → consumer adoption. From f1abe1e378471fc680ee426a2020ef92e4a24ba3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 12:03:54 +0900 Subject: [PATCH 07/87] test(persistence): verify rollback survives reopening --- tests/test_persistence.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index e4baac801..6e307f334 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -139,11 +139,17 @@ def test_failed_keyed_save_preserves_previous_committed_record() -> None: pass else: raise AssertionError("the injected write failure did not occur") + assert not store._conn.in_transaction store.save("workflow_run", "run_other", {"version": 3}) assert store.load("workflow_run") == [{"version": 1}, {"version": 3}] assert not store._conn.in_transaction finally: store.close() + reopened = _StateStore(os.path.join(directory, "state.db")) + try: + assert reopened.load("workflow_run") == [{"version": 1}, {"version": 3}] + finally: + reopened.close() def test_store_treats_kind_key_and_limit_as_sql_parameters() -> None: From 877d5112ed470d851afaa2c746b94393cc768ee7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 12:04:17 +0900 Subject: [PATCH 08/87] docs(gap): track persistence fix and skipped review evidence --- docs/product-technical-gap-baseline.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index f89ee7553..be673655c 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,22 @@ # Contextual Orchestrator: Product & Technical Gap Baseline +## 2026-09-09 State persistence integrity prerequisite + +PR [#1108](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1108) +repairs a reproduced failed-replacement data-loss case in the common SQLite +state writer. Code `d7bba88f3d711883a37effe49ab4f503c4fb8e01` rolls back failed +writes under the existing lock. Test follow-up `f1abe1e3` checks that no +transaction remains immediately after failure and that both the previous and +unrelated subsequent records survive reopening the database. The persistence +suite passed 19 tests in 9.28s; this is not a latency or customer-accuracy result. + +At the earlier PR head `aa674187b0341c7852f85c27fb696aec21f1a799`, GitHub +reported zero check runs and two success statuses whose descriptions explicitly +said reviews were skipped (Draft; expired trial/no credits). Those statuses do +not establish review approval or security validation. Keep protected merge and +release pending actual exact-head evidence. The root cause and reproduction are +in the [canonical runbook](doctoring/autonomous_kpi_runbook.md). + ## 2026-09-09 Numerical candidate and rendered KPI evidence The [autonomous KPI runbook](doctoring/autonomous_kpi_runbook.md) records a From 716e012dcb50857000b0fc53c89c6434fdf7e7c2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 12:28:21 +0900 Subject: [PATCH 09/87] test(persistence): cover deferred commit rollback --- tests/test_persistence.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 6e307f334..492f5493f 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -16,6 +16,8 @@ import threading import time +import pytest + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 @@ -121,17 +123,29 @@ def test_store_upserts_keyed_records_and_appends_streams() -> None: store.close() -def test_failed_keyed_save_preserves_previous_committed_record() -> None: +@pytest.mark.parametrize("failure_phase", ["insert", "commit"]) +def test_failed_keyed_save_preserves_previous_committed_record(failure_phase: str) -> None: """A failed replacement must not leak its deletion into the next commit.""" with tempfile.TemporaryDirectory() as directory: store = _StateStore(os.path.join(directory, "state.db")) try: + store._conn.execute("PRAGMA foreign_keys = ON") store.save("workflow_run", "run_existing", {"version": 1}) - store._conn.execute( - "CREATE TRIGGER reject_replacement BEFORE INSERT ON orchestration_records " - "WHEN NEW.payload = '{\"version\": 2}' " - "BEGIN SELECT RAISE(FAIL, 'injected write failure'); END" - ) + if failure_phase == "insert": + store._conn.execute( + "CREATE TRIGGER reject_replacement BEFORE INSERT ON orchestration_records " + "WHEN NEW.payload = '{\"version\": 2}' " + "BEGIN SELECT RAISE(FAIL, 'injected write failure'); END" + ) + else: + # Both writes succeed; the deferred constraint fails only at commit. + store._conn.execute( + "CREATE TABLE linked_record (record_seq INTEGER REFERENCES " + "orchestration_records(seq) DEFERRABLE INITIALLY DEFERRED)" + ) + store._conn.execute( + "INSERT INTO linked_record SELECT seq FROM orchestration_records" + ) store._conn.commit() try: store.save("workflow_run", "run_existing", {"version": 2}) From 4316be850175c1832cc5524c13203c5baff60c23 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 12:28:53 +0900 Subject: [PATCH 10/87] docs(persistence): record full-suite and commit-failure evidence --- docs/doctoring/autonomous_kpi_runbook.md | 12 ++++++++++++ docs/product-technical-gap-baseline.md | 7 +++++++ 2 files changed, 19 insertions(+) diff --git a/docs/doctoring/autonomous_kpi_runbook.md b/docs/doctoring/autonomous_kpi_runbook.md index 27cc08b3f..3961d0089 100644 --- a/docs/doctoring/autonomous_kpi_runbook.md +++ b/docs/doctoring/autonomous_kpi_runbook.md @@ -109,6 +109,18 @@ full-suite success, protected merge, or production deployment. Next implement decision timing before provider invocation with explicit failure denominators; do not relabel existing response-generation timings. +Follow-up validation: full `python -m pytest -q` at +`877d5112ed470d851afaa2c746b94393cc768ee7` exited 0 with 3,396 passed and +2 skipped in 883.03s. The quiet log does not identify the skip reasons; neither +skip is counted as passed. Test-only commit +`716e012dcb50857000b0fc53c89c6434fdf7e7c2` extends the regression to a real +deferred foreign-key violation at commit, in addition to insertion failure. +The existing transaction implementation is unchanged. At that commit, +`python -m pytest tests/test_persistence.py +tests/test_workflow_run_object_authorization.py tests/test_governance_runtime.py -q` +exited 0: 29 passed in 4.89s. The new test head has not had a full-suite run; +these local results do not establish hosted security checks or independent review. + ## Break release cycles without copying implementation Minimum contract → owner RED test → owner implementation → exact-SHA/digest diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index be673655c..2d1881ff1 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -10,6 +10,13 @@ transaction remains immediately after failure and that both the previous and unrelated subsequent records survive reopening the database. The persistence suite passed 19 tests in 9.28s; this is not a latency or customer-accuracy result. +Full local suite at `877d5112ed470d851afaa2c746b94393cc768ee7`: 3,396 passed, +2 skipped, exit 0 (883.03s). Test-only follow-up +`716e012dcb50857000b0fc53c89c6434fdf7e7c2` covers a deferred commit failure +with real SQLite constraints; persistence, workflow authorization, and governance +tests pass together (29 passed, 4.89s). Full-suite evidence remains attached to +the earlier head, not silently reassigned to the new regression. + At the earlier PR head `aa674187b0341c7852f85c27fb696aec21f1a799`, GitHub reported zero check runs and two success statuses whose descriptions explicitly said reviews were skipped (Draft; expired trial/no credits). Those statuses do From f7f05569a54a51c4cd6e42418a0c079b032d45f6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 12:43:45 +0900 Subject: [PATCH 11/87] test(telemetry): reproduce missing provider request correlation --- tests/test_telemetry.py | 53 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 17448e17f..3489bdff2 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -451,6 +451,59 @@ def test_http_error_ids_correlate_over_real_connections(caplog): assert len(set(request_ids)) == 2 +def test_provider_attempts_share_http_error_identity(monkeypatch, caplog): + """Same-session HTTP requests need distinct identities before provider failure.""" + import http.client + import threading + from contextual_orchestrator import TaskOrchestrator + from contextual_orchestrator.server import SecurityConfig + + model_agent = ModelAgent("correlation_agent", "mock-model") + model_client = ModelClient(max_retries=0) + router = TaskOrchestrator([model_agent], client=model_client) + + def reject_send(*args, **kwargs): + raise RuntimeError("controlled provider failure") + + def fail_completion(*args, **kwargs): + return model_client._send_with_retry(model_agent, {}) + + monkeypatch.setattr(model_client, "_send", reject_send) + monkeypatch.setattr(router, "complete", fail_completion) + server = build_server(router, port=0, security=SecurityConfig(auth_token="test-correlation-token")) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + connection = http.client.HTTPConnection(*server.server_address, timeout=5) + request_ids = [] + try: + with caplog.at_level("DEBUG"): + for request_index in range(2): + first_record = len(caplog.records) + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "mock-model", "messages": [{"role": "user", "content": "unit request"}], + }), { + "Content-Type": "application/json", "Authorization": "Bearer test-correlation-token", + "X-LineageWeave-Session-Id": "shared-private-session", + }) + response = connection.getresponse() + assert response.status >= 400 + response_body = json.loads(response.read()) + request_id = response_body["error"]["detail"]["request_id"] + request_ids.append(request_id) + attempt_logs = [record.getMessage() for record in caplog.records[first_record:] + if record.getMessage().startswith(("provider_attempt ", "provider_attempt_failed "))] + assert len(attempt_logs) == 2, (request_index, attempt_logs) + assert all(f"request_id={request_id}" in message for message in attempt_logs) + assert len(set(request_ids)) == 2 + assert "shared-private-session" not in "\n".join(attempt_logs) + finally: + connection.close() + server.shutdown() + server_thread.join(timeout=5) + server.server_close() + router.close() + + def test_http_diagnostics_exclude_raw_path_and_swallow_client_disconnect(monkeypatch, caplog): """Client cancellation cannot create a second error or leak path identifiers.""" server = build_server(SimpleNamespace(agents=[], candidates=[]), port=0) From c7468ecea009bae3a62413b4c54ab5a9b242bdff Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 12:45:47 +0900 Subject: [PATCH 12/87] fix(telemetry): bind provider attempts to HTTP request identity --- contextual_orchestrator/orchestrator.py | 7 +++++-- contextual_orchestrator/server.py | 15 +++++++++------ contextual_orchestrator/telemetry.py | 20 ++++++++++++++++++++ tests/test_telemetry.py | 19 +++++++++++++++++++ 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index fa85ba533..f03127ad8 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -56,6 +56,7 @@ ) from .telemetry import ( annotate_current_span, + current_request_id, inject_trace_context, record_provider_usage, traced, @@ -1334,11 +1335,12 @@ def _log_provider_attempt(agent: ModelAgent, attempt: int, retry_limit: int) -> """DEBUG-log one provider call attempt before it is made.""" if _LOGGER.isEnabledFor(logging.DEBUG): _LOGGER.debug( - "provider_attempt agent_id=%s model=%s attempt=%d/%d", + "provider_attempt agent_id=%s model=%s attempt=%d/%d request_id=%s", agent.id, agent.model, attempt + 1, retry_limit + 1, + current_request_id() or "-", ) @@ -1348,13 +1350,14 @@ def _log_provider_attempt_failed( """DEBUG-log one failed provider attempt with a redacted, bounded error message.""" if _LOGGER.isEnabledFor(logging.DEBUG): _LOGGER.debug( - "provider_attempt_failed agent_id=%s model=%s attempt=%d error_type=%s transient=%s error_message=%s", + "provider_attempt_failed agent_id=%s model=%s attempt=%d error_type=%s transient=%s error_message=%s request_id=%s", agent.id, agent.model, attempt + 1, type(exc).__name__, transient, redact_text(str(exc))[:500], + current_request_id() or "-", ) diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 112e047b7..9fcad47d2 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -65,6 +65,8 @@ attach_trace_context, configure_telemetry, current_session_id, + current_request_id, + request_identity, detach_trace_context, reset_session_id, session_id_from_headers, @@ -5627,11 +5629,12 @@ def handle_one_request(self) -> None: self.command = None self.path = None self._request_started = None - try: - super().handle_one_request() - finally: - self._log_request_summary(self._request_started) - self._reset_session() + with request_identity(): + try: + super().handle_one_request() + finally: + self._log_request_summary(self._request_started) + self._reset_session() # A request that declared a body it never delivered (unsupported # method, rejected route) must not leave those bytes on a reusable # connection for the stdlib to reparse as the next request. @@ -8193,7 +8196,7 @@ def _send_error( message: str, detail: dict[str, Any] | None = None, ) -> None: - request_id = uuid.uuid4().hex + request_id = current_request_id() or uuid.uuid4().hex _LOGGER.warning( "request_failed status=%s code=%s request_id=%s", status, code, request_id ) diff --git a/contextual_orchestrator/telemetry.py b/contextual_orchestrator/telemetry.py index be1295947..e710d7112 100644 --- a/contextual_orchestrator/telemetry.py +++ b/contextual_orchestrator/telemetry.py @@ -6,6 +6,7 @@ import ipaddress import logging import re +import uuid from collections.abc import Iterator, Mapping from contextlib import contextmanager from contextvars import ContextVar, Token @@ -38,6 +39,25 @@ "contextual_orchestrator_session_id", default=None ) _CONFIGURED = False +_CURRENT_REQUEST_ID: ContextVar[str | None] = ContextVar("gateway_request_id", default=None) + + +def current_request_id() -> str | None: + """Return the server-generated request identity, never a caller value.""" + return _CURRENT_REQUEST_ID.get() + + +@contextmanager +def request_identity() -> Iterator[str]: + """Bind a fresh identity and restore the enclosing context on every exit.""" + request_id = uuid.uuid4().hex + request_token = _CURRENT_REQUEST_ID.set(request_id) + try: + yield request_id + finally: + _CURRENT_REQUEST_ID.reset(request_token) + + # Match the OpenTelemetry SDK's default span-attribute budget so a single # sequence-valued attribute cannot exceed the span's default evidence budget. _MAX_ATTRIBUTE_SEQUENCE_ITEMS = 128 diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 3489bdff2..5822486a9 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -451,6 +451,25 @@ def test_http_error_ids_correlate_over_real_connections(caplog): assert len(set(request_ids)) == 2 +def test_request_identity_restores_context_across_threads_and_failure(): + """Copied work inherits identity; reused workers and failed scopes do not leak it.""" + from concurrent.futures import ThreadPoolExecutor + from contextvars import copy_context + + assert telemetry_module.current_request_id() is None + with ThreadPoolExecutor(max_workers=1) as executor: + with telemetry_module.request_identity() as outer_id: + assert executor.submit(copy_context().run, telemetry_module.current_request_id).result() == outer_id + assert executor.submit(telemetry_module.current_request_id).result() is None + with pytest.raises(RuntimeError): + with telemetry_module.request_identity() as inner_id: + assert inner_id != outer_id + raise RuntimeError("controlled failure") + assert telemetry_module.current_request_id() == outer_id + assert telemetry_module.current_request_id() is None + assert executor.submit(telemetry_module.current_request_id).result() is None + + def test_provider_attempts_share_http_error_identity(monkeypatch, caplog): """Same-session HTTP requests need distinct identities before provider failure.""" import http.client From 8b82235e2b6db19682b0a255758c6796c41b3f55 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 12:48:15 +0900 Subject: [PATCH 13/87] fix(telemetry): correlate retry and terminal provider diagnostics --- contextual_orchestrator/orchestrator.py | 19 ++++++++++++------- tests/test_telemetry.py | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index f03127ad8..777386720 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -1350,14 +1350,14 @@ def _log_provider_attempt_failed( """DEBUG-log one failed provider attempt with a redacted, bounded error message.""" if _LOGGER.isEnabledFor(logging.DEBUG): _LOGGER.debug( - "provider_attempt_failed agent_id=%s model=%s attempt=%d error_type=%s transient=%s error_message=%s request_id=%s", + "provider_attempt_failed agent_id=%s model=%s attempt=%d error_type=%s transient=%s request_id=%s error_message=%s", agent.id, agent.model, attempt + 1, type(exc).__name__, transient, - redact_text(str(exc))[:500], current_request_id() or "-", + redact_text(str(exc))[:500], ) @@ -1365,10 +1365,11 @@ def _log_provider_backoff(agent: ModelAgent, attempt: int, delay: float) -> None """DEBUG-log one backoff sleep before the next retry attempt.""" if _LOGGER.isEnabledFor(logging.DEBUG): _LOGGER.debug( - "provider_backoff agent_id=%s attempt=%d delay_seconds=%.3f", + "provider_backoff agent_id=%s attempt=%d delay_seconds=%.3f request_id=%s", agent.id, attempt + 1, delay, + current_request_id() or "-", ) @@ -1389,11 +1390,12 @@ def _log_provider_exhausted(agent: ModelAgent, attempts: int, last_error: Except first place" or "was never allowed to be retried at all". """ _LOGGER.warning( - "provider_exhausted agent_id=%s model=%s attempts=%s final_error_type=%s", + "provider_exhausted agent_id=%s model=%s attempts=%s final_error_type=%s request_id=%s", agent.id, agent.model, attempts, type(last_error).__name__, + current_request_id() or "-", ) @@ -1418,12 +1420,13 @@ def _log_provider_no_retry_budget( from "this wouldn't have been retried anyway" from this one event name. """ _LOGGER.warning( - "provider_no_retry_budget agent_id=%s model=%s attempts=%s final_error_type=%s transient=%s", + "provider_no_retry_budget agent_id=%s model=%s attempts=%s final_error_type=%s transient=%s request_id=%s", agent.id, agent.model, attempts, type(last_error).__name__, transient, + current_request_id() or "-", ) @@ -1449,12 +1452,13 @@ def _log_provider_one_shot_call_failed( :func:`_log_provider_no_retry_budget`. """ _LOGGER.warning( - "provider_one_shot_call_failed agent_id=%s model=%s attempts=%s final_error_type=%s transient=%s", + "provider_one_shot_call_failed agent_id=%s model=%s attempts=%s final_error_type=%s transient=%s request_id=%s", agent.id, agent.model, attempts, type(last_error).__name__, transient, + current_request_id() or "-", ) @@ -1470,11 +1474,12 @@ def _log_provider_rejected_permanent(agent: ModelAgent, attempts: int, last_erro for the separate case where no retry budget was configured at all. """ _LOGGER.warning( - "provider_rejected_permanent agent_id=%s model=%s attempts=%s final_error_type=%s", + "provider_rejected_permanent agent_id=%s model=%s attempts=%s final_error_type=%s request_id=%s", agent.id, agent.model, attempts, type(last_error).__name__, + current_request_id() or "-", ) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 5822486a9..8a4b9da02 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -451,6 +451,24 @@ def test_http_error_ids_correlate_over_real_connections(caplog): assert len(set(request_ids)) == 2 +def test_provider_diagnostic_events_preserve_request_identity(caplog): + """Every retry outcome keeps trusted identity before untrusted error text.""" + agent = ModelAgent("diagnostic_agent", "mock-model") + failure = RuntimeError("controlled error") + with caplog.at_level("DEBUG"), telemetry_module.request_identity() as request_id: + orchestrator_module._log_provider_attempt(agent, 0, 1) + orchestrator_module._log_provider_attempt_failed(agent, 0, failure, False) + orchestrator_module._log_provider_backoff(agent, 0, 0.0) + orchestrator_module._log_provider_exhausted(agent, 2, failure) + orchestrator_module._log_provider_no_retry_budget(agent, 1, failure, transient=False) + orchestrator_module._log_provider_one_shot_call_failed(agent, 1, failure, transient=False) + orchestrator_module._log_provider_rejected_permanent(agent, 1, failure) + messages = [row.getMessage() for row in caplog.records if row.name == orchestrator_module.__name__] + assert len(messages) == 7 + assert all(f"request_id={request_id}" in message for message in messages) + assert f"request_id={request_id} error_message=" in messages[1] + + def test_request_identity_restores_context_across_threads_and_failure(): """Copied work inherits identity; reused workers and failed scopes do not leak it.""" from concurrent.futures import ThreadPoolExecutor From 7b7b32006e7ae498db2ee781bd423d9c7b6774fc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 12:50:09 +0900 Subject: [PATCH 14/87] docs(telemetry): record correlation evidence and collector boundary --- AGENTS.md | 4 ++ CLAUDE.md | 4 ++ .../doctoring/provider_request_correlation.md | 45 +++++++++++++++++++ docs/product-technical-gap-baseline.md | 9 ++++ 4 files changed, 62 insertions(+) create mode 100644 docs/doctoring/provider_request_correlation.md diff --git a/AGENTS.md b/AGENTS.md index 52b268819..6c6f77f8e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,6 +41,10 @@ push or open a PR. ### Code exploration +- Provider logs need server-generated per-request identity, not a session hash. + Preserve context cleanup and validate the central collector before adoption. + Reproduction and exact evidence: `docs/doctoring/provider_request_correlation.md`. + - This repo has **no `.codegraph/` index**, so use normal search (grep/ripgrep/find, file reads) to locate and understand code. If a `.codegraph/` directory is ever added at the repo root, prefer CodeGraph diff --git a/CLAUDE.md b/CLAUDE.md index 655d98a23..479bb83d5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,6 +19,10 @@ This file complements AGENTS.md with commands and architecture; where they diffe ## Common commands +For missing provider/error correlation, run the focused telemetry and debug-log +tests in `docs/doctoring/provider_request_correlation.md`. A green local producer +test does not prove the central collector preserves the new field. + ```bash # Install (pinned, hash-locked — always this two-step form) python -m pip install --require-hashes -r requirements.lock diff --git a/docs/doctoring/provider_request_correlation.md b/docs/doctoring/provider_request_correlation.md new file mode 100644 index 000000000..846910372 --- /dev/null +++ b/docs/doctoring/provider_request_correlation.md @@ -0,0 +1,45 @@ +# Provider request correlation + +Status: proposed in PR #1105; not deployed. Runtime owner: CO. Log collector +owner: ContextualWisdomLab/.github. + +Central run 34299034731/job 102308769876 used trusted workflow revision +`7fd571dbcdbae6acf29d8f4ee704d7ba6297e4db` and logged installation of CO +`414f22973658c4ddc3d4320fcf7acd9b4e8ba991`. Artifact 10085227363 contains +provider attempts without request identities. Timestamps and a final served +model cannot establish which earlier calls belong to the failed request. + +At local regression commit `f7f05569a54a51c4cd6e42418a0c079b032d45f6`, a real +HTTP request reached the real client retry wrapper with a controlled failing +transport. Its two provider diagnostic lines lacked the error response ID: +1 failed, 46 deselected (5.34s). No external provider was called. + +`c7468ecea009bae3a62413b4c54ab5a9b242bdff` binds a fresh server-generated UUID +to each HTTP handling scope using ContextVar and resets it in finally. The +error response reuses that identity; caller headers and error details cannot +replace it. A session hash is insufficient because several requests can share +a session. Copied contexts inherit identity, while reused workers without a +copied context do not. `8b82235e2b6db19682b0a255758c6796c41b3f55` extends the +identity to all seven provider attempt/retry/terminal diagnostic helpers. + +Reproduce from this checkout with its project environment: + +```sh +python -m pytest tests/test_telemetry.py tests/test_orchestrator_debug_logging.py -q +``` + +Result at the latter code revision: 68 passed in 25.60s. This covers controlled +HTTP failures, sequential same-session requests, copied worker contexts, nested +exception cleanup, and all seven diagnostic helpers. It does not prove +simultaneous HTTP isolation, every orchestration thread path, success-response +correlation, full-suite success, or customer latency improvement. + +Collector compatibility remains a release prerequisite. The trusted central +collector uses end-anchored patterns for several events and truncates failure +messages before error text. It rejects new fields until repaired. The new ID is +32 lowercase hexadecimal characters, or `-` outside an HTTP request; for failed +attempts it precedes `error_message`, for other events it is the final field. +Never recover an ID from untrusted error text. Preserve old-format inputs and +verify exact-SHA producer/collector integration before adoption. Do not copy +collector code into CO or enable unfiltered logs. Timeouts and replay policy +are unchanged by this patch. Browser rendering remains unverified. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index e2be75366..62fbdf045 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,14 @@ # Contextual Orchestrator: Product & Technical Gap Baseline +## 2026-09-09 Request-to-provider diagnostic correlation + +PR #1105 candidate `8b82235e2b6db19682b0a255758c6796c41b3f55` connects HTTP +error identity to seven provider diagnostic events (68 focused tests passed). +The [runbook](doctoring/provider_request_correlation.md) records RED evidence, +cleanup tests, limitations, and the central collector compatibility prerequisite. +Not yet established: simultaneous HTTP isolation, full-suite/security gates, +protected release, actual collector adoption, or customer KPI improvement. + ## 2026-09-08 error-response correlation repair ConceptWeave run 33938445050, job 101256562088, preserves a client-side HTTP From df39ff85ad9ed4a91acf49d8a587bbc236ebddc6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 12:55:51 +0900 Subject: [PATCH 15/87] fix(telemetry): identify successful HTTP request summaries --- contextual_orchestrator/server.py | 4 +++- tests/test_telemetry.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 9fcad47d2..fc706a11a 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -5692,13 +5692,15 @@ def _log_request_summary(self, started: float | None) -> None: if not method and not path and status is None: return _LOGGER.info( + "%s request_id=%s", summarize_request_for_log( method=method or "-", path=path or "-", status=status, latency_ms=(time.monotonic() - (started or time.monotonic())) * 1000.0, session_id_hash=session_id_hash(), - ) + ), + current_request_id() or "-", ) def do_GET(self) -> None: # noqa: N802 diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 8a4b9da02..413cacc81 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -725,6 +725,11 @@ def test_per_request_info_summary_reports_method_path_and_status(caplog): assert "method=GET" in caplog.text assert "path=/healthz" in caplog.text assert "status=200" in caplog.text + import re + summary_lines = [row.getMessage() for row in caplog.records + if row.getMessage().startswith("http_request ")] + assert len(summary_lines) == 1 + assert re.search(r" request_id=[0-9a-f]{32}$", summary_lines[0]) def test_per_request_info_summary_never_includes_query_string(caplog): From 130a049990f2e14c5cf02b11af45205d1ce46121 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 12:56:59 +0900 Subject: [PATCH 16/87] test(telemetry): prove identity rotation on a reused socket --- tests/test_telemetry.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 413cacc81..c4c40d97c 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -512,6 +512,7 @@ def fail_completion(*args, **kwargs): server_thread.start() connection = http.client.HTTPConnection(*server.server_address, timeout=5) request_ids = [] + first_socket = None try: with caplog.at_level("DEBUG"): for request_index in range(2): @@ -522,8 +523,14 @@ def fail_completion(*args, **kwargs): "Content-Type": "application/json", "Authorization": "Bearer test-correlation-token", "X-LineageWeave-Session-Id": "shared-private-session", }) + if first_socket is None: + first_socket = connection.sock + assert first_socket is not None + else: + assert connection.sock is first_socket response = connection.getresponse() assert response.status >= 400 + assert not response.will_close response_body = json.loads(response.read()) request_id = response_body["error"]["detail"]["request_id"] request_ids.append(request_id) From 467353c8e3586fca42c6f1ba6b38b231f3fdbad7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 12:59:18 +0900 Subject: [PATCH 17/87] test(telemetry): isolate overlapping same-session HTTP requests --- tests/test_telemetry.py | 64 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index c4c40d97c..493498bff 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -548,6 +548,70 @@ def fail_completion(*args, **kwargs): router.close() +def test_concurrent_http_provider_identity_isolation(monkeypatch, caplog): + """Overlapping same-session requests keep their own provider/error identities.""" + import http.client + import threading + from concurrent.futures import ThreadPoolExecutor + from contextual_orchestrator import TaskOrchestrator + from contextual_orchestrator.server import SecurityConfig + + rendezvous = threading.Barrier(2, timeout=10) + overlapping_threads = set() + overlap_lock = threading.Lock() + model_agent = ModelAgent("parallel_agent", "mock-model") + model_client = ModelClient(max_retries=0) + router = TaskOrchestrator([model_agent], client=model_client) + + def reject_send(*args, **kwargs): + rendezvous.wait() + with overlap_lock: + overlapping_threads.add(threading.get_ident()) + raise RuntimeError("controlled overlapping failure") + + def fail_completion(*args, **kwargs): + return model_client._send_with_retry(model_agent, {}) + + monkeypatch.setattr(model_client, "_send", reject_send) + monkeypatch.setattr(router, "complete", fail_completion) + server = build_server(router, port=0, security=SecurityConfig(auth_token="parallel-test-token")) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + + def send_request(): + connection = http.client.HTTPConnection(*server.server_address, timeout=15) + try: + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "mock-model", "messages": [{"role": "user", "content": "unit request"}], + }), {"Content-Type": "application/json", "Authorization": "Bearer parallel-test-token", + "X-LineageWeave-Session-Id": "same-private-session"}) + response = connection.getresponse() + assert response.status == 502 + return json.loads(response.read())["error"]["detail"]["request_id"] + finally: + connection.close() + + try: + with caplog.at_level("DEBUG"), ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(send_request) for _ in range(2)] + request_ids = [future.result(timeout=20) for future in futures] + assert len(overlapping_threads) == 2 + assert len(set(request_ids)) == 2 + provider_logs = [row.getMessage() for row in caplog.records + if row.getMessage().startswith(("provider_attempt ", "provider_attempt_failed "))] + assert len(provider_logs) == 4 + for request_id in request_ids: + matching = [message for message in provider_logs if f"request_id={request_id}" in message] + assert len(matching) == 2 + assert sum(message.startswith("provider_attempt ") for message in matching) == 1 + assert "same-private-session" not in "\n".join(provider_logs) + finally: + server.shutdown() + server_thread.join(timeout=5) + server.server_close() + router.close() + + def test_http_diagnostics_exclude_raw_path_and_swallow_client_disconnect(monkeypatch, caplog): """Client cancellation cannot create a second error or leak path identifiers.""" server = build_server(SimpleNamespace(agents=[], candidates=[]), port=0) From 6b24fe96141762b1270a9232677a255ded8f94fc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 13:02:46 +0900 Subject: [PATCH 18/87] docs(telemetry): distinguish second 429 incident from timeout --- docs/doctoring/provider_request_correlation.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/doctoring/provider_request_correlation.md b/docs/doctoring/provider_request_correlation.md index 846910372..eaca45709 100644 --- a/docs/doctoring/provider_request_correlation.md +++ b/docs/doctoring/provider_request_correlation.md @@ -43,3 +43,18 @@ Never recover an ID from untrusted error text. Preserve old-format inputs and verify exact-SHA producer/collector integration before adoption. Do not copy collector code into CO or enable unfiltered logs. Timeouts and replay policy are unchanged by this patch. Browser rendering remains unverified. + +## Independent second incident + +Run 34306399309/job 102324739644 used the same trusted workflow and CO pins. +Artifact 10087151196 again records candidate_count=24, ready_count=1, +deferred_count=8, rejected_count=7, and account diversity=3. The caller reported +429 after 125.2s at 2026-09-09 03:31:41 UTC. Unlike the earlier incident, +the review interval beginning 03:29:35 contains no logged TimeoutError: +several Llama attempts precede DeepSeek at 03:31:40.995, which fails at +03:31:41.035 (about 40ms). TimeoutErrors at 03:27:17 and 03:28:47 belong to +the earlier preflight interval, not this review interval. These logs therefore +do not support attributing the final 429 to the default 90-second timeout. +Role/request attribution remains incomplete without correlation; keep the +timeout repair and this diagnostic repair as separate claims. No rerun or +provider call was issued during this read-only investigation. From 7cb97ec8e2979d35b72c86a801ab18f0fd9c213d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 13:04:42 +0900 Subject: [PATCH 19/87] docs(telemetry): record combined HTTP and isolation verification --- docs/doctoring/provider_request_correlation.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/doctoring/provider_request_correlation.md b/docs/doctoring/provider_request_correlation.md index eaca45709..c437ee526 100644 --- a/docs/doctoring/provider_request_correlation.md +++ b/docs/doctoring/provider_request_correlation.md @@ -46,6 +46,16 @@ are unchanged by this patch. Browser rendering remains unverified. ## Independent second incident +Follow-up at `6b24fe96` (local candidate, not the full-suite head): the success +summary also carries the request ID. A socket-identity assertion proves actual +keep-alive reuse; a two-party barrier followed by two distinct server-thread IDs +proves overlapping same-session HTTP handling rather than merely submitting +two tasks. Both requests retain distinct IDs and matching attempt/failure logs. +`python -m pytest tests/test_telemetry.py tests/test_orchestrator_debug_logging.py +tests/test_request_framing.py -q` passed 81 tests in 20.76s. This supersedes the +earlier simultaneous-HTTP limitation for these controlled failure cases only; +real provider integration and every orchestration worker path remain unverified. + Run 34306399309/job 102324739644 used the same trusted workflow and CO pins. Artifact 10087151196 again records candidate_count=24, ready_count=1, deferred_count=8, rejected_count=7, and account diversity=3. The caller reported From f46786320a7148c4f3207225ff4e077749dbe003 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 13:15:25 +0900 Subject: [PATCH 20/87] docs: record exact-revision collector contract evidence --- docs/doctoring/provider_request_correlation.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/doctoring/provider_request_correlation.md b/docs/doctoring/provider_request_correlation.md index c437ee526..f0caaba1a 100644 --- a/docs/doctoring/provider_request_correlation.md +++ b/docs/doctoring/provider_request_correlation.md @@ -44,7 +44,7 @@ verify exact-SHA producer/collector integration before adoption. Do not copy collector code into CO or enable unfiltered logs. Timeouts and replay policy are unchanged by this patch. Browser rendering remains unverified. -## Independent second incident +## Concurrent HTTP follow-up Follow-up at `6b24fe96` (local candidate, not the full-suite head): the success summary also carries the request ID. A socket-identity assertion proves actual @@ -56,6 +56,8 @@ tests/test_request_framing.py -q` passed 81 tests in 20.76s. This supersedes the earlier simultaneous-HTTP limitation for these controlled failure cases only; real provider integration and every orchestration worker path remain unverified. +## Independent second incident + Run 34306399309/job 102324739644 used the same trusted workflow and CO pins. Artifact 10087151196 again records candidate_count=24, ready_count=1, deferred_count=8, rejected_count=7, and account diversity=3. The caller reported @@ -68,3 +70,17 @@ do not support attributing the final 429 to the default 90-second timeout. Role/request attribution remains incomplete without correlation; keep the timeout repair and this diagnostic repair as separate claims. No rerun or provider call was issued during this read-only investigation. + +## Exact-revision collector contract + +The producer candidate `7cb97ec8e2979d35b72c86a801ab18f0fd9c213d` +was cross-executed with the sanitizer from central PR #2053, +`fc0ab87bfde0900461034be815046914f9019bfc`. All seven actual provider +logging functions produced records whose trusted request ID survived sanitization. +A controlled error body containing a second, forged ID was omitted, as was its +controlled sensitive-text sentinel. Replacing the trusted ID with `INVALID` +or appending an embedded newline caused rejection for all seven records. +The final `request_failed` summary retained its ID and omitted trailing detail. +These are isolated, exact-revision contract checks, not live provider or release +evidence. The consumer deliberately leaves successful HTTP summaries outside +this PR's allowlist; that follow-up contract remains unverified. From f588ca8c093ea7c9a86b857685bfbb1ce3c05fe2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 13:17:55 +0900 Subject: [PATCH 21/87] docs: record completed suite and bounded visual inspection --- docs/doctoring/provider_request_correlation.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/doctoring/provider_request_correlation.md b/docs/doctoring/provider_request_correlation.md index f0caaba1a..3476bcb0c 100644 --- a/docs/doctoring/provider_request_correlation.md +++ b/docs/doctoring/provider_request_correlation.md @@ -73,6 +73,17 @@ provider call was issued during this read-only investigation. ## Exact-revision collector contract +The full suite at `7b7b32006e7ae498db2ee781bd423d9c7b6774fc` terminated +with exit status 0: **3399 passed, 2 skipped in 1594.26s**. This result predates +the success-summary and concurrent-HTTP follow-up; it must not be attributed +to their later revision. Those changes have the focused 81-test evidence above. + +The published Markdown at that same revision was opened in Edge and its +1897 × 949 screenshot directly inspected. The visible upper document had readable +heading/body contrast, wrapped paragraphs and commit identifiers, and an unclipped +test command. This is an English desktop upper-viewport inspection only, not a +full-document, responsive, interaction, or product-UI visual acceptance result. + The producer candidate `7cb97ec8e2979d35b72c86a801ab18f0fd9c213d` was cross-executed with the sanitizer from central PR #2053, `fc0ab87bfde0900461034be815046914f9019bfc`. All seven actual provider From f30fa7853a27ac4e7e150675d24dbef68f3e7d8f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 13:19:50 +0900 Subject: [PATCH 22/87] docs: reconcile diagnostic gap with exact-head evidence --- docs/product-technical-gap-baseline.md | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 62fbdf045..13f52fe7b 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2,12 +2,26 @@ ## 2026-09-09 Request-to-provider diagnostic correlation -PR #1105 candidate `8b82235e2b6db19682b0a255758c6796c41b3f55` connects HTTP -error identity to seven provider diagnostic events (68 focused tests passed). -The [runbook](doctoring/provider_request_correlation.md) records RED evidence, -cleanup tests, limitations, and the central collector compatibility prerequisite. -Not yet established: simultaneous HTTP isolation, full-suite/security gates, -protected release, actual collector adoption, or customer KPI improvement. +PR #1105 candidate `f588ca8c093ea7c9a86b857685bfbb1ce3c05fe2` connects HTTP +identity to seven provider diagnostic events and the successful request summary. +The predecessor `7b7b32006e7ae498db2ee781bd423d9c7b6774fc` completed its full +suite with 3399 passed, 2 skipped (1594.26s, exit 0). Follow-up code at +`6b24fe96` passed 81 focused tests, including actual same-socket reuse and +overlapping same-session HTTP requests with two distinct server thread IDs. +The integrated candidate's full suite is a separate, still-pending verification. + +Actual output from all seven provider diagnostic functions at +`7cb97ec8e2979d35b72c86a801ab18f0fd9c213d` was cross-checked with the central +PR #2053 sanitizer at `fc0ab87bfde0900461034be815046914f9019bfc`: trusted IDs +survived, untrusted error-body IDs and text were omitted, and malformed IDs and +embedded newlines were rejected. This isolated contract test does not establish +collector adoption; successful HTTP summaries remain outside that candidate's +allowlist. The [runbook](doctoring/provider_request_correlation.md) records +RED evidence, exact revisions, cleanup tests, and bounded visual inspection. +Not yet established: every orchestration worker path, integrated full-suite and +security gates, protected release, live collector adoption, or customer KPI +improvement. Diagnostic traceability is a prerequisite for attributing failures, +not a substitute for accuracy or decision-latency measurements. ## 2026-09-08 error-response correlation repair From 89ccb0413b5072ac3f8a3464bd4cb3d9fdddd054 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 13:37:41 +0900 Subject: [PATCH 23/87] docs: verify success-summary collector contract --- docs/doctoring/provider_request_correlation.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/doctoring/provider_request_correlation.md b/docs/doctoring/provider_request_correlation.md index 3476bcb0c..c7db0f45f 100644 --- a/docs/doctoring/provider_request_correlation.md +++ b/docs/doctoring/provider_request_correlation.md @@ -95,3 +95,13 @@ The final `request_failed` summary retained its ID and omitted trailing detail. These are isolated, exact-revision contract checks, not live provider or release evidence. The consumer deliberately leaves successful HTTP summaries outside this PR's allowlist; that follow-up contract remains unverified. + +Follow-up consumer `4a0125bf9f50d4d26355249011df03c3735b3abc` adds a strict +HTTP-summary allowlist. Against producer +`f588ca8c093ea7c9a86b857685bfbb1ce3c05fe2`, an actual local HTTP GET to +`/healthz` returned 200 and emitted one request summary. The new sanitizer +retained it verbatim, including the generated request ID. Appending a controlled +extra detail field or substituting a non-allowlisted path caused rejection. +This supersedes the missing-success-summary contract limitation for that one +route/state; it does not verify all routes, current production adoption, or +latency improvement. No external provider was called. From 7744468af38098f0dca2b56f47876baf5a54c7df Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 13:49:22 +0900 Subject: [PATCH 24/87] docs: retain integrated suite failure evidence --- docs/doctoring/provider_request_correlation.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/doctoring/provider_request_correlation.md b/docs/doctoring/provider_request_correlation.md index c7db0f45f..0cf43c9dc 100644 --- a/docs/doctoring/provider_request_correlation.md +++ b/docs/doctoring/provider_request_correlation.md @@ -105,3 +105,15 @@ extra detail field or substituting a non-allowlisted path caused rejection. This supersedes the missing-success-summary contract limitation for that one route/state; it does not verify all routes, current production adoption, or latency improvement. No external provider was called. + +## Integrated full-suite failure + +At `f588ca8c093ea7c9a86b857685bfbb1ce3c05fe2`, the integrated full suite +terminated with exit 1: **3399 passed, 2 skipped, 1 failed in 1767.82s**. +`test_http_responses_rejects_store_true` failed while constructing the server's +ModelClient, before HTTP assertions: `context.load_verify_locations` raised +`InterruptedError` (errno 4) while loading the certifi CA bundle. This does not +prove a response-correlation regression, but it also does not establish a green +suite or justify classifying the failure as a flake. Same-head isolated +reproduction is the next diagnostic step. Preserve this failure receipt even +if a subsequent isolated test passes. From 01db64ace49fb92fbb73cf8838e804643485d1ba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 13:50:18 +0900 Subject: [PATCH 25/87] docs: distinguish isolated retry from full-suite verification --- docs/doctoring/provider_request_correlation.md | 6 ++++++ docs/product-technical-gap-baseline.md | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/doctoring/provider_request_correlation.md b/docs/doctoring/provider_request_correlation.md index 0cf43c9dc..a28a186d5 100644 --- a/docs/doctoring/provider_request_correlation.md +++ b/docs/doctoring/provider_request_correlation.md @@ -117,3 +117,9 @@ prove a response-correlation regression, but it also does not establish a green suite or justify classifying the failure as a flake. Same-head isolated reproduction is the next diagnostic step. Preserve this failure receipt even if a subsequent isolated test passes. + +The same-head isolated command `python -m pytest +tests/test_responses_store_http_honesty.py -q` then completed with exit 0: +4 passed in 20.05s. The interruption did not recur in this run; its signal or +operating-system trigger remains unproven. No TLS checks were bypassed and no +retry was added. A clean isolated run does not replace full-suite verification. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 13f52fe7b..4e4b9bac7 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -8,7 +8,11 @@ The predecessor `7b7b32006e7ae498db2ee781bd423d9c7b6774fc` completed its full suite with 3399 passed, 2 skipped (1594.26s, exit 0). Follow-up code at `6b24fe96` passed 81 focused tests, including actual same-socket reuse and overlapping same-session HTTP requests with two distinct server thread IDs. -The integrated candidate's full suite is a separate, still-pending verification. +The integrated `f588ca8c` suite terminated with 3399 passed, 2 skipped and +1 failure (1767.82s, exit 1): certifi CA loading raised InterruptedError before +the Responses HTTP test could send a request. Same-head isolated HTTP tests +then passed 4/4 in 20.05s. The original failure remains unresolved evidence; +do not infer full-suite success from the isolated pass. Actual output from all seven provider diagnostic functions at `7cb97ec8e2979d35b72c86a801ab18f0fd9c213d` was cross-checked with the central From b655fe1b1bc515d2f2b9952a2dbb90176d9c9a7b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 13:54:15 +0900 Subject: [PATCH 26/87] docs(gap): reconcile verified success-summary contract --- docs/product-technical-gap-baseline.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 4e4b9bac7..74d1a24a3 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -19,8 +19,12 @@ Actual output from all seven provider diagnostic functions at PR #2053 sanitizer at `fc0ab87bfde0900461034be815046914f9019bfc`: trusted IDs survived, untrusted error-body IDs and text were omitted, and malformed IDs and embedded newlines were rejected. This isolated contract test does not establish -collector adoption; successful HTTP summaries remain outside that candidate's -allowlist. The [runbook](doctoring/provider_request_correlation.md) records +collector adoption. The later sanitizer `4a0125bf9f50d4d26355249011df03c3735b3abc` +also preserved an actual local GET `/healthz` 200 summary from producer +`f588ca8c`, including its request ID, while rejecting extra detail and an +unapproved path. This supersedes the earlier missing-success-summary limitation +for that route/state only, not every HTTP route. The +[runbook](doctoring/provider_request_correlation.md) records RED evidence, exact revisions, cleanup tests, and bounded visual inspection. Not yet established: every orchestration worker path, integrated full-suite and security gates, protected release, live collector adoption, or customer KPI From 85a0b2d24f2a7ffce31b7d0b9942a7ed5d329de6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 14:55:39 +0900 Subject: [PATCH 27/87] test: require durable initial decision receipt over HTTP --- tests/test_decision_receipts.py | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 tests/test_decision_receipts.py diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py new file mode 100644 index 000000000..c34279d5c --- /dev/null +++ b/tests/test_decision_receipts.py @@ -0,0 +1,41 @@ +"""Initial decisions must be acknowledged before any answer is generated.""" + +import http.client +import json +import threading + +from contextual_orchestrator import ModelAgent, TaskOrchestrator +from contextual_orchestrator.server import build_server + + +def test_http_route_persists_initial_decision(tmp_path): + """A real HTTP route retains one native-clock receipt before completion.""" + orchestrator = TaskOrchestrator( + [ModelAgent("worker_one", "mock/worker")], state_db=tmp_path / "state.db" + ) + server = build_server(orchestrator, port=0, decision_receipts=True) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address) + try: + connection.request( + "POST", "/v1/chat/completions", + json.dumps({"model": "orchestrator/route", "messages": [ + {"role": "user", "content": "hello"} + ]}), {"Content-Type": "application/json"}, + ) + response = connection.getresponse() + response.read() + assert response.status == 200 + records = orchestrator._store.load("decision_receipt") + assert len(records) == 1 + assert records[0]["status"] == "acknowledged" + assert records[0]["selected_agent_ids"] == ["worker_one"] + assert records[0]["selection_elapsed_ns"] <= records[0]["durable_ack_elapsed_ns"] + assert len(orchestrator._store.load("initial_decision")) == 1 + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() From 814b9ea7f445087c2d62d02f1ebf8d6d327b7ee6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 15:17:28 +0900 Subject: [PATCH 28/87] feat: connect opt-in Rust decision receipt to HTTP routing candidate --- contextual_orchestrator/decision_receipts.py | 74 +++++++++++++++ contextual_orchestrator/orchestrator.py | 9 ++ contextual_orchestrator/server.py | 50 +++++++++-- .../doctoring/decision_receipt_integration.md | 43 +++++++++ rust/Cargo.lock | 7 ++ rust/Cargo.toml | 2 +- rust/decision_receipt/Cargo.toml | 11 +++ rust/decision_receipt/pyproject.toml | 13 +++ rust/decision_receipt/src/lib.rs | 90 +++++++++++++++++++ tests/test_decision_receipts.py | 90 +++++++++++++++++-- 10 files changed, 378 insertions(+), 11 deletions(-) create mode 100644 contextual_orchestrator/decision_receipts.py create mode 100644 docs/doctoring/decision_receipt_integration.md create mode 100644 rust/decision_receipt/Cargo.toml create mode 100644 rust/decision_receipt/pyproject.toml create mode 100644 rust/decision_receipt/src/lib.rs diff --git a/contextual_orchestrator/decision_receipts.py b/contextual_orchestrator/decision_receipts.py new file mode 100644 index 000000000..b819f2ca0 --- /dev/null +++ b/contextual_orchestrator/decision_receipts.py @@ -0,0 +1,74 @@ +"""Persistence boundary for Rust-owned initial-decision measurements.""" + +from contextvars import ContextVar +import logging +import threading +import uuid + +_CURRENT_DECISION = ContextVar("initial_decision", default=None) +_LOGGER = logging.getLogger(__name__) + + +class DecisionMeasurement: + """Bind one native clock to an HTTP request and its durable store.""" + + def __init__(self, store): + """Require the native module only when measurement is explicitly enabled.""" + from ._decision_receipt import DecisionReceipt + + self.receipt = DecisionReceipt() + self.store = store + self.request_id = uuid.uuid4().hex + self.selected_agent_ids = [] + self._lock = threading.Lock() + self._token = _CURRENT_DECISION.set(self) + + def snapshot(self): + """Return no prompts, provider credentials, or fabricated missing timings.""" + return { + "request_id": self.request_id, + "status": self.receipt.status, + "selected_agent_ids": list(self.selected_agent_ids), + "selection_elapsed_ns": self.receipt.selection_elapsed_ns, + "durable_ack_elapsed_ns": self.receipt.durable_ack_elapsed_ns, + "metric_scope": "initial_provider_dispatch", + } + + def select(self, agent_ids): + """Acknowledge the first decision synchronously before provider dispatch.""" + with self._lock: + if self.receipt.status != "accepted": + return + self.receipt.record_selection() + self.selected_agent_ids = list(agent_ids) + if self.store is None: + self.receipt.record_failure("store_unavailable") + return + try: + self.store.save("initial_decision", None, self.snapshot(), durable=True) + except Exception as exc: + self.receipt.record_failure("write_failed") + _LOGGER.warning("Initial decision measurement write failed error_type=%s", type(exc).__name__) + return + self.receipt.record_durable_ack() + + def close(self, reason="unfinished"): + """Persist the post-commit measurement separately; never claim its own ack.""" + try: + with self._lock: + if self.receipt.status in ("accepted", "selected"): + self.receipt.record_failure(reason) + if self.store is not None: + try: + self.store.save("decision_receipt", None, self.snapshot(), durable=True) + except Exception as exc: + _LOGGER.warning("Decision receipt export failed error_type=%s", type(exc).__name__) + finally: + _CURRENT_DECISION.reset(self._token) + + +def record_initial_selection(agent_ids): + """Record selection only within an explicitly enabled request measurement.""" + measurement = _CURRENT_DECISION.get() + if measurement is not None: + measurement.select(agent_ids) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index fa85ba533..ce407248f 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -6,6 +6,7 @@ from collections.abc import Iterable, Mapping from contextlib import contextmanager, nullcontext from contextvars import ContextVar, copy_context +from .decision_receipts import record_initial_selection from concurrent.futures import ThreadPoolExecutor import copy import hashlib @@ -4329,6 +4330,7 @@ def proxy_completion( agent, upstream, effort_profile, api_surface=api_surface ) measured = bool(agent.group_name or requested_model == self.FREE_MODEL) + record_initial_selection([agent.id]) started_at = time.perf_counter() try: result = self.client.proxy_send(agent, endpoint, upstream) @@ -4394,6 +4396,7 @@ def proxy_completion( last_failure: tuple[Exception, ModelAgent] | None = None every_failure_was_request_too_large = True for candidate in candidates: + record_initial_selection([candidate.id]) started_at = time.perf_counter() candidate_payload = dict(upstream) candidate_payload["model"] = candidate.model @@ -5255,6 +5258,7 @@ def stream_route( stream_kwargs["effort_profile"] = effort_profile if include_usage: stream_kwargs["include_usage"] = True + record_initial_selection([agent.id]) stream = self.client.stream_chat(agent, messages, **stream_kwargs) started_at = time.perf_counter() try: @@ -6780,6 +6784,7 @@ def _plan_generated(self, task: str) -> list[WorkflowStep]: {"role": "user", "content": task}, ] effort_profile = self._role_effort_profile("planner") + record_initial_selection([planner.id]) raw = ( self.client.chat(planner, planner_messages, effort_profile=effort_profile) if effort_profile is not None @@ -7614,6 +7619,7 @@ def call(agent: ModelAgent) -> dict[str, Any] | tuple[bytes, str]: if agent.provider_name == "openrouter" and endpoint == "images/generations" else endpoint ) + record_initial_selection([member.id for member in race_members]) return ( self.client.proxy_send_bytes(agent, provider_endpoint, payload) if binary else self.client.proxy_send(agent, provider_endpoint, payload) @@ -7672,6 +7678,7 @@ def call(agent: ModelAgent) -> dict[str, Any] | tuple[bytes, str]: ) started_at = time.perf_counter() try: + record_initial_selection([agent.id]) result = ( self.client.proxy_send_bytes(agent, provider_endpoint, payload) if binary @@ -7761,6 +7768,7 @@ def _invoke( def call(agent: ModelAgent) -> tuple[str, str, str, dict[str, Any] | None]: with self.client.request_settings(**request_settings): + record_initial_selection([member.id for member in race_members]) output = ( self.client.chat(agent, messages, effort_profile=effort_profile) if effort_profile is not None @@ -7833,6 +7841,7 @@ def call(agent: ModelAgent) -> tuple[str, str, str, dict[str, Any] | None]: single_attempt = getattr(self.client, "single_attempt_transport", None) transport_scope = single_attempt() if callable(single_attempt) else nullcontext() with transport_scope: + record_initial_selection([agent.id]) output = ( self.client.chat(agent, messages, effort_profile=effort_profile) if effort_profile is not None diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index eb6a77519..eed6cc20a 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -15,6 +15,7 @@ import secrets import socket import struct +import sys import tempfile import threading import time @@ -25,6 +26,7 @@ import uuid from .admin import ADMIN_HTML, ADMIN_TRANSLATIONS +from .decision_receipts import DecisionMeasurement from .api_contract import OPENAPI_SPEC from .cost_ledger import ATTRIBUTION_DIMENSIONS, dimension_catalog from .cost_router import ( @@ -5474,6 +5476,7 @@ def build_server( clearfolio_url: str | None = None, coordinator: CostRoutingCoordinator | None = None, release_authority: Mapping[str, Any] | None = None, + decision_receipts: bool = False, ) -> ThreadingHTTPServer: """Build, but do not start, the orchestration HTTP server. @@ -5482,6 +5485,12 @@ def build_server( every completion is priced, recorded, and sync/batch routed. """ security = security or SecurityConfig() + if type(decision_receipts) is not bool: + raise TypeError("decision_receipts must be a boolean") + if decision_receipts: + from ._decision_receipt import DecisionReceipt # noqa: F401 + if orchestrator._store is None: + raise ValueError("decision receipts require a durable state store") security.check_bind(host) release_authority = verify_release_authority_snapshot(release_authority) coordinator = coordinator or CostRoutingCoordinator(orchestrator) @@ -8129,10 +8138,37 @@ def _validate_trace_request(self, body: dict[str, Any]) -> bool: return include_trace def _run(self, callback: Any) -> dict[str, Any]: - security.acquire_run_slot() + self._acquire_measured_slot() try: return callback() finally: + self._release_measured_slot() + + def _acquire_measured_slot(self) -> None: + self._decision_failure_reason = "unfinished" + measurement = DecisionMeasurement(orchestrator._store) if decision_receipts else None + self._decision_measurement = measurement + try: + security.acquire_run_slot() + except Exception: + if measurement is not None: + measurement.close("capacity_rejected") + self._decision_measurement = None + raise + + def _release_measured_slot(self) -> None: + try: + measurement = self._decision_measurement + if measurement is not None: + active_error = sys.exc_info()[1] + reason = self._decision_failure_reason + if isinstance(active_error, (ConnectionError, BrokenPipeError, GeneratorExit)): + reason = "cancelled" + elif active_error is not None: + reason = "selection_failed" + measurement.close(reason) + finally: + self._decision_measurement = None security.release_run_slot() def _parse_positive_int(self, raw: str | None, field_name: str, default: int, max_value: int | None = None) -> int: @@ -8243,6 +8279,7 @@ def _write_response(self, writer: Callable[[], None]) -> bool: writer() return True except (BrokenPipeError, ConnectionError, OSError): + self._decision_failure_reason = "cancelled" _LOGGER.debug("client_disconnected") # `_send*`/`_begin_sse` writers record their *intended* # status in `self._last_status` before calling this method @@ -8438,7 +8475,7 @@ def progress(role: str, status: str) -> None: part={"type": "summary_text", "text": text}, ) - security.acquire_run_slot() + self._acquire_measured_slot() try: if not self._begin_sse(): return False @@ -8486,6 +8523,7 @@ def progress(role: str, status: str) -> None: conduct_kwargs["workflow_run_id"] = f"run_{uuid.uuid4().hex}" result = orchestrator.conduct(messages, **conduct_kwargs) except ConnectionAbortedError: + self._decision_failure_reason = "cancelled" raise except ProviderUpstreamError as exc: failed = { @@ -8595,9 +8633,10 @@ def progress(role: str, status: str) -> None: self._write_sse("data: [DONE]\n\n") return True except ConnectionAbortedError: + self._decision_failure_reason = "cancelled" return False finally: - security.release_run_slot() + self._release_measured_slot() def _stream_route_completion( self, @@ -8643,7 +8682,7 @@ def usage_frame(usage: dict[str, Any]) -> str: } return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" - security.acquire_run_slot() + self._acquire_measured_slot() try: if not self._begin_sse() or not self._write_sse( frame({"role": "assistant"}) @@ -8698,11 +8737,12 @@ def usage_frame(usage: dict[str, Any]) -> str: if not self._write_sse(frame({}, finish="error")): return except Exception: # noqa: BLE001 - headers already sent; surface as a terminal error frame + self._decision_failure_reason = "selection_failed" if not self._write_sse(frame({}, finish="error")): return self._write_sse("data: [DONE]\n\n") finally: - security.release_run_slot() + self._release_measured_slot() def _send_security_headers(self) -> None: if getattr(self, "close_connection", False): diff --git a/docs/doctoring/decision_receipt_integration.md b/docs/doctoring/decision_receipt_integration.md new file mode 100644 index 000000000..0a86a3b69 --- /dev/null +++ b/docs/doctoring/decision_receipt_integration.md @@ -0,0 +1,43 @@ +# Initial decision receipt integration candidate + +Status: incomplete implementation for issue #1110; not release evidence. + +RED `85a0b2d2` exposed the missing HTTP measurement option (1 failed, +3.67 s). The working candidate compiles a separate PyO3 extension using the +workspace's existing pyo3 0.29.2 lock resolution. It introduces no new +third-party dependency. The first build failed because a field-level getter +attribute was invalid; explicit read-only getters corrected compilation. +Context7 lookup hit its monthly quota; local compiler/API checks were used. + +The separate Rust receipt owns an Instant clock and state transitions. Python +only associates that receipt with the existing HTTP context and SQLite store; +it computes no durations or statistical estimates. Opt-in requires the native +extension and a configured durable store. The ordinary callback admission and +two direct SSE admissions establish scopes before the nonblocking capacity check. +Initial selection hooks cover _invoke, streaming, generated planning, and +plain/capability proxy candidates. These source hooks are not all validated. + +The initial decision is synchronously committed before the native acknowledgement +timestamp is captured. A separate receipt is exported at scope exit; that export +cannot be its own acknowledgement. A process crash between these writes leaves +an initial decision without a receipt, which must remain in the unfinished +denominator. Persisted successful receipts alone are not an all-request sample. + +Focused verification in the isolated local Python 3.14 environment: 4 tests +passed in 1.49 s. The real HTTP route's dispatch callback queries a separate +SQLite connection (never flush-on-read store.load), verifies the committed +decision, captures its native acknowledgement, and holds answer generation +behind an event. The final acknowledgement must equal that pre-generation value. +Other tests cover native invalid/duplicate acknowledgement, missing-store +startup rejection, write failure, snapshot isolation and secret-safe errors. +Controlled mock provider output is unit/integration evidence, not customer data. + +Remaining before review/merge: direct coordinator/structured synthesis and all +SSE/provider paths need executable coverage; metadata must bind policy revision +and route mode; unsupported and selection/cancellation outcomes need complete +denominator tests; race size/contract/preparation and concurrency need negative +tests; both receipt kinds need an explicit retention/export contract. Validate +actual wheel installation outside the source checkout (the current editable +extension is not package proof). Add the Proposed ADR with the completed seam +inventory, dependency graph and rendering inspection. Do not claim production +instrumentation, p95, accuracy improvement, protected CI, or release. diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 96d3c6a1d..58fc95de5 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -49,6 +49,13 @@ dependencies = [ "serde_core", ] +[[package]] +name = "contextual-decision-receipt" +version = "0.1.0" +dependencies = [ + "pyo3", +] + [[package]] name = "contextual-token-packer" version = "0.1.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index b7d83efbb..c69753fbb 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["token_counter"] +members = ["token_counter", "decision_receipt"] resolver = "2" diff --git a/rust/decision_receipt/Cargo.toml b/rust/decision_receipt/Cargo.toml new file mode 100644 index 000000000..8e1c1d19a --- /dev/null +++ b/rust/decision_receipt/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "contextual-decision-receipt" +version = "0.1.0" +edition = "2021" + +[dependencies] +pyo3 = { version = "0.29", features = ["abi3-py310", "auto-initialize"] } + +[lib] +name = "_decision_receipt" +crate-type = ["cdylib"] diff --git a/rust/decision_receipt/pyproject.toml b/rust/decision_receipt/pyproject.toml new file mode 100644 index 000000000..2e72d0a41 --- /dev/null +++ b/rust/decision_receipt/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["maturin>=1.8,<2"] +build-backend = "maturin" + +[project] +name = "contextual-decision-receipt" +version = "0.1.0" +requires-python = ">=3.10" + +[tool.maturin] +module-name = "contextual_orchestrator._decision_receipt" +python-source = "../.." +features = ["pyo3/extension-module"] diff --git a/rust/decision_receipt/src/lib.rs b/rust/decision_receipt/src/lib.rs new file mode 100644 index 000000000..75f015170 --- /dev/null +++ b/rust/decision_receipt/src/lib.rs @@ -0,0 +1,90 @@ +//! One-clock initial-decision measurement; storage acknowledgement is caller-owned. +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use std::time::Instant; + +/// Request-local monotonic state with no caller-supplied clock values. +#[pyclass] +pub struct DecisionReceipt { + accepted_at: Instant, + #[pyo3(get)] + status: String, + #[pyo3(get)] + selection_elapsed_ns: Option, + #[pyo3(get)] + durable_ack_elapsed_ns: Option, +} + +impl DecisionReceipt { + fn elapsed_ns(&self) -> PyResult { + self.accepted_at + .elapsed() + .as_nanos() + .try_into() + .map_err(|_| PyValueError::new_err("decision elapsed time overflow")) + } +} + +#[pymethods] +impl DecisionReceipt { + /// Start at validated acceptance, before capacity admission. + #[new] + fn accepted() -> Self { + Self { + accepted_at: Instant::now(), + status: "accepted".into(), + selection_elapsed_ns: None, + durable_ack_elapsed_ns: None, + } + } + + /// Record the initial selection without allowing later attempts to replace it. + fn record_selection(&mut self) -> PyResult<()> { + if self.status != "accepted" { + return Err(PyValueError::new_err("selection requires accepted state")); + } + let elapsed = self.elapsed_ns()?; + self.selection_elapsed_ns = Some(elapsed); + self.status = "selected".into(); + Ok(()) + } + + /// Call only after a synchronous durable decision write returns successfully. + fn record_durable_ack(&mut self) -> PyResult<()> { + if self.status != "selected" { + return Err(PyValueError::new_err( + "acknowledgement requires selected state", + )); + } + let elapsed = self.elapsed_ns()?; + self.durable_ack_elapsed_ns = Some(elapsed); + self.status = "acknowledged".into(); + Ok(()) + } + + /// Preserve a denominator-only terminal observation without a success duration. + fn record_failure(&mut self, reason: &str) -> PyResult<()> { + if !matches!(self.status.as_str(), "accepted" | "selected") { + return Err(PyValueError::new_err("observation is already terminal")); + } + if !matches!( + reason, + "capacity_rejected" + | "selection_failed" + | "write_failed" + | "cancelled" + | "unfinished" + | "store_unavailable" + ) { + return Err(PyValueError::new_err("unknown failure reason")); + } + self.status = reason.into(); + Ok(()) + } +} + +/// Export the receipt separately from token counting. +#[pymodule] +fn _decision_receipt(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py index c34279d5c..42a6e34bc 100644 --- a/tests/test_decision_receipts.py +++ b/tests/test_decision_receipts.py @@ -3,39 +3,119 @@ import http.client import json import threading +import sqlite3 +import pytest from contextual_orchestrator import ModelAgent, TaskOrchestrator -from contextual_orchestrator.server import build_server +from contextual_orchestrator.server import build_server, SecurityConfig -def test_http_route_persists_initial_decision(tmp_path): +def test_http_route_persists_initial_decision(tmp_path, monkeypatch): """A real HTTP route retains one native-clock receipt before completion.""" orchestrator = TaskOrchestrator( [ModelAgent("worker_one", "mock/worker")], state_db=tmp_path / "state.db" ) - server = build_server(orchestrator, port=0, decision_receipts=True) + original_chat = orchestrator.client.chat + dispatch_records = [] + dispatch_ready = threading.Event() + generation_allowed = threading.Event() + dispatched_snapshots = [] + + def inspect_committed_decision(*args, **kwargs): + with sqlite3.connect(tmp_path / "state.db") as independent: + rows = independent.execute( + "SELECT payload FROM orchestration_records WHERE kind = 'initial_decision'" + ).fetchall() + assert len(rows) == 1 + dispatch_records.extend(rows) + from contextual_orchestrator.decision_receipts import _CURRENT_DECISION + dispatched_snapshots.append(_CURRENT_DECISION.get().snapshot()) + dispatch_ready.set() + assert generation_allowed.wait(10) + return original_chat(*args, **kwargs) + + monkeypatch.setattr(orchestrator.client, "chat", inspect_committed_decision) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) worker = threading.Thread(target=server.serve_forever, daemon=True) worker.start() connection = http.client.HTTPConnection(*server.server_address) try: connection.request( "POST", "/v1/chat/completions", - json.dumps({"model": "orchestrator/route", "messages": [ + json.dumps({"model": "orchestrator/auto", "mode": "route", "messages": [ {"role": "user", "content": "hello"} - ]}), {"Content-Type": "application/json"}, + ]}), {"Content-Type": "application/json", "Authorization": "Bearer test-token"}, ) + assert dispatch_ready.wait(10) + assert dispatched_snapshots[0]["status"] == "acknowledged" + assert dispatched_snapshots[0]["durable_ack_elapsed_ns"] is not None + # Hold generation after the ack: the final interval must remain exactly + # the pre-generation native value, irrespective of the hold duration. + assert not generation_allowed.is_set() + generation_allowed.set() response = connection.getresponse() response.read() assert response.status == 200 + assert dispatch_records records = orchestrator._store.load("decision_receipt") assert len(records) == 1 assert records[0]["status"] == "acknowledged" assert records[0]["selected_agent_ids"] == ["worker_one"] assert records[0]["selection_elapsed_ns"] <= records[0]["durable_ack_elapsed_ns"] + assert records[0]["durable_ack_elapsed_ns"] == dispatched_snapshots[0]["durable_ack_elapsed_ns"] assert len(orchestrator._store.load("initial_decision")) == 1 finally: + generation_allowed.set() connection.close() server.shutdown() worker.join() server.server_close() orchestrator.close() + + +def test_measurement_requires_durable_store(): + """Opt-in cannot silently lose every missing-store denominator observation.""" + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")]) + with pytest.raises(ValueError, match="durable state store"): + build_server(orchestrator, port=0, decision_receipts=True) + orchestrator.close() + + +def test_native_rejects_out_of_order_and_duplicate_ack(): + """Native transitions never manufacture missing elapsed times.""" + from contextual_orchestrator._decision_receipt import DecisionReceipt + + receipt = DecisionReceipt() + with pytest.raises(ValueError): + receipt.record_durable_ack() + assert receipt.status == "accepted" + assert receipt.durable_ack_elapsed_ns is None + receipt.record_selection() + receipt.record_durable_ack() + before = receipt.durable_ack_elapsed_ns + with pytest.raises(ValueError): + receipt.record_durable_ack() + assert receipt.durable_ack_elapsed_ns == before + + +def test_write_failure_has_no_ack_and_does_not_log_exception_contents(caplog): + """Failed commits cannot become successful timing samples or leak detail.""" + from contextual_orchestrator.decision_receipts import DecisionMeasurement + + class FailedStore: + def save(self, *args, **kwargs): + raise RuntimeError("secret-canary-never-log") + + measurement = DecisionMeasurement(FailedStore()) + try: + measurement.select(["worker_one"]) + snapshot = measurement.snapshot() + assert snapshot["status"] == "write_failed" + assert snapshot["durable_ack_elapsed_ns"] is None + snapshot["selected_agent_ids"].append("untrusted_mutation") + assert measurement.snapshot()["selected_agent_ids"] == ["worker_one"] + finally: + measurement.close() + assert "secret-canary-never-log" not in caplog.text + assert "error_type=RuntimeError" in caplog.text From 920df5a3a3843f380ae6c365cabac1a1a43c4335 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 15:19:34 +0900 Subject: [PATCH 29/87] test: require persisted admission before chat and SSE dispatch --- tests/test_decision_receipts.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py index 42a6e34bc..078354575 100644 --- a/tests/test_decision_receipts.py +++ b/tests/test_decision_receipts.py @@ -10,31 +10,38 @@ from contextual_orchestrator.server import build_server, SecurityConfig -def test_http_route_persists_initial_decision(tmp_path, monkeypatch): +@pytest.mark.parametrize("stream", [False, True]) +def test_http_route_persists_initial_decision(tmp_path, monkeypatch, stream): """A real HTTP route retains one native-clock receipt before completion.""" orchestrator = TaskOrchestrator( [ModelAgent("worker_one", "mock/worker")], state_db=tmp_path / "state.db" ) - original_chat = orchestrator.client.chat dispatch_records = [] dispatch_ready = threading.Event() generation_allowed = threading.Event() dispatched_snapshots = [] - def inspect_committed_decision(*args, **kwargs): + def inspect_committed_decision(original_call, *args, **kwargs): with sqlite3.connect(tmp_path / "state.db") as independent: rows = independent.execute( "SELECT payload FROM orchestration_records WHERE kind = 'initial_decision'" ).fetchall() + assert independent.execute( + "SELECT COUNT(*) FROM orchestration_records WHERE kind = 'accepted_request'" + ).fetchone()[0] == 1 assert len(rows) == 1 dispatch_records.extend(rows) from contextual_orchestrator.decision_receipts import _CURRENT_DECISION dispatched_snapshots.append(_CURRENT_DECISION.get().snapshot()) dispatch_ready.set() assert generation_allowed.wait(10) - return original_chat(*args, **kwargs) + return original_call(*args, **kwargs) - monkeypatch.setattr(orchestrator.client, "chat", inspect_committed_decision) + for method_name in ("chat", "stream_chat"): + original_call = getattr(orchestrator.client, method_name) + monkeypatch.setattr(orchestrator.client, method_name, + lambda *args, _call=original_call, **kwargs: + inspect_committed_decision(_call, *args, **kwargs)) server = build_server(orchestrator, port=0, decision_receipts=True, security=SecurityConfig(auth_token="test-token")) worker = threading.Thread(target=server.serve_forever, daemon=True) @@ -43,7 +50,7 @@ def inspect_committed_decision(*args, **kwargs): try: connection.request( "POST", "/v1/chat/completions", - json.dumps({"model": "orchestrator/auto", "mode": "route", "messages": [ + json.dumps({"model": "orchestrator/auto", "mode": "route", "stream": stream, "messages": [ {"role": "user", "content": "hello"} ]}), {"Content-Type": "application/json", "Authorization": "Bearer test-token"}, ) From df6eb4a75dac83cac9baebfed1525e5e551792aa Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 15:26:36 +0900 Subject: [PATCH 30/87] feat: retain admission denominator and expose incomplete receipt joins --- contextual_orchestrator/decision_receipts.py | 59 +++++++++++-- contextual_orchestrator/orchestrator.py | 16 ++-- contextual_orchestrator/server.py | 18 +++- tests/test_decision_receipts.py | 93 ++++++++++++++++++-- 4 files changed, 162 insertions(+), 24 deletions(-) diff --git a/contextual_orchestrator/decision_receipts.py b/contextual_orchestrator/decision_receipts.py index b819f2ca0..de8c92ec0 100644 --- a/contextual_orchestrator/decision_receipts.py +++ b/contextual_orchestrator/decision_receipts.py @@ -4,6 +4,8 @@ import logging import threading import uuid +import hashlib +import json _CURRENT_DECISION = ContextVar("initial_decision", default=None) _LOGGER = logging.getLogger(__name__) @@ -12,21 +14,37 @@ class DecisionMeasurement: """Bind one native clock to an HTTP request and its durable store.""" - def __init__(self, store): + def __init__(self, store, *, policy=None, route_mode="unclassified", request_id=None): """Require the native module only when measurement is explicitly enabled.""" from ._decision_receipt import DecisionReceipt self.receipt = DecisionReceipt() self.store = store - self.request_id = uuid.uuid4().hex + self.request_id = request_id or uuid.uuid4().hex + self.identity_source = "http_request" if request_id else "measurement_scope" + self.policy_hash = hashlib.sha256(json.dumps( + policy, sort_keys=True, separators=(",", ":") + ).encode()).hexdigest() if policy is not None else None + self.route_mode = route_mode self.selected_agent_ids = [] self._lock = threading.Lock() self._token = _CURRENT_DECISION.set(self) + try: + if self.store is None: + raise RuntimeError("no durable measurement store") + self.store.save("accepted_request", None, self.snapshot(), durable=True) + except Exception as exc: + _CURRENT_DECISION.reset(self._token) + _LOGGER.warning("Decision admission write failed error_type=%s", type(exc).__name__) + raise RuntimeError("decision measurement admission could not be persisted") from None def snapshot(self): """Return no prompts, provider credentials, or fabricated missing timings.""" return { "request_id": self.request_id, + "identity_source": self.identity_source, + "policy_snapshot_hash": self.policy_hash, + "route_mode": self.route_mode, "status": self.receipt.status, "selected_agent_ids": list(self.selected_agent_ids), "selection_elapsed_ns": self.receipt.selection_elapsed_ns, @@ -34,12 +52,13 @@ def snapshot(self): "metric_scope": "initial_provider_dispatch", } - def select(self, agent_ids): + def select(self, agent_ids, route_mode): """Acknowledge the first decision synchronously before provider dispatch.""" with self._lock: if self.receipt.status != "accepted": return self.receipt.record_selection() + self.route_mode = route_mode self.selected_agent_ids = list(agent_ids) if self.store is None: self.receipt.record_failure("store_unavailable") @@ -67,8 +86,38 @@ def close(self, reason="unfinished"): _CURRENT_DECISION.reset(self._token) -def record_initial_selection(agent_ids): +def record_initial_selection(agent_ids, route_mode="unclassified"): """Record selection only within an explicitly enabled request measurement.""" measurement = _CURRENT_DECISION.get() if measurement is not None: - measurement.select(agent_ids) + measurement.select(agent_ids, route_mode) + + +def export_decision_receipts(store): + """Join retained admissions without dropping missing final acknowledgements. + + This is a local ledger view, not an all-ingress census. Storage outages need + external ingress reconciliation. Concurrent finalization may appear on the + next read; missing values remain unfinished rather than successful zeros. + """ + accepted = store.load("accepted_request") + decisions = {row["request_id"]: row for row in store.load("initial_decision")} + receipts = {row["request_id"]: row for row in store.load("decision_receipt")} + observations = [] + for admission in accepted: + request_id = admission["request_id"] + row = dict(admission) + row["status"] = "unfinished" + if request_id in decisions: + row.update(decisions[request_id]) + row["status"] = "acknowledgement_unobserved" + if request_id in receipts: + row.update(receipts[request_id]) + observations.append(row) + return { + "schema_version": 1, + "measurement_complete": False, + "reconciliation_required": True, + "scope": "retained_local_admissions", + "observations": observations, + } diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index ce407248f..f23f7949c 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -4330,7 +4330,7 @@ def proxy_completion( agent, upstream, effort_profile, api_surface=api_surface ) measured = bool(agent.group_name or requested_model == self.FREE_MODEL) - record_initial_selection([agent.id]) + record_initial_selection([agent.id], "explicit_proxy") started_at = time.perf_counter() try: result = self.client.proxy_send(agent, endpoint, upstream) @@ -4396,7 +4396,7 @@ def proxy_completion( last_failure: tuple[Exception, ModelAgent] | None = None every_failure_was_request_too_large = True for candidate in candidates: - record_initial_selection([candidate.id]) + record_initial_selection([candidate.id], "automatic_proxy") started_at = time.perf_counter() candidate_payload = dict(upstream) candidate_payload["model"] = candidate.model @@ -5258,7 +5258,7 @@ def stream_route( stream_kwargs["effort_profile"] = effort_profile if include_usage: stream_kwargs["include_usage"] = True - record_initial_selection([agent.id]) + record_initial_selection([agent.id], "stream_route") stream = self.client.stream_chat(agent, messages, **stream_kwargs) started_at = time.perf_counter() try: @@ -6784,7 +6784,7 @@ def _plan_generated(self, task: str) -> list[WorkflowStep]: {"role": "user", "content": task}, ] effort_profile = self._role_effort_profile("planner") - record_initial_selection([planner.id]) + record_initial_selection([planner.id], "generated_planner") raw = ( self.client.chat(planner, planner_messages, effort_profile=effort_profile) if effort_profile is not None @@ -7619,7 +7619,7 @@ def call(agent: ModelAgent) -> dict[str, Any] | tuple[bytes, str]: if agent.provider_name == "openrouter" and endpoint == "images/generations" else endpoint ) - record_initial_selection([member.id for member in race_members]) + record_initial_selection([member.id for member in race_members], "capability_race") return ( self.client.proxy_send_bytes(agent, provider_endpoint, payload) if binary else self.client.proxy_send(agent, provider_endpoint, payload) @@ -7678,7 +7678,7 @@ def call(agent: ModelAgent) -> dict[str, Any] | tuple[bytes, str]: ) started_at = time.perf_counter() try: - record_initial_selection([agent.id]) + record_initial_selection([agent.id], "capability_proxy") result = ( self.client.proxy_send_bytes(agent, provider_endpoint, payload) if binary @@ -7768,7 +7768,7 @@ def _invoke( def call(agent: ModelAgent) -> tuple[str, str, str, dict[str, Any] | None]: with self.client.request_settings(**request_settings): - record_initial_selection([member.id for member in race_members]) + record_initial_selection([member.id for member in race_members], "text_race") output = ( self.client.chat(agent, messages, effort_profile=effort_profile) if effort_profile is not None @@ -7841,7 +7841,7 @@ def call(agent: ModelAgent) -> tuple[str, str, str, dict[str, Any] | None]: single_attempt = getattr(self.client, "single_attempt_transport", None) transport_scope = single_attempt() if callable(single_attempt) else nullcontext() with transport_scope: - record_initial_selection([agent.id]) + record_initial_selection([agent.id], "invocation_" + role) output = ( self.client.chat(agent, messages, effort_profile=effort_profile) if effort_profile is not None diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index eed6cc20a..1e4eae3f6 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -26,7 +26,7 @@ import uuid from .admin import ADMIN_HTML, ADMIN_TRANSLATIONS -from .decision_receipts import DecisionMeasurement +from .decision_receipts import DecisionMeasurement, export_decision_receipts from .api_contract import OPENAPI_SPEC from .cost_ledger import ATTRIBUTION_DIMENSIONS, dimension_catalog from .cost_router import ( @@ -5985,7 +5985,10 @@ def do_GET(self) -> None: # noqa: N802 self._send(orchestrator.provider_readiness_report(refresh=raw_refresh == "true")) return if path == "/api/v1/analytics_snapshots/latest": - self._send(orchestrator.analytics_snapshot(locale_bundles=ADMIN_TRANSLATIONS)) + snapshot = orchestrator.analytics_snapshot(locale_bundles=ADMIN_TRANSLATIONS) + if decision_receipts: + snapshot["initial_decision_measurements"] = export_decision_receipts(orchestrator._store) + self._send(snapshot) return if path == "/api/v1/spend_analytics/latest": self._send(orchestrator.spend_analytics()) @@ -8146,7 +8149,16 @@ def _run(self, callback: Any) -> dict[str, Any]: def _acquire_measured_slot(self) -> None: self._decision_failure_reason = "unfinished" - measurement = DecisionMeasurement(orchestrator._store) if decision_receipts else None + try: + measurement = DecisionMeasurement( + orchestrator._store, policy=orchestrator.policy.as_dict() + ) if decision_receipts else None + except RuntimeError: + raise RequestError( + 503, "measurement_unavailable", + "Request measurement is unavailable; retry after service recovery.", + {"measurement_complete": False, "reconciliation_required": True}, + ) from None self._decision_measurement = measurement try: security.acquire_run_slot() diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py index 078354575..45de3077a 100644 --- a/tests/test_decision_receipts.py +++ b/tests/test_decision_receipts.py @@ -10,8 +10,11 @@ from contextual_orchestrator.server import build_server, SecurityConfig -@pytest.mark.parametrize("stream", [False, True]) -def test_http_route_persists_initial_decision(tmp_path, monkeypatch, stream): +@pytest.mark.parametrize("endpoint,stream", [ + ("/v1/chat/completions", False), ("/v1/chat/completions", True), + ("/v1/responses", True), +]) +def test_http_route_persists_initial_decision(tmp_path, monkeypatch, endpoint, stream): """A real HTTP route retains one native-clock receipt before completion.""" orchestrator = TaskOrchestrator( [ModelAgent("worker_one", "mock/worker")], state_db=tmp_path / "state.db" @@ -48,11 +51,13 @@ def inspect_committed_decision(original_call, *args, **kwargs): worker.start() connection = http.client.HTTPConnection(*server.server_address) try: + body = {"model": "orchestrator/auto", "mode": "route", "stream": stream, + "messages": [{"role": "user", "content": "hello"}]} + if endpoint == "/v1/responses": + body = {"model": "orchestrator/auto", "stream": True, "input": "hello"} connection.request( - "POST", "/v1/chat/completions", - json.dumps({"model": "orchestrator/auto", "mode": "route", "stream": stream, "messages": [ - {"role": "user", "content": "hello"} - ]}), {"Content-Type": "application/json", "Authorization": "Bearer test-token"}, + "POST", endpoint, json.dumps(body), + {"Content-Type": "application/json", "Authorization": "Bearer test-token"}, ) assert dispatch_ready.wait(10) assert dispatched_snapshots[0]["status"] == "acknowledged" @@ -112,11 +117,12 @@ def test_write_failure_has_no_ack_and_does_not_log_exception_contents(caplog): class FailedStore: def save(self, *args, **kwargs): - raise RuntimeError("secret-canary-never-log") + if args[0] == "initial_decision": + raise RuntimeError("secret-canary-never-log") measurement = DecisionMeasurement(FailedStore()) try: - measurement.select(["worker_one"]) + measurement.select(["worker_one"], "route") snapshot = measurement.snapshot() assert snapshot["status"] == "write_failed" assert snapshot["durable_ack_elapsed_ns"] is None @@ -126,3 +132,74 @@ def save(self, *args, **kwargs): measurement.close() assert "secret-canary-never-log" not in caplog.text assert "error_type=RuntimeError" in caplog.text + + +def test_admission_write_failure_rejects_before_dispatch_and_recovers(tmp_path, monkeypatch): + """A failed ingress receipt returns safe 503 and does not poison the next context.""" + orchestrator = TaskOrchestrator( + [ModelAgent("worker_one", "mock/worker")], state_db=tmp_path / "state.db" + ) + original_save = orchestrator._store.save + original_chat = orchestrator.client.chat + failed_once = [] + dispatched = [] + + def fail_first_admission(kind, *args, **kwargs): + if kind == "accepted_request" and not failed_once: + failed_once.append(True) + raise RuntimeError("never-disclose-storage-secret") + return original_save(kind, *args, **kwargs) + + def record_dispatch(*args, **kwargs): + dispatched.append(True) + return original_chat(*args, **kwargs) + + monkeypatch.setattr(orchestrator._store, "save", fail_first_admission) + monkeypatch.setattr(orchestrator.client, "chat", record_dispatch) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address) + try: + for expected_status in (503, 200): + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "route", + "messages": [{"role": "user", "content": "hello"}], + }), {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + payload = response.read().decode() + assert response.status == expected_status + assert "never-disclose-storage-secret" not in payload + if expected_status == 503: + assert not dispatched + assert '"measurement_complete": false' in payload + assert len(orchestrator._store.load("accepted_request")) == 1 + assert len(orchestrator._store.load("decision_receipt")) == 1 + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +def test_export_retains_accepted_request_without_finalization(tmp_path): + """Crash-like missing finalization stays an unfinished denominator row.""" + from contextual_orchestrator.decision_receipts import export_decision_receipts + from contextual_orchestrator.orchestrator import _StateStore + + store = _StateStore(tmp_path / "state.db") + try: + store.save("accepted_request", None, { + "request_id": "accepted_only", "status": "accepted", + "selection_elapsed_ns": None, "durable_ack_elapsed_ns": None, + }, durable=True) + exported = export_decision_receipts(store) + assert exported["measurement_complete"] is False + assert exported["reconciliation_required"] is True + assert exported["observations"][0]["request_id"] == "accepted_only" + assert exported["observations"][0]["status"] == "unfinished" + assert exported["observations"][0]["durable_ack_elapsed_ns"] is None + finally: + store.close() From 01ce9035715fab4ed60e7352caa85512f855e0bb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 15:28:29 +0900 Subject: [PATCH 31/87] test: verify race receipt identity and incomplete admission evidence --- contextual_orchestrator/server.py | 2 + .../doctoring/decision_receipt_integration.md | 38 +++++++++++++++++ tests/test_decision_receipts.py | 41 +++++++++++++++++++ 3 files changed, 81 insertions(+) diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 1e4eae3f6..2bc6bfdcd 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -8148,6 +8148,7 @@ def _run(self, callback: Any) -> dict[str, Any]: self._release_measured_slot() def _acquire_measured_slot(self) -> None: + """Retain accepted admission before the nonblocking capacity decision.""" self._decision_failure_reason = "unfinished" try: measurement = DecisionMeasurement( @@ -8169,6 +8170,7 @@ def _acquire_measured_slot(self) -> None: raise def _release_measured_slot(self) -> None: + """Finalize measurement independently of releasing the execution slot.""" try: measurement = self._decision_measurement if measurement is not None: diff --git a/docs/doctoring/decision_receipt_integration.md b/docs/doctoring/decision_receipt_integration.md index 0a86a3b69..965a4d34e 100644 --- a/docs/doctoring/decision_receipt_integration.md +++ b/docs/doctoring/decision_receipt_integration.md @@ -41,3 +41,41 @@ actual wheel installation outside the source checkout (the current editable extension is not package proof). Add the Proposed ADR with the completed seam inventory, dependency graph and rendering inspection. Do not claim production instrumentation, p95, accuracy improvement, protected CI, or release. + +## Admission and SSE follow-up + +At candidate descended from `df6eb4a7`, ten focused tests passed in 4.72 s. +These include ordinary HTTP chat, direct chat SSE, direct Responses SSE, +actual race worker context propagation and full candidate-set identity, and +oversized-race rejection before acknowledgement. The first race test fixture +omitted the required group identity and correctly exercised sequential routing; +adding the operator-contract group activated the intended race path. + +An immutable accepted_request row now precedes the capacity check. Export joins +that row with initial_decision and final decision_receipt rather than dropping +unfinished requests. Rows are append-only and not automatically pruned; operators +must retain them for reconciliation and archive after verified export. A storage +outage can prevent the admission itself from being recorded: opt-in then returns +503 before dispatch, states measurement_complete=false and reconciliation_required=true, +and requires external ingress evidence. The local export never claims all-ingress +completeness. This is a deliberate limitation, not a zero-duration observation. + +The existing analytics snapshot carries the joined records only with measurement +enabled and retains its local-runtime labeling. A separate Rust clock remains +the only duration implementation. A trusted request_id argument is the explicit +port for PR #1105; until integrated, identity_source=measurement_scope is honest. +The policy snapshot is hashed; route_mode currently identifies the dispatch +kind/role, not a complete top-level requested-mode contract. + +An isolated noneditable core install plus native wheel, outside the checkout, +passed six tests (one then-current test deselected) in 9.13 s. Both imports were +verified under the isolated environment's site-packages. That build used an +uncommitted candidate and is not final exact-head package evidence. Dependencies +resolved anew there; locked PyO3 remained 0.29.2. Rebuild the final commit before +release. No binary is committed. + +Still unverified: cache-hit and other callback-bypassing accepted-request coverage, +all structured/coordinator/direct-provider paths, selection/cancellation/capacity +HTTP negatives, complete ingress reconciliation, transactional snapshot export, +declared retention window and actual release packaging workflow. No customer KPI +gain is established. The Proposed ADR and full rendering inspection remain due. diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py index 45de3077a..56eca723f 100644 --- a/tests/test_decision_receipts.py +++ b/tests/test_decision_receipts.py @@ -203,3 +203,44 @@ def test_export_retains_accepted_request_without_finalization(tmp_path): assert exported["observations"][0]["durable_ack_elapsed_ns"] is None finally: store.close() + + +@pytest.mark.parametrize("invalid_capacity", [False, True]) +def test_race_receipt_retains_candidate_set_not_winner(tmp_path, monkeypatch, invalid_capacity): + """Real race workers share one clock; rejected races cannot acknowledge selection.""" + from contextual_orchestrator.decision_receipts import DecisionMeasurement + from contextual_orchestrator.orchestrator import MAX_LOCAL_CONCURRENCY + + contract = { + "contract_id": "test_contract", "model_revision": "test_revision", + "reasoning_effort_profile": "worker_medium", "capability_set": ["text"], + "structured_output_contract": "openai_response_v1", "accuracy_class": "full_precision", + "data_residency_policy": "test_region", "retention_policy": "zero_retention", + "context_limit": 128000, "pricing_evidence_id": "test_price_evidence", + "hedge_eligible": True, "cancellation_supported": False, + "execution_policy": "immediate_race", + } + agents = [ModelAgent(f"worker_{index}", "mock/worker", group_name="shared_group", + endpoint_equivalence=contract) + for index in range(2)] + orchestrator = TaskOrchestrator(agents, state_db=tmp_path / "state.db") + measurement = DecisionMeasurement(orchestrator._store) + try: + if invalid_capacity: + monkeypatch.setattr(orchestrator, "_equivalent_race_members", + lambda *args, **kwargs: [agents[0]] * (MAX_LOCAL_CONCURRENCY + 1)) + with pytest.raises(ValueError, match="concurrency capacity"): + orchestrator._invoke(agents[0], [{"role": "user", "content": "hello"}], + text="hello", role="worker") + assert measurement.receipt.status == "accepted" + assert not orchestrator._store.load("initial_decision") + else: + orchestrator._invoke(agents[0], [{"role": "user", "content": "hello"}], + text="hello", role="worker") + snapshot = measurement.snapshot() + assert snapshot["status"] == "acknowledged" + assert set(snapshot["selected_agent_ids"]) == {"worker_0", "worker_1"} + assert len(orchestrator._store.load("initial_decision")) == 1 + finally: + measurement.close() + orchestrator.close() From aeb66e94d7ea4ddef5b04267661aca63c02ae329 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 15:39:53 +0900 Subject: [PATCH 32/87] fix: preserve one admission across embedding failover and bound receipt exports --- contextual_orchestrator/cost_router.py | 5 ++ contextual_orchestrator/decision_receipts.py | 39 +++++++-- contextual_orchestrator/orchestrator.py | 36 +++++++- contextual_orchestrator/server.py | 74 +++++++++------- tests/test_decision_receipts.py | 92 ++++++++++++++++++++ 5 files changed, 207 insertions(+), 39 deletions(-) diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index 751c9e366..880ac9119 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -24,6 +24,7 @@ from dataclasses import replace from threading import Lock from typing import Any, Dict, List, Optional +from .decision_receipts import record_initial_selection from .batch_routing import ( BatchBackend, @@ -1332,6 +1333,8 @@ def submit_embeddings_batch( if callable(reserve) and callable(start): job = reserve(requests, metadata=metadata) else: + if resolved_agent_id is not None: + record_initial_selection([resolved_agent_id], "embedding_submission") job = backend.submit(requests, metadata=metadata) self._embedding_models[job.job_id] = resolved_model self._embedding_owners[job.job_id] = owner_id @@ -1341,6 +1344,8 @@ def submit_embeddings_batch( self._embedding_part_limits[job.job_id] = part_limits self._embedding_jobs[job.job_id] = job if callable(reserve) and callable(start): + if resolved_agent_id is not None: + record_initial_selection([resolved_agent_id], "embedding_submission") start(job) return job diff --git a/contextual_orchestrator/decision_receipts.py b/contextual_orchestrator/decision_receipts.py index de8c92ec0..6e0aae2f6 100644 --- a/contextual_orchestrator/decision_receipts.py +++ b/contextual_orchestrator/decision_receipts.py @@ -14,7 +14,9 @@ class DecisionMeasurement: """Bind one native clock to an HTTP request and its durable store.""" - def __init__(self, store, *, policy=None, route_mode="unclassified", request_id=None): + def __init__(self, store, *, policy=None, route_mode="unclassified", request_id=None, + endpoint_path=None, request_method=None, + admission_boundary="explicit_scope"): """Require the native module only when measurement is explicitly enabled.""" from ._decision_receipt import DecisionReceipt @@ -22,11 +24,16 @@ def __init__(self, store, *, policy=None, route_mode="unclassified", request_id= self.store = store self.request_id = request_id or uuid.uuid4().hex self.identity_source = "http_request" if request_id else "measurement_scope" + policy_snapshot = policy.as_dict() if policy is not None else None self.policy_hash = hashlib.sha256(json.dumps( - policy, sort_keys=True, separators=(",", ":") - ).encode()).hexdigest() if policy is not None else None + policy_snapshot, sort_keys=True, separators=(",", ":") + ).encode()).hexdigest() if policy_snapshot is not None else None self.route_mode = route_mode + self.endpoint_path = endpoint_path + self.request_method = request_method + self.admission_boundary = admission_boundary self.selected_agent_ids = [] + self.selection_attempt_count = 0 self._lock = threading.Lock() self._token = _CURRENT_DECISION.set(self) try: @@ -45,8 +52,13 @@ def snapshot(self): "identity_source": self.identity_source, "policy_snapshot_hash": self.policy_hash, "route_mode": self.route_mode, + "endpoint_path": self.endpoint_path, + "request_method": self.request_method, + "measurement_unit": "http_request" if self.endpoint_path else "explicit_scope", + "admission_boundary": self.admission_boundary, "status": self.receipt.status, "selected_agent_ids": list(self.selected_agent_ids), + "selection_attempt_count": self.selection_attempt_count, "selection_elapsed_ns": self.receipt.selection_elapsed_ns, "durable_ack_elapsed_ns": self.receipt.durable_ack_elapsed_ns, "metric_scope": "initial_provider_dispatch", @@ -55,7 +67,18 @@ def snapshot(self): def select(self, agent_ids, route_mode): """Acknowledge the first decision synchronously before provider dispatch.""" with self._lock: + if route_mode in ("text_race", "capability_race") and self.route_mode == route_mode: + return # Parallel replicas belong to the same selected candidate set. + self.selection_attempt_count += 1 if self.receipt.status != "accepted": + try: + self.store.save("selection_attempt", None, { + "request_id": self.request_id, + "attempt_number": self.selection_attempt_count, + "selected_agent_ids": list(agent_ids), "route_mode": route_mode, + }, durable=True) + except Exception as exc: + _LOGGER.warning("Decision attempt export failed error_type=%s", type(exc).__name__) return self.receipt.record_selection() self.route_mode = route_mode @@ -93,16 +116,17 @@ def record_initial_selection(agent_ids, route_mode="unclassified"): measurement.select(agent_ids, route_mode) -def export_decision_receipts(store): +def export_decision_receipts(store, limit=256): """Join retained admissions without dropping missing final acknowledgements. This is a local ledger view, not an all-ingress census. Storage outages need external ingress reconciliation. Concurrent finalization may appear on the next read; missing values remain unfinished rather than successful zeros. """ - accepted = store.load("accepted_request") - decisions = {row["request_id"]: row for row in store.load("initial_decision")} - receipts = {row["request_id"]: row for row in store.load("decision_receipt")} + cohort = store.load_decision_window(limit) + accepted = cohort["accepted"] + decisions = {row["request_id"]: row for row in cohort["decisions"]} + receipts = {row["request_id"]: row for row in cohort["receipts"]} observations = [] for admission in accepted: request_id = admission["request_id"] @@ -119,5 +143,6 @@ def export_decision_receipts(store): "measurement_complete": False, "reconciliation_required": True, "scope": "retained_local_admissions", + "window": cohort["window"], "observations": observations, } diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index f23f7949c..147e7bcf7 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -3724,6 +3724,40 @@ def save(self, kind: str, key: str | None, payload: dict[str, Any], *, durable: return self._save_sync(kind, key, payload) + def load_decision_window(self, limit: int = 256) -> dict[str, Any]: + """Read a bounded shared admission cohort without deleting historical rows.""" + if type(limit) is not int or not 1 <= limit <= 1000: + raise ValueError("decision window limit must be between 1 and 1000") + with self._lock: + admissions = self._conn.execute( + "SELECT seq, payload FROM orchestration_records WHERE kind = ? ORDER BY seq DESC LIMIT ?", + ("accepted_request", limit + 1), + ).fetchall() + truncated = len(admissions) > limit + admissions = list(reversed(admissions[:limit])) + accepted = [json.loads(payload) for _, payload in admissions] + request_ids = [row["request_id"] for row in accepted] + phases = [] + if request_ids: + placeholders = ",".join("?" for _ in request_ids) + phases = self._conn.execute( + "SELECT kind, payload FROM orchestration_records " + "WHERE kind IN ('initial_decision', 'decision_receipt') " + "AND json_extract(payload, '$.request_id') IN (" + placeholders + ") " + "ORDER BY seq DESC LIMIT ?", + (*request_ids, 2 * limit + 1), + ).fetchall() + phase_truncated = len(phases) > 2 * limit + phases = list(reversed(phases[:2 * limit])) + return { + "accepted": accepted, + "decisions": [json.loads(payload) for kind, payload in phases if kind == "initial_decision"], + "receipts": [json.loads(payload) for kind, payload in phases if kind == "decision_receipt"], + "window": {"limit": limit, "truncated": truncated, "phase_truncated": phase_truncated, + "first_admission_seq": admissions[0][0] if admissions else None, + "last_admission_seq": admissions[-1][0] if admissions else None}, + } + def _save_sync(self, kind: str, key: str | None, payload: dict[str, Any]) -> None: blob = json.dumps(payload, ensure_ascii=False) with self._lock: @@ -4396,7 +4430,6 @@ def proxy_completion( last_failure: tuple[Exception, ModelAgent] | None = None every_failure_was_request_too_large = True for candidate in candidates: - record_initial_selection([candidate.id], "automatic_proxy") started_at = time.perf_counter() candidate_payload = dict(upstream) candidate_payload["model"] = candidate.model @@ -4415,6 +4448,7 @@ def proxy_completion( send_once = getattr(self.client, "proxy_send_once", None) if not callable(send_once): send_once = self.client.proxy_send + record_initial_selection([candidate.id], "automatic_proxy") result = send_once(candidate, endpoint, candidate_payload) except Exception as exc: # noqa: BLE001 - provider trust boundary if not _is_passthrough_failover_error(exc): diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 2bc6bfdcd..c07f4c5db 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -5636,9 +5636,14 @@ def handle_one_request(self) -> None: self.command = None self.path = None self._request_started = None + self._decision_measurement = None + self._decision_failure_reason = "unfinished" try: super().handle_one_request() finally: + if self._decision_measurement is not None: + self._decision_measurement.close(self._decision_failure_reason) + self._decision_measurement = None self._log_request_summary(self._request_started) self._reset_session() # A request that declared a body it never delivered (unsupported @@ -7241,13 +7246,6 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di ) # Same pool honesty as chat/Completions: do not silently serve # a different embedding deployment than the client requested. - embedding_agents = orchestrator._capability_agents( - "embedding", - TaskOrchestrator.AUTO_MODEL if model_was_omitted else model_name, - ) - embedding_agents = coordinator._cost_ordered_capability_candidates( - embedding_agents - ) encoding_format = _validate_embeddings_encoding_format(body) _validate_embeddings_dimensions(body) end_user_id = _validate_completions_user(body) @@ -7297,6 +7295,12 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di attribution["model_name"] = model_name if not attribution.get("service"): attribution["service"] = "embeddings_api" + self._ensure_decision_measurement("validated_endpoint") + embedding_agents = coordinator._cost_ordered_capability_candidates( + orchestrator._capability_agents( + "embedding", TaskOrchestrator.AUTO_MODEL if model_was_omitted else model_name + ) + ) started_at = time.perf_counter() embedding_deadline = time.monotonic() + float( orchestrator.client.timeout @@ -7383,13 +7387,6 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di _require_pool_model( orchestrator, model_name, required_capability="embedding" ) - embedding_agents = orchestrator._capability_agents( - "embedding", - TaskOrchestrator.AUTO_MODEL if model_was_omitted else model_name, - ) - embedding_agents = coordinator._cost_ordered_capability_candidates( - embedding_agents - ) _validate_embeddings_encoding_format(body) _validate_embeddings_dimensions(body) # OpenAI ``user`` end-user id — same fail-closed shape as sync embeddings. @@ -7408,6 +7405,12 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di endpoint_alias = _validate_batch_embeddings_endpoint(body) if endpoint_alias is not None: submit_metadata["endpoint_alias"] = endpoint_alias + self._ensure_decision_measurement("validated_endpoint") + embedding_agents = coordinator._cost_ordered_capability_candidates( + orchestrator._capability_agents( + "embedding", TaskOrchestrator.AUTO_MODEL if model_was_omitted else model_name + ) + ) document = None last_embedding_error: Exception | None = None for embedding_agent in embedding_agents: @@ -8149,26 +8152,35 @@ def _run(self, callback: Any) -> dict[str, Any]: def _acquire_measured_slot(self) -> None: """Retain accepted admission before the nonblocking capacity decision.""" - self._decision_failure_reason = "unfinished" - try: - measurement = DecisionMeasurement( - orchestrator._store, policy=orchestrator.policy.as_dict() - ) if decision_receipts else None - except RuntimeError: - raise RequestError( - 503, "measurement_unavailable", - "Request measurement is unavailable; retry after service recovery.", - {"measurement_complete": False, "reconciliation_required": True}, - ) from None - self._decision_measurement = measurement + self._ensure_decision_measurement() try: security.acquire_run_slot() except Exception: - if measurement is not None: - measurement.close("capacity_rejected") - self._decision_measurement = None + self._decision_failure_reason = "capacity_rejected" raise + def _ensure_decision_measurement(self, boundary="first_execution_slot") -> None: + """Open one request scope, with explicit timing eligibility at this boundary.""" + if decision_receipts and self._decision_measurement is None: + endpoint = urllib.parse.urlsplit(self.path).path + if endpoint not in { + "/v1/chat/completions", "/v1/completions", "/v1/responses", + "/v1/embeddings", "/v1/batch/embeddings", + }: + endpoint = "other_execution_endpoint" + try: + self._decision_measurement = DecisionMeasurement( + orchestrator._store, policy=orchestrator.policy, + endpoint_path=endpoint, request_method=self.command, + admission_boundary=boundary, + ) + except RuntimeError: + raise RequestError( + 503, "measurement_unavailable", + "Request measurement is unavailable; retry after service recovery.", + {"measurement_complete": False, "reconciliation_required": True}, + ) from None + def _release_measured_slot(self) -> None: """Finalize measurement independently of releasing the execution slot.""" try: @@ -8180,9 +8192,8 @@ def _release_measured_slot(self) -> None: reason = "cancelled" elif active_error is not None: reason = "selection_failed" - measurement.close(reason) + self._decision_failure_reason = reason finally: - self._decision_measurement = None security.release_run_slot() def _parse_positive_int(self, raw: str | None, field_name: str, default: int, max_value: int | None = None) -> int: @@ -8553,6 +8564,7 @@ def progress(role: str, status: str) -> None: self._write_sse("data: [DONE]\n\n") return False except Exception: # noqa: BLE001 - headers sent; terminate with a valid Responses event + self._decision_failure_reason = "selection_failed" failed = { **created_response, "status": "failed", diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py index 56eca723f..e42523662 100644 --- a/tests/test_decision_receipts.py +++ b/tests/test_decision_receipts.py @@ -205,6 +205,45 @@ def test_export_retains_accepted_request_without_finalization(tmp_path): store.close() +def test_export_window_keeps_unfinished_selected_cohort(tmp_path): + """A bounded admission cohort never silently drops its unfinished member.""" + from contextual_orchestrator.decision_receipts import export_decision_receipts + from contextual_orchestrator.orchestrator import _StateStore + + store = _StateStore(tmp_path / "state.db") + try: + for request_id in ("older_request", "unfinished_request", "newest_request"): + store.save("accepted_request", None, {"request_id": request_id, "status": "accepted"}, durable=True) + exported = export_decision_receipts(store, limit=2) + assert exported["window"]["truncated"] is True + assert [row["request_id"] for row in exported["observations"]] == ["unfinished_request", "newest_request"] + assert all(row["status"] == "unfinished" for row in exported["observations"]) + assert len(store.load("accepted_request")) == 3 # No retention deletion. + finally: + store.close() + + +def test_failed_nested_admission_restores_same_thread_context(tmp_path): + """A constructor failure restores its prior token before leaving the thread.""" + from contextual_orchestrator.decision_receipts import DecisionMeasurement, _CURRENT_DECISION + from contextual_orchestrator.orchestrator import _StateStore + + class FailedStore: + def save(self, *args, **kwargs): + raise RuntimeError("unavailable") + + store = _StateStore(tmp_path / "state.db") + outer = DecisionMeasurement(store) + try: + with pytest.raises(RuntimeError, match="could not be persisted"): + DecisionMeasurement(FailedStore()) + assert _CURRENT_DECISION.get() is outer + finally: + outer.close() + store.close() + assert _CURRENT_DECISION.get() is None + + @pytest.mark.parametrize("invalid_capacity", [False, True]) def test_race_receipt_retains_candidate_set_not_winner(tmp_path, monkeypatch, invalid_capacity): """Real race workers share one clock; rejected races cannot acknowledge selection.""" @@ -244,3 +283,56 @@ def test_race_receipt_retains_candidate_set_not_winner(tmp_path, monkeypatch, in finally: measurement.close() orchestrator.close() + + +def test_embedding_failover_has_one_admission_and_two_selection_attempts(tmp_path, monkeypatch): + """Two backend submissions remain children of one validated HTTP request.""" + from contextual_orchestrator.cost_router import CostRoutingCoordinator + + agents = [ModelAgent(f"embedding_{index}", f"mock-embedding-{index}", tags=("embedding",)) + for index in range(2)] + orchestrator = TaskOrchestrator(agents, state_db=tmp_path / "state.db") + class FixtureTokenCounter: + def count_text(self, text, model): + assert text == "hello" + return 1 + + coordinator = CostRoutingCoordinator(orchestrator, embedding_token_counter=FixtureTokenCounter()) + backend = coordinator.embedding_batch_backend + original_submit = backend.submit + submissions = [] + + def fail_first_submission(*args, **kwargs): + submissions.append(True) + if len(submissions) == 1: + raise RuntimeError("controlled first member failure") + return original_submit(*args, **kwargs) + + monkeypatch.setattr(backend, "submit", fail_first_submission) + server = build_server(orchestrator, port=0, coordinator=coordinator, + decision_receipts=True, security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address) + try: + connection.request("POST", "/v1/embeddings", json.dumps({"input": "hello"}), + {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + response.read() + assert response.status == 200 + connection.close() + server.shutdown() + server.server_close() + assert len(submissions) == 2 + assert len(orchestrator._store.load("accepted_request")) == 1 + receipts = orchestrator._store.load("decision_receipt") + assert len(receipts) == 1 + assert receipts[0]["selection_attempt_count"] == 2 + assert receipts[0]["admission_boundary"] == "validated_endpoint" + assert len(orchestrator._store.load("selection_attempt")) == 1 + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() From 6e4f87aa3a27e1f1ccdb435a6c039145617060ed Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 15:44:41 +0900 Subject: [PATCH 33/87] fix: index receipt cohorts and distinguish repeated race decisions --- contextual_orchestrator/decision_receipts.py | 13 +++--- contextual_orchestrator/orchestrator.py | 47 ++++++++++++++++---- tests/test_decision_receipts.py | 40 ++++++++++++++++- 3 files changed, 85 insertions(+), 15 deletions(-) diff --git a/contextual_orchestrator/decision_receipts.py b/contextual_orchestrator/decision_receipts.py index 6e0aae2f6..5db7601e4 100644 --- a/contextual_orchestrator/decision_receipts.py +++ b/contextual_orchestrator/decision_receipts.py @@ -34,6 +34,7 @@ def __init__(self, store, *, policy=None, route_mode="unclassified", request_id= self.admission_boundary = admission_boundary self.selected_agent_ids = [] self.selection_attempt_count = 0 + self._race_attempt_ids = set() self._lock = threading.Lock() self._token = _CURRENT_DECISION.set(self) try: @@ -64,11 +65,13 @@ def snapshot(self): "metric_scope": "initial_provider_dispatch", } - def select(self, agent_ids, route_mode): + def select(self, agent_ids, route_mode, *, attempt_id=None): """Acknowledge the first decision synchronously before provider dispatch.""" with self._lock: - if route_mode in ("text_race", "capability_race") and self.route_mode == route_mode: - return # Parallel replicas belong to the same selected candidate set. + if attempt_id is not None: + if attempt_id in self._race_attempt_ids: + return + self._race_attempt_ids.add(attempt_id) self.selection_attempt_count += 1 if self.receipt.status != "accepted": try: @@ -109,11 +112,11 @@ def close(self, reason="unfinished"): _CURRENT_DECISION.reset(self._token) -def record_initial_selection(agent_ids, route_mode="unclassified"): +def record_initial_selection(agent_ids, route_mode="unclassified", *, attempt_id=None): """Record selection only within an explicitly enabled request measurement.""" measurement = _CURRENT_DECISION.get() if measurement is not None: - measurement.select(agent_ids, route_mode) + measurement.select(agent_ids, route_mode, attempt_id=attempt_id) def export_decision_receipts(store, limit=256): diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index d8c36135e..e77d060bc 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -3645,6 +3645,7 @@ class _StateStore: _LEGACY_INDEX_NAME = "records_kind_seq" _INDEX_NAME = "orchestration_records_kind_seq" _STREAM_LIMITS = {"audit": 256, "authorization": 256, "analytics": 256} + _MEASUREMENT_KINDS = ("accepted_request", "initial_decision", "decision_receipt", "selection_attempt") _CREATE_RECORDS_SQL = ( "CREATE TABLE IF NOT EXISTS orchestration_records (" "seq INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT NOT NULL, key TEXT, payload TEXT NOT NULL)" @@ -3669,6 +3670,18 @@ def __init__(self, path: str) -> None: self._migrate_legacy_table() self._conn.execute(self._CREATE_RECORDS_SQL) self._conn.execute(self._CREATE_RECORDS_KIND_SEQ_INDEX_SQL) + self._conn.execute( + "CREATE INDEX IF NOT EXISTS orchestration_records_kind_key_seq " + "ON orchestration_records(kind, key, seq)" + ) + self._conn.execute( + "UPDATE orchestration_records SET key = json_extract(payload, '$.request_id') " + "WHERE kind IN (?, ?, ?, ?) AND key IS NULL " + "AND CASE WHEN json_valid(payload) THEN " + "json_type(payload, '$.request_id') = 'text' " + "AND length(json_extract(payload, '$.request_id')) > 0 ELSE 0 END", + self._MEASUREMENT_KINDS, + ) self._conn.commit() except Exception: self._conn.rollback() @@ -3730,35 +3743,49 @@ def load_decision_window(self, limit: int = 256) -> dict[str, Any]: raise ValueError("decision window limit must be between 1 and 1000") with self._lock: admissions = self._conn.execute( - "SELECT seq, payload FROM orchestration_records WHERE kind = ? ORDER BY seq DESC LIMIT ?", + "SELECT seq, key, payload FROM orchestration_records WHERE kind = ? ORDER BY seq DESC LIMIT ?", ("accepted_request", limit + 1), ).fetchall() truncated = len(admissions) > limit admissions = list(reversed(admissions[:limit])) - accepted = [json.loads(payload) for _, payload in admissions] + accepted = [json.loads(payload) for _, _, payload in admissions] + if any(key != row.get("request_id") for (_, key, _), row in zip(admissions, accepted)): + raise ValueError("measurement admission identity mismatch") request_ids = [row["request_id"] for row in accepted] phases = [] if request_ids: placeholders = ",".join("?" for _ in request_ids) phases = self._conn.execute( - "SELECT kind, payload FROM orchestration_records " + "SELECT kind, key, payload FROM orchestration_records " "WHERE kind IN ('initial_decision', 'decision_receipt') " - "AND json_extract(payload, '$.request_id') IN (" + placeholders + ") " + "AND key IN (" + placeholders + ") " "ORDER BY seq DESC LIMIT ?", (*request_ids, 2 * limit + 1), ).fetchall() phase_truncated = len(phases) > 2 * limit phases = list(reversed(phases[:2 * limit])) + if any(key != json.loads(payload).get("request_id") for _, key, payload in phases): + raise ValueError("measurement phase identity mismatch") + unresolved_legacy = self._conn.execute( + "SELECT 1 FROM orchestration_records WHERE kind IN (?, ?, ?, ?) " + "AND key IS NULL LIMIT 1", self._MEASUREMENT_KINDS, + ).fetchone() is not None return { "accepted": accepted, - "decisions": [json.loads(payload) for kind, payload in phases if kind == "initial_decision"], - "receipts": [json.loads(payload) for kind, payload in phases if kind == "decision_receipt"], + "decisions": [json.loads(payload) for kind, _, payload in phases if kind == "initial_decision"], + "receipts": [json.loads(payload) for kind, _, payload in phases if kind == "decision_receipt"], "window": {"limit": limit, "truncated": truncated, "phase_truncated": phase_truncated, + "unresolved_legacy_identity": unresolved_legacy, "first_admission_seq": admissions[0][0] if admissions else None, "last_admission_seq": admissions[-1][0] if admissions else None}, } def _save_sync(self, kind: str, key: str | None, payload: dict[str, Any]) -> None: + if kind in self._MEASUREMENT_KINDS: + request_id = payload.get("request_id") + if not isinstance(request_id, str) or not request_id or (key is not None and key != request_id): + raise ValueError("measurement record requires a consistent request identity") + key = request_id blob = json.dumps(payload, ensure_ascii=False) with self._lock, self._conn: if kind in self._KEYED: @@ -7652,7 +7679,8 @@ def call(agent: ModelAgent) -> dict[str, Any] | tuple[bytes, str]: if agent.provider_name == "openrouter" and endpoint == "images/generations" else endpoint ) - record_initial_selection([member.id for member in race_members], "capability_race") + record_initial_selection([member.id for member in race_members], "capability_race", + attempt_id=decision_attempt_id) return ( self.client.proxy_send_bytes(agent, provider_endpoint, payload) if binary else self.client.proxy_send(agent, provider_endpoint, payload) @@ -7660,6 +7688,7 @@ def call(agent: ModelAgent) -> dict[str, Any] | tuple[bytes, str]: contract = EndpointEquivalenceContract(**race_members[0].endpoint_equivalence) # type: ignore[arg-type] attempt_completed, finalize_attempts = self._race_attempt_collector(capability) + decision_attempt_id = uuid.uuid4().hex try: outcome = race_first_valid( [ @@ -7801,7 +7830,8 @@ def _invoke( def call(agent: ModelAgent) -> tuple[str, str, str, dict[str, Any] | None]: with self.client.request_settings(**request_settings): - record_initial_selection([member.id for member in race_members], "text_race") + record_initial_selection([member.id for member in race_members], "text_race", + attempt_id=decision_attempt_id) output = ( self.client.chat(agent, messages, effort_profile=effort_profile) if effort_profile is not None @@ -7812,6 +7842,7 @@ def call(agent: ModelAgent) -> tuple[str, str, str, dict[str, Any] | None]: contract = EndpointEquivalenceContract(**race_members[0].endpoint_equivalence) # type: ignore[arg-type] attempt_completed, finalize_attempts = self._race_attempt_collector("text") + decision_attempt_id = uuid.uuid4().hex try: outcome = race_first_valid( [ diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py index e42523662..a09e2bc39 100644 --- a/tests/test_decision_receipts.py +++ b/tests/test_decision_receipts.py @@ -274,10 +274,12 @@ def test_race_receipt_retains_candidate_set_not_winner(tmp_path, monkeypatch, in assert measurement.receipt.status == "accepted" assert not orchestrator._store.load("initial_decision") else: - orchestrator._invoke(agents[0], [{"role": "user", "content": "hello"}], - text="hello", role="worker") + for _ in range(2): + orchestrator._invoke(agents[0], [{"role": "user", "content": "hello"}], + text="hello", role="worker") snapshot = measurement.snapshot() assert snapshot["status"] == "acknowledged" + assert snapshot["selection_attempt_count"] == 2 assert set(snapshot["selected_agent_ids"]) == {"worker_0", "worker_1"} assert len(orchestrator._store.load("initial_decision")) == 1 finally: @@ -336,3 +338,37 @@ def fail_first_submission(*args, **kwargs): worker.join() server.server_close() orchestrator.close() + + +def test_legacy_identity_backfill_and_indexed_window(tmp_path): + """Legacy keys migrate without touching unrelated data or scanning every phase.""" + from contextual_orchestrator.orchestrator import _StateStore + + database = tmp_path / "state.db" + store = _StateStore(database) + with store._conn: + store._conn.executemany( + "INSERT INTO orchestration_records(kind, key, payload) VALUES (?, NULL, ?)", + [(kind, json.dumps({"request_id": f"request_{index}", "status": "accepted"})) + for index in range(1000) for kind in ("accepted_request", "initial_decision")] + + [("unrelated_legacy", "not valid JSON")], + ) + store.close() + store = _StateStore(database) + traced = [] + try: + assert store._conn.execute("SELECT COUNT(*) FROM orchestration_records").fetchone()[0] == 2001 + assert store._conn.execute( + "SELECT key FROM orchestration_records WHERE kind = 'unrelated_legacy'" + ).fetchone()[0] is None + store._conn.set_trace_callback(traced.append) + cohort = store.load_decision_window(2) + store._conn.set_trace_callback(None) + assert len(cohort["accepted"]) == len(cohort["decisions"]) == 2 + phase_query = next(query for query in traced if query.startswith("SELECT kind, key, payload")) + plan = store._conn.execute("EXPLAIN QUERY PLAN " + phase_query).fetchall() + assert any("orchestration_records_kind_key_seq" in row[3] + and "kind=? AND key=?" in row[3] for row in plan) + assert cohort["window"]["truncated"] is True + finally: + store.close() From 2bd3fcd082499b3dfd152c5d076e2fa7e2e8a742 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 15:47:05 +0900 Subject: [PATCH 34/87] docs: record indexed receipt and identity integration evidence --- .../doctoring/decision_receipt_integration.md | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/doctoring/decision_receipt_integration.md b/docs/doctoring/decision_receipt_integration.md index 965a4d34e..6f0c637e7 100644 --- a/docs/doctoring/decision_receipt_integration.md +++ b/docs/doctoring/decision_receipt_integration.md @@ -79,3 +79,40 @@ all structured/coordinator/direct-provider paths, selection/cancellation/capacit HTTP negatives, complete ingress reconciliation, transactional snapshot export, declared retention window and actual release packaging workflow. No customer KPI gain is established. The Proposed ADR and full rendering inspection remain due. + +## Integrated request identity and indexed cohort checkpoint + +At `fe2db40e`, receipt, persistence and debug/correlation suites passed together: +54 passed in 9.26 s. Normal merges preserve rollback owner #1108 at `129a6650` +and request identity owner #1105 at `b655fe1b`. The handler now owns one +measurement lifetime across repeated slot acquisitions, and passes its trusted +request identity explicitly. Embedding fallback tests observe one admission and +two backend selections; explicit validated-endpoint admission precedes embedding +candidate ordering. Other endpoints remain labeled first_execution_slot and +must not be assumed to include prior selection work. + +`6e4f87aa` adds a shared per-race invocation identity: replicas deduplicate within +one invocation, while a second race in the same request remains a new attempt. +The regression first failed with one attempt instead of two, then passed. +The earlier route-mode-only deduplication is superseded. + +The state store backfills null keys only for valid measurement JSON identities +inside its startup transaction, never overwrites non-null keys, and adds the +(kind, key, seq) index. Its 2,000-measurement-row unit fixture plus malformed +unrelated row retains all 2,001 rows. EXPLAIN for the actual phase query shows +kind/key index search; this is query-plan evidence, not a measured latency gain. +Export reads a bounded shared admission cohort, exposes sequence/truncation +boundaries and unresolved legacy identities, and never independently prunes +phase rows. Historical storage remains append-only pending an archival policy. + +An exact `01ce9035` native wheel and separately built noneditable core package +passed 10 tests in 4.90 s outside the checkout with isolated imports. That proof +does not cover the later merged runtime: rebuild the final candidate again. + +Outstanding semantic gap: structured triage and ranking-evidence embeddings may +invoke providers before the current task-selection hook. Initial provider +dispatch and final task-route decision must be distinguished; no current receipt +is evidence of the headline routing-decision p95. Cache hits, non-generation +operations, auxiliary dispatch, all preselection SSE failures and cancellation +paths still require complete request-boundary coverage and executable evidence. +No protected merge, production publication, or customer KPI gain is claimed. From 8c4edbc605b5bec3188b7ee268e7b28e967536f2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 15:55:59 +0900 Subject: [PATCH 35/87] feat: separate auxiliary provider timing from task decisions --- contextual_orchestrator/decision_receipts.py | 55 ++++++++++++++++- contextual_orchestrator/orchestrator.py | 15 +++-- rust/decision_receipt/src/lib.rs | 18 ++++++ tests/test_decision_receipts.py | 64 ++++++++++++++++++++ 4 files changed, 145 insertions(+), 7 deletions(-) diff --git a/contextual_orchestrator/decision_receipts.py b/contextual_orchestrator/decision_receipts.py index 5db7601e4..d0b7be6b0 100644 --- a/contextual_orchestrator/decision_receipts.py +++ b/contextual_orchestrator/decision_receipts.py @@ -1,6 +1,7 @@ """Persistence boundary for Rust-owned initial-decision measurements.""" from contextvars import ContextVar +from contextlib import contextmanager import logging import threading import uuid @@ -35,6 +36,7 @@ def __init__(self, store, *, policy=None, route_mode="unclassified", request_id= self.selected_agent_ids = [] self.selection_attempt_count = 0 self._race_attempt_ids = set() + self.first_provider_phase = None self._lock = threading.Lock() self._token = _CURRENT_DECISION.set(self) try: @@ -62,7 +64,9 @@ def snapshot(self): "selection_attempt_count": self.selection_attempt_count, "selection_elapsed_ns": self.receipt.selection_elapsed_ns, "durable_ack_elapsed_ns": self.receipt.durable_ack_elapsed_ns, - "metric_scope": "initial_provider_dispatch", + "first_provider_elapsed_ns": self.receipt.first_provider_elapsed_ns, + "first_provider_phase": self.first_provider_phase, + "metric_scope": "initial_task_route_decision", } def select(self, agent_ids, route_mode, *, attempt_id=None): @@ -96,6 +100,44 @@ def select(self, agent_ids, route_mode, *, attempt_id=None): _LOGGER.warning("Initial decision measurement write failed error_type=%s", type(exc).__name__) return self.receipt.record_durable_ack() + self._record_provider_locked(agent_ids, "task_execution") + + def _record_provider_locked(self, agent_ids, phase): + """Keep a first-provider diagnostic independent of the task-route clock.""" + if self.first_provider_phase is not None: + return + self.receipt.record_provider_dispatch() + self.first_provider_phase = phase + try: + self.store.save("provider_dispatch", self.request_id, { + **self.snapshot(), "provider_agent_ids": list(agent_ids), + }, durable=True) + except Exception as exc: + _LOGGER.warning("Provider diagnostic write failed error_type=%s", type(exc).__name__) + + @contextmanager + def auxiliary_call(self, agent_ids, phase): + """Retain auxiliary work without subtracting it from the task-route interval.""" + with self._lock: + self._record_provider_locked(agent_ids, phase) + started = self.receipt.current_elapsed_ns() + outcome = "completed" + try: + yield + except BaseException: + outcome = "failed" + raise + finally: + with self._lock: + finished = self.receipt.current_elapsed_ns() + try: + self.store.save("auxiliary_dispatch", self.request_id, { + "request_id": self.request_id, "phase": phase, + "provider_agent_ids": list(agent_ids), "outcome": outcome, + "started_elapsed_ns": started, "finished_elapsed_ns": finished, + }, durable=True) + except Exception as exc: + _LOGGER.warning("Auxiliary diagnostic write failed error_type=%s", type(exc).__name__) def close(self, reason="unfinished"): """Persist the post-commit measurement separately; never claim its own ack.""" @@ -119,6 +161,17 @@ def record_initial_selection(agent_ids, route_mode="unclassified", *, attempt_id measurement.select(agent_ids, route_mode, attempt_id=attempt_id) +@contextmanager +def observe_auxiliary_dispatch(agent_ids, phase): + """Use the request's native clock for an actual auxiliary provider call.""" + measurement = _CURRENT_DECISION.get() + if measurement is None: + yield + else: + with measurement.auxiliary_call(agent_ids, phase): + yield + + def export_decision_receipts(store, limit=256): """Join retained admissions without dropping missing final acknowledgements. diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 8ea4df932..139b40f8e 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -6,7 +6,7 @@ from collections.abc import Iterable, Mapping from contextlib import contextmanager, nullcontext from contextvars import ContextVar, copy_context -from .decision_receipts import record_initial_selection +from .decision_receipts import observe_auxiliary_dispatch, record_initial_selection from concurrent.futures import ThreadPoolExecutor import copy import hashlib @@ -7288,7 +7288,8 @@ def _embed_cached(self, text: str) -> list[float] | None: if embedding_member is None: return None try: - vectors = self.client.embed(self._agent(embedding_member), [text]) + with observe_auxiliary_dispatch([embedding_member], "routing_evidence_embedding"): + vectors = self.client.embed(self._agent(embedding_member), [text]) except Exception: # noqa: BLE001 - similarity is best-effort evidence return None vector = vectors[0] if vectors else None @@ -7315,9 +7316,10 @@ def _descriptor_vector_cached(self, agent: ModelAgent) -> list[float] | None: if embedding_member is None: return None try: - vectors = self.client.embed( - self._agent(embedding_member), [self._agent_descriptor_text(agent)] - ) + with observe_auxiliary_dispatch([embedding_member], "routing_evidence_embedding"): + vectors = self.client.embed( + self._agent(embedding_member), [self._agent_descriptor_text(agent)] + ) except Exception: # noqa: BLE001 - similarity is best-effort evidence return None vector = vectors[0] if vectors else None @@ -7403,7 +7405,8 @@ def _compute_triage_verdict(self, text: str) -> bool: {"role": "user", "content": text}, ] try: - reply = self.client.chat(triage_agent, messages, temperature=0.0) + with observe_auxiliary_dispatch([triage_agent.id], "structured_triage"): + reply = self.client.chat(triage_agent, messages, temperature=0.0) return _parse_triage_reply(reply) except Exception: # noqa: BLE001 - fail closed toward verified orchestration return True diff --git a/rust/decision_receipt/src/lib.rs b/rust/decision_receipt/src/lib.rs index 75f015170..49b5dbc73 100644 --- a/rust/decision_receipt/src/lib.rs +++ b/rust/decision_receipt/src/lib.rs @@ -13,6 +13,8 @@ pub struct DecisionReceipt { selection_elapsed_ns: Option, #[pyo3(get)] durable_ack_elapsed_ns: Option, + #[pyo3(get)] + first_provider_elapsed_ns: Option, } impl DecisionReceipt { @@ -35,7 +37,23 @@ impl DecisionReceipt { status: "accepted".into(), selection_elapsed_ns: None, durable_ack_elapsed_ns: None, + first_provider_elapsed_ns: None, + } + } + + /// Mark the first provider-ready boundary without advancing task selection. + fn record_provider_dispatch(&mut self) -> PyResult { + if let Some(elapsed) = self.first_provider_elapsed_ns { + return Ok(elapsed); } + let elapsed = self.elapsed_ns()?; + self.first_provider_elapsed_ns = Some(elapsed); + Ok(elapsed) + } + + /// Timestamp an auxiliary phase in the same acceptance clock domain. + fn current_elapsed_ns(&self) -> PyResult { + self.elapsed_ns() } /// Record the initial selection without allowing later attempts to replace it. diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py index a09e2bc39..ee35a70c6 100644 --- a/tests/test_decision_receipts.py +++ b/tests/test_decision_receipts.py @@ -372,3 +372,67 @@ def test_legacy_identity_backfill_and_indexed_window(tmp_path): assert cohort["window"]["truncated"] is True finally: store.close() + + +def test_http_cold_and_cached_triage_keep_task_ack_after_auxiliary_work(tmp_path, monkeypatch): + """Cold triage is diagnostic only; warm triage still measures the task decision.""" + from contextual_orchestrator.decision_receipts import _CURRENT_DECISION + + orchestrator = TaskOrchestrator( + [ModelAgent("worker_one", "mock/worker", tags=("writing",))], + state_db=tmp_path / "state.db", + ) + original_chat = orchestrator.client.chat + auxiliary_ready = threading.Event() + auxiliary_release = threading.Event() + auxiliary_snapshots = [] + task_snapshots = [] + + def controlled_chat(agent, messages, **kwargs): + measurement = _CURRENT_DECISION.get() + if messages[0]["content"] == orchestrator.TRIAGE_SYSTEM_PROMPT: + auxiliary_snapshots.append(measurement.snapshot()) + auxiliary_ready.set() + assert auxiliary_release.wait(10) + return '{"workflow_required": false}' + task_snapshots.append(measurement.snapshot()) + return original_chat(agent, messages, **kwargs) + + monkeypatch.setattr(orchestrator.client, "chat", controlled_chat) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + for request_index in range(2): + connection = http.client.HTTPConnection(*server.server_address) + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "auto", + "messages": [{"role": "user", "content": "same question"}], + }), {"Content-Type": "application/json", "Authorization": "Bearer test-token", + "x-cache-bypass": "true"}) + if request_index == 0: + assert auxiliary_ready.wait(10) + snapshot = auxiliary_snapshots[0] + assert snapshot["durable_ack_elapsed_ns"] is None + assert snapshot.get("first_provider_elapsed_ns") is not None + assert snapshot.get("first_provider_phase") == "structured_triage" + auxiliary_release.set() + response = connection.getresponse() + response.read() + assert response.status == 200 + connection.close() + server.shutdown() + server.server_close() + assert len(auxiliary_snapshots) == 1 + assert len(task_snapshots) == 2 + cold, warm = task_snapshots + assert cold["first_provider_elapsed_ns"] < cold["selection_elapsed_ns"] <= cold["durable_ack_elapsed_ns"] + assert warm["first_provider_phase"] != "structured_triage" + assert warm["durable_ack_elapsed_ns"] is not None + finally: + auxiliary_release.set() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() From 0e6cd58ea16e2206f9855867ecd51c28c1745a3e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 15:58:10 +0900 Subject: [PATCH 36/87] fix: export bounded auxiliary timing diagnostics --- contextual_orchestrator/decision_receipts.py | 9 +++++++++ contextual_orchestrator/orchestrator.py | 13 +++++++++++++ tests/test_decision_receipts.py | 10 ++++++++++ 3 files changed, 32 insertions(+) diff --git a/contextual_orchestrator/decision_receipts.py b/contextual_orchestrator/decision_receipts.py index d0b7be6b0..5dc3f33a6 100644 --- a/contextual_orchestrator/decision_receipts.py +++ b/contextual_orchestrator/decision_receipts.py @@ -66,6 +66,7 @@ def snapshot(self): "durable_ack_elapsed_ns": self.receipt.durable_ack_elapsed_ns, "first_provider_elapsed_ns": self.receipt.first_provider_elapsed_ns, "first_provider_phase": self.first_provider_phase, + "first_provider_boundary": "provider_ready_before_diagnostic_commit", "metric_scope": "initial_task_route_decision", } @@ -183,6 +184,9 @@ def export_decision_receipts(store, limit=256): accepted = cohort["accepted"] decisions = {row["request_id"]: row for row in cohort["decisions"]} receipts = {row["request_id"]: row for row in cohort["receipts"]} + diagnostics = {} + for diagnostic in cohort["diagnostics"]: + diagnostics.setdefault(diagnostic["request_id"], []).append(diagnostic) observations = [] for admission in accepted: request_id = admission["request_id"] @@ -193,6 +197,11 @@ def export_decision_receipts(store, limit=256): row["status"] = "acknowledgement_unobserved" if request_id in receipts: row.update(receipts[request_id]) + request_diagnostics = diagnostics.get(request_id, []) + row["auxiliary_dispatches"] = [entry for entry in request_diagnostics + if entry["record_kind"] == "auxiliary_dispatch"] + row["provider_diagnostics"] = [entry for entry in request_diagnostics + if entry["record_kind"] == "provider_dispatch"] observations.append(row) return { "schema_version": 1, diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 139b40f8e..029eadddf 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -3761,6 +3761,7 @@ def load_decision_window(self, limit: int = 256) -> dict[str, Any]: raise ValueError("measurement admission identity mismatch") request_ids = [row["request_id"] for row in accepted] phases = [] + diagnostics = [] if request_ids: placeholders = ",".join("?" for _ in request_ids) phases = self._conn.execute( @@ -3770,6 +3771,16 @@ def load_decision_window(self, limit: int = 256) -> dict[str, Any]: "ORDER BY seq DESC LIMIT ?", (*request_ids, 2 * limit + 1), ).fetchall() + diagnostics = self._conn.execute( + "SELECT kind, key, payload FROM orchestration_records " + "WHERE kind IN ('provider_dispatch', 'auxiliary_dispatch') " + "AND key IN (" + placeholders + ") ORDER BY seq DESC LIMIT ?", + (*request_ids, 8 * limit + 1), + ).fetchall() + diagnostic_truncated = len(diagnostics) > 8 * limit + diagnostics = list(reversed(diagnostics[:8 * limit])) + if any(key != json.loads(payload).get("request_id") for _, key, payload in diagnostics): + raise ValueError("measurement diagnostic identity mismatch") phase_truncated = len(phases) > 2 * limit phases = list(reversed(phases[:2 * limit])) if any(key != json.loads(payload).get("request_id") for _, key, payload in phases): @@ -3782,7 +3793,9 @@ def load_decision_window(self, limit: int = 256) -> dict[str, Any]: "accepted": accepted, "decisions": [json.loads(payload) for kind, _, payload in phases if kind == "initial_decision"], "receipts": [json.loads(payload) for kind, _, payload in phases if kind == "decision_receipt"], + "diagnostics": [{"record_kind": kind, **json.loads(payload)} for kind, _, payload in diagnostics], "window": {"limit": limit, "truncated": truncated, "phase_truncated": phase_truncated, + "diagnostic_limit": 8 * limit, "diagnostic_truncated": diagnostic_truncated, "unresolved_legacy_identity": unresolved_legacy, "first_admission_seq": admissions[0][0] if admissions else None, "last_admission_seq": admissions[-1][0] if admissions else None}, diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py index ee35a70c6..7b4b1109b 100644 --- a/tests/test_decision_receipts.py +++ b/tests/test_decision_receipts.py @@ -430,6 +430,16 @@ def controlled_chat(agent, messages, **kwargs): assert cold["first_provider_elapsed_ns"] < cold["selection_elapsed_ns"] <= cold["durable_ack_elapsed_ns"] assert warm["first_provider_phase"] != "structured_triage" assert warm["durable_ack_elapsed_ns"] is not None + from contextual_orchestrator.decision_receipts import export_decision_receipts + exported = export_decision_receipts(orchestrator._store) + cold_export, warm_export = exported["observations"] + assert cold_export["first_provider_boundary"] == "provider_ready_before_diagnostic_commit" + assert len(cold_export["auxiliary_dispatches"]) == 1 + auxiliary = cold_export["auxiliary_dispatches"][0] + assert auxiliary["phase"] == "structured_triage" + assert auxiliary["outcome"] == "completed" + assert auxiliary["finished_elapsed_ns"] <= cold_export["selection_elapsed_ns"] + assert warm_export["auxiliary_dispatches"] == [] finally: auxiliary_release.set() server.shutdown() From 21d1e88b0129b2e1be662306a0f19385eee995e0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 16:01:22 +0900 Subject: [PATCH 37/87] fix: retain cache-hit admissions without provider timings --- contextual_orchestrator/decision_receipts.py | 8 +++++ contextual_orchestrator/orchestrator.py | 3 +- rust/decision_receipt/src/lib.rs | 9 +++++ tests/test_decision_receipts.py | 38 ++++++++++++++++++++ 4 files changed, 57 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/decision_receipts.py b/contextual_orchestrator/decision_receipts.py index 5dc3f33a6..dac23c393 100644 --- a/contextual_orchestrator/decision_receipts.py +++ b/contextual_orchestrator/decision_receipts.py @@ -162,6 +162,14 @@ def record_initial_selection(agent_ids, route_mode="unclassified", *, attempt_id measurement.select(agent_ids, route_mode, attempt_id=attempt_id) +def record_answer_cache_hit(): + """Retain answer-cache admissions without attributing cached provider timings.""" + measurement = _CURRENT_DECISION.get() + if measurement is not None: + with measurement._lock: + measurement.receipt.record_cache_hit() + + @contextmanager def observe_auxiliary_dispatch(agent_ids, phase): """Use the request's native clock for an actual auxiliary provider call.""" diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 029eadddf..23514ff4e 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -6,7 +6,7 @@ from collections.abc import Iterable, Mapping from contextlib import contextmanager, nullcontext from contextvars import ContextVar, copy_context -from .decision_receipts import observe_auxiliary_dispatch, record_initial_selection +from .decision_receipts import observe_auxiliary_dispatch, record_answer_cache_hit, record_initial_selection from concurrent.futures import ThreadPoolExecutor import copy import hashlib @@ -5278,6 +5278,7 @@ def complete( ): result = copy.deepcopy(dict(cached)) result["cache_status"] = "hit" + record_answer_cache_hit() return result result = self._dispatch(messages, mode, model_name) try: diff --git a/rust/decision_receipt/src/lib.rs b/rust/decision_receipt/src/lib.rs index 49b5dbc73..5630c1dfc 100644 --- a/rust/decision_receipt/src/lib.rs +++ b/rust/decision_receipt/src/lib.rs @@ -56,6 +56,15 @@ impl DecisionReceipt { self.elapsed_ns() } + /// Complete answer reuse without inventing a provider selection or acknowledgement. + fn record_cache_hit(&mut self) -> PyResult<()> { + if self.status != "accepted" { + return Err(PyValueError::new_err("cache hit requires accepted state")); + } + self.status = "cache_hit".into(); + Ok(()) + } + /// Record the initial selection without allowing later attempts to replace it. fn record_selection(&mut self) -> PyResult<()> { if self.status != "accepted" { diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py index 7b4b1109b..bb4fee7ad 100644 --- a/tests/test_decision_receipts.py +++ b/tests/test_decision_receipts.py @@ -10,6 +10,44 @@ from contextual_orchestrator.server import build_server, SecurityConfig +def test_http_answer_cache_keeps_admission_without_provider_duration(tmp_path): + """Answer reuse has its own terminal outcome, never a copied provider timing.""" + from contextual_orchestrator.decision_receipts import export_decision_receipts + orchestrator = TaskOrchestrator( + [ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db", cache_ttl=60, + ) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + for _ in range(2): + connection = http.client.HTTPConnection(*server.server_address) + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "route", + "messages": [{"role": "user", "content": "same answer"}], + }), {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + response.read() + assert response.status == 200 + connection.close() + server.shutdown() + observations = export_decision_receipts(orchestrator._store)["observations"] + assert len(observations) == 2 + assert observations[0]["status"] == "acknowledged" + assert observations[1]["status"] == "cache_hit" + assert observations[1]["selection_elapsed_ns"] is None + assert observations[1]["durable_ack_elapsed_ns"] is None + assert observations[1]["first_provider_elapsed_ns"] is None + assert len(orchestrator._store.load("provider_dispatch")) == 1 + finally: + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + @pytest.mark.parametrize("endpoint,stream", [ ("/v1/chat/completions", False), ("/v1/chat/completions", True), ("/v1/responses", True), From 3478b90fb3284e1c640638fd9d322e818bd41239 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 16:02:38 +0900 Subject: [PATCH 38/87] test: cover diagnostic cohort export cap boundaries --- tests/test_decision_receipts.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py index bb4fee7ad..6eb1debce 100644 --- a/tests/test_decision_receipts.py +++ b/tests/test_decision_receipts.py @@ -10,6 +10,34 @@ from contextual_orchestrator.server import build_server, SecurityConfig +@pytest.mark.parametrize("diagnostic_count", [16, 17]) +def test_diagnostic_window_cap_preserves_shared_admissions(tmp_path, diagnostic_count): + """One noisy request cannot remove another admitted request from the export.""" + from contextual_orchestrator.orchestrator import _StateStore + from contextual_orchestrator.decision_receipts import export_decision_receipts + store = _StateStore(tmp_path / "state.db") + try: + for request_id in ("outside_cohort", "quiet_request", "noisy_request"): + store.save("accepted_request", None, {"request_id": request_id}, durable=True) + store.save("auxiliary_dispatch", "outside_cohort", + {"request_id": "outside_cohort"}, durable=True) + for index in range(diagnostic_count): + store.save("auxiliary_dispatch", "noisy_request", + {"request_id": "noisy_request", "test_index": index}, durable=True) + exported = export_decision_receipts(store, limit=2) + assert exported["window"]["diagnostic_limit"] == 16 + assert exported["window"]["diagnostic_truncated"] is (diagnostic_count > 16) + quiet, noisy = exported["observations"] + assert quiet["request_id"] == "quiet_request" + assert quiet["status"] == "unfinished" + assert quiet["auxiliary_dispatches"] == [] + assert noisy["request_id"] == "noisy_request" + assert len(noisy["auxiliary_dispatches"]) == 16 + assert all(row["request_id"] == "noisy_request" for row in noisy["auxiliary_dispatches"]) + finally: + store.close() + + def test_http_answer_cache_keeps_admission_without_provider_duration(tmp_path): """Answer reuse has its own terminal outcome, never a copied provider timing.""" from contextual_orchestrator.decision_receipts import export_decision_receipts From 6ddddb8f6303476a43d13e22a489eeadc4ad5fe3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 16:04:49 +0900 Subject: [PATCH 39/87] fix: keep generated planning inside task decision interval --- contextual_orchestrator/orchestrator.py | 12 ++++----- tests/test_decision_receipts.py | 36 +++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 23514ff4e..5cae0e049 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -6866,12 +6866,12 @@ def _plan_generated(self, task: str) -> list[WorkflowStep]: {"role": "user", "content": task}, ] effort_profile = self._role_effort_profile("planner") - record_initial_selection([planner.id], "generated_planner") - raw = ( - self.client.chat(planner, planner_messages, effort_profile=effort_profile) - if effort_profile is not None - else self.client.chat(planner, planner_messages) - ) + with observe_auxiliary_dispatch([planner.id], "generated_planner"): + raw = ( + self.client.chat(planner, planner_messages, effort_profile=effort_profile) + if effort_profile is not None + else self.client.chat(planner, planner_messages) + ) return self._parse_workflow_plan(raw) def _parse_workflow_plan(self, raw: str) -> list[WorkflowStep]: diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py index 6eb1debce..1df10caf2 100644 --- a/tests/test_decision_receipts.py +++ b/tests/test_decision_receipts.py @@ -10,6 +10,29 @@ from contextual_orchestrator.server import build_server, SecurityConfig +def test_generated_planner_is_auxiliary_until_worker_selection(tmp_path, monkeypatch): + """A planning provider does not freeze the task-execution route interval.""" + from contextual_orchestrator.decision_receipts import DecisionMeasurement + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + measurement = DecisionMeasurement(orchestrator._store) + def plan_reply(agent, messages, **kwargs): + assert measurement.snapshot()["durable_ack_elapsed_ns"] is None + return json.dumps({"steps": [{"id": 0, "role": "worker", "agent_id": agent.id, + "subtask": "work", "access": []}, + {"id": 1, "role": "synthesizer", "agent_id": agent.id, + "subtask": "answer", "access": [0]}]}) + monkeypatch.setattr(orchestrator.client, "chat", plan_reply) + try: + orchestrator._plan_generated("answer") + assert measurement.snapshot()["status"] == "accepted" + assert measurement.snapshot()["first_provider_phase"] == "generated_planner" + assert not orchestrator._store.load("initial_decision") + finally: + measurement.close() + orchestrator.close() + + @pytest.mark.parametrize("diagnostic_count", [16, 17]) def test_diagnostic_window_cap_preserves_shared_admissions(tmp_path, diagnostic_count): """One noisy request cannot remove another admitted request from the export.""" @@ -38,17 +61,24 @@ def test_diagnostic_window_cap_preserves_shared_admissions(tmp_path, diagnostic_ store.close() -def test_http_answer_cache_keeps_admission_without_provider_duration(tmp_path): +def test_http_answer_cache_keeps_admission_without_provider_duration(tmp_path, monkeypatch): """Answer reuse has its own terminal outcome, never a copied provider timing.""" from contextual_orchestrator.decision_receipts import export_decision_receipts orchestrator = TaskOrchestrator( [ModelAgent("worker_one", "mock/worker")], state_db=tmp_path / "state.db", cache_ttl=60, ) + original_chat = orchestrator.client.chat + provider_calls = [] + def counted_chat(*args, **kwargs): + provider_calls.append(True) + return original_chat(*args, **kwargs) + monkeypatch.setattr(orchestrator.client, "chat", counted_chat) server = build_server(orchestrator, port=0, decision_receipts=True, security=SecurityConfig(auth_token="test-token")) worker = threading.Thread(target=server.serve_forever, daemon=True) worker.start() + answers = [] try: for _ in range(2): connection = http.client.HTTPConnection(*server.server_address) @@ -57,7 +87,7 @@ def test_http_answer_cache_keeps_admission_without_provider_duration(tmp_path): "messages": [{"role": "user", "content": "same answer"}], }), {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) response = connection.getresponse() - response.read() + answers.append(json.loads(response.read())["choices"][0]["message"]["content"]) assert response.status == 200 connection.close() server.shutdown() @@ -69,6 +99,8 @@ def test_http_answer_cache_keeps_admission_without_provider_duration(tmp_path): assert observations[1]["durable_ack_elapsed_ns"] is None assert observations[1]["first_provider_elapsed_ns"] is None assert len(orchestrator._store.load("provider_dispatch")) == 1 + assert len(provider_calls) == 1 + assert answers[0] == answers[1] finally: server.shutdown() worker.join() From 9707a5e14336c518abd417467a82f5528594a7f1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 16:09:34 +0900 Subject: [PATCH 40/87] fix: distinguish post-decision evidence embedding observations --- contextual_orchestrator/decision_receipts.py | 2 + tests/test_decision_receipts.py | 62 ++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/contextual_orchestrator/decision_receipts.py b/contextual_orchestrator/decision_receipts.py index dac23c393..618106392 100644 --- a/contextual_orchestrator/decision_receipts.py +++ b/contextual_orchestrator/decision_receipts.py @@ -120,6 +120,8 @@ def _record_provider_locked(self, agent_ids, phase): def auxiliary_call(self, agent_ids, phase): """Retain auxiliary work without subtracting it from the task-route interval.""" with self._lock: + if phase == "routing_evidence_embedding" and self.receipt.selection_elapsed_ns is not None: + phase = "post_decision_evidence_embedding" self._record_provider_locked(agent_ids, phase) started = self.receipt.current_elapsed_ns() outcome = "completed" diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py index 1df10caf2..954d403d3 100644 --- a/tests/test_decision_receipts.py +++ b/tests/test_decision_receipts.py @@ -10,6 +10,62 @@ from contextual_orchestrator.server import build_server, SecurityConfig +def test_http_evidence_embedding_cold_and_warm_keep_task_interval(tmp_path, monkeypatch): + """Routing evidence calls occur only cold and remain before task acknowledgement.""" + from contextual_orchestrator.decision_receipts import _CURRENT_DECISION, export_decision_receipts + orchestrator = TaskOrchestrator([ + ModelAgent("worker_one", "mock/worker", tags=("writing",)), + ModelAgent("embedding_one", "mock-embedding", tags=("embedding",)), + ], state_db=tmp_path / "state.db") + original_embed = orchestrator.client.embed + embedding_snapshots = [] + embedding_inputs = [] + def observed_embed(*args, **kwargs): + embedding_inputs.extend(args[1]) + snapshot = _CURRENT_DECISION.get().snapshot() + embedding_snapshots.append(snapshot) + return original_embed(*args, **kwargs) + monkeypatch.setattr(orchestrator.client, "embed", observed_embed) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + counts = [] + for _ in range(2): + connection = http.client.HTTPConnection(*server.server_address) + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "route", + "messages": [{"role": "user", "content": "same evidence request"}], + }), {"Content-Type": "application/json", "Authorization": "Bearer test-token", + "x-cache-bypass": "true"}) + response = connection.getresponse() + response.read() + assert response.status == 200 + connection.close() + counts.append(len(embedding_snapshots)) + server.shutdown() + assert counts[0] > 0 + assert counts[1] == counts[0] + assert embedding_inputs.count("same evidence request") == 1 + assert all(snapshot["first_provider_phase"] == "routing_evidence_embedding" + for snapshot in embedding_snapshots) + cold, warm = export_decision_receipts(orchestrator._store)["observations"] + assert cold["auxiliary_dispatches"] + assert all(row["finished_elapsed_ns"] <= cold["selection_elapsed_ns"] + for row in cold["auxiliary_dispatches"] if row["phase"] == "routing_evidence_embedding") + assert any(row["phase"] == "post_decision_evidence_embedding" + for row in cold["auxiliary_dispatches"]) + assert warm["auxiliary_dispatches"] == [] + assert cold["durable_ack_elapsed_ns"] is not None + assert warm["durable_ack_elapsed_ns"] is not None + finally: + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + def test_generated_planner_is_auxiliary_until_worker_selection(tmp_path, monkeypatch): """A planning provider does not freeze the task-execution route interval.""" from contextual_orchestrator.decision_receipts import DecisionMeasurement @@ -213,10 +269,13 @@ def test_write_failure_has_no_ack_and_does_not_log_exception_contents(caplog): """Failed commits cannot become successful timing samples or leak detail.""" from contextual_orchestrator.decision_receipts import DecisionMeasurement + auxiliary_records = [] class FailedStore: def save(self, *args, **kwargs): if args[0] == "initial_decision": raise RuntimeError("secret-canary-never-log") + if args[0] == "auxiliary_dispatch": + auxiliary_records.append(args[2]) measurement = DecisionMeasurement(FailedStore()) try: @@ -226,6 +285,9 @@ def save(self, *args, **kwargs): assert snapshot["durable_ack_elapsed_ns"] is None snapshot["selected_agent_ids"].append("untrusted_mutation") assert measurement.snapshot()["selected_agent_ids"] == ["worker_one"] + with measurement.auxiliary_call(["embedding_one"], "routing_evidence_embedding"): + pass + assert auxiliary_records[0]["phase"] == "post_decision_evidence_embedding" finally: measurement.close() assert "secret-canary-never-log" not in caplog.text From bbe7eae1a24a95e17b5933ca75cc6b2598f896e4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 16:13:55 +0900 Subject: [PATCH 41/87] build: verify locked native receipts before CI acceptance --- .github/workflows/security.yml | 17 +++++++-- .../doctoring/decision_receipt_integration.md | 37 +++++++++++++++++++ mise.toml | 2 + pyproject.toml | 1 + rust-toolchain.toml | 3 ++ rust/decision_receipt/pyproject.toml | 2 +- scripts/verify_decision_wheel_manifest.py | 23 ++++++++++++ tests/test_decision_wheel_manifest.py | 31 ++++++++++++++++ tests/test_repository_security_metadata.py | 5 ++- uv.lock | 28 ++++++++++++++ 10 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 mise.toml create mode 100644 rust-toolchain.toml create mode 100644 scripts/verify_decision_wheel_manifest.py create mode 100644 tests/test_decision_wheel_manifest.py diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 428cbfaf1..7e9864390 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -44,8 +44,16 @@ jobs: with: version: "0.12.5" + - name: Build locked native decision measurement + run: | + set -euo pipefail + rustup show active-toolchain + uv sync --locked --extra api --extra db --extra queue --group dev --group native-build + uv run --no-sync maturin develop --locked --release --features pyo3/extension-module --manifest-path rust/decision_receipt/Cargo.toml + uv run --no-sync maturin build --locked --release --features pyo3/extension-module --manifest-path rust/decision_receipt/Cargo.toml --out "$RUNNER_TEMP/decision-wheels" + - name: Run full test suite - run: uv run --locked --extra api --extra db --extra queue --group dev python -m pytest -q + run: uv run --no-sync python -m pytest -q - name: Install hash-locked quality tools run: | @@ -74,12 +82,15 @@ jobs: set -euo pipefail rm -rf dist "$RUNNER_TEMP/nim-wheel-site" python -m pip wheel --no-deps --no-build-isolation . --wheel-dir dist + python scripts/verify_decision_wheel_manifest.py dist/contextual_orchestrator-*.whl "$RUNNER_TEMP"/decision-wheels/*.whl python -m pip install --no-deps \ --target "$RUNNER_TEMP/nim-wheel-site" \ - dist/contextual_orchestrator-*.whl + dist/contextual_orchestrator-*.whl "$RUNNER_TEMP"/decision-wheels/*.whl cd "$RUNNER_TEMP" PYTHONPATH="$RUNNER_TEMP/nim-wheel-site" \ - python -c "import contextual_orchestrator; import contextual_orchestrator.nim_benchmark" + python -c "import pathlib, contextual_orchestrator as core, contextual_orchestrator.nim_benchmark, contextual_orchestrator._decision_receipt as native; root = pathlib.Path('nim-wheel-site').resolve(); assert pathlib.Path(core.__file__).is_relative_to(root); assert pathlib.Path(native.__file__).is_relative_to(root)" + PYTHONPATH="$RUNNER_TEMP/nim-wheel-site" \ + python -m pytest --noconftest --import-mode=importlib "$GITHUB_WORKSPACE/tests/test_decision_receipts.py" -q fuzz: name: Property and coverage-guided fuzzing diff --git a/docs/doctoring/decision_receipt_integration.md b/docs/doctoring/decision_receipt_integration.md index 6f0c637e7..7db2f9009 100644 --- a/docs/doctoring/decision_receipt_integration.md +++ b/docs/doctoring/decision_receipt_integration.md @@ -116,3 +116,40 @@ is evidence of the headline routing-decision p95. Cache hits, non-generation operations, auxiliary dispatch, all preselection SSE failures and cancellation paths still require complete request-boundary coverage and executable evidence. No protected merge, production publication, or customer KPI gain is claimed. +# Auxiliary and package acceptance checkpoint (2026-09-09) + +At `9707a5e1`, focused receipt/persistence tests passed 41 cases in 8.43 s. +The Rust clock now separates provider-ready diagnostics from initial task-route +acknowledgement. Structured triage, generated planning, and pre-selection evidence +embedding remain inside the task-route interval. Evidence embedding after native +selection is labelled `post_decision_evidence_embedding`, including write-failed +selection; it is not subtracted or presented as preceding routing work. Answer +cache reuse has a distinct terminal outcome with absent provider timing values. +The bounded admission cohort exports auxiliary records with an explicit diagnostic +cap and truncation indicator. Historical retention remains unresolved. + +The initial embedding test asserted inside a best-effort transport callback. +That assertion was swallowed by the existing best-effort path and prevented cache +fill, causing a test-induced warm retry. The corrected spy only collects values; +assertions run after the response. No descriptor invalidation was established. + +Native packaging was rebuilt from the repository root with +`maturin build --locked --release --manifest-path rust/decision_receipt/Cargo.toml +--interpreter .venv/bin/python --out /tmp/co-decision-receipt-wheels`. +Rust 1.97.1, maturin 1.15.0, Python 3.14.6, macOS arm64 produced the native wheel +SHA-256 `e0bf63d790256c6d4eba8598c131d63188a994c899df5124bd9eadf2cc39c568`. +Its five ZIP entries contain only the receipt extension and distribution metadata; +the independently built core wheel has no overlapping files. A separate +noneditable installation outside the checkout passed all 20 receipt tests in +11.83 s using `python -I`, with both import origins under that environment's +site-packages. This proves that local artifact matrix only, not Linux/Python 3.12 +hosted acceptance or a released package. The namespace was retained because the +suspected core-file collision was not observed in either inspected native wheel. + +CI now installs locked native build tooling, builds the extension before the full +suite, validates disjoint wheel ownership before installation, and exercises the +wheel-installed HTTP receipt tests outside the checkout. The existing benchmark +import smoke is preserved. These workflow changes remain pending hosted evidence. +Full endpoint admission validation, cancellation/error-path coverage, deployed +retention/reconciliation, and customer accuracy/latency measurements remain open; +none of the unit or mock-provider evidence establishes a customer KPI gain. diff --git a/mise.toml b/mise.toml new file mode 100644 index 000000000..19305ad64 --- /dev/null +++ b/mise.toml @@ -0,0 +1,2 @@ +[tools] +rust = "1.97.1" diff --git a/pyproject.toml b/pyproject.toml index de9f90de1..fcce4241c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ queue = [ ] [dependency-groups] +native-build = ["maturin==1.15.0"] dev = [ "hypothesis>=6.100", "pytest>=8.0", diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 000000000..725551148 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.97.1" +profile = "minimal" diff --git a/rust/decision_receipt/pyproject.toml b/rust/decision_receipt/pyproject.toml index 2e72d0a41..6c498c112 100644 --- a/rust/decision_receipt/pyproject.toml +++ b/rust/decision_receipt/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=1.8,<2"] +requires = ["maturin==1.15.0"] build-backend = "maturin" [project] diff --git a/scripts/verify_decision_wheel_manifest.py b/scripts/verify_decision_wheel_manifest.py new file mode 100644 index 000000000..461dd6379 --- /dev/null +++ b/scripts/verify_decision_wheel_manifest.py @@ -0,0 +1,23 @@ +"""Reject native/core wheel ownership overlap before isolated installation.""" + +import sys +import zipfile + + +def verify_wheels(core_path, native_path): + """Require a native-only extension distribution disjoint from the core wheel.""" + with zipfile.ZipFile(core_path) as core_archive, zipfile.ZipFile(native_path) as native_archive: + core_files = set(core_archive.namelist()) + native_files = set(native_archive.namelist()) + if core_files & native_files: + raise ValueError("core and native wheel files overlap") + payload = {name for name in native_files if ".dist-info/" not in name} + if len(payload) != 1 or not all( + name.startswith("contextual_orchestrator/_decision_receipt.") + and name.endswith((".so", ".pyd")) for name in payload + ): + raise ValueError("native wheel must contain only the receipt extension") + + +if __name__ == "__main__": + verify_wheels(*sys.argv[1:]) diff --git a/tests/test_decision_wheel_manifest.py b/tests/test_decision_wheel_manifest.py new file mode 100644 index 000000000..5921ce11e --- /dev/null +++ b/tests/test_decision_wheel_manifest.py @@ -0,0 +1,31 @@ +"""Wheel ownership is validated before installation can mask package omissions.""" + +from pathlib import Path +import runpy +import zipfile + +import pytest + +verify_wheels = runpy.run_path( + str(Path(__file__).resolve().parents[1] / "scripts" / "verify_decision_wheel_manifest.py") +)["verify_wheels"] + + +@pytest.mark.parametrize("native_payload,expected_error", [ + ("contextual_orchestrator/_decision_receipt.abi3.so", None), + ("contextual_orchestrator/__init__.py", "overlap"), + ("unexpected_package/module.py", "only the receipt extension"), +]) +def test_native_wheel_ownership(tmp_path, native_payload, expected_error): + """Reject overlapping or unexpected native payloads; accept disjoint bindings.""" + core_path, native_path = tmp_path / "core.whl", tmp_path / "native.whl" + with zipfile.ZipFile(core_path, "w") as archive: + archive.writestr("contextual_orchestrator/__init__.py", "") + with zipfile.ZipFile(native_path, "w") as archive: + archive.writestr(native_payload, "unit fixture") + archive.writestr("native.dist-info/METADATA", "") + if expected_error: + with pytest.raises(ValueError, match=expected_error): + verify_wheels(core_path, native_path) + else: + verify_wheels(core_path, native_path) diff --git a/tests/test_repository_security_metadata.py b/tests/test_repository_security_metadata.py index f2d82ccf5..090934bc7 100644 --- a/tests/test_repository_security_metadata.py +++ b/tests/test_repository_security_metadata.py @@ -216,7 +216,10 @@ def test_unit_workflow_uses_the_project_lock_for_git_runtime_dependencies(): assert re.search(r"@[0-9a-f]{40}(?:\s+#|$)", setup_uv_line) assert "# v" in setup_uv_line assert 'version: "0.12.5"' in workflow_text - assert "uv run --locked --extra api --extra db --extra queue --group dev python -m pytest -q" in workflow_text + locked_sync = "uv sync --locked --extra api --extra db --extra queue --group dev --group native-build" + native_build = "uv run --no-sync maturin develop --locked --release --features pyo3/extension-module" + full_tests = "uv run --no-sync python -m pytest -q" + assert workflow_text.index(locked_sync) < workflow_text.index(native_build) < workflow_text.index(full_tests) def test_local_full_suite_installs_runtime_and_test_lockfiles(): diff --git a/uv.lock b/uv.lock index 7a11636f2..05a583ae4 100644 --- a/uv.lock +++ b/uv.lock @@ -424,6 +424,9 @@ dev = [ { name = "hypothesis" }, { name = "pytest" }, ] +native-build = [ + { name = "maturin" }, +] [package.metadata] requires-dist = [ @@ -450,6 +453,7 @@ dev = [ { name = "hypothesis", specifier = ">=6.100" }, { name = "pytest", specifier = ">=8.0" }, ] +native-build = [{ name = "maturin", specifier = "==1.15.0" }] [[package]] name = "cryptography" @@ -897,6 +901,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "maturin" +version = "1.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/c8/22e5e21b2679c9bce6415ca578034ca2cc9316be0642ae21e051a2d5198c/maturin-1.15.0.tar.gz", hash = "sha256:94b26cc8e8aba61a5f2099715fe640e18c5f678e9a500408b38761263954228a", size = 385504, upload-time = "2026-08-24T12:11:22.665Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/69/5c01b461044eb1f45ddcce006706eb88110c793cdb11c7ae0b5e08492e94/maturin-1.15.0-py3-none-linux_armv6l.whl", hash = "sha256:6bf6dc62e22d4dcfd5a51244ff0d58975fa4979c48209fe84159617648956d82", size = 10206220, upload-time = "2026-08-24T12:10:53.327Z" }, + { url = "https://files.pythonhosted.org/packages/eb/1f/2b431554e11687cdb1077e0cdadcc118c53f611086b3af00c8545a67c6a5/maturin-1.15.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:cd35772633f489841132bc8e71d6fc7f842df30b9c05cd5cdf1ee1ddcb744cc7", size = 19416513, upload-time = "2026-08-24T12:10:56.126Z" }, + { url = "https://files.pythonhosted.org/packages/51/36/e23a21cb34a648b711036b9b2fe1d4f3f4ee24f8db54215d73f1a9a3a3ec/maturin-1.15.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c40b4eae7bf5ef1f4b1af8d623fe4105016f93578fb15b764e741d08ec3b92dd", size = 10014962, upload-time = "2026-08-24T12:10:58.486Z" }, + { url = "https://files.pythonhosted.org/packages/8c/80/33b15cb2d8f30f12c807955e8f2fd775027692904e30ec0784744ce8cd83/maturin-1.15.0-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:7eb066372f541f8eb4909c79c5d9bd0b9e8125980bdf1ec9e8aba23c6c8d6c55", size = 10196223, upload-time = "2026-08-24T12:11:00.696Z" }, + { url = "https://files.pythonhosted.org/packages/fe/91/b495e19e2f5c503b540452b2039115e7b2363867e8c5ad4179eb752fa92c/maturin-1.15.0-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:653020a63525bb224e5ab0adf02e17a2e08bc86dbea7fc1399c9a56d7529b99e", size = 10541186, upload-time = "2026-08-24T12:11:02.857Z" }, + { url = "https://files.pythonhosted.org/packages/5d/2b/2abff58037188d852b124871b1f0d720e1c2bfb3d4f1b03d87c52cd66488/maturin-1.15.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:0ebf9767892725083138e671c34482c660317a2f3d6a29fc0e0f34e9d8c99136", size = 10083468, upload-time = "2026-08-24T12:11:05.012Z" }, + { url = "https://files.pythonhosted.org/packages/3f/07/b7e9f8be99a6627849e81ac7b6694876bce8f50a92995fe17e3cf2610f0a/maturin-1.15.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:7ab7eebffd7b8debca2265985de4eaeb332141276d24b9560b5ad484d4b3add1", size = 10047786, upload-time = "2026-08-24T12:11:07.042Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ab/167e3cb7accee11b507dbe53e0e87aeccb376d44ae66284c96ee4df3a9fd/maturin-1.15.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:126e12e618b4db42f68c779a56d41f82a390145ba36ac3f621d057eb34f5ad9d", size = 13315332, upload-time = "2026-08-24T12:11:09.433Z" }, + { url = "https://files.pythonhosted.org/packages/14/4d/801379f646cbc6b00998e5289b0630a886be3a4ee4c75b6bc9b87478a7f1/maturin-1.15.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4f9d33e6c3f9615c8caceecbbbd440f8eb25a3ddeb687077682cd5eca2e9ae15", size = 10807183, upload-time = "2026-08-24T12:11:11.73Z" }, + { url = "https://files.pythonhosted.org/packages/89/27/2e612e1cbd1580e9e94d4722c227b5180dca27b32b955a34b79918aa1292/maturin-1.15.0-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:bf29beddd0c6708f112db51d5275fc28b28b9e9c9c5faae387eaef662918b176", size = 10413274, upload-time = "2026-08-24T12:11:14.04Z" }, + { url = "https://files.pythonhosted.org/packages/70/d8/202a7b4d75a51f20f84ec9ce3b7345b12b822207072164e2b1c6ef665125/maturin-1.15.0-py3-none-win32.whl", hash = "sha256:da649988be98e87e009e51b1bf0d301b6a301bc0cecbdd60d40d8ba60748d1ca", size = 8928744, upload-time = "2026-08-24T12:11:16.269Z" }, + { url = "https://files.pythonhosted.org/packages/40/dc/4e90da594986ba78dd3bc8a5921ecdcdb11085b22b02a412caab3b225601/maturin-1.15.0-py3-none-win_amd64.whl", hash = "sha256:552c2be4afd43fe8d5c9f3ec8d4c4756d973b8dcbe94c14084390301f50243e1", size = 10335085, upload-time = "2026-08-24T12:11:18.326Z" }, + { url = "https://files.pythonhosted.org/packages/8b/10/15d4314edf130955edf2dc237aa393a8a7c10f2b9b57b89fa2f61f915659/maturin-1.15.0-py3-none-win_arm64.whl", hash = "sha256:c7dc0c66c78d3debdd9c5aa807e861fbcbf07f3505d34b125df74c03986b0f48", size = 9713795, upload-time = "2026-08-24T12:11:20.83Z" }, +] + [[package]] name = "numpy" version = "2.5.2" From 0b10b553ab916f341ddfbb5c1fc6989c23e0f293 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 16:30:30 +0900 Subject: [PATCH 42/87] fix: preserve unmeasured streaming handler compatibility --- .github/workflows/security.yml | 2 +- contextual_orchestrator/server.py | 25 +++++++++++++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 7e9864390..d17717983 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -88,7 +88,7 @@ jobs: dist/contextual_orchestrator-*.whl "$RUNNER_TEMP"/decision-wheels/*.whl cd "$RUNNER_TEMP" PYTHONPATH="$RUNNER_TEMP/nim-wheel-site" \ - python -c "import pathlib, contextual_orchestrator as core, contextual_orchestrator.nim_benchmark, contextual_orchestrator._decision_receipt as native; root = pathlib.Path('nim-wheel-site').resolve(); assert pathlib.Path(core.__file__).is_relative_to(root); assert pathlib.Path(native.__file__).is_relative_to(root)" + python -c "import pathlib, contextual_orchestrator as core; import contextual_orchestrator.nim_benchmark; import contextual_orchestrator._decision_receipt as native; root = pathlib.Path('nim-wheel-site').resolve(); assert pathlib.Path(core.__file__).is_relative_to(root); assert pathlib.Path(native.__file__).is_relative_to(root)" PYTHONPATH="$RUNNER_TEMP/nim-wheel-site" \ python -m pytest --noconftest --import-mode=importlib "$GITHUB_WORKSPACE/tests/test_decision_receipts.py" -q diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 00b1f256c..7866fa306 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -8190,7 +8190,7 @@ def _ensure_decision_measurement(self, boundary="first_execution_slot") -> None: def _release_measured_slot(self) -> None: """Finalize measurement independently of releasing the execution slot.""" try: - measurement = self._decision_measurement + measurement = self._decision_measurement if decision_receipts else None if measurement is not None: active_error = sys.exc_info()[1] reason = self._decision_failure_reason @@ -8313,7 +8313,8 @@ def _write_response(self, writer: Callable[[], None]) -> bool: writer() return True except (BrokenPipeError, ConnectionError, OSError): - self._decision_failure_reason = "cancelled" + if decision_receipts: + self._decision_failure_reason = "cancelled" _LOGGER.debug("client_disconnected") # `_send*`/`_begin_sse` writers record their *intended* # status in `self._last_status` before calling this method @@ -8509,7 +8510,10 @@ def progress(role: str, status: str) -> None: part={"type": "summary_text", "text": text}, ) - self._acquire_measured_slot() + if decision_receipts: + self._acquire_measured_slot() + else: + security.acquire_run_slot() try: if not self._begin_sse(): return False @@ -8671,7 +8675,10 @@ def progress(role: str, status: str) -> None: self._decision_failure_reason = "cancelled" return False finally: - self._release_measured_slot() + if decision_receipts: + self._release_measured_slot() + else: + security.release_run_slot() def _stream_route_completion( self, @@ -8717,7 +8724,10 @@ def usage_frame(usage: dict[str, Any]) -> str: } return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" - self._acquire_measured_slot() + if decision_receipts: + self._acquire_measured_slot() + else: + security.acquire_run_slot() try: if not self._begin_sse() or not self._write_sse( frame({"role": "assistant"}) @@ -8777,7 +8787,10 @@ def usage_frame(usage: dict[str, Any]) -> str: return self._write_sse("data: [DONE]\n\n") finally: - self._release_measured_slot() + if decision_receipts: + self._release_measured_slot() + else: + security.release_run_slot() def _send_security_headers(self) -> None: if getattr(self, "close_connection", False): From daa5e3636342644c49a037e705e2ecf85e642c71 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 16:34:17 +0900 Subject: [PATCH 43/87] fix: preserve trusted request identity in typed SSE failures --- contextual_orchestrator/server.py | 6 +- .../doctoring/provider_request_correlation.md | 14 ++++ tests/test_stream_error_identity.py | 73 +++++++++++++++++++ 3 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 tests/test_stream_error_identity.py diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index fc706a11a..bc1ecd32f 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -8502,7 +8502,7 @@ def progress(role: str, status: str) -> None: "error": _error_payload( exc.error_code, _provider_upstream_message(exc), - {"request_id": uuid.uuid4().hex, **exc.detail}, + {**exc.detail, "request_id": current_request_id() or uuid.uuid4().hex}, )["error"], } emit("response.failed", response=failed) @@ -8679,8 +8679,8 @@ def usage_frame(usage: dict[str, Any]) -> str: return except ToolFallbackStoppedError as exc: detail = { - "request_id": uuid.uuid4().hex, **_tool_fallback_error_detail(exc), + "request_id": current_request_id() or uuid.uuid4().hex, } payload = _error_payload( TOOL_FALLBACK_STOPPED_CODE, @@ -8697,7 +8697,7 @@ def usage_frame(usage: dict[str, Any]) -> str: payload = _error_payload( exc.error_code, _provider_upstream_message(exc), - {"request_id": uuid.uuid4().hex, **exc.detail}, + {**exc.detail, "request_id": current_request_id() or uuid.uuid4().hex}, ) if not self._write_sse( f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" diff --git a/docs/doctoring/provider_request_correlation.md b/docs/doctoring/provider_request_correlation.md index a28a186d5..dd99deec8 100644 --- a/docs/doctoring/provider_request_correlation.md +++ b/docs/doctoring/provider_request_correlation.md @@ -123,3 +123,17 @@ tests/test_responses_store_http_honesty.py -q` then completed with exit 0: 4 passed in 20.05s. The interruption did not recur in this run; its signal or operating-system trigger remains unproven. No TLS checks were bypassed and no retry was added. A clean isolated run does not replace full-suite verification. +# Typed streaming error correlation + +The chat and Responses SSE error adapters must retain the request identity +observed inside the provider invocation. They previously generated another UUID +when framing typed failures, breaking correlation despite correct provider logs. +The repair uses the ordinary HTTP error adapter's trusted identity convention. +This is an identity mismatch fix, not evidence of upstream detail injection. + +Reproduce with `python -m pytest tests/test_stream_error_identity.py -q`. +Three real HTTP cases cover chat provider errors, Responses provider errors, and +chat stopped-tool errors. The Responses fixture fixes the routing choice to +isolate SSE framing; it does not test routing policy. The original cases failed +identity equality; after repair, these and both debug logging suites passed +(40 tests, 2.75 seconds). No hosted check or deployment is implied. diff --git a/tests/test_stream_error_identity.py b/tests/test_stream_error_identity.py new file mode 100644 index 000000000..a04281313 --- /dev/null +++ b/tests/test_stream_error_identity.py @@ -0,0 +1,73 @@ +"""Real HTTP streaming errors retain the provider-observed request identity.""" + +import http.client +import json +import threading + +import pytest + +from contextual_orchestrator import ModelAgent, TaskOrchestrator +from contextual_orchestrator.provider_errors import ProviderUpstreamError +from contextual_orchestrator.server import SecurityConfig, build_server +from contextual_orchestrator.telemetry import current_request_id +from contextual_orchestrator.tool_fallback import ( + ToolFailureDecision, ToolFailureKind, ToolFallbackAction, ToolFallbackStoppedError, +) + + +@pytest.mark.parametrize("endpoint,error_kind", [ + ("/v1/chat/completions", "provider"), + ("/v1/responses", "provider"), + ("/v1/chat/completions", "tool"), +]) +def test_stream_error_preserves_request_identity(endpoint, error_kind, monkeypatch): + """Typed SSE failures use the same trusted ID as their provider invocation.""" + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")]) + observed_ids = [] + + def fail_stream(*args, **kwargs): + observed_ids.append(current_request_id()) + if error_kind == "tool": + raise ToolFallbackStoppedError("worker_one", ToolFailureDecision( + ToolFailureKind.AMBIGUOUS_OUTCOME, ToolFallbackAction.FAIL_CLOSED, + "ambiguous_outcome", False, False, + )) + raise ProviderUpstreamError( + agent_id="worker_one", model="mock/worker", error_code="unit_failure", + message="unit failure", client_status=502, transport="stream", + ) + + monkeypatch.setattr(orchestrator.client, "stream_chat", fail_stream) + monkeypatch.setattr(orchestrator, "would_route", lambda *args, **kwargs: True) + server = build_server(orchestrator, port=0, + security=SecurityConfig(auth_token="unit-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address, timeout=20) + try: + body = {"model": "orchestrator/auto", "mode": "route", "stream": True, + "messages": [{"role": "user", "content": "unit request"}]} + if endpoint == "/v1/responses": + body = {"model": "orchestrator/auto", "stream": True, "input": "hello"} + connection.request("POST", endpoint, json.dumps(body), { + "Content-Type": "application/json", "Authorization": "Bearer unit-token", + "X-Request-ID": "untrusted-client-id", + }) + response = connection.getresponse() + payload = response.read().decode() + assert response.status == 200 + events = [json.loads(line[6:]) for line in payload.splitlines() + if line.startswith("data: ") and line != "data: [DONE]"] + errors = [event["error"] for event in events if "error" in event] + errors += [event["response"]["error"] for event in events + if event.get("type") == "response.failed"] + assert len(observed_ids) == 1 + assert observed_ids[0] and observed_ids[0] != "untrusted-client-id" + assert len(errors) == 1 + assert errors[0]["detail"]["request_id"] == observed_ids[0] + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() From bafbaf9571420314b98d4a00686287e4454ee773 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 16:35:24 +0900 Subject: [PATCH 44/87] fix: admit streaming triage and retain typed selection failures --- contextual_orchestrator/server.py | 32 ++++++---- tests/test_decision_receipts.py | 103 ++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 13 deletions(-) diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 7866fa306..0a66dda39 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -7120,19 +7120,6 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di return messages = _validate_messages(body.get("messages")) mode = _validate_mode(body.get("orchestration") or body.get("orchestration_mode") or body.get("mode") or "auto") - route_stream = bool( - stream and orchestrator.would_route(messages, mode, model_name) - ) - if route_stream: - if explicit_trace: - raise RequestError( - 400, - "unsupported_trace_disclosure", - "remove include_orchestration_trace or use Responses streaming", - ) - include_trace = False - elif include_trace: - self._authorize_trace_access() # stream + stream_options already coerced/validated before passthrough. attribution = _validate_attribution(body.get("attribution")) routing = _validate_routing( @@ -7156,6 +7143,22 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di # sampling/controls already validated before passthrough branch. if "metadata" in body: _validate_openai_metadata(body) + if explicit_trace: + self._authorize_trace_access() + self._ensure_decision_measurement("validated_endpoint") + route_stream = bool( + stream and orchestrator.would_route(messages, mode, model_name) + ) + if route_stream: + if explicit_trace: + raise RequestError( + 400, + "unsupported_trace_disclosure", + "remove include_orchestration_trace or use Responses streaming", + ) + include_trace = False + elif include_trace and not explicit_trace: + self._authorize_trace_access() started_at = time.perf_counter() model_client = orchestrator.client with model_client.request_settings( @@ -8564,6 +8567,7 @@ def progress(role: str, status: str) -> None: self._decision_failure_reason = "cancelled" raise except ProviderUpstreamError as exc: + self._decision_failure_reason = "selection_failed" failed = { **created_response, "status": "failed", @@ -8754,6 +8758,7 @@ def usage_frame(usage: dict[str, Any]) -> str: ): return except ToolFallbackStoppedError as exc: + self._decision_failure_reason = "selection_failed" detail = { "request_id": uuid.uuid4().hex, **_tool_fallback_error_detail(exc), @@ -8770,6 +8775,7 @@ def usage_frame(usage: dict[str, Any]) -> str: if not self._write_sse(frame({}, finish="error")): return except ProviderUpstreamError as exc: + self._decision_failure_reason = "selection_failed" payload = _error_payload( exc.error_code, _provider_upstream_message(exc), diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py index 954d403d3..22fadb9cd 100644 --- a/tests/test_decision_receipts.py +++ b/tests/test_decision_receipts.py @@ -10,6 +10,109 @@ from contextual_orchestrator.server import build_server, SecurityConfig +@pytest.mark.parametrize("invalid_field", [None, "routing", "attribution", "user", "metadata", "implicit_trace", "explicit_trace"]) +def test_http_auto_stream_admits_before_triage(tmp_path, monkeypatch, invalid_field): + """Auto stream classification must share the eventual task's admission clock.""" + from contextual_orchestrator.decision_receipts import _CURRENT_DECISION, export_decision_receipts + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + snapshots = [] + original_chat = orchestrator.client.chat + def observed_chat(agent, messages, **kwargs): + current = _CURRENT_DECISION.get() + snapshots.append(current.snapshot() if current else None) + if messages[0]["content"] == orchestrator.TRIAGE_SYSTEM_PROMPT: + return '{"workflow_required": false}' + return original_chat(agent, messages, **kwargs) + monkeypatch.setattr(orchestrator.client, "chat", observed_chat) + security = SecurityConfig(auth_token="test-token") + if invalid_field in ("implicit_trace", "explicit_trace"): + security = SecurityConfig(bearer_verifier=lambda token, scope: token == "test-token" and scope == "inference", + expose_trace_by_default=True) + server = build_server(orchestrator, port=0, decision_receipts=True, security=security) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address) + try: + request_body = { + "model": "orchestrator/auto", "mode": "auto", "stream": True, + "messages": [{"role": "user", "content": "streamed question"}], + } + if invalid_field in ("routing", "attribution", "user", "metadata"): + request_body[invalid_field] = "" if invalid_field == "user" else [] + if invalid_field == "explicit_trace": + request_body["include_orchestration_trace"] = True + connection.request("POST", "/v1/chat/completions", json.dumps(request_body), + {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + response.read() + expected_status = 401 if invalid_field == "explicit_trace" else (400 if invalid_field in ("routing", "attribution", "user", "metadata") else 200) + assert response.status == expected_status + connection.close() + server.shutdown() + if expected_status != 200: + assert snapshots == [] + assert orchestrator._store.load("accepted_request") == [] + return + assert snapshots and all(snapshot is not None for snapshot in snapshots) + observation, = export_decision_receipts(orchestrator._store)["observations"] + assert observation["first_provider_phase"] == "structured_triage" + assert observation["durable_ack_elapsed_ns"] is not None + assert snapshots[0]["request_id"] == observation["request_id"] + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +@pytest.mark.parametrize("endpoint", ["/v1/chat/completions", "/v1/responses"]) +def test_http_typed_stream_failure_before_selection_is_retained(tmp_path, monkeypatch, endpoint): + """Handled provider failures before task selection retain a failure denominator.""" + from contextual_orchestrator.provider_errors import ProviderUpstreamError + from contextual_orchestrator.decision_receipts import export_decision_receipts + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + def rejected(*args, **kwargs): + raise ProviderUpstreamError(agent_id="worker_one", model="mock/worker", + error_code="rate_limit_exceeded", message="unit rejection", + client_status=429, provider_status=429, retryable=True) + monkeypatch.setattr(orchestrator, "stream_route", rejected) + monkeypatch.setattr(orchestrator, "conduct", rejected) + if endpoint == "/v1/responses": + monkeypatch.setattr(orchestrator, "would_route", lambda *args, **kwargs: False) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address) + try: + request_body = {"model": "orchestrator/auto", "mode": "route", "stream": True} + if endpoint == "/v1/responses": + request_body["input"] = "question" + request_body.pop("mode") + else: + request_body["messages"] = [{"role": "user", "content": "question"}] + connection.request("POST", endpoint, json.dumps(request_body), + {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + payload = response.read() + assert response.status == 200 + assert b"rate_limit_exceeded" in payload + connection.close() + server.shutdown() + observation, = export_decision_receipts(orchestrator._store)["observations"] + assert observation["status"] == "selection_failed" + assert observation["durable_ack_elapsed_ns"] is None + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + def test_http_evidence_embedding_cold_and_warm_keep_task_interval(tmp_path, monkeypatch): """Routing evidence calls occur only cold and remain before task acknowledgement.""" from contextual_orchestrator.decision_receipts import _CURRENT_DECISION, export_decision_receipts From 3b6dd47ebb0f88802bacdd302051d2f03e7d5003 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 16:39:54 +0900 Subject: [PATCH 45/87] fix: finalize admitted request errors without erasing acknowledgements --- contextual_orchestrator/server.py | 6 ++++++ tests/test_decision_receipts.py | 12 +++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index e49651001..23990cd60 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -8263,6 +8263,12 @@ def _send_error( message: str, detail: dict[str, Any] | None = None, ) -> None: + if decision_receipts and status >= 400: + measurement = self._decision_measurement + if (measurement is not None + and measurement.receipt.status in ("accepted", "selected") + and self._decision_failure_reason == "unfinished"): + self._decision_failure_reason = "selection_failed" request_id = current_request_id() or uuid.uuid4().hex _LOGGER.warning( "request_failed status=%s code=%s request_id=%s", status, code, request_id diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py index 22fadb9cd..937a4f2ac 100644 --- a/tests/test_decision_receipts.py +++ b/tests/test_decision_receipts.py @@ -10,7 +10,7 @@ from contextual_orchestrator.server import build_server, SecurityConfig -@pytest.mark.parametrize("invalid_field", [None, "routing", "attribution", "user", "metadata", "implicit_trace", "explicit_trace"]) +@pytest.mark.parametrize("invalid_field", [None, "routing", "attribution", "user", "metadata", "implicit_trace", "explicit_trace", "authorized_trace"]) def test_http_auto_stream_admits_before_triage(tmp_path, monkeypatch, invalid_field): """Auto stream classification must share the eventual task's admission clock.""" from contextual_orchestrator.decision_receipts import _CURRENT_DECISION, export_decision_receipts @@ -40,16 +40,22 @@ def observed_chat(agent, messages, **kwargs): } if invalid_field in ("routing", "attribution", "user", "metadata"): request_body[invalid_field] = "" if invalid_field == "user" else [] - if invalid_field == "explicit_trace": + if invalid_field in ("explicit_trace", "authorized_trace"): request_body["include_orchestration_trace"] = True connection.request("POST", "/v1/chat/completions", json.dumps(request_body), {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) response = connection.getresponse() response.read() - expected_status = 401 if invalid_field == "explicit_trace" else (400 if invalid_field in ("routing", "attribution", "user", "metadata") else 200) + expected_status = 401 if invalid_field == "explicit_trace" else (400 if invalid_field in ("routing", "attribution", "user", "metadata", "authorized_trace") else 200) assert response.status == expected_status connection.close() server.shutdown() + if invalid_field == "authorized_trace": + observation, = export_decision_receipts(orchestrator._store)["observations"] + assert observation["status"] == "selection_failed" + assert observation["durable_ack_elapsed_ns"] is None + assert snapshots[0]["request_id"] == observation["request_id"] + return if expected_status != 200: assert snapshots == [] assert orchestrator._store.load("accepted_request") == [] From c7345670e08f029ad3aa5dd1133037bb4b451d9b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 16:58:07 +0900 Subject: [PATCH 46/87] fix: reserve chat streaming capacity before classification --- contextual_orchestrator/server.py | 24 +++++-- .../doctoring/decision_receipt_integration.md | 16 +++++ tests/test_decision_receipts.py | 67 +++++++++++++++++++ 3 files changed, 101 insertions(+), 6 deletions(-) diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 23990cd60..9813c5a83 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -5509,6 +5509,7 @@ class Handler(BaseHTTPRequestHandler): """Handle authenticated orchestration, administration, and health routes.""" _session_token = None _trace_token = None + _classification_slot_held = False def _bind_session(self, session_id: str | None) -> None: """Bind validated request correlation to this handler context.""" @@ -5640,10 +5641,13 @@ def handle_one_request(self) -> None: self._request_started = None self._decision_measurement = None self._decision_failure_reason = "unfinished" + self._classification_slot_held = False with request_identity(): try: super().handle_one_request() finally: + if self._classification_slot_held: + self._release_measured_slot() if self._decision_measurement is not None: self._decision_measurement.close(self._decision_failure_reason) self._decision_measurement = None @@ -7146,6 +7150,9 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di if explicit_trace: self._authorize_trace_access() self._ensure_decision_measurement("validated_endpoint") + if stream: + self._acquire_measured_slot() + self._classification_slot_held = True route_stream = bool( stream and orchestrator.would_route(messages, mode, model_name) ) @@ -7175,6 +7182,7 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di messages, model_name, include_usage=include_usage, + slot_acquired=True, ) orchestrator.record_analytics_event( "chat_completion_requested", @@ -8152,7 +8160,8 @@ def _validate_trace_request(self, body: dict[str, Any]) -> bool: return include_trace def _run(self, callback: Any) -> dict[str, Any]: - self._acquire_measured_slot() + if not self._classification_slot_held: + self._acquire_measured_slot() try: return callback() finally: @@ -8204,6 +8213,7 @@ def _release_measured_slot(self) -> None: self._decision_failure_reason = reason finally: security.release_run_slot() + self._classification_slot_held = False def _parse_positive_int(self, raw: str | None, field_name: str, default: int, max_value: int | None = None) -> int: value = default if raw is None else int(raw) @@ -8698,6 +8708,7 @@ def _stream_route_completion( model_name: str, *, include_usage: bool = False, + slot_acquired: bool = False, ) -> None: """Pipe live provider deltas as OpenAI chat-completion SSE frames.""" run_id = f"run_{uuid.uuid4().hex}" @@ -8734,10 +8745,11 @@ def usage_frame(usage: dict[str, Any]) -> str: } return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" - if decision_receipts: - self._acquire_measured_slot() - else: - security.acquire_run_slot() + if not slot_acquired: + if decision_receipts: + self._acquire_measured_slot() + else: + security.acquire_run_slot() try: if not self._begin_sse() or not self._write_sse( frame({"role": "assistant"}) @@ -8799,7 +8811,7 @@ def usage_frame(usage: dict[str, Any]) -> str: return self._write_sse("data: [DONE]\n\n") finally: - if decision_receipts: + if slot_acquired or decision_receipts: self._release_measured_slot() else: security.release_run_slot() diff --git a/docs/doctoring/decision_receipt_integration.md b/docs/doctoring/decision_receipt_integration.md index 7db2f9009..46530d3ce 100644 --- a/docs/doctoring/decision_receipt_integration.md +++ b/docs/doctoring/decision_receipt_integration.md @@ -118,6 +118,22 @@ paths still require complete request-boundary coverage and executable evidence. No protected merge, production publication, or customer KPI gain is claimed. # Auxiliary and package acceptance checkpoint (2026-09-09) +The later frozen `3b6dd47ebb0f88802bacdd302051d2f03e7d5003` full suite passed +3,442 tests with two skips in 771.43 s; its noneditable wheel-only receipt and SSE +identity matrix passed 33 tests in 14.21 s. Independent saturation inspection then +found chat streaming could call triage before acquiring capacity, despite a 503 +response. This was present with measurement disabled too; sibling nonstream chat +and Responses streaming already rejected without a provider call. + +The capacity repair holds one explicit chat-classification lease through either +the direct stream or conducted `_run` path. Request finalization releases the +lease if classification or trace validation exits early. It does not acquire a +second slot, release a slot it never acquired, or depend on measurement being +enabled. Actual HTTP RED covered saturation and early exits (three failures, +one passing control); the expanded enabled/disabled route/conduct/trace/error +matrix plus streaming, disconnect, identity, and trace regression tests passed +93 cases in 33.87 s. Those are local correctness results, not a latency gain. + At `9707a5e1`, focused receipt/persistence tests passed 41 cases in 8.43 s. The Rust clock now separates provider-ready diagnostics from initial task-route acknowledgement. Structured triage, generated planning, and pre-selection evidence diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py index 937a4f2ac..0a06b04e0 100644 --- a/tests/test_decision_receipts.py +++ b/tests/test_decision_receipts.py @@ -10,6 +10,73 @@ from contextual_orchestrator.server import build_server, SecurityConfig +@pytest.mark.parametrize("scenario", ["saturated", "success", "conduct_success", "classifier_error", "trace_rejection"]) +@pytest.mark.parametrize("measurement_enabled", [False, True]) +def test_chat_stream_classification_owns_one_capacity_lease(tmp_path, monkeypatch, scenario, measurement_enabled): + """Classification and task execution share capacity and release it exactly once.""" + from contextual_orchestrator.server import RequestError + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + security = SecurityConfig(auth_token="test-token", max_concurrent_runs=1) + original_acquire, original_release = security.acquire_run_slot, security.release_run_slot + acquired, released, calls = [], [], [] + def acquire(): + original_acquire() + acquired.append(True) + def release(): + released.append(True) + original_release() + monkeypatch.setattr(security, "acquire_run_slot", acquire) + monkeypatch.setattr(security, "release_run_slot", release) + original_chat = orchestrator.client.chat + def chat(agent, messages, **kwargs): + calls.append(True) + if messages[0]["content"] == orchestrator.TRIAGE_SYSTEM_PROMPT: + return json.dumps({"workflow_required": scenario == "conduct_success"}) + return original_chat(agent, messages, **kwargs) + monkeypatch.setattr(orchestrator.client, "chat", chat) + if scenario == "classifier_error": + def reject_classifier(*args, **kwargs): + raise ValueError("unit classifier rejection") + monkeypatch.setattr(orchestrator, "would_route", reject_classifier) + server = build_server(orchestrator, port=0, decision_receipts=measurement_enabled, security=security) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + if scenario == "saturated": + original_acquire() + connection = http.client.HTTPConnection(*server.server_address) + try: + body = {"model": "orchestrator/auto", "mode": "auto", "stream": True, + "messages": [{"role": "user", "content": "capacity question"}]} + if scenario == "trace_rejection": + body["include_orchestration_trace"] = True + connection.request("POST", "/v1/chat/completions", json.dumps(body), + {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + response.read() + assert response.status == {"saturated": 503, "success": 200, "conduct_success": 200, + "classifier_error": 400, "trace_rejection": 400}[scenario] + connection.close() + server.shutdown() + if scenario == "saturated": + assert calls == [] + assert acquired == released == [] + else: + assert len(acquired) == len(released) == 1 + original_acquire() + with pytest.raises(RequestError): + original_acquire() + original_release() + finally: + if scenario == "saturated": + original_release() + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + @pytest.mark.parametrize("invalid_field", [None, "routing", "attribution", "user", "metadata", "implicit_trace", "explicit_trace", "authorized_trace"]) def test_http_auto_stream_admits_before_triage(tmp_path, monkeypatch, invalid_field): """Auto stream classification must share the eventual task's admission clock.""" From 6b40db4695f61ba29a485fc01c59ebae4bc566d3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:06:09 +0900 Subject: [PATCH 47/87] fix: retain durable workflow request origin identity --- AGENTS.md | 3 + CLAUDE.md | 3 + contextual_orchestrator/orchestrator.py | 13 +- docs/doctoring/workflow_request_link.md | 30 +++++ tests/test_workflow_request_link.py | 151 ++++++++++++++++++++++++ 5 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 docs/doctoring/workflow_request_link.md create mode 100644 tests/test_workflow_request_link.py diff --git a/AGENTS.md b/AGENTS.md index a8c04409e..f76a887d8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,8 @@ # AGENTS.md +Workflow origin identity and persistence limitations are documented in +`docs/doctoring/workflow_request_link.md`; preserve origin on replacements and reload. + Cross-agent conventions for `contextual-orchestrator`, readable by any coding agent (Claude, Codex, Cursor, opencode, …). Keep this file tool-agnostic. diff --git a/CLAUDE.md b/CLAUDE.md index 9a8d237c1..309ed0c7c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,5 +1,8 @@ # CLAUDE.md +See `docs/doctoring/workflow_request_link.md` for request-to-workflow correlation +tests, cache semantics, and the distinction between in-memory and durable outcomes. + This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. ## Read AGENTS.md first diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 5cae0e049..007282a13 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -4212,7 +4212,7 @@ def _reload_state(self) -> None: observation.get("irt_row", ()), ) for record in self._store.load("workflow_run"): - self._replace_workflow_run(record) + self._replace_workflow_run(record, restored=True) # A batch_route row persisted before judging (see batch_route's # own pending-record comment) carries an explicit # "pending_verification" marker and intentionally never reaches @@ -5402,6 +5402,8 @@ def stream_route( record["owner_id"] = owner_id self._replace_workflow_run(record) self._run_order.appendleft(record["workflow_run_id"]) + if self._store is not None: + self._store.save("workflow_run", record["workflow_run_id"], record) self._append_audit_event( "workflow_run_created", {"workflow_run_id": record["workflow_run_id"], "mode": "route", "agent_count": 1}, @@ -8790,7 +8792,7 @@ def _run_budget_output_by_model( ) return output_by_model, True - def _replace_workflow_run(self, record: dict[str, Any]) -> None: + def _replace_workflow_run(self, record: dict[str, Any], *, restored: bool = False) -> None: """Store one run and update its constant-time budget meter atomically.""" model_by_agent = {agent.id: agent.model for agent in self.candidates} for step in record.get("trace", []): @@ -8800,6 +8802,13 @@ def _replace_workflow_run(self, record: dict[str, Any]) -> None: run_id = record["workflow_run_id"] with self._budget_spend_lock: previous = self._workflow_runs.get(run_id) + origin_request_id = (previous.get("request_id") if previous is not None + else record.get("request_id") if restored + else current_request_id()) + if origin_request_id is not None: + record["request_id"] = origin_request_id + else: + record.pop("request_id", None) for sign, run in ((-1, previous), (1, record)): if run is None: continue diff --git a/docs/doctoring/workflow_request_link.md b/docs/doctoring/workflow_request_link.md new file mode 100644 index 000000000..b00042e82 --- /dev/null +++ b/docs/doctoring/workflow_request_link.md @@ -0,0 +1,30 @@ +# Durable workflow origin identity + +The accepted-request ledger and completed workflow records previously had no +durable join. At base c7345670, real HTTP route, conduct and streamed route +requests could not match their persisted outcome to the trusted request ID. + +The shared workflow replacement seam now records the trusted HTTP identity on +first creation and preserves it on replacement. Explicit state restoration +retains stored identity instead of attributing history to the loading context. +Non-HTTP runs remain valid without an ID. Cache hits create separate outcome +records with their existing cache-hit classification; the earlier run's origin +is unchanged. This does not make reused output a new provider execution. + +The regression also exposed that stream_route never saved its completed run. +It now uses the same synchronous workflow store path as run/conduct. A failed +save emits the existing terminal SSE error, retains the admission and does not +rewrite an earlier initial-decision acknowledgement. As in run/conduct, the +in-memory run and budget update precede storage: memory is not durable proof, +and no atomic memory/database transaction is claimed. Consumers must reconcile +against persisted records rather than count the in-memory run as stored. + +Run tests/test_workflow_request_link.py with the native extension built for the +same candidate. Initial four cases failed; subsequent HTTP enabled/disabled, +cache, reload, persistence and SSE regressions passed. Tests use controlled +provider output and establish identity, not independently adjudicated accuracy. +The request may map to multiple workflow outcomes; it is not a one-to-one join +contract for every API. Detached background execution without propagated HTTP +context remains unlinked, rather than guessed from the current thread. Explicit +batch-parent/item linkage and crash recovery after an in-memory-only update +remain outside this bounded repair. diff --git a/tests/test_workflow_request_link.py b/tests/test_workflow_request_link.py new file mode 100644 index 000000000..3f35cb053 --- /dev/null +++ b/tests/test_workflow_request_link.py @@ -0,0 +1,151 @@ +"""Trusted HTTP request identity survives durable workflow persistence.""" + +import http.client +import json +import threading + +import pytest + +from contextual_orchestrator import ModelAgent, TaskOrchestrator +from contextual_orchestrator.server import SecurityConfig, build_server +from contextual_orchestrator.telemetry import current_request_id, request_identity + + +@pytest.mark.parametrize("mode,stream", [("route", False), ("conduct", False), ("route", True), ("write_failure", True)]) +@pytest.mark.parametrize("measurement_enabled", [False, True]) +def test_http_workflow_retains_origin_request(tmp_path, monkeypatch, mode, stream, measurement_enabled): + """Provider-observed identity joins the stored run without caller identity trust.""" + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + observed_ids = [] + if mode == "write_failure": + original_save = orchestrator._store.save + + def fail_workflow_write(kind, *args, **kwargs): + if kind == "workflow_run": + raise RuntimeError("private-storage-failure") + return original_save(kind, *args, **kwargs) + + monkeypatch.setattr(orchestrator._store, "save", fail_workflow_write) + for method_name in ("chat", "stream_chat"): + original_method = getattr(orchestrator.client, method_name) + + def observe(*args, _method=original_method, **kwargs): + observed_ids.append(current_request_id()) + return _method(*args, **kwargs) + + monkeypatch.setattr(orchestrator.client, method_name, observe) + server = build_server(orchestrator, port=0, decision_receipts=measurement_enabled, + security=SecurityConfig(auth_token="unit-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address, timeout=20) + try: + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "route" if mode == "write_failure" else mode, "stream": stream, + "messages": [{"role": "user", "content": "Explain addition briefly."}], + }), {"Content-Type": "application/json", "Authorization": "Bearer unit-token", + "X-Request-ID": "untrusted-client-id"}) + response = connection.getresponse() + response_payload = response.read().decode() + connection.close() + server.shutdown() + server.server_close() + assert response.status == 200 + stored_runs = orchestrator._store.load("workflow_run") + if mode == "write_failure": + assert not stored_runs + assert len(orchestrator._workflow_runs) == 1 + assert '"finish_reason": "error"' in response_payload + assert "private-storage-failure" not in response_payload + if measurement_enabled: + from contextual_orchestrator.decision_receipts import export_decision_receipts + receipts = export_decision_receipts(orchestrator._store)["observations"] + assert len(receipts) == 1 + assert receipts[0]["status"] == "acknowledged" + return + assert len(stored_runs) == 1 + assert observed_ids and len(set(observed_ids)) == 1 + assert observed_ids[0] and observed_ids[0] != "untrusted-client-id" + assert stored_runs[0]["request_id"] == observed_ids[0] + if measurement_enabled: + from contextual_orchestrator.decision_receipts import export_decision_receipts + receipts = export_decision_receipts(orchestrator._store)["observations"] + assert len(receipts) == 1 + assert receipts[0]["request_id"] == stored_runs[0]["request_id"] + run_id = stored_runs[0]["workflow_run_id"] + orchestrator.close() + restored = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + try: + assert restored.get_workflow_run(run_id)["request_id"] == observed_ids[0] + finally: + restored.close() + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +def test_workflow_update_preserves_original_request_identity(): + """Later request contexts cannot claim an existing run or a non-HTTP run.""" + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")]) + try: + for origin_http in (False, True): + run_id = f"run_{origin_http}" + record = {"workflow_run_id": run_id, "trace": []} + if origin_http: + with request_identity() as origin_id: + orchestrator._replace_workflow_run(record) + else: + origin_id = None + orchestrator._replace_workflow_run(record) + with request_identity(): + replacement = {"workflow_run_id": run_id, "trace": []} + orchestrator._replace_workflow_run(replacement) + assert replacement.get("request_id") == origin_id + finally: + orchestrator.close() + + +def test_http_cache_hit_keeps_distinct_outcome_identity(tmp_path): + """A reused answer creates a cache-hit outcome, not a reassigned execution.""" + from contextual_orchestrator.decision_receipts import export_decision_receipts + + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db", cache_ttl=60) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="unit-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + for _ in range(2): + connection = http.client.HTTPConnection(*server.server_address, timeout=20) + try: + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "route", + "messages": [{"role": "user", "content": "Explain addition briefly."}], + }), {"Content-Type": "application/json", "Authorization": "Bearer unit-token"}) + response = connection.getresponse() + response.read() + assert response.status == 200 + finally: + connection.close() + server.shutdown() + server.server_close() + stored_runs = orchestrator._store.load("workflow_run") + receipts = export_decision_receipts(orchestrator._store)["observations"] + assert [record["cache_status"] for record in stored_runs] == ["miss", "hit"] + assert len({record["workflow_run_id"] for record in stored_runs}) == 2 + assert len({record["request_id"] for record in stored_runs}) == 2 + assert {record["request_id"] for record in stored_runs} == { + receipt["request_id"] for receipt in receipts + } + assert sorted(receipt["status"] for receipt in receipts) == ["acknowledged", "cache_hit"] + finally: + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() From 4cf7feafd554fbbd65dfc3b790f1081623b0d05a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:07:06 +0900 Subject: [PATCH 48/87] docs: record exact workflow linkage reproduction environment --- docs/doctoring/workflow_request_link.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/doctoring/workflow_request_link.md b/docs/doctoring/workflow_request_link.md index b00042e82..f2b2c4a50 100644 --- a/docs/doctoring/workflow_request_link.md +++ b/docs/doctoring/workflow_request_link.md @@ -19,8 +19,21 @@ in-memory run and budget update precede storage: memory is not durable proof, and no atomic memory/database transaction is claimed. Consumers must reconcile against persisted records rather than count the in-memory run as stored. -Run tests/test_workflow_request_link.py with the native extension built for the -same candidate. Initial four cases failed; subsequent HTTP enabled/disabled, +Reproduction prerequisite: the unchanged native extension built at base +`c7345670e08f029ad3aa5dd1133037bb4b451d9b` is installed in the existing isolated +Python 3.14 environment below. From `/tmp/co-outcome-request-link-20260909`, run: + +```sh +/tmp/co-receipt-wheel-install-20260909/bin/python -c 'import contextual_orchestrator; contextual_orchestrator.__path__.append("/tmp/co-receipt-wheel-install-20260909/lib/python3.14/site-packages/contextual_orchestrator"); import pytest; raise SystemExit(pytest.main(["tests/test_workflow_request_link.py", "tests/test_persistence.py", "tests/test_stream_error_identity.py", "-q"]))' +``` + +This explicitly loads successor Python source and the unchanged base native +extension. The namespace append is test-only; it does not install the successor +or establish wheel/release acceptance. Preserve that environment unchanged. +The three-file command passed 34 tests in 9.36 seconds. A final linkage-only +run after adding the in-memory failure-state assertion passed 10 in 5.43 seconds. +The independent original HTTP probe also confirmed one admission joins one +durably stored route outcome. Initial four cases failed; subsequent HTTP enabled/disabled, cache, reload, persistence and SSE regressions passed. Tests use controlled provider output and establish identity, not independently adjudicated accuracy. The request may map to multiple workflow outcomes; it is not a one-to-one join From af8d732e6cfc9c0169ac850f875f42f1db7eecd4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:11:55 +0900 Subject: [PATCH 49/87] docs: repair locked full-suite environment prerequisite --- docs/doctoring/workflow_request_link.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/doctoring/workflow_request_link.md b/docs/doctoring/workflow_request_link.md index f2b2c4a50..66b0cedd5 100644 --- a/docs/doctoring/workflow_request_link.md +++ b/docs/doctoring/workflow_request_link.md @@ -1,5 +1,25 @@ # Durable workflow origin identity +## Full-suite environment repair + +Full collection at `4cf7feafd554fbbd65dfc3b790f1081623b0d05a` failed with +exit 2 after 5.41 seconds: the shared wheel-smoke environment lacked Hypothesis. +No tests completed in that run. The remedy is a separate project-local environment, +not a runtime change or an installation into the shared historical environment: + +```sh +uv sync --locked --extra api --extra db --extra queue --group dev +.venv/bin/python -c 'import contextual_orchestrator; contextual_orchestrator.__path__.append("/tmp/co-receipt-wheel-install-20260909/lib/python3.14/site-packages/contextual_orchestrator"); import pytest; raise SystemExit(pytest.main(["-q"]))' +``` + +Setup completed with CPython 3.14.6 and the existing lock, including Hypothesis +6.165.10. The reused base-native artifact is `_decision_receipt.abi3.so` under +that explicitly named site-packages directory, SHA-256 +`ddac17f8c6a25e52a9233bd9e75f5ca3c754eb24641caa58113148df8ef60b4d`. +Native source is unchanged from `c7345670`; this is source-integration evidence, +not an installed successor-wheel or release result. Do not modify the shared +native environment. Full-suite results remain pending until its handle terminates. + The accepted-request ledger and completed workflow records previously had no durable join. At base c7345670, real HTTP route, conduct and streamed route requests could not match their persisted outcome to the trusted request ID. From 387aa2111142b13b327f7065226c0f22c305b872 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:32:08 +0900 Subject: [PATCH 50/87] test: reproduce missing durable batch request lineage --- tests/test_batch_request_lineage.py | 81 +++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 tests/test_batch_request_lineage.py diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py new file mode 100644 index 000000000..6005d9237 --- /dev/null +++ b/tests/test_batch_request_lineage.py @@ -0,0 +1,81 @@ +"""Deferred batch outcomes retain their original trusted HTTP admission.""" + +import copy +import threading + +from contextual_orchestrator import CostRoutingCoordinator, ModelAgent, TaskOrchestrator +from contextual_orchestrator.batch_routing import PgLlmBatchBackend +from contextual_orchestrator.server import SecurityConfig, build_server +from contextual_orchestrator.telemetry import current_request_id +from test_batch_routing import _FakeBatchApiClient +from test_cost_review_server import _request + + +def test_http_batch_origin_survives_distinct_retrieval_and_reload(tmp_path): + """One submission joins two item outcomes without trusting their custom IDs.""" + class ObservedBatchClient(_FakeBatchApiClient): + """Reuse the existing offline provider contract with passive identity capture.""" + + async def create_batch_job(self, *args, **kwargs): + self.submission_request_id = current_request_id() + return await super().create_batch_job(*args, **kwargs) + + async def download_results(self, *args, **kwargs): + self.retrieval_request_id = current_request_id() + payload = await super().download_results(*args, **kwargs) + second_result = copy.deepcopy(payload["responses"][0]) + second_result["custom_id"] = "b" + payload["responses"].append(second_result) + return payload + + state_path = tmp_path / "state.db" + agents = [ModelAgent("worker_one", "mock/worker")] + orchestrator = TaskOrchestrator(agents, state_db=state_path) + batch_client = ObservedBatchClient() + coordinator = CostRoutingCoordinator( + orchestrator, batch_backend=PgLlmBatchBackend(batch_client) + ) + server = build_server(orchestrator, port=0, coordinator=coordinator, + security=SecurityConfig(auth_token="unit-token")) + worker_thread = threading.Thread(target=server.serve_forever, daemon=True) + worker_thread.start() + base_url = f"http://127.0.0.1:{server.server_address[1]}" + try: + status, submitted = _request("POST", f"{base_url}/api/v1/batch_routing_jobs", + "unit-token", {"requests": [ + {"custom_id": item_id, "model": "mock/worker", "mode": "route", + "messages": [{"role": "user", "content": "Offline contract fixture."}]} + for item_id in ("a", "b") + ]}) + assert status == 201, submitted + status, retrieved = _request( + "POST", f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results", + "unit-token", + ) + assert status == 200, retrieved + assert {item["custom_id"] for item in retrieved["results"]} == {"a", "b"} + assert batch_client.submission_request_id + assert batch_client.retrieval_request_id + assert batch_client.submission_request_id != batch_client.retrieval_request_id + assert batch_client.submission_request_id not in {"a", "b"} + finally: + server.shutdown() + worker_thread.join() + server.server_close() + orchestrator.close() + + restored = TaskOrchestrator(agents, state_db=state_path) + try: + # Proposed CO-owned association contract: one row per submission/job/item, + # not a replacement of provider custom_id or one origin per eventual job. + links = restored._store.load("batch_request_link") + actual_links = { + (row["request_id"], row["batch_job_id"], row["custom_id"]) + for row in links + } + assert actual_links == { + (batch_client.submission_request_id, submitted["job_id"], item["custom_id"]) + for item in retrieved["results"] + } + finally: + restored.close() From 9b7f49198e7995e77efe270dca96a2094b289051 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:32:59 +0900 Subject: [PATCH 51/87] docs: record deferred batch lineage red evidence --- docs/doctoring/batch_request_lineage.md | 33 +++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 docs/doctoring/batch_request_lineage.md diff --git a/docs/doctoring/batch_request_lineage.md b/docs/doctoring/batch_request_lineage.md new file mode 100644 index 000000000..cfedee463 --- /dev/null +++ b/docs/doctoring/batch_request_lineage.md @@ -0,0 +1,33 @@ +# Deferred batch request lineage + +## RED reproduction, 2026-09-09 + +Base `af8d732e6cfc9c0169ac850f875f42f1db7eecd4`; test commit +`387aa211` on isolated branch `codex/batch-request-lineage-20260909`. +Command: `.venv/bin/python -m pytest tests/test_batch_request_lineage.py -q`. +Terminal result: **1 failed in 3.75s**. Submission returned HTTP 201; +retrieval returned HTTP 200 with two distinct caller custom IDs. Passive backend +observations confirmed distinct trusted submission/retrieval request identities. +After closing and reopening SQLite, the proposed submission/job/item association +was absent: an empty set instead of two links. No runtime implementation changed. + +This test proposes the `batch_request_link` association shape; that shape is not +an existing released contract. It demonstrates the missing durable association, +not a defect in an already-promised schema. The fixture reuses the existing +offline pg-llm-batch client double. No external model was invoked. This is not +remote integration, process-restart batch execution, numerical accuracy, or latency +evidence. The fixture preserves results in test memory across state-store reload; +it does not claim that the default in-memory batch registry survives restart. + +The smallest repair belongs in CO's submission coordinator and durable job +association boundary, with a trusted identity argument from the HTTP adapter. +Preserve multiple submission/job/item associations and keep provider custom IDs +separate. Do not reassign origin when a later HTTP request retrieves outcomes. +Remote execution provenance requires the released pg-llm-batch contract separately. + +Environment: project-local Python 3.14.6, dependencies synchronized with +`uv sync --frozen`. An initial pytest-only environment failed collection because +certifi was absent; this was an environment error, not the RED result. The attempted +`uv sync --frozen --group test` was invalid because test is an optional extra, +not a dependency group; the frozen default synchronization repaired collection. +CodeGraph initialization completed (438 files); no other worktree was modified. From 919c945aff92ff98912f8ba63743d7cdd4bd12c6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:33:59 +0900 Subject: [PATCH 52/87] test: require explicit batch lineage write failure outcome --- tests/test_batch_request_lineage.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index 6005d9237..8fe57b43b 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -2,6 +2,7 @@ import copy import threading +import pytest from contextual_orchestrator import CostRoutingCoordinator, ModelAgent, TaskOrchestrator from contextual_orchestrator.batch_routing import PgLlmBatchBackend @@ -11,7 +12,8 @@ from test_cost_review_server import _request -def test_http_batch_origin_survives_distinct_retrieval_and_reload(tmp_path): +@pytest.mark.parametrize("write_failure", [False, True]) +def test_http_batch_origin_survives_distinct_retrieval_and_reload(tmp_path, monkeypatch, write_failure): """One submission joins two item outcomes without trusting their custom IDs.""" class ObservedBatchClient(_FakeBatchApiClient): """Reuse the existing offline provider contract with passive identity capture.""" @@ -31,6 +33,15 @@ async def download_results(self, *args, **kwargs): state_path = tmp_path / "state.db" agents = [ModelAgent("worker_one", "mock/worker")] orchestrator = TaskOrchestrator(agents, state_db=state_path) + if write_failure: + original_save = orchestrator._store.save + + def reject_link(kind, *args, **kwargs): + if kind == "batch_request_link": + raise RuntimeError("private-store-secret") + return original_save(kind, *args, **kwargs) + + monkeypatch.setattr(orchestrator._store, "save", reject_link) batch_client = ObservedBatchClient() coordinator = CostRoutingCoordinator( orchestrator, batch_backend=PgLlmBatchBackend(batch_client) @@ -48,6 +59,8 @@ async def download_results(self, *args, **kwargs): for item_id in ("a", "b") ]}) assert status == 201, submitted + assert submitted["request_link_status"] == ("write_failed" if write_failure else "durable") + assert "private-store-secret" not in str(submitted) status, retrieved = _request( "POST", f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results", "unit-token", @@ -58,6 +71,8 @@ async def download_results(self, *args, **kwargs): assert batch_client.retrieval_request_id assert batch_client.submission_request_id != batch_client.retrieval_request_id assert batch_client.submission_request_id not in {"a", "b"} + assert batch_client.calls.count("create_batch_job") == 1 + assert coordinator._batch_jobs[submitted["job_id"]].job_id == submitted["job_id"] finally: server.shutdown() worker_thread.join() @@ -70,9 +85,12 @@ async def download_results(self, *args, **kwargs): # not a replacement of provider custom_id or one origin per eventual job. links = restored._store.load("batch_request_link") actual_links = { - (row["request_id"], row["batch_job_id"], row["custom_id"]) - for row in links + (row["request_id"], row["batch_job_id"], custom_id) + for row in links for custom_id in row["custom_ids"] } + if write_failure: + assert not actual_links + return assert actual_links == { (batch_client.submission_request_id, submitted["job_id"], item["custom_id"]) for item in retrieved["results"] From 9356ac6201b5f2f46cc2b93dccecbc23e23b5779 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:34:27 +0900 Subject: [PATCH 53/87] fix: retain durable batch submission identity associations --- contextual_orchestrator/batch_routing.py | 1 + contextual_orchestrator/cost_router.py | 18 ++++++++++++++++++ contextual_orchestrator/server.py | 2 ++ 3 files changed, 21 insertions(+) diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index 5650e441d..3097535b3 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -218,6 +218,7 @@ class BatchJob: # Prompt-token fallback estimates are safe metadata, stored atomically with # the job handle rather than retaining submitted prompt text. prompt_token_estimates: Dict[str, int] = field(default_factory=dict) + request_link_status: str = "unavailable" @dataclass diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index 880ac9119..2eb51e65a 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -973,6 +973,7 @@ def submit_batch( requests: List[BatchRequest], metadata: Optional[Dict[str, Any]] = None, owner_id: Optional[str] = None, + request_id: Optional[str] = None, ) -> BatchJob: """Submit a batch, resolve its targets, and bind its authenticated owner.""" try: @@ -997,6 +998,23 @@ def submit_batch( job.owner_id = owner_id job.prompt_token_estimates = prompt_token_estimates self._batch_jobs[job.job_id] = job + if request_id is not None and self.orchestrator._store is not None: + try: + # One append-only submission envelope commits all item links + # together. A later retrieval never rewrites this origin. + self.orchestrator._store.save("batch_request_link", None, { + "request_id": request_id, + "batch_job_id": job.job_id, + "custom_ids": [request.custom_id for request in prepared_requests], + "owner_id": owner_id, + }, durable=True) + except Exception: + # The upstream submission already happened. Preserve its handle + # and report incomplete lineage instead of inviting a resubmit. + job.request_link_status = "write_failed" + else: + job.request_link_status = "durable" + self._batch_jobs[job.job_id] = job return job def _resolve_batch_request(self, request: BatchRequest) -> BatchRequest: diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 9813c5a83..11a8abfee 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -7485,6 +7485,7 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di batch_requests, metadata=metadata, owner_id=security.principal_id(self.headers), + request_id=current_request_id(), ) ) except InvalidBatchModelError as exc: @@ -7505,6 +7506,7 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di "backend": job.backend, "status": job.status, "request_count": job.request_count, + "request_link_status": job.request_link_status, }, 201) return if path.startswith("/api/v1/batch_routing_jobs/") and path.endswith("/results"): From b565173e915f9997de508b541e39537ee23d5785 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:35:13 +0900 Subject: [PATCH 54/87] test: preserve repeated batch origins and standalone compatibility --- tests/test_batch_request_lineage.py | 45 +++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index 8fe57b43b..525bf64f4 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -71,6 +71,12 @@ def reject_link(kind, *args, **kwargs): assert batch_client.retrieval_request_id assert batch_client.submission_request_id != batch_client.retrieval_request_id assert batch_client.submission_request_id not in {"a", "b"} + repeated_status, repeated = _request( + "POST", f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results", + "unit-token", + ) + assert repeated_status == 200 + assert {item["custom_id"] for item in repeated["results"]} == {"a", "b"} assert batch_client.calls.count("create_batch_job") == 1 assert coordinator._batch_jobs[submitted["job_id"]].job_id == submitted["job_id"] finally: @@ -97,3 +103,42 @@ def reject_link(kind, *args, **kwargs): } finally: restored.close() + + +def test_batch_submission_links_keep_job_scoped_item_ids(tmp_path): + """Repeated item identifiers across submissions retain every origin association.""" + from contextual_orchestrator.batch_routing import BatchRequest + + agents = [ModelAgent("worker_one", "mock/worker")] + orchestrator = TaskOrchestrator(agents, state_db=tmp_path / "state.db") + coordinator = CostRoutingCoordinator( + orchestrator, batch_backend=PgLlmBatchBackend(_FakeBatchApiClient()) + ) + try: + for request_id in ("trusted_origin_one", "trusted_origin_two"): + coordinator.submit_batch([ + BatchRequest(messages=[{"role": "user", "content": "Fixture"}], custom_id="a") + ], owner_id="owner_one", request_id=request_id) + links = orchestrator._store.load("batch_request_link") + assert {row["request_id"] for row in links} == {"trusted_origin_one", "trusted_origin_two"} + assert [row["custom_ids"] for row in links] == [["a"], ["a"]] + finally: + orchestrator.close() + + +def test_library_batch_without_state_store_keeps_legacy_submission(): + """Standalone calls explicitly report unavailable durable request lineage.""" + from contextual_orchestrator.batch_routing import BatchRequest + + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")]) + coordinator = CostRoutingCoordinator( + orchestrator, batch_backend=PgLlmBatchBackend(_FakeBatchApiClient()) + ) + try: + job = coordinator.submit_batch([BatchRequest( + messages=[{"role": "user", "content": "Fixture"}], custom_id="a" + )]) + assert job.request_link_status == "unavailable" + assert job.job_id == "batch-789" + finally: + orchestrator.close() From b5baf420626bba9357c5b0f68aec47cd02ec58cb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:35:46 +0900 Subject: [PATCH 55/87] test: reject redundant remote batch registry write --- tests/test_batch_request_lineage.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index 525bf64f4..d20597e42 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -142,3 +142,30 @@ def test_library_batch_without_state_store_keeps_legacy_submission(): assert job.job_id == "batch-789" finally: orchestrator.close() + + +def test_batch_link_does_not_rewrite_submitted_registry_handle(tmp_path): + """Lineage status must not add another failure-prone registry assignment.""" + from contextual_orchestrator.batch_routing import BatchRequest + + class SingleWriteRegistry(dict): + """Reject a redundant second remote-registry assignment.""" + + def __setitem__(self, key, value): + if key in self: + raise RuntimeError("second registry write failed") + super().__setitem__(key, value) + + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + coordinator = CostRoutingCoordinator( + orchestrator, batch_backend=PgLlmBatchBackend(_FakeBatchApiClient()) + ) + coordinator._batch_jobs = SingleWriteRegistry() + try: + job = coordinator.submit_batch([BatchRequest( + messages=[{"role": "user", "content": "Fixture"}], custom_id="a" + )], request_id="trusted_origin_one") + assert job.request_link_status == "durable" + finally: + orchestrator.close() From 6223962421fa48d8e48a519c25496331a6df12db Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:35:59 +0900 Subject: [PATCH 56/87] fix: avoid second remote registry write for lineage status --- contextual_orchestrator/batch_routing.py | 2 ++ contextual_orchestrator/cost_router.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index 3097535b3..d96e03f23 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -218,6 +218,8 @@ class BatchJob: # Prompt-token fallback estimates are safe metadata, stored atomically with # the job handle rather than retaining submitted prompt text. prompt_token_estimates: Dict[str, int] = field(default_factory=dict) + # Submission-response diagnostic; durable truth is the append-only link + # event, not this snapshot in a separately persisted remote registry. request_link_status: str = "unavailable" diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index 2eb51e65a..bc974c9d4 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -1014,7 +1014,6 @@ def submit_batch( job.request_link_status = "write_failed" else: job.request_link_status = "durable" - self._batch_jobs[job.job_id] = job return job def _resolve_batch_request(self, request: BatchRequest) -> BatchRequest: From f61f2067e2be7a9608379132df7d5f99479d8b6a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:36:49 +0900 Subject: [PATCH 57/87] docs: qualify batch lineage event and failure evidence --- AGENTS.md | 4 +++ CLAUDE.md | 4 +++ docs/doctoring/batch_request_lineage.md | 34 +++++++++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index f76a887d8..5d074dd30 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,9 @@ # AGENTS.md +Deferred batch lineage: read `docs/doctoring/batch_request_lineage.md` for the +HTTP reproduction, atomic submission-event projection, and remote/local failure +boundary. Do not retry a remotely submitted job after local lineage failure. + Workflow origin identity and persistence limitations are documented in `docs/doctoring/workflow_request_link.md`; preserve origin on replacements and reload. diff --git a/CLAUDE.md b/CLAUDE.md index 309ed0c7c..03f002367 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,5 +1,9 @@ # CLAUDE.md +Batch request lineage evidence and unresolved registry failure semantics live +in `docs/doctoring/batch_request_lineage.md`; HTTP 201 alone does not establish +durable lineage. Preserve job-scoped item IDs and original submission identity. + See `docs/doctoring/workflow_request_link.md` for request-to-workflow correlation tests, cache semantics, and the distinction between in-memory and durable outcomes. diff --git a/docs/doctoring/batch_request_lineage.md b/docs/doctoring/batch_request_lineage.md index cfedee463..493f36b64 100644 --- a/docs/doctoring/batch_request_lineage.md +++ b/docs/doctoring/batch_request_lineage.md @@ -31,3 +31,37 @@ certifi was absent; this was an environment error, not the RED result. The attem `uv sync --frozen --group test` was invalid because test is an optional extra, not a dependency group; the frozen default synchronization repaired collection. CodeGraph initialization completed (438 files); no other worktree was modified. + +## Candidate implementation and review repair + +`62239624`: focused lineage, batch backend, and cost-review HTTP tests passed +**42 tests in 12.33s**. Includes existing cross-principal denial coverage. +The initial status/failure contract at `919c945a` failed twice in 1.54s; initial +implementation `9356ac62` passed 39 tests in 14.35s. Follow-up repeated-origin, +repeated-retrieval and no-store cases passed 41 tests at `b565173e` in 11.63s. + +The association is an append-only typed event in the existing state store: +`request_id`, `batch_job_id`, `custom_ids`, and `owner_id`. A single event commits +the complete submission cohort using the existing rollback-safe transaction. +Consumers project one association per custom ID; this replaces the original +RED's proposed per-item event shape. It is not a new relational entity claimed +to satisfy 3NF. Item IDs remain job-scoped and repeated submissions append, +never replace prior origins. No prompt or answer enters the event. + +HTTP 201 means the upstream job was submitted, not that all local state is +durable. `request_link_status` is `durable`, `write_failed`, or `unavailable`. +After remote success, a failed association commit returns the original handle +with `write_failed` and no raw exception. It never automatically resubmits. +No-store/library calls remain supported with unavailable lineage. This response +diagnostic is not reconstructed from remote registry snapshots after restart; +the committed association event is the durable evidence. + +Review found an added second registry assignment could lose the successful +remote handle behind a new exception. Regression `b5baf420` failed once in +0.39s; `62239624` removes that assignment. Existing Valkey writes are separate +`hset` and `expire` calls, not atomic with SQLite. Failure of the original +registry assignment after remote submission remains an unresolved existing +ambiguity; no cross-store transaction, complete recovery, retention/export +window, API reconciliation surface, or remote integration is claimed here. +The event is currently only consumed by the test projection. Full-suite, +package, hosted checks, independent approval, and protected release are pending. From 3773f0dc703e5279d85893c7fbbe0bce1071b53d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:37:17 +0900 Subject: [PATCH 58/87] test: distinguish registry snapshots from durable lineage proof --- tests/test_batch_request_lineage.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index d20597e42..b06222001 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -169,3 +169,31 @@ def __setitem__(self, key, value): assert job.request_link_status == "durable" finally: orchestrator.close() + + +def test_valkey_job_snapshot_does_not_prove_lineage_commit(tmp_path): + """Decoded registry status is non-authoritative; committed events supply proof.""" + from contextual_orchestrator.batch_job_registry import ValkeyJsonMapping + from contextual_orchestrator.batch_routing import BatchJob, BatchRequest + from test_batch_job_registry import FakeValkeyClient + + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + coordinator = CostRoutingCoordinator( + orchestrator, batch_backend=PgLlmBatchBackend(_FakeBatchApiClient()) + ) + coordinator._batch_jobs = ValkeyJsonMapping( + FakeValkeyClient(), "jobs", decode=lambda raw: BatchJob(**raw) + ) + try: + response_job = coordinator.submit_batch([BatchRequest( + messages=[{"role": "user", "content": "Fixture"}], custom_id="a" + )], owner_id="owner_one", request_id="trusted_origin_one") + decoded_job = coordinator._batch_jobs[response_job.job_id] + assert decoded_job is not response_job + assert decoded_job.request_link_status == "unavailable" + assert decoded_job.owner_id == "owner_one" + assert response_job.request_link_status == "durable" + assert orchestrator._store.load("batch_request_link")[0]["request_id"] == "trusted_origin_one" + finally: + orchestrator.close() From 6966974aa4a2580085401406aa135be941957d6a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:38:26 +0900 Subject: [PATCH 59/87] test: retain remote batch handle after registry failure --- tests/test_batch_request_lineage.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index b06222001..75a6ee6e5 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -13,7 +13,8 @@ @pytest.mark.parametrize("write_failure", [False, True]) -def test_http_batch_origin_survives_distinct_retrieval_and_reload(tmp_path, monkeypatch, write_failure): +@pytest.mark.parametrize("registry_failure", [False, True]) +def test_http_batch_origin_survives_distinct_retrieval_and_reload(tmp_path, monkeypatch, write_failure, registry_failure): """One submission joins two item outcomes without trusting their custom IDs.""" class ObservedBatchClient(_FakeBatchApiClient): """Reuse the existing offline provider contract with passive identity capture.""" @@ -46,6 +47,14 @@ def reject_link(kind, *args, **kwargs): coordinator = CostRoutingCoordinator( orchestrator, batch_backend=PgLlmBatchBackend(batch_client) ) + if registry_failure: + class RejectingRegistry(dict): + """Simulate a registry write failing after remote acceptance.""" + + def __setitem__(self, key, value): + raise RuntimeError("private-registry-secret") + + coordinator._batch_jobs = RejectingRegistry() server = build_server(orchestrator, port=0, coordinator=coordinator, security=SecurityConfig(auth_token="unit-token")) worker_thread = threading.Thread(target=server.serve_forever, daemon=True) @@ -61,6 +70,16 @@ def reject_link(kind, *args, **kwargs): assert status == 201, submitted assert submitted["request_link_status"] == ("write_failed" if write_failure else "durable") assert "private-store-secret" not in str(submitted) + assert submitted["registry_persistence_status"] == ("write_failed" if registry_failure else "stored") + assert "private-registry-secret" not in str(submitted) + if registry_failure: + assert submitted["job_id"] == "batch-789" + assert batch_client.calls.count("create_batch_job") == 1 + links = orchestrator._store.load("batch_request_link") + assert len(links) == (0 if write_failure else 1) + if links: + assert links[0]["request_id"] == batch_client.submission_request_id + return status, retrieved = _request( "POST", f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results", "unit-token", @@ -191,7 +210,7 @@ def test_valkey_job_snapshot_does_not_prove_lineage_commit(tmp_path): )], owner_id="owner_one", request_id="trusted_origin_one") decoded_job = coordinator._batch_jobs[response_job.job_id] assert decoded_job is not response_job - assert decoded_job.request_link_status == "unavailable" + assert decoded_job.request_link_status == "durable" assert decoded_job.owner_id == "owner_one" assert response_job.request_link_status == "durable" assert orchestrator._store.load("batch_request_link")[0]["request_id"] == "trusted_origin_one" From d95ad53e40e11e0105c7d976f01eba42887ed7de Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:38:50 +0900 Subject: [PATCH 60/87] fix: preserve applied batch outcome across registry failures --- contextual_orchestrator/batch_routing.py | 5 +++-- contextual_orchestrator/cost_router.py | 8 +++++++- contextual_orchestrator/server.py | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index d96e03f23..4dd236e47 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -218,9 +218,10 @@ class BatchJob: # Prompt-token fallback estimates are safe metadata, stored atomically with # the job handle rather than retaining submitted prompt text. prompt_token_estimates: Dict[str, int] = field(default_factory=dict) - # Submission-response diagnostic; durable truth is the append-only link - # event, not this snapshot in a separately persisted remote registry. + # Finalized before the registry snapshot is written; the append-only event + # supplies the durable request association independently of that registry. request_link_status: str = "unavailable" + registry_persistence_status: str = "unavailable" @dataclass diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index bc974c9d4..d113f3b14 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -997,7 +997,6 @@ def submit_batch( job = self.batch_backend.submit(prepared_requests, metadata=metadata) job.owner_id = owner_id job.prompt_token_estimates = prompt_token_estimates - self._batch_jobs[job.job_id] = job if request_id is not None and self.orchestrator._store is not None: try: # One append-only submission envelope commits all item links @@ -1014,6 +1013,13 @@ def submit_batch( job.request_link_status = "write_failed" else: job.request_link_status = "durable" + job.registry_persistence_status = "stored" + try: + self._batch_jobs[job.job_id] = job + except Exception: + # Submission already applied remotely; an HSET/expiry failure may + # itself be partially applied. Return the handle without replay. + job.registry_persistence_status = "write_failed" return job def _resolve_batch_request(self, request: BatchRequest) -> BatchRequest: diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 11a8abfee..320e2ab69 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -7507,6 +7507,7 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di "status": job.status, "request_count": job.request_count, "request_link_status": job.request_link_status, + "registry_persistence_status": job.registry_persistence_status, }, 201) return if path.startswith("/api/v1/batch_routing_jobs/") and path.endswith("/results"): From 369ea1e34dd0f4b3da9672ceae068656af6a69a8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:39:25 +0900 Subject: [PATCH 61/87] fix: keep registry write result outside persisted snapshot --- contextual_orchestrator/batch_routing.py | 4 +++- tests/test_batch_request_lineage.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index 4dd236e47..a46769819 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -221,7 +221,9 @@ class BatchJob: # Finalized before the registry snapshot is written; the append-only event # supplies the durable request association independently of that registry. request_link_status: str = "unavailable" - registry_persistence_status: str = "unavailable" + # Deliberately not a dataclass field: HSET may succeed before expiry fails, + # so an operation result must never be serialized into its own snapshot. + registry_persistence_status = "unavailable" @dataclass diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index 75a6ee6e5..bee6d39e5 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -211,6 +211,8 @@ def test_valkey_job_snapshot_does_not_prove_lineage_commit(tmp_path): decoded_job = coordinator._batch_jobs[response_job.job_id] assert decoded_job is not response_job assert decoded_job.request_link_status == "durable" + assert decoded_job.registry_persistence_status == "unavailable" + assert response_job.registry_persistence_status == "stored" assert decoded_job.owner_id == "owner_one" assert response_job.request_link_status == "durable" assert orchestrator._store.load("batch_request_link")[0]["request_id"] == "trusted_origin_one" From 18af2947ccd5ae8368ce307a8bfb87f6895dd639 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:40:08 +0900 Subject: [PATCH 62/87] test: distinguish absent and partially written batch registry --- tests/test_batch_request_lineage.py | 30 ++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index bee6d39e5..96f0629f8 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -13,7 +13,7 @@ @pytest.mark.parametrize("write_failure", [False, True]) -@pytest.mark.parametrize("registry_failure", [False, True]) +@pytest.mark.parametrize("registry_failure", [None, "hset", "expire"]) def test_http_batch_origin_survives_distinct_retrieval_and_reload(tmp_path, monkeypatch, write_failure, registry_failure): """One submission joins two item outcomes without trusting their custom IDs.""" class ObservedBatchClient(_FakeBatchApiClient): @@ -48,13 +48,25 @@ def reject_link(kind, *args, **kwargs): orchestrator, batch_backend=PgLlmBatchBackend(batch_client) ) if registry_failure: - class RejectingRegistry(dict): - """Simulate a registry write failing after remote acceptance.""" + from contextual_orchestrator.batch_job_registry import ValkeyJsonMapping + from contextual_orchestrator.batch_routing import BatchJob + from test_batch_job_registry import FakeValkeyClient - def __setitem__(self, key, value): + class RejectingClient(FakeValkeyClient): + """Distinguish no registry write from partial HSET-before-expiry.""" + + def hset(self, *args, **kwargs): + if registry_failure == "hset": + raise RuntimeError("private-registry-secret") + return super().hset(*args, **kwargs) + + def expire(self, *args, **kwargs): raise RuntimeError("private-registry-secret") - coordinator._batch_jobs = RejectingRegistry() + registry_client = RejectingClient() + coordinator._batch_jobs = ValkeyJsonMapping( + registry_client, "jobs", decode=lambda raw: BatchJob(**raw) + ) server = build_server(orchestrator, port=0, coordinator=coordinator, security=SecurityConfig(auth_token="unit-token")) worker_thread = threading.Thread(target=server.serve_forever, daemon=True) @@ -75,6 +87,14 @@ def __setitem__(self, key, value): if registry_failure: assert submitted["job_id"] == "batch-789" assert batch_client.calls.count("create_batch_job") == 1 + stored_handles = registry_client.hashes.get("jobs", {}) + assert bool(stored_handles) == (registry_failure == "expire") + denied_status, _ = _request( + "POST", f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results", + "other-owner-token", + ) + assert denied_status in {401, 403} + assert "download_results" not in batch_client.calls links = orchestrator._store.load("batch_request_link") assert len(links) == (0 if write_failure else 1) if links: From b12981d6226a98ef9f7b1166724d6763fbdb582b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:40:28 +0900 Subject: [PATCH 63/87] test: inspect canonical registry key in partial write probe --- tests/test_batch_request_lineage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index 96f0629f8..78fbd2cc2 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -87,7 +87,7 @@ def expire(self, *args, **kwargs): if registry_failure: assert submitted["job_id"] == "batch-789" assert batch_client.calls.count("create_batch_job") == 1 - stored_handles = registry_client.hashes.get("jobs", {}) + stored_handles = registry_client.hashes.get("batch_job_registry:jobs", {}) assert bool(stored_handles) == (registry_failure == "expire") denied_status, _ = _request( "POST", f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results", From d71bcc0aebc3138c2aeece54196bfc336820dc4e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:42:08 +0900 Subject: [PATCH 64/87] test: reproduce missing owner-bound batch restart recovery --- tests/test_batch_request_lineage.py | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index 78fbd2cc2..645a3df37 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -238,3 +238,51 @@ def test_valkey_job_snapshot_does_not_prove_lineage_commit(tmp_path): assert orchestrator._store.load("batch_request_link")[0]["request_id"] == "trusted_origin_one" finally: orchestrator.close() + + +def test_http_batch_failed_registry_recovers_authorized_job_after_restart(tmp_path): + """SQLite recovery binds the original owner without another remote submission.""" + class MissingRegistry(dict): + """Lose the registry assignment, retaining only committed SQLite evidence.""" + + def __setitem__(self, key, value): + raise RuntimeError("registry unavailable") + + state_path = tmp_path / "state.db" + agents = [ModelAgent("worker_one", "mock/worker")] + security = SecurityConfig(bearer_verifier=lambda token, scope: + token in {"owner-one", "owner-two"}) + clients = [] + for restarted in (False, True): + orchestrator = TaskOrchestrator(agents, state_db=state_path) + client = _FakeBatchApiClient() + clients.append(client) + coordinator = CostRoutingCoordinator(orchestrator, + batch_backend=PgLlmBatchBackend(client, endpoint_alias="original-endpoint")) + if not restarted: + coordinator._batch_jobs = MissingRegistry() + server = build_server(orchestrator, port=0, coordinator=coordinator, security=security) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + base_url = f"http://127.0.0.1:{server.server_address[1]}" + try: + if not restarted: + status, submitted = _request("POST", f"{base_url}/api/v1/batch_routing_jobs", + "owner-one", {"requests": [{"custom_id": "a", "model": "mock/worker", + "messages": [{"role": "user", "content": "Never persist this prompt."}]}]}) + assert status == 201 + assert submitted["registry_persistence_status"] == "write_failed" + continue + result_url = f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results" + denied_status, _ = _request("POST", result_url, "owner-two") + assert denied_status == 404 + assert "download_results" not in client.calls + status, retrieved = _request("POST", result_url, "owner-one") + assert status == 200, retrieved + assert retrieved["results"][0]["custom_id"] == "a" + assert sum(item.calls.count("create_batch_job") for item in clients) == 1 + finally: + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() From 6b40cce68a49e9428ca505a9fa1fdb27540d050c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:43:42 +0900 Subject: [PATCH 65/87] fix: recover owner-bound remote batch descriptors from durable events --- contextual_orchestrator/batch_routing.py | 32 ++++++++++++++++++++++++ contextual_orchestrator/cost_router.py | 32 +++++++++++++++++++++--- contextual_orchestrator/orchestrator.py | 9 +++++++ 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index a46769819..6f02f5043 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -470,6 +470,38 @@ def __init__( job_registry.mapping("pg_llm_batch_jobs") if job_registry is not None else {} ) + def recovery_descriptor(self, requests: List[BatchRequest]) -> Dict[str, Any]: + """Describe exact target and item metadata without submitted prompt text.""" + from .cost_ledger import Attribution + return { + "backend_name": self.name, + "endpoint_alias": self._endpoint_alias, + "endpoint": self._endpoint, + "items": [{"custom_id": item.custom_id, "model": item.model, + "mode": item.mode, "attribution": Attribution.from_mapping(item.attribution).as_dict()} + for item in requests], + } + + def restore_descriptor(self, job: BatchJob, descriptor: Dict[str, Any]) -> None: + """Restore prompt-free item identity only for this exact configured target.""" + if (descriptor.get("backend_name") != self.name + or descriptor.get("endpoint_alias") != self._endpoint_alias + or descriptor.get("endpoint") != self._endpoint): + raise ValueError("batch target mismatch") + items = descriptor.get("items") + if not isinstance(items, list) or len(items) != job.request_count: + raise ValueError("batch item count mismatch") + restored = {} + for item in items: + if not isinstance(item, dict) or set(item) != {"custom_id", "model", "mode", "attribution"}: + raise ValueError("invalid batch item descriptor") + if any(not isinstance(item[field], str) or not item[field] for field in ("custom_id", "model", "mode")): + raise ValueError("invalid batch item identity") + if item["custom_id"] in restored or not isinstance(item["attribution"], dict): + raise ValueError("invalid batch item metadata") + restored[item["custom_id"]] = {**item, "messages": []} + self._jobs[job.job_id] = {"endpoint_alias": self._endpoint_alias, "requests": restored} + def _assemble_payload(self, requests: List[BatchRequest]) -> str: if self._assembler is not None: return self._assembler.assemble( diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index d113f3b14..d9fd06f93 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -20,8 +20,9 @@ import hashlib import re +import time from contextvars import ContextVar -from dataclasses import replace +from dataclasses import asdict, replace from threading import Lock from typing import Any, Dict, List, Optional from .decision_receipts import record_initial_selection @@ -38,10 +39,11 @@ LocalBatchBackend, LocalEmbeddingBatchBackend, ProviderEmbeddingBatchBackend, + PgLlmBatchBackend, RoutingHints, RoutingPolicy, ) -from .batch_job_registry import ClaimNotAcquired, JobRegistryFactory, build_job_registry +from .batch_job_registry import ClaimNotAcquired, JobRegistryFactory, build_job_registry, DEFAULT_RETENTION_SECONDS from .cost_ledger import CostLedger, PriceBook, PriceEntry from .kv_config import InMemoryConfigStore from .model_discovery import _currency_is_comparable @@ -1001,11 +1003,16 @@ def submit_batch( try: # One append-only submission envelope commits all item links # together. A later retrieval never rewrites this origin. - self.orchestrator._store.save("batch_request_link", None, { + self.orchestrator._store.save("batch_request_link", job.job_id, { "request_id": request_id, "batch_job_id": job.job_id, "custom_ids": [request.custom_id for request in prepared_requests], "owner_id": owner_id, + "recovery_descriptor": ({ + "job": asdict(job), + "expires_at": job.submitted_at + DEFAULT_RETENTION_SECONDS, + "backend": self.batch_backend.recovery_descriptor(prepared_requests), + } if isinstance(self.batch_backend, PgLlmBatchBackend) else None), }, durable=True) except Exception: # The upstream submission already happened. Preserve its handle @@ -1311,6 +1318,25 @@ def _resolve_batch_provider_model(self, item: BatchResultItem) -> tuple[str, str def _require_job(self, job_id: str, *, owner_id: Optional[str] = None) -> BatchJob: job = self._batch_jobs.get(job_id) + if job is None and owner_id is not None and self.orchestrator._store is not None: + record = self.orchestrator._store.load_latest_key("batch_request_link", job_id) + if (isinstance(record, dict) and record.get("owner_id") == owner_id + and record.get("batch_job_id") == job_id + and isinstance(self.batch_backend, PgLlmBatchBackend)): + descriptor = record.get("recovery_descriptor") + try: + if not isinstance(descriptor, dict) or type(descriptor.get("expires_at")) is not int: + raise ValueError("invalid descriptor") + if descriptor["expires_at"] <= time.time(): + raise ValueError("expired descriptor") + recovered = BatchJob(**descriptor["job"]) + if recovered.job_id != job_id or recovered.owner_id != owner_id or recovered.backend != self.batch_backend.name: + raise ValueError("mismatched descriptor") + self.batch_backend.restore_descriptor(recovered, descriptor["backend"]) + recovered.request_link_status = "durable" + job = recovered + except (KeyError, TypeError, ValueError): + job = None if job is None or job.owner_id != owner_id: raise KeyError(f"batch job {job_id!r} not found") return job diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 007282a13..5030f42d6 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -3863,6 +3863,15 @@ def load(self, kind: str, limit: int | None = None) -> list[dict[str, Any]]: rows = list(reversed(rows)) return [json.loads(row[0]) for row in rows] + def load_latest_key(self, kind: str, key: str) -> dict[str, Any] | None: + """Read one exact-key durable event using the existing identity index.""" + with self._lock: + row = self._conn.execute( + "SELECT payload FROM orchestration_records WHERE kind = ? AND key = ? " + "ORDER BY seq DESC LIMIT 1", (kind, key), + ).fetchone() + return json.loads(row[0]) if row is not None else None + def prune_keyed(self, kind: str, retained_keys: set[str]) -> None: """Delete keyed rows outside the caller's bounded in-memory set.""" if kind not in self._KEYED: From e8d8c8aea4fd7972d6c835de67957873f3d85391 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:44:12 +0900 Subject: [PATCH 66/87] fix: use existing attribution dimensions normalizer --- contextual_orchestrator/batch_routing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index 6f02f5043..a8391332f 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -472,13 +472,13 @@ def __init__( def recovery_descriptor(self, requests: List[BatchRequest]) -> Dict[str, Any]: """Describe exact target and item metadata without submitted prompt text.""" - from .cost_ledger import Attribution + from .cost_ledger import AttributionDimensions return { "backend_name": self.name, "endpoint_alias": self._endpoint_alias, "endpoint": self._endpoint, "items": [{"custom_id": item.custom_id, "model": item.model, - "mode": item.mode, "attribution": Attribution.from_mapping(item.attribution).as_dict()} + "mode": item.mode, "attribution": AttributionDimensions.from_mapping(item.attribution).as_dict()} for item in requests], } From 65b416030aef0795d12b327607025481671a5da8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:45:08 +0900 Subject: [PATCH 67/87] test: reject unsafe batch recovery descriptors and result identities --- tests/test_batch_request_lineage.py | 37 ++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index 645a3df37..b7cbfcb1c 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -240,7 +240,8 @@ def test_valkey_job_snapshot_does_not_prove_lineage_commit(tmp_path): orchestrator.close() -def test_http_batch_failed_registry_recovers_authorized_job_after_restart(tmp_path): +@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage"]) +def test_http_batch_failed_registry_recovers_authorized_job_after_restart(tmp_path, recovery_case): """SQLite recovery binds the original owner without another remote submission.""" class MissingRegistry(dict): """Lose the registry assignment, retaining only committed SQLite evidence.""" @@ -255,10 +256,23 @@ def __setitem__(self, key, value): clients = [] for restarted in (False, True): orchestrator = TaskOrchestrator(agents, state_db=state_path) - client = _FakeBatchApiClient() + class RecoveryClient(_FakeBatchApiClient): + """Return controlled result variants without changing routing execution.""" + + async def download_results(self, *args, **kwargs): + result = await super().download_results(*args, **kwargs) + if recovery_case == "unexpected_item": + result["responses"][0]["custom_id"] = "unsubmitted-item" + if recovery_case == "missing_usage": + result["responses"][0]["response"]["body"].pop("usage") + return result + + client = RecoveryClient() clients.append(client) coordinator = CostRoutingCoordinator(orchestrator, - batch_backend=PgLlmBatchBackend(client, endpoint_alias="original-endpoint")) + batch_backend=PgLlmBatchBackend(client, endpoint_alias=( + "changed-endpoint" if restarted and recovery_case == "backend_mismatch" + else "original-endpoint"))) if not restarted: coordinator._batch_jobs = MissingRegistry() server = build_server(orchestrator, port=0, coordinator=coordinator, security=security) @@ -272,14 +286,31 @@ def __setitem__(self, key, value): "messages": [{"role": "user", "content": "Never persist this prompt."}]}]}) assert status == 201 assert submitted["registry_persistence_status"] == "write_failed" + record = orchestrator._store.load("batch_request_link")[0] + assert "Never persist this prompt." not in str(record) + if recovery_case == "expired": + record["recovery_descriptor"]["expires_at"] = 0 + if recovery_case == "malformed": + record["recovery_descriptor"]["backend"] = [] + if recovery_case in {"expired", "malformed"}: + orchestrator._store.save("batch_request_link", submitted["job_id"], record, durable=True) continue result_url = f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results" denied_status, _ = _request("POST", result_url, "owner-two") assert denied_status == 404 assert "download_results" not in client.calls status, retrieved = _request("POST", result_url, "owner-one") + if recovery_case in {"expired", "malformed", "backend_mismatch"}: + assert status == 404, retrieved + assert "download_results" not in client.calls + continue + if recovery_case == "unexpected_item": + assert status != 200 + continue assert status == 200, retrieved assert retrieved["results"][0]["custom_id"] == "a" + if recovery_case == "missing_usage": + assert retrieved["results"][0]["measurement_status"] != "measured" assert sum(item.calls.count("create_batch_job") for item in clients) == 1 finally: server.shutdown() From 853e860946d2d55a06d824e042057806c3ad79d8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:45:39 +0900 Subject: [PATCH 68/87] fix: validate batch recovery shape and preserve configured expiry --- contextual_orchestrator/batch_job_registry.py | 5 +++++ contextual_orchestrator/batch_routing.py | 2 +- contextual_orchestrator/cost_router.py | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/contextual_orchestrator/batch_job_registry.py b/contextual_orchestrator/batch_job_registry.py index 21c7fc8a7..25a8a56a9 100644 --- a/contextual_orchestrator/batch_job_registry.py +++ b/contextual_orchestrator/batch_job_registry.py @@ -199,6 +199,11 @@ def __init__(self, client: Any = None, *, retention_seconds: int = DEFAULT_RETEN ) self._local_locks_guard = threading.Lock() + @property + def retention_seconds(self) -> int: + """Return the configured registry retention for durable recovery expiry.""" + return self._retention_seconds + def lock( self, name: str, diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index a8391332f..7ba494311 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -484,7 +484,7 @@ def recovery_descriptor(self, requests: List[BatchRequest]) -> Dict[str, Any]: def restore_descriptor(self, job: BatchJob, descriptor: Dict[str, Any]) -> None: """Restore prompt-free item identity only for this exact configured target.""" - if (descriptor.get("backend_name") != self.name + if (not isinstance(descriptor, dict) or descriptor.get("backend_name") != self.name or descriptor.get("endpoint_alias") != self._endpoint_alias or descriptor.get("endpoint") != self._endpoint): raise ValueError("batch target mismatch") diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index d9fd06f93..ae632b4c0 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -43,7 +43,7 @@ RoutingHints, RoutingPolicy, ) -from .batch_job_registry import ClaimNotAcquired, JobRegistryFactory, build_job_registry, DEFAULT_RETENTION_SECONDS +from .batch_job_registry import ClaimNotAcquired, JobRegistryFactory, build_job_registry from .cost_ledger import CostLedger, PriceBook, PriceEntry from .kv_config import InMemoryConfigStore from .model_discovery import _currency_is_comparable @@ -1010,7 +1010,7 @@ def submit_batch( "owner_id": owner_id, "recovery_descriptor": ({ "job": asdict(job), - "expires_at": job.submitted_at + DEFAULT_RETENTION_SECONDS, + "expires_at": job.submitted_at + self._job_registry.retention_seconds, "backend": self.batch_backend.recovery_descriptor(prepared_requests), } if isinstance(self.batch_backend, PgLlmBatchBackend) else None), }, durable=True) From b4abdd660b0ee44fd1a5b7a8b8c7a7ad5d1e1f92 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:46:34 +0900 Subject: [PATCH 69/87] docs: record owner-bound batch restart recovery evidence --- docs/doctoring/batch_request_lineage.md | 40 +++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/doctoring/batch_request_lineage.md b/docs/doctoring/batch_request_lineage.md index 493f36b64..f8a9dd9a3 100644 --- a/docs/doctoring/batch_request_lineage.md +++ b/docs/doctoring/batch_request_lineage.md @@ -65,3 +65,43 @@ ambiguity; no cross-store transaction, complete recovery, retention/export window, API reconciliation surface, or remote integration is claimed here. The event is currently only consumed by the test projection. Full-suite, package, hosted checks, independent approval, and protected release are pending. + +## Recovery successor (supersedes status-only limitations above) + +Actual HTTP restart regression `d71bcc0a` failed once in 1.96s: an accepted +remote job returned 201 despite a failed registry write, but its rightful owner +received 404 after restart. A different authenticated owner also received 404. +This established unrecoverable work, not just missing status metadata. + +Candidate `853e8609` passes **73 focused tests in 21.17s** across lineage, +batch routing, cost-review HTTP, and registry files. New coverage distinguishes +HSET failure (no handle stored) from expiry failure (HSET partially applied), +and returns the original upstream handle without a second submission. Earlier +`18af2947` expiry tests incorrectly inspected an unprefixed fake-registry key; +`b12981d6` corrected the test and passed ten cases in 6.18s. + +The single durable submission event now includes a prompt-free recovery +descriptor: exact backend name/endpoint alias/endpoint, owner-bound job snapshot, +original token estimates, expected job-scoped item IDs, model/mode/normalized +attribution, and expiry tied to the configured registry retention. This is only +supported for PgLlmBatchBackend, not arbitrary local or embedding backends. +No prompts or credentials are included. Absent source messages remain unavailable; +their contents are never fabricated for token estimates. + +An indexed exact-key lookup permits the original owner to recover a missing +registry handle after SQLite reload. Wrong owner, expired descriptor, malformed +backend object, or changed backend target fail closed. Restored expected item +IDs retain the existing response-identity validator; unexpected result IDs are +rejected. Missing usage remains estimated or unavailable, never relabeled measured. +The malformed-backend test initially returned 500 at `65b41603` (one failure, +five passes in 9.90s); explicit type validation changed that to a concealed 404. + +Request-link status is finalized before the single registry write and survives +Valkey decoding consistently. Registry-write outcome is response-only and is +excluded from dataclass serialization, since HSET may apply before expiry fails. +`stored` means that configured registry operation returned, not proof of process +restart durability for a local dictionary. HTTP 201 still means remote submission; +clients must preserve the handle and must not resubmit solely because local +persistence is incomplete. If both SQLite and the registry fail, automatic recovery +remains unavailable. This is a tested recovery slice, not complete cross-store +atomicity, retention cleanup, production integration, or customer KPI evidence. From e372bc542f8fbec8d047a9affae15c018698d48a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:47:39 +0900 Subject: [PATCH 70/87] test: reject inconsistent recovery identities and survive registry outage --- tests/test_batch_request_lineage.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index b7cbfcb1c..30f747b9d 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -240,7 +240,7 @@ def test_valkey_job_snapshot_does_not_prove_lineage_commit(tmp_path): orchestrator.close() -@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage"]) +@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch"]) def test_http_batch_failed_registry_recovers_authorized_job_after_restart(tmp_path, recovery_case): """SQLite recovery binds the original owner without another remote submission.""" class MissingRegistry(dict): @@ -275,6 +275,15 @@ async def download_results(self, *args, **kwargs): else "original-endpoint"))) if not restarted: coordinator._batch_jobs = MissingRegistry() + elif recovery_case == "registry_outage": + class UnavailableRegistry(MissingRegistry): + """All reads and writes remain unavailable during recovery.""" + + def get(self, *args, **kwargs): + raise RuntimeError("registry still unavailable") + + coordinator._batch_jobs = UnavailableRegistry() + coordinator.batch_backend._jobs = UnavailableRegistry() server = build_server(orchestrator, port=0, coordinator=coordinator, security=security) worker = threading.Thread(target=server.serve_forever, daemon=True) worker.start() @@ -292,7 +301,11 @@ async def download_results(self, *args, **kwargs): record["recovery_descriptor"]["expires_at"] = 0 if recovery_case == "malformed": record["recovery_descriptor"]["backend"] = [] - if recovery_case in {"expired", "malformed"}: + if recovery_case == "item_mismatch": + record["custom_ids"] = ["different-original-item"] + if recovery_case == "estimate_mismatch": + record["recovery_descriptor"]["job"]["prompt_token_estimates"] = {"different-item": 99} + if recovery_case in {"expired", "malformed", "item_mismatch", "estimate_mismatch"}: orchestrator._store.save("batch_request_link", submitted["job_id"], record, durable=True) continue result_url = f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results" @@ -300,7 +313,7 @@ async def download_results(self, *args, **kwargs): assert denied_status == 404 assert "download_results" not in client.calls status, retrieved = _request("POST", result_url, "owner-one") - if recovery_case in {"expired", "malformed", "backend_mismatch"}: + if recovery_case in {"expired", "malformed", "backend_mismatch", "item_mismatch", "estimate_mismatch"}: assert status == 404, retrieved assert "download_results" not in client.calls continue From 7fb1a71ce1b4c14d1ba29e12501ba22fb589cca9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:48:17 +0900 Subject: [PATCH 71/87] fix: isolate authorized batch recovery from unavailable registry --- contextual_orchestrator/batch_routing.py | 13 +++++++++++-- contextual_orchestrator/cost_router.py | 19 +++++++++++++++++-- tests/test_batch_request_lineage.py | 2 +- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index 7ba494311..497da1424 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -224,6 +224,7 @@ class BatchJob: # Deliberately not a dataclass field: HSET may succeed before expiry fails, # so an operation result must never be serialized into its own snapshot. registry_persistence_status = "unavailable" + recovered_request_metadata = None @dataclass @@ -459,11 +460,15 @@ def __init__( endpoint: str = "/v1/chat/completions", payload_assembler: Any = None, job_registry: Any = None, + recovery_identity: str | None = None, ) -> None: self._client = client self._endpoint_alias = endpoint_alias self._endpoint = endpoint self._assembler = payload_assembler + if recovery_identity is not None and (not isinstance(recovery_identity, str) or not recovery_identity.strip()): + raise ValueError("recovery identity must be a nonempty operator-controlled identifier") + self._recovery_identity = recovery_identity # Tracked requests survive a restart when a Valkey-backed registry # is injected; a plain dict preserves the historical behavior. self._jobs: Dict[str, Dict[str, Any]] = ( @@ -474,6 +479,7 @@ def recovery_descriptor(self, requests: List[BatchRequest]) -> Dict[str, Any]: """Describe exact target and item metadata without submitted prompt text.""" from .cost_ledger import AttributionDimensions return { + "recovery_identity": self._recovery_identity, "backend_name": self.name, "endpoint_alias": self._endpoint_alias, "endpoint": self._endpoint, @@ -485,6 +491,8 @@ def recovery_descriptor(self, requests: List[BatchRequest]) -> Dict[str, Any]: def restore_descriptor(self, job: BatchJob, descriptor: Dict[str, Any]) -> None: """Restore prompt-free item identity only for this exact configured target.""" if (not isinstance(descriptor, dict) or descriptor.get("backend_name") != self.name + or self._recovery_identity is None + or descriptor.get("recovery_identity") != self._recovery_identity or descriptor.get("endpoint_alias") != self._endpoint_alias or descriptor.get("endpoint") != self._endpoint): raise ValueError("batch target mismatch") @@ -500,7 +508,7 @@ def restore_descriptor(self, job: BatchJob, descriptor: Dict[str, Any]) -> None: if item["custom_id"] in restored or not isinstance(item["attribution"], dict): raise ValueError("invalid batch item metadata") restored[item["custom_id"]] = {**item, "messages": []} - self._jobs[job.job_id] = {"endpoint_alias": self._endpoint_alias, "requests": restored} + job.recovered_request_metadata = restored def _assemble_payload(self, requests: List[BatchRequest]) -> str: if self._assembler is not None: @@ -580,7 +588,8 @@ async def _download() -> Dict[str, Any]: reason, ) raise BatchDownloadError(job.job_id, reason) - tracked = self._jobs.get(job.job_id, {}).get("requests", {}) + tracked = (job.recovered_request_metadata if job.recovered_request_metadata is not None + else self._jobs.get(job.job_id, {}).get("requests", {})) responses = _validated_download_responses( payload, expected_custom_ids=set(tracked), diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index ae632b4c0..93cdfea84 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -1077,7 +1077,7 @@ def retrieve_batch(self, job_id: str, *, owner_id: Optional[str] = None) -> Dict not self._batch_item_usage_valid(item) and item.custom_id not in prompt_token_estimates for item in items - ) + ) and job.recovered_request_metadata is None request_by_custom_id = ( self._legacy_batch_requests(job) if needs_legacy_lookup else {} ) @@ -1317,7 +1317,10 @@ def _resolve_batch_provider_model(self, item: BatchResultItem) -> tuple[str, str return provider, item.model def _require_job(self, job_id: str, *, owner_id: Optional[str] = None) -> BatchJob: - job = self._batch_jobs.get(job_id) + try: + job = self._batch_jobs.get(job_id) + except Exception: + job = None if job is None and owner_id is not None and self.orchestrator._store is not None: record = self.orchestrator._store.load_latest_key("batch_request_link", job_id) if (isinstance(record, dict) and record.get("owner_id") == owner_id @@ -1332,7 +1335,19 @@ def _require_job(self, job_id: str, *, owner_id: Optional[str] = None) -> BatchJ recovered = BatchJob(**descriptor["job"]) if recovered.job_id != job_id or recovered.owner_id != owner_id or recovered.backend != self.batch_backend.name: raise ValueError("mismatched descriptor") + custom_ids = record.get("custom_ids") + if (type(recovered.request_count) is not int or recovered.request_count < 1 + or not isinstance(custom_ids, list) + or any(not isinstance(item, str) or not item for item in custom_ids) + or len(set(custom_ids)) != recovered.request_count): + raise ValueError("invalid recovery item identities") + estimates = recovered.prompt_token_estimates + if (not isinstance(estimates, dict) or not set(estimates).issubset(custom_ids) + or any(type(value) is not int or value < 0 for value in estimates.values())): + raise ValueError("invalid recovery estimates") self.batch_backend.restore_descriptor(recovered, descriptor["backend"]) + if set(recovered.recovered_request_metadata) != set(custom_ids): + raise ValueError("mismatched recovery items") recovered.request_link_status = "durable" job = recovered except (KeyError, TypeError, ValueError): diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index 30f747b9d..329075904 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -272,7 +272,7 @@ async def download_results(self, *args, **kwargs): coordinator = CostRoutingCoordinator(orchestrator, batch_backend=PgLlmBatchBackend(client, endpoint_alias=( "changed-endpoint" if restarted and recovery_case == "backend_mismatch" - else "original-endpoint"))) + else "original-endpoint"), recovery_identity="unit-deployment-account")) if not restarted: coordinator._batch_jobs = MissingRegistry() elif recovery_case == "registry_outage": From 83394afa94e22feb073bb76b0035cd32b5d841ba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:49:11 +0900 Subject: [PATCH 72/87] test: bind recovery to deployment and reject malformed job fields --- tests/test_batch_request_lineage.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index 329075904..a65b24904 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -240,7 +240,7 @@ def test_valkey_job_snapshot_does_not_prove_lineage_commit(tmp_path): orchestrator.close() -@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch"]) +@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity"]) def test_http_batch_failed_registry_recovers_authorized_job_after_restart(tmp_path, recovery_case): """SQLite recovery binds the original owner without another remote submission.""" class MissingRegistry(dict): @@ -272,7 +272,10 @@ async def download_results(self, *args, **kwargs): coordinator = CostRoutingCoordinator(orchestrator, batch_backend=PgLlmBatchBackend(client, endpoint_alias=( "changed-endpoint" if restarted and recovery_case == "backend_mismatch" - else "original-endpoint"), recovery_identity="unit-deployment-account")) + else "original-endpoint"), recovery_identity=( + None if restarted and recovery_case == "missing_identity" else + "different-deployment" if restarted and recovery_case == "deployment_mismatch" + else "unit-deployment-account"))) if not restarted: coordinator._batch_jobs = MissingRegistry() elif recovery_case == "registry_outage": @@ -305,7 +308,11 @@ def get(self, *args, **kwargs): record["custom_ids"] = ["different-original-item"] if recovery_case == "estimate_mismatch": record["recovery_descriptor"]["job"]["prompt_token_estimates"] = {"different-item": 99} - if recovery_case in {"expired", "malformed", "item_mismatch", "estimate_mismatch"}: + if recovery_case == "null_estimates": + record["recovery_descriptor"]["job"]["prompt_token_estimates"] = None + if recovery_case == "boolean_count": + record["recovery_descriptor"]["job"]["request_count"] = True + if recovery_case in {"expired", "malformed", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count"}: orchestrator._store.save("batch_request_link", submitted["job_id"], record, durable=True) continue result_url = f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results" @@ -313,7 +320,7 @@ def get(self, *args, **kwargs): assert denied_status == 404 assert "download_results" not in client.calls status, retrieved = _request("POST", result_url, "owner-one") - if recovery_case in {"expired", "malformed", "backend_mismatch", "item_mismatch", "estimate_mismatch"}: + if recovery_case in {"expired", "malformed", "backend_mismatch", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity"}: assert status == 404, retrieved assert "download_results" not in client.calls continue From f4d036bfb2ce6839d78610e73e17f32618d9f073 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:50:03 +0900 Subject: [PATCH 73/87] test: recover backend metadata despite healthy coordinator handle --- tests/test_batch_request_lineage.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index a65b24904..c51fb49eb 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -240,7 +240,7 @@ def test_valkey_job_snapshot_does_not_prove_lineage_commit(tmp_path): orchestrator.close() -@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity"]) +@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "coordinator_hit"]) def test_http_batch_failed_registry_recovers_authorized_job_after_restart(tmp_path, recovery_case): """SQLite recovery binds the original owner without another remote submission.""" class MissingRegistry(dict): @@ -278,6 +278,9 @@ async def download_results(self, *args, **kwargs): else "unit-deployment-account"))) if not restarted: coordinator._batch_jobs = MissingRegistry() + elif recovery_case == "coordinator_hit": + from contextual_orchestrator.batch_routing import BatchJob + coordinator._batch_jobs[submitted["job_id"]] = BatchJob(**record["recovery_descriptor"]["job"]) elif recovery_case == "registry_outage": class UnavailableRegistry(MissingRegistry): """All reads and writes remain unavailable during recovery.""" From 6cb530a7396285f4cb69e30e150cdf06fdcba453 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:50:29 +0900 Subject: [PATCH 74/87] fix: recover partial registry state and expose descriptor availability --- contextual_orchestrator/batch_routing.py | 6 ++++++ contextual_orchestrator/cost_router.py | 9 ++++++++- contextual_orchestrator/server.py | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index 497da1424..5af23bc79 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -221,6 +221,7 @@ class BatchJob: # Finalized before the registry snapshot is written; the append-only event # supplies the durable request association independently of that registry. request_link_status: str = "unavailable" + recovery_status: str = "unavailable" # Deliberately not a dataclass field: HSET may succeed before expiry fails, # so an operation result must never be serialized into its own snapshot. registry_persistence_status = "unavailable" @@ -475,6 +476,11 @@ def __init__( job_registry.mapping("pg_llm_batch_jobs") if job_registry is not None else {} ) + @property + def recovery_enabled(self) -> bool: + """Whether the operator supplied a stable deployment/account binding.""" + return self._recovery_identity is not None + def recovery_descriptor(self, requests: List[BatchRequest]) -> Dict[str, Any]: """Describe exact target and item metadata without submitted prompt text.""" from .cost_ledger import AttributionDimensions diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index 93cdfea84..ab4559bef 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -1020,6 +1020,8 @@ def submit_batch( job.request_link_status = "write_failed" else: job.request_link_status = "durable" + if isinstance(self.batch_backend, PgLlmBatchBackend) and self.batch_backend.recovery_enabled: + job.recovery_status = "durable_descriptor" job.registry_persistence_status = "stored" try: self._batch_jobs[job.job_id] = job @@ -1321,7 +1323,11 @@ def _require_job(self, job_id: str, *, owner_id: Optional[str] = None) -> BatchJ job = self._batch_jobs.get(job_id) except Exception: job = None - if job is None and owner_id is not None and self.orchestrator._store is not None: + if job is not None and job.owner_id != owner_id: + raise KeyError(f"batch job {job_id!r} not found") + if (owner_id is not None and self.orchestrator._store is not None + and (job is None or (isinstance(self.batch_backend, PgLlmBatchBackend) + and self.batch_backend.recovery_enabled))): record = self.orchestrator._store.load_latest_key("batch_request_link", job_id) if (isinstance(record, dict) and record.get("owner_id") == owner_id and record.get("batch_job_id") == job_id @@ -1349,6 +1355,7 @@ def _require_job(self, job_id: str, *, owner_id: Optional[str] = None) -> BatchJ if set(recovered.recovered_request_metadata) != set(custom_ids): raise ValueError("mismatched recovery items") recovered.request_link_status = "durable" + recovered.recovery_status = "durable_descriptor" job = recovered except (KeyError, TypeError, ValueError): job = None diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 320e2ab69..c9fb6c2aa 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -7508,6 +7508,7 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di "request_count": job.request_count, "request_link_status": job.request_link_status, "registry_persistence_status": job.registry_persistence_status, + "recovery_status": job.recovery_status, }, 201) return if path.startswith("/api/v1/batch_routing_jobs/") and path.endswith("/results"): From 42ad396c8f768693faa0b678d97a450f675225df Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:51:27 +0900 Subject: [PATCH 75/87] test: reject duplicate persisted items and verify recovery polling --- tests/test_batch_request_lineage.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index c51fb49eb..87dd1a3ad 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -82,6 +82,7 @@ def expire(self, *args, **kwargs): assert status == 201, submitted assert submitted["request_link_status"] == ("write_failed" if write_failure else "durable") assert "private-store-secret" not in str(submitted) + assert submitted["recovery_status"] == "unavailable" assert submitted["registry_persistence_status"] == ("write_failed" if registry_failure else "stored") assert "private-registry-secret" not in str(submitted) if registry_failure: @@ -240,7 +241,7 @@ def test_valkey_job_snapshot_does_not_prove_lineage_commit(tmp_path): orchestrator.close() -@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "coordinator_hit"]) +@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "coordinator_hit", "duplicate_ids"]) def test_http_batch_failed_registry_recovers_authorized_job_after_restart(tmp_path, recovery_case): """SQLite recovery binds the original owner without another remote submission.""" class MissingRegistry(dict): @@ -301,6 +302,7 @@ def get(self, *args, **kwargs): "messages": [{"role": "user", "content": "Never persist this prompt."}]}]}) assert status == 201 assert submitted["registry_persistence_status"] == "write_failed" + assert submitted["recovery_status"] == "durable_descriptor" record = orchestrator._store.load("batch_request_link")[0] assert "Never persist this prompt." not in str(record) if recovery_case == "expired": @@ -315,7 +317,9 @@ def get(self, *args, **kwargs): record["recovery_descriptor"]["job"]["prompt_token_estimates"] = None if recovery_case == "boolean_count": record["recovery_descriptor"]["job"]["request_count"] = True - if recovery_case in {"expired", "malformed", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count"}: + if recovery_case == "duplicate_ids": + record["custom_ids"] = ["a", "a"] + if recovery_case in {"expired", "malformed", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "duplicate_ids"}: orchestrator._store.save("batch_request_link", submitted["job_id"], record, durable=True) continue result_url = f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results" @@ -323,7 +327,7 @@ def get(self, *args, **kwargs): assert denied_status == 404 assert "download_results" not in client.calls status, retrieved = _request("POST", result_url, "owner-one") - if recovery_case in {"expired", "malformed", "backend_mismatch", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity"}: + if recovery_case in {"expired", "malformed", "backend_mismatch", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "duplicate_ids"}: assert status == 404, retrieved assert "download_results" not in client.calls continue @@ -331,6 +335,8 @@ def get(self, *args, **kwargs): assert status != 200 continue assert status == 200, retrieved + poll_status, _ = _request("GET", result_url.removesuffix("/results"), "owner-one") + assert poll_status == 200 assert retrieved["results"][0]["custom_id"] == "a" if recovery_case == "missing_usage": assert retrieved["results"][0]["measurement_status"] != "measured" From aaa9b13399aa4b8120c6ee1cf3ba01655e9e4355 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:51:40 +0900 Subject: [PATCH 76/87] fix: require exact persisted batch item cardinality --- contextual_orchestrator/cost_router.py | 1 + 1 file changed, 1 insertion(+) diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index ab4559bef..0323f2baa 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -1345,6 +1345,7 @@ def _require_job(self, job_id: str, *, owner_id: Optional[str] = None) -> BatchJ if (type(recovered.request_count) is not int or recovered.request_count < 1 or not isinstance(custom_ids, list) or any(not isinstance(item, str) or not item for item in custom_ids) + or len(custom_ids) != recovered.request_count or len(set(custom_ids)) != recovered.request_count): raise ValueError("invalid recovery item identities") estimates = recovered.prompt_token_estimates From 17cfa6112d75612ee686f85e0660aee56dee34c1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:53:06 +0900 Subject: [PATCH 77/87] test: retain applied remote submission after backend registry failure --- tests/test_batch_request_lineage.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index 87dd1a3ad..ff073c698 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -241,7 +241,7 @@ def test_valkey_job_snapshot_does_not_prove_lineage_commit(tmp_path): orchestrator.close() -@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "coordinator_hit", "duplicate_ids"]) +@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "coordinator_hit", "duplicate_ids", "backend_write_outage"]) def test_http_batch_failed_registry_recovers_authorized_job_after_restart(tmp_path, recovery_case): """SQLite recovery binds the original owner without another remote submission.""" class MissingRegistry(dict): @@ -278,7 +278,10 @@ async def download_results(self, *args, **kwargs): "different-deployment" if restarted and recovery_case == "deployment_mismatch" else "unit-deployment-account"))) if not restarted: - coordinator._batch_jobs = MissingRegistry() + if recovery_case == "backend_write_outage": + coordinator.batch_backend._jobs = MissingRegistry() + else: + coordinator._batch_jobs = MissingRegistry() elif recovery_case == "coordinator_hit": from contextual_orchestrator.batch_routing import BatchJob coordinator._batch_jobs[submitted["job_id"]] = BatchJob(**record["recovery_descriptor"]["job"]) @@ -301,7 +304,10 @@ def get(self, *args, **kwargs): "owner-one", {"requests": [{"custom_id": "a", "model": "mock/worker", "messages": [{"role": "user", "content": "Never persist this prompt."}]}]}) assert status == 201 - assert submitted["registry_persistence_status"] == "write_failed" + assert submitted["registry_persistence_status"] == ( + "stored" if recovery_case == "backend_write_outage" else "write_failed") + if recovery_case == "backend_write_outage": + assert submitted["backend_registry_persistence_status"] == "write_failed" assert submitted["recovery_status"] == "durable_descriptor" record = orchestrator._store.load("batch_request_link")[0] assert "Never persist this prompt." not in str(record) From 8ddfeb9f641984a4e0920d7059fe608fa7ba3292 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:53:34 +0900 Subject: [PATCH 78/87] fix: return applied batch handle despite backend registry failure --- contextual_orchestrator/batch_routing.py | 21 ++++++++++++++------- contextual_orchestrator/server.py | 1 + 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index 5af23bc79..df5c61ed3 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -225,6 +225,7 @@ class BatchJob: # Deliberately not a dataclass field: HSET may succeed before expiry fails, # so an operation result must never be serialized into its own snapshot. registry_persistence_status = "unavailable" + backend_registry_persistence_status = "unavailable" recovered_request_metadata = None @@ -548,18 +549,24 @@ async def _submit() -> Dict[str, Any]: # Tracked requests are stored as JSON primitives (not dataclass # instances) so the registry can be a JSON-backed Valkey mapping; # retrieve() rebuilds the dataclass view it needs. - self._jobs[batch_id] = { - "endpoint_alias": self._endpoint_alias, - "requests": { - request.custom_id: dataclasses.asdict(request) for request in requests - }, - } - return BatchJob( + registry_status = "stored" + try: + self._jobs[batch_id] = { + "endpoint_alias": self._endpoint_alias, + "requests": { + request.custom_id: dataclasses.asdict(request) for request in requests + }, + } + except Exception: + registry_status = "write_failed" + job = BatchJob( job_id=batch_id, backend=self.name, status=job_payload.get("status", "validating"), request_count=len(requests), ) + job.backend_registry_persistence_status = registry_status + return job def poll(self, job: BatchJob) -> Dict[str, Any]: """Poll batch status via the pg-llm-batch client.""" diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index c9fb6c2aa..bf194b8c4 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -7509,6 +7509,7 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di "request_link_status": job.request_link_status, "registry_persistence_status": job.registry_persistence_status, "recovery_status": job.recovery_status, + "backend_registry_persistence_status": job.backend_registry_persistence_status, }, 201) return if path.startswith("/api/v1/batch_routing_jobs/") and path.endswith("/results"): From df638d6cf76b7af25730f8f2f1d421fad8c23633 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:53:56 +0900 Subject: [PATCH 79/87] test: preserve healthy active batch beyond recovery expiry --- tests/test_batch_request_lineage.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index ff073c698..4a17b4b66 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -241,7 +241,7 @@ def test_valkey_job_snapshot_does_not_prove_lineage_commit(tmp_path): orchestrator.close() -@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "coordinator_hit", "duplicate_ids", "backend_write_outage"]) +@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "coordinator_hit", "duplicate_ids", "backend_write_outage", "healthy_expired"]) def test_http_batch_failed_registry_recovers_authorized_job_after_restart(tmp_path, recovery_case): """SQLite recovery binds the original owner without another remote submission.""" class MissingRegistry(dict): @@ -282,9 +282,11 @@ async def download_results(self, *args, **kwargs): coordinator.batch_backend._jobs = MissingRegistry() else: coordinator._batch_jobs = MissingRegistry() - elif recovery_case == "coordinator_hit": + elif recovery_case in {"coordinator_hit", "healthy_expired"}: from contextual_orchestrator.batch_routing import BatchJob coordinator._batch_jobs[submitted["job_id"]] = BatchJob(**record["recovery_descriptor"]["job"]) + if recovery_case == "healthy_expired": + coordinator.batch_backend._jobs = retained_backend_metadata elif recovery_case == "registry_outage": class UnavailableRegistry(MissingRegistry): """All reads and writes remain unavailable during recovery.""" @@ -311,7 +313,8 @@ def get(self, *args, **kwargs): assert submitted["recovery_status"] == "durable_descriptor" record = orchestrator._store.load("batch_request_link")[0] assert "Never persist this prompt." not in str(record) - if recovery_case == "expired": + retained_backend_metadata = coordinator.batch_backend._jobs + if recovery_case in {"expired", "healthy_expired"}: record["recovery_descriptor"]["expires_at"] = 0 if recovery_case == "malformed": record["recovery_descriptor"]["backend"] = [] @@ -325,7 +328,7 @@ def get(self, *args, **kwargs): record["recovery_descriptor"]["job"]["request_count"] = True if recovery_case == "duplicate_ids": record["custom_ids"] = ["a", "a"] - if recovery_case in {"expired", "malformed", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "duplicate_ids"}: + if recovery_case in {"expired", "healthy_expired", "malformed", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "duplicate_ids"}: orchestrator._store.save("batch_request_link", submitted["job_id"], record, durable=True) continue result_url = f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results" From a6b94855f1cc0b0d28917d05b811a777bbff8ee0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:54:27 +0900 Subject: [PATCH 80/87] fix: preserve healthy batch registry lifecycle during recovery --- contextual_orchestrator/batch_routing.py | 11 +++++++++++ contextual_orchestrator/cost_router.py | 3 +++ 2 files changed, 14 insertions(+) diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index df5c61ed3..e6b5f79aa 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -482,6 +482,17 @@ def recovery_enabled(self) -> bool: """Whether the operator supplied a stable deployment/account binding.""" return self._recovery_identity is not None + def has_job_metadata(self, job: BatchJob) -> bool: + """Check whether active registry metadata supports the existing job.""" + try: + document = self._jobs.get(job.job_id) + except Exception: + return False + return (isinstance(document, dict) + and document.get("endpoint_alias") == self._endpoint_alias + and isinstance(document.get("requests"), dict) + and len(document["requests"]) == job.request_count) + def recovery_descriptor(self, requests: List[BatchRequest]) -> Dict[str, Any]: """Describe exact target and item metadata without submitted prompt text.""" from .cost_ledger import AttributionDimensions diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index 0323f2baa..4a195199c 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -1325,6 +1325,9 @@ def _require_job(self, job_id: str, *, owner_id: Optional[str] = None) -> BatchJ job = None if job is not None and job.owner_id != owner_id: raise KeyError(f"batch job {job_id!r} not found") + if (job is not None and isinstance(self.batch_backend, PgLlmBatchBackend) + and self.batch_backend.has_job_metadata(job)): + return job if (owner_id is not None and self.orchestrator._store is not None and (job is None or (isinstance(self.batch_backend, PgLlmBatchBackend) and self.batch_backend.recovery_enabled))): From 3f98f922a385f67f98292bf0bd5ee7df0f7e50c0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:55:46 +0900 Subject: [PATCH 81/87] docs: record batch recovery review repairs and deployment binding --- docs/doctoring/batch_request_lineage.md | 51 +++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/docs/doctoring/batch_request_lineage.md b/docs/doctoring/batch_request_lineage.md index f8a9dd9a3..a1dcd425a 100644 --- a/docs/doctoring/batch_request_lineage.md +++ b/docs/doctoring/batch_request_lineage.md @@ -105,3 +105,54 @@ clients must preserve the handle and must not resubmit solely because local persistence is incomplete. If both SQLite and the registry fail, automatic recovery remains unavailable. This is a tested recovery slice, not complete cross-store atomicity, retention cleanup, production integration, or customer KPI evidence. + +## Independent review repairs and injected-adapter configuration + +At `a6b94855`, the four-file focused suite passes **84 tests in 32.63s**. +Intermediate failures and successful checks remain attributed to their commits: + +| Source | Check | Result | +| --- | --- | --- | +| `e372bc54` | Registry outage, inconsistent item IDs and estimate keys | 3 failed, 3.59s | +| `7fb1a71c` | Lineage file after recovery isolation | 19 passed, 14.94s | +| `83394afa` | Malformed job fields and deployment binding added | 23 passed, 18.26s | +| `f4d036bf` | Healthy coordinator handle, missing backend metadata | 1 failed, 1.37s | +| `6cb530a7` | Four-file suite | 81 passed, 29.90s | +| `42ad396c` | Duplicate persisted item IDs | 1 failed, 1.52s | +| `aaa9b133` | Four-file suite | 82 passed, 31.21s | +| `17cfa611` | Backend registry write fails after remote creation | 1 failed, 0.95s | +| `8ddfeb9f` | Same backend-failure HTTP test | 1 passed, 1.79s | +| `df638d6c` | Healthy active job with expired recovery descriptor | 1 failed, 1.51s | + +The Pg adapter now retains the upstream handle even when its own metadata +registry write fails before returning to the coordinator. The response separates +`backend_registry_persistence_status`, `registry_persistence_status`, +`request_link_status`, and `recovery_status`. The first two describe individual +write outcomes, not restart durability. HSET may apply before expiry fails; +response-only write results are not serialized as claims about their own writes. +Missing durable lineage still remains explicit; no handler repeats submission. + +Recovery tolerates continuing coordinator/backend registry outages. Validated +item metadata is carried on the returned job for that retrieval, without writing +back through the failed registry. The stored envelope's item list, descriptor +items, job count, and estimate keys must agree before downloading. Invalid count +types and null estimates fail closed. A healthy authorized registry job retains +its original refresh-on-read lifecycle even if its separate fixed-deadline +recovery descriptor has expired; missing metadata cannot use expired recovery. + +There is no built-in production Pg adapter constructor in this source tree: +the CLI/default coordinator uses the local backend. Integrators injecting +`PgLlmBatchBackend` configure its optional `recovery_identity` argument with a +stable, non-secret service/deployment/account identifier. Keep this identifier +stable during credential rotation and change it when the service or account +changes. Do not use a credential, infer equivalence from an endpoint alias, or +accept an HTTP caller's value. The default `None` disables durable recovery, +and submission reports `recovery_status=unavailable`; a committed, explicitly +bound Pg descriptor reports `durable_descriptor`. A different or missing binding +cannot recover the old job, even if endpoint aliases are identical. + +The real-HTTP tests use offline injected clients; no deployed integration is +claimed. Simultaneous loss of both durable submission evidence and job registries +still cannot promise recovery. Full suite, clean wheel, hosted review and release +remain pending. Do not stop at a status-only response when recoverable evidence +exists, and do not label a retained remote handle complete recovery by itself. From 08a660dc82839c0d634827b07369814febb466b0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:56:29 +0900 Subject: [PATCH 82/87] test: prevent healthy registry from bypassing deployment binding --- tests/test_batch_request_lineage.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index 4a17b4b66..3c19881de 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -241,7 +241,7 @@ def test_valkey_job_snapshot_does_not_prove_lineage_commit(tmp_path): orchestrator.close() -@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "coordinator_hit", "duplicate_ids", "backend_write_outage", "healthy_expired"]) +@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "coordinator_hit", "duplicate_ids", "backend_write_outage", "healthy_expired", "healthy_deployment_mismatch"]) def test_http_batch_failed_registry_recovers_authorized_job_after_restart(tmp_path, recovery_case): """SQLite recovery binds the original owner without another remote submission.""" class MissingRegistry(dict): @@ -275,17 +275,17 @@ async def download_results(self, *args, **kwargs): "changed-endpoint" if restarted and recovery_case == "backend_mismatch" else "original-endpoint"), recovery_identity=( None if restarted and recovery_case == "missing_identity" else - "different-deployment" if restarted and recovery_case == "deployment_mismatch" + "different-deployment" if restarted and recovery_case in {"deployment_mismatch", "healthy_deployment_mismatch"} else "unit-deployment-account"))) if not restarted: if recovery_case == "backend_write_outage": coordinator.batch_backend._jobs = MissingRegistry() else: coordinator._batch_jobs = MissingRegistry() - elif recovery_case in {"coordinator_hit", "healthy_expired"}: + elif recovery_case in {"coordinator_hit", "healthy_expired", "healthy_deployment_mismatch"}: from contextual_orchestrator.batch_routing import BatchJob coordinator._batch_jobs[submitted["job_id"]] = BatchJob(**record["recovery_descriptor"]["job"]) - if recovery_case == "healthy_expired": + if recovery_case in {"healthy_expired", "healthy_deployment_mismatch"}: coordinator.batch_backend._jobs = retained_backend_metadata elif recovery_case == "registry_outage": class UnavailableRegistry(MissingRegistry): @@ -336,7 +336,7 @@ def get(self, *args, **kwargs): assert denied_status == 404 assert "download_results" not in client.calls status, retrieved = _request("POST", result_url, "owner-one") - if recovery_case in {"expired", "malformed", "backend_mismatch", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "duplicate_ids"}: + if recovery_case in {"expired", "malformed", "backend_mismatch", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "duplicate_ids", "healthy_deployment_mismatch"}: assert status == 404, retrieved assert "download_results" not in client.calls continue From cd38d9c4811deb6e9fde9c0c11a78869d9f39dcf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:56:46 +0900 Subject: [PATCH 83/87] fix: validate deployment binding before healthy batch fast path --- contextual_orchestrator/batch_routing.py | 2 ++ contextual_orchestrator/cost_router.py | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index e6b5f79aa..6434eebab 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -490,6 +490,7 @@ def has_job_metadata(self, job: BatchJob) -> bool: return False return (isinstance(document, dict) and document.get("endpoint_alias") == self._endpoint_alias + and document.get("recovery_identity") == self._recovery_identity and isinstance(document.get("requests"), dict) and len(document["requests"]) == job.request_count) @@ -564,6 +565,7 @@ async def _submit() -> Dict[str, Any]: try: self._jobs[batch_id] = { "endpoint_alias": self._endpoint_alias, + "recovery_identity": self._recovery_identity, "requests": { request.custom_id: dataclasses.asdict(request) for request in requests }, diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index 4a195199c..d4f8c45ea 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -1325,9 +1325,12 @@ def _require_job(self, job_id: str, *, owner_id: Optional[str] = None) -> BatchJ job = None if job is not None and job.owner_id != owner_id: raise KeyError(f"batch job {job_id!r} not found") - if (job is not None and isinstance(self.batch_backend, PgLlmBatchBackend) - and self.batch_backend.has_job_metadata(job)): - return job + if job is not None and isinstance(self.batch_backend, PgLlmBatchBackend): + if self.batch_backend.has_job_metadata(job): + return job + # Missing or differently bound metadata cannot use the ordinary + # retrieval path; only a validated durable descriptor may recover. + job = None if (owner_id is not None and self.orchestrator._store is not None and (job is None or (isinstance(self.batch_backend, PgLlmBatchBackend) and self.batch_backend.recovery_enabled))): From 2d68f7052389f43e7d2abf6cc6aee1a3a101d929 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:57:37 +0900 Subject: [PATCH 84/87] test: preserve explicitly unbound legacy batch compatibility --- tests/test_batch_request_lineage.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index 3c19881de..d22877b9e 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -180,6 +180,10 @@ def test_library_batch_without_state_store_keeps_legacy_submission(): )]) assert job.request_link_status == "unavailable" assert job.job_id == "batch-789" + # Legacy metadata without a deployment binding remains usable only + # while the injected backend also has no recovery identity configured. + coordinator.batch_backend._jobs[job.job_id].pop("recovery_identity") + assert coordinator.poll_batch(job.job_id)["is_complete"] is True finally: orchestrator.close() From c06615ab31fc1d47ee5d3b9a6082b9608aad85cd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:58:46 +0900 Subject: [PATCH 85/87] test: preserve endpoint binding on healthy batch metadata --- tests/test_batch_request_lineage.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py index d22877b9e..eaeef59ca 100644 --- a/tests/test_batch_request_lineage.py +++ b/tests/test_batch_request_lineage.py @@ -245,7 +245,7 @@ def test_valkey_job_snapshot_does_not_prove_lineage_commit(tmp_path): orchestrator.close() -@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "coordinator_hit", "duplicate_ids", "backend_write_outage", "healthy_expired", "healthy_deployment_mismatch"]) +@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "coordinator_hit", "duplicate_ids", "backend_write_outage", "healthy_expired", "healthy_deployment_mismatch", "healthy_endpoint_mismatch"]) def test_http_batch_failed_registry_recovers_authorized_job_after_restart(tmp_path, recovery_case): """SQLite recovery binds the original owner without another remote submission.""" class MissingRegistry(dict): @@ -277,7 +277,9 @@ async def download_results(self, *args, **kwargs): coordinator = CostRoutingCoordinator(orchestrator, batch_backend=PgLlmBatchBackend(client, endpoint_alias=( "changed-endpoint" if restarted and recovery_case == "backend_mismatch" - else "original-endpoint"), recovery_identity=( + else "original-endpoint"), endpoint=( + "/v1/completions" if restarted and recovery_case == "healthy_endpoint_mismatch" + else "/v1/chat/completions"), recovery_identity=( None if restarted and recovery_case == "missing_identity" else "different-deployment" if restarted and recovery_case in {"deployment_mismatch", "healthy_deployment_mismatch"} else "unit-deployment-account"))) @@ -286,10 +288,10 @@ async def download_results(self, *args, **kwargs): coordinator.batch_backend._jobs = MissingRegistry() else: coordinator._batch_jobs = MissingRegistry() - elif recovery_case in {"coordinator_hit", "healthy_expired", "healthy_deployment_mismatch"}: + elif recovery_case in {"coordinator_hit", "healthy_expired", "healthy_deployment_mismatch", "healthy_endpoint_mismatch"}: from contextual_orchestrator.batch_routing import BatchJob coordinator._batch_jobs[submitted["job_id"]] = BatchJob(**record["recovery_descriptor"]["job"]) - if recovery_case in {"healthy_expired", "healthy_deployment_mismatch"}: + if recovery_case in {"healthy_expired", "healthy_deployment_mismatch", "healthy_endpoint_mismatch"}: coordinator.batch_backend._jobs = retained_backend_metadata elif recovery_case == "registry_outage": class UnavailableRegistry(MissingRegistry): @@ -340,7 +342,7 @@ def get(self, *args, **kwargs): assert denied_status == 404 assert "download_results" not in client.calls status, retrieved = _request("POST", result_url, "owner-one") - if recovery_case in {"expired", "malformed", "backend_mismatch", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "duplicate_ids", "healthy_deployment_mismatch"}: + if recovery_case in {"expired", "malformed", "backend_mismatch", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "duplicate_ids", "healthy_deployment_mismatch", "healthy_endpoint_mismatch"}: assert status == 404, retrieved assert "download_results" not in client.calls continue From 450667593285679c92d1d0a698f35eadb2b2c879 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 17:58:57 +0900 Subject: [PATCH 86/87] fix: enforce exact endpoint on bound batch metadata --- contextual_orchestrator/batch_routing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index 6434eebab..7cb8be038 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -491,6 +491,8 @@ def has_job_metadata(self, job: BatchJob) -> bool: return (isinstance(document, dict) and document.get("endpoint_alias") == self._endpoint_alias and document.get("recovery_identity") == self._recovery_identity + and (document.get("endpoint") == self._endpoint + or ("endpoint" not in document and self._recovery_identity is None)) and isinstance(document.get("requests"), dict) and len(document["requests"]) == job.request_count) @@ -566,6 +568,7 @@ async def _submit() -> Dict[str, Any]: self._jobs[batch_id] = { "endpoint_alias": self._endpoint_alias, "recovery_identity": self._recovery_identity, + "endpoint": self._endpoint, "requests": { request.custom_id: dataclasses.asdict(request) for request in requests }, From 73404f89d7a90f7a6b98caf514af859b02a375dd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 18:00:01 +0900 Subject: [PATCH 87/87] docs: freeze reviewed batch recovery acceptance scope --- docs/doctoring/batch_request_lineage.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/doctoring/batch_request_lineage.md b/docs/doctoring/batch_request_lineage.md index a1dcd425a..d8130d517 100644 --- a/docs/doctoring/batch_request_lineage.md +++ b/docs/doctoring/batch_request_lineage.md @@ -156,3 +156,17 @@ claimed. Simultaneous loss of both durable submission evidence and job registrie still cannot promise recovery. Full suite, clean wheel, hosted review and release remain pending. Do not stop at a status-only response when recoverable evidence exists, and do not label a retained remote handle complete recovery by itself. + +## Final focused acceptance before full regression + +Runtime `45066759`: **86 passed in 36.58s** across the four focused files. +The last review exposed inconsistent endpoint binding on healthy metadata: +`c06615ab` failed once in 2.09s. New metadata now stores the exact endpoint and +stable deployment binding before the healthy fast path can use it. A changed +binding, changed endpoint, or missing binding on opted-in metadata cannot bypass +the validated descriptor path. This fixes a contract inconsistency; no cross-service +data leak was established by the offline test. Unbound legacy metadata remains +compatible only with an unbound backend; its missing identity is never described +as validated recovery. The legacy-focused suite passed 85 tests in 35.09s at +`2d68f705`. Independent review accepted freezing this bounded implementation for +full regression and separate installed-artifact verification, not publication.