diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 428cbfaf1..d17717983 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -44,8 +44,16 @@ jobs: with: version: "0.12.5" + - name: Build locked native decision measurement + run: | + set -euo pipefail + rustup show active-toolchain + uv sync --locked --extra api --extra db --extra queue --group dev --group native-build + uv run --no-sync maturin develop --locked --release --features pyo3/extension-module --manifest-path rust/decision_receipt/Cargo.toml + uv run --no-sync maturin build --locked --release --features pyo3/extension-module --manifest-path rust/decision_receipt/Cargo.toml --out "$RUNNER_TEMP/decision-wheels" + - name: Run full test suite - run: uv run --locked --extra api --extra db --extra queue --group dev python -m pytest -q + run: uv run --no-sync python -m pytest -q - name: Install hash-locked quality tools run: | @@ -74,12 +82,15 @@ jobs: set -euo pipefail rm -rf dist "$RUNNER_TEMP/nim-wheel-site" python -m pip wheel --no-deps --no-build-isolation . --wheel-dir dist + python scripts/verify_decision_wheel_manifest.py dist/contextual_orchestrator-*.whl "$RUNNER_TEMP"/decision-wheels/*.whl python -m pip install --no-deps \ --target "$RUNNER_TEMP/nim-wheel-site" \ - dist/contextual_orchestrator-*.whl + dist/contextual_orchestrator-*.whl "$RUNNER_TEMP"/decision-wheels/*.whl cd "$RUNNER_TEMP" PYTHONPATH="$RUNNER_TEMP/nim-wheel-site" \ - python -c "import contextual_orchestrator; import contextual_orchestrator.nim_benchmark" + python -c "import pathlib, contextual_orchestrator as core; import contextual_orchestrator.nim_benchmark; import contextual_orchestrator._decision_receipt as native; root = pathlib.Path('nim-wheel-site').resolve(); assert pathlib.Path(core.__file__).is_relative_to(root); assert pathlib.Path(native.__file__).is_relative_to(root)" + PYTHONPATH="$RUNNER_TEMP/nim-wheel-site" \ + python -m pytest --noconftest --import-mode=importlib "$GITHUB_WORKSPACE/tests/test_decision_receipts.py" -q fuzz: name: Property and coverage-guided fuzzing diff --git a/AGENTS.md b/AGENTS.md index ee030590a..5d074dd30 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,12 @@ # AGENTS.md +Deferred batch lineage: read `docs/doctoring/batch_request_lineage.md` for the +HTTP reproduction, atomic submission-event projection, and remote/local failure +boundary. Do not retry a remotely submitted job after local lineage failure. + +Workflow origin identity and persistence limitations are documented in +`docs/doctoring/workflow_request_link.md`; preserve origin on replacements and reload. + Cross-agent conventions for `contextual-orchestrator`, readable by any coding agent (Claude, Codex, Cursor, opencode, …). Keep this file tool-agnostic. @@ -50,6 +57,10 @@ push or open a PR. ### Code exploration +- Provider logs need server-generated per-request identity, not a session hash. + Preserve context cleanup and validate the central collector before adoption. + Reproduction and exact evidence: `docs/doctoring/provider_request_correlation.md`. + - This repo has **no `.codegraph/` index**, so use normal search (grep/ripgrep/find, file reads) to locate and understand code. If a `.codegraph/` directory is ever added at the repo root, prefer CodeGraph diff --git a/CLAUDE.md b/CLAUDE.md index edcea2356..03f002367 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,5 +1,12 @@ # CLAUDE.md +Batch request lineage evidence and unresolved registry failure semantics live +in `docs/doctoring/batch_request_lineage.md`; HTTP 201 alone does not establish +durable lineage. Preserve job-scoped item IDs and original submission identity. + +See `docs/doctoring/workflow_request_link.md` for request-to-workflow correlation +tests, cache semantics, and the distinction between in-memory and durable outcomes. + This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. ## Read AGENTS.md first @@ -27,6 +34,10 @@ This file complements AGENTS.md with commands and architecture; where they diffe ## Common commands +For missing provider/error correlation, run the focused telemetry and debug-log +tests in `docs/doctoring/provider_request_correlation.md`. A green local producer +test does not prove the central collector preserves the new field. + ```bash # Install (pinned, hash-locked — always this two-step form) python -m pip install --require-hashes -r requirements.lock diff --git a/contextual_orchestrator/batch_job_registry.py b/contextual_orchestrator/batch_job_registry.py index 21c7fc8a7..25a8a56a9 100644 --- a/contextual_orchestrator/batch_job_registry.py +++ b/contextual_orchestrator/batch_job_registry.py @@ -199,6 +199,11 @@ def __init__(self, client: Any = None, *, retention_seconds: int = DEFAULT_RETEN ) self._local_locks_guard = threading.Lock() + @property + def retention_seconds(self) -> int: + """Return the configured registry retention for durable recovery expiry.""" + return self._retention_seconds + def lock( self, name: str, diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index 5650e441d..7cb8be038 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -218,6 +218,15 @@ class BatchJob: # Prompt-token fallback estimates are safe metadata, stored atomically with # the job handle rather than retaining submitted prompt text. prompt_token_estimates: Dict[str, int] = field(default_factory=dict) + # Finalized before the registry snapshot is written; the append-only event + # supplies the durable request association independently of that registry. + request_link_status: str = "unavailable" + recovery_status: str = "unavailable" + # Deliberately not a dataclass field: HSET may succeed before expiry fails, + # so an operation result must never be serialized into its own snapshot. + registry_persistence_status = "unavailable" + backend_registry_persistence_status = "unavailable" + recovered_request_metadata = None @dataclass @@ -453,17 +462,75 @@ def __init__( endpoint: str = "/v1/chat/completions", payload_assembler: Any = None, job_registry: Any = None, + recovery_identity: str | None = None, ) -> None: self._client = client self._endpoint_alias = endpoint_alias self._endpoint = endpoint self._assembler = payload_assembler + if recovery_identity is not None and (not isinstance(recovery_identity, str) or not recovery_identity.strip()): + raise ValueError("recovery identity must be a nonempty operator-controlled identifier") + self._recovery_identity = recovery_identity # Tracked requests survive a restart when a Valkey-backed registry # is injected; a plain dict preserves the historical behavior. self._jobs: Dict[str, Dict[str, Any]] = ( job_registry.mapping("pg_llm_batch_jobs") if job_registry is not None else {} ) + @property + def recovery_enabled(self) -> bool: + """Whether the operator supplied a stable deployment/account binding.""" + return self._recovery_identity is not None + + def has_job_metadata(self, job: BatchJob) -> bool: + """Check whether active registry metadata supports the existing job.""" + try: + document = self._jobs.get(job.job_id) + except Exception: + return False + return (isinstance(document, dict) + and document.get("endpoint_alias") == self._endpoint_alias + and document.get("recovery_identity") == self._recovery_identity + and (document.get("endpoint") == self._endpoint + or ("endpoint" not in document and self._recovery_identity is None)) + and isinstance(document.get("requests"), dict) + and len(document["requests"]) == job.request_count) + + def recovery_descriptor(self, requests: List[BatchRequest]) -> Dict[str, Any]: + """Describe exact target and item metadata without submitted prompt text.""" + from .cost_ledger import AttributionDimensions + return { + "recovery_identity": self._recovery_identity, + "backend_name": self.name, + "endpoint_alias": self._endpoint_alias, + "endpoint": self._endpoint, + "items": [{"custom_id": item.custom_id, "model": item.model, + "mode": item.mode, "attribution": AttributionDimensions.from_mapping(item.attribution).as_dict()} + for item in requests], + } + + def restore_descriptor(self, job: BatchJob, descriptor: Dict[str, Any]) -> None: + """Restore prompt-free item identity only for this exact configured target.""" + if (not isinstance(descriptor, dict) or descriptor.get("backend_name") != self.name + or self._recovery_identity is None + or descriptor.get("recovery_identity") != self._recovery_identity + or descriptor.get("endpoint_alias") != self._endpoint_alias + or descriptor.get("endpoint") != self._endpoint): + raise ValueError("batch target mismatch") + items = descriptor.get("items") + if not isinstance(items, list) or len(items) != job.request_count: + raise ValueError("batch item count mismatch") + restored = {} + for item in items: + if not isinstance(item, dict) or set(item) != {"custom_id", "model", "mode", "attribution"}: + raise ValueError("invalid batch item descriptor") + if any(not isinstance(item[field], str) or not item[field] for field in ("custom_id", "model", "mode")): + raise ValueError("invalid batch item identity") + if item["custom_id"] in restored or not isinstance(item["attribution"], dict): + raise ValueError("invalid batch item metadata") + restored[item["custom_id"]] = {**item, "messages": []} + job.recovered_request_metadata = restored + def _assemble_payload(self, requests: List[BatchRequest]) -> str: if self._assembler is not None: return self._assembler.assemble( @@ -496,18 +563,26 @@ async def _submit() -> Dict[str, Any]: # Tracked requests are stored as JSON primitives (not dataclass # instances) so the registry can be a JSON-backed Valkey mapping; # retrieve() rebuilds the dataclass view it needs. - self._jobs[batch_id] = { - "endpoint_alias": self._endpoint_alias, - "requests": { - request.custom_id: dataclasses.asdict(request) for request in requests - }, - } - return BatchJob( + registry_status = "stored" + try: + self._jobs[batch_id] = { + "endpoint_alias": self._endpoint_alias, + "recovery_identity": self._recovery_identity, + "endpoint": self._endpoint, + "requests": { + request.custom_id: dataclasses.asdict(request) for request in requests + }, + } + except Exception: + registry_status = "write_failed" + job = BatchJob( job_id=batch_id, backend=self.name, status=job_payload.get("status", "validating"), request_count=len(requests), ) + job.backend_registry_persistence_status = registry_status + return job def poll(self, job: BatchJob) -> Dict[str, Any]: """Poll batch status via the pg-llm-batch client.""" @@ -542,7 +617,8 @@ async def _download() -> Dict[str, Any]: reason, ) raise BatchDownloadError(job.job_id, reason) - tracked = self._jobs.get(job.job_id, {}).get("requests", {}) + tracked = (job.recovered_request_metadata if job.recovered_request_metadata is not None + else self._jobs.get(job.job_id, {}).get("requests", {})) responses = _validated_download_responses( payload, expected_custom_ids=set(tracked), diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index 751c9e366..d4f8c45ea 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -20,10 +20,12 @@ import hashlib import re +import time from contextvars import ContextVar -from dataclasses import replace +from dataclasses import asdict, replace from threading import Lock from typing import Any, Dict, List, Optional +from .decision_receipts import record_initial_selection from .batch_routing import ( BatchBackend, @@ -37,6 +39,7 @@ LocalBatchBackend, LocalEmbeddingBatchBackend, ProviderEmbeddingBatchBackend, + PgLlmBatchBackend, RoutingHints, RoutingPolicy, ) @@ -972,6 +975,7 @@ def submit_batch( requests: List[BatchRequest], metadata: Optional[Dict[str, Any]] = None, owner_id: Optional[str] = None, + request_id: Optional[str] = None, ) -> BatchJob: """Submit a batch, resolve its targets, and bind its authenticated owner.""" try: @@ -995,7 +999,36 @@ def submit_batch( job = self.batch_backend.submit(prepared_requests, metadata=metadata) job.owner_id = owner_id job.prompt_token_estimates = prompt_token_estimates - self._batch_jobs[job.job_id] = job + if request_id is not None and self.orchestrator._store is not None: + try: + # One append-only submission envelope commits all item links + # together. A later retrieval never rewrites this origin. + self.orchestrator._store.save("batch_request_link", job.job_id, { + "request_id": request_id, + "batch_job_id": job.job_id, + "custom_ids": [request.custom_id for request in prepared_requests], + "owner_id": owner_id, + "recovery_descriptor": ({ + "job": asdict(job), + "expires_at": job.submitted_at + self._job_registry.retention_seconds, + "backend": self.batch_backend.recovery_descriptor(prepared_requests), + } if isinstance(self.batch_backend, PgLlmBatchBackend) else None), + }, durable=True) + except Exception: + # The upstream submission already happened. Preserve its handle + # and report incomplete lineage instead of inviting a resubmit. + job.request_link_status = "write_failed" + else: + job.request_link_status = "durable" + if isinstance(self.batch_backend, PgLlmBatchBackend) and self.batch_backend.recovery_enabled: + job.recovery_status = "durable_descriptor" + job.registry_persistence_status = "stored" + try: + self._batch_jobs[job.job_id] = job + except Exception: + # Submission already applied remotely; an HSET/expiry failure may + # itself be partially applied. Return the handle without replay. + job.registry_persistence_status = "write_failed" return job def _resolve_batch_request(self, request: BatchRequest) -> BatchRequest: @@ -1046,7 +1079,7 @@ def retrieve_batch(self, job_id: str, *, owner_id: Optional[str] = None) -> Dict not self._batch_item_usage_valid(item) and item.custom_id not in prompt_token_estimates for item in items - ) + ) and job.recovered_request_metadata is None request_by_custom_id = ( self._legacy_batch_requests(job) if needs_legacy_lookup else {} ) @@ -1286,7 +1319,53 @@ def _resolve_batch_provider_model(self, item: BatchResultItem) -> tuple[str, str return provider, item.model def _require_job(self, job_id: str, *, owner_id: Optional[str] = None) -> BatchJob: - job = self._batch_jobs.get(job_id) + try: + job = self._batch_jobs.get(job_id) + except Exception: + job = None + if job is not None and job.owner_id != owner_id: + raise KeyError(f"batch job {job_id!r} not found") + if job is not None and isinstance(self.batch_backend, PgLlmBatchBackend): + if self.batch_backend.has_job_metadata(job): + return job + # Missing or differently bound metadata cannot use the ordinary + # retrieval path; only a validated durable descriptor may recover. + job = None + if (owner_id is not None and self.orchestrator._store is not None + and (job is None or (isinstance(self.batch_backend, PgLlmBatchBackend) + and self.batch_backend.recovery_enabled))): + record = self.orchestrator._store.load_latest_key("batch_request_link", job_id) + if (isinstance(record, dict) and record.get("owner_id") == owner_id + and record.get("batch_job_id") == job_id + and isinstance(self.batch_backend, PgLlmBatchBackend)): + descriptor = record.get("recovery_descriptor") + try: + if not isinstance(descriptor, dict) or type(descriptor.get("expires_at")) is not int: + raise ValueError("invalid descriptor") + if descriptor["expires_at"] <= time.time(): + raise ValueError("expired descriptor") + recovered = BatchJob(**descriptor["job"]) + if recovered.job_id != job_id or recovered.owner_id != owner_id or recovered.backend != self.batch_backend.name: + raise ValueError("mismatched descriptor") + custom_ids = record.get("custom_ids") + if (type(recovered.request_count) is not int or recovered.request_count < 1 + or not isinstance(custom_ids, list) + or any(not isinstance(item, str) or not item for item in custom_ids) + or len(custom_ids) != recovered.request_count + or len(set(custom_ids)) != recovered.request_count): + raise ValueError("invalid recovery item identities") + estimates = recovered.prompt_token_estimates + if (not isinstance(estimates, dict) or not set(estimates).issubset(custom_ids) + or any(type(value) is not int or value < 0 for value in estimates.values())): + raise ValueError("invalid recovery estimates") + self.batch_backend.restore_descriptor(recovered, descriptor["backend"]) + if set(recovered.recovered_request_metadata) != set(custom_ids): + raise ValueError("mismatched recovery items") + recovered.request_link_status = "durable" + recovered.recovery_status = "durable_descriptor" + job = recovered + except (KeyError, TypeError, ValueError): + job = None if job is None or job.owner_id != owner_id: raise KeyError(f"batch job {job_id!r} not found") return job @@ -1332,6 +1411,8 @@ def submit_embeddings_batch( if callable(reserve) and callable(start): job = reserve(requests, metadata=metadata) else: + if resolved_agent_id is not None: + record_initial_selection([resolved_agent_id], "embedding_submission") job = backend.submit(requests, metadata=metadata) self._embedding_models[job.job_id] = resolved_model self._embedding_owners[job.job_id] = owner_id @@ -1341,6 +1422,8 @@ def submit_embeddings_batch( self._embedding_part_limits[job.job_id] = part_limits self._embedding_jobs[job.job_id] = job if callable(reserve) and callable(start): + if resolved_agent_id is not None: + record_initial_selection([resolved_agent_id], "embedding_submission") start(job) return job diff --git a/contextual_orchestrator/decision_receipts.py b/contextual_orchestrator/decision_receipts.py new file mode 100644 index 000000000..618106392 --- /dev/null +++ b/contextual_orchestrator/decision_receipts.py @@ -0,0 +1,223 @@ +"""Persistence boundary for Rust-owned initial-decision measurements.""" + +from contextvars import ContextVar +from contextlib import contextmanager +import logging +import threading +import uuid +import hashlib +import json + +_CURRENT_DECISION = ContextVar("initial_decision", default=None) +_LOGGER = logging.getLogger(__name__) + + +class DecisionMeasurement: + """Bind one native clock to an HTTP request and its durable store.""" + + def __init__(self, store, *, policy=None, route_mode="unclassified", request_id=None, + endpoint_path=None, request_method=None, + admission_boundary="explicit_scope"): + """Require the native module only when measurement is explicitly enabled.""" + from ._decision_receipt import DecisionReceipt + + self.receipt = DecisionReceipt() + self.store = store + self.request_id = request_id or uuid.uuid4().hex + self.identity_source = "http_request" if request_id else "measurement_scope" + policy_snapshot = policy.as_dict() if policy is not None else None + self.policy_hash = hashlib.sha256(json.dumps( + policy_snapshot, sort_keys=True, separators=(",", ":") + ).encode()).hexdigest() if policy_snapshot is not None else None + self.route_mode = route_mode + self.endpoint_path = endpoint_path + self.request_method = request_method + self.admission_boundary = admission_boundary + self.selected_agent_ids = [] + self.selection_attempt_count = 0 + self._race_attempt_ids = set() + self.first_provider_phase = None + self._lock = threading.Lock() + self._token = _CURRENT_DECISION.set(self) + try: + if self.store is None: + raise RuntimeError("no durable measurement store") + self.store.save("accepted_request", None, self.snapshot(), durable=True) + except Exception as exc: + _CURRENT_DECISION.reset(self._token) + _LOGGER.warning("Decision admission write failed error_type=%s", type(exc).__name__) + raise RuntimeError("decision measurement admission could not be persisted") from None + + def snapshot(self): + """Return no prompts, provider credentials, or fabricated missing timings.""" + return { + "request_id": self.request_id, + "identity_source": self.identity_source, + "policy_snapshot_hash": self.policy_hash, + "route_mode": self.route_mode, + "endpoint_path": self.endpoint_path, + "request_method": self.request_method, + "measurement_unit": "http_request" if self.endpoint_path else "explicit_scope", + "admission_boundary": self.admission_boundary, + "status": self.receipt.status, + "selected_agent_ids": list(self.selected_agent_ids), + "selection_attempt_count": self.selection_attempt_count, + "selection_elapsed_ns": self.receipt.selection_elapsed_ns, + "durable_ack_elapsed_ns": self.receipt.durable_ack_elapsed_ns, + "first_provider_elapsed_ns": self.receipt.first_provider_elapsed_ns, + "first_provider_phase": self.first_provider_phase, + "first_provider_boundary": "provider_ready_before_diagnostic_commit", + "metric_scope": "initial_task_route_decision", + } + + def select(self, agent_ids, route_mode, *, attempt_id=None): + """Acknowledge the first decision synchronously before provider dispatch.""" + with self._lock: + if attempt_id is not None: + if attempt_id in self._race_attempt_ids: + return + self._race_attempt_ids.add(attempt_id) + self.selection_attempt_count += 1 + if self.receipt.status != "accepted": + try: + self.store.save("selection_attempt", None, { + "request_id": self.request_id, + "attempt_number": self.selection_attempt_count, + "selected_agent_ids": list(agent_ids), "route_mode": route_mode, + }, durable=True) + except Exception as exc: + _LOGGER.warning("Decision attempt export failed error_type=%s", type(exc).__name__) + return + self.receipt.record_selection() + self.route_mode = route_mode + self.selected_agent_ids = list(agent_ids) + if self.store is None: + self.receipt.record_failure("store_unavailable") + return + try: + self.store.save("initial_decision", None, self.snapshot(), durable=True) + except Exception as exc: + self.receipt.record_failure("write_failed") + _LOGGER.warning("Initial decision measurement write failed error_type=%s", type(exc).__name__) + return + self.receipt.record_durable_ack() + self._record_provider_locked(agent_ids, "task_execution") + + def _record_provider_locked(self, agent_ids, phase): + """Keep a first-provider diagnostic independent of the task-route clock.""" + if self.first_provider_phase is not None: + return + self.receipt.record_provider_dispatch() + self.first_provider_phase = phase + try: + self.store.save("provider_dispatch", self.request_id, { + **self.snapshot(), "provider_agent_ids": list(agent_ids), + }, durable=True) + except Exception as exc: + _LOGGER.warning("Provider diagnostic write failed error_type=%s", type(exc).__name__) + + @contextmanager + def auxiliary_call(self, agent_ids, phase): + """Retain auxiliary work without subtracting it from the task-route interval.""" + with self._lock: + if phase == "routing_evidence_embedding" and self.receipt.selection_elapsed_ns is not None: + phase = "post_decision_evidence_embedding" + self._record_provider_locked(agent_ids, phase) + started = self.receipt.current_elapsed_ns() + outcome = "completed" + try: + yield + except BaseException: + outcome = "failed" + raise + finally: + with self._lock: + finished = self.receipt.current_elapsed_ns() + try: + self.store.save("auxiliary_dispatch", self.request_id, { + "request_id": self.request_id, "phase": phase, + "provider_agent_ids": list(agent_ids), "outcome": outcome, + "started_elapsed_ns": started, "finished_elapsed_ns": finished, + }, durable=True) + except Exception as exc: + _LOGGER.warning("Auxiliary diagnostic write failed error_type=%s", type(exc).__name__) + + def close(self, reason="unfinished"): + """Persist the post-commit measurement separately; never claim its own ack.""" + try: + with self._lock: + if self.receipt.status in ("accepted", "selected"): + self.receipt.record_failure(reason) + if self.store is not None: + try: + self.store.save("decision_receipt", None, self.snapshot(), durable=True) + except Exception as exc: + _LOGGER.warning("Decision receipt export failed error_type=%s", type(exc).__name__) + finally: + _CURRENT_DECISION.reset(self._token) + + +def record_initial_selection(agent_ids, route_mode="unclassified", *, attempt_id=None): + """Record selection only within an explicitly enabled request measurement.""" + measurement = _CURRENT_DECISION.get() + if measurement is not None: + measurement.select(agent_ids, route_mode, attempt_id=attempt_id) + + +def record_answer_cache_hit(): + """Retain answer-cache admissions without attributing cached provider timings.""" + measurement = _CURRENT_DECISION.get() + if measurement is not None: + with measurement._lock: + measurement.receipt.record_cache_hit() + + +@contextmanager +def observe_auxiliary_dispatch(agent_ids, phase): + """Use the request's native clock for an actual auxiliary provider call.""" + measurement = _CURRENT_DECISION.get() + if measurement is None: + yield + else: + with measurement.auxiliary_call(agent_ids, phase): + yield + + +def export_decision_receipts(store, limit=256): + """Join retained admissions without dropping missing final acknowledgements. + + This is a local ledger view, not an all-ingress census. Storage outages need + external ingress reconciliation. Concurrent finalization may appear on the + next read; missing values remain unfinished rather than successful zeros. + """ + cohort = store.load_decision_window(limit) + accepted = cohort["accepted"] + decisions = {row["request_id"]: row for row in cohort["decisions"]} + receipts = {row["request_id"]: row for row in cohort["receipts"]} + diagnostics = {} + for diagnostic in cohort["diagnostics"]: + diagnostics.setdefault(diagnostic["request_id"], []).append(diagnostic) + observations = [] + for admission in accepted: + request_id = admission["request_id"] + row = dict(admission) + row["status"] = "unfinished" + if request_id in decisions: + row.update(decisions[request_id]) + row["status"] = "acknowledgement_unobserved" + if request_id in receipts: + row.update(receipts[request_id]) + request_diagnostics = diagnostics.get(request_id, []) + row["auxiliary_dispatches"] = [entry for entry in request_diagnostics + if entry["record_kind"] == "auxiliary_dispatch"] + row["provider_diagnostics"] = [entry for entry in request_diagnostics + if entry["record_kind"] == "provider_dispatch"] + observations.append(row) + return { + "schema_version": 1, + "measurement_complete": False, + "reconciliation_required": True, + "scope": "retained_local_admissions", + "window": cohort["window"], + "observations": observations, + } diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index fa85ba533..5030f42d6 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -6,6 +6,7 @@ from collections.abc import Iterable, Mapping from contextlib import contextmanager, nullcontext from contextvars import ContextVar, copy_context +from .decision_receipts import observe_auxiliary_dispatch, record_answer_cache_hit, record_initial_selection from concurrent.futures import ThreadPoolExecutor import copy import hashlib @@ -56,6 +57,7 @@ ) from .telemetry import ( annotate_current_span, + current_request_id, inject_trace_context, record_provider_usage, traced, @@ -1334,11 +1336,12 @@ def _log_provider_attempt(agent: ModelAgent, attempt: int, retry_limit: int) -> """DEBUG-log one provider call attempt before it is made.""" if _LOGGER.isEnabledFor(logging.DEBUG): _LOGGER.debug( - "provider_attempt agent_id=%s model=%s attempt=%d/%d", + "provider_attempt agent_id=%s model=%s attempt=%d/%d request_id=%s", agent.id, agent.model, attempt + 1, retry_limit + 1, + current_request_id() or "-", ) @@ -1348,12 +1351,13 @@ def _log_provider_attempt_failed( """DEBUG-log one failed provider attempt with a redacted, bounded error message.""" if _LOGGER.isEnabledFor(logging.DEBUG): _LOGGER.debug( - "provider_attempt_failed agent_id=%s model=%s attempt=%d error_type=%s transient=%s error_message=%s", + "provider_attempt_failed agent_id=%s model=%s attempt=%d error_type=%s transient=%s request_id=%s error_message=%s", agent.id, agent.model, attempt + 1, type(exc).__name__, transient, + current_request_id() or "-", redact_text(str(exc))[:500], ) @@ -1362,10 +1366,11 @@ def _log_provider_backoff(agent: ModelAgent, attempt: int, delay: float) -> None """DEBUG-log one backoff sleep before the next retry attempt.""" if _LOGGER.isEnabledFor(logging.DEBUG): _LOGGER.debug( - "provider_backoff agent_id=%s attempt=%d delay_seconds=%.3f", + "provider_backoff agent_id=%s attempt=%d delay_seconds=%.3f request_id=%s", agent.id, attempt + 1, delay, + current_request_id() or "-", ) @@ -1386,11 +1391,12 @@ def _log_provider_exhausted(agent: ModelAgent, attempts: int, last_error: Except first place" or "was never allowed to be retried at all". """ _LOGGER.warning( - "provider_exhausted agent_id=%s model=%s attempts=%s final_error_type=%s", + "provider_exhausted agent_id=%s model=%s attempts=%s final_error_type=%s request_id=%s", agent.id, agent.model, attempts, type(last_error).__name__, + current_request_id() or "-", ) @@ -1415,12 +1421,13 @@ def _log_provider_no_retry_budget( from "this wouldn't have been retried anyway" from this one event name. """ _LOGGER.warning( - "provider_no_retry_budget agent_id=%s model=%s attempts=%s final_error_type=%s transient=%s", + "provider_no_retry_budget agent_id=%s model=%s attempts=%s final_error_type=%s transient=%s request_id=%s", agent.id, agent.model, attempts, type(last_error).__name__, transient, + current_request_id() or "-", ) @@ -1446,12 +1453,13 @@ def _log_provider_one_shot_call_failed( :func:`_log_provider_no_retry_budget`. """ _LOGGER.warning( - "provider_one_shot_call_failed agent_id=%s model=%s attempts=%s final_error_type=%s transient=%s", + "provider_one_shot_call_failed agent_id=%s model=%s attempts=%s final_error_type=%s transient=%s request_id=%s", agent.id, agent.model, attempts, type(last_error).__name__, transient, + current_request_id() or "-", ) @@ -1467,11 +1475,12 @@ def _log_provider_rejected_permanent(agent: ModelAgent, attempts: int, last_erro for the separate case where no retry budget was configured at all. """ _LOGGER.warning( - "provider_rejected_permanent agent_id=%s model=%s attempts=%s final_error_type=%s", + "provider_rejected_permanent agent_id=%s model=%s attempts=%s final_error_type=%s request_id=%s", agent.id, agent.model, attempts, type(last_error).__name__, + current_request_id() or "-", ) @@ -3644,6 +3653,7 @@ class _StateStore: _LEGACY_INDEX_NAME = "records_kind_seq" _INDEX_NAME = "orchestration_records_kind_seq" _STREAM_LIMITS = {"audit": 256, "authorization": 256, "analytics": 256} + _MEASUREMENT_KINDS = ("accepted_request", "initial_decision", "decision_receipt", "selection_attempt") _CREATE_RECORDS_SQL = ( "CREATE TABLE IF NOT EXISTS orchestration_records (" "seq INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT NOT NULL, key TEXT, payload TEXT NOT NULL)" @@ -3668,6 +3678,18 @@ def __init__(self, path: str) -> None: self._migrate_legacy_table() self._conn.execute(self._CREATE_RECORDS_SQL) self._conn.execute(self._CREATE_RECORDS_KIND_SEQ_INDEX_SQL) + self._conn.execute( + "CREATE INDEX IF NOT EXISTS orchestration_records_kind_key_seq " + "ON orchestration_records(kind, key, seq)" + ) + self._conn.execute( + "UPDATE orchestration_records SET key = json_extract(payload, '$.request_id') " + "WHERE kind IN (?, ?, ?, ?) AND key IS NULL " + "AND CASE WHEN json_valid(payload) THEN " + "json_type(payload, '$.request_id') = 'text' " + "AND length(json_extract(payload, '$.request_id')) > 0 ELSE 0 END", + self._MEASUREMENT_KINDS, + ) self._conn.commit() except Exception: self._conn.rollback() @@ -3723,16 +3745,76 @@ def save(self, kind: str, key: str | None, payload: dict[str, Any], *, durable: return self._save_sync(kind, key, payload) + def load_decision_window(self, limit: int = 256) -> dict[str, Any]: + """Read a bounded shared admission cohort without deleting historical rows.""" + if type(limit) is not int or not 1 <= limit <= 1000: + raise ValueError("decision window limit must be between 1 and 1000") + with self._lock: + admissions = self._conn.execute( + "SELECT seq, key, payload FROM orchestration_records WHERE kind = ? ORDER BY seq DESC LIMIT ?", + ("accepted_request", limit + 1), + ).fetchall() + truncated = len(admissions) > limit + admissions = list(reversed(admissions[:limit])) + accepted = [json.loads(payload) for _, _, payload in admissions] + if any(key != row.get("request_id") for (_, key, _), row in zip(admissions, accepted)): + raise ValueError("measurement admission identity mismatch") + request_ids = [row["request_id"] for row in accepted] + phases = [] + diagnostics = [] + if request_ids: + placeholders = ",".join("?" for _ in request_ids) + phases = self._conn.execute( + "SELECT kind, key, payload FROM orchestration_records " + "WHERE kind IN ('initial_decision', 'decision_receipt') " + "AND key IN (" + placeholders + ") " + "ORDER BY seq DESC LIMIT ?", + (*request_ids, 2 * limit + 1), + ).fetchall() + diagnostics = self._conn.execute( + "SELECT kind, key, payload FROM orchestration_records " + "WHERE kind IN ('provider_dispatch', 'auxiliary_dispatch') " + "AND key IN (" + placeholders + ") ORDER BY seq DESC LIMIT ?", + (*request_ids, 8 * limit + 1), + ).fetchall() + diagnostic_truncated = len(diagnostics) > 8 * limit + diagnostics = list(reversed(diagnostics[:8 * limit])) + if any(key != json.loads(payload).get("request_id") for _, key, payload in diagnostics): + raise ValueError("measurement diagnostic identity mismatch") + phase_truncated = len(phases) > 2 * limit + phases = list(reversed(phases[:2 * limit])) + if any(key != json.loads(payload).get("request_id") for _, key, payload in phases): + raise ValueError("measurement phase identity mismatch") + unresolved_legacy = self._conn.execute( + "SELECT 1 FROM orchestration_records WHERE kind IN (?, ?, ?, ?) " + "AND key IS NULL LIMIT 1", self._MEASUREMENT_KINDS, + ).fetchone() is not None + return { + "accepted": accepted, + "decisions": [json.loads(payload) for kind, _, payload in phases if kind == "initial_decision"], + "receipts": [json.loads(payload) for kind, _, payload in phases if kind == "decision_receipt"], + "diagnostics": [{"record_kind": kind, **json.loads(payload)} for kind, _, payload in diagnostics], + "window": {"limit": limit, "truncated": truncated, "phase_truncated": phase_truncated, + "diagnostic_limit": 8 * limit, "diagnostic_truncated": diagnostic_truncated, + "unresolved_legacy_identity": unresolved_legacy, + "first_admission_seq": admissions[0][0] if admissions else None, + "last_admission_seq": admissions[-1][0] if admissions else None}, + } + def _save_sync(self, kind: str, key: str | None, payload: dict[str, Any]) -> None: + if kind in self._MEASUREMENT_KINDS: + request_id = payload.get("request_id") + if not isinstance(request_id, str) or not request_id or (key is not None and key != request_id): + raise ValueError("measurement record requires a consistent request identity") + key = request_id blob = json.dumps(payload, ensure_ascii=False) - with self._lock: + with self._lock, self._conn: if kind in self._KEYED: self._conn.execute(self._DELETE_KEYED_SQL, (kind, key)) self._conn.execute(self._INSERT_SQL, (kind, key, blob)) if kind in self._STREAM_LIMITS: limit = self._STREAM_LIMITS[kind] self._conn.execute(self._PRUNE_STREAM_SQL, (kind, kind, limit)) - self._conn.commit() def _drain_stream_queue(self) -> None: while True: @@ -3781,6 +3863,15 @@ def load(self, kind: str, limit: int | None = None) -> list[dict[str, Any]]: rows = list(reversed(rows)) return [json.loads(row[0]) for row in rows] + def load_latest_key(self, kind: str, key: str) -> dict[str, Any] | None: + """Read one exact-key durable event using the existing identity index.""" + with self._lock: + row = self._conn.execute( + "SELECT payload FROM orchestration_records WHERE kind = ? AND key = ? " + "ORDER BY seq DESC LIMIT 1", (kind, key), + ).fetchone() + return json.loads(row[0]) if row is not None else None + def prune_keyed(self, kind: str, retained_keys: set[str]) -> None: """Delete keyed rows outside the caller's bounded in-memory set.""" if kind not in self._KEYED: @@ -4130,7 +4221,7 @@ def _reload_state(self) -> None: observation.get("irt_row", ()), ) for record in self._store.load("workflow_run"): - self._replace_workflow_run(record) + self._replace_workflow_run(record, restored=True) # A batch_route row persisted before judging (see batch_route's # own pending-record comment) carries an explicit # "pending_verification" marker and intentionally never reaches @@ -4329,6 +4420,7 @@ def proxy_completion( agent, upstream, effort_profile, api_surface=api_surface ) measured = bool(agent.group_name or requested_model == self.FREE_MODEL) + record_initial_selection([agent.id], "explicit_proxy") started_at = time.perf_counter() try: result = self.client.proxy_send(agent, endpoint, upstream) @@ -4412,6 +4504,7 @@ def proxy_completion( send_once = getattr(self.client, "proxy_send_once", None) if not callable(send_once): send_once = self.client.proxy_send + record_initial_selection([candidate.id], "automatic_proxy") result = send_once(candidate, endpoint, candidate_payload) except Exception as exc: # noqa: BLE001 - provider trust boundary if not _is_passthrough_failover_error(exc): @@ -5194,6 +5287,7 @@ def complete( ): result = copy.deepcopy(dict(cached)) result["cache_status"] = "hit" + record_answer_cache_hit() return result result = self._dispatch(messages, mode, model_name) try: @@ -5255,6 +5349,7 @@ def stream_route( stream_kwargs["effort_profile"] = effort_profile if include_usage: stream_kwargs["include_usage"] = True + record_initial_selection([agent.id], "stream_route") stream = self.client.stream_chat(agent, messages, **stream_kwargs) started_at = time.perf_counter() try: @@ -5316,6 +5411,8 @@ def stream_route( record["owner_id"] = owner_id self._replace_workflow_run(record) self._run_order.appendleft(record["workflow_run_id"]) + if self._store is not None: + self._store.save("workflow_run", record["workflow_run_id"], record) self._append_audit_event( "workflow_run_created", {"workflow_run_id": record["workflow_run_id"], "mode": "route", "agent_count": 1}, @@ -6780,11 +6877,12 @@ def _plan_generated(self, task: str) -> list[WorkflowStep]: {"role": "user", "content": task}, ] effort_profile = self._role_effort_profile("planner") - raw = ( - self.client.chat(planner, planner_messages, effort_profile=effort_profile) - if effort_profile is not None - else self.client.chat(planner, planner_messages) - ) + with observe_auxiliary_dispatch([planner.id], "generated_planner"): + raw = ( + self.client.chat(planner, planner_messages, effort_profile=effort_profile) + if effort_profile is not None + else self.client.chat(planner, planner_messages) + ) return self._parse_workflow_plan(raw) def _parse_workflow_plan(self, raw: str) -> list[WorkflowStep]: @@ -7215,7 +7313,8 @@ def _embed_cached(self, text: str) -> list[float] | None: if embedding_member is None: return None try: - vectors = self.client.embed(self._agent(embedding_member), [text]) + with observe_auxiliary_dispatch([embedding_member], "routing_evidence_embedding"): + vectors = self.client.embed(self._agent(embedding_member), [text]) except Exception: # noqa: BLE001 - similarity is best-effort evidence return None vector = vectors[0] if vectors else None @@ -7242,9 +7341,10 @@ def _descriptor_vector_cached(self, agent: ModelAgent) -> list[float] | None: if embedding_member is None: return None try: - vectors = self.client.embed( - self._agent(embedding_member), [self._agent_descriptor_text(agent)] - ) + with observe_auxiliary_dispatch([embedding_member], "routing_evidence_embedding"): + vectors = self.client.embed( + self._agent(embedding_member), [self._agent_descriptor_text(agent)] + ) except Exception: # noqa: BLE001 - similarity is best-effort evidence return None vector = vectors[0] if vectors else None @@ -7330,7 +7430,8 @@ def _compute_triage_verdict(self, text: str) -> bool: {"role": "user", "content": text}, ] try: - reply = self.client.chat(triage_agent, messages, temperature=0.0) + with observe_auxiliary_dispatch([triage_agent.id], "structured_triage"): + reply = self.client.chat(triage_agent, messages, temperature=0.0) return _parse_triage_reply(reply) except Exception: # noqa: BLE001 - fail closed toward verified orchestration return True @@ -7614,6 +7715,8 @@ def call(agent: ModelAgent) -> dict[str, Any] | tuple[bytes, str]: if agent.provider_name == "openrouter" and endpoint == "images/generations" else endpoint ) + record_initial_selection([member.id for member in race_members], "capability_race", + attempt_id=decision_attempt_id) return ( self.client.proxy_send_bytes(agent, provider_endpoint, payload) if binary else self.client.proxy_send(agent, provider_endpoint, payload) @@ -7621,6 +7724,7 @@ def call(agent: ModelAgent) -> dict[str, Any] | tuple[bytes, str]: contract = EndpointEquivalenceContract(**race_members[0].endpoint_equivalence) # type: ignore[arg-type] attempt_completed, finalize_attempts = self._race_attempt_collector(capability) + decision_attempt_id = uuid.uuid4().hex try: outcome = race_first_valid( [ @@ -7672,6 +7776,7 @@ def call(agent: ModelAgent) -> dict[str, Any] | tuple[bytes, str]: ) started_at = time.perf_counter() try: + record_initial_selection([agent.id], "capability_proxy") result = ( self.client.proxy_send_bytes(agent, provider_endpoint, payload) if binary @@ -7761,6 +7866,8 @@ def _invoke( def call(agent: ModelAgent) -> tuple[str, str, str, dict[str, Any] | None]: with self.client.request_settings(**request_settings): + record_initial_selection([member.id for member in race_members], "text_race", + attempt_id=decision_attempt_id) output = ( self.client.chat(agent, messages, effort_profile=effort_profile) if effort_profile is not None @@ -7771,6 +7878,7 @@ def call(agent: ModelAgent) -> tuple[str, str, str, dict[str, Any] | None]: contract = EndpointEquivalenceContract(**race_members[0].endpoint_equivalence) # type: ignore[arg-type] attempt_completed, finalize_attempts = self._race_attempt_collector("text") + decision_attempt_id = uuid.uuid4().hex try: outcome = race_first_valid( [ @@ -7833,6 +7941,7 @@ def call(agent: ModelAgent) -> tuple[str, str, str, dict[str, Any] | None]: single_attempt = getattr(self.client, "single_attempt_transport", None) transport_scope = single_attempt() if callable(single_attempt) else nullcontext() with transport_scope: + record_initial_selection([agent.id], "invocation_" + role) output = ( self.client.chat(agent, messages, effort_profile=effort_profile) if effort_profile is not None @@ -8692,7 +8801,7 @@ def _run_budget_output_by_model( ) return output_by_model, True - def _replace_workflow_run(self, record: dict[str, Any]) -> None: + def _replace_workflow_run(self, record: dict[str, Any], *, restored: bool = False) -> None: """Store one run and update its constant-time budget meter atomically.""" model_by_agent = {agent.id: agent.model for agent in self.candidates} for step in record.get("trace", []): @@ -8702,6 +8811,13 @@ def _replace_workflow_run(self, record: dict[str, Any]) -> None: run_id = record["workflow_run_id"] with self._budget_spend_lock: previous = self._workflow_runs.get(run_id) + origin_request_id = (previous.get("request_id") if previous is not None + else record.get("request_id") if restored + else current_request_id()) + if origin_request_id is not None: + record["request_id"] = origin_request_id + else: + record.pop("request_id", None) for sign, run in ((-1, previous), (1, record)): if run is None: continue diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index eb6a77519..bf194b8c4 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -15,6 +15,7 @@ import secrets import socket import struct +import sys import tempfile import threading import time @@ -25,6 +26,7 @@ import uuid from .admin import ADMIN_HTML, ADMIN_TRANSLATIONS +from .decision_receipts import DecisionMeasurement, export_decision_receipts from .api_contract import OPENAPI_SPEC from .cost_ledger import ATTRIBUTION_DIMENSIONS, dimension_catalog from .cost_router import ( @@ -65,6 +67,8 @@ attach_trace_context, configure_telemetry, current_session_id, + current_request_id, + request_identity, detach_trace_context, reset_session_id, session_id_from_headers, @@ -5474,6 +5478,7 @@ def build_server( clearfolio_url: str | None = None, coordinator: CostRoutingCoordinator | None = None, release_authority: Mapping[str, Any] | None = None, + decision_receipts: bool = False, ) -> ThreadingHTTPServer: """Build, but do not start, the orchestration HTTP server. @@ -5482,6 +5487,12 @@ def build_server( every completion is priced, recorded, and sync/batch routed. """ security = security or SecurityConfig() + if type(decision_receipts) is not bool: + raise TypeError("decision_receipts must be a boolean") + if decision_receipts: + from ._decision_receipt import DecisionReceipt # noqa: F401 + if orchestrator._store is None: + raise ValueError("decision receipts require a durable state store") security.check_bind(host) release_authority = verify_release_authority_snapshot(release_authority) coordinator = coordinator or CostRoutingCoordinator(orchestrator) @@ -5498,6 +5509,7 @@ class Handler(BaseHTTPRequestHandler): """Handle authenticated orchestration, administration, and health routes.""" _session_token = None _trace_token = None + _classification_slot_held = False def _bind_session(self, session_id: str | None) -> None: """Bind validated request correlation to this handler context.""" @@ -5627,11 +5639,20 @@ def handle_one_request(self) -> None: self.command = None self.path = None self._request_started = None - try: - super().handle_one_request() - finally: - self._log_request_summary(self._request_started) - self._reset_session() + self._decision_measurement = None + self._decision_failure_reason = "unfinished" + self._classification_slot_held = False + with request_identity(): + try: + super().handle_one_request() + finally: + if self._classification_slot_held: + self._release_measured_slot() + if self._decision_measurement is not None: + self._decision_measurement.close(self._decision_failure_reason) + self._decision_measurement = None + self._log_request_summary(self._request_started) + self._reset_session() # A request that declared a body it never delivered (unsupported # method, rejected route) must not leave those bytes on a reusable # connection for the stdlib to reparse as the next request. @@ -5689,13 +5710,15 @@ def _log_request_summary(self, started: float | None) -> None: if not method and not path and status is None: return _LOGGER.info( + "%s request_id=%s", summarize_request_for_log( method=method or "-", path=path or "-", status=status, latency_ms=(time.monotonic() - (started or time.monotonic())) * 1000.0, session_id_hash=session_id_hash(), - ) + ), + current_request_id() or "-", ) def do_GET(self) -> None: # noqa: N802 @@ -5976,7 +5999,10 @@ def do_GET(self) -> None: # noqa: N802 self._send(orchestrator.provider_readiness_report(refresh=raw_refresh == "true")) return if path == "/api/v1/analytics_snapshots/latest": - self._send(orchestrator.analytics_snapshot(locale_bundles=ADMIN_TRANSLATIONS)) + snapshot = orchestrator.analytics_snapshot(locale_bundles=ADMIN_TRANSLATIONS) + if decision_receipts: + snapshot["initial_decision_measurements"] = export_decision_receipts(orchestrator._store) + self._send(snapshot) return if path == "/api/v1/spend_analytics/latest": self._send(orchestrator.spend_analytics()) @@ -7098,19 +7124,6 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di return messages = _validate_messages(body.get("messages")) mode = _validate_mode(body.get("orchestration") or body.get("orchestration_mode") or body.get("mode") or "auto") - route_stream = bool( - stream and orchestrator.would_route(messages, mode, model_name) - ) - if route_stream: - if explicit_trace: - raise RequestError( - 400, - "unsupported_trace_disclosure", - "remove include_orchestration_trace or use Responses streaming", - ) - include_trace = False - elif include_trace: - self._authorize_trace_access() # stream + stream_options already coerced/validated before passthrough. attribution = _validate_attribution(body.get("attribution")) routing = _validate_routing( @@ -7134,6 +7147,25 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di # sampling/controls already validated before passthrough branch. if "metadata" in body: _validate_openai_metadata(body) + if explicit_trace: + self._authorize_trace_access() + self._ensure_decision_measurement("validated_endpoint") + if stream: + self._acquire_measured_slot() + self._classification_slot_held = True + route_stream = bool( + stream and orchestrator.would_route(messages, mode, model_name) + ) + if route_stream: + if explicit_trace: + raise RequestError( + 400, + "unsupported_trace_disclosure", + "remove include_orchestration_trace or use Responses streaming", + ) + include_trace = False + elif include_trace and not explicit_trace: + self._authorize_trace_access() started_at = time.perf_counter() model_client = orchestrator.client with model_client.request_settings( @@ -7150,6 +7182,7 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di messages, model_name, include_usage=include_usage, + slot_acquired=True, ) orchestrator.record_analytics_event( "chat_completion_requested", @@ -7229,13 +7262,6 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di ) # Same pool honesty as chat/Completions: do not silently serve # a different embedding deployment than the client requested. - embedding_agents = orchestrator._capability_agents( - "embedding", - TaskOrchestrator.AUTO_MODEL if model_was_omitted else model_name, - ) - embedding_agents = coordinator._cost_ordered_capability_candidates( - embedding_agents - ) encoding_format = _validate_embeddings_encoding_format(body) _validate_embeddings_dimensions(body) end_user_id = _validate_completions_user(body) @@ -7285,6 +7311,12 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di attribution["model_name"] = model_name if not attribution.get("service"): attribution["service"] = "embeddings_api" + self._ensure_decision_measurement("validated_endpoint") + embedding_agents = coordinator._cost_ordered_capability_candidates( + orchestrator._capability_agents( + "embedding", TaskOrchestrator.AUTO_MODEL if model_was_omitted else model_name + ) + ) started_at = time.perf_counter() embedding_deadline = time.monotonic() + float( orchestrator.client.timeout @@ -7371,13 +7403,6 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di _require_pool_model( orchestrator, model_name, required_capability="embedding" ) - embedding_agents = orchestrator._capability_agents( - "embedding", - TaskOrchestrator.AUTO_MODEL if model_was_omitted else model_name, - ) - embedding_agents = coordinator._cost_ordered_capability_candidates( - embedding_agents - ) _validate_embeddings_encoding_format(body) _validate_embeddings_dimensions(body) # OpenAI ``user`` end-user id — same fail-closed shape as sync embeddings. @@ -7396,6 +7421,12 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di endpoint_alias = _validate_batch_embeddings_endpoint(body) if endpoint_alias is not None: submit_metadata["endpoint_alias"] = endpoint_alias + self._ensure_decision_measurement("validated_endpoint") + embedding_agents = coordinator._cost_ordered_capability_candidates( + orchestrator._capability_agents( + "embedding", TaskOrchestrator.AUTO_MODEL if model_was_omitted else model_name + ) + ) document = None last_embedding_error: Exception | None = None for embedding_agent in embedding_agents: @@ -7454,6 +7485,7 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di batch_requests, metadata=metadata, owner_id=security.principal_id(self.headers), + request_id=current_request_id(), ) ) except InvalidBatchModelError as exc: @@ -7474,6 +7506,10 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di "backend": job.backend, "status": job.status, "request_count": job.request_count, + "request_link_status": job.request_link_status, + "registry_persistence_status": job.registry_persistence_status, + "recovery_status": job.recovery_status, + "backend_registry_persistence_status": job.backend_registry_persistence_status, }, 201) return if path.startswith("/api/v1/batch_routing_jobs/") and path.endswith("/results"): @@ -8129,11 +8165,60 @@ def _validate_trace_request(self, body: dict[str, Any]) -> bool: return include_trace def _run(self, callback: Any) -> dict[str, Any]: - security.acquire_run_slot() + if not self._classification_slot_held: + self._acquire_measured_slot() try: return callback() + finally: + self._release_measured_slot() + + def _acquire_measured_slot(self) -> None: + """Retain accepted admission before the nonblocking capacity decision.""" + self._ensure_decision_measurement() + try: + security.acquire_run_slot() + except Exception: + self._decision_failure_reason = "capacity_rejected" + raise + + def _ensure_decision_measurement(self, boundary="first_execution_slot") -> None: + """Open one request scope, with explicit timing eligibility at this boundary.""" + if decision_receipts and self._decision_measurement is None: + endpoint = urllib.parse.urlsplit(self.path).path + if endpoint not in { + "/v1/chat/completions", "/v1/completions", "/v1/responses", + "/v1/embeddings", "/v1/batch/embeddings", + }: + endpoint = "other_execution_endpoint" + try: + self._decision_measurement = DecisionMeasurement( + orchestrator._store, policy=orchestrator.policy, + endpoint_path=endpoint, request_method=self.command, + admission_boundary=boundary, + request_id=current_request_id(), + ) + except RuntimeError: + raise RequestError( + 503, "measurement_unavailable", + "Request measurement is unavailable; retry after service recovery.", + {"measurement_complete": False, "reconciliation_required": True}, + ) from None + + def _release_measured_slot(self) -> None: + """Finalize measurement independently of releasing the execution slot.""" + try: + measurement = self._decision_measurement if decision_receipts else None + if measurement is not None: + active_error = sys.exc_info()[1] + reason = self._decision_failure_reason + if isinstance(active_error, (ConnectionError, BrokenPipeError, GeneratorExit)): + reason = "cancelled" + elif active_error is not None: + reason = "selection_failed" + self._decision_failure_reason = reason finally: security.release_run_slot() + self._classification_slot_held = False def _parse_positive_int(self, raw: str | None, field_name: str, default: int, max_value: int | None = None) -> int: value = default if raw is None else int(raw) @@ -8193,8 +8278,17 @@ def _send_error( message: str, detail: dict[str, Any] | None = None, ) -> None: - _LOGGER.warning("request_failed status=%s code=%s", status, code) - self._send(_error_payload(code, message, {"request_id": uuid.uuid4().hex, **(detail or {})}), status) + if decision_receipts and status >= 400: + measurement = self._decision_measurement + if (measurement is not None + and measurement.receipt.status in ("accepted", "selected") + and self._decision_failure_reason == "unfinished"): + self._decision_failure_reason = "selection_failed" + request_id = current_request_id() or uuid.uuid4().hex + _LOGGER.warning( + "request_failed status=%s code=%s request_id=%s", status, code, request_id + ) + self._send(_error_payload(code, message, {**(detail or {}), "request_id": request_id}), status) def _write_response(self, writer: Callable[[], None]) -> bool: """Run a response-writing callback, swallowing a dead-peer disconnect. @@ -8243,6 +8337,8 @@ def _write_response(self, writer: Callable[[], None]) -> bool: writer() return True except (BrokenPipeError, ConnectionError, OSError): + if decision_receipts: + self._decision_failure_reason = "cancelled" _LOGGER.debug("client_disconnected") # `_send*`/`_begin_sse` writers record their *intended* # status in `self._last_status` before calling this method @@ -8438,7 +8534,10 @@ def progress(role: str, status: str) -> None: part={"type": "summary_text", "text": text}, ) - security.acquire_run_slot() + if decision_receipts: + self._acquire_measured_slot() + else: + security.acquire_run_slot() try: if not self._begin_sse(): return False @@ -8486,21 +8585,24 @@ def progress(role: str, status: str) -> None: conduct_kwargs["workflow_run_id"] = f"run_{uuid.uuid4().hex}" result = orchestrator.conduct(messages, **conduct_kwargs) except ConnectionAbortedError: + self._decision_failure_reason = "cancelled" raise except ProviderUpstreamError as exc: + self._decision_failure_reason = "selection_failed" failed = { **created_response, "status": "failed", "error": _error_payload( exc.error_code, _provider_upstream_message(exc), - {"request_id": uuid.uuid4().hex, **exc.detail}, + {**exc.detail, "request_id": current_request_id() or uuid.uuid4().hex}, )["error"], } emit("response.failed", response=failed) self._write_sse("data: [DONE]\n\n") return False except Exception: # noqa: BLE001 - headers sent; terminate with a valid Responses event + self._decision_failure_reason = "selection_failed" failed = { **created_response, "status": "failed", @@ -8595,9 +8697,13 @@ def progress(role: str, status: str) -> None: self._write_sse("data: [DONE]\n\n") return True except ConnectionAbortedError: + self._decision_failure_reason = "cancelled" return False finally: - security.release_run_slot() + if decision_receipts: + self._release_measured_slot() + else: + security.release_run_slot() def _stream_route_completion( self, @@ -8607,6 +8713,7 @@ def _stream_route_completion( model_name: str, *, include_usage: bool = False, + slot_acquired: bool = False, ) -> None: """Pipe live provider deltas as OpenAI chat-completion SSE frames.""" run_id = f"run_{uuid.uuid4().hex}" @@ -8643,7 +8750,11 @@ def usage_frame(usage: dict[str, Any]) -> str: } return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" - security.acquire_run_slot() + if not slot_acquired: + if decision_receipts: + self._acquire_measured_slot() + else: + security.acquire_run_slot() try: if not self._begin_sse() or not self._write_sse( frame({"role": "assistant"}) @@ -8670,9 +8781,10 @@ def usage_frame(usage: dict[str, Any]) -> str: ): return except ToolFallbackStoppedError as exc: + self._decision_failure_reason = "selection_failed" detail = { - "request_id": uuid.uuid4().hex, **_tool_fallback_error_detail(exc), + "request_id": current_request_id() or uuid.uuid4().hex, } payload = _error_payload( TOOL_FALLBACK_STOPPED_CODE, @@ -8686,10 +8798,11 @@ def usage_frame(usage: dict[str, Any]) -> str: if not self._write_sse(frame({}, finish="error")): return except ProviderUpstreamError as exc: + self._decision_failure_reason = "selection_failed" payload = _error_payload( exc.error_code, _provider_upstream_message(exc), - {"request_id": uuid.uuid4().hex, **exc.detail}, + {**exc.detail, "request_id": current_request_id() or uuid.uuid4().hex}, ) if not self._write_sse( f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" @@ -8698,11 +8811,15 @@ def usage_frame(usage: dict[str, Any]) -> str: if not self._write_sse(frame({}, finish="error")): return except Exception: # noqa: BLE001 - headers already sent; surface as a terminal error frame + self._decision_failure_reason = "selection_failed" if not self._write_sse(frame({}, finish="error")): return self._write_sse("data: [DONE]\n\n") finally: - security.release_run_slot() + if slot_acquired or decision_receipts: + self._release_measured_slot() + else: + security.release_run_slot() def _send_security_headers(self) -> None: if getattr(self, "close_connection", False): diff --git a/contextual_orchestrator/telemetry.py b/contextual_orchestrator/telemetry.py index be1295947..e710d7112 100644 --- a/contextual_orchestrator/telemetry.py +++ b/contextual_orchestrator/telemetry.py @@ -6,6 +6,7 @@ import ipaddress import logging import re +import uuid from collections.abc import Iterator, Mapping from contextlib import contextmanager from contextvars import ContextVar, Token @@ -38,6 +39,25 @@ "contextual_orchestrator_session_id", default=None ) _CONFIGURED = False +_CURRENT_REQUEST_ID: ContextVar[str | None] = ContextVar("gateway_request_id", default=None) + + +def current_request_id() -> str | None: + """Return the server-generated request identity, never a caller value.""" + return _CURRENT_REQUEST_ID.get() + + +@contextmanager +def request_identity() -> Iterator[str]: + """Bind a fresh identity and restore the enclosing context on every exit.""" + request_id = uuid.uuid4().hex + request_token = _CURRENT_REQUEST_ID.set(request_id) + try: + yield request_id + finally: + _CURRENT_REQUEST_ID.reset(request_token) + + # Match the OpenTelemetry SDK's default span-attribute budget so a single # sequence-valued attribute cannot exceed the span's default evidence budget. _MAX_ATTRIBUTE_SEQUENCE_ITEMS = 128 diff --git a/docs/doctoring/autonomous_kpi_runbook.md b/docs/doctoring/autonomous_kpi_runbook.md index 692fc4d74..d50532388 100644 --- a/docs/doctoring/autonomous_kpi_runbook.md +++ b/docs/doctoring/autonomous_kpi_runbook.md @@ -223,6 +223,45 @@ UI acceptance. Viewport dimensions, durable image export, responsive sizes, other locales, and error/loading states remain unverified. Do not mark the full visual-inspection requirement complete from this receipt. +## State-write atomicity prerequisite (2026-09-09) + +At CO `ab8a7caa6c00a49eede17a03d0897865cfdce9f5`, `run()` calls +`complete()` before saving its workflow record. Its duration includes generation +and cannot represent accepted-request-to-decision latency. Analytics stream +`save()` normally enqueues, whereas `durable=True` reaches the synchronous +commit path. Neither queue acceptance nor final workflow storage is a substitute +for a pre-invocation decision acknowledgement. + +Tracing that common synchronous path exposed an atomicity defect: a keyed +replacement deletes the old row before inserting the new one. An insertion +failure left the deletion in an open transaction, and a subsequent unrelated +save committed it. A real SQLite trigger injecting an insertion failure reproduced +the loss: the previous version disappeared. The new regression failed before +the fix (1 failed, 18 deselected, 6.17s). + +Code commit `d7bba88f3d711883a37effe49ab4f503c4fb8e01` uses the existing +connection's transaction context inside the existing lock. Success commits; +failure rolls back before another writer acquires the lock. No new dependency, +schema, numerical implementation, or production routing default was introduced. +The same project-local Python environment ran `python -m pytest +tests/test_persistence.py -q` from the isolated worktree: 19 passed in 16.38s. +This establishes the tested SQLite failure case, not customer KPI improvement, +full-suite success, protected merge, or production deployment. Next implement +decision timing before provider invocation with explicit failure denominators; +do not relabel existing response-generation timings. + +Follow-up validation: full `python -m pytest -q` at +`877d5112ed470d851afaa2c746b94393cc768ee7` exited 0 with 3,396 passed and +2 skipped in 883.03s. The quiet log does not identify the skip reasons; neither +skip is counted as passed. Test-only commit +`716e012dcb50857000b0fc53c89c6434fdf7e7c2` extends the regression to a real +deferred foreign-key violation at commit, in addition to insertion failure. +The existing transaction implementation is unchanged. At that commit, +`python -m pytest tests/test_persistence.py +tests/test_workflow_run_object_authorization.py tests/test_governance_runtime.py -q` +exited 0: 29 passed in 4.89s. The new test head has not had a full-suite run; +these local results do not establish hosted security checks or independent review. + ## Break release cycles without copying implementation Minimum contract → owner RED test → owner implementation → exact-SHA/digest diff --git a/docs/doctoring/batch_request_lineage.md b/docs/doctoring/batch_request_lineage.md new file mode 100644 index 000000000..d8130d517 --- /dev/null +++ b/docs/doctoring/batch_request_lineage.md @@ -0,0 +1,172 @@ +# Deferred batch request lineage + +## RED reproduction, 2026-09-09 + +Base `af8d732e6cfc9c0169ac850f875f42f1db7eecd4`; test commit +`387aa211` on isolated branch `codex/batch-request-lineage-20260909`. +Command: `.venv/bin/python -m pytest tests/test_batch_request_lineage.py -q`. +Terminal result: **1 failed in 3.75s**. Submission returned HTTP 201; +retrieval returned HTTP 200 with two distinct caller custom IDs. Passive backend +observations confirmed distinct trusted submission/retrieval request identities. +After closing and reopening SQLite, the proposed submission/job/item association +was absent: an empty set instead of two links. No runtime implementation changed. + +This test proposes the `batch_request_link` association shape; that shape is not +an existing released contract. It demonstrates the missing durable association, +not a defect in an already-promised schema. The fixture reuses the existing +offline pg-llm-batch client double. No external model was invoked. This is not +remote integration, process-restart batch execution, numerical accuracy, or latency +evidence. The fixture preserves results in test memory across state-store reload; +it does not claim that the default in-memory batch registry survives restart. + +The smallest repair belongs in CO's submission coordinator and durable job +association boundary, with a trusted identity argument from the HTTP adapter. +Preserve multiple submission/job/item associations and keep provider custom IDs +separate. Do not reassign origin when a later HTTP request retrieves outcomes. +Remote execution provenance requires the released pg-llm-batch contract separately. + +Environment: project-local Python 3.14.6, dependencies synchronized with +`uv sync --frozen`. An initial pytest-only environment failed collection because +certifi was absent; this was an environment error, not the RED result. The attempted +`uv sync --frozen --group test` was invalid because test is an optional extra, +not a dependency group; the frozen default synchronization repaired collection. +CodeGraph initialization completed (438 files); no other worktree was modified. + +## Candidate implementation and review repair + +`62239624`: focused lineage, batch backend, and cost-review HTTP tests passed +**42 tests in 12.33s**. Includes existing cross-principal denial coverage. +The initial status/failure contract at `919c945a` failed twice in 1.54s; initial +implementation `9356ac62` passed 39 tests in 14.35s. Follow-up repeated-origin, +repeated-retrieval and no-store cases passed 41 tests at `b565173e` in 11.63s. + +The association is an append-only typed event in the existing state store: +`request_id`, `batch_job_id`, `custom_ids`, and `owner_id`. A single event commits +the complete submission cohort using the existing rollback-safe transaction. +Consumers project one association per custom ID; this replaces the original +RED's proposed per-item event shape. It is not a new relational entity claimed +to satisfy 3NF. Item IDs remain job-scoped and repeated submissions append, +never replace prior origins. No prompt or answer enters the event. + +HTTP 201 means the upstream job was submitted, not that all local state is +durable. `request_link_status` is `durable`, `write_failed`, or `unavailable`. +After remote success, a failed association commit returns the original handle +with `write_failed` and no raw exception. It never automatically resubmits. +No-store/library calls remain supported with unavailable lineage. This response +diagnostic is not reconstructed from remote registry snapshots after restart; +the committed association event is the durable evidence. + +Review found an added second registry assignment could lose the successful +remote handle behind a new exception. Regression `b5baf420` failed once in +0.39s; `62239624` removes that assignment. Existing Valkey writes are separate +`hset` and `expire` calls, not atomic with SQLite. Failure of the original +registry assignment after remote submission remains an unresolved existing +ambiguity; no cross-store transaction, complete recovery, retention/export +window, API reconciliation surface, or remote integration is claimed here. +The event is currently only consumed by the test projection. Full-suite, +package, hosted checks, independent approval, and protected release are pending. + +## Recovery successor (supersedes status-only limitations above) + +Actual HTTP restart regression `d71bcc0a` failed once in 1.96s: an accepted +remote job returned 201 despite a failed registry write, but its rightful owner +received 404 after restart. A different authenticated owner also received 404. +This established unrecoverable work, not just missing status metadata. + +Candidate `853e8609` passes **73 focused tests in 21.17s** across lineage, +batch routing, cost-review HTTP, and registry files. New coverage distinguishes +HSET failure (no handle stored) from expiry failure (HSET partially applied), +and returns the original upstream handle without a second submission. Earlier +`18af2947` expiry tests incorrectly inspected an unprefixed fake-registry key; +`b12981d6` corrected the test and passed ten cases in 6.18s. + +The single durable submission event now includes a prompt-free recovery +descriptor: exact backend name/endpoint alias/endpoint, owner-bound job snapshot, +original token estimates, expected job-scoped item IDs, model/mode/normalized +attribution, and expiry tied to the configured registry retention. This is only +supported for PgLlmBatchBackend, not arbitrary local or embedding backends. +No prompts or credentials are included. Absent source messages remain unavailable; +their contents are never fabricated for token estimates. + +An indexed exact-key lookup permits the original owner to recover a missing +registry handle after SQLite reload. Wrong owner, expired descriptor, malformed +backend object, or changed backend target fail closed. Restored expected item +IDs retain the existing response-identity validator; unexpected result IDs are +rejected. Missing usage remains estimated or unavailable, never relabeled measured. +The malformed-backend test initially returned 500 at `65b41603` (one failure, +five passes in 9.90s); explicit type validation changed that to a concealed 404. + +Request-link status is finalized before the single registry write and survives +Valkey decoding consistently. Registry-write outcome is response-only and is +excluded from dataclass serialization, since HSET may apply before expiry fails. +`stored` means that configured registry operation returned, not proof of process +restart durability for a local dictionary. HTTP 201 still means remote submission; +clients must preserve the handle and must not resubmit solely because local +persistence is incomplete. If both SQLite and the registry fail, automatic recovery +remains unavailable. This is a tested recovery slice, not complete cross-store +atomicity, retention cleanup, production integration, or customer KPI evidence. + +## Independent review repairs and injected-adapter configuration + +At `a6b94855`, the four-file focused suite passes **84 tests in 32.63s**. +Intermediate failures and successful checks remain attributed to their commits: + +| Source | Check | Result | +| --- | --- | --- | +| `e372bc54` | Registry outage, inconsistent item IDs and estimate keys | 3 failed, 3.59s | +| `7fb1a71c` | Lineage file after recovery isolation | 19 passed, 14.94s | +| `83394afa` | Malformed job fields and deployment binding added | 23 passed, 18.26s | +| `f4d036bf` | Healthy coordinator handle, missing backend metadata | 1 failed, 1.37s | +| `6cb530a7` | Four-file suite | 81 passed, 29.90s | +| `42ad396c` | Duplicate persisted item IDs | 1 failed, 1.52s | +| `aaa9b133` | Four-file suite | 82 passed, 31.21s | +| `17cfa611` | Backend registry write fails after remote creation | 1 failed, 0.95s | +| `8ddfeb9f` | Same backend-failure HTTP test | 1 passed, 1.79s | +| `df638d6c` | Healthy active job with expired recovery descriptor | 1 failed, 1.51s | + +The Pg adapter now retains the upstream handle even when its own metadata +registry write fails before returning to the coordinator. The response separates +`backend_registry_persistence_status`, `registry_persistence_status`, +`request_link_status`, and `recovery_status`. The first two describe individual +write outcomes, not restart durability. HSET may apply before expiry fails; +response-only write results are not serialized as claims about their own writes. +Missing durable lineage still remains explicit; no handler repeats submission. + +Recovery tolerates continuing coordinator/backend registry outages. Validated +item metadata is carried on the returned job for that retrieval, without writing +back through the failed registry. The stored envelope's item list, descriptor +items, job count, and estimate keys must agree before downloading. Invalid count +types and null estimates fail closed. A healthy authorized registry job retains +its original refresh-on-read lifecycle even if its separate fixed-deadline +recovery descriptor has expired; missing metadata cannot use expired recovery. + +There is no built-in production Pg adapter constructor in this source tree: +the CLI/default coordinator uses the local backend. Integrators injecting +`PgLlmBatchBackend` configure its optional `recovery_identity` argument with a +stable, non-secret service/deployment/account identifier. Keep this identifier +stable during credential rotation and change it when the service or account +changes. Do not use a credential, infer equivalence from an endpoint alias, or +accept an HTTP caller's value. The default `None` disables durable recovery, +and submission reports `recovery_status=unavailable`; a committed, explicitly +bound Pg descriptor reports `durable_descriptor`. A different or missing binding +cannot recover the old job, even if endpoint aliases are identical. + +The real-HTTP tests use offline injected clients; no deployed integration is +claimed. Simultaneous loss of both durable submission evidence and job registries +still cannot promise recovery. Full suite, clean wheel, hosted review and release +remain pending. Do not stop at a status-only response when recoverable evidence +exists, and do not label a retained remote handle complete recovery by itself. + +## Final focused acceptance before full regression + +Runtime `45066759`: **86 passed in 36.58s** across the four focused files. +The last review exposed inconsistent endpoint binding on healthy metadata: +`c06615ab` failed once in 2.09s. New metadata now stores the exact endpoint and +stable deployment binding before the healthy fast path can use it. A changed +binding, changed endpoint, or missing binding on opted-in metadata cannot bypass +the validated descriptor path. This fixes a contract inconsistency; no cross-service +data leak was established by the offline test. Unbound legacy metadata remains +compatible only with an unbound backend; its missing identity is never described +as validated recovery. The legacy-focused suite passed 85 tests in 35.09s at +`2d68f705`. Independent review accepted freezing this bounded implementation for +full regression and separate installed-artifact verification, not publication. diff --git a/docs/doctoring/decision_receipt_integration.md b/docs/doctoring/decision_receipt_integration.md new file mode 100644 index 000000000..46530d3ce --- /dev/null +++ b/docs/doctoring/decision_receipt_integration.md @@ -0,0 +1,171 @@ +# Initial decision receipt integration candidate + +Status: incomplete implementation for issue #1110; not release evidence. + +RED `85a0b2d2` exposed the missing HTTP measurement option (1 failed, +3.67 s). The working candidate compiles a separate PyO3 extension using the +workspace's existing pyo3 0.29.2 lock resolution. It introduces no new +third-party dependency. The first build failed because a field-level getter +attribute was invalid; explicit read-only getters corrected compilation. +Context7 lookup hit its monthly quota; local compiler/API checks were used. + +The separate Rust receipt owns an Instant clock and state transitions. Python +only associates that receipt with the existing HTTP context and SQLite store; +it computes no durations or statistical estimates. Opt-in requires the native +extension and a configured durable store. The ordinary callback admission and +two direct SSE admissions establish scopes before the nonblocking capacity check. +Initial selection hooks cover _invoke, streaming, generated planning, and +plain/capability proxy candidates. These source hooks are not all validated. + +The initial decision is synchronously committed before the native acknowledgement +timestamp is captured. A separate receipt is exported at scope exit; that export +cannot be its own acknowledgement. A process crash between these writes leaves +an initial decision without a receipt, which must remain in the unfinished +denominator. Persisted successful receipts alone are not an all-request sample. + +Focused verification in the isolated local Python 3.14 environment: 4 tests +passed in 1.49 s. The real HTTP route's dispatch callback queries a separate +SQLite connection (never flush-on-read store.load), verifies the committed +decision, captures its native acknowledgement, and holds answer generation +behind an event. The final acknowledgement must equal that pre-generation value. +Other tests cover native invalid/duplicate acknowledgement, missing-store +startup rejection, write failure, snapshot isolation and secret-safe errors. +Controlled mock provider output is unit/integration evidence, not customer data. + +Remaining before review/merge: direct coordinator/structured synthesis and all +SSE/provider paths need executable coverage; metadata must bind policy revision +and route mode; unsupported and selection/cancellation outcomes need complete +denominator tests; race size/contract/preparation and concurrency need negative +tests; both receipt kinds need an explicit retention/export contract. Validate +actual wheel installation outside the source checkout (the current editable +extension is not package proof). Add the Proposed ADR with the completed seam +inventory, dependency graph and rendering inspection. Do not claim production +instrumentation, p95, accuracy improvement, protected CI, or release. + +## Admission and SSE follow-up + +At candidate descended from `df6eb4a7`, ten focused tests passed in 4.72 s. +These include ordinary HTTP chat, direct chat SSE, direct Responses SSE, +actual race worker context propagation and full candidate-set identity, and +oversized-race rejection before acknowledgement. The first race test fixture +omitted the required group identity and correctly exercised sequential routing; +adding the operator-contract group activated the intended race path. + +An immutable accepted_request row now precedes the capacity check. Export joins +that row with initial_decision and final decision_receipt rather than dropping +unfinished requests. Rows are append-only and not automatically pruned; operators +must retain them for reconciliation and archive after verified export. A storage +outage can prevent the admission itself from being recorded: opt-in then returns +503 before dispatch, states measurement_complete=false and reconciliation_required=true, +and requires external ingress evidence. The local export never claims all-ingress +completeness. This is a deliberate limitation, not a zero-duration observation. + +The existing analytics snapshot carries the joined records only with measurement +enabled and retains its local-runtime labeling. A separate Rust clock remains +the only duration implementation. A trusted request_id argument is the explicit +port for PR #1105; until integrated, identity_source=measurement_scope is honest. +The policy snapshot is hashed; route_mode currently identifies the dispatch +kind/role, not a complete top-level requested-mode contract. + +An isolated noneditable core install plus native wheel, outside the checkout, +passed six tests (one then-current test deselected) in 9.13 s. Both imports were +verified under the isolated environment's site-packages. That build used an +uncommitted candidate and is not final exact-head package evidence. Dependencies +resolved anew there; locked PyO3 remained 0.29.2. Rebuild the final commit before +release. No binary is committed. + +Still unverified: cache-hit and other callback-bypassing accepted-request coverage, +all structured/coordinator/direct-provider paths, selection/cancellation/capacity +HTTP negatives, complete ingress reconciliation, transactional snapshot export, +declared retention window and actual release packaging workflow. No customer KPI +gain is established. The Proposed ADR and full rendering inspection remain due. + +## Integrated request identity and indexed cohort checkpoint + +At `fe2db40e`, receipt, persistence and debug/correlation suites passed together: +54 passed in 9.26 s. Normal merges preserve rollback owner #1108 at `129a6650` +and request identity owner #1105 at `b655fe1b`. The handler now owns one +measurement lifetime across repeated slot acquisitions, and passes its trusted +request identity explicitly. Embedding fallback tests observe one admission and +two backend selections; explicit validated-endpoint admission precedes embedding +candidate ordering. Other endpoints remain labeled first_execution_slot and +must not be assumed to include prior selection work. + +`6e4f87aa` adds a shared per-race invocation identity: replicas deduplicate within +one invocation, while a second race in the same request remains a new attempt. +The regression first failed with one attempt instead of two, then passed. +The earlier route-mode-only deduplication is superseded. + +The state store backfills null keys only for valid measurement JSON identities +inside its startup transaction, never overwrites non-null keys, and adds the +(kind, key, seq) index. Its 2,000-measurement-row unit fixture plus malformed +unrelated row retains all 2,001 rows. EXPLAIN for the actual phase query shows +kind/key index search; this is query-plan evidence, not a measured latency gain. +Export reads a bounded shared admission cohort, exposes sequence/truncation +boundaries and unresolved legacy identities, and never independently prunes +phase rows. Historical storage remains append-only pending an archival policy. + +An exact `01ce9035` native wheel and separately built noneditable core package +passed 10 tests in 4.90 s outside the checkout with isolated imports. That proof +does not cover the later merged runtime: rebuild the final candidate again. + +Outstanding semantic gap: structured triage and ranking-evidence embeddings may +invoke providers before the current task-selection hook. Initial provider +dispatch and final task-route decision must be distinguished; no current receipt +is evidence of the headline routing-decision p95. Cache hits, non-generation +operations, auxiliary dispatch, all preselection SSE failures and cancellation +paths still require complete request-boundary coverage and executable evidence. +No protected merge, production publication, or customer KPI gain is claimed. +# Auxiliary and package acceptance checkpoint (2026-09-09) + +The later frozen `3b6dd47ebb0f88802bacdd302051d2f03e7d5003` full suite passed +3,442 tests with two skips in 771.43 s; its noneditable wheel-only receipt and SSE +identity matrix passed 33 tests in 14.21 s. Independent saturation inspection then +found chat streaming could call triage before acquiring capacity, despite a 503 +response. This was present with measurement disabled too; sibling nonstream chat +and Responses streaming already rejected without a provider call. + +The capacity repair holds one explicit chat-classification lease through either +the direct stream or conducted `_run` path. Request finalization releases the +lease if classification or trace validation exits early. It does not acquire a +second slot, release a slot it never acquired, or depend on measurement being +enabled. Actual HTTP RED covered saturation and early exits (three failures, +one passing control); the expanded enabled/disabled route/conduct/trace/error +matrix plus streaming, disconnect, identity, and trace regression tests passed +93 cases in 33.87 s. Those are local correctness results, not a latency gain. + +At `9707a5e1`, focused receipt/persistence tests passed 41 cases in 8.43 s. +The Rust clock now separates provider-ready diagnostics from initial task-route +acknowledgement. Structured triage, generated planning, and pre-selection evidence +embedding remain inside the task-route interval. Evidence embedding after native +selection is labelled `post_decision_evidence_embedding`, including write-failed +selection; it is not subtracted or presented as preceding routing work. Answer +cache reuse has a distinct terminal outcome with absent provider timing values. +The bounded admission cohort exports auxiliary records with an explicit diagnostic +cap and truncation indicator. Historical retention remains unresolved. + +The initial embedding test asserted inside a best-effort transport callback. +That assertion was swallowed by the existing best-effort path and prevented cache +fill, causing a test-induced warm retry. The corrected spy only collects values; +assertions run after the response. No descriptor invalidation was established. + +Native packaging was rebuilt from the repository root with +`maturin build --locked --release --manifest-path rust/decision_receipt/Cargo.toml +--interpreter .venv/bin/python --out /tmp/co-decision-receipt-wheels`. +Rust 1.97.1, maturin 1.15.0, Python 3.14.6, macOS arm64 produced the native wheel +SHA-256 `e0bf63d790256c6d4eba8598c131d63188a994c899df5124bd9eadf2cc39c568`. +Its five ZIP entries contain only the receipt extension and distribution metadata; +the independently built core wheel has no overlapping files. A separate +noneditable installation outside the checkout passed all 20 receipt tests in +11.83 s using `python -I`, with both import origins under that environment's +site-packages. This proves that local artifact matrix only, not Linux/Python 3.12 +hosted acceptance or a released package. The namespace was retained because the +suspected core-file collision was not observed in either inspected native wheel. + +CI now installs locked native build tooling, builds the extension before the full +suite, validates disjoint wheel ownership before installation, and exercises the +wheel-installed HTTP receipt tests outside the checkout. The existing benchmark +import smoke is preserved. These workflow changes remain pending hosted evidence. +Full endpoint admission validation, cancellation/error-path coverage, deployed +retention/reconciliation, and customer accuracy/latency measurements remain open; +none of the unit or mock-provider evidence establishes a customer KPI gain. diff --git a/docs/doctoring/provider_request_correlation.md b/docs/doctoring/provider_request_correlation.md new file mode 100644 index 000000000..dd99deec8 --- /dev/null +++ b/docs/doctoring/provider_request_correlation.md @@ -0,0 +1,139 @@ +# Provider request correlation + +Status: proposed in PR #1105; not deployed. Runtime owner: CO. Log collector +owner: ContextualWisdomLab/.github. + +Central run 34299034731/job 102308769876 used trusted workflow revision +`7fd571dbcdbae6acf29d8f4ee704d7ba6297e4db` and logged installation of CO +`414f22973658c4ddc3d4320fcf7acd9b4e8ba991`. Artifact 10085227363 contains +provider attempts without request identities. Timestamps and a final served +model cannot establish which earlier calls belong to the failed request. + +At local regression commit `f7f05569a54a51c4cd6e42418a0c079b032d45f6`, a real +HTTP request reached the real client retry wrapper with a controlled failing +transport. Its two provider diagnostic lines lacked the error response ID: +1 failed, 46 deselected (5.34s). No external provider was called. + +`c7468ecea009bae3a62413b4c54ab5a9b242bdff` binds a fresh server-generated UUID +to each HTTP handling scope using ContextVar and resets it in finally. The +error response reuses that identity; caller headers and error details cannot +replace it. A session hash is insufficient because several requests can share +a session. Copied contexts inherit identity, while reused workers without a +copied context do not. `8b82235e2b6db19682b0a255758c6796c41b3f55` extends the +identity to all seven provider attempt/retry/terminal diagnostic helpers. + +Reproduce from this checkout with its project environment: + +```sh +python -m pytest tests/test_telemetry.py tests/test_orchestrator_debug_logging.py -q +``` + +Result at the latter code revision: 68 passed in 25.60s. This covers controlled +HTTP failures, sequential same-session requests, copied worker contexts, nested +exception cleanup, and all seven diagnostic helpers. It does not prove +simultaneous HTTP isolation, every orchestration thread path, success-response +correlation, full-suite success, or customer latency improvement. + +Collector compatibility remains a release prerequisite. The trusted central +collector uses end-anchored patterns for several events and truncates failure +messages before error text. It rejects new fields until repaired. The new ID is +32 lowercase hexadecimal characters, or `-` outside an HTTP request; for failed +attempts it precedes `error_message`, for other events it is the final field. +Never recover an ID from untrusted error text. Preserve old-format inputs and +verify exact-SHA producer/collector integration before adoption. Do not copy +collector code into CO or enable unfiltered logs. Timeouts and replay policy +are unchanged by this patch. Browser rendering remains unverified. + +## Concurrent HTTP follow-up + +Follow-up at `6b24fe96` (local candidate, not the full-suite head): the success +summary also carries the request ID. A socket-identity assertion proves actual +keep-alive reuse; a two-party barrier followed by two distinct server-thread IDs +proves overlapping same-session HTTP handling rather than merely submitting +two tasks. Both requests retain distinct IDs and matching attempt/failure logs. +`python -m pytest tests/test_telemetry.py tests/test_orchestrator_debug_logging.py +tests/test_request_framing.py -q` passed 81 tests in 20.76s. This supersedes the +earlier simultaneous-HTTP limitation for these controlled failure cases only; +real provider integration and every orchestration worker path remain unverified. + +## Independent second incident + +Run 34306399309/job 102324739644 used the same trusted workflow and CO pins. +Artifact 10087151196 again records candidate_count=24, ready_count=1, +deferred_count=8, rejected_count=7, and account diversity=3. The caller reported +429 after 125.2s at 2026-09-09 03:31:41 UTC. Unlike the earlier incident, +the review interval beginning 03:29:35 contains no logged TimeoutError: +several Llama attempts precede DeepSeek at 03:31:40.995, which fails at +03:31:41.035 (about 40ms). TimeoutErrors at 03:27:17 and 03:28:47 belong to +the earlier preflight interval, not this review interval. These logs therefore +do not support attributing the final 429 to the default 90-second timeout. +Role/request attribution remains incomplete without correlation; keep the +timeout repair and this diagnostic repair as separate claims. No rerun or +provider call was issued during this read-only investigation. + +## Exact-revision collector contract + +The full suite at `7b7b32006e7ae498db2ee781bd423d9c7b6774fc` terminated +with exit status 0: **3399 passed, 2 skipped in 1594.26s**. This result predates +the success-summary and concurrent-HTTP follow-up; it must not be attributed +to their later revision. Those changes have the focused 81-test evidence above. + +The published Markdown at that same revision was opened in Edge and its +1897 × 949 screenshot directly inspected. The visible upper document had readable +heading/body contrast, wrapped paragraphs and commit identifiers, and an unclipped +test command. This is an English desktop upper-viewport inspection only, not a +full-document, responsive, interaction, or product-UI visual acceptance result. + +The producer candidate `7cb97ec8e2979d35b72c86a801ab18f0fd9c213d` +was cross-executed with the sanitizer from central PR #2053, +`fc0ab87bfde0900461034be815046914f9019bfc`. All seven actual provider +logging functions produced records whose trusted request ID survived sanitization. +A controlled error body containing a second, forged ID was omitted, as was its +controlled sensitive-text sentinel. Replacing the trusted ID with `INVALID` +or appending an embedded newline caused rejection for all seven records. +The final `request_failed` summary retained its ID and omitted trailing detail. +These are isolated, exact-revision contract checks, not live provider or release +evidence. The consumer deliberately leaves successful HTTP summaries outside +this PR's allowlist; that follow-up contract remains unverified. + +Follow-up consumer `4a0125bf9f50d4d26355249011df03c3735b3abc` adds a strict +HTTP-summary allowlist. Against producer +`f588ca8c093ea7c9a86b857685bfbb1ce3c05fe2`, an actual local HTTP GET to +`/healthz` returned 200 and emitted one request summary. The new sanitizer +retained it verbatim, including the generated request ID. Appending a controlled +extra detail field or substituting a non-allowlisted path caused rejection. +This supersedes the missing-success-summary contract limitation for that one +route/state; it does not verify all routes, current production adoption, or +latency improvement. No external provider was called. + +## Integrated full-suite failure + +At `f588ca8c093ea7c9a86b857685bfbb1ce3c05fe2`, the integrated full suite +terminated with exit 1: **3399 passed, 2 skipped, 1 failed in 1767.82s**. +`test_http_responses_rejects_store_true` failed while constructing the server's +ModelClient, before HTTP assertions: `context.load_verify_locations` raised +`InterruptedError` (errno 4) while loading the certifi CA bundle. This does not +prove a response-correlation regression, but it also does not establish a green +suite or justify classifying the failure as a flake. Same-head isolated +reproduction is the next diagnostic step. Preserve this failure receipt even +if a subsequent isolated test passes. + +The same-head isolated command `python -m pytest +tests/test_responses_store_http_honesty.py -q` then completed with exit 0: +4 passed in 20.05s. The interruption did not recur in this run; its signal or +operating-system trigger remains unproven. No TLS checks were bypassed and no +retry was added. A clean isolated run does not replace full-suite verification. +# Typed streaming error correlation + +The chat and Responses SSE error adapters must retain the request identity +observed inside the provider invocation. They previously generated another UUID +when framing typed failures, breaking correlation despite correct provider logs. +The repair uses the ordinary HTTP error adapter's trusted identity convention. +This is an identity mismatch fix, not evidence of upstream detail injection. + +Reproduce with `python -m pytest tests/test_stream_error_identity.py -q`. +Three real HTTP cases cover chat provider errors, Responses provider errors, and +chat stopped-tool errors. The Responses fixture fixes the routing choice to +isolate SSE framing; it does not test routing policy. The original cases failed +identity equality; after repair, these and both debug logging suites passed +(40 tests, 2.75 seconds). No hosted check or deployment is implied. diff --git a/docs/doctoring/workflow_request_link.md b/docs/doctoring/workflow_request_link.md new file mode 100644 index 000000000..66b0cedd5 --- /dev/null +++ b/docs/doctoring/workflow_request_link.md @@ -0,0 +1,63 @@ +# Durable workflow origin identity + +## Full-suite environment repair + +Full collection at `4cf7feafd554fbbd65dfc3b790f1081623b0d05a` failed with +exit 2 after 5.41 seconds: the shared wheel-smoke environment lacked Hypothesis. +No tests completed in that run. The remedy is a separate project-local environment, +not a runtime change or an installation into the shared historical environment: + +```sh +uv sync --locked --extra api --extra db --extra queue --group dev +.venv/bin/python -c 'import contextual_orchestrator; contextual_orchestrator.__path__.append("/tmp/co-receipt-wheel-install-20260909/lib/python3.14/site-packages/contextual_orchestrator"); import pytest; raise SystemExit(pytest.main(["-q"]))' +``` + +Setup completed with CPython 3.14.6 and the existing lock, including Hypothesis +6.165.10. The reused base-native artifact is `_decision_receipt.abi3.so` under +that explicitly named site-packages directory, SHA-256 +`ddac17f8c6a25e52a9233bd9e75f5ca3c754eb24641caa58113148df8ef60b4d`. +Native source is unchanged from `c7345670`; this is source-integration evidence, +not an installed successor-wheel or release result. Do not modify the shared +native environment. Full-suite results remain pending until its handle terminates. + +The accepted-request ledger and completed workflow records previously had no +durable join. At base c7345670, real HTTP route, conduct and streamed route +requests could not match their persisted outcome to the trusted request ID. + +The shared workflow replacement seam now records the trusted HTTP identity on +first creation and preserves it on replacement. Explicit state restoration +retains stored identity instead of attributing history to the loading context. +Non-HTTP runs remain valid without an ID. Cache hits create separate outcome +records with their existing cache-hit classification; the earlier run's origin +is unchanged. This does not make reused output a new provider execution. + +The regression also exposed that stream_route never saved its completed run. +It now uses the same synchronous workflow store path as run/conduct. A failed +save emits the existing terminal SSE error, retains the admission and does not +rewrite an earlier initial-decision acknowledgement. As in run/conduct, the +in-memory run and budget update precede storage: memory is not durable proof, +and no atomic memory/database transaction is claimed. Consumers must reconcile +against persisted records rather than count the in-memory run as stored. + +Reproduction prerequisite: the unchanged native extension built at base +`c7345670e08f029ad3aa5dd1133037bb4b451d9b` is installed in the existing isolated +Python 3.14 environment below. From `/tmp/co-outcome-request-link-20260909`, run: + +```sh +/tmp/co-receipt-wheel-install-20260909/bin/python -c 'import contextual_orchestrator; contextual_orchestrator.__path__.append("/tmp/co-receipt-wheel-install-20260909/lib/python3.14/site-packages/contextual_orchestrator"); import pytest; raise SystemExit(pytest.main(["tests/test_workflow_request_link.py", "tests/test_persistence.py", "tests/test_stream_error_identity.py", "-q"]))' +``` + +This explicitly loads successor Python source and the unchanged base native +extension. The namespace append is test-only; it does not install the successor +or establish wheel/release acceptance. Preserve that environment unchanged. +The three-file command passed 34 tests in 9.36 seconds. A final linkage-only +run after adding the in-memory failure-state assertion passed 10 in 5.43 seconds. +The independent original HTTP probe also confirmed one admission joins one +durably stored route outcome. Initial four cases failed; subsequent HTTP enabled/disabled, +cache, reload, persistence and SSE regressions passed. Tests use controlled +provider output and establish identity, not independently adjudicated accuracy. +The request may map to multiple workflow outcomes; it is not a one-to-one join +contract for every API. Detached background execution without propagated HTTP +context remains unlinked, rather than guessed from the current thread. Explicit +batch-parent/item linkage and crash recovery after an in-memory-only update +remain outside this bounded repair. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index b8671342c..3c41bdef7 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,29 @@ # Contextual Orchestrator: Product & Technical Gap Baseline +## 2026-09-09 State persistence integrity prerequisite + +PR [#1108](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1108) +repairs a reproduced failed-replacement data-loss case in the common SQLite +state writer. Code `d7bba88f3d711883a37effe49ab4f503c4fb8e01` rolls back failed +writes under the existing lock. Test follow-up `f1abe1e3` checks that no +transaction remains immediately after failure and that both the previous and +unrelated subsequent records survive reopening the database. The persistence +suite passed 19 tests in 9.28s; this is not a latency or customer-accuracy result. + +Full local suite at `877d5112ed470d851afaa2c746b94393cc768ee7`: 3,396 passed, +2 skipped, exit 0 (883.03s). Test-only follow-up +`716e012dcb50857000b0fc53c89c6434fdf7e7c2` covers a deferred commit failure +with real SQLite constraints; persistence, workflow authorization, and governance +tests pass together (29 passed, 4.89s). Full-suite evidence remains attached to +the earlier head, not silently reassigned to the new regression. + +At the earlier PR head `aa674187b0341c7852f85c27fb696aec21f1a799`, GitHub +reported zero check runs and two success statuses whose descriptions explicitly +said reviews were skipped (Draft; expired trial/no credits). Those statuses do +not establish review approval or security validation. Keep protected merge and +release pending actual exact-head evidence. The root cause and reproduction are +in the [canonical runbook](doctoring/autonomous_kpi_runbook.md). + ## 2026-09-09 Stacked quality-trigger repair Correction: PR #1066 at `59a8f4eadfe0e0dcc5ff47cf1acfb80403e241ad` already @@ -269,6 +293,54 @@ inner products both equal 3.5 while unaligned coordinate RMSE equals 1.0. This demonstrates the identification pitfall, not estimator accuracy or a latency improvement. It is a manual documentation check, not yet a hosted CI gate or a test of the released fast-mlsirm implementation. +## 2026-09-09 Request-to-provider diagnostic correlation + +PR #1105 candidate `f588ca8c093ea7c9a86b857685bfbb1ce3c05fe2` connects HTTP +identity to seven provider diagnostic events and the successful request summary. +The predecessor `7b7b32006e7ae498db2ee781bd423d9c7b6774fc` completed its full +suite with 3399 passed, 2 skipped (1594.26s, exit 0). Follow-up code at +`6b24fe96` passed 81 focused tests, including actual same-socket reuse and +overlapping same-session HTTP requests with two distinct server thread IDs. +The integrated `f588ca8c` suite terminated with 3399 passed, 2 skipped and +1 failure (1767.82s, exit 1): certifi CA loading raised InterruptedError before +the Responses HTTP test could send a request. Same-head isolated HTTP tests +then passed 4/4 in 20.05s. The original failure remains unresolved evidence; +do not infer full-suite success from the isolated pass. + +Actual output from all seven provider diagnostic functions at +`7cb97ec8e2979d35b72c86a801ab18f0fd9c213d` was cross-checked with the central +PR #2053 sanitizer at `fc0ab87bfde0900461034be815046914f9019bfc`: trusted IDs +survived, untrusted error-body IDs and text were omitted, and malformed IDs and +embedded newlines were rejected. This isolated contract test does not establish +collector adoption. The later sanitizer `4a0125bf9f50d4d26355249011df03c3735b3abc` +also preserved an actual local GET `/healthz` 200 summary from producer +`f588ca8c`, including its request ID, while rejecting extra detail and an +unapproved path. This supersedes the earlier missing-success-summary limitation +for that route/state only, not every HTTP route. The +[runbook](doctoring/provider_request_correlation.md) records +RED evidence, exact revisions, cleanup tests, and bounded visual inspection. +Not yet established: every orchestration worker path, integrated full-suite and +security gates, protected release, live collector adoption, or customer KPI +improvement. Diagnostic traceability is a prerequisite for attributing failures, +not a substitute for accuracy or decision-latency measurements. + +## 2026-09-08 error-response correlation repair + +ConceptWeave run 33938445050, job 101256562088, preserves a client-side HTTP +500 with request ID `175d6d59c5294b0e8a21548193b90482`. Its surviving artifact +9969701340 contains gateway stderr but only generic request-failure messages; +it cannot correlate that ID to an internal cause. The job installed CO source +`2e414d15ba58f28597751b625a8a2f00fc9fadcf`. This is not proof of free-pool +exhaustion, a disappeared run, or a currently released fix. + +The same correlation gap was reproduced on main +`414f22973658c4ddc3d4320fcf7acd9b4e8ba991`: the common HTTP error response had +a generated ID absent from its log. The proposed repair generates one ID for +both response and warning, prevents detail fields from overriding it, and logs +neither session values nor error details. RED: one missing-correlation failure; +GREEN: 45 telemetry tests passed in 6.58 seconds. This improves future failure +correlation only; it does not recover the historical exception, cover every +streaming-error path, or prove immutable publication or deployed behavior. ## 2026-09-01 Autonomous Commercialization Loop: PR #970 Merge, Token Accounting & Cost Gateway Harmonization diff --git a/mise.toml b/mise.toml new file mode 100644 index 000000000..19305ad64 --- /dev/null +++ b/mise.toml @@ -0,0 +1,2 @@ +[tools] +rust = "1.97.1" diff --git a/pyproject.toml b/pyproject.toml index de9f90de1..fcce4241c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ queue = [ ] [dependency-groups] +native-build = ["maturin==1.15.0"] dev = [ "hypothesis>=6.100", "pytest>=8.0", diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 000000000..725551148 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.97.1" +profile = "minimal" diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 96d3c6a1d..58fc95de5 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -49,6 +49,13 @@ dependencies = [ "serde_core", ] +[[package]] +name = "contextual-decision-receipt" +version = "0.1.0" +dependencies = [ + "pyo3", +] + [[package]] name = "contextual-token-packer" version = "0.1.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index b7d83efbb..c69753fbb 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["token_counter"] +members = ["token_counter", "decision_receipt"] resolver = "2" diff --git a/rust/decision_receipt/Cargo.toml b/rust/decision_receipt/Cargo.toml new file mode 100644 index 000000000..8e1c1d19a --- /dev/null +++ b/rust/decision_receipt/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "contextual-decision-receipt" +version = "0.1.0" +edition = "2021" + +[dependencies] +pyo3 = { version = "0.29", features = ["abi3-py310", "auto-initialize"] } + +[lib] +name = "_decision_receipt" +crate-type = ["cdylib"] diff --git a/rust/decision_receipt/pyproject.toml b/rust/decision_receipt/pyproject.toml new file mode 100644 index 000000000..6c498c112 --- /dev/null +++ b/rust/decision_receipt/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["maturin==1.15.0"] +build-backend = "maturin" + +[project] +name = "contextual-decision-receipt" +version = "0.1.0" +requires-python = ">=3.10" + +[tool.maturin] +module-name = "contextual_orchestrator._decision_receipt" +python-source = "../.." +features = ["pyo3/extension-module"] diff --git a/rust/decision_receipt/src/lib.rs b/rust/decision_receipt/src/lib.rs new file mode 100644 index 000000000..5630c1dfc --- /dev/null +++ b/rust/decision_receipt/src/lib.rs @@ -0,0 +1,117 @@ +//! One-clock initial-decision measurement; storage acknowledgement is caller-owned. +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use std::time::Instant; + +/// Request-local monotonic state with no caller-supplied clock values. +#[pyclass] +pub struct DecisionReceipt { + accepted_at: Instant, + #[pyo3(get)] + status: String, + #[pyo3(get)] + selection_elapsed_ns: Option, + #[pyo3(get)] + durable_ack_elapsed_ns: Option, + #[pyo3(get)] + first_provider_elapsed_ns: Option, +} + +impl DecisionReceipt { + fn elapsed_ns(&self) -> PyResult { + self.accepted_at + .elapsed() + .as_nanos() + .try_into() + .map_err(|_| PyValueError::new_err("decision elapsed time overflow")) + } +} + +#[pymethods] +impl DecisionReceipt { + /// Start at validated acceptance, before capacity admission. + #[new] + fn accepted() -> Self { + Self { + accepted_at: Instant::now(), + status: "accepted".into(), + selection_elapsed_ns: None, + durable_ack_elapsed_ns: None, + first_provider_elapsed_ns: None, + } + } + + /// Mark the first provider-ready boundary without advancing task selection. + fn record_provider_dispatch(&mut self) -> PyResult { + if let Some(elapsed) = self.first_provider_elapsed_ns { + return Ok(elapsed); + } + let elapsed = self.elapsed_ns()?; + self.first_provider_elapsed_ns = Some(elapsed); + Ok(elapsed) + } + + /// Timestamp an auxiliary phase in the same acceptance clock domain. + fn current_elapsed_ns(&self) -> PyResult { + self.elapsed_ns() + } + + /// Complete answer reuse without inventing a provider selection or acknowledgement. + fn record_cache_hit(&mut self) -> PyResult<()> { + if self.status != "accepted" { + return Err(PyValueError::new_err("cache hit requires accepted state")); + } + self.status = "cache_hit".into(); + Ok(()) + } + + /// Record the initial selection without allowing later attempts to replace it. + fn record_selection(&mut self) -> PyResult<()> { + if self.status != "accepted" { + return Err(PyValueError::new_err("selection requires accepted state")); + } + let elapsed = self.elapsed_ns()?; + self.selection_elapsed_ns = Some(elapsed); + self.status = "selected".into(); + Ok(()) + } + + /// Call only after a synchronous durable decision write returns successfully. + fn record_durable_ack(&mut self) -> PyResult<()> { + if self.status != "selected" { + return Err(PyValueError::new_err( + "acknowledgement requires selected state", + )); + } + let elapsed = self.elapsed_ns()?; + self.durable_ack_elapsed_ns = Some(elapsed); + self.status = "acknowledged".into(); + Ok(()) + } + + /// Preserve a denominator-only terminal observation without a success duration. + fn record_failure(&mut self, reason: &str) -> PyResult<()> { + if !matches!(self.status.as_str(), "accepted" | "selected") { + return Err(PyValueError::new_err("observation is already terminal")); + } + if !matches!( + reason, + "capacity_rejected" + | "selection_failed" + | "write_failed" + | "cancelled" + | "unfinished" + | "store_unavailable" + ) { + return Err(PyValueError::new_err("unknown failure reason")); + } + self.status = reason.into(); + Ok(()) + } +} + +/// Export the receipt separately from token counting. +#[pymodule] +fn _decision_receipt(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/scripts/verify_decision_wheel_manifest.py b/scripts/verify_decision_wheel_manifest.py new file mode 100644 index 000000000..461dd6379 --- /dev/null +++ b/scripts/verify_decision_wheel_manifest.py @@ -0,0 +1,23 @@ +"""Reject native/core wheel ownership overlap before isolated installation.""" + +import sys +import zipfile + + +def verify_wheels(core_path, native_path): + """Require a native-only extension distribution disjoint from the core wheel.""" + with zipfile.ZipFile(core_path) as core_archive, zipfile.ZipFile(native_path) as native_archive: + core_files = set(core_archive.namelist()) + native_files = set(native_archive.namelist()) + if core_files & native_files: + raise ValueError("core and native wheel files overlap") + payload = {name for name in native_files if ".dist-info/" not in name} + if len(payload) != 1 or not all( + name.startswith("contextual_orchestrator/_decision_receipt.") + and name.endswith((".so", ".pyd")) for name in payload + ): + raise ValueError("native wheel must contain only the receipt extension") + + +if __name__ == "__main__": + verify_wheels(*sys.argv[1:]) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py new file mode 100644 index 000000000..eaeef59ca --- /dev/null +++ b/tests/test_batch_request_lineage.py @@ -0,0 +1,363 @@ +"""Deferred batch outcomes retain their original trusted HTTP admission.""" + +import copy +import threading +import pytest + +from contextual_orchestrator import CostRoutingCoordinator, ModelAgent, TaskOrchestrator +from contextual_orchestrator.batch_routing import PgLlmBatchBackend +from contextual_orchestrator.server import SecurityConfig, build_server +from contextual_orchestrator.telemetry import current_request_id +from test_batch_routing import _FakeBatchApiClient +from test_cost_review_server import _request + + +@pytest.mark.parametrize("write_failure", [False, True]) +@pytest.mark.parametrize("registry_failure", [None, "hset", "expire"]) +def test_http_batch_origin_survives_distinct_retrieval_and_reload(tmp_path, monkeypatch, write_failure, registry_failure): + """One submission joins two item outcomes without trusting their custom IDs.""" + class ObservedBatchClient(_FakeBatchApiClient): + """Reuse the existing offline provider contract with passive identity capture.""" + + async def create_batch_job(self, *args, **kwargs): + self.submission_request_id = current_request_id() + return await super().create_batch_job(*args, **kwargs) + + async def download_results(self, *args, **kwargs): + self.retrieval_request_id = current_request_id() + payload = await super().download_results(*args, **kwargs) + second_result = copy.deepcopy(payload["responses"][0]) + second_result["custom_id"] = "b" + payload["responses"].append(second_result) + return payload + + state_path = tmp_path / "state.db" + agents = [ModelAgent("worker_one", "mock/worker")] + orchestrator = TaskOrchestrator(agents, state_db=state_path) + if write_failure: + original_save = orchestrator._store.save + + def reject_link(kind, *args, **kwargs): + if kind == "batch_request_link": + raise RuntimeError("private-store-secret") + return original_save(kind, *args, **kwargs) + + monkeypatch.setattr(orchestrator._store, "save", reject_link) + batch_client = ObservedBatchClient() + coordinator = CostRoutingCoordinator( + orchestrator, batch_backend=PgLlmBatchBackend(batch_client) + ) + if registry_failure: + from contextual_orchestrator.batch_job_registry import ValkeyJsonMapping + from contextual_orchestrator.batch_routing import BatchJob + from test_batch_job_registry import FakeValkeyClient + + class RejectingClient(FakeValkeyClient): + """Distinguish no registry write from partial HSET-before-expiry.""" + + def hset(self, *args, **kwargs): + if registry_failure == "hset": + raise RuntimeError("private-registry-secret") + return super().hset(*args, **kwargs) + + def expire(self, *args, **kwargs): + raise RuntimeError("private-registry-secret") + + registry_client = RejectingClient() + coordinator._batch_jobs = ValkeyJsonMapping( + registry_client, "jobs", decode=lambda raw: BatchJob(**raw) + ) + server = build_server(orchestrator, port=0, coordinator=coordinator, + security=SecurityConfig(auth_token="unit-token")) + worker_thread = threading.Thread(target=server.serve_forever, daemon=True) + worker_thread.start() + base_url = f"http://127.0.0.1:{server.server_address[1]}" + try: + status, submitted = _request("POST", f"{base_url}/api/v1/batch_routing_jobs", + "unit-token", {"requests": [ + {"custom_id": item_id, "model": "mock/worker", "mode": "route", + "messages": [{"role": "user", "content": "Offline contract fixture."}]} + for item_id in ("a", "b") + ]}) + assert status == 201, submitted + assert submitted["request_link_status"] == ("write_failed" if write_failure else "durable") + assert "private-store-secret" not in str(submitted) + assert submitted["recovery_status"] == "unavailable" + assert submitted["registry_persistence_status"] == ("write_failed" if registry_failure else "stored") + assert "private-registry-secret" not in str(submitted) + if registry_failure: + assert submitted["job_id"] == "batch-789" + assert batch_client.calls.count("create_batch_job") == 1 + stored_handles = registry_client.hashes.get("batch_job_registry:jobs", {}) + assert bool(stored_handles) == (registry_failure == "expire") + denied_status, _ = _request( + "POST", f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results", + "other-owner-token", + ) + assert denied_status in {401, 403} + assert "download_results" not in batch_client.calls + links = orchestrator._store.load("batch_request_link") + assert len(links) == (0 if write_failure else 1) + if links: + assert links[0]["request_id"] == batch_client.submission_request_id + return + status, retrieved = _request( + "POST", f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results", + "unit-token", + ) + assert status == 200, retrieved + assert {item["custom_id"] for item in retrieved["results"]} == {"a", "b"} + assert batch_client.submission_request_id + assert batch_client.retrieval_request_id + assert batch_client.submission_request_id != batch_client.retrieval_request_id + assert batch_client.submission_request_id not in {"a", "b"} + repeated_status, repeated = _request( + "POST", f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results", + "unit-token", + ) + assert repeated_status == 200 + assert {item["custom_id"] for item in repeated["results"]} == {"a", "b"} + assert batch_client.calls.count("create_batch_job") == 1 + assert coordinator._batch_jobs[submitted["job_id"]].job_id == submitted["job_id"] + finally: + server.shutdown() + worker_thread.join() + server.server_close() + orchestrator.close() + + restored = TaskOrchestrator(agents, state_db=state_path) + try: + # Proposed CO-owned association contract: one row per submission/job/item, + # not a replacement of provider custom_id or one origin per eventual job. + links = restored._store.load("batch_request_link") + actual_links = { + (row["request_id"], row["batch_job_id"], custom_id) + for row in links for custom_id in row["custom_ids"] + } + if write_failure: + assert not actual_links + return + assert actual_links == { + (batch_client.submission_request_id, submitted["job_id"], item["custom_id"]) + for item in retrieved["results"] + } + finally: + restored.close() + + +def test_batch_submission_links_keep_job_scoped_item_ids(tmp_path): + """Repeated item identifiers across submissions retain every origin association.""" + from contextual_orchestrator.batch_routing import BatchRequest + + agents = [ModelAgent("worker_one", "mock/worker")] + orchestrator = TaskOrchestrator(agents, state_db=tmp_path / "state.db") + coordinator = CostRoutingCoordinator( + orchestrator, batch_backend=PgLlmBatchBackend(_FakeBatchApiClient()) + ) + try: + for request_id in ("trusted_origin_one", "trusted_origin_two"): + coordinator.submit_batch([ + BatchRequest(messages=[{"role": "user", "content": "Fixture"}], custom_id="a") + ], owner_id="owner_one", request_id=request_id) + links = orchestrator._store.load("batch_request_link") + assert {row["request_id"] for row in links} == {"trusted_origin_one", "trusted_origin_two"} + assert [row["custom_ids"] for row in links] == [["a"], ["a"]] + finally: + orchestrator.close() + + +def test_library_batch_without_state_store_keeps_legacy_submission(): + """Standalone calls explicitly report unavailable durable request lineage.""" + from contextual_orchestrator.batch_routing import BatchRequest + + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")]) + coordinator = CostRoutingCoordinator( + orchestrator, batch_backend=PgLlmBatchBackend(_FakeBatchApiClient()) + ) + try: + job = coordinator.submit_batch([BatchRequest( + messages=[{"role": "user", "content": "Fixture"}], custom_id="a" + )]) + assert job.request_link_status == "unavailable" + assert job.job_id == "batch-789" + # Legacy metadata without a deployment binding remains usable only + # while the injected backend also has no recovery identity configured. + coordinator.batch_backend._jobs[job.job_id].pop("recovery_identity") + assert coordinator.poll_batch(job.job_id)["is_complete"] is True + finally: + orchestrator.close() + + +def test_batch_link_does_not_rewrite_submitted_registry_handle(tmp_path): + """Lineage status must not add another failure-prone registry assignment.""" + from contextual_orchestrator.batch_routing import BatchRequest + + class SingleWriteRegistry(dict): + """Reject a redundant second remote-registry assignment.""" + + def __setitem__(self, key, value): + if key in self: + raise RuntimeError("second registry write failed") + super().__setitem__(key, value) + + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + coordinator = CostRoutingCoordinator( + orchestrator, batch_backend=PgLlmBatchBackend(_FakeBatchApiClient()) + ) + coordinator._batch_jobs = SingleWriteRegistry() + try: + job = coordinator.submit_batch([BatchRequest( + messages=[{"role": "user", "content": "Fixture"}], custom_id="a" + )], request_id="trusted_origin_one") + assert job.request_link_status == "durable" + finally: + orchestrator.close() + + +def test_valkey_job_snapshot_does_not_prove_lineage_commit(tmp_path): + """Decoded registry status is non-authoritative; committed events supply proof.""" + from contextual_orchestrator.batch_job_registry import ValkeyJsonMapping + from contextual_orchestrator.batch_routing import BatchJob, BatchRequest + from test_batch_job_registry import FakeValkeyClient + + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + coordinator = CostRoutingCoordinator( + orchestrator, batch_backend=PgLlmBatchBackend(_FakeBatchApiClient()) + ) + coordinator._batch_jobs = ValkeyJsonMapping( + FakeValkeyClient(), "jobs", decode=lambda raw: BatchJob(**raw) + ) + try: + response_job = coordinator.submit_batch([BatchRequest( + messages=[{"role": "user", "content": "Fixture"}], custom_id="a" + )], owner_id="owner_one", request_id="trusted_origin_one") + decoded_job = coordinator._batch_jobs[response_job.job_id] + assert decoded_job is not response_job + assert decoded_job.request_link_status == "durable" + assert decoded_job.registry_persistence_status == "unavailable" + assert response_job.registry_persistence_status == "stored" + assert decoded_job.owner_id == "owner_one" + assert response_job.request_link_status == "durable" + assert orchestrator._store.load("batch_request_link")[0]["request_id"] == "trusted_origin_one" + finally: + orchestrator.close() + + +@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "coordinator_hit", "duplicate_ids", "backend_write_outage", "healthy_expired", "healthy_deployment_mismatch", "healthy_endpoint_mismatch"]) +def test_http_batch_failed_registry_recovers_authorized_job_after_restart(tmp_path, recovery_case): + """SQLite recovery binds the original owner without another remote submission.""" + class MissingRegistry(dict): + """Lose the registry assignment, retaining only committed SQLite evidence.""" + + def __setitem__(self, key, value): + raise RuntimeError("registry unavailable") + + state_path = tmp_path / "state.db" + agents = [ModelAgent("worker_one", "mock/worker")] + security = SecurityConfig(bearer_verifier=lambda token, scope: + token in {"owner-one", "owner-two"}) + clients = [] + for restarted in (False, True): + orchestrator = TaskOrchestrator(agents, state_db=state_path) + class RecoveryClient(_FakeBatchApiClient): + """Return controlled result variants without changing routing execution.""" + + async def download_results(self, *args, **kwargs): + result = await super().download_results(*args, **kwargs) + if recovery_case == "unexpected_item": + result["responses"][0]["custom_id"] = "unsubmitted-item" + if recovery_case == "missing_usage": + result["responses"][0]["response"]["body"].pop("usage") + return result + + client = RecoveryClient() + clients.append(client) + coordinator = CostRoutingCoordinator(orchestrator, + batch_backend=PgLlmBatchBackend(client, endpoint_alias=( + "changed-endpoint" if restarted and recovery_case == "backend_mismatch" + else "original-endpoint"), endpoint=( + "/v1/completions" if restarted and recovery_case == "healthy_endpoint_mismatch" + else "/v1/chat/completions"), recovery_identity=( + None if restarted and recovery_case == "missing_identity" else + "different-deployment" if restarted and recovery_case in {"deployment_mismatch", "healthy_deployment_mismatch"} + else "unit-deployment-account"))) + if not restarted: + if recovery_case == "backend_write_outage": + coordinator.batch_backend._jobs = MissingRegistry() + else: + coordinator._batch_jobs = MissingRegistry() + elif recovery_case in {"coordinator_hit", "healthy_expired", "healthy_deployment_mismatch", "healthy_endpoint_mismatch"}: + from contextual_orchestrator.batch_routing import BatchJob + coordinator._batch_jobs[submitted["job_id"]] = BatchJob(**record["recovery_descriptor"]["job"]) + if recovery_case in {"healthy_expired", "healthy_deployment_mismatch", "healthy_endpoint_mismatch"}: + coordinator.batch_backend._jobs = retained_backend_metadata + elif recovery_case == "registry_outage": + class UnavailableRegistry(MissingRegistry): + """All reads and writes remain unavailable during recovery.""" + + def get(self, *args, **kwargs): + raise RuntimeError("registry still unavailable") + + coordinator._batch_jobs = UnavailableRegistry() + coordinator.batch_backend._jobs = UnavailableRegistry() + server = build_server(orchestrator, port=0, coordinator=coordinator, security=security) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + base_url = f"http://127.0.0.1:{server.server_address[1]}" + try: + if not restarted: + status, submitted = _request("POST", f"{base_url}/api/v1/batch_routing_jobs", + "owner-one", {"requests": [{"custom_id": "a", "model": "mock/worker", + "messages": [{"role": "user", "content": "Never persist this prompt."}]}]}) + assert status == 201 + assert submitted["registry_persistence_status"] == ( + "stored" if recovery_case == "backend_write_outage" else "write_failed") + if recovery_case == "backend_write_outage": + assert submitted["backend_registry_persistence_status"] == "write_failed" + assert submitted["recovery_status"] == "durable_descriptor" + record = orchestrator._store.load("batch_request_link")[0] + assert "Never persist this prompt." not in str(record) + retained_backend_metadata = coordinator.batch_backend._jobs + if recovery_case in {"expired", "healthy_expired"}: + record["recovery_descriptor"]["expires_at"] = 0 + if recovery_case == "malformed": + record["recovery_descriptor"]["backend"] = [] + if recovery_case == "item_mismatch": + record["custom_ids"] = ["different-original-item"] + if recovery_case == "estimate_mismatch": + record["recovery_descriptor"]["job"]["prompt_token_estimates"] = {"different-item": 99} + if recovery_case == "null_estimates": + record["recovery_descriptor"]["job"]["prompt_token_estimates"] = None + if recovery_case == "boolean_count": + record["recovery_descriptor"]["job"]["request_count"] = True + if recovery_case == "duplicate_ids": + record["custom_ids"] = ["a", "a"] + if recovery_case in {"expired", "healthy_expired", "malformed", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "duplicate_ids"}: + orchestrator._store.save("batch_request_link", submitted["job_id"], record, durable=True) + continue + result_url = f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results" + denied_status, _ = _request("POST", result_url, "owner-two") + assert denied_status == 404 + assert "download_results" not in client.calls + status, retrieved = _request("POST", result_url, "owner-one") + if recovery_case in {"expired", "malformed", "backend_mismatch", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "duplicate_ids", "healthy_deployment_mismatch", "healthy_endpoint_mismatch"}: + assert status == 404, retrieved + assert "download_results" not in client.calls + continue + if recovery_case == "unexpected_item": + assert status != 200 + continue + assert status == 200, retrieved + poll_status, _ = _request("GET", result_url.removesuffix("/results"), "owner-one") + assert poll_status == 200 + assert retrieved["results"][0]["custom_id"] == "a" + if recovery_case == "missing_usage": + assert retrieved["results"][0]["measurement_status"] != "measured" + assert sum(item.calls.count("create_batch_job") for item in clients) == 1 + finally: + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py new file mode 100644 index 000000000..0a06b04e0 --- /dev/null +++ b/tests/test_decision_receipts.py @@ -0,0 +1,784 @@ +"""Initial decisions must be acknowledged before any answer is generated.""" + +import http.client +import json +import threading +import sqlite3 +import pytest + +from contextual_orchestrator import ModelAgent, TaskOrchestrator +from contextual_orchestrator.server import build_server, SecurityConfig + + +@pytest.mark.parametrize("scenario", ["saturated", "success", "conduct_success", "classifier_error", "trace_rejection"]) +@pytest.mark.parametrize("measurement_enabled", [False, True]) +def test_chat_stream_classification_owns_one_capacity_lease(tmp_path, monkeypatch, scenario, measurement_enabled): + """Classification and task execution share capacity and release it exactly once.""" + from contextual_orchestrator.server import RequestError + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + security = SecurityConfig(auth_token="test-token", max_concurrent_runs=1) + original_acquire, original_release = security.acquire_run_slot, security.release_run_slot + acquired, released, calls = [], [], [] + def acquire(): + original_acquire() + acquired.append(True) + def release(): + released.append(True) + original_release() + monkeypatch.setattr(security, "acquire_run_slot", acquire) + monkeypatch.setattr(security, "release_run_slot", release) + original_chat = orchestrator.client.chat + def chat(agent, messages, **kwargs): + calls.append(True) + if messages[0]["content"] == orchestrator.TRIAGE_SYSTEM_PROMPT: + return json.dumps({"workflow_required": scenario == "conduct_success"}) + return original_chat(agent, messages, **kwargs) + monkeypatch.setattr(orchestrator.client, "chat", chat) + if scenario == "classifier_error": + def reject_classifier(*args, **kwargs): + raise ValueError("unit classifier rejection") + monkeypatch.setattr(orchestrator, "would_route", reject_classifier) + server = build_server(orchestrator, port=0, decision_receipts=measurement_enabled, security=security) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + if scenario == "saturated": + original_acquire() + connection = http.client.HTTPConnection(*server.server_address) + try: + body = {"model": "orchestrator/auto", "mode": "auto", "stream": True, + "messages": [{"role": "user", "content": "capacity question"}]} + if scenario == "trace_rejection": + body["include_orchestration_trace"] = True + connection.request("POST", "/v1/chat/completions", json.dumps(body), + {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + response.read() + assert response.status == {"saturated": 503, "success": 200, "conduct_success": 200, + "classifier_error": 400, "trace_rejection": 400}[scenario] + connection.close() + server.shutdown() + if scenario == "saturated": + assert calls == [] + assert acquired == released == [] + else: + assert len(acquired) == len(released) == 1 + original_acquire() + with pytest.raises(RequestError): + original_acquire() + original_release() + finally: + if scenario == "saturated": + original_release() + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +@pytest.mark.parametrize("invalid_field", [None, "routing", "attribution", "user", "metadata", "implicit_trace", "explicit_trace", "authorized_trace"]) +def test_http_auto_stream_admits_before_triage(tmp_path, monkeypatch, invalid_field): + """Auto stream classification must share the eventual task's admission clock.""" + from contextual_orchestrator.decision_receipts import _CURRENT_DECISION, export_decision_receipts + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + snapshots = [] + original_chat = orchestrator.client.chat + def observed_chat(agent, messages, **kwargs): + current = _CURRENT_DECISION.get() + snapshots.append(current.snapshot() if current else None) + if messages[0]["content"] == orchestrator.TRIAGE_SYSTEM_PROMPT: + return '{"workflow_required": false}' + return original_chat(agent, messages, **kwargs) + monkeypatch.setattr(orchestrator.client, "chat", observed_chat) + security = SecurityConfig(auth_token="test-token") + if invalid_field in ("implicit_trace", "explicit_trace"): + security = SecurityConfig(bearer_verifier=lambda token, scope: token == "test-token" and scope == "inference", + expose_trace_by_default=True) + server = build_server(orchestrator, port=0, decision_receipts=True, security=security) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address) + try: + request_body = { + "model": "orchestrator/auto", "mode": "auto", "stream": True, + "messages": [{"role": "user", "content": "streamed question"}], + } + if invalid_field in ("routing", "attribution", "user", "metadata"): + request_body[invalid_field] = "" if invalid_field == "user" else [] + if invalid_field in ("explicit_trace", "authorized_trace"): + request_body["include_orchestration_trace"] = True + connection.request("POST", "/v1/chat/completions", json.dumps(request_body), + {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + response.read() + expected_status = 401 if invalid_field == "explicit_trace" else (400 if invalid_field in ("routing", "attribution", "user", "metadata", "authorized_trace") else 200) + assert response.status == expected_status + connection.close() + server.shutdown() + if invalid_field == "authorized_trace": + observation, = export_decision_receipts(orchestrator._store)["observations"] + assert observation["status"] == "selection_failed" + assert observation["durable_ack_elapsed_ns"] is None + assert snapshots[0]["request_id"] == observation["request_id"] + return + if expected_status != 200: + assert snapshots == [] + assert orchestrator._store.load("accepted_request") == [] + return + assert snapshots and all(snapshot is not None for snapshot in snapshots) + observation, = export_decision_receipts(orchestrator._store)["observations"] + assert observation["first_provider_phase"] == "structured_triage" + assert observation["durable_ack_elapsed_ns"] is not None + assert snapshots[0]["request_id"] == observation["request_id"] + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +@pytest.mark.parametrize("endpoint", ["/v1/chat/completions", "/v1/responses"]) +def test_http_typed_stream_failure_before_selection_is_retained(tmp_path, monkeypatch, endpoint): + """Handled provider failures before task selection retain a failure denominator.""" + from contextual_orchestrator.provider_errors import ProviderUpstreamError + from contextual_orchestrator.decision_receipts import export_decision_receipts + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + def rejected(*args, **kwargs): + raise ProviderUpstreamError(agent_id="worker_one", model="mock/worker", + error_code="rate_limit_exceeded", message="unit rejection", + client_status=429, provider_status=429, retryable=True) + monkeypatch.setattr(orchestrator, "stream_route", rejected) + monkeypatch.setattr(orchestrator, "conduct", rejected) + if endpoint == "/v1/responses": + monkeypatch.setattr(orchestrator, "would_route", lambda *args, **kwargs: False) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address) + try: + request_body = {"model": "orchestrator/auto", "mode": "route", "stream": True} + if endpoint == "/v1/responses": + request_body["input"] = "question" + request_body.pop("mode") + else: + request_body["messages"] = [{"role": "user", "content": "question"}] + connection.request("POST", endpoint, json.dumps(request_body), + {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + payload = response.read() + assert response.status == 200 + assert b"rate_limit_exceeded" in payload + connection.close() + server.shutdown() + observation, = export_decision_receipts(orchestrator._store)["observations"] + assert observation["status"] == "selection_failed" + assert observation["durable_ack_elapsed_ns"] is None + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +def test_http_evidence_embedding_cold_and_warm_keep_task_interval(tmp_path, monkeypatch): + """Routing evidence calls occur only cold and remain before task acknowledgement.""" + from contextual_orchestrator.decision_receipts import _CURRENT_DECISION, export_decision_receipts + orchestrator = TaskOrchestrator([ + ModelAgent("worker_one", "mock/worker", tags=("writing",)), + ModelAgent("embedding_one", "mock-embedding", tags=("embedding",)), + ], state_db=tmp_path / "state.db") + original_embed = orchestrator.client.embed + embedding_snapshots = [] + embedding_inputs = [] + def observed_embed(*args, **kwargs): + embedding_inputs.extend(args[1]) + snapshot = _CURRENT_DECISION.get().snapshot() + embedding_snapshots.append(snapshot) + return original_embed(*args, **kwargs) + monkeypatch.setattr(orchestrator.client, "embed", observed_embed) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + counts = [] + for _ in range(2): + connection = http.client.HTTPConnection(*server.server_address) + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "route", + "messages": [{"role": "user", "content": "same evidence request"}], + }), {"Content-Type": "application/json", "Authorization": "Bearer test-token", + "x-cache-bypass": "true"}) + response = connection.getresponse() + response.read() + assert response.status == 200 + connection.close() + counts.append(len(embedding_snapshots)) + server.shutdown() + assert counts[0] > 0 + assert counts[1] == counts[0] + assert embedding_inputs.count("same evidence request") == 1 + assert all(snapshot["first_provider_phase"] == "routing_evidence_embedding" + for snapshot in embedding_snapshots) + cold, warm = export_decision_receipts(orchestrator._store)["observations"] + assert cold["auxiliary_dispatches"] + assert all(row["finished_elapsed_ns"] <= cold["selection_elapsed_ns"] + for row in cold["auxiliary_dispatches"] if row["phase"] == "routing_evidence_embedding") + assert any(row["phase"] == "post_decision_evidence_embedding" + for row in cold["auxiliary_dispatches"]) + assert warm["auxiliary_dispatches"] == [] + assert cold["durable_ack_elapsed_ns"] is not None + assert warm["durable_ack_elapsed_ns"] is not None + finally: + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +def test_generated_planner_is_auxiliary_until_worker_selection(tmp_path, monkeypatch): + """A planning provider does not freeze the task-execution route interval.""" + from contextual_orchestrator.decision_receipts import DecisionMeasurement + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + measurement = DecisionMeasurement(orchestrator._store) + def plan_reply(agent, messages, **kwargs): + assert measurement.snapshot()["durable_ack_elapsed_ns"] is None + return json.dumps({"steps": [{"id": 0, "role": "worker", "agent_id": agent.id, + "subtask": "work", "access": []}, + {"id": 1, "role": "synthesizer", "agent_id": agent.id, + "subtask": "answer", "access": [0]}]}) + monkeypatch.setattr(orchestrator.client, "chat", plan_reply) + try: + orchestrator._plan_generated("answer") + assert measurement.snapshot()["status"] == "accepted" + assert measurement.snapshot()["first_provider_phase"] == "generated_planner" + assert not orchestrator._store.load("initial_decision") + finally: + measurement.close() + orchestrator.close() + + +@pytest.mark.parametrize("diagnostic_count", [16, 17]) +def test_diagnostic_window_cap_preserves_shared_admissions(tmp_path, diagnostic_count): + """One noisy request cannot remove another admitted request from the export.""" + from contextual_orchestrator.orchestrator import _StateStore + from contextual_orchestrator.decision_receipts import export_decision_receipts + store = _StateStore(tmp_path / "state.db") + try: + for request_id in ("outside_cohort", "quiet_request", "noisy_request"): + store.save("accepted_request", None, {"request_id": request_id}, durable=True) + store.save("auxiliary_dispatch", "outside_cohort", + {"request_id": "outside_cohort"}, durable=True) + for index in range(diagnostic_count): + store.save("auxiliary_dispatch", "noisy_request", + {"request_id": "noisy_request", "test_index": index}, durable=True) + exported = export_decision_receipts(store, limit=2) + assert exported["window"]["diagnostic_limit"] == 16 + assert exported["window"]["diagnostic_truncated"] is (diagnostic_count > 16) + quiet, noisy = exported["observations"] + assert quiet["request_id"] == "quiet_request" + assert quiet["status"] == "unfinished" + assert quiet["auxiliary_dispatches"] == [] + assert noisy["request_id"] == "noisy_request" + assert len(noisy["auxiliary_dispatches"]) == 16 + assert all(row["request_id"] == "noisy_request" for row in noisy["auxiliary_dispatches"]) + finally: + store.close() + + +def test_http_answer_cache_keeps_admission_without_provider_duration(tmp_path, monkeypatch): + """Answer reuse has its own terminal outcome, never a copied provider timing.""" + from contextual_orchestrator.decision_receipts import export_decision_receipts + orchestrator = TaskOrchestrator( + [ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db", cache_ttl=60, + ) + original_chat = orchestrator.client.chat + provider_calls = [] + def counted_chat(*args, **kwargs): + provider_calls.append(True) + return original_chat(*args, **kwargs) + monkeypatch.setattr(orchestrator.client, "chat", counted_chat) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + answers = [] + try: + for _ in range(2): + connection = http.client.HTTPConnection(*server.server_address) + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "route", + "messages": [{"role": "user", "content": "same answer"}], + }), {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + answers.append(json.loads(response.read())["choices"][0]["message"]["content"]) + assert response.status == 200 + connection.close() + server.shutdown() + observations = export_decision_receipts(orchestrator._store)["observations"] + assert len(observations) == 2 + assert observations[0]["status"] == "acknowledged" + assert observations[1]["status"] == "cache_hit" + assert observations[1]["selection_elapsed_ns"] is None + assert observations[1]["durable_ack_elapsed_ns"] is None + assert observations[1]["first_provider_elapsed_ns"] is None + assert len(orchestrator._store.load("provider_dispatch")) == 1 + assert len(provider_calls) == 1 + assert answers[0] == answers[1] + finally: + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +@pytest.mark.parametrize("endpoint,stream", [ + ("/v1/chat/completions", False), ("/v1/chat/completions", True), + ("/v1/responses", True), +]) +def test_http_route_persists_initial_decision(tmp_path, monkeypatch, endpoint, stream): + """A real HTTP route retains one native-clock receipt before completion.""" + orchestrator = TaskOrchestrator( + [ModelAgent("worker_one", "mock/worker")], state_db=tmp_path / "state.db" + ) + dispatch_records = [] + dispatch_ready = threading.Event() + generation_allowed = threading.Event() + dispatched_snapshots = [] + + def inspect_committed_decision(original_call, *args, **kwargs): + with sqlite3.connect(tmp_path / "state.db") as independent: + rows = independent.execute( + "SELECT payload FROM orchestration_records WHERE kind = 'initial_decision'" + ).fetchall() + assert independent.execute( + "SELECT COUNT(*) FROM orchestration_records WHERE kind = 'accepted_request'" + ).fetchone()[0] == 1 + assert len(rows) == 1 + dispatch_records.extend(rows) + from contextual_orchestrator.decision_receipts import _CURRENT_DECISION + dispatched_snapshots.append(_CURRENT_DECISION.get().snapshot()) + dispatch_ready.set() + assert generation_allowed.wait(10) + return original_call(*args, **kwargs) + + for method_name in ("chat", "stream_chat"): + original_call = getattr(orchestrator.client, method_name) + monkeypatch.setattr(orchestrator.client, method_name, + lambda *args, _call=original_call, **kwargs: + inspect_committed_decision(_call, *args, **kwargs)) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address) + try: + body = {"model": "orchestrator/auto", "mode": "route", "stream": stream, + "messages": [{"role": "user", "content": "hello"}]} + if endpoint == "/v1/responses": + body = {"model": "orchestrator/auto", "stream": True, "input": "hello"} + connection.request( + "POST", endpoint, json.dumps(body), + {"Content-Type": "application/json", "Authorization": "Bearer test-token"}, + ) + assert dispatch_ready.wait(10) + assert dispatched_snapshots[0]["status"] == "acknowledged" + assert dispatched_snapshots[0]["durable_ack_elapsed_ns"] is not None + # Hold generation after the ack: the final interval must remain exactly + # the pre-generation native value, irrespective of the hold duration. + assert not generation_allowed.is_set() + generation_allowed.set() + response = connection.getresponse() + response.read() + assert response.status == 200 + assert dispatch_records + records = orchestrator._store.load("decision_receipt") + assert len(records) == 1 + assert records[0]["status"] == "acknowledged" + assert records[0]["selected_agent_ids"] == ["worker_one"] + assert records[0]["selection_elapsed_ns"] <= records[0]["durable_ack_elapsed_ns"] + assert records[0]["durable_ack_elapsed_ns"] == dispatched_snapshots[0]["durable_ack_elapsed_ns"] + assert len(orchestrator._store.load("initial_decision")) == 1 + finally: + generation_allowed.set() + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +def test_measurement_requires_durable_store(): + """Opt-in cannot silently lose every missing-store denominator observation.""" + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")]) + with pytest.raises(ValueError, match="durable state store"): + build_server(orchestrator, port=0, decision_receipts=True) + orchestrator.close() + + +def test_native_rejects_out_of_order_and_duplicate_ack(): + """Native transitions never manufacture missing elapsed times.""" + from contextual_orchestrator._decision_receipt import DecisionReceipt + + receipt = DecisionReceipt() + with pytest.raises(ValueError): + receipt.record_durable_ack() + assert receipt.status == "accepted" + assert receipt.durable_ack_elapsed_ns is None + receipt.record_selection() + receipt.record_durable_ack() + before = receipt.durable_ack_elapsed_ns + with pytest.raises(ValueError): + receipt.record_durable_ack() + assert receipt.durable_ack_elapsed_ns == before + + +def test_write_failure_has_no_ack_and_does_not_log_exception_contents(caplog): + """Failed commits cannot become successful timing samples or leak detail.""" + from contextual_orchestrator.decision_receipts import DecisionMeasurement + + auxiliary_records = [] + class FailedStore: + def save(self, *args, **kwargs): + if args[0] == "initial_decision": + raise RuntimeError("secret-canary-never-log") + if args[0] == "auxiliary_dispatch": + auxiliary_records.append(args[2]) + + measurement = DecisionMeasurement(FailedStore()) + try: + measurement.select(["worker_one"], "route") + snapshot = measurement.snapshot() + assert snapshot["status"] == "write_failed" + assert snapshot["durable_ack_elapsed_ns"] is None + snapshot["selected_agent_ids"].append("untrusted_mutation") + assert measurement.snapshot()["selected_agent_ids"] == ["worker_one"] + with measurement.auxiliary_call(["embedding_one"], "routing_evidence_embedding"): + pass + assert auxiliary_records[0]["phase"] == "post_decision_evidence_embedding" + finally: + measurement.close() + assert "secret-canary-never-log" not in caplog.text + assert "error_type=RuntimeError" in caplog.text + + +def test_admission_write_failure_rejects_before_dispatch_and_recovers(tmp_path, monkeypatch): + """A failed ingress receipt returns safe 503 and does not poison the next context.""" + orchestrator = TaskOrchestrator( + [ModelAgent("worker_one", "mock/worker")], state_db=tmp_path / "state.db" + ) + original_save = orchestrator._store.save + original_chat = orchestrator.client.chat + failed_once = [] + dispatched = [] + + def fail_first_admission(kind, *args, **kwargs): + if kind == "accepted_request" and not failed_once: + failed_once.append(True) + raise RuntimeError("never-disclose-storage-secret") + return original_save(kind, *args, **kwargs) + + def record_dispatch(*args, **kwargs): + dispatched.append(True) + return original_chat(*args, **kwargs) + + monkeypatch.setattr(orchestrator._store, "save", fail_first_admission) + monkeypatch.setattr(orchestrator.client, "chat", record_dispatch) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address) + try: + for expected_status in (503, 200): + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "route", + "messages": [{"role": "user", "content": "hello"}], + }), {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + payload = response.read().decode() + assert response.status == expected_status + assert "never-disclose-storage-secret" not in payload + if expected_status == 503: + assert not dispatched + assert '"measurement_complete": false' in payload + assert len(orchestrator._store.load("accepted_request")) == 1 + assert len(orchestrator._store.load("decision_receipt")) == 1 + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +def test_export_retains_accepted_request_without_finalization(tmp_path): + """Crash-like missing finalization stays an unfinished denominator row.""" + from contextual_orchestrator.decision_receipts import export_decision_receipts + from contextual_orchestrator.orchestrator import _StateStore + + store = _StateStore(tmp_path / "state.db") + try: + store.save("accepted_request", None, { + "request_id": "accepted_only", "status": "accepted", + "selection_elapsed_ns": None, "durable_ack_elapsed_ns": None, + }, durable=True) + exported = export_decision_receipts(store) + assert exported["measurement_complete"] is False + assert exported["reconciliation_required"] is True + assert exported["observations"][0]["request_id"] == "accepted_only" + assert exported["observations"][0]["status"] == "unfinished" + assert exported["observations"][0]["durable_ack_elapsed_ns"] is None + finally: + store.close() + + +def test_export_window_keeps_unfinished_selected_cohort(tmp_path): + """A bounded admission cohort never silently drops its unfinished member.""" + from contextual_orchestrator.decision_receipts import export_decision_receipts + from contextual_orchestrator.orchestrator import _StateStore + + store = _StateStore(tmp_path / "state.db") + try: + for request_id in ("older_request", "unfinished_request", "newest_request"): + store.save("accepted_request", None, {"request_id": request_id, "status": "accepted"}, durable=True) + exported = export_decision_receipts(store, limit=2) + assert exported["window"]["truncated"] is True + assert [row["request_id"] for row in exported["observations"]] == ["unfinished_request", "newest_request"] + assert all(row["status"] == "unfinished" for row in exported["observations"]) + assert len(store.load("accepted_request")) == 3 # No retention deletion. + finally: + store.close() + + +def test_failed_nested_admission_restores_same_thread_context(tmp_path): + """A constructor failure restores its prior token before leaving the thread.""" + from contextual_orchestrator.decision_receipts import DecisionMeasurement, _CURRENT_DECISION + from contextual_orchestrator.orchestrator import _StateStore + + class FailedStore: + def save(self, *args, **kwargs): + raise RuntimeError("unavailable") + + store = _StateStore(tmp_path / "state.db") + outer = DecisionMeasurement(store) + try: + with pytest.raises(RuntimeError, match="could not be persisted"): + DecisionMeasurement(FailedStore()) + assert _CURRENT_DECISION.get() is outer + finally: + outer.close() + store.close() + assert _CURRENT_DECISION.get() is None + + +@pytest.mark.parametrize("invalid_capacity", [False, True]) +def test_race_receipt_retains_candidate_set_not_winner(tmp_path, monkeypatch, invalid_capacity): + """Real race workers share one clock; rejected races cannot acknowledge selection.""" + from contextual_orchestrator.decision_receipts import DecisionMeasurement + from contextual_orchestrator.orchestrator import MAX_LOCAL_CONCURRENCY + + contract = { + "contract_id": "test_contract", "model_revision": "test_revision", + "reasoning_effort_profile": "worker_medium", "capability_set": ["text"], + "structured_output_contract": "openai_response_v1", "accuracy_class": "full_precision", + "data_residency_policy": "test_region", "retention_policy": "zero_retention", + "context_limit": 128000, "pricing_evidence_id": "test_price_evidence", + "hedge_eligible": True, "cancellation_supported": False, + "execution_policy": "immediate_race", + } + agents = [ModelAgent(f"worker_{index}", "mock/worker", group_name="shared_group", + endpoint_equivalence=contract) + for index in range(2)] + orchestrator = TaskOrchestrator(agents, state_db=tmp_path / "state.db") + measurement = DecisionMeasurement(orchestrator._store) + try: + if invalid_capacity: + monkeypatch.setattr(orchestrator, "_equivalent_race_members", + lambda *args, **kwargs: [agents[0]] * (MAX_LOCAL_CONCURRENCY + 1)) + with pytest.raises(ValueError, match="concurrency capacity"): + orchestrator._invoke(agents[0], [{"role": "user", "content": "hello"}], + text="hello", role="worker") + assert measurement.receipt.status == "accepted" + assert not orchestrator._store.load("initial_decision") + else: + for _ in range(2): + orchestrator._invoke(agents[0], [{"role": "user", "content": "hello"}], + text="hello", role="worker") + snapshot = measurement.snapshot() + assert snapshot["status"] == "acknowledged" + assert snapshot["selection_attempt_count"] == 2 + assert set(snapshot["selected_agent_ids"]) == {"worker_0", "worker_1"} + assert len(orchestrator._store.load("initial_decision")) == 1 + finally: + measurement.close() + orchestrator.close() + + +def test_embedding_failover_has_one_admission_and_two_selection_attempts(tmp_path, monkeypatch): + """Two backend submissions remain children of one validated HTTP request.""" + from contextual_orchestrator.cost_router import CostRoutingCoordinator + + agents = [ModelAgent(f"embedding_{index}", f"mock-embedding-{index}", tags=("embedding",)) + for index in range(2)] + orchestrator = TaskOrchestrator(agents, state_db=tmp_path / "state.db") + class FixtureTokenCounter: + def count_text(self, text, model): + assert text == "hello" + return 1 + + coordinator = CostRoutingCoordinator(orchestrator, embedding_token_counter=FixtureTokenCounter()) + backend = coordinator.embedding_batch_backend + original_submit = backend.submit + submissions = [] + + def fail_first_submission(*args, **kwargs): + submissions.append(True) + if len(submissions) == 1: + raise RuntimeError("controlled first member failure") + return original_submit(*args, **kwargs) + + monkeypatch.setattr(backend, "submit", fail_first_submission) + server = build_server(orchestrator, port=0, coordinator=coordinator, + decision_receipts=True, security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address) + try: + connection.request("POST", "/v1/embeddings", json.dumps({"input": "hello"}), + {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + response.read() + assert response.status == 200 + connection.close() + server.shutdown() + server.server_close() + assert len(submissions) == 2 + assert len(orchestrator._store.load("accepted_request")) == 1 + receipts = orchestrator._store.load("decision_receipt") + assert len(receipts) == 1 + assert receipts[0]["selection_attempt_count"] == 2 + assert receipts[0]["admission_boundary"] == "validated_endpoint" + assert len(orchestrator._store.load("selection_attempt")) == 1 + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +def test_legacy_identity_backfill_and_indexed_window(tmp_path): + """Legacy keys migrate without touching unrelated data or scanning every phase.""" + from contextual_orchestrator.orchestrator import _StateStore + + database = tmp_path / "state.db" + store = _StateStore(database) + with store._conn: + store._conn.executemany( + "INSERT INTO orchestration_records(kind, key, payload) VALUES (?, NULL, ?)", + [(kind, json.dumps({"request_id": f"request_{index}", "status": "accepted"})) + for index in range(1000) for kind in ("accepted_request", "initial_decision")] + + [("unrelated_legacy", "not valid JSON")], + ) + store.close() + store = _StateStore(database) + traced = [] + try: + assert store._conn.execute("SELECT COUNT(*) FROM orchestration_records").fetchone()[0] == 2001 + assert store._conn.execute( + "SELECT key FROM orchestration_records WHERE kind = 'unrelated_legacy'" + ).fetchone()[0] is None + store._conn.set_trace_callback(traced.append) + cohort = store.load_decision_window(2) + store._conn.set_trace_callback(None) + assert len(cohort["accepted"]) == len(cohort["decisions"]) == 2 + phase_query = next(query for query in traced if query.startswith("SELECT kind, key, payload")) + plan = store._conn.execute("EXPLAIN QUERY PLAN " + phase_query).fetchall() + assert any("orchestration_records_kind_key_seq" in row[3] + and "kind=? AND key=?" in row[3] for row in plan) + assert cohort["window"]["truncated"] is True + finally: + store.close() + + +def test_http_cold_and_cached_triage_keep_task_ack_after_auxiliary_work(tmp_path, monkeypatch): + """Cold triage is diagnostic only; warm triage still measures the task decision.""" + from contextual_orchestrator.decision_receipts import _CURRENT_DECISION + + orchestrator = TaskOrchestrator( + [ModelAgent("worker_one", "mock/worker", tags=("writing",))], + state_db=tmp_path / "state.db", + ) + original_chat = orchestrator.client.chat + auxiliary_ready = threading.Event() + auxiliary_release = threading.Event() + auxiliary_snapshots = [] + task_snapshots = [] + + def controlled_chat(agent, messages, **kwargs): + measurement = _CURRENT_DECISION.get() + if messages[0]["content"] == orchestrator.TRIAGE_SYSTEM_PROMPT: + auxiliary_snapshots.append(measurement.snapshot()) + auxiliary_ready.set() + assert auxiliary_release.wait(10) + return '{"workflow_required": false}' + task_snapshots.append(measurement.snapshot()) + return original_chat(agent, messages, **kwargs) + + monkeypatch.setattr(orchestrator.client, "chat", controlled_chat) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + for request_index in range(2): + connection = http.client.HTTPConnection(*server.server_address) + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "auto", + "messages": [{"role": "user", "content": "same question"}], + }), {"Content-Type": "application/json", "Authorization": "Bearer test-token", + "x-cache-bypass": "true"}) + if request_index == 0: + assert auxiliary_ready.wait(10) + snapshot = auxiliary_snapshots[0] + assert snapshot["durable_ack_elapsed_ns"] is None + assert snapshot.get("first_provider_elapsed_ns") is not None + assert snapshot.get("first_provider_phase") == "structured_triage" + auxiliary_release.set() + response = connection.getresponse() + response.read() + assert response.status == 200 + connection.close() + server.shutdown() + server.server_close() + assert len(auxiliary_snapshots) == 1 + assert len(task_snapshots) == 2 + cold, warm = task_snapshots + assert cold["first_provider_elapsed_ns"] < cold["selection_elapsed_ns"] <= cold["durable_ack_elapsed_ns"] + assert warm["first_provider_phase"] != "structured_triage" + assert warm["durable_ack_elapsed_ns"] is not None + from contextual_orchestrator.decision_receipts import export_decision_receipts + exported = export_decision_receipts(orchestrator._store) + cold_export, warm_export = exported["observations"] + assert cold_export["first_provider_boundary"] == "provider_ready_before_diagnostic_commit" + assert len(cold_export["auxiliary_dispatches"]) == 1 + auxiliary = cold_export["auxiliary_dispatches"][0] + assert auxiliary["phase"] == "structured_triage" + assert auxiliary["outcome"] == "completed" + assert auxiliary["finished_elapsed_ns"] <= cold_export["selection_elapsed_ns"] + assert warm_export["auxiliary_dispatches"] == [] + finally: + auxiliary_release.set() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() diff --git a/tests/test_decision_wheel_manifest.py b/tests/test_decision_wheel_manifest.py new file mode 100644 index 000000000..5921ce11e --- /dev/null +++ b/tests/test_decision_wheel_manifest.py @@ -0,0 +1,31 @@ +"""Wheel ownership is validated before installation can mask package omissions.""" + +from pathlib import Path +import runpy +import zipfile + +import pytest + +verify_wheels = runpy.run_path( + str(Path(__file__).resolve().parents[1] / "scripts" / "verify_decision_wheel_manifest.py") +)["verify_wheels"] + + +@pytest.mark.parametrize("native_payload,expected_error", [ + ("contextual_orchestrator/_decision_receipt.abi3.so", None), + ("contextual_orchestrator/__init__.py", "overlap"), + ("unexpected_package/module.py", "only the receipt extension"), +]) +def test_native_wheel_ownership(tmp_path, native_payload, expected_error): + """Reject overlapping or unexpected native payloads; accept disjoint bindings.""" + core_path, native_path = tmp_path / "core.whl", tmp_path / "native.whl" + with zipfile.ZipFile(core_path, "w") as archive: + archive.writestr("contextual_orchestrator/__init__.py", "") + with zipfile.ZipFile(native_path, "w") as archive: + archive.writestr(native_payload, "unit fixture") + archive.writestr("native.dist-info/METADATA", "") + if expected_error: + with pytest.raises(ValueError, match=expected_error): + verify_wheels(core_path, native_path) + else: + verify_wheels(core_path, native_path) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index a6db1ec2f..02e0a7912 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -16,6 +16,8 @@ import threading import time +import pytest + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 @@ -121,6 +123,49 @@ def test_store_upserts_keyed_records_and_appends_streams() -> None: store.close() +@pytest.mark.parametrize("failure_phase", ["insert", "commit"]) +def test_failed_keyed_save_preserves_previous_committed_record(failure_phase: str) -> None: + """A failed replacement must not leak its deletion into the next commit.""" + with tempfile.TemporaryDirectory() as directory: + store = _StateStore(os.path.join(directory, "state.db")) + try: + store._conn.execute("PRAGMA foreign_keys = ON") + store.save("workflow_run", "run_existing", {"version": 1}) + if failure_phase == "insert": + store._conn.execute( + "CREATE TRIGGER reject_replacement BEFORE INSERT ON orchestration_records " + "WHEN NEW.payload = '{\"version\": 2}' " + "BEGIN SELECT RAISE(FAIL, 'injected write failure'); END" + ) + else: + # Both writes succeed; the deferred constraint fails only at commit. + store._conn.execute( + "CREATE TABLE linked_record (record_seq INTEGER REFERENCES " + "orchestration_records(seq) DEFERRABLE INITIALLY DEFERRED)" + ) + store._conn.execute( + "INSERT INTO linked_record SELECT seq FROM orchestration_records" + ) + store._conn.commit() + try: + store.save("workflow_run", "run_existing", {"version": 2}) + except sqlite3.IntegrityError: + pass + else: + raise AssertionError("the injected write failure did not occur") + assert not store._conn.in_transaction + store.save("workflow_run", "run_other", {"version": 3}) + assert store.load("workflow_run") == [{"version": 1}, {"version": 3}] + assert not store._conn.in_transaction + finally: + store.close() + reopened = _StateStore(os.path.join(directory, "state.db")) + try: + assert reopened.load("workflow_run") == [{"version": 1}, {"version": 3}] + finally: + reopened.close() + + def test_store_treats_kind_key_and_limit_as_sql_parameters() -> None: with tempfile.TemporaryDirectory() as directory: store = _StateStore(os.path.join(directory, "s.db")) diff --git a/tests/test_repository_security_metadata.py b/tests/test_repository_security_metadata.py index f2d82ccf5..090934bc7 100644 --- a/tests/test_repository_security_metadata.py +++ b/tests/test_repository_security_metadata.py @@ -216,7 +216,10 @@ def test_unit_workflow_uses_the_project_lock_for_git_runtime_dependencies(): assert re.search(r"@[0-9a-f]{40}(?:\s+#|$)", setup_uv_line) assert "# v" in setup_uv_line assert 'version: "0.12.5"' in workflow_text - assert "uv run --locked --extra api --extra db --extra queue --group dev python -m pytest -q" in workflow_text + locked_sync = "uv sync --locked --extra api --extra db --extra queue --group dev --group native-build" + native_build = "uv run --no-sync maturin develop --locked --release --features pyo3/extension-module" + full_tests = "uv run --no-sync python -m pytest -q" + assert workflow_text.index(locked_sync) < workflow_text.index(native_build) < workflow_text.index(full_tests) def test_local_full_suite_installs_runtime_and_test_lockfiles(): diff --git a/tests/test_stream_error_identity.py b/tests/test_stream_error_identity.py new file mode 100644 index 000000000..a04281313 --- /dev/null +++ b/tests/test_stream_error_identity.py @@ -0,0 +1,73 @@ +"""Real HTTP streaming errors retain the provider-observed request identity.""" + +import http.client +import json +import threading + +import pytest + +from contextual_orchestrator import ModelAgent, TaskOrchestrator +from contextual_orchestrator.provider_errors import ProviderUpstreamError +from contextual_orchestrator.server import SecurityConfig, build_server +from contextual_orchestrator.telemetry import current_request_id +from contextual_orchestrator.tool_fallback import ( + ToolFailureDecision, ToolFailureKind, ToolFallbackAction, ToolFallbackStoppedError, +) + + +@pytest.mark.parametrize("endpoint,error_kind", [ + ("/v1/chat/completions", "provider"), + ("/v1/responses", "provider"), + ("/v1/chat/completions", "tool"), +]) +def test_stream_error_preserves_request_identity(endpoint, error_kind, monkeypatch): + """Typed SSE failures use the same trusted ID as their provider invocation.""" + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")]) + observed_ids = [] + + def fail_stream(*args, **kwargs): + observed_ids.append(current_request_id()) + if error_kind == "tool": + raise ToolFallbackStoppedError("worker_one", ToolFailureDecision( + ToolFailureKind.AMBIGUOUS_OUTCOME, ToolFallbackAction.FAIL_CLOSED, + "ambiguous_outcome", False, False, + )) + raise ProviderUpstreamError( + agent_id="worker_one", model="mock/worker", error_code="unit_failure", + message="unit failure", client_status=502, transport="stream", + ) + + monkeypatch.setattr(orchestrator.client, "stream_chat", fail_stream) + monkeypatch.setattr(orchestrator, "would_route", lambda *args, **kwargs: True) + server = build_server(orchestrator, port=0, + security=SecurityConfig(auth_token="unit-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address, timeout=20) + try: + body = {"model": "orchestrator/auto", "mode": "route", "stream": True, + "messages": [{"role": "user", "content": "unit request"}]} + if endpoint == "/v1/responses": + body = {"model": "orchestrator/auto", "stream": True, "input": "hello"} + connection.request("POST", endpoint, json.dumps(body), { + "Content-Type": "application/json", "Authorization": "Bearer unit-token", + "X-Request-ID": "untrusted-client-id", + }) + response = connection.getresponse() + payload = response.read().decode() + assert response.status == 200 + events = [json.loads(line[6:]) for line in payload.splitlines() + if line.startswith("data: ") and line != "data: [DONE]"] + errors = [event["error"] for event in events if "error" in event] + errors += [event["response"]["error"] for event in events + if event.get("type") == "response.failed"] + assert len(observed_ids) == 1 + assert observed_ids[0] and observed_ids[0] != "untrusted-client-id" + assert len(errors) == 1 + assert errors[0]["detail"]["request_id"] == observed_ids[0] + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 542cf2a23..493498bff 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -384,19 +384,234 @@ def test_http_error_log_excludes_raw_session_id(monkeypatch, caplog): server = build_server(SimpleNamespace(agents=[], candidates=[]), port=0) handler = server.RequestHandlerClass.__new__(server.RequestHandlerClass) handler.path = "/v1/chat/completions" - monkeypatch.setattr(handler, "_send", lambda *_args, **_kwargs: None) + captured_send = MagicMock() + monkeypatch.setattr(handler, "_send", captured_send) token = set_session_id("session-secret") try: with caplog.at_level("WARNING"): - handler._send_error(401, "unauthorized", "not authorized") + handler._send_error( + 401, "unauthorized", "not authorized", + {"request_id": "untrusted\nlog-injection", "reason": "private-detail"}, + ) finally: reset_session_id(token) server.server_close() assert "request_failed" in caplog.text + response_payload = captured_send.call_args.args[0] + response_request_id = response_payload["error"]["detail"]["request_id"] + warning_messages = [ + record.getMessage() for record in caplog.records + if record.name == "contextual_orchestrator.server" + and record.getMessage().startswith("request_failed ") + ] + assert warning_messages == [ + f"request_failed status=401 code=unauthorized request_id={response_request_id}" + ] + assert response_request_id != "untrusted\nlog-injection" + assert len(response_request_id) == 32 + assert all(character in "0123456789abcdef" for character in response_request_id) + assert response_payload["error"]["detail"]["reason"] == "private-detail" + assert "untrusted" not in caplog.text + assert "private-detail" not in caplog.text assert "session-secret" not in caplog.text +def test_http_error_ids_correlate_over_real_connections(caplog): + """Separate HTTP errors carry distinct IDs matching their server warnings.""" + import http.client + import threading + + server = build_server(SimpleNamespace(agents=[], candidates=[]), port=0) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + request_ids = [] + try: + with caplog.at_level("WARNING", logger="contextual_orchestrator.server"): + for _request_index in range(2): + connection = http.client.HTTPConnection(*server.server_address, timeout=5) + try: + connection.request("GET", "/v1/models") + response = connection.getresponse() + assert response.status == 401 + payload = json.loads(response.read()) + request_id = payload["error"]["detail"]["request_id"] + request_ids.append(request_id) + expected_message = ( + f"request_failed status=401 code={payload['error']['code']} " + f"request_id={request_id}" + ) + assert expected_message in [record.getMessage() for record in caplog.records] + finally: + connection.close() + finally: + server.shutdown() + server_thread.join(timeout=5) + server.server_close() + assert len(set(request_ids)) == 2 + + +def test_provider_diagnostic_events_preserve_request_identity(caplog): + """Every retry outcome keeps trusted identity before untrusted error text.""" + agent = ModelAgent("diagnostic_agent", "mock-model") + failure = RuntimeError("controlled error") + with caplog.at_level("DEBUG"), telemetry_module.request_identity() as request_id: + orchestrator_module._log_provider_attempt(agent, 0, 1) + orchestrator_module._log_provider_attempt_failed(agent, 0, failure, False) + orchestrator_module._log_provider_backoff(agent, 0, 0.0) + orchestrator_module._log_provider_exhausted(agent, 2, failure) + orchestrator_module._log_provider_no_retry_budget(agent, 1, failure, transient=False) + orchestrator_module._log_provider_one_shot_call_failed(agent, 1, failure, transient=False) + orchestrator_module._log_provider_rejected_permanent(agent, 1, failure) + messages = [row.getMessage() for row in caplog.records if row.name == orchestrator_module.__name__] + assert len(messages) == 7 + assert all(f"request_id={request_id}" in message for message in messages) + assert f"request_id={request_id} error_message=" in messages[1] + + +def test_request_identity_restores_context_across_threads_and_failure(): + """Copied work inherits identity; reused workers and failed scopes do not leak it.""" + from concurrent.futures import ThreadPoolExecutor + from contextvars import copy_context + + assert telemetry_module.current_request_id() is None + with ThreadPoolExecutor(max_workers=1) as executor: + with telemetry_module.request_identity() as outer_id: + assert executor.submit(copy_context().run, telemetry_module.current_request_id).result() == outer_id + assert executor.submit(telemetry_module.current_request_id).result() is None + with pytest.raises(RuntimeError): + with telemetry_module.request_identity() as inner_id: + assert inner_id != outer_id + raise RuntimeError("controlled failure") + assert telemetry_module.current_request_id() == outer_id + assert telemetry_module.current_request_id() is None + assert executor.submit(telemetry_module.current_request_id).result() is None + + +def test_provider_attempts_share_http_error_identity(monkeypatch, caplog): + """Same-session HTTP requests need distinct identities before provider failure.""" + import http.client + import threading + from contextual_orchestrator import TaskOrchestrator + from contextual_orchestrator.server import SecurityConfig + + model_agent = ModelAgent("correlation_agent", "mock-model") + model_client = ModelClient(max_retries=0) + router = TaskOrchestrator([model_agent], client=model_client) + + def reject_send(*args, **kwargs): + raise RuntimeError("controlled provider failure") + + def fail_completion(*args, **kwargs): + return model_client._send_with_retry(model_agent, {}) + + monkeypatch.setattr(model_client, "_send", reject_send) + monkeypatch.setattr(router, "complete", fail_completion) + server = build_server(router, port=0, security=SecurityConfig(auth_token="test-correlation-token")) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + connection = http.client.HTTPConnection(*server.server_address, timeout=5) + request_ids = [] + first_socket = None + try: + with caplog.at_level("DEBUG"): + for request_index in range(2): + first_record = len(caplog.records) + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "mock-model", "messages": [{"role": "user", "content": "unit request"}], + }), { + "Content-Type": "application/json", "Authorization": "Bearer test-correlation-token", + "X-LineageWeave-Session-Id": "shared-private-session", + }) + if first_socket is None: + first_socket = connection.sock + assert first_socket is not None + else: + assert connection.sock is first_socket + response = connection.getresponse() + assert response.status >= 400 + assert not response.will_close + response_body = json.loads(response.read()) + request_id = response_body["error"]["detail"]["request_id"] + request_ids.append(request_id) + attempt_logs = [record.getMessage() for record in caplog.records[first_record:] + if record.getMessage().startswith(("provider_attempt ", "provider_attempt_failed "))] + assert len(attempt_logs) == 2, (request_index, attempt_logs) + assert all(f"request_id={request_id}" in message for message in attempt_logs) + assert len(set(request_ids)) == 2 + assert "shared-private-session" not in "\n".join(attempt_logs) + finally: + connection.close() + server.shutdown() + server_thread.join(timeout=5) + server.server_close() + router.close() + + +def test_concurrent_http_provider_identity_isolation(monkeypatch, caplog): + """Overlapping same-session requests keep their own provider/error identities.""" + import http.client + import threading + from concurrent.futures import ThreadPoolExecutor + from contextual_orchestrator import TaskOrchestrator + from contextual_orchestrator.server import SecurityConfig + + rendezvous = threading.Barrier(2, timeout=10) + overlapping_threads = set() + overlap_lock = threading.Lock() + model_agent = ModelAgent("parallel_agent", "mock-model") + model_client = ModelClient(max_retries=0) + router = TaskOrchestrator([model_agent], client=model_client) + + def reject_send(*args, **kwargs): + rendezvous.wait() + with overlap_lock: + overlapping_threads.add(threading.get_ident()) + raise RuntimeError("controlled overlapping failure") + + def fail_completion(*args, **kwargs): + return model_client._send_with_retry(model_agent, {}) + + monkeypatch.setattr(model_client, "_send", reject_send) + monkeypatch.setattr(router, "complete", fail_completion) + server = build_server(router, port=0, security=SecurityConfig(auth_token="parallel-test-token")) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + + def send_request(): + connection = http.client.HTTPConnection(*server.server_address, timeout=15) + try: + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "mock-model", "messages": [{"role": "user", "content": "unit request"}], + }), {"Content-Type": "application/json", "Authorization": "Bearer parallel-test-token", + "X-LineageWeave-Session-Id": "same-private-session"}) + response = connection.getresponse() + assert response.status == 502 + return json.loads(response.read())["error"]["detail"]["request_id"] + finally: + connection.close() + + try: + with caplog.at_level("DEBUG"), ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(send_request) for _ in range(2)] + request_ids = [future.result(timeout=20) for future in futures] + assert len(overlapping_threads) == 2 + assert len(set(request_ids)) == 2 + provider_logs = [row.getMessage() for row in caplog.records + if row.getMessage().startswith(("provider_attempt ", "provider_attempt_failed "))] + assert len(provider_logs) == 4 + for request_id in request_ids: + matching = [message for message in provider_logs if f"request_id={request_id}" in message] + assert len(matching) == 2 + assert sum(message.startswith("provider_attempt ") for message in matching) == 1 + assert "same-private-session" not in "\n".join(provider_logs) + finally: + server.shutdown() + server_thread.join(timeout=5) + server.server_close() + router.close() + + def test_http_diagnostics_exclude_raw_path_and_swallow_client_disconnect(monkeypatch, caplog): """Client cancellation cannot create a second error or leak path identifiers.""" server = build_server(SimpleNamespace(agents=[], candidates=[]), port=0) @@ -581,6 +796,11 @@ def test_per_request_info_summary_reports_method_path_and_status(caplog): assert "method=GET" in caplog.text assert "path=/healthz" in caplog.text assert "status=200" in caplog.text + import re + summary_lines = [row.getMessage() for row in caplog.records + if row.getMessage().startswith("http_request ")] + assert len(summary_lines) == 1 + assert re.search(r" request_id=[0-9a-f]{32}$", summary_lines[0]) def test_per_request_info_summary_never_includes_query_string(caplog): diff --git a/tests/test_workflow_request_link.py b/tests/test_workflow_request_link.py new file mode 100644 index 000000000..3f35cb053 --- /dev/null +++ b/tests/test_workflow_request_link.py @@ -0,0 +1,151 @@ +"""Trusted HTTP request identity survives durable workflow persistence.""" + +import http.client +import json +import threading + +import pytest + +from contextual_orchestrator import ModelAgent, TaskOrchestrator +from contextual_orchestrator.server import SecurityConfig, build_server +from contextual_orchestrator.telemetry import current_request_id, request_identity + + +@pytest.mark.parametrize("mode,stream", [("route", False), ("conduct", False), ("route", True), ("write_failure", True)]) +@pytest.mark.parametrize("measurement_enabled", [False, True]) +def test_http_workflow_retains_origin_request(tmp_path, monkeypatch, mode, stream, measurement_enabled): + """Provider-observed identity joins the stored run without caller identity trust.""" + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + observed_ids = [] + if mode == "write_failure": + original_save = orchestrator._store.save + + def fail_workflow_write(kind, *args, **kwargs): + if kind == "workflow_run": + raise RuntimeError("private-storage-failure") + return original_save(kind, *args, **kwargs) + + monkeypatch.setattr(orchestrator._store, "save", fail_workflow_write) + for method_name in ("chat", "stream_chat"): + original_method = getattr(orchestrator.client, method_name) + + def observe(*args, _method=original_method, **kwargs): + observed_ids.append(current_request_id()) + return _method(*args, **kwargs) + + monkeypatch.setattr(orchestrator.client, method_name, observe) + server = build_server(orchestrator, port=0, decision_receipts=measurement_enabled, + security=SecurityConfig(auth_token="unit-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address, timeout=20) + try: + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "route" if mode == "write_failure" else mode, "stream": stream, + "messages": [{"role": "user", "content": "Explain addition briefly."}], + }), {"Content-Type": "application/json", "Authorization": "Bearer unit-token", + "X-Request-ID": "untrusted-client-id"}) + response = connection.getresponse() + response_payload = response.read().decode() + connection.close() + server.shutdown() + server.server_close() + assert response.status == 200 + stored_runs = orchestrator._store.load("workflow_run") + if mode == "write_failure": + assert not stored_runs + assert len(orchestrator._workflow_runs) == 1 + assert '"finish_reason": "error"' in response_payload + assert "private-storage-failure" not in response_payload + if measurement_enabled: + from contextual_orchestrator.decision_receipts import export_decision_receipts + receipts = export_decision_receipts(orchestrator._store)["observations"] + assert len(receipts) == 1 + assert receipts[0]["status"] == "acknowledged" + return + assert len(stored_runs) == 1 + assert observed_ids and len(set(observed_ids)) == 1 + assert observed_ids[0] and observed_ids[0] != "untrusted-client-id" + assert stored_runs[0]["request_id"] == observed_ids[0] + if measurement_enabled: + from contextual_orchestrator.decision_receipts import export_decision_receipts + receipts = export_decision_receipts(orchestrator._store)["observations"] + assert len(receipts) == 1 + assert receipts[0]["request_id"] == stored_runs[0]["request_id"] + run_id = stored_runs[0]["workflow_run_id"] + orchestrator.close() + restored = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + try: + assert restored.get_workflow_run(run_id)["request_id"] == observed_ids[0] + finally: + restored.close() + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +def test_workflow_update_preserves_original_request_identity(): + """Later request contexts cannot claim an existing run or a non-HTTP run.""" + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")]) + try: + for origin_http in (False, True): + run_id = f"run_{origin_http}" + record = {"workflow_run_id": run_id, "trace": []} + if origin_http: + with request_identity() as origin_id: + orchestrator._replace_workflow_run(record) + else: + origin_id = None + orchestrator._replace_workflow_run(record) + with request_identity(): + replacement = {"workflow_run_id": run_id, "trace": []} + orchestrator._replace_workflow_run(replacement) + assert replacement.get("request_id") == origin_id + finally: + orchestrator.close() + + +def test_http_cache_hit_keeps_distinct_outcome_identity(tmp_path): + """A reused answer creates a cache-hit outcome, not a reassigned execution.""" + from contextual_orchestrator.decision_receipts import export_decision_receipts + + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db", cache_ttl=60) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="unit-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + for _ in range(2): + connection = http.client.HTTPConnection(*server.server_address, timeout=20) + try: + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "route", + "messages": [{"role": "user", "content": "Explain addition briefly."}], + }), {"Content-Type": "application/json", "Authorization": "Bearer unit-token"}) + response = connection.getresponse() + response.read() + assert response.status == 200 + finally: + connection.close() + server.shutdown() + server.server_close() + stored_runs = orchestrator._store.load("workflow_run") + receipts = export_decision_receipts(orchestrator._store)["observations"] + assert [record["cache_status"] for record in stored_runs] == ["miss", "hit"] + assert len({record["workflow_run_id"] for record in stored_runs}) == 2 + assert len({record["request_id"] for record in stored_runs}) == 2 + assert {record["request_id"] for record in stored_runs} == { + receipt["request_id"] for receipt in receipts + } + assert sorted(receipt["status"] for receipt in receipts) == ["acknowledged", "cache_hit"] + finally: + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() diff --git a/uv.lock b/uv.lock index 7a11636f2..05a583ae4 100644 --- a/uv.lock +++ b/uv.lock @@ -424,6 +424,9 @@ dev = [ { name = "hypothesis" }, { name = "pytest" }, ] +native-build = [ + { name = "maturin" }, +] [package.metadata] requires-dist = [ @@ -450,6 +453,7 @@ dev = [ { name = "hypothesis", specifier = ">=6.100" }, { name = "pytest", specifier = ">=8.0" }, ] +native-build = [{ name = "maturin", specifier = "==1.15.0" }] [[package]] name = "cryptography" @@ -897,6 +901,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "maturin" +version = "1.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/c8/22e5e21b2679c9bce6415ca578034ca2cc9316be0642ae21e051a2d5198c/maturin-1.15.0.tar.gz", hash = "sha256:94b26cc8e8aba61a5f2099715fe640e18c5f678e9a500408b38761263954228a", size = 385504, upload-time = "2026-08-24T12:11:22.665Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/69/5c01b461044eb1f45ddcce006706eb88110c793cdb11c7ae0b5e08492e94/maturin-1.15.0-py3-none-linux_armv6l.whl", hash = "sha256:6bf6dc62e22d4dcfd5a51244ff0d58975fa4979c48209fe84159617648956d82", size = 10206220, upload-time = "2026-08-24T12:10:53.327Z" }, + { url = "https://files.pythonhosted.org/packages/eb/1f/2b431554e11687cdb1077e0cdadcc118c53f611086b3af00c8545a67c6a5/maturin-1.15.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:cd35772633f489841132bc8e71d6fc7f842df30b9c05cd5cdf1ee1ddcb744cc7", size = 19416513, upload-time = "2026-08-24T12:10:56.126Z" }, + { url = "https://files.pythonhosted.org/packages/51/36/e23a21cb34a648b711036b9b2fe1d4f3f4ee24f8db54215d73f1a9a3a3ec/maturin-1.15.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c40b4eae7bf5ef1f4b1af8d623fe4105016f93578fb15b764e741d08ec3b92dd", size = 10014962, upload-time = "2026-08-24T12:10:58.486Z" }, + { url = "https://files.pythonhosted.org/packages/8c/80/33b15cb2d8f30f12c807955e8f2fd775027692904e30ec0784744ce8cd83/maturin-1.15.0-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:7eb066372f541f8eb4909c79c5d9bd0b9e8125980bdf1ec9e8aba23c6c8d6c55", size = 10196223, upload-time = "2026-08-24T12:11:00.696Z" }, + { url = "https://files.pythonhosted.org/packages/fe/91/b495e19e2f5c503b540452b2039115e7b2363867e8c5ad4179eb752fa92c/maturin-1.15.0-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:653020a63525bb224e5ab0adf02e17a2e08bc86dbea7fc1399c9a56d7529b99e", size = 10541186, upload-time = "2026-08-24T12:11:02.857Z" }, + { url = "https://files.pythonhosted.org/packages/5d/2b/2abff58037188d852b124871b1f0d720e1c2bfb3d4f1b03d87c52cd66488/maturin-1.15.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:0ebf9767892725083138e671c34482c660317a2f3d6a29fc0e0f34e9d8c99136", size = 10083468, upload-time = "2026-08-24T12:11:05.012Z" }, + { url = "https://files.pythonhosted.org/packages/3f/07/b7e9f8be99a6627849e81ac7b6694876bce8f50a92995fe17e3cf2610f0a/maturin-1.15.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:7ab7eebffd7b8debca2265985de4eaeb332141276d24b9560b5ad484d4b3add1", size = 10047786, upload-time = "2026-08-24T12:11:07.042Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ab/167e3cb7accee11b507dbe53e0e87aeccb376d44ae66284c96ee4df3a9fd/maturin-1.15.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:126e12e618b4db42f68c779a56d41f82a390145ba36ac3f621d057eb34f5ad9d", size = 13315332, upload-time = "2026-08-24T12:11:09.433Z" }, + { url = "https://files.pythonhosted.org/packages/14/4d/801379f646cbc6b00998e5289b0630a886be3a4ee4c75b6bc9b87478a7f1/maturin-1.15.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4f9d33e6c3f9615c8caceecbbbd440f8eb25a3ddeb687077682cd5eca2e9ae15", size = 10807183, upload-time = "2026-08-24T12:11:11.73Z" }, + { url = "https://files.pythonhosted.org/packages/89/27/2e612e1cbd1580e9e94d4722c227b5180dca27b32b955a34b79918aa1292/maturin-1.15.0-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:bf29beddd0c6708f112db51d5275fc28b28b9e9c9c5faae387eaef662918b176", size = 10413274, upload-time = "2026-08-24T12:11:14.04Z" }, + { url = "https://files.pythonhosted.org/packages/70/d8/202a7b4d75a51f20f84ec9ce3b7345b12b822207072164e2b1c6ef665125/maturin-1.15.0-py3-none-win32.whl", hash = "sha256:da649988be98e87e009e51b1bf0d301b6a301bc0cecbdd60d40d8ba60748d1ca", size = 8928744, upload-time = "2026-08-24T12:11:16.269Z" }, + { url = "https://files.pythonhosted.org/packages/40/dc/4e90da594986ba78dd3bc8a5921ecdcdb11085b22b02a412caab3b225601/maturin-1.15.0-py3-none-win_amd64.whl", hash = "sha256:552c2be4afd43fe8d5c9f3ec8d4c4756d973b8dcbe94c14084390301f50243e1", size = 10335085, upload-time = "2026-08-24T12:11:18.326Z" }, + { url = "https://files.pythonhosted.org/packages/8b/10/15d4314edf130955edf2dc237aa393a8a7c10f2b9b57b89fa2f61f915659/maturin-1.15.0-py3-none-win_arm64.whl", hash = "sha256:c7dc0c66c78d3debdd9c5aa807e861fbcbf07f3505d34b125df74c03986b0f48", size = 9713795, upload-time = "2026-08-24T12:11:20.83Z" }, +] + [[package]] name = "numpy" version = "2.5.2"