fix(embeddings): restore provider-backed batches on current main

.gitignore

@@ -17,3 +17,4 @@ tempcred.txt
 
 # background-agent isolated worktree scratch state (nested git checkouts)
 .claude/worktrees/
+.worktrees/

CHANGELOG.md

@@ -20,6 +20,38 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html)
 
 ### Fixed
 
+- Workflow workers now preserve the caller message array exactly once, while
+  the added envelope carries only the subtask and Conductor-style prior-step
+  access list instead of duplicating the task or source attachments.
+- Configured-gateway discovery now removes its blank bootstrap row after a
+  concrete catalog's chat candidates fail bounded readiness, so virtual
+  requests cannot bypass an authentication failure through an unprobed seed;
+  this retirement is process-local so a later startup can probe recovered
+  credentials, while explicit model pins still return their own typed error.
+- Queued embedding admissions now carry the durable registry's result
+  retention and the selected backend's polling cadence, so clients can poll
+  within the actual job lifecycle instead of guessing or failing closed on
+  missing lifecycle metadata.
+- Virtual structured workflows now exclude a same-endpoint candidate only
+  after both its synthesis and bounded repair violate the caller's schema,
+  then continue with the next eligible model on that endpoint. Explicit model
+  pins remain single-model and exhausted virtual pools return a typed error.
+- Configured-gateway runtime discovery now retains chat rows only after a
+  bounded structured-output probe, and virtual structured workflows share one
+  request-scoped missing-model exclusion set across evidence and synthesis.
+  Probe telemetry is separate from caller attempts, and explicit structured
+  requests keep their model pin throughout evidence, judgment, and synthesis.
+- Chat token accounting now uses valid provider usage or exact Rust raw-output
+  counts for ADR-declared tokenizer mappings. Unreconstructible prompts, tools,
+  multimodal input, unknown models, and missing stream usage are explicitly
+  unavailable; token-threshold routing stays synchronous, enabled budgets fail
+  closed, and API usage/cost fields no longer publish heuristic estimates
+  (ADR 0006).
+- Provider-embedding workers now propagate durable-claim renewal loss and
+  refresh ownership before terminal publication. Embedding token accounting
+  uses configured `pg_tiktoken` or the packaged Rust cl100k counter for exact
+  declared models, and otherwise fails closed without publishing estimated
+  usage or cost (ADR 0005). Chat accounting is governed separately by ADR 0006.
 - The HTTP embedding endpoints (`/v1/embeddings`, `/v1/batch/embeddings`)
   now correctly wire the coordinator's cheapest-price selection into an
   *omitted* `model` (the common case), not only an explicitly-named model or

Dockerfile

@@ -9,12 +9,29 @@
 # see docs/kv-credentials.md for the bootstrap flow.
 # Agents: defaults to the bundled mock pool; mount your own and set AGENTS_FILE:
 #           -v ./agents.json:/app/agents.json -e AGENTS_FILE=/app/agents.json
+ARG MATURIN_BUILDER_IMAGE=ghcr.io/pyo3/maturin@sha256:b6c8b59a0170b77eb31a35b56034abd39972483ad0ebfff344deaa42a85f3bd3
+FROM ${MATURIN_BUILDER_IMAGE} AS maturin-tools
+FROM rust:1.97.1-slim-bookworm@sha256:2775a09d208ff0d7c1f50490c45b62db929e87ba1dcbc3f2132ac71a704bcdd3 AS dependency-builder
+RUN apt-get update \
+    && apt-get install --no-install-recommends --yes build-essential \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=maturin-tools /usr/local/bin/uv /usr/local/bin/uv
+COPY --from=maturin-tools /usr/bin/maturin /usr/local/bin/maturin
+COPY requirements.lock /build/requirements.lock
+COPY rust/Cargo.toml rust/Cargo.lock /build/rust/
+COPY rust/token_counter/ /build/rust/token_counter/
+COPY contextual_orchestrator/ /build/contextual_orchestrator/
+RUN uv python install 3.12 \
+    && uv pip install --python 3.12 --require-hashes -r /build/requirements.lock --target /build/deps \
+    && maturin build --locked --release --manifest-path /build/rust/token_counter/Cargo.toml --out /build/wheels \
+    && uv pip install --python 3.12 /build/wheels/*.whl --target /build/deps
+
 # python:3.12-slim
 FROM python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf
 
 WORKDIR /app
 COPY pyproject.toml requirements.lock README.md LICENSE ./
-RUN pip install --no-cache-dir --require-hashes -r requirements.lock
+COPY --from=dependency-builder /build/deps/ /usr/local/lib/python3.12/site-packages/
 COPY contextual_orchestrator/ /usr/local/lib/python3.12/site-packages/contextual_orchestrator/
 COPY examples/ examples/
 

README.md

@@ -154,15 +154,15 @@ Non-mock providers must use `https://` URLs and a **resolvable KV credential** 
 One public interface:
 
 - `contextual-orchestrator` is the model-like control-plane candidate exposed to callers. `/v1/models` lists it first, followed by every configured worker candidate, including disabled candidates with their status.
-- `/v1/chat/completions` accepts normal chat messages, and `"stream": true` returns an OpenAI-compatible `text/event-stream` of `chat.completion.chunk` deltas terminated by `data: [DONE]`. `stream_options.include_usage=true` is accepted for ordinary chat streams and emits a usage-only chunk after the terminal stop chunk, labeled `usage_source: reported` when the provider returned usage or `usage_source: estimated` (never mislabeled `reported`) otherwise; single-agent `tools` passthrough accepts it the same way from the one non-streaming upstream call — the provider's own usage field when present, an honest estimate when the provider omits it; `response_format`-only structured passthrough (conduct mode) still rejects the combination before provider execution, since its usage comes from a multi-step workflow's cost ledger and may be unmeasured. In **route** mode the worker's tokens are streamed live as they arrive from the provider (real token streaming); in **conduct** mode the multi-step answer is produced then framed as deltas (a workflow can't honestly token-stream a synthesizer that hasn't run yet).
+- `/v1/chat/completions` accepts normal chat messages, and `"stream": true` returns an OpenAI-compatible `text/event-stream` of `chat.completion.chunk` deltas terminated by `data: [DONE]`. `stream_options.include_usage=true` is accepted for ordinary chat streams and emits a usage-only chunk after the terminal stop chunk. Valid provider counts carry `usage_source: reported` and `usage_measurement_status: measured`; missing or malformed counts carry `usage: null` and `usage_measurement_status: unavailable`. Single-agent `tools` passthrough follows the same rule and never reconstructs tool or multimodal framing. `response_format`-only structured passthrough (conduct mode) still rejects the combination before provider execution when workflow-level usage is unavailable. In **route** mode the worker's tokens are streamed live as they arrive from the provider (real token streaming); in **conduct** mode the multi-step answer is produced then framed as deltas (a workflow can't honestly token-stream a synthesizer that hasn't run yet).
 - `TaskOrchestrator.complete()` decides whether to route to one worker or run a short workflow.
 - `TaskOrchestrator.compare_to_baseline(prompts, mode)` (CLI `--eval PROMPT...`) measures the orchestration engine against a single-worker baseline — per-prompt and aggregate latency plus a structural coverage delta (contributing steps + verifier-pass presence). It is a measured tradeoff report, not a human-quality claim.
 - Responses include orchestration mode metadata, and trusted callers can request the full trace for audit.
 - `/admin` exposes an operator console for agent pool, policy, trace, and audit review.
 - The admin console can use [Clearfolio](https://github.com/ContextualWisdomLab/clearfolio) as its document viewer: pass `--clearfolio-url URL` (or `CONTEXTUAL_ORCHESTRATOR_CLEARFOLIO_URL`) and the Integrations view gains a Document Viewer card (open viewer / deep-link `{url}/viewer/{docId}`). Default: disabled, console unchanged.
 - `/api/v1/provider_readiness/latest` reports provider liveness separately from an explicit chat readiness probe; `?refresh=true` re-probes instead of returning the cached result.
 - `/api/v1/analytics_snapshots/latest` returns source-backed local KPI definitions (trace completeness, policy-safe run rate, successful chat requests, and related event-derived counts) from in-memory runtime state, localized via the same locale bundles as the admin console.
-- `/api/v1/spend_analytics/latest` exposes per-model token and cost spend aggregated from workflow runs. Output tokens use provider-reported `usage` when available and fall back to a ~4 chars/token estimate otherwise (each model row is labeled `usage_source: reported | mixed | estimated`); cost is computed only for models with an operator-supplied price (`TaskOrchestrator(price_per_million=...)`), otherwise reported as null with the model listed under `unpriced_models`. See [Observability & spend](#observability--spend).
+- `/api/v1/spend_analytics/latest` exposes per-model token and cost spend aggregated from workflow runs. Valid provider usage is authoritative; declared model IDs may use the packaged Rust tokenizer for exact raw textual output. Prompt framing, tools, multimodal input, unknown tokenizers, and missing native code remain unavailable. Cost is computed only when every required count and operator-supplied price is available. See [Observability & spend](#observability--spend).
 - `/api/v1/sales_readiness/latest` exposes a local enterprise-pilot readiness gate for API compatibility, operator evidence, workflow traces, evaluation replay, security posture, analytics truthfulness, locale parity, and provider egress safety. It is process-local evidence, not a production compliance certificate.
 - `/api/v1/commercial_readiness/latest` exposes a KRW 2,000,000,000 commercial due-diligence readiness gate. It is a buyer-review evidence snapshot, not a valuation guarantee or purchase commitment.
 - `/api/v1/commercial_evidence_manifests/latest` shows the evidence gaps to resolve before commercial due diligence. The former `/api/v1/buyer_evidence_manifests/latest` route remains a deprecated compatibility alias.
@@ -201,23 +201,23 @@ See [docs/architecture.md](docs/architecture.md) for the source-backed analysis.
 
 ## Observability & spend
 
-Local spend observability, aggregated from in-memory workflow runs. It is honest by construction — estimates are labeled, and cost is only reported when a price is configured.
+Local spend observability, aggregated from in-memory workflow runs. It is honest by construction: counts are authoritative or explicitly unavailable, and cost is reported only when its required counts and prices are available.
 
 ```bash
 curl -s http://127.0.0.1:8000/api/v1/spend_analytics/latest \
   -H "authorization: Bearer $local_token" | jq '.totals, .by_model, .budget'
 ```
 
-- **Tokens.** `by_model[].output_tokens` uses the provider-reported `usage.completion_tokens` when a real worker returns it, and falls back to a `~4 chars/token` estimate otherwise. Each row carries `usage_source`: `reported` (all steps reported), `mixed`, or `estimated`. `estimated_output_tokens` is always the estimate, kept alongside for comparison. `measurement_status` is `local_runtime_estimate`, not production telemetry.
-- **Cost.** Supply a price table to turn tokens into money — `TaskOrchestrator(price_per_million={"gpt-5.5": 10.0})` (USD per 1M output tokens). Models without a price appear under `unpriced_models` with `estimated_cost_usd: null`. No prices are assumed or fabricated.
+- **Tokens.** `by_model[].output_tokens` uses provider-reported completion/output tokens first. For exact full model IDs declared by ADR 0006, a missing output count may use the packaged Rust tokenizer over raw textual output only. Rows carry `usage_source: reported | tokenizer | mixed | unavailable`; unavailable rows return `output_tokens: null`. Prompt tokens are provider-reported or null because chat framing is not reconstructed.
+- **Cost.** Supply a price table to turn authoritative output tokens into money — `TaskOrchestrator(price_per_million={"gpt-5.5": 10.0})` (USD per 1M output tokens). Models without a price appear under `unpriced_models`; `cost_usd` remains null when a price or required token count is unavailable. No prices or token counts are assumed.
 - **Budget cap.** Set an operator cap to refuse runaway spend (default: no cap):
 
   ```bash
   python -m contextual_orchestrator --serve --agents examples/agents.mock.json \
     --budget-max-output-tokens 2000000 --budget-max-cost-usd 50
   ```
 
-  Or in code: `TaskOrchestrator(budget_max_output_tokens=..., budget_max_cost_usd=...)`. Once spend reaches a cap, the next run is refused — `run()` raises `BudgetExceededError` and `/v1/chat/completions` returns HTTP `429 budget_exceeded`. Current state is in `spend_analytics()["budget"]` (`enabled`, limits, `spent_*`, `remaining_*`, `exceeded`). Cost caps require a price table; token caps do not.
+  Or in code: `TaskOrchestrator(budget_max_output_tokens=..., budget_max_cost_usd=...)`. Once spend reaches a cap, the next run is refused — `run()` raises `BudgetExceededError` and `/v1/chat/completions` returns HTTP `429 budget_exceeded`. An enabled budget also fails closed when a required count or price is unavailable. Current state is in `spend_analytics()["budget"]` (`enabled`, limits, nullable `spent_*`/`remaining_*`, `measurement_status`, `enforcement_status`, `exceeded`). Cost caps require a complete price table; token caps require authoritative output counts.
 - **Admin.** The `/admin` **Observability** view renders the totals and the per-model table (unpriced models show an `unpriced` chip).
 
 These are process-local measured signals for a stdlib lab, not a billing system or production compliance data.
@@ -240,7 +240,9 @@ is read from a **KV config store**, never `os.getenv`.
   first-class dimensions catalogued in `cost_attribution_dimensions`: **account,
   service, upstream API/provider, model name, team, group, company**. Token
   counts reuse `pg-llm-batch`'s `pg_tiktoken` counter when a Postgres DSN is
-  configured, and fall back to a deterministic heuristic otherwise.
+  configured. Valid provider usage is authoritative; missing chat framing,
+  tool, multimodal, or unknown-tokenizer counts remain explicitly unavailable
+  instead of falling back to a deterministic heuristic.
 - **Canonical Billing export.** Install the published `metering_billing`
   producer SDK, create its durable outbox, and pass
   `CanonicalUsageRecordSink(event_builder=build_contextual_usage_event,

contextual_orchestrator/__init__.py

@@ -11,6 +11,7 @@
     LocalEmbeddingBatchBackend,
     PgLlmBatchBackend,
     PgLlmBatchEmbeddingBackend,
+    ProviderEmbeddingBatchBackend,
     RoutingDecision,
     RoutingHints,
     RoutingPolicy,
@@ -61,7 +62,14 @@
     parse_reasoning_effort_profile,
     snapshot_role_effort_catalog,
 )
-from .token_counting import HeuristicTokenCounter, build_token_counter
+from .token_counting import (
+    NativeCl100kTokenCounter,
+    NativeExactTokenCounter,
+    TokenCountUnavailable,
+    UnavailableTokenCounter,
+    build_embedding_token_counter,
+    build_token_counter,
+)
 from .response_cache import (
     RedisResponseCacheProvider,
     ResponseCacheProvider,
@@ -119,7 +127,11 @@
     # config / tokens
     "InMemoryConfigStore",
     "get_config_store",
-    "HeuristicTokenCounter",
+    "NativeCl100kTokenCounter",
+    "NativeExactTokenCounter",
+    "TokenCountUnavailable",
+    "UnavailableTokenCounter",
+    "build_embedding_token_counter",
     "build_token_counter",
     "ResponseCacheProvider",
     "RedisResponseCacheProvider",
@@ -139,6 +151,7 @@
     "EmbeddingBatchResultItem",
     "LocalEmbeddingBatchBackend",
     "PgLlmBatchEmbeddingBackend",
+    "ProviderEmbeddingBatchBackend",
     "heuristic_embedding",
     "build_embeddings_jsonl_body",
     "cheapest_upstream",