diff --git a/.github/workflows/downstream-canary.yml b/.github/workflows/downstream-canary.yml index 8d66249..7ce2fb8 100644 --- a/.github/workflows/downstream-canary.yml +++ b/.github/workflows/downstream-canary.yml @@ -9,10 +9,23 @@ name: Downstream Canary # downstream consumer (stella) against THIS repo's HEAD so that break is # visible before the freeze, not after. # -# Deliberately advisory, not a required check: this repo's own gate must stay -# green on this repo's own guarantees, not on a downstream project's -# unrelated churn. `continue-on-error` + an explicit ::warning:: keeps the -# signal visible without letting a foreign repo block a merge here. +# Advisory on a pull request, and NOT advisory on the schedule. This repo's own +# gate must stay green on this repo's own guarantees, so a downstream project's +# unrelated churn must never block a merge here — hence `continue-on-error` on +# `pull_request`. +# +# But "advisory" was originally implemented as advisory *everywhere*, and that +# made the canary useless in exactly the case it was built for. On 2026-07-29 it +# caught a real break — stella's `Host::add_http` call had not been updated for +# the C7/C8 credential parameter — emitted its `::warning::`, wrote its step +# summary, and reported the run as **success**. Nobody saw it. stella stayed +# thirteen commits behind for as long as it took a human to go looking. +# +# A warning on a green run is not a signal; it is a note in a file nobody opens. +# So the scheduled run now *fails* on a downstream break. It gates no PR and +# blocks no merge — it has nothing to block — so failing costs nothing and buys +# a red run in the Actions list plus GitHub's scheduled-failure notification. +# The PR path is untouched and still cannot block a merge here. on: schedule: @@ -41,7 +54,7 @@ env: jobs: stella-canary: - name: stella builds against CGP HEAD (advisory) + name: stella builds against CGP HEAD (advisory on PR, gating on schedule) runs-on: ubuntu-latest steps: - name: Checkout context-graph-protocol (this repo, HEAD) @@ -70,7 +83,10 @@ jobs: STELLA_DIR: ${{ github.workspace }}/stella run: ./cgp/.github/scripts/downstream-canary-stella.sh - - name: Flag the break (advisory — does not fail the job) + # `continue-on-error` above is unconditional so that this step always runs + # and always records what happened. Whether the *job* then goes red is + # decided here, by event: a scheduled run fails, a pull request does not. + - name: Flag the break (fails the scheduled run; advisory on a PR) if: steps.build.outcome == 'failure' run: | echo "::warning title=downstream canary::stella no longer builds against context-graph-protocol HEAD (${{ github.sha }}) — a breaking change to contextgraph-types::ContextFrame or another wire type likely needs a coordinated stella update before the next freeze/tag." @@ -79,6 +95,17 @@ jobs: echo echo "stella (macanderson/stella) no longer builds/tests against this repo's HEAD (\`${{ github.sha }}\`). See the \`build\` step log above for the compiler error." } >> "$GITHUB_STEP_SUMMARY" + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + echo "Advisory on a pull request: a downstream repo does not block a merge here." + echo "_Advisory on a PR — this does not block the merge._" >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + # Scheduled or manually dispatched: gate nothing, block nothing, so + # failing is free — and a red run is the only form of this signal that + # actually reaches a human. + echo "Scheduled run: failing so the break is visible rather than a warning on a green run." + echo "_Scheduled run — failed deliberately so this reaches a human._" >> "$GITHUB_STEP_SUMMARY" + exit 1 oxagen-canary: name: oxagen conformance fixtures pinned to CGP HEAD (advisory, deferred) diff --git a/SPEC.md b/SPEC.md index ac0da14..a54ec5e 100644 --- a/SPEC.md +++ b/SPEC.md @@ -632,6 +632,25 @@ excluded while a healthy provider fanned out concurrently beside it still return its frames, so one leg's crash never poisons a `query_all`. Run it: `contextgraph-inspect host` (CI: `host-conformance.sh`). +That harness drives *this* repository's host. A **composition harness** +(`contextgraph-conformance`'s `composition_conformance` module) covers the step +above it, in whatever host implements it: given a `ComposingHost` — anything that +answers "with these providers and this query, what reaches the prompt, and what +did you drop getting there?" — it checks the rules binding a host's merge across +providers. `Host::query_all` audits budget honesty **per provider**, so a set of +individually conformant providers can still overflow a shared budget in +aggregate: three providers each returning one honest 400-token frame against a +1000-token query are each within budget and jointly 200 over. The checks are the +cross-provider **token bound** (§7); the **total partition** — every offered frame +is admitted or reported dropped, never silently truncated (issue #15); the +**quarantine** (§7 B2/B4) — a provider the audit rejected contributes nothing, +checked with a *frame flooder* whose frames are individually cheap, so only having +consulted the audit keeps them out; and **determinism** — an unchanged frame set +composes to the same render order, the prompt-cache guarantee of +`docs/context-reuse.md` §1. `ReferenceComposingHost` (`query_all` plus +`compose_for_prompt`) is the worked example that passes it. A host with its own +merge implements the trait and gets the same audit instead of an assurance. + What remains genuinely unchecked: - **C4, C7, C8 — the HTTP transport rules.** These bind the host's HTTP client. diff --git a/contextgraph-conformance/src/composition_conformance.rs b/contextgraph-conformance/src/composition_conformance.rs new file mode 100644 index 0000000..55c3b92 --- /dev/null +++ b/contextgraph-conformance/src/composition_conformance.rs @@ -0,0 +1,493 @@ +//! Composition conformance (`SPEC.md` §11.1) — the suite a **downstream** host +//! can run against its own composition layer. +//! +//! [`run_host_conformance`](crate::run_host_conformance) drives +//! [`contextgraph_host::Host`] itself, so it certifies the reference host and +//! nothing else. That leaves a real gap, because `Host::query_all` is not the +//! whole host: it audits budget honesty **per provider**, and then hands back a +//! fan-out. Something above it has to turn N providers' accepted frames into the +//! one frame set that reaches a prompt, and that step is where a host makes its +//! own decisions — which frames win a shared budget, what happens to the losers, +//! and in what order the survivors render. +//! +//! That step is not covered by the per-provider audit, and the gap is not +//! theoretical. Three providers each returning one honest 400-token frame against +//! a 1000-token query are *individually* conformant — no `token_cost` lie, no +//! frame flood — and `FanOut::accepted_frames()` yields all three, for 1200 +//! tokens. Whether the prompt ends up over budget, and whether anyone is told +//! which evidence was dropped to keep it under, is entirely up to the composing +//! host. A downstream host that got this wrong would pass every check in the +//! provider suite and every check in the host suite. +//! +//! So this module inverts the dependency: instead of driving a fixed host, it +//! takes a [`ComposingHost`] — anything that can answer "given these providers +//! and this query, what reaches the prompt, and what did you drop getting +//! there?" — and holds it to the rules that bind that answer. The reference +//! implementation is [`compose_for_prompt`](contextgraph_host::compose_for_prompt), +//! which passes; a downstream host with its own merge (stella's `recall_via_host` +//! is the known one) implements the trait and gets the same audit. +//! +//! # The rules checked +//! +//! - **[`CCHECK_BUDGET_BOUND`]** — the admitted set's summed token cost does not +//! exceed the query's `max_tokens`, *including* when every individual provider +//! was honest and only the sum overflows (§7). +//! - **[`CCHECK_TOTAL_PARTITION`]** — every frame the host was offered is either +//! admitted or reported as dropped. A frame that is neither has been *silently +//! truncated*, which is the one outcome an evidence audit cannot tolerate +//! (issue #15's total-partition requirement). +//! - **[`CCHECK_QUARANTINE`]** — frames from a provider the host's own audit +//! rejected never reach the prompt. A composing host that reads raw provider +//! results instead of `accepted_frames()` re-admits exactly what B2/B4 dropped. +//! - **[`CCHECK_DETERMINISM`]** — the same frame set composes to the same +//! admitted sequence twice running. This is the prompt-cache guarantee +//! (`docs/context-reuse.md` §1): a turn whose underlying frames did not change +//! must emit byte-identical text, so selection may depend on score but +//! *rendering* must not. +//! +//! Every check is **adversarial by construction**, the same discipline +//! [`host_conformance`](crate::host_conformance) uses: each one points the host at +//! input that tries to make it fail *and* at a well-behaved counterpart it must +//! accept, so a check can only pass if the host **discriminates**. A host that +//! admitted nothing at all, or reported every frame as dropped, would fail its +//! counterpart rather than passing vacuously. +//! +//! # Honest residual +//! +//! This suite sees a host's composition as a black box over frames: it cannot +//! check *rendering* (R3 fencing is [`host_conformance`]'s `host-content-quoting`, +//! against the reference renderer), and it cannot check that a host's stated drop +//! *reason* is the true one — only that a drop is reported at all. A host that +//! reported every over-budget drop as a duplicate would pass. Reason fidelity +//! needs a vocabulary this trait deliberately does not impose, because a +//! downstream host's drop reasons are its own (stella has `FrameCount`, +//! `TokenBudget`, `RequiredOverBudget`; the reference has `Duplicate` and +//! `OverBudget`). + +use std::collections::BTreeSet; + +use async_trait::async_trait; +use contextgraph_types::{ + BYTES_PER_BUDGET_TOKEN, ContextFrame, ContextQuery, FrameId, FrameKind, budget_tokens, +}; + +use contextgraph_host::{ContextProvider, Host, ProviderResult, compose_for_prompt}; + +use crate::host_conformance::{ProbeProvider, probe_query}; +use crate::report::{CheckResult, ConformanceReport}; + +/// §7 — the admitted set fits the query's token budget, including when only the +/// cross-provider sum overflows. +pub const CCHECK_BUDGET_BOUND: &str = "composition-budget-bound"; +/// Issue #15 — every offered frame is admitted or reported dropped, never +/// silently truncated. +pub const CCHECK_TOTAL_PARTITION: &str = "composition-total-partition"; +/// §7 B2/B4 — frames the host's own audit rejected never reach the prompt. +pub const CCHECK_QUARANTINE: &str = "composition-quarantine"; +/// `docs/context-reuse.md` §1 — an unchanged frame set composes identically. +pub const CCHECK_DETERMINISM: &str = "composition-determinism"; + +/// One frame a composing host declined to admit. +/// +/// Deliberately does **not** carry a reason enum. The suite's contract is that a +/// dropped frame is *accounted for*, not that it is accounted for in the +/// protocol's vocabulary — a downstream host's drop reasons are its own product +/// vocabulary (see the module's honest residual). Imposing one here would make +/// the trait unimplementable without a lossy mapping, and a lossy mapping is +/// worse evidence than an honest identity. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExcludedFrame { + /// The provider that served it. + pub provider_id: String, + /// The provider's own frame id. + pub frame_id: String, +} + +/// What a composing host did with a fan-out: what reaches the prompt, and what +/// it dropped getting there. +#[derive(Debug, Clone, Default)] +pub struct Composition { + /// The frames that reach the prompt, in the order the host renders them, + /// each paired with the provider that served it. + pub admitted: Vec<(String, ContextFrame)>, + /// Every frame the host was offered and did not admit. + pub dropped: Vec, +} + +impl Composition { + /// The summed declared token cost of the admitted frames. + fn admitted_tokens(&self) -> u64 { + self.admitted + .iter() + .map(|(_, frame)| u64::from(frame.token_cost)) + .sum() + } + + /// The `(provider, frame id)` pairs accounted for — admitted or dropped. + fn accounted(&self) -> BTreeSet<(String, String)> { + self.admitted + .iter() + .map(|(provider, frame)| (provider.clone(), frame.id.clone())) + .chain( + self.dropped + .iter() + .map(|drop| (drop.provider_id.clone(), drop.frame_id.clone())), + ) + .collect() + } + + /// The admitted frames as identity pairs, in render order — the sequence + /// [`CCHECK_DETERMINISM`] compares across runs. + fn render_order(&self) -> Vec<(String, String)> { + self.admitted + .iter() + .map(|(provider, frame)| (provider.clone(), frame.id.clone())) + .collect() + } +} + +/// A host's composition layer, as this suite needs to see it. +/// +/// One method, because one method is the whole contract: a composing host is a +/// function from *(providers, query)* to *what reached the prompt*. Taking the +/// providers rather than a pre-built fan-out is deliberate — it lets the suite +/// hand over adversarial providers and still exercise the host's **own** fan-out +/// and audit path, which is where [`CCHECK_QUARANTINE`] lives. A trait that took +/// an already-audited frame list could not tell whether the host ran the audit. +#[async_trait] +pub trait ComposingHost: Send + Sync { + /// Register exactly `providers`, execute `query`, and report the result. + /// + /// Implementations should build a fresh host per call: the suite relies on + /// calls being independent, and reuses ids across checks. + async fn compose( + &self, + providers: Vec>, + query: &ContextQuery, + ) -> Composition; +} + +/// Run every composition check against `host`, returning a typed +/// [`ConformanceReport`]. +/// +/// `target` names the implementation under test and appears in the report, so a +/// downstream host's CI output says which host was certified rather than just +/// "passed". +pub async fn run_composition_conformance( + host: &dyn ComposingHost, + target: impl Into, +) -> ConformanceReport { + let checks = vec![ + check_budget_bound(host).await, + check_total_partition(host).await, + check_quarantine(host).await, + check_determinism(host).await, + ]; + ConformanceReport { + target: target.into(), + checks, + } +} + +/// **§7** — the cross-provider budget bound. +/// +/// The adversarial input is the case the per-provider audit structurally cannot +/// catch: three providers, each returning a single honest 400-token frame against +/// `max_tokens: 1000`. Nobody lied — every provider is within budget on its own — +/// and the sum is 1200. A composing host must drop something. +/// +/// The well-behaved counterpart is the same shape under the budget (3 × 200 = +/// 600), where the host must admit **all three**. Without it a host that always +/// returned an empty composition would pass. +async fn check_budget_bound(host: &dyn ComposingHost) -> CheckResult { + let query = probe_query(); // max_tokens: 1000, max_frames: 8 + let over = host + .compose(three_providers_each_costing(400), &query) + .await; + let within_budget = over.admitted_tokens() <= u64::from(query.max_tokens); + let dropped_something = !over.dropped.is_empty(); + + let under = host + .compose(three_providers_each_costing(200), &query) + .await; + let kept_all = under.admitted.len() == 3 && under.dropped.is_empty(); + + CheckResult::from_bool( + CCHECK_BUDGET_BOUND, + within_budget && dropped_something && kept_all, + format!( + "§7 (composition): three individually-honest providers summing 1200 against a \ + 1000-token budget compose within budget={within_budget} \ + (admitted {} tokens) and report a drop={dropped_something}; the same shape summing \ + 600 keeps all three frames={kept_all}", + over.admitted_tokens() + ), + ) +} + +/// **Issue #15 total partition** — no frame vanishes unaccounted. +/// +/// Uses the same over-budget input as [`check_budget_bound`], because that is the +/// case where a host *must* shed frames and therefore the case where silent +/// truncation is tempting: the naive implementation stops walking the moment the +/// budget fills, and everything after it disappears uncounted as well as unkept. +/// +/// The counterpart is the under-budget input, where the host must report *no* +/// drops — otherwise a host could pass by declaring every frame dropped. +async fn check_total_partition(host: &dyn ComposingHost) -> CheckResult { + let query = probe_query(); + let offered: BTreeSet<(String, String)> = (0..3) + .map(|i| (format!("p{i}"), format!("p{i}-f"))) + .collect(); + + let over = host + .compose(three_providers_each_costing(400), &query) + .await; + let accounted = over.accounted(); + let missing: Vec<_> = offered.difference(&accounted).collect(); + let total = missing.is_empty(); + // A host cannot satisfy the partition by inventing frames it was never + // offered, either: the accounted set must not exceed the offered one. + let no_phantoms = accounted.difference(&offered).count() == 0; + + let under = host + .compose(three_providers_each_costing(200), &query) + .await; + let no_spurious_drops = under.dropped.is_empty(); + + CheckResult::from_bool( + CCHECK_TOTAL_PARTITION, + total && no_phantoms && no_spurious_drops, + format!( + "issue #15 (composition): every offered frame is admitted or reported \ + dropped={total} (unaccounted: {missing:?}), no frame is reported that was never \ + offered={no_phantoms}; a composition that drops nothing reports \ + nothing={no_spurious_drops}" + ), + ) +} + +/// **§7 B2/B4** — a provider the audit rejected stays out of the prompt. +/// +/// The adversarial provider is a **frame flooder**: `max_frames + 9` frames, each +/// individually cheap. The host's own audit is required to reject the whole set +/// (B4) and does. The composing layer must not put it back. +/// +/// A flooder rather than a `token_cost` liar, and the distinction matters. A +/// liar's frames are *also* over the token budget, so a host that skipped the +/// audit entirely would still drop them while packing — and would pass this check +/// by accident, for the wrong reason. (That is not hypothetical; it is what the +/// first version of this check did.) A flooder's frames are cheap: they sail +/// through any token-budget pack, so the **only** thing that keeps them out of +/// the prompt is having consulted the audit. That makes the check load-bearing +/// instead of incidental. +/// +/// The counterpart pairs the flooder with an honest provider whose frame **must** +/// still arrive: quarantining one leg may not take the other down with it, which +/// is the crash-isolation posture applied to the budget audit. +async fn check_quarantine(host: &dyn ComposingHost) -> CheckResult { + let query = probe_query(); // max_frames: 8, max_tokens: 1000 + let flood: Vec = (0..query.max_frames + 9) + .map(|i| honest_frame(&format!("flood-{i}"), 1)) + .collect(); + let providers: Vec> = vec![ + Box::new(ProbeProvider::local("flooder", flood)), + Box::new(ProbeProvider::local( + "honest", + vec![honest_frame("honest-f", 100)], + )), + ]; + let composed = host.compose(providers, &query).await; + + let flooder_excluded = !composed + .admitted + .iter() + .any(|(provider, _)| provider == "flooder"); + let honest_admitted = composed + .admitted + .iter() + .any(|(provider, frame)| provider == "honest" && frame.id == "honest-f"); + + CheckResult::from_bool( + CCHECK_QUARANTINE, + flooder_excluded && honest_admitted, + format!( + "§7 B2/B4 (composition): the frames of a provider the audit rejected (a frame \ + flooder, whose frames are individually cheap enough to pass any token pack) never \ + reach the prompt={flooder_excluded}; an honest provider queried alongside it still \ + arrives={honest_admitted}" + ), + ) +} + +/// **`docs/context-reuse.md` §1** — an unchanged frame set composes identically. +/// +/// Composes the same input twice and compares the admitted sequence. This is the +/// prompt-cache guarantee: selection may depend on score, but *rendering order* +/// must be a function of the frame set alone, so a turn whose underlying frames +/// did not change emits byte-identical text and rides the provider's cache +/// instead of busting it. +/// +/// The frames are given deliberately **tied scores** — the reference frame +/// fixture scores every frame 0.5 — because a tie is where an unstable sort or a +/// hash-ordered map leaks nondeterminism. A host that ordered by score alone +/// would pass on distinct scores and fail here, which is the point. +/// +/// The counterpart is inverted: rather than a second input the host must accept, +/// the check also asserts the composition is **non-empty**, so a host that +/// admitted nothing cannot pass by being trivially stable. +async fn check_determinism(host: &dyn ComposingHost) -> CheckResult { + let query = probe_query(); + let first = host + .compose(three_providers_each_costing(100), &query) + .await; + let second = host + .compose(three_providers_each_costing(100), &query) + .await; + + let stable = first.render_order() == second.render_order(); + let non_empty = !first.admitted.is_empty(); + + CheckResult::from_bool( + CCHECK_DETERMINISM, + stable && non_empty, + format!( + "context-reuse §1 (composition): an unchanged frame set composes to the same render \ + order twice={stable} (first {:?}, second {:?}); the composition is non-empty, so \ + stability is not vacuous={non_empty}", + first.render_order(), + second.render_order() + ), + ) +} + +/// The reference composing host: [`Host::query_all`] for the fan-out and audit, +/// then [`compose_for_prompt`] for the shared-budget pack. +/// +/// Two jobs. It is the proof the suite is **satisfiable** — a suite no +/// implementation passes is a suite with a bug, not a bar — and it is the worked +/// example a downstream host implements against, which is why the body is short +/// enough to read: fan out, keep what the audit accepted, pack it, read the +/// partition back off the audit. +/// +/// Note what it can and cannot report. Frames the audit quarantined (a +/// `token_cost` liar's whole set) are **absent** from `dropped`, because +/// [`ProviderResult::BudgetLie`](contextgraph_host::ProviderResult) carries a +/// *count* of dropped frames and not their ids — the host knows how many it threw +/// out, not which. That is why [`CCHECK_QUARANTINE`] asserts only that those +/// frames stay out of the prompt, and why [`CCHECK_TOTAL_PARTITION`] is posed +/// over honest providers: demanding that a quarantined frame be named would be +/// demanding information the fan-out does not carry. +pub struct ReferenceComposingHost; + +#[async_trait] +impl ComposingHost for ReferenceComposingHost { + async fn compose( + &self, + providers: Vec>, + query: &ContextQuery, + ) -> Composition { + let mut host = Host::new(); + for provider in providers { + host.register(provider); + } + let fanout = host.query_all(query).await; + + // Compose from the **audited** accepted set, never from raw provider + // results — this line is the whole of `CCHECK_QUARANTINE`. + let offered: Vec<(String, ContextFrame)> = fanout + .outcomes + .iter() + .filter_map(|outcome| match &outcome.result { + ProviderResult::Frames(result) => Some( + result + .frames + .iter() + .map(|frame| (outcome.provider_id.clone(), frame.clone())), + ), + _ => None, + }) + .flatten() + .collect(); + + let composed = compose_for_prompt( + offered + .iter() + .map(|(provider, frame)| (provider.as_str(), frame)), + query.max_tokens, + ); + + // The audit is a total partition of `offered`, so reading `admitted` and + // `dropped` straight off it is what makes this host's own + // `CCHECK_TOTAL_PARTITION` hold by construction rather than by care. + let included: Vec<&FrameId> = composed.audit.included().collect(); + let admitted = included + .iter() + .filter_map(|id| { + offered + .iter() + .find(|(provider, frame)| { + provider == &id.provider_id && frame.id == id.frame_id + }) + .cloned() + }) + .collect(); + let dropped = composed + .audit + .excluded() + .map(|entry| ExcludedFrame { + provider_id: entry.frame.provider_id.clone(), + frame_id: entry.frame.frame_id.clone(), + }) + .collect(); + + Composition { admitted, dropped } + } +} + +/// A frame whose declared `token_cost` is the **honest** canonical count for its +/// own content (§7 B3): the content is exactly `token_cost * +/// BYTES_PER_BUDGET_TOKEN` bytes, so `ceil(len / 4) == token_cost`. +/// +/// This is load-bearing, and getting it wrong is the first mistake this suite +/// invites — it is the bug the suite's own first run had. A composing host is +/// entitled to pack by a frame's **canonical** cost rather than its declared one; +/// the reference host does, deliberately, so an under-declared frame cannot sneak +/// past the budget. A fixture declaring `token_cost: 400` on a one-byte body is +/// therefore measured as costing 1 by the host and 400 by the suite, and the +/// check fails a *correct* host over a fixture defect. +/// +/// So every frame this suite offers satisfies B3. The rule under test is the +/// cross-provider **sum**; posing it over frames that already lie about their +/// individual cost would test something else entirely — something the provider +/// suite's `budget-honesty` check already covers. +fn honest_frame(id: &str, token_cost: u32) -> ContextFrame { + let content = "x".repeat(token_cost as usize * BYTES_PER_BUDGET_TOKEN); + debug_assert_eq!( + budget_tokens(&content), + token_cost, + "the fixture must satisfy B3 or the suite measures the wrong thing" + ); + let mut frame = ContextFrame::full(id, FrameKind::Doc, id, &content, 0.5, token_cost); + frame.citation_label = Some(id.into()); + frame +} + +/// Three local providers, each serving exactly one B3-honest frame costing +/// `token_cost`. +/// +/// Each is individually honest for any budget at or above `token_cost`, so +/// whether the set overflows is purely a property of the *sum* — which is what +/// makes this the fixture the per-provider audit cannot help with. +fn three_providers_each_costing(token_cost: u32) -> Vec> { + (0..3) + .map(|i| { + let id = format!("p{i}"); + let frame = honest_frame(&format!("{id}-f"), token_cost); + Box::new(ProbeProvider::local(&id, vec![frame])) as Box + }) + .collect() +} + +#[cfg(test)] +mod tests; diff --git a/contextgraph-conformance/src/composition_conformance/tests.rs b/contextgraph-conformance/src/composition_conformance/tests.rs new file mode 100644 index 0000000..dc43a50 --- /dev/null +++ b/contextgraph-conformance/src/composition_conformance/tests.rs @@ -0,0 +1,304 @@ +//! Tests for the composition suite. +//! +//! Two obligations, and the second matters more than the first. The reference +//! host must **pass** (a bar nothing clears is a bug in the bar), and each check +//! must **fail** a host that violates exactly the rule it names — otherwise the +//! suite is decoration. So every check gets a purpose-built saboteur: a +//! `ComposingHost` that is correct in every respect except one. +//! +//! This is the in-module equivalent of `.github/scripts/conformance-red.sh`, +//! which proves the provider suite bites by running it against a deliberately +//! broken provider. A green suite is only evidence if red is reachable. + +use super::*; +use crate::report::CheckStatus; + +/// The outcome of one check by name, so a test can assert on the check it is +/// about rather than on report-wide `passed()`. +fn status(report: &ConformanceReport, name: &str) -> CheckStatus { + report + .checks + .iter() + .find(|check| check.name == name) + .unwrap_or_else(|| panic!("check `{name}` missing from report")) + .status +} + +fn evidence(report: &ConformanceReport, name: &str) -> String { + report + .checks + .iter() + .find(|check| check.name == name) + .map(|check| check.evidence.clone()) + .unwrap_or_default() +} + +#[tokio::test] +async fn the_reference_composing_host_passes_every_check() { + let report = run_composition_conformance(&ReferenceComposingHost, "reference").await; + assert!( + report.passed(), + "the reference composition layer must satisfy the suite it defines; failures: {:?}", + report + .failures() + .map(|check| format!("{}: {}", check.name, check.evidence)) + .collect::>() + ); + assert_eq!(report.checks.len(), 4, "all four checks ran"); + assert_eq!(report.target, "reference"); +} + +/// A host that runs the fan-out and admits **everything** the providers returned, +/// with no shared-budget pack and no drop report. This is the naive composition — +/// and precisely the shape a host lands on when it assumes `query_all`'s +/// per-provider audit already bounded the total. +struct AdmitsEverything; + +#[async_trait] +impl ComposingHost for AdmitsEverything { + async fn compose( + &self, + providers: Vec>, + query: &ContextQuery, + ) -> Composition { + let mut host = Host::new(); + for provider in providers { + host.register(provider); + } + let fanout = host.query_all(query).await; + let admitted = fanout + .outcomes + .iter() + .filter_map(|outcome| match &outcome.result { + ProviderResult::Frames(result) => Some( + result + .frames + .iter() + .map(|frame| (outcome.provider_id.clone(), frame.clone())), + ), + _ => None, + }) + .flatten() + .collect(); + Composition { + admitted, + dropped: vec![], + } + } +} + +#[tokio::test] +async fn admitting_every_frame_fails_the_cross_provider_budget_bound() { + let report = run_composition_conformance(&AdmitsEverything, "admits-everything").await; + assert_eq!( + status(&report, CCHECK_BUDGET_BOUND), + CheckStatus::Fail, + "three honest 400-token providers against a 1000-token budget sum to 1200 — a host \ + that admits them all is over budget: {}", + evidence(&report, CCHECK_BUDGET_BOUND) + ); + // It passes quarantine and determinism: it *is* composing from the audited + // accepted set, and it *is* stable. That non-overlap is the point — each + // check isolates one rule instead of every check failing together. + assert_eq!(status(&report, CCHECK_QUARANTINE), CheckStatus::Pass); + assert_eq!(status(&report, CCHECK_DETERMINISM), CheckStatus::Pass); +} + +/// A host that packs to the token budget correctly but stops walking the moment +/// the budget fills, so the frames after that point are neither admitted nor +/// reported. The classic silent truncation: `break` where the code needed +/// `continue`-with-a-report. +struct SilentlyTruncates; + +#[async_trait] +impl ComposingHost for SilentlyTruncates { + async fn compose( + &self, + providers: Vec>, + query: &ContextQuery, + ) -> Composition { + let mut host = Host::new(); + for provider in providers { + host.register(provider); + } + let fanout = host.query_all(query).await; + let mut admitted = Vec::new(); + let mut spent = 0u32; + for outcome in &fanout.outcomes { + if let ProviderResult::Frames(result) = &outcome.result { + for frame in &result.frames { + if spent.saturating_add(frame.token_cost) > query.max_tokens { + // The bug: stop, and say nothing about the rest. + break; + } + spent += frame.token_cost; + admitted.push((outcome.provider_id.clone(), frame.clone())); + } + } + } + Composition { + admitted, + dropped: vec![], + } + } +} + +#[tokio::test] +async fn silent_truncation_fails_the_total_partition() { + let report = run_composition_conformance(&SilentlyTruncates, "silently-truncates").await; + assert_eq!( + status(&report, CCHECK_TOTAL_PARTITION), + CheckStatus::Fail, + "a frame that is neither admitted nor reported has vanished unaccounted: {}", + evidence(&report, CCHECK_TOTAL_PARTITION) + ); + // Staying within budget is not the failing rule here — this host does that. + // The check that fires is the one about accounting, which is the distinction + // the two checks exist to draw. + assert_eq!(status(&report, CCHECK_BUDGET_BOUND), CheckStatus::Fail); + assert!( + evidence(&report, CCHECK_BUDGET_BOUND).contains("report a drop=false"), + "budget-bound fails on the missing drop report, not on the bound itself: {}", + evidence(&report, CCHECK_BUDGET_BOUND) + ); +} + +/// A host that composes from **raw provider results**, skipping the audit — so a +/// frame flooder's frames are put back after `query_all` rejected them. Every +/// frame it admits is well-formed and individually cheap, which is what makes +/// this failure invisible both to a frame-validity check and to its own +/// token-budget pack: nothing except the audit was ever going to keep them out. +struct IgnoresTheAudit; + +#[async_trait] +impl ComposingHost for IgnoresTheAudit { + async fn compose( + &self, + providers: Vec>, + query: &ContextQuery, + ) -> Composition { + // Query the providers directly, bypassing the host's audit entirely. + let mut admitted = Vec::new(); + let mut dropped = Vec::new(); + let mut spent = 0u32; + for provider in &providers { + let Ok(result) = provider.query(query).await else { + continue; + }; + for frame in result.frames { + if spent.saturating_add(frame.token_cost) > query.max_tokens { + dropped.push(ExcludedFrame { + provider_id: provider.id().to_string(), + frame_id: frame.id.clone(), + }); + continue; + } + spent += frame.token_cost; + admitted.push((provider.id().to_string(), frame)); + } + } + Composition { admitted, dropped } + } +} + +#[tokio::test] +async fn composing_from_raw_provider_results_fails_quarantine() { + let report = run_composition_conformance(&IgnoresTheAudit, "ignores-the-audit").await; + assert_eq!( + status(&report, CCHECK_QUARANTINE), + CheckStatus::Fail, + "a host that skips the audit re-admits exactly what B4 rejected: {}", + evidence(&report, CCHECK_QUARANTINE) + ); + // It respects the budget and accounts for its drops — it is wrong in exactly + // one way, and that is the way the check names. + assert_eq!(status(&report, CCHECK_BUDGET_BOUND), CheckStatus::Pass); + assert_eq!(status(&report, CCHECK_TOTAL_PARTITION), CheckStatus::Pass); +} + +/// A host whose render order depends on something outside the frame set. Rather +/// than fake a clock or a hash seed, it alternates on an internal counter — the +/// observable behavior of any host whose ordering leaks nondeterminism, and the +/// reason the determinism check uses tied scores. +struct UnstableOrder { + calls: std::sync::atomic::AtomicUsize, +} + +#[async_trait] +impl ComposingHost for UnstableOrder { + async fn compose( + &self, + providers: Vec>, + query: &ContextQuery, + ) -> Composition { + let inner = ReferenceComposingHost.compose(providers, query).await; + let n = self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let mut admitted = inner.admitted; + if n % 2 == 1 { + admitted.reverse(); + } + Composition { + admitted, + dropped: inner.dropped, + } + } +} + +#[tokio::test] +async fn an_order_that_varies_between_identical_calls_fails_determinism() { + let host = UnstableOrder { + calls: std::sync::atomic::AtomicUsize::new(0), + }; + let report = run_composition_conformance(&host, "unstable-order").await; + assert_eq!( + status(&report, CCHECK_DETERMINISM), + CheckStatus::Fail, + "an unchanged frame set that renders in a different order busts the prompt cache: {}", + evidence(&report, CCHECK_DETERMINISM) + ); +} + +/// A host that admits nothing at all. It cannot exceed a budget, cannot silently +/// truncate (it reports every frame as dropped), and is perfectly stable — so it +/// would sail through a suite written without well-behaved counterparts. Every +/// check must reject it. +struct AdmitsNothing; + +#[async_trait] +impl ComposingHost for AdmitsNothing { + async fn compose( + &self, + providers: Vec>, + query: &ContextQuery, + ) -> Composition { + let mut dropped = Vec::new(); + for provider in &providers { + if let Ok(result) = provider.query(query).await { + for frame in result.frames { + dropped.push(ExcludedFrame { + provider_id: provider.id().to_string(), + frame_id: frame.id, + }); + } + } + } + Composition { + admitted: vec![], + dropped, + } + } +} + +#[tokio::test] +async fn a_host_that_admits_nothing_passes_no_check_vacuously() { + let report = run_composition_conformance(&AdmitsNothing, "admits-nothing").await; + for check in &report.checks { + assert_eq!( + check.status, + CheckStatus::Fail, + "`{}` must not pass vacuously for a host that serves no context: {}", + check.name, + check.evidence + ); + } +} diff --git a/contextgraph-conformance/src/host_conformance.rs b/contextgraph-conformance/src/host_conformance.rs index 3f73033..b58b019 100644 --- a/contextgraph-conformance/src/host_conformance.rs +++ b/contextgraph-conformance/src/host_conformance.rs @@ -652,7 +652,7 @@ fn fenced_between(rendered: &str, needle: &str) -> bool { /// The query every host-side check probes with — a modest budget so an /// over-budget or flooding provider is unambiguously over the line. -fn probe_query() -> ContextQuery { +pub(crate) fn probe_query() -> ContextQuery { ContextQuery { goal: "host-conformance probe".into(), query_text: None, @@ -668,7 +668,7 @@ fn probe_query() -> ContextQuery { /// A minimal well-formed frame declaring `token_cost` — the unit the host's B1/B2 /// budget audit sums. -fn frame(id: &str, token_cost: u32) -> ContextFrame { +pub(crate) fn frame(id: &str, token_cost: u32) -> ContextFrame { let mut frame = ContextFrame::full(id, FrameKind::Doc, id, "c", 0.5, token_cost); frame.citation_label = Some(id.into()); frame @@ -766,7 +766,7 @@ fn frames_line() -> String { /// host-side equivalent of a `--misbehave` mode. It records whether its `query` /// was ever invoked, so a check can prove the host never transmitted a payload /// it was required to gate (§4 C2). -struct ProbeProvider { +pub(crate) struct ProbeProvider { id: String, info: ProviderInfo, capabilities: Capabilities, @@ -775,7 +775,7 @@ struct ProbeProvider { } impl ProbeProvider { - fn with_data_flow(id: &str, data_flow: DataFlow, frames: Vec) -> Self { + pub(crate) fn with_data_flow(id: &str, data_flow: DataFlow, frames: Vec) -> Self { Self { id: id.into(), info: ProviderInfo { @@ -795,7 +795,7 @@ impl ProbeProvider { } /// A local, egress-free provider — always queryable without consent. - fn local(id: &str, frames: Vec) -> Self { + pub(crate) fn local(id: &str, frames: Vec) -> Self { Self::with_data_flow(id, local_flow(), frames) } @@ -825,7 +825,7 @@ impl ProbeProvider { } } -fn local_flow() -> DataFlow { +pub(crate) fn local_flow() -> DataFlow { DataFlow { reads: true, writes: false, diff --git a/contextgraph-conformance/src/lib.rs b/contextgraph-conformance/src/lib.rs index 4aebef8..5ec8094 100644 --- a/contextgraph-conformance/src/lib.rs +++ b/contextgraph-conformance/src/lib.rs @@ -58,6 +58,17 @@ //! [`host_conformance`], the dual suite: [`run_host_conformance`] drives the //! reference [`Host`] against adversarial in-process providers and asserts it //! upholds them (`SPEC.md` §11.1; issue #14). +//! +//! Both of those suites certify code in *this* repository. A third, +//! [`composition_conformance`], is for code that is not: it takes a +//! [`ComposingHost`] and certifies **someone else's** composition layer — the step +//! above [`Host::query_all`] that turns a fan-out across several providers into +//! the one frame set that reaches a prompt. That step is where a downstream host +//! makes its own calls about a shared budget, and neither suite above can see it: +//! three providers each returning one honest 400-token frame against a +//! 1000-token query are individually conformant and jointly 200 over. Run +//! [`run_composition_conformance`] against your own host; +//! [`ReferenceComposingHost`] is the worked example that passes it. use contextgraph_host::{ ConsentRecord, ContextProvider, DigestVerification, DropReason, Host, HostError, @@ -69,9 +80,14 @@ use contextgraph_types::{ Grantor, ProviderInfo, }; +pub mod composition_conformance; pub mod host_conformance; mod report; +pub use composition_conformance::{ + CCHECK_BUDGET_BOUND, CCHECK_DETERMINISM, CCHECK_QUARANTINE, CCHECK_TOTAL_PARTITION, + ComposingHost, Composition, ExcludedFrame, ReferenceComposingHost, run_composition_conformance, +}; pub use host_conformance::{ HCHECK_BUDGET_DROP, HCHECK_COMPOSITION_AUDIT, HCHECK_CONSENT_GATE, HCHECK_CONTENT_QUOTING, HCHECK_CRASH_ISOLATION, HCHECK_FRAME_LIMIT, HCHECK_PROVENANCE_BYTES, HCHECK_SCOPE_RECEIPT, diff --git a/contextgraph-conformance/tests/composition_conformance_suite.rs b/contextgraph-conformance/tests/composition_conformance_suite.rs new file mode 100644 index 0000000..174dc93 --- /dev/null +++ b/contextgraph-conformance/tests/composition_conformance_suite.rs @@ -0,0 +1,53 @@ +//! Composition conformance (`SPEC.md` §11.1) — the suite a *downstream* host runs +//! against its own composition layer. +//! +//! `host_conformance_suite.rs` certifies the reference host; this certifies the +//! **contract** a non-reference host is held to. The distinction matters for what +//! this file can assert: the interesting subject is code in another repository, so +//! what is testable here is that the reference implementation satisfies the bar +//! and that the bar is reachable through the crate's public API — the same way a +//! downstream host will reach it. +//! +//! The adversarial half (a saboteur per check, each failing exactly the rule it +//! violates) lives in the module's unit tests, where the saboteurs can stay +//! private. + +use contextgraph_conformance::{ + CCHECK_BUDGET_BOUND, CCHECK_DETERMINISM, CCHECK_QUARANTINE, CCHECK_TOTAL_PARTITION, + CheckStatus, ReferenceComposingHost, run_composition_conformance, +}; + +#[tokio::test] +async fn the_reference_composition_layer_upholds_every_composition_rule() { + let report = + run_composition_conformance(&ReferenceComposingHost, "reference: compose_for_prompt").await; + assert!( + report.passed(), + "the reference composition layer must satisfy the suite it defines; failures: {:?}", + report + .failures() + .map(|check| format!("{}: {}", check.name, check.evidence)) + .collect::>() + ); + + // Every check ran and passed — none skipped, none vacuous. + assert_eq!(report.checks.len(), 4); + for name in [ + CCHECK_BUDGET_BOUND, + CCHECK_TOTAL_PARTITION, + CCHECK_QUARANTINE, + CCHECK_DETERMINISM, + ] { + let status = report + .checks + .iter() + .find(|check| check.name == name) + .unwrap_or_else(|| panic!("report is missing the `{name}` check")) + .status; + assert_eq!(status, CheckStatus::Pass, "{name}: {report:?}"); + } + + // The target string is carried through verbatim, so a downstream host's CI + // output names the host that was certified rather than just "passed". + assert_eq!(report.target, "reference: compose_for_prompt"); +} diff --git a/contextgraph-host/src/http.rs b/contextgraph-host/src/http.rs index f9a9d68..fb3c237 100644 --- a/contextgraph-host/src/http.rs +++ b/contextgraph-host/src/http.rs @@ -101,7 +101,32 @@ fn is_loopback_host(host: &str) -> bool { /// allowed (the bytes never leave the machine); every `https://` target is /// allowed. Called before the client is built or DNS is resolved, so a refusal /// short-circuits with zero network activity. -fn refuse_insecure_transport(id: &str, url: &str) -> Result<(), HostError> { +/// +/// # Why this is public +/// +/// [`Host::add_http`](crate::Host::add_http) already calls it, so C7 holds +/// whether or not a caller does. It is exported for the case a host wants to +/// classify a URL *before* attempting the connection — typically to report a +/// plaintext endpoint as the configuration error it is, rather than as a +/// connection failure or a non-conformant provider. +/// +/// The alternative is that every host re-derives "which hosts are loopback" +/// locally, and C7 ends up with one implementation per host, free to disagree +/// about `[::1]`, `127.0.0.2`, or the casing of `LOCALHOST`. A normative rule +/// with N implementations is N rules. This is the one. +/// +/// ```no_run +/// use contextgraph_host::{HostError, refuse_insecure_transport}; +/// +/// // Plaintext to a remote peer: refused, with the peer named. +/// let refusal = refuse_insecure_transport("acme", "http://cgp.example.com/q"); +/// assert!(matches!(refusal, Err(HostError::InsecureTransport { .. }))); +/// +/// // Loopback plaintext and TLS are both fine. +/// assert!(refuse_insecure_transport("local", "http://127.0.0.1:8080/q").is_ok()); +/// assert!(refuse_insecure_transport("acme", "https://cgp.example.com/q").is_ok()); +/// ``` +pub fn refuse_insecure_transport(id: &str, url: &str) -> Result<(), HostError> { let parsed = reqwest::Url::parse(url).map_err(|e| HostError::Transport { id: id.to_string(), message: format!("invalid provider url: {e}"), @@ -562,6 +587,58 @@ mod tests { } } + /// The C7 rule is now public API ([`refuse_insecure_transport`]) so a host can + /// classify a URL without re-deriving "which hosts are loopback" locally. That + /// makes these edge cases part of the exported contract rather than an + /// internal detail, so they are pinned directly instead of only through + /// `connect`: they are exactly the cases an independent reimplementation gets + /// wrong, and the reason the rule is exported at all. + #[test] + fn the_exported_c7_rule_classifies_every_loopback_spelling() { + // Allowed: TLS anywhere, and plaintext to loopback in each of its + // spellings — the literal name (any casing), all of `127.0.0.0/8` rather + // than just `127.0.0.1`, and bracketed IPv6 `::1`. + for allowed in [ + "https://example.com/cgp", + "http://localhost:8080/cgp", + "http://LOCALHOST:8080/cgp", + "http://127.0.0.1/cgp", + "http://127.0.0.2/cgp", + "http://[::1]:8080/cgp", + ] { + assert!( + refuse_insecure_transport("p", allowed).is_ok(), + "C7 must allow {allowed}" + ); + } + + // Refused: plaintext to anything off-machine. `127.0.0.1.example.com` is + // the prefix-matching trap — it *starts with* a loopback IP and is a + // remote DNS name. + for refused in [ + "http://example.com/cgp", + "http://127.0.0.1.example.com/cgp", + "http://[2001:db8::1]/cgp", + "http://10.0.0.5/cgp", + ] { + assert!( + matches!( + refuse_insecure_transport("p", refused), + Err(HostError::InsecureTransport { .. }) + ), + "C7 must refuse {refused}" + ); + } + + // An unparseable URL is a config error, not a security verdict: reporting + // it as `InsecureTransport` would tell an operator to add TLS to a string + // that is not a URL at all. + assert!(matches!( + refuse_insecure_transport("p", "not a url"), + Err(HostError::Transport { .. }) + )); + } + #[tokio::test] async fn a_plaintext_loopback_transport_is_allowed() { // The C7 loopback exception: wiremock serves plain `http://` on diff --git a/contextgraph-host/src/lib.rs b/contextgraph-host/src/lib.rs index 51ec2e5..ce57bf7 100644 --- a/contextgraph-host/src/lib.rs +++ b/contextgraph-host/src/lib.rs @@ -82,7 +82,7 @@ pub use error::HostError; pub use host::{ DropReason, DroppedFrame, FanOut, Host, ProviderOutcome, ProviderResult, VerifyOutcome, }; -pub use http::{Credential, HttpProvider}; +pub use http::{Credential, HttpProvider, refuse_insecure_transport}; pub use ingest::{ IngestBundle, IngestConfig, IngestProvider, PasteIngest, SegmentKind, SegmentOutcome, SegmentReport, ingest_paste,