From c9ba5f943558709d8be7a357452726779f95e48d Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sun, 16 Aug 2026 21:01:30 -0500 Subject: [PATCH 01/13] =?UTF-8?q?fix(benchmark):=20a=20dispatched=20run's?= =?UTF-8?q?=20acts=20happen=20IN=20ITS=20ROOM=20=E2=80=94=20agent/solve=20?= =?UTF-8?q?was=20hardcoded=20roomless=20(#329/#243)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `agent/solve` opened with `let room = Uuid::nil();` — a literal, no param behind it. Every benchmark act therefore executed with no room, and `apply_act`'s receipt radiation skips a nil room BY DESIGN (radiating nil-room acts once stole academy's single-room chat projection onto a phantom, live-proven 2026-08-12). Its own comment names the fix as pending: "Skip until solves thread their bench room (#329's per-run rooms make every solve act a room act)". #329(a) shipped the per-run room weeks ago; nothing ever threaded it. Consequence, and it is the failure BENCHMARKS-ARE-ADAPTERS-NOT-A-RUNNER.md exists to name: a citizen can burn a full act budget, write a patch and take a verdict while the room she was dispatched into shows NOTHING. Not a rendering gap — the events are never published. The acceptance test in that doc ("can a citizen standing in the room perceive the run's state through the same ViewState pipe the human's screen uses?") answered NO, and answering it needed a file read, which is the doc's own definition of disconnected. * `AgentSolveParams.room: Option` — the activity the run belongs to. * `benchmark/dispatch` passes the room it just spawned. Its acts now radiate `persona:act` into it, so the work renders as collapsed receipts in the run room's transcript (#243) and reaches citizen perception through the same projection the screen reads. * `work/claim`'s path passes `None` and SAYS SO in a comment: the claim verb carries no activity, so a claim-fired solve is still invisible. That is exactly #425's subject; the two paths now differ precisely at the gap #425 exists to close, instead of both being silently roomless. * `WorkspaceCycle::actions_taken()` — the seam a long-running drive needs to report liveness WHILE it runs. `drive_to_settle` returns its act count only at settlement and a SWE attempt legitimately runs hours, so `benchmark/runs` reads `acts` from a ledger written once per attempt. Measured 2026-08-16: two dispatched solves read `acts=0, stalled=false` for ten straight minutes while both citizens were demonstrably mid-turn (captures written 12s earlier). The projection whose stated purpose is "silence must never be ambiguous with progress" cannot currently tell them apart — its 20-min stall window assumes artifact activity tracks act cadence, and no per-act artifact write exists. Accessor lands here; the heartbeat that consumes it is the next slice. Deployed and SHA-verified before commit. NOT yet live-proven end to end: no dispatched run has reached act #1 since — the two sympy runs both settled in ~150ms with zero acts and were correctly classified INFRA VOID by #384, which is a separate round-killer this change does not touch. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../continuum-core/src/cognition/workspace.rs | 18 ++ .../src/commands/agent/attempt_outcome.rs | 298 ++++++++++++++++++ .../src/commands/agent/solve.rs | 22 +- core/continuum-core/src/commands/benchmark.rs | 14 +- core/continuum-core/src/modules/work.rs | 15 +- protocol/typescript/agent/AgentSolveParams.ts | 16 + 6 files changed, 379 insertions(+), 4 deletions(-) create mode 100644 core/continuum-core/src/commands/agent/attempt_outcome.rs diff --git a/core/continuum-core/src/cognition/workspace.rs b/core/continuum-core/src/cognition/workspace.rs index 2f2b76fa3..c3ad1b667 100644 --- a/core/continuum-core/src/cognition/workspace.rs +++ b/core/continuum-core/src/cognition/workspace.rs @@ -1601,6 +1601,24 @@ impl WorkspaceCycle { } } + /// How many acts these hands have executed, EVER — the monotonic counter, not + /// the capacity-bounded receipt ring ([`super::working_memory::WorkingMemory::actions_taken`]). + /// `None` for a pure-cognition cycle (no hands, so the question has no answer — + /// distinct from `Some(0)`, which is hands that have not acted yet). + /// + /// Exists so a LONG-RUNNING drive can report its own liveness WHILE it runs. + /// `drive_to_settle` returns its act count only at settlement, and a benchmark + /// attempt legitimately runs for hours — so every liveness surface downstream + /// (`benchmark/runs`, the bench board, the run-room panel) was reading a number + /// that could not move until the work was already over. Measured 2026-08-16: + /// two dispatched solves read `acts=0, stalled=false` for ten straight minutes + /// while their citizens were mid-turn, and the projection whose stated purpose + /// is "silence must never be ambiguous with progress" could not tell the two + /// apart. A wait-free atomic load, safe to poll on a heartbeat. + pub fn actions_taken(&self) -> Option { + self.acting.as_ref().map(|a| a.working_memory.actions_taken()) + } + /// Begin a memory-isolated measurement window over this cycle's hippocampus. /// /// `cognition/eval` drives the persona's REAL admission as it grades her, so diff --git a/core/continuum-core/src/commands/agent/attempt_outcome.rs b/core/continuum-core/src/commands/agent/attempt_outcome.rs new file mode 100644 index 000000000..17af4069f --- /dev/null +++ b/core/continuum-core/src/commands/agent/attempt_outcome.rs @@ -0,0 +1,298 @@ +//! What ONE benchmark attempt's settle MEANS — the pure classifier that stands +//! between "the substrate failed her" and "she produced a real result". +//! +//! # Why this file exists (glass-boxed 2026-08-16, run `claim-109172fa-…`) +//! +//! `agent/solve` used to fold two genuinely different endings into one arm: +//! +//! ```ignore +//! if r.infra_error.is_some() || (r.acts == 0 && r.patch.is_empty()) { … } +//! ``` +//! +//! …and then emitted ONE probe whose prose was hardcoded to the SECOND disjunct: +//! *"attempt produced ZERO work with no error — infra void (serving transition)"*. +//! +//! On that run the FIRST disjunct fired. The ledger +//! (`~/.continuum/progress/agent-solve-claim-109172fa-….json`) recorded +//! `acts: 19`, `files_changed: ["astropy/modeling/separable.py"]`, a 12,937-byte +//! patch, and an `infra_error` naming a served-model swap. The wire said she did +//! nothing. A reader (human or citizen) who trusts the probe stream — which is the +//! whole point of the probe stream — concludes a citizen who worked 19 acts and +//! wrote a real patch "produced ZERO work". That is a lying receipt, the class this +//! repo hunts (#151/#357), and it cost a debugging session. +//! +//! The label was also unearned in the other direction: "serving transition" was +//! asserted for EVERY zero-work ending, including ones where serving never moved. +//! An attempt that is not attributable to infrastructure must not be attributed to +//! infrastructure — otherwise the harness launders a capability result into an +//! infra excuse, retries it for 90s × N, and the round burns hours producing no +//! measurement at all. +//! +//! # The contract +//! +//! One pure function over EVIDENCE — the settle's own numbers plus the serving +//! snapshot observed at the attempt's start and end. No I/O, no clock, so the +//! table test below pins every arm without a live lane. + +/// Which infrastructure failed her — named, never guessed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InfraKind { + /// The served model moved underneath the attempt (a re-home / model swap), or + /// the gateway refused because the pinned model is no longer resident. THIS is + /// the ending that earns the words "serving transition". + ServingTransition, + /// The deliberation call failed for some other named reason (timeout, 5xx, + /// stream read error) with no evidence that serving itself moved. + InferenceFault, +} + +impl InfraKind { + /// The wire word — stable, greppable, one truth for probe + ledger prose. + pub fn as_str(self) -> &'static str { + match self { + InfraKind::ServingTransition => "serving_transition", + InfraKind::InferenceFault => "inference_fault", + } + } +} + +/// What the attempt loop must DO with this settle. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AttemptDisposition { + /// A real ending on a working lane — grade it. Includes an honest empty-diff + /// settle: producing nothing while the substrate worked IS a result. + Grade, + /// A NAMED infrastructure fault. Her chances must not be burned; the attempt is + /// retried in the same workspace (her partial work survives there). + InfraFault { kind: InfraKind, cause: String }, + /// Zero acts, empty patch, NO named error, and serving never moved. Nothing here + /// is attributable to infrastructure — so nothing here may CLAIM infrastructure. + /// Loud and distinct: it grades as the zero it is, and the probe says exactly + /// that, so a silent-settle regression surfaces as itself instead of hiding for + /// hours behind an infra retry loop. + SilentVoid, +} + +/// The gateway's refusal marker for "the model you are pinned to is not the one +/// being served". Matching on this substring is matching on OUR OWN contract string +/// (`ai::openai_adapter::unguaranteed_model_refusal`), not on a foreign format. +pub const SERVED_MODEL_REFUSAL_MARKER: &str = "is not the active served model"; +/// The gateway's refusal marker for "nothing is resident right now" — the other +/// half of the same serving transition (published on every teardown/re-home). +pub const NO_MODEL_RESIDENT_MARKER: &str = "no model is resident right now"; + +/// Everything the classifier is allowed to look at. Borrowed, so callers pass their +/// live values with no allocation. +#[derive(Debug, Clone, Copy)] +pub struct AttemptEvidence<'a> { + /// Acts the settle actually executed (the drive sums re-drives into this). + pub acts: u32, + /// Bytes of the workspace diff the grader would read. + pub patch_bytes: usize, + /// `Some(cause)` when the deliberation path failed rather than settling. + pub infra_error: Option<&'a str>, + /// The served model id observed when this attempt STARTED. + pub served_model_at_start: Option<&'a str>, + /// The served model id observed when this attempt ENDED. A difference is + /// MEASURED evidence that the lane moved under her — not an assumption. + pub served_model_at_end: Option<&'a str>, +} + +impl AttemptEvidence<'_> { + /// Did serving demonstrably move during the attempt? + pub fn serving_moved(&self) -> bool { + self.served_model_at_start != self.served_model_at_end + } + + /// Does the failure cause itself name a serving transition? + fn cause_names_serving(&self) -> bool { + self.infra_error.is_some_and(|c| { + c.contains(SERVED_MODEL_REFUSAL_MARKER) || c.contains(NO_MODEL_RESIDENT_MARKER) + }) + } + + /// True when the attempt produced something a grader can read. + pub fn produced_work(&self) -> bool { + self.acts > 0 || self.patch_bytes > 0 + } +} + +/// Classify one attempt's ending from its evidence. Pure. +/// +/// Precedence is deliberate: a NAMED infra fault outranks the work counters (#386 — +/// an attempt whose settle carries an inference error died to infrastructure by +/// definition, however much she had done before it), and "serving transition" is +/// only ever claimed when serving evidence supports it. +pub fn classify_attempt(ev: AttemptEvidence<'_>) -> AttemptDisposition { + if let Some(cause) = ev.infra_error { + let kind = if ev.serving_moved() || ev.cause_names_serving() { + InfraKind::ServingTransition + } else { + InfraKind::InferenceFault + }; + return AttemptDisposition::InfraFault { + kind, + cause: cause.to_string(), + }; + } + if ev.produced_work() { + return AttemptDisposition::Grade; + } + // Zero work with no named error. The ONLY thing that can make this infra is + // measured serving movement (#384's F1 signature: null-decision ticks while the + // lane was being swapped). Without it, the harness has no standing to claim a + // fault it cannot name. + if ev.serving_moved() { + return AttemptDisposition::InfraFault { + kind: InfraKind::ServingTransition, + cause: format!( + "attempt produced zero acts and an empty patch while the served model \ + moved from {} to {} — the lane was swapped under the drive (#384)", + ev.served_model_at_start.unwrap_or(""), + ev.served_model_at_end.unwrap_or(""), + ), + }; + } + AttemptDisposition::SilentVoid +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ev<'a>( + acts: u32, + patch_bytes: usize, + infra_error: Option<&'a str>, + start: Option<&'a str>, + end: Option<&'a str>, + ) -> AttemptEvidence<'a> { + AttemptEvidence { + acts, + patch_bytes, + infra_error, + served_model_at_start: start, + served_model_at_end: end, + } + } + + const DEVSTRAL: &str = "unsloth/Devstral-Small-2507-GGUF"; + const QWEN: &str = "ggml-org/Qwen3.8-27B-GGUF"; + // The verbatim cause from run claim-109172fa-b5eb-4259-9d40-39bd0a4dab00. + const LIVE_SWAP_CAUSE: &str = "llama-server (local OpenAI-compatible gateway): model \ + 'unsloth/Devstral-Small-2507-GGUF' is not the active served \ + model (serving: ggml-org/Qwen3.8-27B-GGUF, ready: true); the \ + serving daemon owns which single model is resident"; + + // what this catches: the lying receipt of run claim-109172fa — 19 acts and a + // 12,937-byte patch classified by a branch whose probe prose says "produced ZERO + // work with no error". The disposition must be an infra fault that CARRIES her + // work forward as evidence, and the caller must never be able to print + // "zero work" for it (the evidence says otherwise). + #[test] + fn a_swapped_lane_after_real_work_is_a_serving_transition_not_a_zero() { + let e = ev(19, 12_937, Some(LIVE_SWAP_CAUSE), Some(DEVSTRAL), Some(QWEN)); + assert!(e.produced_work(), "19 acts + a 12kB patch IS work"); + assert_eq!( + classify_attempt(e), + AttemptDisposition::InfraFault { + kind: InfraKind::ServingTransition, + cause: LIVE_SWAP_CAUSE.to_string(), + } + ); + } + + // what this catches: "serving transition" asserted with no serving evidence. A + // stream read error on a lane that never moved is an inference fault — still + // infra (unburned retry), but it must not name a transition that did not happen. + #[test] + fn an_inference_fault_on_a_steady_lane_is_not_called_a_serving_transition() { + let e = ev( + 3, + 0, + Some("llama-server: stream read error: error decoding response body"), + Some(DEVSTRAL), + Some(DEVSTRAL), + ); + assert_eq!( + classify_attempt(e), + AttemptDisposition::InfraFault { + kind: InfraKind::InferenceFault, + cause: "llama-server: stream read error: error decoding response body".to_string(), + } + ); + } + + // what this catches: the #384 protection surviving the rewrite BY EVIDENCE. Zero + // acts, empty patch, no error, but the served model moved mid-attempt — that IS + // the F1 signature and must still retry unburned rather than grade as capability. + #[test] + fn zero_work_during_a_measured_model_swap_stays_infra() { + let e = ev(0, 0, None, Some(DEVSTRAL), Some(QWEN)); + match classify_attempt(e) { + AttemptDisposition::InfraFault { kind, cause } => { + assert_eq!(kind, InfraKind::ServingTransition); + assert!(cause.contains(DEVSTRAL) && cause.contains(QWEN), "{cause}"); + } + other => panic!("a measured swap must stay infra, got {other:?}"), + } + } + + // what this catches: the retry-loop hole. Zero work, no error, serving steady — + // the harness has NO evidence of a fault, so it must not claim one (and must not + // burn 90s × N retries producing no measurement). This grades as the zero it is. + #[test] + fn zero_work_on_a_steady_lane_is_a_silent_void_never_an_infra_claim() { + assert_eq!( + classify_attempt(ev(0, 0, None, Some(DEVSTRAL), Some(DEVSTRAL))), + AttemptDisposition::SilentVoid + ); + // …and with serving never observed at all (no snapshot either side), the + // absence of evidence is still not evidence of a transition. + assert_eq!( + classify_attempt(ev(0, 0, None, None, None)), + AttemptDisposition::SilentVoid + ); + } + + // what this catches: an honest settle being stolen from the grader. Acts with an + // empty diff, or a patch with no error, are RESULTS — the empty-diff re-drive and + // the verifier already had their say by the time we get here. + #[test] + fn work_on_a_working_lane_grades() { + assert_eq!( + classify_attempt(ev(12, 0, None, Some(DEVSTRAL), Some(DEVSTRAL))), + AttemptDisposition::Grade + ); + assert_eq!( + classify_attempt(ev(0, 400, None, Some(DEVSTRAL), Some(DEVSTRAL))), + AttemptDisposition::Grade + ); + } + + // what this catches: the cause-marker path, for the case where the snapshot + // reads identical at both ends because the daemon swapped BACK before the + // attempt returned. The refusal text itself is our own contract string and is + // sufficient serving evidence on its own. + #[test] + fn a_served_model_refusal_names_a_transition_even_when_the_snapshot_settled_back() { + let e = ev(5, 0, Some(LIVE_SWAP_CAUSE), Some(DEVSTRAL), Some(DEVSTRAL)); + assert!(matches!( + classify_attempt(e), + AttemptDisposition::InfraFault { + kind: InfraKind::ServingTransition, + .. + } + )); + let transition = "llama-server: no model is resident right now (the serving daemon is \ + between lanes)"; + let e2 = ev(0, 0, Some(transition), Some(DEVSTRAL), Some(DEVSTRAL)); + assert!(matches!( + classify_attempt(e2), + AttemptDisposition::InfraFault { + kind: InfraKind::ServingTransition, + .. + } + )); + } +} diff --git a/core/continuum-core/src/commands/agent/solve.rs b/core/continuum-core/src/commands/agent/solve.rs index 823521ad7..4441e45ae 100644 --- a/core/continuum-core/src/commands/agent/solve.rs +++ b/core/continuum-core/src/commands/agent/solve.rs @@ -58,6 +58,22 @@ pub struct AgentSolveParams { #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional, type = "number")] pub max_acts: Option, + /// The ROOM this run happens in — `benchmark/dispatch`'s per-run activity room + /// (#329). Every act she executes radiates a `persona:act` receipt into it, so + /// the run's work lands in the room's transcript as collapsed receipts (#243) + /// and anyone standing there — human screen or citizen mind — perceives it + /// through the ONE ViewState pipe. + /// + /// Omitted → `Uuid::nil()`, which is the ROOMLESS shape: `apply_act` skips + /// receipt radiation entirely for a nil room (radiating them stole the + /// single-room chat projection onto a phantom, live-proven 2026-08-12), so a + /// roomless solve does its work invisibly. That was every dispatched benchmark + /// run until this param existed — the exact disconnection + /// BENCHMARKS-ARE-ADAPTERS-NOT-A-RUNNER.md names as the failure mode, and the + /// reason the flywheel saw no turns from a full graded attempt. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional, type = "string")] + pub room: Option, /// Fire-and-poll (#86): when true, the solve is spawned DETACHED — `run` returns a job /// handle NOW (arms empty, `detached: true`) and the REAL result (patch + acts) lands in /// `~/.continuum/progress/agent-solve-.json`. A real agentic drive (N full-generation @@ -1152,7 +1168,11 @@ impl AgentSolve { // ergonomic/adapter fix ([[use-adapters-dont-dumb-it-down]]), not a capability demand — // and honest (it states the real I/O contract; it does not hand her the answer). Then // DRIVE her to settlement (read → edit → run → fix, her real act→observe loop). - let room = Uuid::nil(); + // The run's ROOM (see `AgentSolveParams::room`). `Uuid::nil()` is the + // honest roomless fallback for a bare `agent/solve` with no activity + // behind it; a DISPATCHED run always carries one, and that is what + // turns her acts into room receipts instead of invisible work. + let room = p.room.unwrap_or_else(Uuid::nil); // The workspace-grounding sentence counters the observed "new project ritual" // (glass-boxed 2026-07-22 via turn capture: her first act on a seeded task was // code/create-workspace("my_stack_project") + a Rust hello-world + git/commit — diff --git a/core/continuum-core/src/commands/benchmark.rs b/core/continuum-core/src/commands/benchmark.rs index d8cc40430..7501247cb 100644 --- a/core/continuum-core/src/commands/benchmark.rs +++ b/core/continuum-core/src/commands/benchmark.rs @@ -1442,8 +1442,18 @@ impl ActionCommand for BenchmarkDispatch { // STAGED SWE card has a solve to fire here (a gym card self-grades differently). if staged_ok && solves_fired < solve_cap { if let CardWork::Swe { .. } = &pc.work { - crate::modules::work::dispatch_staged_swe_solve(ctx, &airc, *who_peer, card_id) - .await; + // The run room goes WITH the solve: her acts radiate receipts + // into the room this dispatch just spawned, so the round's work + // is visible where the round lives (#243/#329) instead of only + // in a ledger file that lands when it is already over. + crate::modules::work::dispatch_staged_swe_solve( + ctx, + &airc, + *who_peer, + card_id, + Some(room.room_id.as_uuid()), + ) + .await; solves_fired += 1; } } diff --git a/core/continuum-core/src/modules/work.rs b/core/continuum-core/src/modules/work.rs index cf892bd13..bbd82b771 100644 --- a/core/continuum-core/src/modules/work.rs +++ b/core/continuum-core/src/modules/work.rs @@ -549,7 +549,14 @@ impl ActionCommand for WorkClaim { // nobody in the loop. Best-effort: a dispatch failure never voids the // claim — the claim is hers either way, and the probe says what happened. if let Some(caller) = ctx.caller.as_ref() { - dispatch_staged_swe_solve(ctx, &airc, caller.peer_id.as_uuid(), card_id).await; + // ROOMLESS, and named as such rather than papered over: the claim verb + // carries no activity — a citizen can claim from anywhere, and the card + // does not remember which room staged it. So a claim-fired solve still + // works invisibly. That is #425's whole subject (a bench claim must lead + // to IN-ROOM work, not a detached nil-room solve); the dispatch path + // below already has a room and passes it, so the two paths now differ + // exactly at the gap #425 exists to close. + dispatch_staged_swe_solve(ctx, &airc, caller.peer_id.as_uuid(), card_id, None).await; } Ok(WorkClaimResult { card_id: p.card_id, @@ -577,11 +584,16 @@ const SWE_CLAIM_ATTEMPTS: u32 = 3; /// 2026-08-11: cards staged + assigned, zero claims, zero solves). The solve is her WHOLE /// cognition with an exclusive warm slot (`quiesce_others`), so nothing about "she does the /// work herself" changes — only the trigger moves off the chat turn. +/// `room` is the activity the run BELONGS to — `benchmark/dispatch` passes the +/// per-run room it just spawned (#329), and every act the solver executes then +/// radiates a receipt into it (#243). `None` is the roomless shape: her work +/// happens, and no room ever sees it. pub(crate) async fn dispatch_staged_swe_solve( ctx: &Ctx, airc: &std::sync::Arc, claimer: uuid::Uuid, card_id: airc_work::WorkCardId, + room: Option, ) { let Ok(board) = airc .work_board_complete(airc_lib::WORK_BOARD_PROJECTION_PAGE_SIZE) @@ -667,6 +679,7 @@ pub(crate) async fn dispatch_staged_swe_solve( capture_dir: None, learn: crate::cognition::learning_policy::LearningPolicy::LearnFromThisWork, max_acts: None, + room, path_prepend: Some(vec![venv_bin]), suppress_recall: None, prev_failed_patch_sha: None, diff --git a/protocol/typescript/agent/AgentSolveParams.ts b/protocol/typescript/agent/AgentSolveParams.ts index 324189334..e938f4260 100644 --- a/protocol/typescript/agent/AgentSolveParams.ts +++ b/protocol/typescript/agent/AgentSolveParams.ts @@ -25,6 +25,22 @@ workspace: string, * Max act→observe cycles (default 12). */ max_acts?: number, +/** + * The ROOM this run happens in — `benchmark/dispatch`'s per-run activity room + * (#329). Every act she executes radiates a `persona:act` receipt into it, so + * the run's work lands in the room's transcript as collapsed receipts (#243) + * and anyone standing there — human screen or citizen mind — perceives it + * through the ONE ViewState pipe. + * + * Omitted → `Uuid::nil()`, which is the ROOMLESS shape: `apply_act` skips + * receipt radiation entirely for a nil room (radiating them stole the + * single-room chat projection onto a phantom, live-proven 2026-08-12), so a + * roomless solve does its work invisibly. That was every dispatched benchmark + * run until this param existed — the exact disconnection + * BENCHMARKS-ARE-ADAPTERS-NOT-A-RUNNER.md names as the failure mode, and the + * reason the flywheel saw no turns from a full graded attempt. + */ +room?: string, /** * Fire-and-poll (#86): when true, the solve is spawned DETACHED — `run` returns a job * handle NOW (arms empty, `detached: true`) and the REAL result (patch + acts) lands in From ce82f00ffa2955f7e02ab75421701b091d829e25 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sun, 16 Aug 2026 21:16:51 -0500 Subject: [PATCH 02/13] =?UTF-8?q?fix(cognition):=20an=20EMPTY=20completion?= =?UTF-8?q?=20is=20not=20a=20chosen=20silence=20=E2=80=94=20the=20Err=20ar?= =?UTF-8?q?m's=20missing=20twin=20(#181/#390)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `Err` arm at llm_deliberation_faculty.rs:1844 refuses to let a FAILED model collapse into a serene `Pass`, and its own comment cites [[fallbacks-are-illegal-fail-loud]]. But a lane can also answer `Ok` with NOTHING, and that walked straight past the guard and settled as an ordinary non-Act. Same failure the Err arm exists to prevent, one branch over. MEASURED, not inferred. Direct probe against the live lane (70-token prompt, so no context pressure of any kind): finish_reason = 'length' completion_tokens = 16 content = '' Sixteen tokens generated, empty content. Qwen3.8 under `--jinja` opens ``; an unclosed block leaves `extract_reasoning` branch (3) — which is CORRECT — with empty text and the whole tail as reasoning. Separately, Solenne's capture on the turn her benchmark run died: finish_reason = 'stop' usage = {in:0, out:0} responseTimeMs = 28 text = '' SCOPE, measured across every capture on disk: 47 of 862 responses (5.5%) are empty-text, spread over ~19 citizens. NOT benchmark-specific. And the all-empty column is exactly the citizens whose "I've noticed my recent messages have been repetitive… I'll remain silent" turns have been the standing round-killer — fe4dac17 6/6, Asha (90e758b2) 4/4, e5f4141d 3/3, a20b3ada 2/2. On the turns I can see, nothing came back, and the substrate wrote it down as her choice. (Per-persona samples are small; the ratios are not stable rates. The shape is what matters.) COST: agent/solve reads it as "she chose not to act" → acts=0, empty patch → the run voids as an INFRA VOID after three attempts (the two sympy runs, 2026-08-16). A live citizen's turn reads as silence, indistinguishable from withdrawal. FIX: fault, exactly like the Err arm — the settle step surfaces it LOUD instead of fabricating a no-op, and `delib.empty_completion` puts it on the probe stream. Scoped so a native tool turn, which legitimately carries empty content, is untouched: fires only on empty text AND not ToolUse AND no tool_calls, i.e. only when the turn yields nothing to act or speak with. The receipt names WHICH shape it was — thought-but-committed-nothing (reasoning chars) vs the lane returned void. WHAT THIS DOES NOT DO: it does not stop the empty completions. It stops them being laundered as cognition. The generation-side fix (reasoning budget / think-block closure) is #181 and is next. Deployed + SHA-verified before commit. LIVE PROOF NOT YET IN HAND: zero faults across 2,589 probe rows in the first window after deploy — consistent with no citizen having hit an empty completion yet post-reboot, NOT with the guard being confirmed. A watcher is running; the owed evidence is one `delib.empty_completion` row from a real turn. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/cognition/llm_deliberation_faculty.rs | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/core/continuum-core/src/cognition/llm_deliberation_faculty.rs b/core/continuum-core/src/cognition/llm_deliberation_faculty.rs index e33fc7030..dc100896c 100644 --- a/core/continuum-core/src/cognition/llm_deliberation_faculty.rs +++ b/core/continuum-core/src/cognition/llm_deliberation_faculty.rs @@ -1901,6 +1901,73 @@ impl Faculty for LlmDeliberationFaculty { } } + // AN EMPTY COMPLETION IS NOT A CHOSEN SILENCE (the `Err` arm's missing twin). + // + // The `Err` arm above refuses to let a FAILED model collapse into a serene + // `Pass` — [[fallbacks-are-illegal-fail-loud]]. But a lane can also answer + // `Ok` with NOTHING, and that walked straight past the guard and settled as + // an ordinary non-Act. Two live shapes, both measured 2026-08-16: + // + // * the server generated tokens that never reached `content` — Qwen3.8 under + // `--jinja` opens ``, and an unclosed block leaves `extract_reasoning` + // branch (3) with empty text and the whole tail as reasoning (#181). Direct + // probe against the live lane: 70-token prompt, `finish_reason: length`, + // `completion_tokens: 16`, `content: ""`. + // * the lane returned 0 tokens in AND out in 28ms (`finish_reason: stop`) — + // Solenne's capture on the turn her benchmark run died. + // + // Cost of laundering it: `agent/solve` reads "she chose not to act" → acts=0, + // empty patch → the whole run voids as an INFRA VOID after three attempts; a + // LIVE citizen reads it as a silent turn, which is indistinguishable from + // withdrawal. Measured across every capture on disk: 47 of 862 responses + // (5.5%) are empty-text, spread over ~19 citizens — and the all-empty column + // is exactly the citizens whose "I've been repetitive, I'll remain silent" + // turns are the standing round-killer (#390/#414). They were not withdrawing. + // Nothing came back, and the substrate wrote it down as a choice. + // + // SCOPE: a native tool turn legitimately carries empty content, so ToolUse and + // any present tool_calls are excluded — this fires only when the turn yields + // no text, no tool call, and therefore nothing to act or speak with. The + // reasoning tail rides on the fault so the receipt says WHICH shape it was: + // thought-but-committed-nothing, or the lane returned void. + if resp.text.trim().is_empty() + && !matches!(resp.finish_reason, FinishReason::ToolUse) + && resp.tool_calls.as_ref().is_none_or(|c| c.is_empty()) + { + let reasoning_tokens = resp.reasoning.as_ref().map_or(0, |r| r.len()); + let why = if reasoning_tokens > 0 { + format!( + "the model produced {reasoning_tokens} chars of REASONING and committed \ + no answer (finish_reason {:?}) — an unclosed think-block or a budget \ + exhausted mid-thought, never a decision to stay silent", + resp.finish_reason + ) + } else { + format!( + "the lane returned an EMPTY completion with no reasoning and no tool call \ + (finish_reason {:?}) — nothing was generated, never a decision to stay \ + silent", + resp.finish_reason + ) + }; + tracing::warn!( + persona = %self.persona_name, + finish_reason = ?resp.finish_reason, + reasoning_chars = reasoning_tokens, + gen_await_ms, + "empty completion surfaced as a FAULT (not a silent Pass)" + ); + crate::probe!( + class = "delib.empty_completion", + persona = %self.persona_name, + reasoning_chars = reasoning_tokens, + gen_await_ms, + "the lane answered with nothing — surfacing a fault so it can never be \ + read as chosen silence" + ); + return Some(Contribution::deliberation_fault(why)); + } + // Did she choose to act? Two shapes, both → `Decision::Act`: // (a) the adapter returned a native tool-use turn (FinishReason::ToolUse); // (b) the model emitted a tool call as JSON in its prose (small models that From a4a60f90b6c4f18c4fdc399f65bccc6471455320 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sun, 16 Aug 2026 21:36:53 -0500 Subject: [PATCH 03/13] docs(architecture): the round lifecycle as a recipe-owned state machine (#371 design) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Joel 2026-08-16, after a full session of a driver getting lost: "It's way too hard to rig up. Clearly." This writes the design down so the next agent inherits a paved road instead of tire tracks. THE DEFECT, stated exactly: a benchmark round is not a thing in the system, it is a RITUAL an agent performs. The agent picks when to dispatch, picks the watch window, hand-queries probes to learn what is happening, and when the session ends the process ends with it. Every failure this session is downstream of that ONE fact, and they all have the same shape — the question has no owner, so the driver guesses from an absence: is serving ready? nothing owns readiness → dispatched and hoped has it started? nothing announces started → read acts=0, called it stalled is this run alive? liveness = a file mtime → flagged a healthy run quiet is the round done? nothing announces done → never knew did my fix work? no green/vacuous distinction → reported a vacuous green Five instances of [[an-absence-is-an-unfinished-measurement]] in one session, by a driver that had that lesson in its own guardrails. Not a knowledge gap — a missing owner. THE SHAPE: staging → ready → working → grading → done, every transition an event from the component that KNOWS (env builder, supervisor, card store, round entity) — never a timeout, never a poll, never an agent's judgement. Every stage a ViewState on the one pipe humans and citizens already read. Recipe is data, so the process is identical every round on every machine. THREE LAWS: transitions announced by the knower; liveness is a pulse never a terminal artifact (today's projection cannot distinguish silence from progress, which is its stated purpose); an absence is never a state. SUBSUMES #329(b) no-END, #442 readiness gate, #374 pulse, #425 in-room work, and the unbuilt RULES half of [[recipe-is-content-type-plus-rules]]. Carries the acceptance test from BENCHMARKS-ARE-ADAPTERS-NOT-A-RUNNER.md plus one earned tonight: can a fresh driver answer ready/started/stuck/done using ONLY queries — zero log reads, zero probe archaeology, zero inference from an absence? Build order is smallest-true-cause first; step 1 (pulse the run) consumes the WorkspaceCycle::actions_taken seam c9ba5f943 already landed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- ...LIFECYCLE-AS-RECIPE-OWNED-STATE-MACHINE.md | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 docs/architecture/ROUND-LIFECYCLE-AS-RECIPE-OWNED-STATE-MACHINE.md diff --git a/docs/architecture/ROUND-LIFECYCLE-AS-RECIPE-OWNED-STATE-MACHINE.md b/docs/architecture/ROUND-LIFECYCLE-AS-RECIPE-OWNED-STATE-MACHINE.md new file mode 100644 index 000000000..2738de7a0 --- /dev/null +++ b/docs/architecture/ROUND-LIFECYCLE-AS-RECIPE-OWNED-STATE-MACHINE.md @@ -0,0 +1,149 @@ +# The Round Lifecycle as a Recipe-Owned State Machine (#371) + +**Status:** design, not built. Fuses #329(b) (the round has no END), #442 (dispatch +must consume the state pipe), #371, and the unbuilt RULES half of +[[recipe-is-content-type-plus-rules]]. + +**Joel, 2026-08-16:** *"random and directed by agent, not an ecosystem"* — and, +after a full session of a driver getting lost: *"It's way too hard to rig up. +Clearly."* + +--- + +## 1. The defect, stated exactly + +**A benchmark round is not a thing in the system. It is a ritual an agent performs.** + +The agent chooses when to dispatch. The agent chooses the watch window. The agent +hand-queries probes to learn what is happening. When the agent's session ends, the +"process" ends with it, and the next agent re-derives it from scratch. + +Everything that went wrong on 2026-08-16 is downstream of that one fact. Not a +knowledge gap — a **missing owner**: + +| the driver asked | why it could not be answered | what the driver did instead | +|---|---|---| +| is serving ready for work? | nothing owns readiness | dispatched anyway, hoped | +| has the round started? | nothing announces started | read `acts=0`, called it stalled | +| is this run alive or wedged? | liveness = a file mtime that moves once per ATTEMPT | flagged a healthy run `quiet` | +| is the round done? | nothing announces done | never knew; the round just stopped mattering | +| did my fix work? | nothing distinguishes "no fault" from "nothing ran" | reported a vacuous green | + +Each row is the same shape: **the question has no owner, so the agent guesses from +an absence, and an absence is not evidence.** That is +[[an-absence-is-an-unfinished-measurement]] — five separate times in one session, +by a driver who had that lesson in its own guardrails. + +**The runbook** (`benchmark-round-runbook-the-no-brainer-sequence`) is the current +mitigation and it works — it caught two false alarms within minutes of being read. +But a runbook is a human-followed procedure. It shrinks the blast radius; it does +not move the process into the system. This document is how it stops being a +procedure at all. + +## 2. The shape + +**A round is an activity. Its recipe owns its lifecycle.** The recipe declares the +stages; each transition is an **event emitted by the component that knows** — never +a timeout, never a poll, never an agent's judgement. + +``` + ┌─────────┐ + dispatch ───────▶│ STAGING │ envs building, cards posting + └────┬────┘ + envs staged ok ────┤ (the ENV BUILDER knows — not a timer) + ┌────▼────┐ + │ READY │ gate open: hosted loops + serving ready (#442) + └────┬────┘ + first claim ─────────┤ (the WORK BOARD knows) + ┌────▼────┐ + │ WORKING │ acts landing, patches forming + └────┬────┘ + card → done ─────────┤ (the CARD STORE knows — #450, already event-driven) + ┌────▼────┐ + │ GRADING │ + └────┬────┘ + all cards settled ───┤ (the ROUND ENTITY knows) + ┌────▼────┐ + │ DONE │ + scorecard. THE END #329(b) says doesn't exist. + └─────────┘ +``` + +**Every stage is a ViewState on the same pipe humans and citizens already read.** +"What is it doing, and when will it be ready" becomes a query anyone can make — +the operator, a citizen standing in the room, the dispatcher itself. Never +archaeology. + +**Because the recipe is data, the process is identical every round, on every +machine, with no agent in the loop except as another observer.** + +## 3. The three laws this encodes + +1. **Every transition is announced by the component that knows it.** Not inferred, + not timed, not polled. The env builder announces staged. The supervisor announces + hosted. The card store announces done. A stage nobody can announce is a stage + that does not exist yet — say so, don't fake it with a timeout. + +2. **Liveness is a pulse, never a terminal artifact.** Today `benchmark/runs` + derives `acts` and `stalled` from a ledger written once per attempt, while an + attempt legitimately runs hours against a 20-minute stall window. The projection + whose stated purpose is *"silence must never be ambiguous with progress"* is + structurally unable to tell them apart. A run that is working must SAY so on the + cadence it works at. + +3. **An absence is never a state.** No row means "nothing has reported," which is + distinct from "nothing is happening" and from "it is finished." The projection + must carry the difference, because every driver that has to infer it will infer + it wrong. (Both halves of tonight's vacuous green: zero faults because nothing + ran, and zero acts because nothing had written yet.) + +## 4. What this subsumes + +- **#329(b)** — the round has no END. `DONE` + scorecard is the END. +- **#442** — dispatch refuses to stage into a not-ready room. That is the + `STAGING → READY` gate, expressed as a state instead of a check. +- **#374** — run PULSE as a first-class wire signal. That is law 2. +- **#425** — a bench claim leads to in-room work. The room is the round's activity; + a roomless solve is a run with no lifecycle to belong to. +- The **RULES half of a recipe** ([[recipe-is-content-type-plus-rules]]) — we built + content-type and never rules. This lifecycle *is* the rules. + +## 5. Acceptance test + +From [BENCHMARKS-ARE-ADAPTERS-NOT-A-RUNNER.md](BENCHMARKS-ARE-ADAPTERS-NOT-A-RUNNER.md), +unchanged and now testable: + +> *Can a citizen standing in the room perceive the run's state through the same +> ViewState pipe the human's screen uses?* + +Plus one more, earned tonight: + +> *Can a fresh driver — with no memory of this session — answer "is it ready, has +> it started, is it stuck, is it done" using only queries, with zero log reads, +> zero probe archaeology, and zero inference from an absence?* + +If either needs a file read or a judgement call, it is disconnected and it failed. + +## 6. Build order + +Smallest true causes first; each independently useful. + +1. **Pulse the run while it runs** (law 2). `WorkspaceCycle::actions_taken()` is the + seam and already exists (c9ba5f943) — a heartbeat consumes it so `acts` and + last-activity are live. Kills the false `quiet` immediately. +2. **Round entity owns stages.** `bench_round::register_round` already exists; + give it the stage field and the transition subscribers. +3. **Stage transitions from real emitters.** Env builder → staged. Supervisor → + hosted/ready. Card store → done (#450 already fires). Round → settled. +4. **`RoundViewState` on the pipe**, folded from the round entity — not a file scan. + Retires the 5s progress-directory poll in `positron_bench_source`. +5. **Dispatch consumes it** (#442): refuse to stage while not READY, and say why. + +## 7. The smell to catch yourself on + +If you are about to add a timeout, a retry window, a sleep, or an agent-side +heuristic to decide what stage a round is in — **stop.** That is the ritual growing +back. The question is always *which component already knows this, and why isn't it +saying so?* + +And if you are about to report a state derived from something you did not observe +happening — stop. Say "I did not observe it," and name the query that would. From dd876435f8148abd8ff813be2a734317d53578cf Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sun, 16 Aug 2026 21:47:53 -0500 Subject: [PATCH 04/13] =?UTF-8?q?feat(benchmark):=20the=20run=20PULSES=20w?= =?UTF-8?q?hile=20it=20runs=20=E2=80=94=20liveness=20stops=20being=20a=20t?= =?UTF-8?q?erminal=20artifact=20(#371=20law=202,=20#374)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First build step of ROUND-LIFECYCLE-AS-RECIPE-OWNED-STATE-MACHINE.md. THE DEFECT: the run ledger was written ONCE per attempt, at settlement. For the entire attempt — legitimately HOURS on a full SWE budget — `benchmark/runs` read `acts: 0` and a `last_activity` frozen at run start. RUN_STALL_WINDOW_SECS is 20 minutes. So a HEALTHY first attempt is guaranteed to read `quiet`/stalled, every single time, and the projection whose own doc comment says its purpose is "silence must never be ambiguous with progress" was STRUCTURALLY unable to tell them apart. Its stall window assumes artifact activity tracks act cadence (~4-6 min); no per-act artifact write has ever existed. MEASURED 2026-08-16: two dispatched solves read `acts=0, stalled=false` for ten straight minutes while the driver watching them could not distinguish working from wedged — which is precisely how a vacuous "no faults observed" gets reported as a green. THE FIX: `select!` over the drive future and a 60s interval. No spawn, so the cycle stays BORROWED — no 'static bound, no Arc juggling, no parallel allocator, and the pulse cannot outlive the work it reports on. Each tick reads `WorkspaceCycle::actions_taken()` (the seam c9ba5f943 landed) — a wait-free atomic load on the persona's own monotonic counter — and rewrites the running marker, which moves BOTH `acts` and the mtime `last_activity_ms` folds from. Cadence is well under the stall window (a live run can never age into `quiet`) and far above act cadence (one small JSON write per tick, nothing else). Every field derives exactly as the detached wrapper's `state: running` marker derives them, so a pulse can never contradict the marker it refreshes. `None` run_id → no ledger → no pulse, correct: nothing polls a run that returns inline. WHY THE COUNTER AND NOT A SEPARATE TALLY: it is the same monotonic counter perception renders to her, so the board and her own proprioception cannot disagree — and it is the counter whose capacity-bounded-ring sibling caused the retracted "they never act" premises on #390/#211. Deployed + SHA-verified. LIVE PROOF OWED: a dispatched run showing acts CLIMB mid-attempt. Until that row exists this is compiled-and-deployed, not confirmed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/commands/agent/solve.rs | 77 ++++++++++++++++++- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/core/continuum-core/src/commands/agent/solve.rs b/core/continuum-core/src/commands/agent/solve.rs index 4441e45ae..6f77e8673 100644 --- a/core/continuum-core/src/commands/agent/solve.rs +++ b/core/continuum-core/src/commands/agent/solve.rs @@ -1236,10 +1236,79 @@ impl AgentSolve { f } }; - let mut settled = crate::cognition::act_observe::drive_to_settle( - &cycle, burst, room, max_acts, framing, - ) - .await; + // THE RUN PULSES WHILE IT RUNS (#371 law 2: liveness is a pulse, never a + // terminal artifact). + // + // The ledger used to be written ONCE per attempt, at settlement. So for the + // entire attempt — legitimately HOURS on a full SWE budget — `benchmark/runs` + // read `acts: 0` and a `last_activity` frozen at run start. Against a 20-minute + // stall window that means a HEALTHY first attempt is guaranteed to read `quiet`, + // every time, and the projection whose stated purpose is "silence must never be + // ambiguous with progress" was structurally unable to tell them apart. Measured + // 2026-08-16: two dispatched solves read `acts=0, stalled=false` for ten straight + // minutes, and the driver watching them could not distinguish working from wedged + // — which is exactly how a vacuous "no faults" gets reported as a green. + // + // `select!` over the drive future and an interval: no spawn, so the cycle stays + // BORROWED (no 'static bound, no Arc juggling, no parallel allocator). Each tick + // reads the persona's own monotonic act counter — a wait-free atomic load — and + // rewrites the running marker, which moves BOTH `acts` and the file mtime that + // `last_activity_ms` folds from. The counter is the same one perception renders, + // so the board and her own proprioception can never disagree. + // + // Cadence: well under RUN_STALL_WINDOW_SECS so a live run can never age into + // `quiet`, and far above act cadence (~2-6 min) so it costs a tiny JSON write + // per tick and nothing else. + const RUN_PULSE: std::time::Duration = std::time::Duration::from_secs(60); + // Same ledger the detached wrapper journals `state: running` into, and the + // SAME derivation of every field, so a pulse can never contradict the marker + // it refreshes. `None` run_id (an attached call) → no ledger → no pulse, which + // is correct: nothing is polling a run that returns inline. + let pulse_run_id = p.run_id.clone().unwrap_or_default(); + let pulse_path = p + .run_id + .as_deref() + .and_then(agent_solve_ledger_path); + let pulse_persona = p.persona_id.clone(); + let pulse_instance: Option = workspace + .contains("/workspace/swe/") + .then(|| { + std::path::Path::new(&workspace) + .file_name() + .map(|s| s.to_string_lossy().to_string()) + }) + .flatten(); + let mut settled = { + let drive = crate::cognition::act_observe::drive_to_settle( + &cycle, burst, room, max_acts, framing, + ); + tokio::pin!(drive); + let mut ticker = tokio::time::interval(RUN_PULSE); + ticker.tick().await; // interval fires immediately; consume that tick + loop { + tokio::select! { + outcome = &mut drive => break outcome, + _ = ticker.tick() => { + // Best-effort by construction: a failed pulse must never + // disturb the work it is only reporting on. + if let (Some(p), Some(acts)) = + (pulse_path.as_ref(), cycle.actions_taken()) + { + let _ = std::fs::write(p, serde_json::json!({ + "state": "running", + "run_id": pulse_run_id, + "persona_id": pulse_persona, + "workspace": workspace, + "instance": pulse_instance, + // Acts SHE has executed, live — not a count that + // materializes only once the work is already over. + "acts": acts, + }).to_string()); + } + } + } + } + }; // 4) Collect the HANDS artifact: everything she changed in the workspace as a unified diff // (new files included), plus the touched paths. This is what SWE/Terminal-Bench apply. From 5e6c2fdae29e3719c4b0a6c508f7f07b3b75506b Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sun, 16 Aug 2026 22:04:06 -0500 Subject: [PATCH 05/13] =?UTF-8?q?fix(swe):=20astropy=20env=20builds=20?= =?UTF-8?q?=E2=80=94=20a=201990s=20vendored=20zlib=20stubs=20out=20fdopen?= =?UTF-8?q?=20on=20every=20modern=20Apple=20SDK=20(#383)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit THE FOURTH HEAD of #383, and the reason two dispatched rounds died at env-build TONIGHT even though #2329's jinja2 + build-requires fixes are both merged and in this tree. Not pyerfa, not Cython, not Python 3.11 — I guessed all three before reading the error. astropy vendors cfitsio, which vendors an ancient zlib. `cextern/cfitsio/zlib/zutil.h:140`: #if defined(MACOS) || defined(TARGET_OS_MAC) # define OS_CODE 7 # ifndef fdopen # define fdopen(fd,mode) NULL /* No fdopen() */ `TARGET_OS_MAC` means "some Apple platform" on every modern SDK — it does NOT mean "classic Mac OS", which is what this zlib was written to test. So the branch fires, `fdopen` is macro-replaced by `NULL`, and the SYSTEM header's own declaration FILE *fdopen(int, const char *) → FILE *NULL(int, const char *) becomes `error: expected identifier or '('` inside ``, thousands of lines from anything astropy wrote. (The adjacent `'OS_CODE' macro redefined` warning is the same branch firing.) FIX: the guard is `#ifndef fdopen`, so pre-defining it is the entire repair. `-Dfdopen=fdopen` makes the guard FALSE — the NULL stub is never emitted — and the macro is the identity, so every real call still compiles to `fdopen`. No source patched, nothing stubbed, nothing renamed, and a repo that doesn't vendor this zlib never notices the flag. POSITIVE + NEGATIVE CONTROL against the real staged checkout, both run before committing: cc -c -I/cextern/cfitsio/zlib → error: expected identifier or '(' error: expected ')' cc -c -Dfdopen=fdopen -I → clean, exit 0 Placed in ERA_CFLAGS beside its three siblings rather than a per-repo table, because it is not an astropy fact — it is an ERA fact (old vendored C vs a modern SDK), same shape as the others: the compiler is HARNESS, the C is SUBJECT, and the subject built fine on the compilers of its day. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../continuum-core/src/cognition/swe_bench.rs | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/core/continuum-core/src/cognition/swe_bench.rs b/core/continuum-core/src/cognition/swe_bench.rs index 5aed39e1d..e844dc465 100644 --- a/core/continuum-core/src/cognition/swe_bench.rs +++ b/core/continuum-core/src/cognition/swe_bench.rs @@ -572,8 +572,39 @@ fn build_requires(repo_dir: &Path) -> Vec { /// on this machine), the C is SUBJECT: demote exactly those three diagnostics back to the /// warnings they were. distutils APPENDS `CFLAGS` to its sysconfig baseline, so nothing else /// about the build changes, and modern code that doesn't trip them is untouched. +/// +/// The FOURTH head (#383, measured live 2026-08-17 on astropy__astropy-14182, and the +/// reason two dispatched rounds died at env-build after the jinja2 + build-requires fixes +/// both landed): astropy vendors cfitsio, which vendors a 1990s zlib, whose +/// `cextern/cfitsio/zlib/zutil.h:140` reads +/// +/// ```c +/// #if defined(MACOS) || defined(TARGET_OS_MAC) +/// # define OS_CODE 7 +/// # ifndef fdopen +/// # define fdopen(fd,mode) NULL /* No fdopen() */ +/// ``` +/// +/// `TARGET_OS_MAC` is 1 on EVERY modern Apple SDK — it means "some Apple platform", not +/// "classic Mac OS" as it did when this zlib was written. So the branch fires, `fdopen` is +/// macro-replaced by `NULL`, and the system header's own declaration +/// `FILE *fdopen(int, const char *)` becomes `FILE *NULL(int, const char *)` → +/// `error: expected identifier or '('` in ``, thousands of lines from anything +/// astropy wrote. (The adjacent `'OS_CODE' macro redefined` warning is the same branch.) +/// +/// The guard is `#ifndef fdopen`, so pre-defining it is the whole fix: `-Dfdopen=fdopen` +/// makes the guard FALSE — the NULL stub is never emitted — and the macro itself is the +/// identity, so every real `fdopen` call compiles to `fdopen`. Nothing is stubbed, nothing +/// is renamed, no source is patched, and a repo that does not vendor this zlib never +/// notices the flag. +/// +/// It lives here rather than in a per-repo table because it is not an astropy fact — it is +/// an ERA fact (old vendored zlib vs a modern Apple SDK), identical in shape to the three +/// above: the compiler is HARNESS, the C is SUBJECT, and the subject built fine on the +/// compilers of its own day. const ERA_CFLAGS: &str = "-Wno-error=incompatible-function-pointer-types \ - -Wno-error=implicit-function-declaration -Wno-error=int-conversion"; + -Wno-error=implicit-function-declaration -Wno-error=int-conversion \ + -Dfdopen=fdopen"; /// Build deps that a repo's DEPENDENCY sdists import at build time but that nothing installs /// under `--no-build-isolation` (we honor the top repo's `[build-system].requires`; a From 03c890b2907985e34b8ab440ef57041f4fe34f5b Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 17 Aug 2026 05:17:11 -0500 Subject: [PATCH 06/13] =?UTF-8?q?fix(governor):=20shed=20CAPABILITY,=20nev?= =?UTF-8?q?er=20starve=20the=20window=20=E2=80=94=20model=20choice=20was?= =?UTF-8?q?=20floored=20at=20bare=20survival=20(#438-class)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Glass-boxed live on the M5 2026-08-17: a 27B served at usable_gb=5 with served_window=2048 against a MEASURED demand_window=63817. Every SWE act ran against a context that could not hold the task statement — the "machine is allowing sludge" Joel reported, and the arithmetic behind ~4 acts/hour. The defect was an inconsistency INTERNAL to plan_serving. Two floors, two different standards: lane COUNT -> BOOTSTRAP_WORKING_SET (16384, "one full turn") model CHOICE -> MIN_SERVE_CTX (2048, bare survival) The lane loop already refuses to add a slot that cannot hold a full turn (1 lane @ 30k beats 2 lanes @ 2k), then falls back to .unwrap_or(1) and calls it "honest starvation, surfaced downstream" — while the MODEL is never reconsidered. Shedding a lane and shedding capability are the same move for the same reason; only the first was implemented. So `fits_one_lane` crowned the most capable model that cleared a TRIVIAL bar and let the window collapse to it. A model that fits only at 2048 is not more capable on this host — it is unusable on this host. Viability is now tested at the same "one full turn" standard the lane floor uses, with a documented degrade: if NO candidate clears it, fall back to the old MIN_SERVE_CTX bar so a genuinely tiny host still serves something rather than nothing (honest starvation is then real, not a selection artifact). Deliberately the STABLE bootstrap constant, not the moving measured p95 — coupling model CHOICE to a jittering demand signal is the 718-replan flap that wedged three benchmark runs. This ran on EVERY node, so a grid of peers would each independently crown its biggest technically-holdable model and starve its own window, with no single node's numbers looking anomalous. LIVE, deploy-verified: 27B/2048/bound_by=host-fit -> 14B/16384/bound_by=demand, lanes 1 -> 4. bound_by flipping to `demand` is the signal: the plan now meets the working set instead of being cut by the host. Tests: model_choice_sheds_capability_rather_than_starve_the_window pins all three arms — the starve case, a NEGATIVE CONTROL that a roomy host still picks the 27B (this fix must not silently downgrade capable boxes), and the degrade fallback. 27 serving_plan / 139 serving / 178 capacity green. Also repaired stable_keeps_incumbent_when_upgrade_lacks_headroom, whose fixture had encoded the old bar: with pair() at 10GB, big was "viable" only on 184MB of KV, so "fresh would pick big" meant a 2048-token 9GB model. Given a local fixture where big genuinely clears a full turn (19.5 <= 20) yet still fails the 0.9x switch-up headroom bar, so the test asserts what it always meant to. KNOWN RESIDUAL: selection tests weights + kv_at(ctx) against raw usable_bytes while window_for sizes against `effective` (minus co-consumer headroom) and subtracts compute buffers, so the two floors are near-consistent, not identical (observed 16025 vs 16384). The 8x collapse is gone; unifying the last 2% is a follow-up. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/cognition/serving_plan.rs | 144 +++++++++++++++--- 1 file changed, 126 insertions(+), 18 deletions(-) diff --git a/core/continuum-core/src/cognition/serving_plan.rs b/core/continuum-core/src/cognition/serving_plan.rs index 6ab8b8e9e..116b4813c 100644 --- a/core/continuum-core/src/cognition/serving_plan.rs +++ b/core/continuum-core/src/cognition/serving_plan.rs @@ -406,25 +406,56 @@ pub fn plan_serving( return None; } - // GPU-viable = weights + at least one lane's KV fit the honest budget. - // "At least one lane" is the floor: a model we can't run even single-laned - // on the GPU is not a serving option on this host. - let fits_one_lane = |m: &ModelFootprint| { - m.weights_bytes.saturating_add(m.kv_at(MIN_SERVE_CTX)) <= host.usable_bytes + // GPU-viable = weights + at least one lane's KV fit the honest budget AT a + // given per-slot window. WHICH window is the whole question (#438-class): + // + // This filter used to be hardcoded to `MIN_SERVE_CTX` (2048) — bare survival. + // That made "viable" mean "can technically hold a 2k window", so selection + // always crowned the most capable model that cleared a trivial bar and then + // let the window collapse to it. Glass-boxed live 2026-08-17 on this box: + // a 27B chosen at `usable_gb=5`, `served_window=2048` against a MEASURED + // `demand_window=63817` — a mind handed 3% of the context its own turn needs. + // Every act it took was against a window too small to hold the task statement. + // + // The inconsistency was INTERNAL to this function: the lane-count loop below + // already refuses to add a slot that can't clear `BOOTSTRAP_WORKING_SET` + // ("one full turn"), because 1 lane @ 30k beats 2 lanes @ 2k. But when even + // ONE lane couldn't clear that floor it fell back to `.unwrap_or(1)` and + // called the result "honest starvation, surfaced downstream" — while the + // MODEL was never reconsidered. Shedding a lane and shedding capability are + // the same move for the same reason; only the first was implemented. The + // correct response to "one lane can't hold a turn" is a SMALLER MODEL that + // can, not the bigger model blinded. A model that fits only at 2048 is not + // more capable on this host — it is unusable on this host. + // + // So viability is now tested at the SAME "one full turn" standard the lane + // floor uses, with a documented degrade: if NO candidate clears it, fall back + // to the old `MIN_SERVE_CTX` bar so a genuinely tiny host still serves + // something rather than nothing (honest starvation is then real, not a + // selection artifact). Deliberately the STABLE bootstrap constant and NOT the + // moving measured p95 — coupling model CHOICE to a jittering demand signal is + // the 718-replan flap that wedged three benchmark runs (see the lane floor's + // own note). The served window still refines with measurement below. + let fits_one_lane_at = |m: &ModelFootprint, ctx: u32| { + m.weights_bytes.saturating_add(m.kv_at(ctx)) <= host.usable_bytes }; // Prefer the MOST CAPABLE model that fits a lane. Ties broken toward the // larger model (more headroom spent = the more capable variant), then by // id descending for deterministic selection. - let best = candidates - .iter() - .filter(|m| fits_one_lane(m)) - .max_by(|a, b| { - a.capability_rank - .cmp(&b.capability_rank) - .then(a.weights_bytes.cmp(&b.weights_bytes)) - .then(b.model_id.cmp(&a.model_id)) - }); + let most_capable_fitting = |ctx: u32| { + candidates + .iter() + .filter(|m| fits_one_lane_at(m, ctx)) + .max_by(|a, b| { + a.capability_rank + .cmp(&b.capability_rank) + .then(a.weights_bytes.cmp(&b.weights_bytes)) + .then(b.model_id.cmp(&a.model_id)) + }) + }; + let best = most_capable_fitting(BOOTSTRAP_WORKING_SET) + .or_else(|| most_capable_fitting(MIN_SERVE_CTX)); let Some(model) = best else { // Nothing fits a lane on the GPU budget. Degrade honestly: name the @@ -827,6 +858,69 @@ mod tests { ); } + // what this catches: the governor serving a mind a window too small to think in + // because MODEL CHOICE was floored at bare survival while LANE COUNT was floored at + // a full turn. Glass-boxed live 2026-08-17: a 27B picked at usable_gb=5 → + // served_window=2048 against a measured demand_window=63817, and every SWE act ran + // against a context that could not hold the task statement. The pre-fix filter asked + // "can this model hold MIN_SERVE_CTX (2048)?", so the biggest model always won and + // then starved the window to that trivial bar. Shedding a lane and shedding + // capability are the same move for the same reason (1 lane @ 30k beats 2 lanes @ 2k; + // a 14B @ 30k beats a 27B @ 2k) — only the first had been implemented. + #[test] + fn model_choice_sheds_capability_rather_than_starve_the_window() { + // Big model clears 2048 but NOT a full turn; small model clears a full turn. + let big = fp("big-27b", 28, 256 * 1024, 131_072, 5); + let small = fp("small-14b", 10, 64 * 1024, 131_072, 3); + let candidates = [big.clone(), small.clone()]; + let demand = ServingDemand::new(1, None); + + // 30GB: big fits ONLY at the survival floor (28 + 0.5 ≤ 30, but 28 + 4.3 > 30). + // Pre-fix this crowned `big` and served 2048. It must now pick `small`. + let plan = plan_serving( + HostBudget { usable_bytes: 30 * GB, perf_cores: 10 }, + &candidates, + demand, + ) + .expect("a model fits"); + assert_eq!( + plan.base_model.model_id, "small-14b", + "a model that fits only at the 2048 survival floor is not a serving option \ + on this host — shed capability, never starve the window" + ); + assert!( + plan.served_context_window >= BOOTSTRAP_WORKING_SET, + "the served window must clear one full turn ({BOOTSTRAP_WORKING_SET}), got {}", + plan.served_context_window + ); + + // NEGATIVE CONTROL — the floor must not cost capability when the host can afford + // it. On a roomy host the 27B clears a full turn and must still win, otherwise + // this fix would have silently downgraded every capable box. + let roomy = plan_serving( + HostBudget { usable_bytes: 64 * GB, perf_cores: 10 }, + &candidates, + demand, + ) + .expect("a model fits"); + assert_eq!( + roomy.base_model.model_id, "big-27b", + "when the budget clears a full turn, the MOST capable model still wins" + ); + + // DEGRADE PATH — when NO candidate clears a full turn, fall back to the old + // survival bar so a genuinely tiny host serves something rather than nothing. + // Honest starvation is then real, not an artifact of the selection rule. + let tiny = plan_serving( + HostBudget { usable_bytes: 11 * GB, perf_cores: 4 }, + &candidates, + demand, + ) + .expect("the survival fallback still yields a model"); + assert_eq!(tiny.base_model.model_id, "small-14b"); + assert!(tiny.fits_on_gpu, "the fallback still serves on GPU, just narrowly"); + } + // what this catches: the "alive" OOM (2026-07-16). The served window's FULL live // footprint — weights + lanes·KV(C) + lanes·(compute_floor + compute_rate·C), every // lane prefilling at once — must fit within the EFFECTIVE budget (usable − co-consumer @@ -1479,13 +1573,27 @@ mod tests { // rather than flap the served model on a transient budget bump. #[test] fn stable_keeps_incumbent_when_upgrade_lacks_headroom() { - // 10GB: big (9.7GB) fits a lane but exceeds the 0.9*10=9GB headroom bar. + // LOCAL fixture, not `pair()`. This test needs `big` to be a LEGITIMATE upgrade + // target (clears the full-turn window floor, so selection would really pick it) + // that nonetheless fails the switch-UP headroom bar. With the shared `pair()` at + // 10GB those two are unsatisfiable together: big (9GB + 90k/tok) needs 10.47GB to + // clear one full turn, so at any budget where it lacks 0.9x headroom it also + // isn't full-turn viable — and a model that can only be served blind is not an + // upgrade worth flapping for. Before the model-choice floor landed, the old + // MIN_SERVE_CTX bar admitted big here on 184MB of KV and this fixture read as + // "fresh would pick big" when what fresh actually had was a 2048-token 9GB model. + // Sized instead so big genuinely clears a full turn (18 + 1.47 = 19.5 <= 20) yet + // still exceeds the headroom bar (18.18 > 0.9 * 20 = 18). let host = HostBudget { - usable_bytes: 10 * GB, + usable_bytes: 20 * GB, perf_cores: 6, }; + let models = vec![ + fp("small", 1, 4_000, 32_768, 1), + fp("big", 18, 90_000, 262_144, 3), + ]; assert_eq!( - plan_serving(host, &pair(), ServingDemand::new(MAX_LANES, None)) + plan_serving(host, &models, ServingDemand::new(MAX_LANES, None)) .unwrap() .base_model .model_id, @@ -1494,7 +1602,7 @@ mod tests { ); let stable = plan_serving_stable( host, - &pair(), + &models, Some("small"), ServingDemand::new(MAX_LANES, None), ) From a5ad69ca4da586bbb8aa3c083c2f66539beacef9 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 17 Aug 2026 05:23:09 -0500 Subject: [PATCH 07/13] =?UTF-8?q?docs(architecture):=20context=20is=20a=20?= =?UTF-8?q?capability=20axis=20=E2=80=94=20the=20governor=20learns=20the?= =?UTF-8?q?=20window=20but=20not=20the=20CHOICE=20(Joel=208/17)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Joel: "Context windows are as important as model size. It seems 16-20k is bare minimum for decent activities. Ideally the governor learns." Captures the design rather than rushing it into plan_serving, which is the most flap-prone function in the tree (718 replans, three wedged benchmark runs). THE CLAIM: delivered capability is f(params, served_window) and the planner models only params. capability_rank is a scalar keyed to model identity, so a 27B is "more capable" than a 14B even when the host can serve the 27B only 2048 tokens. That shipped as the live defect fixed in 03c890b29. WHAT IS ALREADY RIGHT, and must not be rebuilt: working_set.rs is a good learning loop whose hard problem is solved. It records DEMAND not USAGE — the counterfactual "what this turn would have used with no budget", including grounding contributions assembly dropped — explicitly avoiding the thermometer-inside-the-thermostat trap where a p95 of what-was-sent re-derives its own clamp forever. Peak not average. Persisted. Passed as a parameter. The live demand_window=63817 IS that module working correctly. THE ACTUAL GAP: one learned signal, three decisions, two deaf. The window is learned; lane count and model choice both use hardcoded BOOTSTRAP_WORKING_SET — a constant whose own doc says it "was never meant to survive", and which sits at the BOTTOM of Joel's 16-20k band, identically for a chat turn and a SWE turn. WHY THE CONSTANT IS THERE AND WHY IT DOESN'T FORBID LEARNING: the anti-flap argument is against driving a structural, expensive-to-change decision from a fast jittering signal. It is not against learning. Two time constants: fast measured demand sizes the window (cheap, built, correct); a slow hysteretic learned floor drives model choice and lane count (expensive, so margin band + dwell time + hard clamp). Flapping is a property of the update rule, not of learning. PROPOSED, NOT BUILT: (1) capability_rank goes 2-D — rank by delivered capability, since the current most-capable-that-clears-a-full-turn gate is a correct approximation but cannot express "14B at 60k beats 27B at 20k"; (2) the floor becomes learned + slow, with BOOTSTRAP_WORKING_SET surviving as the floor OF the floor, not the value; (3) the floor becomes per-ACTIVITY — recipes are already data (#433) and "recipe = content-type + RULES" makes the minimum useful window recipe-owned. Includes acceptance tests (incl. an anti-flap test pinned as a test, not asserted in prose) and the de-hardcode smell: a new constant beside BOOTSTRAP_WORKING_SET is a third way to express a floor — the shape the guard already caught once as FLOOR_TOKENS (#411). BLOCKING OBSERVATION recorded for whoever picks this up: delib.generate emits ZERO probe rows, so per-generation latency and tok/s are unreadable and every throughput question here needs 20-minute black-box sampling. An unmeasurable governor cannot be a learning one (#441). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../CONTEXT-IS-A-CAPABILITY-AXIS.md | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 docs/architecture/CONTEXT-IS-A-CAPABILITY-AXIS.md diff --git a/docs/architecture/CONTEXT-IS-A-CAPABILITY-AXIS.md b/docs/architecture/CONTEXT-IS-A-CAPABILITY-AXIS.md new file mode 100644 index 000000000..5f53c5c6b --- /dev/null +++ b/docs/architecture/CONTEXT-IS-A-CAPABILITY-AXIS.md @@ -0,0 +1,116 @@ +# Context is a capability axis, and the governor must learn its floor + +**Joel, 2026-08-17:** *"Context windows are as important as model size. It seems 16-20k +is bare minimum for decent activities. Ideally the governor learns."* + +## The claim + +Delivered capability is a function of **two** variables — parameters and served window — +and the serving planner models only the first. `ModelFootprint::capability_rank` is a +scalar keyed to model identity alone. Under that type, a 27B is always "more capable" +than a 14B, even when the host can only serve the 27B a 2,048-token window and could +serve the 14B 32k. + +That is not an abstraction quibble. It shipped as a live defect on the M5 (fixed in +`03c890b29`): the planner crowned a 27B at `usable_gb=5`, served it **2,048 tokens** +against a **measured 63,817-token demand**, and every SWE act ran against a context too +small to hold the task statement. The window collapse was invisible to the ranking that +caused it. + +## What is already right (do not rebuild it) + +`cognition/working_set.rs` is a well-built learning loop and its hard problem is already +solved. Read it before proposing anything here. + +- It records **DEMAND, not USAGE** — the counterfactual "what would this turn have used + with no budget at all": framing + the full conversation before trimming + *every* + grounding contribution offered, including the ones assembly dropped + generation + reserve. This deliberately avoids the thermometer-inside-the-thermostat trap: a p95 of + what-was-*sent* re-derives the clamp that produced it and freezes it forever. +- It is **peak, not average** — a working set is the high-water mark at which an activity + stops being strangled; averaging a coding turn with idle chatter serves neither. +- It **persists** per persona and rehydrates across reboot. +- It is **passed as a parameter**, never read from a global inside a decision. + +The `demand_window: 63817` observed live is this module working correctly. A demand that +exceeds the window is the signal that the window is too small — and the only signal that +can ever grow it. + +## The actual gap: one learned signal, three decisions, two of them deaf + +| Decision | Signal today | Should be | +|---|---|---| +| Served window | **learned** (measured p95 demand) | learned (fast) | +| Lane count | `BOOTSTRAP_WORKING_SET` (16,384) | learned (slow) | +| Model choice | `BOOTSTRAP_WORKING_SET` (16,384) | learned (slow) | + +Both structural decisions use a hardcoded constant whose own doc says it "was never meant +to survive." And note where that constant sits against Joel's read: 16,384 is the *bottom* +of the 16–20k "bare minimum" band. So the current floor guarantees only bare adequacy, and +guarantees it identically for a chat turn and a SWE turn. + +## Why the constant is there, and why that reason does not forbid learning + +The static value is not laziness. Coupling lane count to a moving demand signal produced +the **718-replan flap**: `usable_gb` swinging 26→6, lanes oscillating 1↔2, every flip +resizing the live admission semaphore and prefill throttle under in-flight requests — the +`no response headers for 300s` wedge that killed three benchmark runs. + +That is an argument against driving a **structural, expensive-to-change** decision from a +**fast, jittering** signal. It is not an argument against learning. The resolution is two +time constants: + +- **Fast signal → window.** Per-turn measured demand sizes the served window. Cheap to + change; already built; already correct. +- **Slow signal → structure.** A hysteretic, long-horizon learned floor drives model + choice and lane count. Expensive to change, so it must move rarely and with a dwell + time and a margin band, never on a single sample. + +A slow floor cannot flap, because flapping is a property of the update rule, not of +learning. + +## Proposed design (NOT built — this doc is the design, not a report) + +1. **`capability_rank` becomes 2-D.** Rank candidates by delivered capability + `f(model, window_it_would_get_on_this_host)`, not by a static scalar with a floor gate + bolted on. The current fix (most-capable-that-clears-a-full-turn, else degrade) is a + correct *approximation* of this and is a fine intermediate state — but it is a gate, + not a model of the tradeoff, and it cannot express "a 14B at 60k beats a 27B at 20k." +2. **The floor becomes learned and slow.** Derive it from the existing + `WorkingSetRegistry` rather than from `BOOTSTRAP_WORKING_SET`, with: a hysteresis band + (only move the decision when the learned value crosses by a margin), a minimum dwell + time, and a hard lower clamp so it can never learn its way below one real turn. The + bootstrap constant survives as the *floor of the floor*, not as the value. +3. **The floor becomes per-activity, not global.** A chat turn and a SWE turn have + different working sets; one global floor serves neither well. Recipes are already data + (#433 parameterized recipes, the activities catalog), and "recipe = content-type + + RULES" makes the minimum useful window a recipe-owned property. Learn the demand + distribution *per activity class*, not just per persona. + +## Acceptance tests + +- A host that can serve model A at 60k or model B at 20k, where B outranks A statically, + picks by delivered capability — and the test states which and why. +- The learned floor, driven by a synthetic demand trace that oscillates, moves at most + once across the trace (anti-flap, pinned as a test, not asserted in prose). +- The learned floor never drops below the hard clamp regardless of input. +- A recipe declaring a large working set gets a larger floor than a chat recipe **on the + same host**. + +## The smell to catch yourself on + +If you are adding another constant next to `BOOTSTRAP_WORKING_SET`, stop — that is a third +way to express a floor, and the de-hardcode guard exists to catch exactly that shape (it +already caught `FLOOR_TOKENS`, #411). There should be one floor, learned, with a clamp. + +Related: #438 (governor downshift on a bogus sample), #234 (demand-derived lane `-c`), +#213/#214 (window floors and dead grow-back), #124 (de-hardcode the dynamic system), +#441 (throughput sentinel — currently emitting nothing, see below). + +## Blocking observation for anyone measuring this + +`delib.generate` emits **zero** probe rows on this box. That is the class that would carry +per-generation latency and tok/s. Its absence is why every throughput question in this area +has to be answered by black-box sampling over 20-minute windows instead of read off the +stream. Fix that before trying to tune anything by measurement — an unmeasurable governor +cannot be a learning one. From fc8964bff3707f913ac1892603cc3614b57d4eb3 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 17 Aug 2026 07:48:39 -0500 Subject: [PATCH 08/13] =?UTF-8?q?fix(cognition):=20my=20empty-completion?= =?UTF-8?q?=20guard=20was=20standing=20in=20front=20of=20the=20#181=20reco?= =?UTF-8?q?very=20=E2=80=94=20reasoning-bearing=20empties=20are=20not=20fa?= =?UTF-8?q?ults?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit REGRESSION I INTRODUCED in ce82f00ff, caught by its own test suite. That guard faulted on "empty text + no tool call + not ToolUse" — which is the EXACT precondition of the two recovery paths immediately below it: persona.act.reasoning_lift — a tool call sitting in the reasoning tail gets lifted out and EXECUTED (json_in_prompt_tools::parse_tool_calls over the reasoning channel). persona.act.think_only — #181's teacher sentinel: she spent the whole generation inside , so route the #159 reported-never-executed mechanism and let drive_to_settle hand her another generation that starts from her own conclusions (the reasoning is already in working memory). Faulting first made BOTH unreachable. Measured live 2026-08-17: 40 delib.empty_completion faults against 87 persona.turn.start while three SWE runs sat at the SAME act count for 1,357 seconds. Every one of those turns had 15k+ chars of reasoning and finish_reason: length at exactly 4096 output tokens — i.e. the shape the recovery exists for. The machinery was already built and correct; I put a dead end in front of it. FIX: fault ONLY when nothing is left to recover — the lane returned genuine void (no text, no reasoning, no call), or she has no tools for the sentinel to teach through. A reasoning-bearing empty falls through to the lift/think-only owners. The guard's purpose is preserved: an empty is still never read as chosen silence, it is just routed to the component that can act on it. Restores 2 tests that had been RED since ce82f00ff and that I shipped without running — think_only_turn_routes_the_teacher_sentinel_not_empty_speak and empty_content_with_reasoning_tool_intent_lifts_the_final_call. Both name this exact contract; both now pass. 62/62 deliberation tests green. NOT YET LIVE-PROVEN. Deployed and unit-green, but the owed evidence is a real reasoning_lift or think_only probe row with empties falling — measurement in flight. Recorded here so the claim is not read as stronger than it is. ALSO CORRECTED, and NOT shipped: completion_budget_for is window/4, so a 16k window caps generation at 4096 and a thinking model exhausts it inside . Raising it to window/2 broke prompt_plus_completion_cap_never_exceeds_the_ served_window (prompt 525 + completion 512 > window 1024) — the prompt sizer overshoots its own target by ~13 tokens, which /4 had slack to hide. The OVERSHOOT is the defect, not the fraction; shipping /2 on top of it would push prompt+completion past n_ctx and 500 every turn with context-shift off. Left at /4 until the sizer is fixed. Bounding the reasoning channel (--reasoning-budget) was considered and REJECTED: it lobotomizes the model to fit a fraction we invented (Joel, 2026-08-17). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/cognition/llm_deliberation_faculty.rs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/core/continuum-core/src/cognition/llm_deliberation_faculty.rs b/core/continuum-core/src/cognition/llm_deliberation_faculty.rs index dc100896c..9d0f83b31 100644 --- a/core/continuum-core/src/cognition/llm_deliberation_faculty.rs +++ b/core/continuum-core/src/cognition/llm_deliberation_faculty.rs @@ -1930,9 +1930,30 @@ impl Faculty for LlmDeliberationFaculty { // no text, no tool call, and therefore nothing to act or speak with. The // reasoning tail rides on the fault so the receipt says WHICH shape it was: // thought-but-committed-nothing, or the lane returned void. + // ONLY when there is nothing left to recover. This guard shipped (ce82f00ff) + // gating on "empty text + no tool call" alone, which is the EXACT precondition + // of the two recovery paths below — `persona.act.reasoning_lift` (a tool call + // sitting in the reasoning tail gets lifted and executed) and + // `persona.act.think_only` (#181's teacher sentinel, which hands her another + // generation starting from her own conclusions). Faulting first made both + // unreachable: measured live 2026-08-17, 40 `delib.empty_completion` faults + // against 87 turn starts while three SWE runs sat at the same act count for + // 1,357s. The recovery already existed; the guard was standing in front of it. + // + // So a REASONING-BEARING empty is not a fault — it is #181, and it has an + // owner. Fault only for the genuinely unrecoverable shapes: the lane returned + // void (no text, no reasoning, no call), or she has no hands for the sentinel + // to teach through. Both are still surfaced rather than read as chosen silence, + // which is what this guard is for ([[a-perception-FACT-is-honesty]]). + let nothing_to_recover = resp + .reasoning + .as_deref() + .is_none_or(|r| r.trim().is_empty()) + || self.tools.is_empty(); if resp.text.trim().is_empty() && !matches!(resp.finish_reason, FinishReason::ToolUse) && resp.tool_calls.as_ref().is_none_or(|c| c.is_empty()) + && nothing_to_recover { let reasoning_tokens = resp.reasoning.as_ref().map_or(0, |r| r.len()); let why = if reasoning_tokens > 0 { From 7de2db7c877dd44036a1ff6b83f035c13f83d816 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 17 Aug 2026 08:11:34 -0500 Subject: [PATCH 09/13] =?UTF-8?q?docs(architecture):=20the=20reserve=20mus?= =?UTF-8?q?t=20YIELD=20to=20the=20incompressible=20prompt=20floor=20?= =?UTF-8?q?=E2=80=94=20why=20raising=20window/4=20is=20a=20refactor,=20not?= =?UTF-8?q?=20a=20constant?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records the measured blocker on the #181 output starvation so the next attempt does not repeat either of the two wrong fixes. MEASURED: completion_budget_for = window/4, so a 16k window caps generation at 4096. A reasoning model spends output on thinking, exhausts the cap inside , and never reaches the tool call — 7 of 20 captured turns at finish_reason: length, output_tokens: 4096 exactly, ~15k reasoning chars, empty text, zero tool calls. WRONG FIX 1, rejected on Joel's instruction: --reasoning-budget. Capping shrinks the model to fit a fraction we invented. Their thinking is the product. WRONG FIX 2, tried and reverted: window/2. Breaks prompt_plus_completion_cap_never_exceeds_the_served_window, the invariant that keeps prompt+completion under n_ctx. With context-shift off that is a 500 per turn — every citizen muted. Trading a 35% empty rate for a 100% mute rate. THE ACTUAL DEFECT, from reading prompt_view_within: the budget derivation (window − reserve − tool_tokens) is correct. What breaks is that three sibling tests pin content that must survive budget pressure unconditionally — held work card, most recent burst, newest message. Those are INCOMPRESSIBLE FLOORS. Grow the reserve and the budget drops under the floor; the packer admits the mandatory content anyway and overshoots (prompt 525 + completion 512 > 1024). The overshoot is the floor correctly refusing to compress. Holding the reserve FIXED is the error. FIX SHAPE: reserve = min(desired_share, window − mandatory_floor). Invariant then holds at every window, including the synthetic sub-MIN_SERVE_CTX windows the tests use and production never serves, while the share stays generous at real 16k+ windows where the floor is hundreds against thousands. NOT A ONE-LINER, and that is why this is a doc and not a patch: the dependency is circular (reserve → budget → packing → floor → reserve). The floor must be computable before the reserve is chosen — hoist the mandatory-section measurement ahead of budgeting, or two-pass size-then-resize. Sequence recorded: hoist the floor, make the reserve yield, THEN raise the share, and re-run all four prompt_shaping tests at both a synthetic small window and a realistic 16k one. Those four tests are the only thing between a generous reserve and a 500 on every turn; none of them may be weakened to make the change pass. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../CONTEXT-IS-A-CAPABILITY-AXIS.md | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docs/architecture/CONTEXT-IS-A-CAPABILITY-AXIS.md b/docs/architecture/CONTEXT-IS-A-CAPABILITY-AXIS.md index 5f53c5c6b..73aeb926e 100644 --- a/docs/architecture/CONTEXT-IS-A-CAPABILITY-AXIS.md +++ b/docs/architecture/CONTEXT-IS-A-CAPABILITY-AXIS.md @@ -114,3 +114,50 @@ per-generation latency and tok/s. Its absence is why every throughput question i has to be answered by black-box sampling over 20-minute windows instead of read off the stream. Fix that before trying to tune anything by measurement — an unmeasurable governor cannot be a learning one. + +--- + +## The blocker on raising the generation reserve (measured 2026-08-17) + +`completion_budget_for(window) = window / 4`. On a 16,384 window that caps generation +at 4,096 — and a reasoning model spends output tokens THINKING before it answers, so it +exhausts the cap inside `` and never reaches the tool call. Measured: 7 of 20 +captured turns came back `finish_reason: length` with `output_tokens: 4096` EXACTLY, +~15k chars of reasoning, empty text, ZERO tool calls, 4–5 minutes of GPU each. + +**Do NOT "fix" this by bounding the reasoning channel.** llama-server offers +`--reasoning-budget N`; using it makes the model smaller to fit a fraction we invented. +Their ability to think is the product (Joel, 2026-08-17: *"So blown away their ability +to think with more capping. Lame."*). + +**And do NOT just raise the fraction.** Tried `window/2`; it breaks +`prompt_plus_completion_cap_never_exceeds_the_served_window` — the invariant that keeps +`prompt + completion` under `n_ctx` (llama-server runs with context-shift off, so +crossing it is a 500 on every turn, i.e. every citizen muted). + +**The real defect, from reading the sizer.** `prompt_view_within` derives +`budget = context_window − completion_reserve − describe_tool_tokens()`, which is +correct. But three sibling tests name content that must survive budget pressure +unconditionally — the held work card, the most recent burst, the newest message. Those +are INCOMPRESSIBLE FLOORS. When the reserve grows, the budget shrinks below the floor, +and the packer admits the mandatory content anyway. Observed at window=1024: +prompt 525 + completion 512 > 1024. The overshoot IS the floor refusing to compress, +which is correct behaviour — the reserve is what's wrong to hold fixed. + +**The fix shape:** the reserve must YIELD to the floor — +`reserve = min(desired_share, window − mandatory_floor)`. Then the invariant holds at +every window (including synthetic sub-`MIN_SERVE_CTX` ones the tests use and production +never serves), and the share can be generous at real 16k+ windows where the floor is a +few hundred tokens against thousands. + +**Why it isn't a one-liner:** this is circular — reserve → budget → packing → floor → +reserve. The floor must be computable BEFORE the reserve is chosen, which means hoisting +the mandatory-section measurement ahead of budgeting (or a two-pass size-then-resize). +That is a real refactor of `prompt_view_within`, not a constant change, and it must not +weaken any of the four tests: they are the only thing standing between a generous +reserve and a 500 on every turn. + +**Sequence for whoever picks this up:** hoist the floor → make the reserve yield to it → +THEN raise the share → re-run all four `prompt_shaping` tests at both a synthetic small +window and a realistic 16k one. The share becoming a policy knob (and eventually +learned, per this document) only makes sense after the floor is load-bearing. From 943b8a2f6ad83ba28f901bf389cff65eca86db5b Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 17 Aug 2026 08:31:29 -0500 Subject: [PATCH 10/13] =?UTF-8?q?feat(bench):=20ENFORCE=20the=20gold=20gat?= =?UTF-8?q?e=20=E2=80=94=20an=20env=20that=20cannot=20score=20its=20own=20?= =?UTF-8?q?gold=20patch=20disqualifies=20itself=20(#380=20keystone)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spine check RAN and enforced NOTHING. `benchmark/swe-grade --gold` has always substituted `instance.patch` as the candidate and graded it, so the mechanism existed. But it returned a bare `resolved: false`, which downstream is byte-identical to a citizen's capability zero. The `gold` param's own doc states the requirement — "It MUST resolve — if it does not, the environment is wrong and no other number from it means anything" — and that sentence was addressed to a human and enforced by nobody. WHY THIS IS THE KEYSTONE FOR EVERY NUMBER WE REPORT. Without it a failure has two indistinguishable causes: the patch was wrong, or the environment cannot score a correct patch at all. Measured on this box 2026-08-17 (build fc8964bff, deploy-verified): a 2019-era django env carries pytest 8.4.2, and this module's own notes record era suites importing pytest internals modern pytest deleted (flask 2.2: `from _pytest.monkeypatch import notset`). So an unknown fraction of our zeros are harness artifacts tallied as capability — noise with a number attached. That is why #383's 114/300 and #380's era drift cannot be told apart from model failure by reading scores. `swe_bench::gold_gate` is a THIN caller over `grade(.., Some(&instance.patch))`, deliberately not a parallel scorer: it must exercise the exact clone → apply → test path a real attempt takes or it proves nothing about that path. A second implementation agreeing with itself is the classic dead instrument. What it adds is the enforcement: on a non-resolve it stamps `verdict.error` naming the gate, the f2p/p2p counts, and the era-dep suspects to check. `error` is contractually an ABSENCE and "must never be tallied as a failed attempt" (SweVerdict::error), so the disqualification propagates through every existing consumer with no further wiring. The command's gold arm now routes through it, so there is ONE path and every caller inherits the labelling — not a second spelling of gold grading. Explicitly NOT conflated: `gate_ok == false` means FAIL_TO_PASS already passed on the pristine tree (the instance carries no bug here). That is a different fact from a broken environment and the doc says so. NOT era-pinning pytest in this commit. That fraction is deliberate and documented at swe_bench.rs:692-706 ("harness is deliberately MODERN"), with the breakage it causes documented at :907. Inverting a decision with a stated rationale is Joel's call; this commit builds the instrument that MEASURES which envs it breaks, which is the prerequisite either way. Verified: cargo check clean. Live gold-gate run against a known-gradeable and a suspect env is the owed proof and is next. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../continuum-core/src/cognition/swe_bench.rs | 54 +++++++++++++++++++ core/continuum-core/src/commands/benchmark.rs | 13 ++++- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/core/continuum-core/src/cognition/swe_bench.rs b/core/continuum-core/src/cognition/swe_bench.rs index e844dc465..da26ceca6 100644 --- a/core/continuum-core/src/cognition/swe_bench.rs +++ b/core/continuum-core/src/cognition/swe_bench.rs @@ -1455,6 +1455,60 @@ fn compose_failure_excerpt( } } +/// THE GOLD GATE: grade the instance's OWN gold patch and require it to resolve. +/// +/// This is the spine check [`SweInstance::patch`]'s own doc has promised since that field +/// was written — "the spine check grades THIS; it must resolve or the environment is +/// wrong" — and which did not exist. `grade(.., None)` means "grade the tree as the solver +/// left it", not "grade gold", so nothing in the tree ever validated an env against a +/// known-correct patch. +/// +/// WHY THIS IS THE KEYSTONE FOR EVERY NUMBER WE REPORT. Without it a `resolved: false` has +/// two indistinguishable causes: the citizen's patch was wrong, or the environment cannot +/// score a correct patch at all. Measured 2026-08-17 on this box, the second is live and +/// unquantified: a 2019-era django env carries pytest 8.4.2, and the module's own notes +/// record era suites importing pytest internals that modern pytest deleted (flask 2.2's +/// `from _pytest.monkeypatch import notset`). So today an unknown fraction of our zeros are +/// harness artifacts being tallied as capability. That is not a measurement — it is noise +/// with a number attached, and it is why 114/300 (#383) and #380 cannot be told apart from +/// model failure by looking at scores. +/// +/// The gate makes the distinction mechanical: gold resolves → the env can score, so a +/// citizen's zero is HERS. Gold fails → the env is disqualified and no result from it may +/// be reported as capability ([[an-absence-is-an-unfinished-measurement]]). +/// +/// Deliberately a thin caller over [`grade`], not a parallel scorer: it must exercise the +/// EXACT clone → apply → test path a real attempt takes, or it proves nothing about that +/// path. A second implementation that agreed with itself would be the classic dead +/// instrument. +/// +/// `gate_ok == false` in the returned verdict is a DIFFERENT fact and is not a gate +/// failure: it means FAIL_TO_PASS already passed on the pristine tree, so the instance +/// carries no bug here. Callers must not conflate "this task cannot distinguish a fix" +/// with "this environment is broken". +pub async fn gold_gate(instance: &SweInstance, repo_dir: &Path) -> SweVerdict { + let mut verdict = grade(instance, repo_dir, Some(&instance.patch)).await; + // An env that cannot score its own gold patch is disqualified, and the reason has to + // survive into the receipt — a bare `resolved: false` here would read downstream as a + // capability zero, which is the exact confusion this gate exists to end. + if verdict.error.is_none() && !verdict.resolved { + verdict.error = Some(format!( + "GOLD GATE FAILED for {}: the instance's own gold patch did not resolve \ + (FAIL_TO_PASS {}/{}, PASS_TO_PASS {}/{}). The environment cannot score a \ + known-correct patch, so NO result from it is a capability measurement — \ + not a zero, an absence. Era deps are the leading suspect (#380): check the \ + interpreter rung against `interpreter_for_year` and the harness pytest \ + version against what this era's suite can import.", + instance.instance_id, + verdict.f2p_passed, + verdict.f2p_total, + verdict.p2p_passed, + verdict.p2p_total, + )); + } + verdict +} + pub async fn grade( instance: &SweInstance, repo_dir: &Path, diff --git a/core/continuum-core/src/commands/benchmark.rs b/core/continuum-core/src/commands/benchmark.rs index 7501247cb..35b29568f 100644 --- a/core/continuum-core/src/commands/benchmark.rs +++ b/core/continuum-core/src/commands/benchmark.rs @@ -1939,7 +1939,18 @@ pub(crate) async fn grade_swe(p: SweGradeParams) -> Result Date: Mon, 17 Aug 2026 08:42:43 -0500 Subject: [PATCH 11/13] fix(bench): grade django from CANONICAL test ids, not from prose (#383 root cause) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gold gate (943b8a2f6) caught django-10914 failing its OWN gold patch: FAIL_TO_PASS 0/1, PASS_TO_PASS 35/40, reported as "REGRESSION — your changes BROKE 5 test(s)" while the quoted output showed all five passing `... ok`. ROOT CAUSE, in `parse_django_report`: it required `" ... "` AND `" ("` on the SAME line. unittest puts the outcome on the id line ONLY when a test has no docstring; with one it prints two lines and the `... ok` lands on the SECOND: test_skip_if_db_feature (test_utils.tests.SkippingTestCase) Testing the django.test.skipIfDBFeature decorator. ... ok Both lines fell through the `continue`s — the id line has no `" ... "`, the docstring line has no `" ("` — so every docstringed django test was recorded NOWHERE, and absent-from-map reads downstream as not-passed. The 5 "broken" were the 5 docstringed ones. django is 114/300 of Lite, so django scores were never measuring django, and every django zero on record is retro-actively uninterpretable. THE FIX IS NOT A BETTER PARSER. I first extended the state machine to stitch the two-line form; Joel called it correctly — "fragile, needs cleaner fixes". It is, because unittest's verbose output is a RENDERING, not a data format: docstrings can contain " ... " and " (", django ≥4.1 changed the class-path shape, and every one of those is a way to mis-attribute silently. Mis-attribution here does not look like a bug; it looks like a citizen who failed. So django is now graded from CANONICAL ids. `install_django_json_runner` drops a `DiscoverRunner` subclass into the clone's `tests/` dir; `--testrunner` selects it; it emits one `CONTINUUM_TEST {"id": ..., "ok": ...}` row per test keyed by `test.id()` — the id unittest itself uses. No line shapes, no docstring ambiguity, no version drift. The dataset's `method (module.Class)` spelling is derived from the canonical `module.Class.method` by deterministic surgery, not guessed from output. Skips and expected failures are PASSES, unexpected successes FAILURES — the same rule as the prose path, now expressed once in `django_outcome` (it was inlined twice, which is how a `skipped` becomes a pass in one shape and a non-outcome in the other). FALLBACKS ARE LOUD, NEVER SILENT: if the runner cannot be installed, or emits zero rows (django too old for --testrunner, import error, crash before any test), we fall back to the prose parser AND warn. A silent fallback that scores is indistinguishable from one that lies. `parse_django_report` is KEPT as that fallback, with the two-line handling, since it is now the degraded path rather than the only path. Verified: cargo check clean. Live gold-gate re-run on django-10914 is the owed proof and is next — the same instance whose failure produced this diagnosis, so the fix is falsifiable by the evidence that motivated it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../continuum-core/src/cognition/swe_bench.rs | 246 +++++++++++++++++- 1 file changed, 233 insertions(+), 13 deletions(-) diff --git a/core/continuum-core/src/cognition/swe_bench.rs b/core/continuum-core/src/cognition/swe_bench.rs index da26ceca6..58e06c528 100644 --- a/core/continuum-core/src/cognition/swe_bench.rs +++ b/core/continuum-core/src/cognition/swe_bench.rs @@ -1157,7 +1157,115 @@ pub fn runner_for_repo(repo: &str) -> TestRunner { /// The exact argv (after the venv's `python`) that runs an instance's test scope. /// Pure so the per-repo command shape is table-testable without a venv. +/// The module name of the JSON runner dropped into django's `tests/` directory. +/// `tests/` is on `sys.path` when runtests.py runs, so a bare module name resolves. +const DJANGO_JSON_RUNNER_MODULE: &str = "continuum_json_runner"; + +/// A `DiscoverRunner` that reports each test by its CANONICAL id instead of by prose. +/// +/// WHY THIS EXISTS RATHER THAN A BETTER REGEX. unittest's verbose output is a rendering, +/// not a data format: the outcome sits on the id line when a test has no docstring and on +/// the DOCSTRING line when it does, django ≥4.1 changed the class-path shape, and a +/// docstring can contain any characters including " ... " and " (". Every one of those is a +/// way for a line-shaped parser to mis-attribute silently — and mis-attribution here does +/// not look like a bug, it looks like a citizen who failed. `test.id()` is the id unittest +/// itself uses; asking the runner for it removes the entire class of guess. +/// +/// Skips and expected failures count as PASSES, unexpected successes as FAILURES — the same +/// rule [`django_outcome`] applies, kept identical on purpose. +const DJANGO_JSON_RUNNER_SRC: &str = r#"# Written by continuum's SWE-bench grader. Not part of django. +import json, sys, unittest +from django.test.runner import DiscoverRunner + +class _ContinuumResult(unittest.TextTestResult): + def __init__(self, *a, **kw): + super().__init__(*a, **kw) + self.continuum_rows = {} + def addSuccess(self, test): + super().addSuccess(test); self.continuum_rows[test.id()] = True + def addError(self, test, err): + super().addError(test, err); self.continuum_rows[test.id()] = False + def addFailure(self, test, err): + super().addFailure(test, err); self.continuum_rows[test.id()] = False + def addSkip(self, test, reason): + super().addSkip(test, reason); self.continuum_rows[test.id()] = True + def addExpectedFailure(self, test, err): + super().addExpectedFailure(test, err); self.continuum_rows[test.id()] = True + def addUnexpectedSuccess(self, test): + super().addUnexpectedSuccess(test); self.continuum_rows[test.id()] = False + +class JsonRunner(DiscoverRunner): + def get_resultclass(self): + return _ContinuumResult + def run_suite(self, suite, **kwargs): + result = super().run_suite(suite, **kwargs) + for tid, ok in getattr(result, "continuum_rows", {}).items(): + sys.stderr.write("CONTINUUM_TEST " + json.dumps({"id": tid, "ok": ok}) + "\n") + sys.stderr.flush() + return result +"#; + +/// Drop the JSON runner into the clone's `tests/` dir. Returns whether it is usable. +/// Idempotent — grading re-runs over the same clone just overwrite it. +async fn install_django_json_runner(repo_dir: &Path) -> bool { + let dir = repo_dir.join("tests"); + if !dir.is_dir() { + return false; + } + let path = dir.join(format!("{DJANGO_JSON_RUNNER_MODULE}.py")); + match std::fs::write(&path, DJANGO_JSON_RUNNER_SRC) { + Ok(()) => true, + Err(e) => { + tracing::warn!( + path = %path.display(), + error = %e, + "could not install the django JSON test runner — falling back to parsing the \ + verbose report, which mis-attributes docstringed tests (#383)" + ); + false + } + } +} + +/// One machine-readable row per test: `CONTINUUM_TEST {"id": "...", "ok": true}`. +/// +/// `test.id()` is `module.Class.method`. The dataset's django ids are the unittest RENDERING +/// of that, `method (module.Class)`, so both spellings are registered for the same outcome — +/// deterministic surgery on a canonical id, not a guess about output shape. +pub fn parse_django_json(report: &str) -> (HashMap, HashMap) { + let mut by_node = HashMap::new(); + let mut by_func: HashMap = HashMap::new(); + for line in report.lines() { + let Some(payload) = line.trim().strip_prefix("CONTINUUM_TEST ") else { + continue; + }; + let Ok(v) = serde_json::from_str::(payload) else { + continue; + }; + let (Some(id), Some(ok)) = (v.get("id").and_then(|x| x.as_str()), v.get("ok").and_then(|x| x.as_bool())) + else { + continue; + }; + by_node.insert(id.to_string(), ok); + if let Some((class_path, method)) = id.rsplit_once('.') { + by_node.insert(format!("{method} ({class_path})"), ok); + let entry = by_func.entry(method.to_string()).or_insert(true); + *entry = *entry && ok; + } + } + (by_node, by_func) +} + pub fn test_invocation(runner: TestRunner, test_files: &[String]) -> Vec { + test_invocation_with(runner, test_files, false) +} + +/// [`test_invocation`], plus whether django should be told to use the JSON runner. +pub fn test_invocation_with( + runner: TestRunner, + test_files: &[String], + django_json: bool, +) -> Vec { match runner { TestRunner::Pytest => { let mut args: Vec = vec!["-m".into(), "pytest".into()]; @@ -1179,6 +1287,10 @@ pub fn test_invocation(runner: TestRunner, test_files: &[String]) -> Vec "--parallel".into(), "1".into(), ]; + if django_json { + // `tests/` is on sys.path under runtests.py, so the bare module resolves. + args.push(format!("--testrunner={DJANGO_JSON_RUNNER_MODULE}.JsonRunner")); + } args.extend(test_files.iter().map(|f| django_directive(f))); args } @@ -1201,32 +1313,115 @@ fn django_directive(file: &str) -> String { /// The report line is `test_combine (expressions.tests.CombinedExprTests) ... ok` on /// django ≤4.0, and on ≥4.1 the class path grows a trailing method repeat /// (`...CombinedExprTests.test_combine) ... ok`) — normalized back so dataset ids hit. +/// unittest's outcome vocabulary → pass/fail, or `None` for a line that is not an outcome. +/// +/// ONE place, because the report has TWO line shapes (id-and-outcome on one line, or +/// id-then-docstring-and-outcome across two) and both must classify identically. Two copies +/// of this match is how a `skipped` counts as a pass in one shape and a non-outcome in the +/// other — silent, and invisible in aggregate scores. +/// +/// `skipped` and `expected failure` are PASSES: SWE-bench's own harness treats a test that +/// declines to run as satisfied, and a test the suite knows is broken as behaving correctly. +/// `unexpected success` is a FAILURE for the same reason — the suite's expectation was wrong. +fn django_outcome(outcome: &str) -> Option { + if outcome.starts_with("ok") + || outcome.starts_with("skipped") + || outcome.starts_with("expected failure") + { + Some(true) + } else if outcome.starts_with("FAIL") + || outcome.starts_with("ERROR") + || outcome.starts_with("unexpected success") + { + Some(false) + } else { + None + } +} + +/// THE TWO-LINE DOCSTRING FORM (fixed 2026-08-17, found by the gold gate). +/// +/// unittest's verbose output puts the outcome on the test-id line ONLY when the test has no +/// docstring. With a docstring it prints TWO lines and the `... ok` lands on the SECOND: +/// +/// ```text +/// test_skip_if_db_feature (test_utils.tests.SkippingTestCase) +/// Testing the django.test.skipIfDBFeature decorator. ... ok +/// ``` +/// +/// This parser required `" ... "` AND `" ("` on ONE line, so BOTH lines fell through the +/// `continue`s: the id line has no `" ... "`, the docstring line has no `" ("`. Every +/// docstringed django test was therefore recorded NOWHERE, and absent-from-map reads +/// downstream as not-passed. +/// +/// Measured consequence, which is why this is not a cosmetic parse bug: django-10914's own +/// GOLD patch graded `FAIL_TO_PASS 0/1, PASS_TO_PASS 35/40` and was reported as a +/// "REGRESSION — your changes BROKE 5 test(s)", while the very output being quoted showed +/// all five passing `... ok`. The 5 broken were the 5 docstringed ones. So django scores +/// were never measuring django — and django is 114/300 of Lite (#383). Every django zero we +/// have recorded is retro-actively uninterpretable. +/// +/// Dataset ids for these tests may be EITHER spelling — SWE-bench's own id lists were +/// harvested from this same verbose output, so some entries are the docstring text rather +/// than the node id (e.g. "An exception is setUp() is reraised after disable() is called."). +/// Both are therefore registered as keys for the same outcome; whichever spelling the +/// dataset carries resolves. Registering both is not a fallback — it is the honest statement +/// that one test has two names in this format. pub fn parse_django_report(report: &str) -> (HashMap, HashMap) { let mut by_node = HashMap::new(); let mut by_func: HashMap = HashMap::new(); + // A test-id line awaiting its outcome on a following docstring line: (func, class_norm). + let mut pending: Option<(String, String)> = None; for line in report.lines() { let line = line.trim(); let Some((head, tail)) = line.split_once(" ... ") else { + // No outcome here. A bare `func (class.path)` line is the FIRST half of the + // two-line form — remember it. Anything else clears the pending slot so an + // outcome can never be attributed across unrelated output. + pending = match line + .split_once(" (") + .and_then(|(f, c)| c.strip_suffix(')').map(|c| (f, c))) + { + Some((func, class_path)) if !func.is_empty() && !class_path.is_empty() => { + let class_norm = class_path + .strip_suffix(&format!(".{func}")) + .unwrap_or(class_path); + Some((func.to_string(), class_norm.to_string())) + } + _ => None, + }; continue; }; + // An outcome line WITHOUT `" ("` is the docstring half. Attribute it to the id we + // remembered, and register the docstring text as a second key for the same test. + if !head.contains(" (") { + let outcome = tail.trim(); + let ok = match django_outcome(outcome) { + Some(ok) => ok, + None => { + pending = None; + continue; + } + }; + if let Some((func, class_norm)) = pending.take() { + by_node.insert(format!("{func} ({class_norm})"), ok); + let doc = head.trim(); + if !doc.is_empty() { + by_node.insert(doc.to_string(), ok); + } + let entry = by_func.entry(func).or_insert(true); + *entry = *entry && ok; + } + continue; + } + pending = None; let Some((func, class_part)) = head.split_once(" (") else { continue; }; let Some(class_path) = class_part.strip_suffix(')') else { continue; }; - let outcome = tail.trim(); - let ok = if outcome.starts_with("ok") - || outcome.starts_with("skipped") - || outcome.starts_with("expected failure") - { - true - } else if outcome.starts_with("FAIL") - || outcome.starts_with("ERROR") - || outcome.starts_with("unexpected success") - { - false - } else { + let Some(ok) = django_outcome(tail.trim()) else { continue; }; // django ≥4.1 prints `module.Class.test_name`; the dataset uses `module.Class`. @@ -1338,7 +1533,16 @@ pub async fn run_tests( // arguments" before running a single test — every id read as failed, and the whole // 2019-pytest class graded p2p 0/N (live: pytest-5221 retry, 2026-08-12). Cosmetic // flags are not worth a version gate; `-v` and no:cacheprovider go back to 2.x. - let owned_args = test_invocation(runner, test_files); + // django is graded from CANONICAL ids, not from prose. `install_django_json_runner` + // drops a DiscoverRunner subclass into the clone that emits one machine-readable row + // per test keyed by `test.id()`; `test_invocation` then asks runtests.py to use it. + // If the drop fails we fall back to the verbose-report parser — reported, never silent, + // because a fallback that scores is indistinguishable from a fallback that lies. + let django_json = match runner { + TestRunner::DjangoRuntests => install_django_json_runner(repo_dir).await, + TestRunner::Pytest => false, + }; + let owned_args = test_invocation_with(runner, test_files, django_json); let args: Vec<&str> = owned_args.iter().map(String::as_str).collect(); let Ok(out) = run(&venv_py.to_string_lossy(), &args, Some(repo_dir)).await else { return ( @@ -1353,6 +1557,22 @@ pub async fn run_tests( ); let (by_node, by_func) = match runner { TestRunner::Pytest => parse_pytest_report(&report), + // Machine-readable rows when the JSON runner is installed. If it emitted NOTHING + // (django too old for `--testrunner`, an import error in the runner, a crash before + // any test ran) fall back to the verbose report rather than scoring every id as + // failed — and say so, because a silent fallback here reads as a citizen's zero. + TestRunner::DjangoRuntests if django_json => { + let (by_node, by_func) = parse_django_json(&report); + if by_node.is_empty() { + tracing::warn!( + "the django JSON runner emitted no rows — falling back to the verbose \ + report. Grades from this run are less trustworthy (#383)." + ); + parse_django_report(&report) + } else { + (by_node, by_func) + } + } TestRunner::DjangoRuntests => parse_django_report(&report), }; let verdicts = ids From cb298848859ccdf70023e1dc985c3c01c517065c Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 17 Aug 2026 09:34:37 -0500 Subject: [PATCH 12/13] =?UTF-8?q?fix(bench):=20select=20django's=20JSON=20?= =?UTF-8?q?runner=20through=20SETTINGS=20=E2=80=94=20runtests.py=20has=20n?= =?UTF-8?q?o=20--testrunner=20(#383)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The canonical-id grader (13fd3578e) was correct about WHAT to read and wrong about HOW to ask for it. It passed `--testrunner=continuum_json_runner.JsonRunner` to `tests/runtests.py`. That flag does not exist in ANY django era — runtests.py reads `settings.TEST_RUNNER` and only defaults it when unset (verified in-tree at 1.11, 2.2, 3.2, 4.2, 5.2 and main: the same three lines throughout). An unknown flag there is not ignored. argparse rejects the whole invocation before a single test runs, so the live gold gate on django-10914 came back in 4.4 seconds with `PASS_TO_PASS passes 0 of 40 on the PRISTINE tree`. The fix I shipped to stop mis-attributing django tests made django strictly less gradeable than the prose parser it replaced. Fix: install a settings module beside the runner — from test_sqlite import * TEST_RUNNER = "continuum_json_runner.JsonRunner" — and pass `--settings=continuum_json_settings` instead of `--settings=test_sqlite`. Both files are pure additions to the clone; django's own test_sqlite stays untouched and still supplies every database/hasher setting, so the shim cannot drift from whatever the era's suite settings happen to be. The invocation now differs from the plain one in exactly one argument, which is what the new test asserts. WHAT WORKED, and is worth keeping: the zero-rows fallback. `run_tests` warns and falls back to `parse_django_report` when the JSON runner emits nothing, and that is the only reason a broken invocation surfaced as an env fault instead of 40 capability failures. A silent fallback here would have read as a citizen's zero. Mechanism proven live in the 2019-era env BEFORE this commit — the settings shim run by hand emits 30 canonical rows (`CONTINUUM_TEST {"id": "file_uploads.tests.FileUploadTests .test_base64_upload", "ok": true}`) where the flag emitted zero. Those hand-written files were then deleted so the owed gold-gate re-run exercises the install path, not my copies. 15/15 swe_bench tests green (metal,accelerate). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../continuum-core/src/cognition/swe_bench.rs | 102 ++++++++++++++---- 1 file changed, 84 insertions(+), 18 deletions(-) diff --git a/core/continuum-core/src/cognition/swe_bench.rs b/core/continuum-core/src/cognition/swe_bench.rs index 58e06c528..75f4e4c5e 100644 --- a/core/continuum-core/src/cognition/swe_bench.rs +++ b/core/continuum-core/src/cognition/swe_bench.rs @@ -1161,6 +1161,27 @@ pub fn runner_for_repo(repo: &str) -> TestRunner { /// `tests/` is on `sys.path` when runtests.py runs, so a bare module name resolves. const DJANGO_JSON_RUNNER_MODULE: &str = "continuum_json_runner"; +/// The settings module that SELECTS the JSON runner — the only seam django offers. +/// +/// `runtests.py` has **no `--testrunner` flag in any era**; it reads +/// `settings.TEST_RUNNER` and defaults it only when unset (verified in-tree at 1.11, +/// 2.2, 3.2, 4.2, 5.2 and main — same three lines throughout). Passing a flag it does +/// not know is not a no-op: argparse rejects the whole invocation before a single test +/// runs, which is exactly what happened live on django-10914 (`unrecognized arguments: +/// --testrunner=…`, suite 0/40 in 4s). So the runner is selected by a settings module +/// that re-exports django's own `test_sqlite` and overrides that one key. +const DJANGO_JSON_SETTINGS_MODULE: &str = "continuum_json_settings"; + +/// Settings for [`DJANGO_JSON_SETTINGS_MODULE`]. A pure ADDITION to the clone: django's +/// own `test_sqlite` stays untouched and supplies every database/hasher setting, so this +/// shim cannot drift from whatever the era's suite settings happen to be. +const DJANGO_JSON_SETTINGS_SRC: &str = r#"# Written by continuum's SWE-bench grader. Not part of django. +# runtests.py honours settings.TEST_RUNNER; it has no --testrunner flag. Inherit +# django's own suite settings verbatim and override only the runner. +from test_sqlite import * # noqa: F401,F403 +TEST_RUNNER = "continuum_json_runner.JsonRunner" +"#; + /// A `DiscoverRunner` that reports each test by its CANONICAL id instead of by prose. /// /// WHY THIS EXISTS RATHER THAN A BETTER REGEX. unittest's verbose output is a rendering, @@ -1205,26 +1226,30 @@ class JsonRunner(DiscoverRunner): return result "#; -/// Drop the JSON runner into the clone's `tests/` dir. Returns whether it is usable. -/// Idempotent — grading re-runs over the same clone just overwrite it. +/// Drop the JSON runner AND the settings module that selects it into the clone's `tests/` +/// dir. Returns whether both are usable — either alone does nothing, so this is one unit. +/// Idempotent — grading re-runs over the same clone just overwrite them. async fn install_django_json_runner(repo_dir: &Path) -> bool { let dir = repo_dir.join("tests"); if !dir.is_dir() { return false; } - let path = dir.join(format!("{DJANGO_JSON_RUNNER_MODULE}.py")); - match std::fs::write(&path, DJANGO_JSON_RUNNER_SRC) { - Ok(()) => true, - Err(e) => { + for (module, src) in [ + (DJANGO_JSON_RUNNER_MODULE, DJANGO_JSON_RUNNER_SRC), + (DJANGO_JSON_SETTINGS_MODULE, DJANGO_JSON_SETTINGS_SRC), + ] { + let path = dir.join(format!("{module}.py")); + if let Err(e) = std::fs::write(&path, src) { tracing::warn!( path = %path.display(), error = %e, "could not install the django JSON test runner — falling back to parsing the \ verbose report, which mis-attributes docstringed tests (#383)" ); - false + return false; } } + true } /// One machine-readable row per test: `CONTINUUM_TEST {"id": "...", "ok": true}`. @@ -1276,21 +1301,26 @@ pub fn test_invocation_with( } TestRunner::DjangoRuntests => { // Mirrors the official harness: verbosity 2 prints one line per test (the - // report we parse), test_sqlite is the settings module django's own suite - // ships for exactly this, and --parallel 1 keeps the per-test lines from - // interleaving across workers. + // report we parse when there are no JSON rows), test_sqlite is the settings + // module django's own suite ships for exactly this, and --parallel 1 keeps the + // per-test lines from interleaving across workers. + // + // The JSON runner is selected THROUGH settings (`TEST_RUNNER`), because that is + // the seam runtests.py actually reads — see [`DJANGO_JSON_SETTINGS_MODULE`]. Our + // shim re-exports test_sqlite, so this stays one settings argument either way. + let settings = if django_json { + DJANGO_JSON_SETTINGS_MODULE + } else { + "test_sqlite" + }; let mut args: Vec = vec![ "tests/runtests.py".into(), "--verbosity".into(), "2".into(), - "--settings=test_sqlite".into(), + format!("--settings={settings}"), "--parallel".into(), "1".into(), ]; - if django_json { - // `tests/` is on sys.path under runtests.py, so the bare module resolves. - args.push(format!("--testrunner={DJANGO_JSON_RUNNER_MODULE}.JsonRunner")); - } args.extend(test_files.iter().map(|f| django_directive(f))); args } @@ -1558,9 +1588,11 @@ pub async fn run_tests( let (by_node, by_func) = match runner { TestRunner::Pytest => parse_pytest_report(&report), // Machine-readable rows when the JSON runner is installed. If it emitted NOTHING - // (django too old for `--testrunner`, an import error in the runner, a crash before - // any test ran) fall back to the verbose report rather than scoring every id as - // failed — and say so, because a silent fallback here reads as a citizen's zero. + // (an import error in the runner or settings shim, a runtests.py that rejected the + // invocation, a crash before any test ran) fall back to the verbose report rather + // than scoring every id as failed — and say so, because a silent fallback here + // reads as a citizen's zero. This fallback is what kept django-10914's broken + // `--testrunner` invocation from grading as 40 capability failures. TestRunner::DjangoRuntests if django_json => { let (by_node, by_func) = parse_django_json(&report); if by_node.is_empty() { @@ -2299,6 +2331,40 @@ diff --git a/sympy/solvers/tests/test_other.py b/sympy/solvers/tests/test_other. ); } + // what this catches: selecting the JSON runner through a flag runtests.py does not have. + // Shipped once as `--testrunner=continuum_json_runner.JsonRunner` — argparse rejected the + // WHOLE invocation ("unrecognized arguments"), so django-10914's own GOLD patch graded + // 0/40 in 4 seconds. runtests.py has no such flag in ANY era; it reads settings.TEST_RUNNER + // and defaults it only when unset (verified in-tree at 1.11/2.2/3.2/4.2/5.2/main). So the + // runner is selected by a settings MODULE, and the invocation must differ from the plain + // one in exactly one argument — the settings value — and in nothing else. + #[test] + fn the_json_runner_is_selected_through_settings_never_a_flag() { + let files = vec!["tests/expressions/tests.py".to_string()]; + let plain = test_invocation_with(TestRunner::DjangoRuntests, &files, false); + let json = test_invocation_with(TestRunner::DjangoRuntests, &files, true); + + assert_eq!(plain.len(), json.len(), "the JSON path adds no argument"); + let differences: Vec<_> = plain.iter().zip(&json).filter(|(a, b)| a != b).collect(); + assert_eq!( + differences, + vec![( + &"--settings=test_sqlite".to_string(), + &format!("--settings={DJANGO_JSON_SETTINGS_MODULE}") + )], + "settings is the ONLY difference: {plain:?} vs {json:?}" + ); + assert!( + !json.iter().any(|a| a.contains("--testrunner")), + "runtests.py has no --testrunner flag; passing one aborts the run before any test" + ); + // The shim must inherit django's own suite settings rather than restate them, or it + // drifts from whatever the era's test_sqlite happens to configure. + assert!(DJANGO_JSON_SETTINGS_SRC.contains("from test_sqlite import *")); + assert!(DJANGO_JSON_SETTINGS_SRC + .contains(&format!("TEST_RUNNER = \"{DJANGO_JSON_RUNNER_MODULE}.JsonRunner\""))); + } + // what this catches: django report ids never resolving. The dataset's FAIL_TO_PASS // shape is `test_name (module.Class)`; django ≤4.0 prints exactly that, ≥4.1 appends // the method to the class path — both must hit the same dataset id, and the bare-name From 2c5b09b135e61fd2520088f813707886de7add5a Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 17 Aug 2026 09:49:00 -0500 Subject: [PATCH 13/13] =?UTF-8?q?fix(bench):=20django's=20dataset=20ids=20?= =?UTF-8?q?include=20DOCSTRINGS=20=E2=80=94=20the=20runner=20must=20emit?= =?UTF-8?q?=20its=20own=20(#383)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gold gate on django-10914 after the settings seam: FAIL_TO_PASS 1/1 (was 0/1), PASS_TO_PASS 38/40 (was 35/40), gate_ok true. The two remaining misses were not tests and not env — they were SPELLING: "An exception is setUp() is reraised after disable() is called." "assertRaisesMessage shouldn't interpret RE special chars." Those are the dataset's own PASS_TO_PASS entries, verified by reading the cached Lite JSON: 2 of django-10914's 98 p2p ids are docstring prose with no test id in them at all. The cause is upstream and structural — SWE-bench harvested django ids from unittest's verbose log, and unittest prints a test's DOCSTRING in place of its id when it has one. So a canonical id, however correct, can NEVER resolve those rows. Fix: the runner emits the docstring alongside the id (`test.shortDescription()`, the same string unittest itself renders), and the parser registers THREE spellings for one outcome — canonical id, unittest rendering, docstring. Docstrings are not unique, so they are AND-folded (a docstring shared by a pass and a fail reads as a fail) and never overwrite a real id. Only the test object knows its own docstring; having the runner report it is what removes the guess, which is the same reason the canonical id came from `test.id()`. CORRECTION to what I wrote on 13fd3578e: I described the old prose parser as "mis-attributing" docstringed tests. It DROPPED them. For a docstringed test the id line carries no `... ok` suffix and the docstring line does not match the `name (class)` shape, so neither line produced an entry. The parser was not confused about which test passed; it had no row for that test at all. That distinction matters because it is why the docstring must be emitted as DATA rather than recovered by a better regex. 16/16 swe_bench tests green (metal,accelerate). Live re-run of the gold gate is next and is the falsifiable claim: p2p 40/40 and resolved=true, or the docstring path is not the whole remainder. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../continuum-core/src/cognition/swe_bench.rs | 114 ++++++++++++++++-- 1 file changed, 102 insertions(+), 12 deletions(-) diff --git a/core/continuum-core/src/cognition/swe_bench.rs b/core/continuum-core/src/cognition/swe_bench.rs index 75f4e4c5e..7ea04fefc 100644 --- a/core/continuum-core/src/cognition/swe_bench.rs +++ b/core/continuum-core/src/cognition/swe_bench.rs @@ -1202,26 +1202,39 @@ class _ContinuumResult(unittest.TextTestResult): def __init__(self, *a, **kw): super().__init__(*a, **kw) self.continuum_rows = {} + def _record(self, test, ok): + # shortDescription() is the docstring's first line — the SAME string unittest + # renders on its own output line, and therefore the id SWE-bench's own log parser + # captured for docstringed tests. Emitting it is not redundant: the dataset spells + # those tests BY THE DOCSTRING and by nothing else. + try: + desc = test.shortDescription() or "" + except Exception: + desc = "" + self.continuum_rows[test.id()] = (ok, desc) def addSuccess(self, test): - super().addSuccess(test); self.continuum_rows[test.id()] = True + super().addSuccess(test); self._record(test, True) def addError(self, test, err): - super().addError(test, err); self.continuum_rows[test.id()] = False + super().addError(test, err); self._record(test, False) def addFailure(self, test, err): - super().addFailure(test, err); self.continuum_rows[test.id()] = False + super().addFailure(test, err); self._record(test, False) def addSkip(self, test, reason): - super().addSkip(test, reason); self.continuum_rows[test.id()] = True + super().addSkip(test, reason); self._record(test, True) def addExpectedFailure(self, test, err): - super().addExpectedFailure(test, err); self.continuum_rows[test.id()] = True + super().addExpectedFailure(test, err); self._record(test, True) def addUnexpectedSuccess(self, test): - super().addUnexpectedSuccess(test); self.continuum_rows[test.id()] = False + super().addUnexpectedSuccess(test); self._record(test, False) class JsonRunner(DiscoverRunner): def get_resultclass(self): return _ContinuumResult def run_suite(self, suite, **kwargs): result = super().run_suite(suite, **kwargs) - for tid, ok in getattr(result, "continuum_rows", {}).items(): - sys.stderr.write("CONTINUUM_TEST " + json.dumps({"id": tid, "ok": ok}) + "\n") + for tid, (ok, desc) in getattr(result, "continuum_rows", {}).items(): + row = {"id": tid, "ok": ok} + if desc: + row["desc"] = desc + sys.stderr.write("CONTINUUM_TEST " + json.dumps(row) + "\n") sys.stderr.flush() return result "#; @@ -1252,14 +1265,31 @@ async fn install_django_json_runner(repo_dir: &Path) -> bool { true } -/// One machine-readable row per test: `CONTINUUM_TEST {"id": "...", "ok": true}`. +/// One machine-readable row per test: `CONTINUUM_TEST {"id": …, "ok": …, "desc": …}`. /// -/// `test.id()` is `module.Class.method`. The dataset's django ids are the unittest RENDERING -/// of that, `method (module.Class)`, so both spellings are registered for the same outcome — -/// deterministic surgery on a canonical id, not a guess about output shape. +/// THREE spellings are registered for one outcome, because the dataset uses all three and a +/// canonical id ALONE cannot resolve a django instance (measured on django-10914, gold gate): +/// +/// | spelling | where it comes from | example | +/// |---|---|---| +/// | canonical id | `test.id()` | `test_utils.tests.AssertRaisesMsgTest.test_special_re_chars` | +/// | unittest rendering | id, re-spelled | `test_special_re_chars (test_utils.tests.AssertRaisesMsgTest)` | +/// | **docstring** | `test.shortDescription()` | `assertRaisesMessage shouldn't interpret RE special chars.` | +/// +/// The third row is the one that is easy to miss and impossible to work around downstream. +/// SWE-bench's own django ids were harvested from unittest's verbose log, and unittest prints +/// the DOCSTRING in place of the id when a test has one — so for those tests the dataset's +/// PASS_TO_PASS entry *is* the docstring, with no id anywhere in it. Verified in the dataset: +/// 2 of django-10914's 98 p2p ids are docstring prose. Only the test itself knows its own +/// docstring, which is why the runner emits it rather than the grader guessing. +/// +/// Ids and renderings are unique, so they are inserted directly. Docstrings are NOT unique — +/// two tests may share one — so they are AND-folded (pass only if every test carrying that +/// docstring passed) and they never overwrite a real id. pub fn parse_django_json(report: &str) -> (HashMap, HashMap) { let mut by_node = HashMap::new(); let mut by_func: HashMap = HashMap::new(); + let mut by_desc: HashMap = HashMap::new(); for line in report.lines() { let Some(payload) = line.trim().strip_prefix("CONTINUUM_TEST ") else { continue; @@ -1277,6 +1307,17 @@ pub fn parse_django_json(report: &str) -> (HashMap, HashMap