diff --git a/CHANGELOG.md b/CHANGELOG.md index ab05bf4..06b34fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## Unreleased + +### Changed + +- A block whose offload preview would tell the model too little now stays in view: prose, whose + digest was reopened 34 to 80% of the time for no net saving, and output under 2 KB whose only + preview is its first line. On held-out recorded sessions this halves reopens on the Claude Code + mod's path, from 4.5 to 2.3 per session, and lowers its context saving from 8.6% to 5.2% of tool + output. Repeats, JSON relevance splits and opt-in prose mode offload as before. +- `secondwind eval --blocks` records each offload's preview kind. + ## v0.5.0 ### Upgrade notes diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 953d9c5..d075d96 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -971,6 +971,7 @@ fn eval( "raw": block.raw_tokens, "sent": block.sent_tokens, "reopened": block.reopened, "faithful": block.faithful, "ctx": block.context_tokens, "after": block.turns_after, + "preview": block.preview, }); let _ = writeln!(out, "{line}"); } diff --git a/crates/optimize/src/cachecost.rs b/crates/optimize/src/cachecost.rs index e52d30c..74f6c44 100644 --- a/crates/optimize/src/cachecost.rs +++ b/crates/optimize/src/cachecost.rs @@ -111,7 +111,7 @@ mod tests { (0..40) .map(|i| { format!( - "The handler_{i} module validates each request against the session store first. " + "fn handler_{i}(request: &Request) -> Result {{ validate(request, {i}) }}\n" ) }) .collect() diff --git a/crates/optimize/src/counterfactual.rs b/crates/optimize/src/counterfactual.rs index 0c4590e..7e9e97b 100644 --- a/crates/optimize/src/counterfactual.rs +++ b/crates/optimize/src/counterfactual.rs @@ -147,6 +147,8 @@ pub struct BlockOutcome { pub context_tokens: u64, // Assistant turns after this result: how many times its tokens are read again. pub turns_after: usize, + // For an offload, the kind of preview the model saw in its place. + pub preview: &'static str, } #[derive(Default, Clone, Copy)] @@ -282,6 +284,11 @@ pub fn evaluate( faithful, context_tokens: next_context[ti], turns_after: assistant_after[ti], + preview: if outcome == "offload" { + preview_kind(&sent) + } else { + "" + }, }); } } @@ -334,6 +341,27 @@ fn same_content(back: &str, original: &str) -> bool { } } +// The preview an offload stub carries, read from its label, so it names what the model saw. +pub fn preview_kind(stub: &str) -> &'static str { + let label = stub.lines().next().unwrap_or(""); + const KINDS: [(&str, &str); 7] = [ + ("offloaded, diff summary", "diff summary"), + ("offloaded, match map", "match map"), + ("offloaded, code outline", "code outline"), + ("offloaded, content table", "content table"), + ("offloaded, json index", "json index"), + ("offloaded, prose digest", "prose digest"), + ("offloaded, preview:", "head"), + ]; + if let Some((_, kind)) = KINDS.iter().find(|(tag, _)| label.contains(tag)) { + return kind; + } + if stub.contains("less relevant to the query") { + return "relevance split"; + } + "other" +} + // What produced a result: the tool, and for a shell call the program whose output it is. pub fn command_class(tool: &str, input: Option<&str>) -> String { if tool != "Bash" { diff --git a/crates/optimize/src/lib.rs b/crates/optimize/src/lib.rs index cb0291c..cb11e33 100644 --- a/crates/optimize/src/lib.rs +++ b/crates/optimize/src/lib.rs @@ -97,6 +97,8 @@ pub enum KeptReason { DetectorFired, // Duplicate JSON keys: ambiguous, so kept verbatim rather than compressed. DuplicateKeys, + // Its offload preview would tell the model too little (offload::worth_offloading). + ThinPreview, } impl KeptReason { @@ -114,6 +116,7 @@ impl KeptReason { KeptReason::NoNetSaving(netcost::Reason::UnknownModel) => "no_saving_unknown_model", KeptReason::DetectorFired => "detector", KeptReason::DuplicateKeys => "duplicate_keys", + KeptReason::ThinPreview => "thin_preview", } } } @@ -701,6 +704,11 @@ impl Optimizer { reason: KeptReason::NotApplicable, }; } + if !offload::worth_offloading(raw) { + return Outcome::KeptVerbatim { + reason: KeptReason::ThinPreview, + }; + } let offloaded = match self.store.offload(raw) { Ok(offloaded) => offloaded, Err(_) => { @@ -1062,7 +1070,7 @@ mod tests { // when allowed; without a resolver stays verbatim rather than emit a marker nothing could resolve. let mut seed = 0xabcd_1234u32; let mut raw = String::new(); - for _ in 0..30 { + for _ in 0..80 { seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); let a = seed; seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); diff --git a/crates/optimize/src/offload.rs b/crates/optimize/src/offload.rs index efbf6b3..8e9e2e9 100644 --- a/crates/optimize/src/offload.rs +++ b/crates/optimize/src/offload.rs @@ -348,46 +348,98 @@ fn preview_budget(body_len: usize) -> usize { ((body_len as f64 * PREVIEW_FRACTION) as usize).max(MIN_PREVIEW_BYTES) } +// Which preview an offload stub carries in place of the block. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum PreviewKind { + DiffSummary, + MatchMap, + CodeOutline, + ContentTable, + JsonIndex, + ProseDigest, + Head, +} + +// A small block whose only preview is its first line stays in view under this size. +const MIN_HEAD_OFFLOAD_BYTES: usize = 2048; + +// Whether the preview tells the model enough for offloading to pay. On 1,491 recorded sessions a +// prose digest was reopened 34 to 80% of the time for no net saving, and a bare head hides the +// result of a short check the model has to read. +pub fn worth_offloading(raw: &str) -> bool { + let canonical = canonical_of(raw); + if canonical.len() < MIN_OFFLOAD_BYTES { + return true; + } + match preview_of(&canonical).0 { + PreviewKind::ProseDigest => false, + PreviewKind::Head => canonical.len() >= MIN_HEAD_OFFLOAD_BYTES, + _ => true, + } +} + fn preview(canonical: &str) -> String { + preview_of(canonical).1 +} + +fn preview_of(canonical: &str) -> (PreviewKind, String) { let budget = preview_budget(canonical.len()); if let Some(summary) = diff_summary(canonical, budget) { - return format!( - "[{} bytes offloaded, diff summary, call resolve for the full patch]\n{summary}", - canonical.len() + return ( + PreviewKind::DiffSummary, + format!( + "[{} bytes offloaded, diff summary, call resolve for the full patch]\n{summary}", + canonical.len() + ), ); } if let Some(map) = crate::search::location_map(canonical, budget) { - return format!( - "[{} bytes offloaded, match map, call resolve for the matched lines]\n{map}", - canonical.len() + return ( + PreviewKind::MatchMap, + format!( + "[{} bytes offloaded, match map, call resolve for the matched lines]\n{map}", + canonical.len() + ), ); } if let Some(outline) = crate::outline::outline(canonical, budget) { - return format!( - "[{} bytes offloaded, code outline, call resolve for the full body]\n{outline}", - canonical.len() + return ( + PreviewKind::CodeOutline, + format!( + "[{} bytes offloaded, code outline, call resolve for the full body]\n{outline}", + canonical.len() + ), ); } // Record arrays: show the short scalar content as a readable table so the agent answers common // lookups inline; long/boilerplate fields and overflow rows stay recoverable through resolve. if let Some(table) = content_table(canonical) { - return table; + return (PreviewKind::ContentTable, table); } // Index shape, never values, so the model can't answer a value from the preview and must resolve. if let Some(index) = json_index(canonical, budget) { - return format!( - "[{} bytes offloaded, json index, values omitted, call resolve for any value]\n{index}", - canonical.len() + return ( + PreviewKind::JsonIndex, + format!( + "[{} bytes offloaded, json index, values omitted, call resolve for any value]\n{index}", + canonical.len() + ), ); } if let Some(digest) = crate::prose::digest(canonical, crate::prose::budget(canonical.len())) { - return format!( - "[{} bytes offloaded, prose digest: some sentences verbatim, each \u{2026} a gap, call resolve for the full text]\n{digest}", - canonical.len() + return ( + PreviewKind::ProseDigest, + format!( + "[{} bytes offloaded, prose digest: some sentences verbatim, each \u{2026} a gap, call resolve for the full text]\n{digest}", + canonical.len() + ), ); } let head: String = canonical.chars().take(80).collect(); - format!("[{} bytes offloaded, preview: {head} ...]", canonical.len()) + ( + PreviewKind::Head, + format!("[{} bytes offloaded, preview: {head} ...]", canonical.len()), + ) } // The first line is always kept, so a preview is never empty. @@ -797,6 +849,45 @@ mod tests { assert_eq!(store.resolve(&out.marker).as_deref(), Some(raw.as_str())); } + #[test] + fn a_block_with_a_thin_preview_stays_in_view() { + let status: String = (0..12) + .map(|i| format!("valid bindings/node/platforms/package-{i}/package.json\n")) + .collect(); + assert!((MIN_OFFLOAD_BYTES..MIN_HEAD_OFFLOAD_BYTES).contains(&status.len())); + assert_eq!(preview_of(&canonical_of(&status)).0, PreviewKind::Head); + assert!(!worth_offloading(&status)); + let mut optimizer = crate::Optimizer::default(); + assert!(!matches!( + optimizer.compress_block(&status), + crate::Outcome::Offloaded { .. } + )); + + assert!( + worth_offloading(&status.repeat(4)), + "a large block still offloads" + ); + assert!( + worth_offloading("tiny"), + "under the offload minimum the store answers, not this rule" + ); + + let prose: String = (0..30) + .map(|i| format!("The handler number {i} validates every request against the session store before it answers. ")) + .collect(); + assert_eq!( + preview_of(&canonical_of(&prose)).0, + PreviewKind::ProseDigest + ); + assert!(!worth_offloading(&prose)); + + let code: String = (0..200) + .map(|i| format!("fn handler_{i}(input: &str) -> usize {{ input.len() * {i} }}\n")) + .collect(); + assert_eq!(preview_of(&canonical_of(&code)).0, PreviewKind::CodeOutline); + assert!(worth_offloading(&code)); + } + #[test] fn a_tampered_on_disk_blob_does_not_resolve() { use std::time::Duration; diff --git a/crates/optimize/src/policy.rs b/crates/optimize/src/policy.rs index 554058b..b82da35 100644 --- a/crates/optimize/src/policy.rs +++ b/crates/optimize/src/policy.rs @@ -135,6 +135,7 @@ mod tests { faithful: true, context_tokens: 0, turns_after: 0, + preview: "head", } } diff --git a/crates/optimize/src/proxy.rs b/crates/optimize/src/proxy.rs index b626407..f02b050 100644 --- a/crates/optimize/src/proxy.rs +++ b/crates/optimize/src/proxy.rs @@ -991,7 +991,13 @@ mod tests { // Default guard: a fresh block with no content-covering preview is kept verbatim (and booked), // not offloaded into a resolve the model would need this turn. let resolver = "mcp__secondwind__resolve"; - let raw = "The service validates every request and logs it. ".repeat(40); + let raw: String = (0..40) + .map(|i| { + format!( + "fn handler_{i}(request: &Request) -> Response {{ validate(request, {i}) }}\n" + ) + }) + .collect(); let mut body = json!({ "tools": [{"name": resolver, "description": "swload fetch"}], "messages": [{"role": "user", "content": [ diff --git a/crates/optimize/src/replay.rs b/crates/optimize/src/replay.rs index 7c03566..83694ec 100644 --- a/crates/optimize/src/replay.rs +++ b/crates/optimize/src/replay.rs @@ -511,11 +511,9 @@ mod tests { fn offloadable_result() -> String { let mut out = String::new(); - for i in 0..12 { + for i in 0..40 { out.push_str(&format!( - "The handler_{i} module validates every inbound request against the shared session store before dispatch. \ -When the token has expired the gateway returns a rejection and records the client address for the audit trail. \ -" + "fn handler_{i}(request: &Request) -> Result {{ validate(request, {i}).and_then(dispatch) }}\n" )); } out