From 2296a3e4328578175c42748955ca50ff8fce7fa9 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Thu, 6 Aug 2026 11:12:30 -0700 Subject: [PATCH 1/3] feat(libsy): de-latch escalation sessions after judged recovery Signed-off-by: Lin Jia --- crates/libsy/src/algorithms/llm_class.rs | 130 +++++++++++++++++- .../libsy/src/algorithms/util/escalation.rs | 6 + docs/reference/toml_schema.md | 1 + .../escalation_router_routing.md | 17 +++ 4 files changed, 152 insertions(+), 2 deletions(-) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index c2cce86e..66350183 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -444,6 +444,9 @@ struct TaskClassifier { /// Session-state key holding the consecutive-escalate streak. const STREAK_KEY: &str = "escalation_streak"; +/// Session-state key holding the consecutive-clear streak while latched. +const RECOVERY_STREAK_KEY: &str = "escalation_recovery_streak"; + fn streak(state: &State) -> u32 { match state.extra.get(STREAK_KEY) { Some(StateValue::Count(n)) => *n, @@ -451,6 +454,13 @@ fn streak(state: &State) -> u32 { } } +fn recovery_streak(state: &State) -> u32 { + match state.extra.get(RECOVERY_STREAK_KEY) { + Some(StateValue::Count(n)) => *n, + _ => 0, + } +} + fn decisive(target: &str) -> Classification { Classification::Scores(vec![Score { target: target.to_string(), @@ -477,6 +487,8 @@ struct EscalationClassifier { efficient: LlmTarget, /// Consecutive escalate verdicts required to latch. confirmations: u32, + /// Consecutive clear verdicts, while latched, required to de-latch. `0` disables recovery. + recovery_confirmations: u32, } #[async_trait] @@ -505,9 +517,43 @@ impl Classifier for EscalationClassifier { }); }; - // A confirmed session stays capable without a judge call. + // A confirmed session stays capable. Without recovery the latch is permanent and + // latched turns skip the judge; with it, the judge keeps reading the trajectory and + // hands the session back to efficient once the trouble has stayed clear long enough. if streak(state) >= self.confirmations { - return Ok((decisive(&self.capable.semantic_name), None)); + if self.recovery_confirmations == 0 { + return Ok((decisive(&self.capable.semantic_name), None)); + } + // Judge the trajectory as it stands; the capable tier's prior replies are already + // in the history. A clear verdict scores the efficient target. + let mut judge_request = request.clone(); + let (classification, _) = self + .judge + .score(state, &mut judge_request, Some(driver)) + .await?; + let recovered = match &classification.argmax(false)? { + Some(score) if score.target == self.efficient.semantic_name => { + recovery_streak(state) + 1 + } + Some(_) => 0, + // Judge outage: hold the latch and the streak — recovery needs live verdicts. + None => recovery_streak(state), + }; + if recovered < self.recovery_confirmations { + state.extra.insert( + RECOVERY_STREAK_KEY.to_string(), + StateValue::Count(recovered), + ); + return Ok((decisive(&self.capable.semantic_name), None)); + } + // De-latch: clear both streaks and fall through to the ordinary efficient-first + // path below, which judges this turn's fresh reply and can re-escalate as usual. + state + .extra + .insert(STREAK_KEY.to_string(), StateValue::Count(0)); + state + .extra + .insert(RECOVERY_STREAK_KEY.to_string(), StateValue::Count(0)); } // Call efficient model and buffer the response so the judge can read it. @@ -824,6 +870,7 @@ impl LlmTaskClassifier { let capable_name = capable_target.semantic_name.clone(); let efficient_name = efficient_target.semantic_name.clone(); let confirmations = config.confirmations; + let recovery_confirmations = config.recovery_confirmations; let esc = Arc::new(EscalationClassifier { judge: escalation::build_judge( judge_target, @@ -836,6 +883,7 @@ impl LlmTaskClassifier { capable: capable_target.clone(), efficient: efficient_target.clone(), confirmations, + recovery_confirmations, }); let inner: Arc> = esc.clone(); let targets = LlmTargetSet::new(vec![capable_target, efficient_target]); @@ -1913,6 +1961,84 @@ mod tests { Ok(()) } + /// Builds an escalation router that latches on the first verdict and de-latches after + /// `recovery_confirmations` clear verdicts. + fn escalation_router_with_recovery( + client: Arc, + judge_client: Arc, + recovery_confirmations: u32, + ) -> Result> { + let target = |name: &str, c: Arc| LlmTarget { + semantic_name: name.to_string(), + llm_client: Some(c), + }; + Ok(Arc::new(LlmTaskClassifier::new( + LlmClassifierConfig::Escalation { + judge_target: target("judge", judge_client), + efficient_target: target("efficient", client.clone()), + capable_target: target("capable", client), + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + confirmations: 1, + recovery_confirmations, + ..EscalationJudgeConfig::default() + }, + max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + }, + )?)) + } + + #[tokio::test] + async fn escalation_router_delatches_after_recovery_streak() -> Result<()> { + // Turn 1: judge escalates and the session latches. + // Turn 2: the latched-turn judge rules clear, the recovery streak confirms, and the + // turn falls through to the efficient-first path (whose own judge also rules clear), + // so efficient is served again. + let judge_client = QueuedClient::new([ + r#"{"escalate":true,"reason":"stuck"}"#, + r#"{"escalate":false,"reason":"blocker resolved"}"#, + r#"{"escalate":false,"reason":"progressing"}"#, + ]); + let model_client = QueuedClient::new(["efficient draft", "capable t1", "efficient t2"]); + let router = escalation_router_with_recovery(model_client, judge_client, 1)?; + + let session_request = classify_session_request(); + router + .clone() + .run(Context::default(), session_request.clone()) + .await?; + let (trace, response) = router.run(Context::default(), session_request).await?; + + assert_eq!(trace.last().map(|d| d.selected_model()), Some("efficient")); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("efficient t2".to_string()) + ); + Ok(()) + } + + #[tokio::test] + async fn escalation_router_holds_latch_until_recovery_confirms() -> Result<()> { + // With recovery_confirmations=2, one clear verdict while latched is not enough: + // the session stays capable and the streak carries to the next turn. + let judge_client = QueuedClient::new([ + r#"{"escalate":true,"reason":"stuck"}"#, + r#"{"escalate":false,"reason":"looks better"}"#, + ]); + let model_client = QueuedClient::new(["efficient draft", "capable t1", "capable t2"]); + let router = escalation_router_with_recovery(model_client, judge_client, 2)?; + + let session_request = classify_session_request(); + router + .clone() + .run(Context::default(), session_request.clone()) + .await?; + let (trace, _) = router.run(Context::default(), session_request).await?; + + assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable")); + Ok(()) + } + #[tokio::test] async fn escalation_classifier_falls_back_to_capable_when_efficient_overflows() -> Result<()> { // When the efficient model exceeds its context window inside score(), the classifier diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index b8387f62..5461cd67 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -52,6 +52,11 @@ pub struct EscalationJudgeConfig { /// `1` escalates on the first verdict; the router's main cost dial. /// `2` or higher needs a session id, since the streak is retained per session. pub confirmations: u32, + /// Consecutive clear verdicts required, while latched, before the session de-latches back + /// to the efficient tier. `0` (the default) disables recovery: a latch is permanent for + /// the session's remainder and latched turns skip the judge entirely. Any escalate verdict + /// clears the recovery streak, and a de-latched session can re-escalate as usual. + pub recovery_confirmations: u32, /// Trailing messages shown on top of the anchors. A loop longer than this is invisible. pub recent_turn_window: usize, /// Per-message cap inside the trailing window. @@ -82,6 +87,7 @@ impl Default for EscalationJudgeConfig { fn default() -> Self { Self { confirmations: 2, + recovery_confirmations: 0, recent_turn_window: 28, window_message_chars: 500, } diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index c42d0560..006d829a 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -142,6 +142,7 @@ Escalation mode serves the weak target first and judges the completed turn. See | `weak_target` | Yes | — | Target served before the latch. | | `prompt` | No | packaged prompt | Replaces the trajectory-judge prompt. | | `escalation.confirmations` | No | `2` | Consecutive escalate verdicts required to latch. Above `1` needs a session ID. | +| `escalation.recovery_confirmations` | No | `0` | Consecutive clear verdicts, while latched, before de-latching back to weak. `0` keeps the latch permanent. | | `escalation.recent_turn_window` | No | `28` | Trailing messages shown to the judge. | | `escalation.window_message_chars` | No | `500` | Per-message cap inside that window. | diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 4d3073c0..8dcab326 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -103,6 +103,7 @@ configuration, so a bare `escalation = {}` is a valid, tuned route: | Key | Default | Meaning | |---|---|---| | `confirmations` | `2` | Consecutive escalate verdicts required before the session latches to strong. Must be at least `1`. | +| `recovery_confirmations` | `0` | Consecutive clear verdicts, while latched, before the session de-latches back to weak. `0` keeps the latch permanent and skips the judge on latched turns. | | `recent_turn_window` | `28` | Trailing messages shown to the judge on top of the anchors. Must be at least `1`. | | `window_message_chars` | `500` | Per-message truncation cap inside that trailing window. Must be at least `50`. | @@ -115,6 +116,22 @@ Anchor and transcript caps remain fixed. Set the route-level `max_output_tokens` key to change the judge's reply budget. Any decline still resets the streak to zero. +## Recovery (de-escalation) + +By default a latch is permanent: latched turns skip the judge and the strong +tier serves the session's remainder, including long stretches of routine work +after the original trouble is fixed. Setting `recovery_confirmations` above +`0` keeps the judge reading the trajectory on latched turns. Once it rules +clear for that many consecutive turns, the session de-latches back to the weak +tier; the de-latching turn itself runs the ordinary weak-first path, so a +fresh escalate verdict can immediately re-latch. Any escalate verdict while +latched clears the recovery streak. The asymmetry is deliberate: escalate +eagerly (`confirmations = 2`), hand back conservatively (for example +`recovery_confirmations = 4`), so the route does not flap around transient +quiet spells. If a de-latched conversation has outgrown the weak model's +context window, the [context-window fallback](../operations/context_window.md) +returns it to strong on the next turn. + ## Run the route After building the Rust server, as described in From 8ddbb7746cf191ec5af79bcaf1faccd3aedd55a3 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Thu, 6 Aug 2026 15:14:13 -0700 Subject: [PATCH 2/3] feat(libsy): recovery prompt, probation, and one-shot de-latch Benchmarking the recovery knob showed three failure modes working together: the trouble rubric reads the latched transcript, which is the capable tier's own healthy-looking work, so it hands sessions back exactly when the strong tier is cruising through the hard part; a wrong hand-back then costs a full re-confirmation streak of weak-tier thrash before the session can re-latch; and nothing stops the cycle from repeating for the rest of the session. This change gives the latched-turn consultation its own packaged hand-back prompt that asks whether the remaining work could be carried by the efficient tier and defaults to staying strong (the route-level prompt override keeps replacing the trouble rubric only), re-latches on a single escalate verdict after a de-latch, and makes the second latch permanent so each session gets at most one recovery. Co-Authored-By: Claude Fable 5 Signed-off-by: Lin Jia --- crates/libsy/src/algorithms/llm_class.rs | 217 ++++++++++++++++-- .../libsy/src/algorithms/util/escalation.rs | 34 +++ .../src/prompts/escalation/recovery_prompt.md | 48 ++++ docs/reference/toml_schema.md | 2 +- .../escalation_router_routing.md | 36 ++- 5 files changed, 310 insertions(+), 27 deletions(-) create mode 100644 crates/libsy/src/prompts/escalation/recovery_prompt.md diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 66350183..74bb1c2a 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -447,6 +447,9 @@ const STREAK_KEY: &str = "escalation_streak"; /// Session-state key holding the consecutive-clear streak while latched. const RECOVERY_STREAK_KEY: &str = "escalation_recovery_streak"; +/// Session-state key marking that the session's one recovery has been used. +const RECOVERY_SPENT_KEY: &str = "escalation_recovery_spent"; + fn streak(state: &State) -> u32 { match state.extra.get(STREAK_KEY) { Some(StateValue::Count(n)) => *n, @@ -461,6 +464,13 @@ fn recovery_streak(state: &State) -> u32 { } } +fn recovery_spent(state: &State) -> bool { + matches!( + state.extra.get(RECOVERY_SPENT_KEY), + Some(StateValue::Count(n)) if *n > 0 + ) +} + fn decisive(target: &str) -> Classification { Classification::Scores(vec![Score { target: target.to_string(), @@ -483,6 +493,10 @@ fn assistant_message(response: &AggLlmResponse) -> Message { /// not pay for a second model call. struct EscalationClassifier { judge: JudgeClassifier, + /// Hand-back judge consulted on latched turns. Uses the packaged recovery prompt, which + /// asks whether the efficient tier could carry the *remaining* work — the trouble + /// rubric would misread the capable tier's own healthy-looking turns as recovery. + recovery_judge: JudgeClassifier, capable: LlmTarget, efficient: LlmTarget, /// Consecutive escalate verdicts required to latch. @@ -518,17 +532,20 @@ impl Classifier for EscalationClassifier { }; // A confirmed session stays capable. Without recovery the latch is permanent and - // latched turns skip the judge; with it, the judge keeps reading the trajectory and - // hands the session back to efficient once the trouble has stayed clear long enough. + // latched turns skip the judge; with it, the hand-back judge keeps reading the + // trajectory and returns the session to efficient once the remaining work no longer + // needs the capable tier. A session gets one recovery: after it has been handed back + // and re-latched, the second latch is permanent, so a wrong hand-back cannot oscillate + // for the rest of the session. if streak(state) >= self.confirmations { - if self.recovery_confirmations == 0 { + if self.recovery_confirmations == 0 || recovery_spent(state) { return Ok((decisive(&self.capable.semantic_name), None)); } - // Judge the trajectory as it stands; the capable tier's prior replies are already - // in the history. A clear verdict scores the efficient target. + // Ask the hand-back judge; the capable tier's prior replies are already in the + // history. A clear verdict scores the efficient target. let mut judge_request = request.clone(); let (classification, _) = self - .judge + .recovery_judge .score(state, &mut judge_request, Some(driver)) .await?; let recovered = match &classification.argmax(false)? { @@ -546,8 +563,12 @@ impl Classifier for EscalationClassifier { ); return Ok((decisive(&self.capable.semantic_name), None)); } - // De-latch: clear both streaks and fall through to the ordinary efficient-first - // path below, which judges this turn's fresh reply and can re-escalate as usual. + // De-latch: spend the session's one recovery, clear both streaks, and fall + // through to the ordinary efficient-first path below, which judges this turn's + // fresh reply and can re-escalate as usual. + state + .extra + .insert(RECOVERY_SPENT_KEY.to_string(), StateValue::Count(1)); state .extra .insert(STREAK_KEY.to_string(), StateValue::Count(0)); @@ -621,8 +642,21 @@ impl Classifier for EscalationClassifier { .extra .insert(STREAK_KEY.to_string(), StateValue::Count(pending)); - if escalate && pending >= self.confirmations { - // Streak confirmed: drop the efficient response, caller will serve capable. + // Probation: a session that has already been handed back once re-latches on a single + // escalate verdict, so a wrong hand-back costs one efficient turn instead of a full + // re-confirmation cycle of thrash. + let required = if recovery_spent(state) { + 1 + } else { + self.confirmations + }; + if escalate && pending >= required { + // Record a full streak so the latched check above recognizes the latch even when + // probation confirmed it early. Drop the efficient response, caller serves capable. + state.extra.insert( + STREAK_KEY.to_string(), + StateValue::Count(self.confirmations), + ); return Ok((decisive(&self.capable.semantic_name), None)); } @@ -873,10 +907,17 @@ impl LlmTaskClassifier { let recovery_confirmations = config.recovery_confirmations; let esc = Arc::new(EscalationClassifier { judge: escalation::build_judge( + judge_target.clone(), + capable_name.clone(), + efficient_name.clone(), + &contract_config, + config.clone(), + max_output_tokens, + )?, + recovery_judge: escalation::build_recovery_judge( judge_target, capable_name, efficient_name, - &contract_config, config, max_output_tokens, )?, @@ -1961,11 +2002,12 @@ mod tests { Ok(()) } - /// Builds an escalation router that latches on the first verdict and de-latches after - /// `recovery_confirmations` clear verdicts. + /// Builds an escalation router that latches after `confirmations` escalate verdicts and + /// de-latches after `recovery_confirmations` clear verdicts. fn escalation_router_with_recovery( client: Arc, judge_client: Arc, + confirmations: u32, recovery_confirmations: u32, ) -> Result> { let target = |name: &str, c: Arc| LlmTarget { @@ -1979,7 +2021,7 @@ mod tests { capable_target: target("capable", client), contract: ClassifierContractConfig::default(), config: EscalationJudgeConfig { - confirmations: 1, + confirmations, recovery_confirmations, ..EscalationJudgeConfig::default() }, @@ -2000,7 +2042,7 @@ mod tests { r#"{"escalate":false,"reason":"progressing"}"#, ]); let model_client = QueuedClient::new(["efficient draft", "capable t1", "efficient t2"]); - let router = escalation_router_with_recovery(model_client, judge_client, 1)?; + let router = escalation_router_with_recovery(model_client, judge_client, 1, 1)?; let session_request = classify_session_request(); router @@ -2026,7 +2068,7 @@ mod tests { r#"{"escalate":false,"reason":"looks better"}"#, ]); let model_client = QueuedClient::new(["efficient draft", "capable t1", "capable t2"]); - let router = escalation_router_with_recovery(model_client, judge_client, 2)?; + let router = escalation_router_with_recovery(model_client, judge_client, 1, 2)?; let session_request = classify_session_request(); router @@ -2039,6 +2081,149 @@ mod tests { Ok(()) } + #[tokio::test] + async fn escalation_router_probation_relatches_on_one_verdict_then_latch_is_permanent() + -> Result<()> { + // Turn 1: trouble judge escalates; streak 1 of 2, efficient still serves. + // Turn 2: trouble judge escalates; streak confirms, capable serves (first latch). + // Turn 3: hand-back judge clears, the session de-latches, and the fall-through + // trouble judge escalates — probation confirms on that single verdict, so capable + // serves again (second latch) instead of restarting the two-verdict streak. + // Turn 4: the second latch is permanent: no judge runs, capable serves. A clear + // verdict is queued as a tripwire — consuming it would de-latch and serve efficient. + let judge_client = QueuedClient::new([ + r#"{"escalate":true,"reason":"stuck"}"#, + r#"{"escalate":true,"reason":"still stuck"}"#, + r#"{"escalate":false,"reason":"remaining work is routine"}"#, + r#"{"escalate":true,"reason":"weak model thrashing again"}"#, + r#"{"escalate":false,"reason":"tripwire: must never be consumed"}"#, + ]); + let model_client = QueuedClient::new([ + "efficient t1", + "efficient d2", + "capable t2", + "efficient d3", + "capable t3", + "capable t4", + ]); + let router = escalation_router_with_recovery(model_client, judge_client, 2, 1)?; + + let session_request = classify_session_request(); + let (trace, _) = router + .clone() + .run(Context::default(), session_request.clone()) + .await?; + assert_eq!(trace.last().map(|d| d.selected_model()), Some("efficient")); + let (trace, _) = router + .clone() + .run(Context::default(), session_request.clone()) + .await?; + assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable")); + let (trace, response) = router + .clone() + .run(Context::default(), session_request.clone()) + .await?; + assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable")); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("capable t3".to_string()) + ); + let (trace, response) = router.run(Context::default(), session_request).await?; + assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable")); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("capable t4".to_string()) + ); + Ok(()) + } + + #[tokio::test] + async fn escalation_recovery_judge_uses_the_packaged_recovery_prompt() -> Result<()> { + // The route-level prompt override replaces the trouble rubric only; the hand-back + // judge consulted on latched turns must keep the packaged recovery prompt, which + // asks about the remaining work rather than trouble in the recent turns. + struct RecordingJudge { + replies: Mutex>, + prompts: Mutex>, + } + #[async_trait] + impl RoutedLlmClient for RecordingJudge { + async fn call( + &self, + _ctx: Context, + request: Request, + _decision: Arc, + ) -> std::result::Result { + self.prompts + .lock() + .extend( + request + .llm_request + .instructions + .first() + .and_then(|instruction| { + instruction.content.iter().find_map(|b| { + if let ContentBlock::Text { text } = b { + Some(text.clone()) + } else { + None + } + }) + }), + ); + let reply = self.replies.lock().pop_front().expect("queued verdict"); + Ok(Response { + llm_response: LlmResponse::Agg(text_response(None, reply)), + metadata: request.metadata, + }) + } + } + let judge_client = Arc::new(RecordingJudge { + replies: Mutex::new( + [ + r#"{"escalate":true,"reason":"stuck"}"#, + r#"{"escalate":false,"reason":"remaining work is routine"}"#, + r#"{"escalate":false,"reason":"progressing"}"#, + ] + .into_iter() + .map(String::from) + .collect(), + ), + prompts: Mutex::new(Vec::new()), + }); + let model_client = QueuedClient::new(["efficient d1", "capable t1", "efficient t2"]); + let target = |name: &str, c: Arc| LlmTarget { + semantic_name: name.to_string(), + llm_client: Some(c), + }; + let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + judge_target: target("judge", judge_client.clone()), + efficient_target: target("efficient", model_client.clone()), + capable_target: target("capable", model_client), + contract: ClassifierContractConfig::default().with_prompt("Custom trajectory rubric."), + config: EscalationJudgeConfig { + confirmations: 1, + recovery_confirmations: 1, + ..EscalationJudgeConfig::default() + }, + max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + })?); + + let session_request = classify_session_request(); + router + .clone() + .run(Context::default(), session_request.clone()) + .await?; + router.run(Context::default(), session_request).await?; + + let prompts = judge_client.prompts.lock().clone(); + assert_eq!(prompts.len(), 3); + assert_eq!(prompts[0], "Custom trajectory rubric."); + assert!(prompts[1].contains("hand-back judge")); + assert_eq!(prompts[2], "Custom trajectory rubric."); + Ok(()) + } + #[tokio::test] async fn escalation_classifier_falls_back_to_capable_when_efficient_overflows() -> Result<()> { // When the efficient model exceeds its context window inside score(), the classifier diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index 5461cd67..c37514d1 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -22,6 +22,7 @@ use crate::{LibsyError, Result}; use switchyard_protocol::Request; const PROMPT_TEMPLATE: &str = include_str!("../../prompts/escalation/prompt.md"); +const RECOVERY_PROMPT_TEMPLATE: &str = include_str!("../../prompts/escalation/recovery_prompt.md"); const SCHEMA_TEMPLATE: &str = include_str!("../../prompts/escalation/schema.json"); /// Separator marking where [`truncate_middle`] dropped a message's interior. @@ -173,6 +174,39 @@ pub(crate) fn build_judge( )) } +/// Builds the hand-back judge consulted on latched turns when recovery is enabled. +/// +/// The recovery question — could the efficient tier carry the *remaining* work — is +/// different from the trouble question the trajectory judge answers, and the latched +/// transcript it reads is the capable tier's own (usually healthy-looking) work. It +/// therefore always uses the packaged recovery prompt: the route-level `prompt` override +/// replaces the trouble rubric only. Same verdict schema, so `escalate: true` keeps the +/// session on the capable tier. +pub(crate) fn build_recovery_judge( + judge_target: LlmTarget, + capable: String, + efficient: String, + config: EscalationJudgeConfig, + max_output_tokens: u64, +) -> Result> { + config.validate()?; + let contract = ClassifierContract::from_config( + &ClassifierContractConfig::default(), + RECOVERY_PROMPT_TEMPLATE, + SCHEMA_TEMPLATE, + )?; + Ok(JudgeClassifier::new( + StructuredJudge::new( + EscalationInput { config }, + contract, + SerdeDecoder::new(), + JudgeRuntimeConfig::new(max_output_tokens)?, + ), + judge_target, + EscalationPolicy { capable, efficient }, + )) +} + /// The 1-indexed model invocation the transcript ends on: one per assistant reply. /// /// The judge reads the turn *including* the reply it is judging, so the newest assistant diff --git a/crates/libsy/src/prompts/escalation/recovery_prompt.md b/crates/libsy/src/prompts/escalation/recovery_prompt.md new file mode 100644 index 00000000..9e7c8f5c --- /dev/null +++ b/crates/libsy/src/prompts/escalation/recovery_prompt.md @@ -0,0 +1,48 @@ +You are a hand-back judge inside an agentic coding router. This session +hit sustained trouble earlier and was escalated: the most recent turns +you see were produced by the STRONG tier (frontier, expensive). The +router wants to hand the session back to the EFFICIENT tier (cheap but +top-class 2026 model) as soon as that is safe. + +You see a condensed view of the session: the task framing (system +prompt + first user message) and the most recent turns of activity. +Return exactly one JSON object: + +{"escalate": boolean, "reason": "one short sentence naming the evidence"} + +`escalate: true` keeps the strong tier. `escalate: false` hands the +session back to the efficient tier. + +# Judge the remaining work, not the recent turns + +The recent turns are the strong tier's own work, so they will usually +look healthy — smooth progress is what the strong tier is paid for and +is NOT evidence that the session has become easy. Do not read the +absence of trouble as recovery. Instead, ask what work REMAINS between +the trajectory's current position and the task being done, and whether +the efficient tier could carry that remainder on its own. + +Hand back (`escalate: false`) only when the evidence shows the hard +part is BEHIND the trajectory, not merely being handled well: +- The blocker that plausibly caused the escalation is visibly resolved + (the failing test now passes, the broken build now compiles, the + missing service now runs) AND what remains is routine: mechanical + edits, running an established verification, cleanup, documentation. +- The task's own success check has already passed and the remaining + turns are wrap-up. + +Stay strong (`escalate: true`) in every other case, including: +- The trajectory is mid-flight through the difficult work: cross-module + synthesis, subtle invariants, multi-step algorithmic or formal + reasoning — even when each recent turn looks clean. +- The original blocker is not yet demonstrably resolved, or nothing in + the visible window shows the task's verification passing. +- The remaining work is unclear from the visible window. + +A wrong hand-back is expensive: the efficient tier inherits a long, +difficult context mid-task, and the router pays for the thrash and the +re-escalation. When the evidence is thin or ambiguous, return +{"escalate": true}. + +Do not emit markdown, commentary, or chain-of-thought — only the JSON +object. diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 006d829a..b30b93c3 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -142,7 +142,7 @@ Escalation mode serves the weak target first and judges the completed turn. See | `weak_target` | Yes | — | Target served before the latch. | | `prompt` | No | packaged prompt | Replaces the trajectory-judge prompt. | | `escalation.confirmations` | No | `2` | Consecutive escalate verdicts required to latch. Above `1` needs a session ID. | -| `escalation.recovery_confirmations` | No | `0` | Consecutive clear verdicts, while latched, before de-latching back to weak. `0` keeps the latch permanent. | +| `escalation.recovery_confirmations` | No | `0` | Consecutive hand-back-judge clear verdicts, while latched, before de-latching back to weak. `0` keeps the latch permanent. One recovery per session; after a de-latch, a single escalate verdict re-latches permanently. | | `escalation.recent_turn_window` | No | `28` | Trailing messages shown to the judge. | | `escalation.window_message_chars` | No | `500` | Per-message cap inside that window. | diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 8dcab326..82365879 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -121,16 +121,32 @@ resets the streak to zero. By default a latch is permanent: latched turns skip the judge and the strong tier serves the session's remainder, including long stretches of routine work after the original trouble is fixed. Setting `recovery_confirmations` above -`0` keeps the judge reading the trajectory on latched turns. Once it rules -clear for that many consecutive turns, the session de-latches back to the weak -tier; the de-latching turn itself runs the ordinary weak-first path, so a -fresh escalate verdict can immediately re-latch. Any escalate verdict while -latched clears the recovery streak. The asymmetry is deliberate: escalate -eagerly (`confirmations = 2`), hand back conservatively (for example -`recovery_confirmations = 4`), so the route does not flap around transient -quiet spells. If a de-latched conversation has outgrown the weak model's -context window, the [context-window fallback](../operations/context_window.md) -returns it to strong on the next turn. +`0` consults a hand-back judge on latched turns. Once it rules clear for that +many consecutive turns, the session de-latches back to the weak tier; the +de-latching turn itself runs the ordinary weak-first path, so a fresh +escalate verdict can immediately re-latch. Any stay-strong verdict while +latched clears the recovery streak. + +The hand-back judge answers a different question than the trajectory judge. +The latched transcript is the strong tier's own work and usually looks +healthy, so "is there trouble?" would hand sessions back exactly when the +strong tier is cruising through the hard part. The packaged recovery prompt +instead asks whether the *remaining* work could be carried by the weak tier, +and defaults to staying strong when the evidence is thin. The route-level +`prompt` key overrides the trajectory-judge rubric only; the recovery prompt +is not configurable. + +Two guards keep a wrong hand-back cheap: + +- **Probation.** After a de-latch, a single escalate verdict re-latches the + session — the weak tier does not get a full `confirmations`-length streak + of turns to prove itself a second time. +- **One recovery per session.** The re-latch is permanent: latched turns stop + consulting the judge again, so a session cannot oscillate between tiers. + +If a de-latched conversation has outgrown the weak model's context window, +the [context-window fallback](../operations/context_window.md) returns it to +strong on the next turn. ## Run the route From df5462f869055ef41faa543f6a72a7e83eba41ad Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Thu, 6 Aug 2026 16:30:59 -0700 Subject: [PATCH 3/3] polish(libsy): overflow re-latch, recovery tracing, judge-builder dedup A handed-back conversation that has outgrown the efficient tier's context window overflows on every future turn, and the overflow arm never reaches the judge, so probation could not re-latch it: the session paid a doomed efficient call plus a capable fallback per turn forever. Overflow after a hand-back now re-latches permanently. Recovery transitions (hand-back, probation re-latch, overflow re-latch) now emit info-level tracing events, the two judge builders share one assembly path, and the module docs name both judges. Co-Authored-By: Claude Fable 5 Signed-off-by: Lin Jia --- crates/libsy/src/algorithms/llm_class.rs | 103 +++++++++++++++++- .../libsy/src/algorithms/util/escalation.rs | 47 +++++--- .../escalation_router_routing.md | 7 +- 3 files changed, 140 insertions(+), 17 deletions(-) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 74bb1c2a..3282cae9 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -566,6 +566,10 @@ impl Classifier for EscalationClassifier { // De-latch: spend the session's one recovery, clear both streaks, and fall // through to the ordinary efficient-first path below, which judges this turn's // fresh reply and can re-escalate as usual. + tracing::info!( + recovery_confirmations = self.recovery_confirmations, + "escalation recovery: session handed back to the efficient tier" + ); state .extra .insert(RECOVERY_SPENT_KEY.to_string(), StateValue::Count(1)); @@ -601,7 +605,23 @@ impl Classifier for EscalationClassifier { Err(LibsyError::ClientCall { source: LlmClientError::ContextWindowExceeded { .. }, .. - }) => return Ok((decisive(&self.capable.semantic_name), None)), + }) => { + // A handed-back conversation that has outgrown the efficient tier's context + // window can never be served weak again, and this arm skips the judge, so + // probation would otherwise never fire: re-latch permanently instead of + // paying a doomed efficient call plus a capable fallback on every turn. + if recovery_spent(state) { + tracing::info!( + "escalation recovery: efficient tier overflowed after hand-back, \ + re-latching permanently" + ); + state.extra.insert( + STREAK_KEY.to_string(), + StateValue::Count(self.confirmations), + ); + } + return Ok((decisive(&self.capable.semantic_name), None)); + } Err(e) => return Err(e), }; let agg = efficient_response @@ -651,6 +671,9 @@ impl Classifier for EscalationClassifier { self.confirmations }; if escalate && pending >= required { + if required < self.confirmations { + tracing::info!("escalation recovery: probation re-latch, latch is now permanent"); + } // Record a full streak so the latched check above recognizes the latch even when // probation confirmed it early. Drop the efficient response, caller serves capable. state.extra.insert( @@ -2137,6 +2160,84 @@ mod tests { Ok(()) } + #[tokio::test] + async fn escalation_relatches_permanently_when_efficient_overflows_after_hand_back() + -> Result<()> { + // Turn 1: trouble judge escalates; capable serves (first latch). + // Turn 2: hand-back judge clears and the session de-latches, but the efficient call + // overflows its context window — the turn falls through to capable and the session + // re-latches permanently, because a conversation the efficient tier cannot even read + // will overflow on every future turn and this arm never reaches the judge. + // Turn 3: capable serves with no judge call; a clear verdict is queued as a tripwire. + struct OverflowMarkerClient { + replies: Mutex>, + } + #[async_trait] + impl RoutedLlmClient for OverflowMarkerClient { + async fn call( + &self, + _ctx: Context, + request: Request, + decision: Arc, + ) -> std::result::Result { + let reply = self.replies.lock().pop_front().expect("queued reply"); + if reply == "OVERFLOW" { + return Err(ClientError::ContextWindowExceeded { + model: decision.selected_model().to_string(), + message: "prompt is too long".to_string(), + }); + } + Ok(Response { + llm_response: LlmResponse::Agg(text_response(None, reply)), + metadata: request.metadata, + }) + } + } + let model_client = Arc::new(OverflowMarkerClient { + replies: Mutex::new( + [ + "efficient d1", + "capable t1", + "OVERFLOW", + "capable t2", + "capable t3", + ] + .into_iter() + .map(String::from) + .collect(), + ), + }); + let judge_client = QueuedClient::new([ + r#"{"escalate":true,"reason":"stuck"}"#, + r#"{"escalate":false,"reason":"remaining work is routine"}"#, + r#"{"escalate":false,"reason":"tripwire: must never be consumed"}"#, + ]); + let router = escalation_router_with_recovery(model_client, judge_client, 1, 1)?; + + let session_request = classify_session_request(); + let (trace, _) = router + .clone() + .run(Context::default(), session_request.clone()) + .await?; + assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable")); + let (trace, response) = router + .clone() + .run(Context::default(), session_request.clone()) + .await?; + assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable")); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("capable t2".to_string()) + ); + let (trace, response) = router.run(Context::default(), session_request).await?; + assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable")); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("capable t3".to_string()) + ); + Ok(()) + } + #[tokio::test] async fn escalation_recovery_judge_uses_the_packaged_recovery_prompt() -> Result<()> { // The route-level prompt override replaces the trouble rubric only; the hand-back diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index c37514d1..f5a428b9 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -1,11 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Trajectory-judge components for the escalation router — the judge, its verdict policy, and -//! the transcript condenser they read. +//! Trajectory-judge components for the escalation router — the judges, their shared verdict +//! policy, and the transcript condenser they read. //! -//! [`build_judge`] is the whole surface; the confirmation policy that consumes its verdicts -//! lives with the assembled algorithm in [`crate::algorithms::escalation`]. +//! [`build_judge`] (the trouble judge) and [`build_recovery_judge`] (the hand-back judge +//! consulted on latched turns) are the whole surface; the confirmation policy that consumes +//! their verdicts lives with the assembled algorithm in [`crate::algorithms::escalation`]. use serde::Deserialize; use switchyard_protocol::{ContentBlock, Message, Role}; @@ -159,19 +160,16 @@ pub(crate) fn build_judge( config: EscalationJudgeConfig, max_output_tokens: u64, ) -> Result> { - config.validate()?; let contract = ClassifierContract::from_config(contract_config, PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?; - Ok(JudgeClassifier::new( - StructuredJudge::new( - EscalationInput { config }, - contract, - SerdeDecoder::new(), - JudgeRuntimeConfig::new(max_output_tokens)?, - ), + assemble_judge( judge_target, - EscalationPolicy { capable, efficient }, - )) + capable, + efficient, + contract, + config, + max_output_tokens, + ) } /// Builds the hand-back judge consulted on latched turns when recovery is enabled. @@ -189,12 +187,31 @@ pub(crate) fn build_recovery_judge( config: EscalationJudgeConfig, max_output_tokens: u64, ) -> Result> { - config.validate()?; let contract = ClassifierContract::from_config( &ClassifierContractConfig::default(), RECOVERY_PROMPT_TEMPLATE, SCHEMA_TEMPLATE, )?; + assemble_judge( + judge_target, + capable, + efficient, + contract, + config, + max_output_tokens, + ) +} + +/// Validates `config` and assembles a structured judge over the rendered `contract`. +fn assemble_judge( + judge_target: LlmTarget, + capable: String, + efficient: String, + contract: ClassifierContract, + config: EscalationJudgeConfig, + max_output_tokens: u64, +) -> Result> { + config.validate()?; Ok(JudgeClassifier::new( StructuredJudge::new( EscalationInput { config }, diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 82365879..46dd7867 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -68,7 +68,8 @@ For each turn on an unlatched session, Switchyard: streak reaches `confirmations`. That turn is billed for a weak call, a judge call, and a strong call. -A latched session routes straight to the strong target with no judge call: +With recovery disabled (the default), a latched session routes straight to the +strong target with no judge call: ```mermaid %%{init: {"flowchart": {"nodeSpacing": 18, "rankSpacing": 26}}}%% @@ -191,6 +192,10 @@ per-session routing stats under the judge's model id, tagged with the `classifier` tier — so per-session token accounting includes judge overhead alongside the tiers the session was served by. +Recovery transitions are logged at `info` level: one event when a session is +handed back to the weak tier, and one when it re-latches (via probation or a +weak-tier context overflow) and the latch becomes permanent. + ## When not to use escalation routing - **One-shot requests.** No trajectory to judge. Use