diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index c2cce86e5..3282cae9e 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -444,6 +444,12 @@ struct TaskClassifier { /// Session-state key holding the consecutive-escalate streak. const STREAK_KEY: &str = "escalation_streak"; +/// Session-state key holding the consecutive-clear streak while latched. +const RECOVERY_STREAK_KEY: &str = "escalation_recovery_streak"; + +/// Session-state key marking that the session's one recovery has been used. +const RECOVERY_SPENT_KEY: &str = "escalation_recovery_spent"; + fn streak(state: &State) -> u32 { match state.extra.get(STREAK_KEY) { Some(StateValue::Count(n)) => *n, @@ -451,6 +457,20 @@ fn streak(state: &State) -> u32 { } } +fn recovery_streak(state: &State) -> u32 { + match state.extra.get(RECOVERY_STREAK_KEY) { + Some(StateValue::Count(n)) => *n, + _ => 0, + } +} + +fn recovery_spent(state: &State) -> bool { + matches!( + state.extra.get(RECOVERY_SPENT_KEY), + Some(StateValue::Count(n)) if *n > 0 + ) +} + fn decisive(target: &str) -> Classification { Classification::Scores(vec![Score { target: target.to_string(), @@ -473,10 +493,16 @@ fn assistant_message(response: &AggLlmResponse) -> Message { /// not pay for a second model call. struct EscalationClassifier { judge: JudgeClassifier, + /// Hand-back judge consulted on latched turns. Uses the packaged recovery prompt, which + /// asks whether the efficient tier could carry the *remaining* work — the trouble + /// rubric would misread the capable tier's own healthy-looking turns as recovery. + recovery_judge: JudgeClassifier, capable: LlmTarget, efficient: LlmTarget, /// Consecutive escalate verdicts required to latch. confirmations: u32, + /// Consecutive clear verdicts, while latched, required to de-latch. `0` disables recovery. + recovery_confirmations: u32, } #[async_trait] @@ -505,9 +531,54 @@ impl Classifier for EscalationClassifier { }); }; - // A confirmed session stays capable without a judge call. + // A confirmed session stays capable. Without recovery the latch is permanent and + // latched turns skip the judge; with it, the hand-back judge keeps reading the + // trajectory and returns the session to efficient once the remaining work no longer + // needs the capable tier. A session gets one recovery: after it has been handed back + // and re-latched, the second latch is permanent, so a wrong hand-back cannot oscillate + // for the rest of the session. if streak(state) >= self.confirmations { - return Ok((decisive(&self.capable.semantic_name), None)); + if self.recovery_confirmations == 0 || recovery_spent(state) { + return Ok((decisive(&self.capable.semantic_name), None)); + } + // Ask the hand-back judge; the capable tier's prior replies are already in the + // history. A clear verdict scores the efficient target. + let mut judge_request = request.clone(); + let (classification, _) = self + .recovery_judge + .score(state, &mut judge_request, Some(driver)) + .await?; + let recovered = match &classification.argmax(false)? { + Some(score) if score.target == self.efficient.semantic_name => { + recovery_streak(state) + 1 + } + Some(_) => 0, + // Judge outage: hold the latch and the streak — recovery needs live verdicts. + None => recovery_streak(state), + }; + if recovered < self.recovery_confirmations { + state.extra.insert( + RECOVERY_STREAK_KEY.to_string(), + StateValue::Count(recovered), + ); + return Ok((decisive(&self.capable.semantic_name), None)); + } + // De-latch: spend the session's one recovery, clear both streaks, and fall + // through to the ordinary efficient-first path below, which judges this turn's + // fresh reply and can re-escalate as usual. + tracing::info!( + recovery_confirmations = self.recovery_confirmations, + "escalation recovery: session handed back to the efficient tier" + ); + state + .extra + .insert(RECOVERY_SPENT_KEY.to_string(), StateValue::Count(1)); + state + .extra + .insert(STREAK_KEY.to_string(), StateValue::Count(0)); + state + .extra + .insert(RECOVERY_STREAK_KEY.to_string(), StateValue::Count(0)); } // Call efficient model and buffer the response so the judge can read it. @@ -534,7 +605,23 @@ impl Classifier for EscalationClassifier { Err(LibsyError::ClientCall { source: LlmClientError::ContextWindowExceeded { .. }, .. - }) => return Ok((decisive(&self.capable.semantic_name), None)), + }) => { + // A handed-back conversation that has outgrown the efficient tier's context + // window can never be served weak again, and this arm skips the judge, so + // probation would otherwise never fire: re-latch permanently instead of + // paying a doomed efficient call plus a capable fallback on every turn. + if recovery_spent(state) { + tracing::info!( + "escalation recovery: efficient tier overflowed after hand-back, \ + re-latching permanently" + ); + state.extra.insert( + STREAK_KEY.to_string(), + StateValue::Count(self.confirmations), + ); + } + return Ok((decisive(&self.capable.semantic_name), None)); + } Err(e) => return Err(e), }; let agg = efficient_response @@ -575,8 +662,24 @@ impl Classifier for EscalationClassifier { .extra .insert(STREAK_KEY.to_string(), StateValue::Count(pending)); - if escalate && pending >= self.confirmations { - // Streak confirmed: drop the efficient response, caller will serve capable. + // Probation: a session that has already been handed back once re-latches on a single + // escalate verdict, so a wrong hand-back costs one efficient turn instead of a full + // re-confirmation cycle of thrash. + let required = if recovery_spent(state) { + 1 + } else { + self.confirmations + }; + if escalate && pending >= required { + if required < self.confirmations { + tracing::info!("escalation recovery: probation re-latch, latch is now permanent"); + } + // Record a full streak so the latched check above recognizes the latch even when + // probation confirmed it early. Drop the efficient response, caller serves capable. + state.extra.insert( + STREAK_KEY.to_string(), + StateValue::Count(self.confirmations), + ); return Ok((decisive(&self.capable.semantic_name), None)); } @@ -824,18 +927,27 @@ impl LlmTaskClassifier { let capable_name = capable_target.semantic_name.clone(); let efficient_name = efficient_target.semantic_name.clone(); let confirmations = config.confirmations; + let recovery_confirmations = config.recovery_confirmations; let esc = Arc::new(EscalationClassifier { judge: escalation::build_judge( + judge_target.clone(), + capable_name.clone(), + efficient_name.clone(), + &contract_config, + config.clone(), + max_output_tokens, + )?, + recovery_judge: escalation::build_recovery_judge( judge_target, capable_name, efficient_name, - &contract_config, config, max_output_tokens, )?, capable: capable_target.clone(), efficient: efficient_target.clone(), confirmations, + recovery_confirmations, }); let inner: Arc> = esc.clone(); let targets = LlmTargetSet::new(vec![capable_target, efficient_target]); @@ -1913,6 +2025,306 @@ mod tests { Ok(()) } + /// Builds an escalation router that latches after `confirmations` escalate verdicts and + /// de-latches after `recovery_confirmations` clear verdicts. + fn escalation_router_with_recovery( + client: Arc, + judge_client: Arc, + confirmations: u32, + recovery_confirmations: u32, + ) -> Result> { + let target = |name: &str, c: Arc| LlmTarget { + semantic_name: name.to_string(), + llm_client: Some(c), + }; + Ok(Arc::new(LlmTaskClassifier::new( + LlmClassifierConfig::Escalation { + judge_target: target("judge", judge_client), + efficient_target: target("efficient", client.clone()), + capable_target: target("capable", client), + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + confirmations, + recovery_confirmations, + ..EscalationJudgeConfig::default() + }, + max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + }, + )?)) + } + + #[tokio::test] + async fn escalation_router_delatches_after_recovery_streak() -> Result<()> { + // Turn 1: judge escalates and the session latches. + // Turn 2: the latched-turn judge rules clear, the recovery streak confirms, and the + // turn falls through to the efficient-first path (whose own judge also rules clear), + // so efficient is served again. + let judge_client = QueuedClient::new([ + r#"{"escalate":true,"reason":"stuck"}"#, + r#"{"escalate":false,"reason":"blocker resolved"}"#, + r#"{"escalate":false,"reason":"progressing"}"#, + ]); + let model_client = QueuedClient::new(["efficient draft", "capable t1", "efficient t2"]); + let router = escalation_router_with_recovery(model_client, judge_client, 1, 1)?; + + let session_request = classify_session_request(); + router + .clone() + .run(Context::default(), session_request.clone()) + .await?; + let (trace, response) = router.run(Context::default(), session_request).await?; + + assert_eq!(trace.last().map(|d| d.selected_model()), Some("efficient")); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("efficient t2".to_string()) + ); + Ok(()) + } + + #[tokio::test] + async fn escalation_router_holds_latch_until_recovery_confirms() -> Result<()> { + // With recovery_confirmations=2, one clear verdict while latched is not enough: + // the session stays capable and the streak carries to the next turn. + let judge_client = QueuedClient::new([ + r#"{"escalate":true,"reason":"stuck"}"#, + r#"{"escalate":false,"reason":"looks better"}"#, + ]); + let model_client = QueuedClient::new(["efficient draft", "capable t1", "capable t2"]); + let router = escalation_router_with_recovery(model_client, judge_client, 1, 2)?; + + let session_request = classify_session_request(); + router + .clone() + .run(Context::default(), session_request.clone()) + .await?; + let (trace, _) = router.run(Context::default(), session_request).await?; + + assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable")); + Ok(()) + } + + #[tokio::test] + async fn escalation_router_probation_relatches_on_one_verdict_then_latch_is_permanent() + -> Result<()> { + // Turn 1: trouble judge escalates; streak 1 of 2, efficient still serves. + // Turn 2: trouble judge escalates; streak confirms, capable serves (first latch). + // Turn 3: hand-back judge clears, the session de-latches, and the fall-through + // trouble judge escalates — probation confirms on that single verdict, so capable + // serves again (second latch) instead of restarting the two-verdict streak. + // Turn 4: the second latch is permanent: no judge runs, capable serves. A clear + // verdict is queued as a tripwire — consuming it would de-latch and serve efficient. + let judge_client = QueuedClient::new([ + r#"{"escalate":true,"reason":"stuck"}"#, + r#"{"escalate":true,"reason":"still stuck"}"#, + r#"{"escalate":false,"reason":"remaining work is routine"}"#, + r#"{"escalate":true,"reason":"weak model thrashing again"}"#, + r#"{"escalate":false,"reason":"tripwire: must never be consumed"}"#, + ]); + let model_client = QueuedClient::new([ + "efficient t1", + "efficient d2", + "capable t2", + "efficient d3", + "capable t3", + "capable t4", + ]); + let router = escalation_router_with_recovery(model_client, judge_client, 2, 1)?; + + let session_request = classify_session_request(); + let (trace, _) = router + .clone() + .run(Context::default(), session_request.clone()) + .await?; + assert_eq!(trace.last().map(|d| d.selected_model()), Some("efficient")); + let (trace, _) = router + .clone() + .run(Context::default(), session_request.clone()) + .await?; + assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable")); + let (trace, response) = router + .clone() + .run(Context::default(), session_request.clone()) + .await?; + assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable")); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("capable t3".to_string()) + ); + let (trace, response) = router.run(Context::default(), session_request).await?; + assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable")); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("capable t4".to_string()) + ); + Ok(()) + } + + #[tokio::test] + async fn escalation_relatches_permanently_when_efficient_overflows_after_hand_back() + -> Result<()> { + // Turn 1: trouble judge escalates; capable serves (first latch). + // Turn 2: hand-back judge clears and the session de-latches, but the efficient call + // overflows its context window — the turn falls through to capable and the session + // re-latches permanently, because a conversation the efficient tier cannot even read + // will overflow on every future turn and this arm never reaches the judge. + // Turn 3: capable serves with no judge call; a clear verdict is queued as a tripwire. + struct OverflowMarkerClient { + replies: Mutex>, + } + #[async_trait] + impl RoutedLlmClient for OverflowMarkerClient { + async fn call( + &self, + _ctx: Context, + request: Request, + decision: Arc, + ) -> std::result::Result { + let reply = self.replies.lock().pop_front().expect("queued reply"); + if reply == "OVERFLOW" { + return Err(ClientError::ContextWindowExceeded { + model: decision.selected_model().to_string(), + message: "prompt is too long".to_string(), + }); + } + Ok(Response { + llm_response: LlmResponse::Agg(text_response(None, reply)), + metadata: request.metadata, + }) + } + } + let model_client = Arc::new(OverflowMarkerClient { + replies: Mutex::new( + [ + "efficient d1", + "capable t1", + "OVERFLOW", + "capable t2", + "capable t3", + ] + .into_iter() + .map(String::from) + .collect(), + ), + }); + let judge_client = QueuedClient::new([ + r#"{"escalate":true,"reason":"stuck"}"#, + r#"{"escalate":false,"reason":"remaining work is routine"}"#, + r#"{"escalate":false,"reason":"tripwire: must never be consumed"}"#, + ]); + let router = escalation_router_with_recovery(model_client, judge_client, 1, 1)?; + + let session_request = classify_session_request(); + let (trace, _) = router + .clone() + .run(Context::default(), session_request.clone()) + .await?; + assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable")); + let (trace, response) = router + .clone() + .run(Context::default(), session_request.clone()) + .await?; + assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable")); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("capable t2".to_string()) + ); + let (trace, response) = router.run(Context::default(), session_request).await?; + assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable")); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("capable t3".to_string()) + ); + Ok(()) + } + + #[tokio::test] + async fn escalation_recovery_judge_uses_the_packaged_recovery_prompt() -> Result<()> { + // The route-level prompt override replaces the trouble rubric only; the hand-back + // judge consulted on latched turns must keep the packaged recovery prompt, which + // asks about the remaining work rather than trouble in the recent turns. + struct RecordingJudge { + replies: Mutex>, + prompts: Mutex>, + } + #[async_trait] + impl RoutedLlmClient for RecordingJudge { + async fn call( + &self, + _ctx: Context, + request: Request, + _decision: Arc, + ) -> std::result::Result { + self.prompts + .lock() + .extend( + request + .llm_request + .instructions + .first() + .and_then(|instruction| { + instruction.content.iter().find_map(|b| { + if let ContentBlock::Text { text } = b { + Some(text.clone()) + } else { + None + } + }) + }), + ); + let reply = self.replies.lock().pop_front().expect("queued verdict"); + Ok(Response { + llm_response: LlmResponse::Agg(text_response(None, reply)), + metadata: request.metadata, + }) + } + } + let judge_client = Arc::new(RecordingJudge { + replies: Mutex::new( + [ + r#"{"escalate":true,"reason":"stuck"}"#, + r#"{"escalate":false,"reason":"remaining work is routine"}"#, + r#"{"escalate":false,"reason":"progressing"}"#, + ] + .into_iter() + .map(String::from) + .collect(), + ), + prompts: Mutex::new(Vec::new()), + }); + let model_client = QueuedClient::new(["efficient d1", "capable t1", "efficient t2"]); + let target = |name: &str, c: Arc| LlmTarget { + semantic_name: name.to_string(), + llm_client: Some(c), + }; + let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + judge_target: target("judge", judge_client.clone()), + efficient_target: target("efficient", model_client.clone()), + capable_target: target("capable", model_client), + contract: ClassifierContractConfig::default().with_prompt("Custom trajectory rubric."), + config: EscalationJudgeConfig { + confirmations: 1, + recovery_confirmations: 1, + ..EscalationJudgeConfig::default() + }, + max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + })?); + + let session_request = classify_session_request(); + router + .clone() + .run(Context::default(), session_request.clone()) + .await?; + router.run(Context::default(), session_request).await?; + + let prompts = judge_client.prompts.lock().clone(); + assert_eq!(prompts.len(), 3); + assert_eq!(prompts[0], "Custom trajectory rubric."); + assert!(prompts[1].contains("hand-back judge")); + assert_eq!(prompts[2], "Custom trajectory rubric."); + Ok(()) + } + #[tokio::test] async fn escalation_classifier_falls_back_to_capable_when_efficient_overflows() -> Result<()> { // When the efficient model exceeds its context window inside score(), the classifier diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index b8387f625..f5a428b96 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -1,11 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Trajectory-judge components for the escalation router — the judge, its verdict policy, and -//! the transcript condenser they read. +//! Trajectory-judge components for the escalation router — the judges, their shared verdict +//! policy, and the transcript condenser they read. //! -//! [`build_judge`] is the whole surface; the confirmation policy that consumes its verdicts -//! lives with the assembled algorithm in [`crate::algorithms::escalation`]. +//! [`build_judge`] (the trouble judge) and [`build_recovery_judge`] (the hand-back judge +//! consulted on latched turns) are the whole surface; the confirmation policy that consumes +//! their verdicts lives with the assembled algorithm in [`crate::algorithms::escalation`]. use serde::Deserialize; use switchyard_protocol::{ContentBlock, Message, Role}; @@ -22,6 +23,7 @@ use crate::{LibsyError, Result}; use switchyard_protocol::Request; const PROMPT_TEMPLATE: &str = include_str!("../../prompts/escalation/prompt.md"); +const RECOVERY_PROMPT_TEMPLATE: &str = include_str!("../../prompts/escalation/recovery_prompt.md"); const SCHEMA_TEMPLATE: &str = include_str!("../../prompts/escalation/schema.json"); /// Separator marking where [`truncate_middle`] dropped a message's interior. @@ -52,6 +54,11 @@ pub struct EscalationJudgeConfig { /// `1` escalates on the first verdict; the router's main cost dial. /// `2` or higher needs a session id, since the streak is retained per session. pub confirmations: u32, + /// Consecutive clear verdicts required, while latched, before the session de-latches back + /// to the efficient tier. `0` (the default) disables recovery: a latch is permanent for + /// the session's remainder and latched turns skip the judge entirely. Any escalate verdict + /// clears the recovery streak, and a de-latched session can re-escalate as usual. + pub recovery_confirmations: u32, /// Trailing messages shown on top of the anchors. A loop longer than this is invisible. pub recent_turn_window: usize, /// Per-message cap inside the trailing window. @@ -82,6 +89,7 @@ impl Default for EscalationJudgeConfig { fn default() -> Self { Self { confirmations: 2, + recovery_confirmations: 0, recent_turn_window: 28, window_message_chars: 500, } @@ -152,9 +160,58 @@ pub(crate) fn build_judge( config: EscalationJudgeConfig, max_output_tokens: u64, ) -> Result> { - config.validate()?; let contract = ClassifierContract::from_config(contract_config, PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?; + assemble_judge( + judge_target, + capable, + efficient, + contract, + config, + max_output_tokens, + ) +} + +/// Builds the hand-back judge consulted on latched turns when recovery is enabled. +/// +/// The recovery question — could the efficient tier carry the *remaining* work — is +/// different from the trouble question the trajectory judge answers, and the latched +/// transcript it reads is the capable tier's own (usually healthy-looking) work. It +/// therefore always uses the packaged recovery prompt: the route-level `prompt` override +/// replaces the trouble rubric only. Same verdict schema, so `escalate: true` keeps the +/// session on the capable tier. +pub(crate) fn build_recovery_judge( + judge_target: LlmTarget, + capable: String, + efficient: String, + config: EscalationJudgeConfig, + max_output_tokens: u64, +) -> Result> { + let contract = ClassifierContract::from_config( + &ClassifierContractConfig::default(), + RECOVERY_PROMPT_TEMPLATE, + SCHEMA_TEMPLATE, + )?; + assemble_judge( + judge_target, + capable, + efficient, + contract, + config, + max_output_tokens, + ) +} + +/// Validates `config` and assembles a structured judge over the rendered `contract`. +fn assemble_judge( + judge_target: LlmTarget, + capable: String, + efficient: String, + contract: ClassifierContract, + config: EscalationJudgeConfig, + max_output_tokens: u64, +) -> Result> { + config.validate()?; Ok(JudgeClassifier::new( StructuredJudge::new( EscalationInput { config }, diff --git a/crates/libsy/src/prompts/escalation/recovery_prompt.md b/crates/libsy/src/prompts/escalation/recovery_prompt.md new file mode 100644 index 000000000..9e7c8f5c8 --- /dev/null +++ b/crates/libsy/src/prompts/escalation/recovery_prompt.md @@ -0,0 +1,48 @@ +You are a hand-back judge inside an agentic coding router. This session +hit sustained trouble earlier and was escalated: the most recent turns +you see were produced by the STRONG tier (frontier, expensive). The +router wants to hand the session back to the EFFICIENT tier (cheap but +top-class 2026 model) as soon as that is safe. + +You see a condensed view of the session: the task framing (system +prompt + first user message) and the most recent turns of activity. +Return exactly one JSON object: + +{"escalate": boolean, "reason": "one short sentence naming the evidence"} + +`escalate: true` keeps the strong tier. `escalate: false` hands the +session back to the efficient tier. + +# Judge the remaining work, not the recent turns + +The recent turns are the strong tier's own work, so they will usually +look healthy — smooth progress is what the strong tier is paid for and +is NOT evidence that the session has become easy. Do not read the +absence of trouble as recovery. Instead, ask what work REMAINS between +the trajectory's current position and the task being done, and whether +the efficient tier could carry that remainder on its own. + +Hand back (`escalate: false`) only when the evidence shows the hard +part is BEHIND the trajectory, not merely being handled well: +- The blocker that plausibly caused the escalation is visibly resolved + (the failing test now passes, the broken build now compiles, the + missing service now runs) AND what remains is routine: mechanical + edits, running an established verification, cleanup, documentation. +- The task's own success check has already passed and the remaining + turns are wrap-up. + +Stay strong (`escalate: true`) in every other case, including: +- The trajectory is mid-flight through the difficult work: cross-module + synthesis, subtle invariants, multi-step algorithmic or formal + reasoning — even when each recent turn looks clean. +- The original blocker is not yet demonstrably resolved, or nothing in + the visible window shows the task's verification passing. +- The remaining work is unclear from the visible window. + +A wrong hand-back is expensive: the efficient tier inherits a long, +difficult context mid-task, and the router pays for the thrash and the +re-escalation. When the evidence is thin or ambiguous, return +{"escalate": true}. + +Do not emit markdown, commentary, or chain-of-thought — only the JSON +object. diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index c42d0560f..b30b93c3b 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -142,6 +142,7 @@ Escalation mode serves the weak target first and judges the completed turn. See | `weak_target` | Yes | — | Target served before the latch. | | `prompt` | No | packaged prompt | Replaces the trajectory-judge prompt. | | `escalation.confirmations` | No | `2` | Consecutive escalate verdicts required to latch. Above `1` needs a session ID. | +| `escalation.recovery_confirmations` | No | `0` | Consecutive hand-back-judge clear verdicts, while latched, before de-latching back to weak. `0` keeps the latch permanent. One recovery per session; after a de-latch, a single escalate verdict re-latches permanently. | | `escalation.recent_turn_window` | No | `28` | Trailing messages shown to the judge. | | `escalation.window_message_chars` | No | `500` | Per-message cap inside that window. | diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 4d3073c05..46dd78677 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -68,7 +68,8 @@ For each turn on an unlatched session, Switchyard: streak reaches `confirmations`. That turn is billed for a weak call, a judge call, and a strong call. -A latched session routes straight to the strong target with no judge call: +With recovery disabled (the default), a latched session routes straight to the +strong target with no judge call: ```mermaid %%{init: {"flowchart": {"nodeSpacing": 18, "rankSpacing": 26}}}%% @@ -103,6 +104,7 @@ configuration, so a bare `escalation = {}` is a valid, tuned route: | Key | Default | Meaning | |---|---|---| | `confirmations` | `2` | Consecutive escalate verdicts required before the session latches to strong. Must be at least `1`. | +| `recovery_confirmations` | `0` | Consecutive clear verdicts, while latched, before the session de-latches back to weak. `0` keeps the latch permanent and skips the judge on latched turns. | | `recent_turn_window` | `28` | Trailing messages shown to the judge on top of the anchors. Must be at least `1`. | | `window_message_chars` | `500` | Per-message truncation cap inside that trailing window. Must be at least `50`. | @@ -115,6 +117,38 @@ Anchor and transcript caps remain fixed. Set the route-level `max_output_tokens` key to change the judge's reply budget. Any decline still resets the streak to zero. +## Recovery (de-escalation) + +By default a latch is permanent: latched turns skip the judge and the strong +tier serves the session's remainder, including long stretches of routine work +after the original trouble is fixed. Setting `recovery_confirmations` above +`0` consults a hand-back judge on latched turns. Once it rules clear for that +many consecutive turns, the session de-latches back to the weak tier; the +de-latching turn itself runs the ordinary weak-first path, so a fresh +escalate verdict can immediately re-latch. Any stay-strong verdict while +latched clears the recovery streak. + +The hand-back judge answers a different question than the trajectory judge. +The latched transcript is the strong tier's own work and usually looks +healthy, so "is there trouble?" would hand sessions back exactly when the +strong tier is cruising through the hard part. The packaged recovery prompt +instead asks whether the *remaining* work could be carried by the weak tier, +and defaults to staying strong when the evidence is thin. The route-level +`prompt` key overrides the trajectory-judge rubric only; the recovery prompt +is not configurable. + +Two guards keep a wrong hand-back cheap: + +- **Probation.** After a de-latch, a single escalate verdict re-latches the + session — the weak tier does not get a full `confirmations`-length streak + of turns to prove itself a second time. +- **One recovery per session.** The re-latch is permanent: latched turns stop + consulting the judge again, so a session cannot oscillate between tiers. + +If a de-latched conversation has outgrown the weak model's context window, +the [context-window fallback](../operations/context_window.md) returns it to +strong on the next turn. + ## Run the route After building the Rust server, as described in @@ -158,6 +192,10 @@ per-session routing stats under the judge's model id, tagged with the `classifier` tier — so per-session token accounting includes judge overhead alongside the tiers the session was served by. +Recovery transitions are logged at `info` level: one event when a session is +handed back to the weak tier, and one when it re-latches (via probation or a +weak-tier context overflow) and the latch becomes permanent. + ## When not to use escalation routing - **One-shot requests.** No trajectory to judge. Use