From d7b05f1a35186cf52a010025cbf89738f9d93d0f Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Thu, 6 Aug 2026 10:12:13 -0700 Subject: [PATCH 1/2] fix(translation): stop same-format replay from dropping router additions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A same-format hop replays the exact inbound body, which is what keeps it lossless. That replay was unconditional, so anything a routing algorithm added to the request IR after decoding never reached the wire: the codec returned the body captured before the addition. stage_router's efficient_system_prompt was silently dropped whenever the inbound request and the selected target shared a format, and handoff notes went the same way. All three buffered codecs had the same short-circuit, so this was not OpenAI-specific — an Anthropic-inbound request routed to an Anthropic target dropped its prompt too, and Responses likewise. Cross-format hops escaped only by accident: the preserved body is keyed by inbound format, so the target codec found nothing to replay and re-encoded from normalized fields. PreservationMetadata now carries a seal recorded when the body is captured. Codecs seal after decoding; exact_preserved_request refuses to replay once the IR no longer matches. Callers that mutate the IR need do nothing and cannot forget — the previous contract asked them to clear the entry by hand, which is what libsy was not doing. `model` is excluded from the seal: routing rewrites it on every hop and the client stamps the resolved name onto the encoded body afterwards, so a replayed body is never wrong about it. Note: exact_preserved_request now takes &LlmRequest rather than &PreservationMetadata. Out-of-tree codecs implementing BufferedCodec need the same one-line change, plus a seal_preservation() call after decoding if they want exact replay. Verified against the reproduction in SWITCH-1224 using a stub upstream that echoes the system message it received: before, the efficient tier saw no system message; after, it sees the configured prompt. Fixes SWITCH-1224 Co-Authored-By: Claude Opus 5 (1M context) --- crates/libsy/src/algorithms/util/prompts.rs | 73 +++++++++++++++++++ crates/protocol/src/llm.rs | 58 ++++++++++++++- .../src/codecs/anthropic/buffered.rs | 7 +- .../src/codecs/openai_chat/buffered.rs | 8 +- .../src/codecs/responses/buffered.rs | 8 +- crates/switchyard-translation/src/util.rs | 11 ++- .../tests/extension_points.rs | 32 ++++---- .../tests/request_translation.rs | 61 ++++++++++++++++ 8 files changed, 230 insertions(+), 28 deletions(-) diff --git a/crates/libsy/src/algorithms/util/prompts.rs b/crates/libsy/src/algorithms/util/prompts.rs index e4270847c..664c28568 100644 --- a/crates/libsy/src/algorithms/util/prompts.rs +++ b/crates/libsy/src/algorithms/util/prompts.rs @@ -234,6 +234,21 @@ mod tests { Ok(request) } + /// A request that already carries an exact inbound body for same-format replay. + fn request_with_preserved_body() -> Request { + let mut request = Request::default(); + request.llm_request.preservation.requests.insert( + "openai_chat".into(), + serde_json::json!({ + "model": "weak-model", + "messages": [{"role": "user", "content": "hi"}], + }), + ); + // A codec seals once decoding is done; without that the body reads as stale. + request.llm_request.seal_preservation(); + request + } + fn prompts() -> TargetPrompts { TargetPrompts::default() .with("strong", STRONG_PROMPT) @@ -304,4 +319,62 @@ mod tests { assert!(instructions(&request).is_empty()); Ok(()) } + + /// A configured tier prompt must survive a same-format hop. The codec replays + /// the preserved inbound body verbatim when it is present, so the processor has + /// to give up exact replay once it has added an instruction the body lacks. + #[tokio::test] + async fn tier_prompt_drops_preserved_body_so_same_format_targets_see_it() -> Result<()> { + let processor = SystemPromptProcessor::new(prompts()); + let mut request = request_with_preserved_body(); + processor + .process( + &mut (), + Event::Decision { + request: &mut request, + decision: &RoutedTo("weak"), + }, + ) + .await?; + + assert_eq!(instructions(&request), vec![WEAK_PROMPT]); + assert!( + !request.llm_request.preserved_request_is_current(), + "preserved inbound body would replay without the tier prompt" + ); + Ok(()) + } + + /// Same contract for the one-off note: it is added to the conversation, so a + /// replayed body would drop it too. + #[test] + fn note_drops_preserved_body() { + let mut request = request_with_preserved_body(); + append_note(&mut request, NOTE); + assert!( + !request.llm_request.preserved_request_is_current(), + "preserved inbound body would replay without the note" + ); + } + + /// A target with no configured prompt is routed untouched, so exact replay stays. + #[tokio::test] + async fn unprompted_target_keeps_preserved_body() -> Result<()> { + let processor = SystemPromptProcessor::new(prompts()); + let mut request = request_with_preserved_body(); + processor + .process( + &mut (), + Event::Decision { + request: &mut request, + decision: &RoutedTo("unconfigured"), + }, + ) + .await?; + assert!( + request.llm_request.preserved_request_is_current(), + "an untouched request keeps exact replay" + ); + Ok(()) + } } diff --git a/crates/protocol/src/llm.rs b/crates/protocol/src/llm.rs index d8def175a..a4aede975 100644 --- a/crates/protocol/src/llm.rs +++ b/crates/protocol/src/llm.rs @@ -284,9 +284,15 @@ pub struct ProviderExtensions { /// Exact source payloads retained for lossless same-format round trips. /// /// Translation's default preservation policy prefers a stored same-format body -/// over reconstructing one from normalized fields. A caller that mutates the IR -/// must clear the corresponding entry or use a policy with preservation disabled -/// when those mutations must be encoded. +/// over reconstructing one from normalized fields, which is what makes a +/// same-format hop lossless. +/// +/// A stored body is only a faithful stand-in while the IR still matches it. +/// [`LlmRequest::seal_preservation`] records what the IR looked like when the +/// body was captured, and [`LlmRequest::preserved_request_is_current`] reports +/// whether it still does. Callers that mutate the IR — adding a system prompt, +/// appending a handoff note — need do nothing: the seal stops matching and +/// codecs re-encode from normalized fields on their own. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] #[serde(default)] pub struct PreservationMetadata { @@ -294,6 +300,9 @@ pub struct PreservationMetadata { pub requests: BTreeMap, /// Original response bodies keyed by source format. pub responses: BTreeMap, + /// Fingerprint of the request IR at the moment the bodies above were + /// captured. `None` means unsealed, which reads as "not current". + pub request_seal: Option, } /// Normalized request representation shared by Switchyard components. @@ -326,6 +335,49 @@ pub struct LlmRequest { pub preservation: PreservationMetadata, } +impl LlmRequest { + /// Records the current shape of this request against its preserved bodies. + /// + /// Codecs call this once decoding is complete. Until it is called the + /// preserved bodies are treated as stale, so an un-sealed request always + /// re-encodes from normalized fields. + pub fn seal_preservation(&mut self) { + self.preservation.request_seal = None; + self.preservation.request_seal = Some(self.shape_fingerprint()); + } + + /// Whether the preserved bodies still describe this request. + /// + /// Returns `false` once anything has changed since [`Self::seal_preservation`] + /// — a routing algorithm inserting a tier system prompt, appending a handoff + /// note, rewriting the model — which is what stops a same-format hop from + /// replaying a body that predates the change. + pub fn preserved_request_is_current(&self) -> bool { + self.preservation.request_seal == Some(self.shape_fingerprint()) + } + + /// Hashes everything except the seal itself, so sealing is idempotent. + /// + /// `model` is deliberately excluded. Routing rewrites it on every hop and the + /// client stamps the resolved name onto the encoded body afterwards, so a + /// replayed body is never wrong about the model — unlike a prompt or a note, + /// which only exist in the IR. + fn shape_fingerprint(&self) -> u64 { + use std::hash::{Hash, Hasher}; + + let mut unsealed = self.clone(); + unsealed.preservation.request_seal = None; + unsealed.model = None; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + // Serialization gives a total order over the IR without requiring `Hash` + // on every nested provider value; `Value` maps are ordered. + serde_json::to_string(&unsealed) + .unwrap_or_default() + .hash(&mut hasher); + hasher.finish() + } +} + /// Normalized token usage counts. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct Usage { diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs index 7a8e79c9e..8b87fd244 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs @@ -150,6 +150,10 @@ impl FormatCodec for AnthropicMessagesCodec { ], ); + // Record the shape the preserved body describes. Anything that mutates the + // IR after this point invalidates exact replay, so a tier prompt or a + // handoff note cannot be silently dropped on a same-format hop. + request.seal_preservation(); Ok(DecodedRequest { request, diagnostics, @@ -161,8 +165,7 @@ impl FormatCodec for AnthropicMessagesCodec { request: &LlmRequest, policy: &TranslationPolicy, ) -> Result { - if let Some(body) = - exact_preserved_request(&request.preservation, WireFormat::AnthropicMessages, policy) + if let Some(body) = exact_preserved_request(request, WireFormat::AnthropicMessages, policy) { return Ok(EncodedRequest { body, diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index dbe77ca2a..3af917b53 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -165,6 +165,10 @@ impl FormatCodec for OpenAiChatCodec { ], ); + // Record the shape the preserved body describes. Anything that mutates the + // IR after this point invalidates exact replay, so a tier prompt or a + // handoff note cannot be silently dropped on a same-format hop. + request.seal_preservation(); Ok(DecodedRequest { request, diagnostics, @@ -176,9 +180,7 @@ impl FormatCodec for OpenAiChatCodec { request: &LlmRequest, policy: &TranslationPolicy, ) -> Result { - if let Some(body) = - exact_preserved_request(&request.preservation, WireFormat::OpenAiChat, policy) - { + if let Some(body) = exact_preserved_request(request, WireFormat::OpenAiChat, policy) { return Ok(EncodedRequest { body, diagnostics: Vec::new(), diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs index 5d3b9e888..4cc9ac33a 100644 --- a/crates/switchyard-translation/src/codecs/responses/buffered.rs +++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs @@ -107,6 +107,10 @@ impl FormatCodec for OpenAiResponsesCodec { "stream", ], ); + // Record the shape the preserved body describes. Anything that mutates the + // IR after this point invalidates exact replay, so a tier prompt or a + // handoff note cannot be silently dropped on a same-format hop. + request.seal_preservation(); Ok(DecodedRequest { request, diagnostics, @@ -118,9 +122,7 @@ impl FormatCodec for OpenAiResponsesCodec { request: &LlmRequest, _policy: &TranslationPolicy, ) -> Result { - if let Some(body) = - exact_preserved_request(&request.preservation, WireFormat::OpenAiResponses, _policy) - { + if let Some(body) = exact_preserved_request(request, WireFormat::OpenAiResponses, _policy) { return Ok(EncodedRequest { body, diagnostics: Vec::new(), diff --git a/crates/switchyard-translation/src/util.rs b/crates/switchyard-translation/src/util.rs index 9fcf769fd..4e67239b0 100644 --- a/crates/switchyard-translation/src/util.rs +++ b/crates/switchyard-translation/src/util.rs @@ -248,14 +248,21 @@ pub fn capture_response_preservation( } /// Returns an exact preserved request for the target format when available. +/// +/// Replay is refused once the request IR has moved on from the body — a routing +/// algorithm having added a tier system prompt or a handoff note, say. Encoding +/// then falls through to normalized fields so the addition reaches the wire. pub fn exact_preserved_request( - preservation: &PreservationMetadata, + request: &LlmRequest, format: impl Into, policy: &TranslationPolicy, ) -> Option { + if !request.preserved_request_is_current() { + return None; + } let format = format.into(); (policy.preservation != PreservationPolicy::Disabled) - .then(|| preservation.requests.get(&format).cloned()) + .then(|| request.preservation.requests.get(&format).cloned()) .flatten() } diff --git a/crates/switchyard-translation/tests/extension_points.rs b/crates/switchyard-translation/tests/extension_points.rs index d7db6f41d..9633147ec 100644 --- a/crates/switchyard-translation/tests/extension_points.rs +++ b/crates/switchyard-translation/tests/extension_points.rs @@ -154,21 +154,23 @@ impl FormatCodec for MinimalCustomCodec { body: &Value, policy: &TranslationPolicy, ) -> switchyard_translation::Result { - Ok(DecodedRequest { - request: LlmRequest { - model: body - .get("model") + let mut request = LlmRequest { + model: body + .get("model") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + messages: vec![Message::text( + Role::User, + body.get("prompt") .and_then(Value::as_str) - .map(ToOwned::to_owned), - messages: vec![Message::text( - Role::User, - body.get("prompt") - .and_then(Value::as_str) - .unwrap_or_default(), - )], - preservation: capture_request_preservation(self.format(), body, policy), - ..LlmRequest::default() - }, + .unwrap_or_default(), + )], + preservation: capture_request_preservation(self.format(), body, policy), + ..LlmRequest::default() + }; + request.seal_preservation(); + Ok(DecodedRequest { + request, diagnostics: Vec::new(), }) } @@ -178,7 +180,7 @@ impl FormatCodec for MinimalCustomCodec { request: &LlmRequest, policy: &TranslationPolicy, ) -> switchyard_translation::Result { - if let Some(body) = exact_preserved_request(&request.preservation, self.format(), policy) { + if let Some(body) = exact_preserved_request(request, self.format(), policy) { return Ok(EncodedRequest { body, diagnostics: Vec::new(), diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs index bfcf78625..277f87277 100644 --- a/crates/switchyard-translation/tests/request_translation.rs +++ b/crates/switchyard-translation/tests/request_translation.rs @@ -5,6 +5,7 @@ use pretty_assertions::assert_eq; use serde_json::{Value, json}; +use switchyard_protocol::{ContentBlock, InstructionBlock, Role}; use switchyard_translation::{TranslationEngine, TranslationPolicy, WireFormat}; type TestResult = std::result::Result<(), Box>; @@ -1490,3 +1491,63 @@ fn anthropic_thinking_is_dropped_from_responses_input() -> TestResult { ); Ok(()) } + +/// A same-format hop replays the exact inbound body, which is what makes it +/// lossless — but only while the IR still matches that body. Once a router has +/// added an instruction the body predates, replay would silently drop it, so +/// every codec must fall back to encoding from normalized fields. +/// +/// Regression test for SWITCH-1224. +#[test] +fn same_format_encoding_drops_exact_replay_once_the_ir_gains_an_instruction() -> TestResult { + const PROMPT: &str = "Respond with exactly EFFICIENT_SYSTEM_SENTINEL and nothing else."; + let engine = TranslationEngine::default(); + + for (format, inbound) in [ + ( + WireFormat::OpenAiChat, + json!({"model": "m", "messages": [{"role": "user", "content": "hi"}]}), + ), + ( + WireFormat::AnthropicMessages, + json!({"model": "m", "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}]}), + ), + ( + WireFormat::OpenAiResponses, + json!({"model": "m", "input": "hi"}), + ), + ] { + let policy = TranslationPolicy::default(); + let mut request = engine + .decode_request(format.clone(), &inbound, &policy)? + .request; + assert!( + request.preserved_request_is_current(), + "{format}: an untouched request should still replay exactly" + ); + + // What a routing algorithm does on the way out. + request.instructions.insert( + 0, + InstructionBlock { + role: Role::System, + content: vec![ContentBlock::Text { + text: PROMPT.to_string(), + }], + }, + ); + assert!( + !request.preserved_request_is_current(), + "{format}: the preserved body no longer describes this request" + ); + + let encoded = engine.encode_request(format.clone(), &request, &policy)?; + assert!( + encoded.body.to_string().contains(PROMPT), + "{format}: tier system prompt missing from the wire body: {}", + encoded.body + ); + } + Ok(()) +} From a845afccb09ec47f74ec746c7942361cd238b321 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Thu, 6 Aug 2026 10:17:45 -0700 Subject: [PATCH 2/2] test(translation): pin tier prompts across every source/target format pair Covers the guarantee the fix has to make, not just the reported symptom: both tier prompts (capable and efficient, and anything else an algorithm wires into the processor chain) reach the wire on all nine source/target format pairs. Same-format is the case that regressed; cross-format never had the replay shortcut available, so those rows pin existing behaviour rather than change it. Co-Authored-By: Claude Opus 5 (1M context) --- crates/libsy/src/algorithms/util/prompts.rs | 43 +++++++++-------- .../tests/request_translation.rs | 47 +++++++++++++++++++ 2 files changed, 70 insertions(+), 20 deletions(-) diff --git a/crates/libsy/src/algorithms/util/prompts.rs b/crates/libsy/src/algorithms/util/prompts.rs index 664c28568..756e6a1f8 100644 --- a/crates/libsy/src/algorithms/util/prompts.rs +++ b/crates/libsy/src/algorithms/util/prompts.rs @@ -320,28 +320,31 @@ mod tests { Ok(()) } - /// A configured tier prompt must survive a same-format hop. The codec replays - /// the preserved inbound body verbatim when it is present, so the processor has - /// to give up exact replay once it has added an instruction the body lacks. + /// Both tier prompts must survive a same-format hop — capable and efficient + /// alike, and whatever else an algorithm wires in. The codec replays the + /// preserved inbound body verbatim when it is still current, so adding an + /// instruction has to stop it being current. #[tokio::test] - async fn tier_prompt_drops_preserved_body_so_same_format_targets_see_it() -> Result<()> { + async fn any_tier_prompt_invalidates_exact_replay() -> Result<()> { let processor = SystemPromptProcessor::new(prompts()); - let mut request = request_with_preserved_body(); - processor - .process( - &mut (), - Event::Decision { - request: &mut request, - decision: &RoutedTo("weak"), - }, - ) - .await?; - - assert_eq!(instructions(&request), vec![WEAK_PROMPT]); - assert!( - !request.llm_request.preserved_request_is_current(), - "preserved inbound body would replay without the tier prompt" - ); + for (target, expected) in [("strong", STRONG_PROMPT), ("weak", WEAK_PROMPT)] { + let mut request = request_with_preserved_body(); + processor + .process( + &mut (), + Event::Decision { + request: &mut request, + decision: &RoutedTo(target), + }, + ) + .await?; + + assert_eq!(instructions(&request), vec![expected]); + assert!( + !request.llm_request.preserved_request_is_current(), + "{target}: preserved inbound body would replay without the tier prompt" + ); + } Ok(()) } diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs index 277f87277..89c986d39 100644 --- a/crates/switchyard-translation/tests/request_translation.rs +++ b/crates/switchyard-translation/tests/request_translation.rs @@ -1551,3 +1551,50 @@ fn same_format_encoding_drops_exact_replay_once_the_ir_gains_an_instruction() -> } Ok(()) } + +/// The same guarantee across formats: a routed request that picked up an +/// instruction must carry it to any target, whether or not the target shares the +/// inbound format. Cross-format never had the same-format shortcut available, so +/// this pins the behaviour rather than fixing it. +#[test] +fn a_router_added_instruction_reaches_every_target_format() -> TestResult { + const PROMPT: &str = "stay on the settled plan"; + let engine = TranslationEngine::default(); + let formats = [ + WireFormat::OpenAiChat, + WireFormat::AnthropicMessages, + WireFormat::OpenAiResponses, + ]; + let inbound = |format: &WireFormat| match format { + WireFormat::AnthropicMessages => json!({ + "model": "m", "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}]}), + WireFormat::OpenAiResponses => json!({"model": "m", "input": "hi"}), + _ => json!({"model": "m", "messages": [{"role": "user", "content": "hi"}]}), + }; + + for source in &formats { + for target in &formats { + let policy = TranslationPolicy::default(); + let mut request = engine + .decode_request(source.clone(), &inbound(source), &policy)? + .request; + request.instructions.insert( + 0, + InstructionBlock { + role: Role::System, + content: vec![ContentBlock::Text { + text: PROMPT.to_string(), + }], + }, + ); + let encoded = engine.encode_request(target.clone(), &request, &policy)?; + assert!( + encoded.body.to_string().contains(PROMPT), + "{source} -> {target}: instruction missing from the wire body: {}", + encoded.body + ); + } + } + Ok(()) +}