diff --git a/crates/libsy/src/algorithms/util/prompts.rs b/crates/libsy/src/algorithms/util/prompts.rs index e4270847..756e6a1f 100644 --- a/crates/libsy/src/algorithms/util/prompts.rs +++ b/crates/libsy/src/algorithms/util/prompts.rs @@ -234,6 +234,21 @@ mod tests { Ok(request) } + /// A request that already carries an exact inbound body for same-format replay. + fn request_with_preserved_body() -> Request { + let mut request = Request::default(); + request.llm_request.preservation.requests.insert( + "openai_chat".into(), + serde_json::json!({ + "model": "weak-model", + "messages": [{"role": "user", "content": "hi"}], + }), + ); + // A codec seals once decoding is done; without that the body reads as stale. + request.llm_request.seal_preservation(); + request + } + fn prompts() -> TargetPrompts { TargetPrompts::default() .with("strong", STRONG_PROMPT) @@ -304,4 +319,65 @@ mod tests { assert!(instructions(&request).is_empty()); Ok(()) } + + /// Both tier prompts must survive a same-format hop — capable and efficient + /// alike, and whatever else an algorithm wires in. The codec replays the + /// preserved inbound body verbatim when it is still current, so adding an + /// instruction has to stop it being current. + #[tokio::test] + async fn any_tier_prompt_invalidates_exact_replay() -> Result<()> { + let processor = SystemPromptProcessor::new(prompts()); + for (target, expected) in [("strong", STRONG_PROMPT), ("weak", WEAK_PROMPT)] { + let mut request = request_with_preserved_body(); + processor + .process( + &mut (), + Event::Decision { + request: &mut request, + decision: &RoutedTo(target), + }, + ) + .await?; + + assert_eq!(instructions(&request), vec![expected]); + assert!( + !request.llm_request.preserved_request_is_current(), + "{target}: preserved inbound body would replay without the tier prompt" + ); + } + Ok(()) + } + + /// Same contract for the one-off note: it is added to the conversation, so a + /// replayed body would drop it too. + #[test] + fn note_drops_preserved_body() { + let mut request = request_with_preserved_body(); + append_note(&mut request, NOTE); + assert!( + !request.llm_request.preserved_request_is_current(), + "preserved inbound body would replay without the note" + ); + } + + /// A target with no configured prompt is routed untouched, so exact replay stays. + #[tokio::test] + async fn unprompted_target_keeps_preserved_body() -> Result<()> { + let processor = SystemPromptProcessor::new(prompts()); + let mut request = request_with_preserved_body(); + processor + .process( + &mut (), + Event::Decision { + request: &mut request, + decision: &RoutedTo("unconfigured"), + }, + ) + .await?; + assert!( + request.llm_request.preserved_request_is_current(), + "an untouched request keeps exact replay" + ); + Ok(()) + } } diff --git a/crates/protocol/src/llm.rs b/crates/protocol/src/llm.rs index d8def175..a4aede97 100644 --- a/crates/protocol/src/llm.rs +++ b/crates/protocol/src/llm.rs @@ -284,9 +284,15 @@ pub struct ProviderExtensions { /// Exact source payloads retained for lossless same-format round trips. /// /// Translation's default preservation policy prefers a stored same-format body -/// over reconstructing one from normalized fields. A caller that mutates the IR -/// must clear the corresponding entry or use a policy with preservation disabled -/// when those mutations must be encoded. +/// over reconstructing one from normalized fields, which is what makes a +/// same-format hop lossless. +/// +/// A stored body is only a faithful stand-in while the IR still matches it. +/// [`LlmRequest::seal_preservation`] records what the IR looked like when the +/// body was captured, and [`LlmRequest::preserved_request_is_current`] reports +/// whether it still does. Callers that mutate the IR — adding a system prompt, +/// appending a handoff note — need do nothing: the seal stops matching and +/// codecs re-encode from normalized fields on their own. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] #[serde(default)] pub struct PreservationMetadata { @@ -294,6 +300,9 @@ pub struct PreservationMetadata { pub requests: BTreeMap, /// Original response bodies keyed by source format. pub responses: BTreeMap, + /// Fingerprint of the request IR at the moment the bodies above were + /// captured. `None` means unsealed, which reads as "not current". + pub request_seal: Option, } /// Normalized request representation shared by Switchyard components. @@ -326,6 +335,49 @@ pub struct LlmRequest { pub preservation: PreservationMetadata, } +impl LlmRequest { + /// Records the current shape of this request against its preserved bodies. + /// + /// Codecs call this once decoding is complete. Until it is called the + /// preserved bodies are treated as stale, so an un-sealed request always + /// re-encodes from normalized fields. + pub fn seal_preservation(&mut self) { + self.preservation.request_seal = None; + self.preservation.request_seal = Some(self.shape_fingerprint()); + } + + /// Whether the preserved bodies still describe this request. + /// + /// Returns `false` once anything has changed since [`Self::seal_preservation`] + /// — a routing algorithm inserting a tier system prompt, appending a handoff + /// note, rewriting the model — which is what stops a same-format hop from + /// replaying a body that predates the change. + pub fn preserved_request_is_current(&self) -> bool { + self.preservation.request_seal == Some(self.shape_fingerprint()) + } + + /// Hashes everything except the seal itself, so sealing is idempotent. + /// + /// `model` is deliberately excluded. Routing rewrites it on every hop and the + /// client stamps the resolved name onto the encoded body afterwards, so a + /// replayed body is never wrong about the model — unlike a prompt or a note, + /// which only exist in the IR. + fn shape_fingerprint(&self) -> u64 { + use std::hash::{Hash, Hasher}; + + let mut unsealed = self.clone(); + unsealed.preservation.request_seal = None; + unsealed.model = None; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + // Serialization gives a total order over the IR without requiring `Hash` + // on every nested provider value; `Value` maps are ordered. + serde_json::to_string(&unsealed) + .unwrap_or_default() + .hash(&mut hasher); + hasher.finish() + } +} + /// Normalized token usage counts. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct Usage { diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs index 7a8e79c9..8b87fd24 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs @@ -150,6 +150,10 @@ impl FormatCodec for AnthropicMessagesCodec { ], ); + // Record the shape the preserved body describes. Anything that mutates the + // IR after this point invalidates exact replay, so a tier prompt or a + // handoff note cannot be silently dropped on a same-format hop. + request.seal_preservation(); Ok(DecodedRequest { request, diagnostics, @@ -161,8 +165,7 @@ impl FormatCodec for AnthropicMessagesCodec { request: &LlmRequest, policy: &TranslationPolicy, ) -> Result { - if let Some(body) = - exact_preserved_request(&request.preservation, WireFormat::AnthropicMessages, policy) + if let Some(body) = exact_preserved_request(request, WireFormat::AnthropicMessages, policy) { return Ok(EncodedRequest { body, diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index dbe77ca2..3af917b5 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -165,6 +165,10 @@ impl FormatCodec for OpenAiChatCodec { ], ); + // Record the shape the preserved body describes. Anything that mutates the + // IR after this point invalidates exact replay, so a tier prompt or a + // handoff note cannot be silently dropped on a same-format hop. + request.seal_preservation(); Ok(DecodedRequest { request, diagnostics, @@ -176,9 +180,7 @@ impl FormatCodec for OpenAiChatCodec { request: &LlmRequest, policy: &TranslationPolicy, ) -> Result { - if let Some(body) = - exact_preserved_request(&request.preservation, WireFormat::OpenAiChat, policy) - { + if let Some(body) = exact_preserved_request(request, WireFormat::OpenAiChat, policy) { return Ok(EncodedRequest { body, diagnostics: Vec::new(), diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs index 5d3b9e88..4cc9ac33 100644 --- a/crates/switchyard-translation/src/codecs/responses/buffered.rs +++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs @@ -107,6 +107,10 @@ impl FormatCodec for OpenAiResponsesCodec { "stream", ], ); + // Record the shape the preserved body describes. Anything that mutates the + // IR after this point invalidates exact replay, so a tier prompt or a + // handoff note cannot be silently dropped on a same-format hop. + request.seal_preservation(); Ok(DecodedRequest { request, diagnostics, @@ -118,9 +122,7 @@ impl FormatCodec for OpenAiResponsesCodec { request: &LlmRequest, _policy: &TranslationPolicy, ) -> Result { - if let Some(body) = - exact_preserved_request(&request.preservation, WireFormat::OpenAiResponses, _policy) - { + if let Some(body) = exact_preserved_request(request, WireFormat::OpenAiResponses, _policy) { return Ok(EncodedRequest { body, diagnostics: Vec::new(), diff --git a/crates/switchyard-translation/src/util.rs b/crates/switchyard-translation/src/util.rs index 9fcf769f..4e67239b 100644 --- a/crates/switchyard-translation/src/util.rs +++ b/crates/switchyard-translation/src/util.rs @@ -248,14 +248,21 @@ pub fn capture_response_preservation( } /// Returns an exact preserved request for the target format when available. +/// +/// Replay is refused once the request IR has moved on from the body — a routing +/// algorithm having added a tier system prompt or a handoff note, say. Encoding +/// then falls through to normalized fields so the addition reaches the wire. pub fn exact_preserved_request( - preservation: &PreservationMetadata, + request: &LlmRequest, format: impl Into, policy: &TranslationPolicy, ) -> Option { + if !request.preserved_request_is_current() { + return None; + } let format = format.into(); (policy.preservation != PreservationPolicy::Disabled) - .then(|| preservation.requests.get(&format).cloned()) + .then(|| request.preservation.requests.get(&format).cloned()) .flatten() } diff --git a/crates/switchyard-translation/tests/extension_points.rs b/crates/switchyard-translation/tests/extension_points.rs index d7db6f41..9633147e 100644 --- a/crates/switchyard-translation/tests/extension_points.rs +++ b/crates/switchyard-translation/tests/extension_points.rs @@ -154,21 +154,23 @@ impl FormatCodec for MinimalCustomCodec { body: &Value, policy: &TranslationPolicy, ) -> switchyard_translation::Result { - Ok(DecodedRequest { - request: LlmRequest { - model: body - .get("model") + let mut request = LlmRequest { + model: body + .get("model") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + messages: vec![Message::text( + Role::User, + body.get("prompt") .and_then(Value::as_str) - .map(ToOwned::to_owned), - messages: vec![Message::text( - Role::User, - body.get("prompt") - .and_then(Value::as_str) - .unwrap_or_default(), - )], - preservation: capture_request_preservation(self.format(), body, policy), - ..LlmRequest::default() - }, + .unwrap_or_default(), + )], + preservation: capture_request_preservation(self.format(), body, policy), + ..LlmRequest::default() + }; + request.seal_preservation(); + Ok(DecodedRequest { + request, diagnostics: Vec::new(), }) } @@ -178,7 +180,7 @@ impl FormatCodec for MinimalCustomCodec { request: &LlmRequest, policy: &TranslationPolicy, ) -> switchyard_translation::Result { - if let Some(body) = exact_preserved_request(&request.preservation, self.format(), policy) { + if let Some(body) = exact_preserved_request(request, self.format(), policy) { return Ok(EncodedRequest { body, diagnostics: Vec::new(), diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs index bfcf7862..89c986d3 100644 --- a/crates/switchyard-translation/tests/request_translation.rs +++ b/crates/switchyard-translation/tests/request_translation.rs @@ -5,6 +5,7 @@ use pretty_assertions::assert_eq; use serde_json::{Value, json}; +use switchyard_protocol::{ContentBlock, InstructionBlock, Role}; use switchyard_translation::{TranslationEngine, TranslationPolicy, WireFormat}; type TestResult = std::result::Result<(), Box>; @@ -1490,3 +1491,110 @@ fn anthropic_thinking_is_dropped_from_responses_input() -> TestResult { ); Ok(()) } + +/// A same-format hop replays the exact inbound body, which is what makes it +/// lossless — but only while the IR still matches that body. Once a router has +/// added an instruction the body predates, replay would silently drop it, so +/// every codec must fall back to encoding from normalized fields. +/// +/// Regression test for SWITCH-1224. +#[test] +fn same_format_encoding_drops_exact_replay_once_the_ir_gains_an_instruction() -> TestResult { + const PROMPT: &str = "Respond with exactly EFFICIENT_SYSTEM_SENTINEL and nothing else."; + let engine = TranslationEngine::default(); + + for (format, inbound) in [ + ( + WireFormat::OpenAiChat, + json!({"model": "m", "messages": [{"role": "user", "content": "hi"}]}), + ), + ( + WireFormat::AnthropicMessages, + json!({"model": "m", "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}]}), + ), + ( + WireFormat::OpenAiResponses, + json!({"model": "m", "input": "hi"}), + ), + ] { + let policy = TranslationPolicy::default(); + let mut request = engine + .decode_request(format.clone(), &inbound, &policy)? + .request; + assert!( + request.preserved_request_is_current(), + "{format}: an untouched request should still replay exactly" + ); + + // What a routing algorithm does on the way out. + request.instructions.insert( + 0, + InstructionBlock { + role: Role::System, + content: vec![ContentBlock::Text { + text: PROMPT.to_string(), + }], + }, + ); + assert!( + !request.preserved_request_is_current(), + "{format}: the preserved body no longer describes this request" + ); + + let encoded = engine.encode_request(format.clone(), &request, &policy)?; + assert!( + encoded.body.to_string().contains(PROMPT), + "{format}: tier system prompt missing from the wire body: {}", + encoded.body + ); + } + Ok(()) +} + +/// The same guarantee across formats: a routed request that picked up an +/// instruction must carry it to any target, whether or not the target shares the +/// inbound format. Cross-format never had the same-format shortcut available, so +/// this pins the behaviour rather than fixing it. +#[test] +fn a_router_added_instruction_reaches_every_target_format() -> TestResult { + const PROMPT: &str = "stay on the settled plan"; + let engine = TranslationEngine::default(); + let formats = [ + WireFormat::OpenAiChat, + WireFormat::AnthropicMessages, + WireFormat::OpenAiResponses, + ]; + let inbound = |format: &WireFormat| match format { + WireFormat::AnthropicMessages => json!({ + "model": "m", "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}]}), + WireFormat::OpenAiResponses => json!({"model": "m", "input": "hi"}), + _ => json!({"model": "m", "messages": [{"role": "user", "content": "hi"}]}), + }; + + for source in &formats { + for target in &formats { + let policy = TranslationPolicy::default(); + let mut request = engine + .decode_request(source.clone(), &inbound(source), &policy)? + .request; + request.instructions.insert( + 0, + InstructionBlock { + role: Role::System, + content: vec![ContentBlock::Text { + text: PROMPT.to_string(), + }], + }, + ); + let encoded = engine.encode_request(target.clone(), &request, &policy)?; + assert!( + encoded.body.to_string().contains(PROMPT), + "{source} -> {target}: instruction missing from the wire body: {}", + encoded.body + ); + } + } + Ok(()) +}