From 9295eb4f022f366c796b961ab65f818cf89d249c Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 29 Jul 2026 17:47:08 -0600 Subject: [PATCH 01/51] feat(translation): preserve raw stream events Signed-off-by: Bryan Bednarski --- crates/protocol/src/stream.rs | 109 +++++++++++++++--- .../src/codecs/anthropic/stream.rs | 9 ++ .../src/codecs/openai_chat/stream.rs | 9 ++ .../src/codecs/responses/stream.rs | 9 ++ .../src/codecs/stream.rs | 15 +++ crates/switchyard-translation/src/engine.rs | 60 ++++++++++ .../tests/stream_translation.rs | 84 ++++++++++++++ 7 files changed, 277 insertions(+), 18 deletions(-) diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rs index 5fd82e687..2697cb90e 100644 --- a/crates/protocol/src/stream.rs +++ b/crates/protocol/src/stream.rs @@ -14,6 +14,7 @@ use serde_json::Value; use crate::{ LlmClientError, + format::FormatId, llm::{AggLlmResponse, ContentBlock, ResponseOutput, Role, StopReason, ToolCall, Usage}, }; @@ -58,21 +59,7 @@ impl LlmResponse { LlmResponse::Stream(mut stream) => { let mut accumulator = ResponseAccumulator::new(); while let Some(item) = stream.next().await { - match item? { - LlmResponseChunk::DecodeError { message } => { - return Err(LlmClientError::ResponseTranslation(message)); - } - LlmResponseChunk::StreamError { message } => { - // The upstream reported the failure inside the response body, so - // there is no real status line to carry; 502 stands in for "the - // upstream failed" the same way a failed non-streaming call would. - return Err(LlmClientError::UpstreamHttp { - status: MID_STREAM_UPSTREAM_STATUS, - body: message, - }); - } - chunk => accumulator.push(chunk), - } + push_checked_chunk(&mut accumulator, item?)?; } Ok(accumulator.finish()) } @@ -144,11 +131,48 @@ impl AggLlmResponse { } } -/// One provider-neutral streaming event — the normalized counterpart to -/// [`AggLlmResponse`](crate::AggLlmResponse), sitting between stream decoders and -/// encoders. `switchyard-translation` re-exports it as `ConversationStreamEvent`. +fn push_checked_chunk( + accumulator: &mut ResponseAccumulator, + chunk: LlmResponseChunk, +) -> Result<(), LlmClientError> { + match chunk { + LlmResponseChunk::ProviderEvent { normalized, .. } => { + for chunk in normalized { + push_checked_chunk(accumulator, chunk)?; + } + Ok(()) + } + LlmResponseChunk::DecodeError { message } => { + Err(LlmClientError::ResponseTranslation(message)) + } + LlmResponseChunk::StreamError { message } => Err(LlmClientError::UpstreamHttp { + status: MID_STREAM_UPSTREAM_STATUS, + body: message, + }), + chunk => { + accumulator.push(chunk); + Ok(()) + } + } +} + +/// One streaming event carried between a host and an algorithm. +/// +/// Normalized variants expose provider-neutral meaning. [`ProviderEvent`](Self::ProviderEvent) +/// additionally retains exact source JSON for a lossless same-format round trip while keeping +/// normalized children available to algorithms and cross-format encoders. +/// `switchyard-translation` re-exports this type as `ConversationStreamEvent`. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum LlmResponseChunk { + /// One exact provider event paired with the neutral events decoded from it. + ProviderEvent { + /// Provider format that produced `raw`. + source: FormatId, + /// Exact parsed provider event. + raw: Value, + /// Provider-neutral events decoded from `raw`, in source order. + normalized: Vec, + }, MessageStart { id: Option, model: Option, @@ -216,6 +240,11 @@ impl ResponseAccumulator { /// earlier ones; text, reasoning, and tool-call arguments append. pub fn push(&mut self, chunk: LlmResponseChunk) { match chunk { + LlmResponseChunk::ProviderEvent { normalized, .. } => { + for chunk in normalized { + self.push(chunk); + } + } LlmResponseChunk::MessageStart { id, model } => { if id.is_some() { self.id = id; @@ -361,6 +390,50 @@ mod tests { ); } + #[test] + fn folds_normalized_chunks_inside_provider_event() { + let aggregate = fold(vec![LlmResponseChunk::ProviderEvent { + source: crate::WireFormat::OpenAiChat.into(), + raw: json!({ + "choices": [{"delta": {"content": "hello"}}], + "system_fingerprint": "fp_exact" + }), + normalized: vec![LlmResponseChunk::TextDelta { + index: 0, + text: "hello".to_string(), + }], + }]); + + assert_eq!( + aggregate.outputs[0].content, + vec![ContentBlock::Text { + text: "hello".to_string() + }] + ); + } + + #[test] + fn stream_errors_inside_provider_events_remain_typed() { + let response = LlmResponse::Stream(Box::pin(stream::iter([Ok( + LlmResponseChunk::ProviderEvent { + source: crate::WireFormat::OpenAiChat.into(), + raw: json!({"error": {"message": "provider failed"}}), + normalized: vec![LlmResponseChunk::StreamError { + message: "provider failed".to_string(), + }], + }, + )]))); + + let error = block_on(response.into_agg()).err(); + assert!(matches!( + error, + Some(LlmClientError::UpstreamHttp { + status: MID_STREAM_UPSTREAM_STATUS, + .. + }) + )); + } + #[test] fn assembles_tool_calls_by_index() { // id/name arrive once, arguments stream across deltas and parse as JSON. diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rs index 0153f8ef9..9f9423a6f 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs @@ -126,6 +126,15 @@ fn encode_anthropic_stream( event: LlmResponseChunk, ) -> Vec { match event { + LlmResponseChunk::ProviderEvent { + source, + raw, + normalized: _, + } if source == WireFormat::AnthropicMessages.into() => vec![raw], + LlmResponseChunk::ProviderEvent { normalized, .. } => normalized + .into_iter() + .flat_map(|event| encode_anthropic_stream(state, event)) + .collect(), LlmResponseChunk::MessageStart { id, model } => { record_source_identity(state, id, model); if state.emitted_message_start { diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs index b11922e07..dd5068151 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs @@ -154,6 +154,15 @@ fn encode_openai_chat_stream( event: LlmResponseChunk, ) -> Vec { match event { + LlmResponseChunk::ProviderEvent { + source, + raw, + normalized: _, + } if source == WireFormat::OpenAiChat.into() => vec![raw], + LlmResponseChunk::ProviderEvent { normalized, .. } => normalized + .into_iter() + .flat_map(|event| encode_openai_chat_stream(state, event)) + .collect(), LlmResponseChunk::MessageStart { id, model } => { record_source_identity(state, id, model); if state.emitted_message_start diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs index 31b269ce2..8bafa1ed9 100644 --- a/crates/switchyard-translation/src/codecs/responses/stream.rs +++ b/crates/switchyard-translation/src/codecs/responses/stream.rs @@ -154,6 +154,15 @@ fn encode_responses_stream( event: LlmResponseChunk, ) -> Vec { match event { + LlmResponseChunk::ProviderEvent { + source, + raw, + normalized: _, + } if source == WireFormat::OpenAiResponses.into() => vec![raw], + LlmResponseChunk::ProviderEvent { normalized, .. } => normalized + .into_iter() + .flat_map(|event| encode_responses_stream(state, event)) + .collect(), LlmResponseChunk::MessageStart { id, model } => { record_source_identity(state, id, model); ensure_responses_created(state) diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs index 318be86e8..a9f179acd 100644 --- a/crates/switchyard-translation/src/codecs/stream.rs +++ b/crates/switchyard-translation/src/codecs/stream.rs @@ -215,6 +215,21 @@ pub fn decode_stream_event( }) } +/// Decodes one provider stream event and retains its exact source JSON. +pub fn decode_stream_event_preserving( + state: &mut StreamTranslationState, + source: impl Into, + event: &Value, +) -> LlmResponseChunk { + let source = source.into(); + state.source = Some(source.clone()); + LlmResponseChunk::ProviderEvent { + source: source.clone(), + raw: event.clone(), + normalized: decode_stream_event(state, source, event), + } +} + /// Encodes one neutral stream event with the built-in codec registry. pub fn encode_stream_event( state: &mut StreamTranslationState, diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rs index e7f646d9d..3c3ba383e 100644 --- a/crates/switchyard-translation/src/engine.rs +++ b/crates/switchyard-translation/src/engine.rs @@ -18,6 +18,7 @@ use crate::error::{Result, TranslationError}; use crate::format::FormatId; use crate::llm::{AggLlmResponse, LlmRequest}; use crate::policy::TranslationPolicy; +use crate::LlmResponseChunk; /// Encoded translation result with any diagnostics emitted along the way. #[derive(Debug)] @@ -243,6 +244,45 @@ impl TranslationEngine { .collect()) } + /// Decodes one provider event while retaining the exact source JSON. + pub fn decode_stream_event( + &self, + state: &mut StreamTranslationState, + source: impl Into, + event: &Value, + ) -> Result { + let source = source.into(); + let source_codec = self.stream_registry.codec(source.clone())?; + state.source = Some(source.clone()); + Ok(LlmResponseChunk::ProviderEvent { + source, + raw: event.clone(), + normalized: source_codec.decode_event(state, event), + }) + } + + /// Encodes one neutral or preserved stream event for a target provider. + /// + /// A preserved event is replayed exactly when its source and target formats + /// match. Cross-format encoding intentionally uses only its normalized + /// events. + pub fn encode_stream_event( + &self, + state: &mut StreamTranslationState, + target: impl Into, + event: LlmResponseChunk, + ) -> Result> { + let target = target.into(); + let target_codec = self.stream_registry.codec(target.clone())?; + state.target = Some(target.clone()); + Ok(encode_stream_chunk( + state, + target_codec.as_ref(), + &target, + event, + )) + } + /// Finishes target-provider stream emission after the source stream closes. pub fn finish_stream( &self, @@ -255,6 +295,26 @@ impl TranslationEngine { } } +fn encode_stream_chunk( + state: &mut StreamTranslationState, + target_codec: &dyn crate::codecs::stream::StreamCodec, + target: &FormatId, + event: LlmResponseChunk, +) -> Vec { + match event { + LlmResponseChunk::ProviderEvent { + source, + raw, + normalized: _, + } if &source == target => vec![raw], + LlmResponseChunk::ProviderEvent { normalized, .. } => normalized + .into_iter() + .flat_map(|event| encode_stream_chunk(state, target_codec, target, event)) + .collect(), + event => target_codec.encode_event(state, event), + } +} + // Attaches source and target formats to every diagnostic emitted across both passes. fn with_formats( decoded: Vec, diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs index 38a6f628c..e67b9140a 100644 --- a/crates/switchyard-translation/tests/stream_translation.rs +++ b/crates/switchyard-translation/tests/stream_translation.rs @@ -8,10 +8,94 @@ use serde_json::json; use switchyard_protocol::{ResponseAccumulator, StopReason}; use switchyard_translation::{ LlmResponseChunk, StreamTranslationState, TranslationEngine, WireFormat, decode_stream_event, + decode_stream_event_preserving, encode_stream_event, }; type TestResult = std::result::Result<(), Box>; +#[test] +fn preserved_same_format_events_replay_unknown_fields_exactly() -> TestResult { + let cases = [ + ( + WireFormat::OpenAiChat, + json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "gpt-4o", + "system_fingerprint": "fp_provider_specific", + "choices": [{ + "index": 0, + "delta": {"content": "Hi"}, + "finish_reason": null + }] + }), + ), + ( + WireFormat::OpenAiResponses, + json!({ + "type": "response.output_text.delta", + "item_id": "item-1", + "output_index": 0, + "content_index": 0, + "delta": "Hi", + "sequence_number": 2, + "provider_extension": {"exact": true} + }), + ), + ( + WireFormat::AnthropicMessages, + json!({ + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "Hi"}, + "provider_extension": {"exact": true} + }), + ), + ]; + + for (format, event) in cases { + let engine = TranslationEngine::default(); + let mut state = StreamTranslationState::new(format, format); + let preserved = engine.decode_stream_event(&mut state, format, &event)?; + let replayed = engine.encode_stream_event(&mut state, format, preserved)?; + assert_eq!(replayed, vec![event.clone()]); + + let mut state = StreamTranslationState::new(format, format); + let preserved = decode_stream_event_preserving(&mut state, format, &event); + let replayed = encode_stream_event(&mut state, format, preserved); + assert_eq!(replayed, vec![event]); + } + Ok(()) +} + +#[test] +fn preserved_cross_format_event_uses_normalized_content() -> TestResult { + let engine = TranslationEngine::default(); + let mut state = + StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::AnthropicMessages); + let event = json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "gpt-4o", + "system_fingerprint": "fp_provider_specific", + "choices": [{ + "index": 0, + "delta": {"content": "Hi"}, + "finish_reason": null + }] + }); + + let preserved = engine.decode_stream_event(&mut state, WireFormat::OpenAiChat, &event)?; + let translated = + engine.encode_stream_event(&mut state, WireFormat::AnthropicMessages, preserved)?; + + assert_eq!(translated[2]["delta"]["text"], "Hi"); + assert!(translated + .iter() + .all(|event| event.get("system_fingerprint").is_none())); + Ok(()) +} + // Verifies an OpenAI text delta opens the expected Anthropic message and content blocks. #[test] fn openai_chat_stream_event_translates_to_anthropic_message_events() -> TestResult { From b4df242fa4ada5f4f7494e4731825b281c605b27 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 30 Jul 2026 08:27:06 -0600 Subject: [PATCH 02/51] docs(translation): describe stream preservation as parsed-JSON-value replay Signed-off-by: Bryan Bednarski --- crates/protocol/src/stream.rs | 9 +++++---- crates/switchyard-translation/src/codecs/stream.rs | 2 +- crates/switchyard-translation/src/engine.rs | 8 ++++---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rs index 2697cb90e..ceb9569f7 100644 --- a/crates/protocol/src/stream.rs +++ b/crates/protocol/src/stream.rs @@ -159,16 +159,17 @@ fn push_checked_chunk( /// One streaming event carried between a host and an algorithm. /// /// Normalized variants expose provider-neutral meaning. [`ProviderEvent`](Self::ProviderEvent) -/// additionally retains exact source JSON for a lossless same-format round trip while keeping -/// normalized children available to algorithms and cross-format encoders. +/// additionally retains the parsed source JSON value, so a same-format round trip replays that +/// value rather than the original bytes: SSE framing, whitespace, and object key order are not +/// preserved. Normalized children stay available to algorithms and cross-format encoders. /// `switchyard-translation` re-exports this type as `ConversationStreamEvent`. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum LlmResponseChunk { - /// One exact provider event paired with the neutral events decoded from it. + /// One preserved provider event paired with the neutral events decoded from it. ProviderEvent { /// Provider format that produced `raw`. source: FormatId, - /// Exact parsed provider event. + /// Source event as a parsed JSON value. raw: Value, /// Provider-neutral events decoded from `raw`, in source order. normalized: Vec, diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs index a9f179acd..fb7b160d0 100644 --- a/crates/switchyard-translation/src/codecs/stream.rs +++ b/crates/switchyard-translation/src/codecs/stream.rs @@ -215,7 +215,7 @@ pub fn decode_stream_event( }) } -/// Decodes one provider stream event and retains its exact source JSON. +/// Decodes one provider stream event and retains its parsed source JSON value. pub fn decode_stream_event_preserving( state: &mut StreamTranslationState, source: impl Into, diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rs index 3c3ba383e..5e4ab6cbd 100644 --- a/crates/switchyard-translation/src/engine.rs +++ b/crates/switchyard-translation/src/engine.rs @@ -244,7 +244,7 @@ impl TranslationEngine { .collect()) } - /// Decodes one provider event while retaining the exact source JSON. + /// Decodes one provider event while retaining its parsed source JSON value. pub fn decode_stream_event( &self, state: &mut StreamTranslationState, @@ -263,9 +263,9 @@ impl TranslationEngine { /// Encodes one neutral or preserved stream event for a target provider. /// - /// A preserved event is replayed exactly when its source and target formats - /// match. Cross-format encoding intentionally uses only its normalized - /// events. + /// A preserved event replays its retained JSON value unchanged when its + /// source and target formats match. Cross-format encoding intentionally + /// uses only its normalized events. pub fn encode_stream_event( &self, state: &mut StreamTranslationState, From c2f089d32750b02ddf2c091ef94e8dd6cd3b8de3 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 30 Jul 2026 10:06:15 -0600 Subject: [PATCH 03/51] fix(translation): suppress synthesized finish after a replayed terminal event Signed-off-by: Bryan Bednarski --- .../src/codecs/anthropic/stream.rs | 12 +++- .../src/codecs/openai_chat/stream.rs | 11 +-- .../src/codecs/responses/stream.rs | 12 +++- .../src/codecs/stream.rs | 15 ++++ crates/switchyard-translation/src/engine.rs | 11 ++- .../tests/stream_translation.rs | 68 +++++++++++++++++++ 6 files changed, 116 insertions(+), 13 deletions(-) diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rs index 9f9423a6f..ab0681820 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs @@ -7,7 +7,7 @@ use serde_json::{Map, Value, json}; use crate::LlmResponseChunk; use crate::codecs::stream::{ - StreamCodec, StreamTranslationState, record_source_identity, + StreamCodec, StreamTranslationState, mark_replayed_terminal, record_source_identity, target_message_id_or_source_message_id, target_model_or_source_model, }; use crate::format::{FormatId, WireFormat}; @@ -129,8 +129,11 @@ fn encode_anthropic_stream( LlmResponseChunk::ProviderEvent { source, raw, - normalized: _, - } if source == WireFormat::AnthropicMessages.into() => vec![raw], + normalized, + } if source == WireFormat::AnthropicMessages.into() => { + mark_replayed_terminal(state, &normalized); + vec![raw] + } LlmResponseChunk::ProviderEvent { normalized, .. } => normalized .into_iter() .flat_map(|event| encode_anthropic_stream(state, event)) @@ -198,6 +201,9 @@ fn encode_anthropic_stream( // Emits any missing Anthropic terminal events and closes open content blocks. fn finish_anthropic_stream(state: &mut StreamTranslationState) -> Vec { + if state.finished { + return Vec::new(); + } let mut out = Vec::new(); if !state.emitted_message_start { out.extend(encode_anthropic_stream( diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs index dd5068151..361435c1f 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs @@ -7,8 +7,8 @@ use serde_json::{Map, Value, json}; use crate::LlmResponseChunk; use crate::codecs::stream::{ - StreamCodec, StreamTranslationState, record_source_identity, state_source_is, string_field, - target_model_or_source_model, + StreamCodec, StreamTranslationState, mark_replayed_terminal, record_source_identity, + state_source_is, string_field, target_model_or_source_model, }; use crate::format::{FormatId, WireFormat}; use crate::llm::Usage; @@ -157,8 +157,11 @@ fn encode_openai_chat_stream( LlmResponseChunk::ProviderEvent { source, raw, - normalized: _, - } if source == WireFormat::OpenAiChat.into() => vec![raw], + normalized, + } if source == WireFormat::OpenAiChat.into() => { + mark_replayed_terminal(state, &normalized); + vec![raw] + } LlmResponseChunk::ProviderEvent { normalized, .. } => normalized .into_iter() .flat_map(|event| encode_openai_chat_stream(state, event)) diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs index 8bafa1ed9..6197c8116 100644 --- a/crates/switchyard-translation/src/codecs/responses/stream.rs +++ b/crates/switchyard-translation/src/codecs/responses/stream.rs @@ -7,7 +7,7 @@ use serde_json::{Value, json}; use crate::LlmResponseChunk; use crate::codecs::stream::{ - StreamCodec, StreamTranslationState, record_source_identity, + StreamCodec, StreamTranslationState, mark_replayed_terminal, record_source_identity, target_message_id_or_source_message_id, target_model_or_source_model, }; use crate::format::{FormatId, WireFormat}; @@ -157,8 +157,11 @@ fn encode_responses_stream( LlmResponseChunk::ProviderEvent { source, raw, - normalized: _, - } if source == WireFormat::OpenAiResponses.into() => vec![raw], + normalized, + } if source == WireFormat::OpenAiResponses.into() => { + mark_replayed_terminal(state, &normalized); + vec![raw] + } LlmResponseChunk::ProviderEvent { normalized, .. } => normalized .into_iter() .flat_map(|event| encode_responses_stream(state, event)) @@ -194,6 +197,9 @@ fn encode_responses_stream( // Emits final OpenAI Responses completion events from accumulated state. fn finish_responses_stream(state: &mut StreamTranslationState) -> Vec { + if state.finished { + return Vec::new(); + } let mut out = ensure_responses_created(state); if state.response_text_started && let Some(output_index) = state.response_text_output_index diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs index fb7b160d0..96884e5a4 100644 --- a/crates/switchyard-translation/src/codecs/stream.rs +++ b/crates/switchyard-translation/src/codecs/stream.rs @@ -215,6 +215,21 @@ pub fn decode_stream_event( }) } +// Replaying a preserved event emits its retained JSON without running the encoder, so the +// terminal bookkeeping the encoder would have done has to happen here. Without it `finish` +// still believes the stream is unterminated and synthesizes a second terminal sequence. +pub(crate) fn mark_replayed_terminal( + state: &mut StreamTranslationState, + normalized: &[LlmResponseChunk], +) { + if normalized + .iter() + .any(|event| matches!(event, LlmResponseChunk::MessageStop { .. })) + { + state.finished = true; + } +} + /// Decodes one provider stream event and retains its parsed source JSON value. pub fn decode_stream_event_preserving( state: &mut StreamTranslationState, diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rs index 5e4ab6cbd..01eaa9a46 100644 --- a/crates/switchyard-translation/src/engine.rs +++ b/crates/switchyard-translation/src/engine.rs @@ -12,7 +12,9 @@ use crate::codecs::FormatCodec; use crate::codecs::anthropic::AnthropicMessagesCodec; use crate::codecs::openai_chat::OpenAiChatCodec; use crate::codecs::responses::OpenAiResponsesCodec; -use crate::codecs::stream::{StreamCodecRegistry, StreamTranslationState}; +use crate::codecs::stream::{ + StreamCodecRegistry, StreamTranslationState, mark_replayed_terminal, +}; use crate::diagnostic::TranslationDiagnostic; use crate::error::{Result, TranslationError}; use crate::format::FormatId; @@ -305,8 +307,11 @@ fn encode_stream_chunk( LlmResponseChunk::ProviderEvent { source, raw, - normalized: _, - } if &source == target => vec![raw], + normalized, + } if &source == target => { + mark_replayed_terminal(state, &normalized); + vec![raw] + } LlmResponseChunk::ProviderEvent { normalized, .. } => normalized .into_iter() .flat_map(|event| encode_stream_chunk(state, target_codec, target, event)) diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs index e67b9140a..7dfaf6901 100644 --- a/crates/switchyard-translation/tests/stream_translation.rs +++ b/crates/switchyard-translation/tests/stream_translation.rs @@ -68,6 +68,74 @@ fn preserved_same_format_events_replay_unknown_fields_exactly() -> TestResult { Ok(()) } +// Replay emits the preserved event without running the encoder, so the encoder never sees the +// stop it would normally record. Both replay paths must still leave the stream marked finished +// or `finish_stream` synthesizes a terminal the client already received. +#[test] +fn replayed_terminal_event_suppresses_synthesized_finish() -> TestResult { + let cases = [ + ( + WireFormat::OpenAiChat, + vec![ + json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "gpt-4o", + "choices": [{"index": 0, "delta": {"content": "Hi"}, "finish_reason": null}] + }), + json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "gpt-4o", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}] + }), + ], + ), + ( + WireFormat::AnthropicMessages, + vec![ + json!({"type": "message_start", "message": {"id": "msg_1", "model": "claude"}}), + json!({"type": "message_stop"}), + ], + ), + ( + WireFormat::OpenAiResponses, + vec![ + json!({"type": "response.created", "response": {"id": "resp_1", "model": "gpt-4o"}}), + json!({"type": "response.completed", "response": {"id": "resp_1", "model": "gpt-4o"}}), + ], + ), + ]; + + let engine = TranslationEngine::default(); + for (format, events) in cases { + let mut state = StreamTranslationState::new(format, format); + let mut replayed = Vec::new(); + for event in &events { + let preserved = engine.decode_stream_event(&mut state, format, event)?; + replayed.extend(engine.encode_stream_event(&mut state, format, preserved)?); + } + assert_eq!(replayed, events, "{format:?} engine replay"); + assert!( + engine.finish_stream(&mut state, format)?.is_empty(), + "{format:?} engine replay already delivered a terminal event", + ); + + let mut state = StreamTranslationState::new(format, format); + let mut replayed = Vec::new(); + for event in &events { + let preserved = decode_stream_event_preserving(&mut state, format, event); + replayed.extend(encode_stream_event(&mut state, format, preserved)); + } + assert_eq!(replayed, events, "{format:?} codec replay"); + assert!( + engine.finish_stream(&mut state, format)?.is_empty(), + "{format:?} codec replay already delivered a terminal event", + ); + } + Ok(()) +} + #[test] fn preserved_cross_format_event_uses_normalized_content() -> TestResult { let engine = TranslationEngine::default(); From 42e23cef12a565cb43db396095ad77506284ea72 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 30 Jul 2026 10:37:22 -0600 Subject: [PATCH 04/51] perf(translation): move preserved stream events Signed-off-by: Bryan Bednarski --- crates/switchyard-translation/src/codecs/stream.rs | 12 ++++++++---- crates/switchyard-translation/src/engine.rs | 10 +++++++--- .../tests/stream_translation.rs | 10 +++++----- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs index 96884e5a4..c3e19bc6b 100644 --- a/crates/switchyard-translation/src/codecs/stream.rs +++ b/crates/switchyard-translation/src/codecs/stream.rs @@ -231,17 +231,21 @@ pub(crate) fn mark_replayed_terminal( } /// Decodes one provider stream event and retains its parsed source JSON value. +/// +/// Takes ownership of `event` so preservation does not deep-copy provider JSON +/// on the per-event streaming path. pub fn decode_stream_event_preserving( state: &mut StreamTranslationState, source: impl Into, - event: &Value, + event: Value, ) -> LlmResponseChunk { let source = source.into(); state.source = Some(source.clone()); + let normalized = decode_stream_event(state, source.clone(), &event); LlmResponseChunk::ProviderEvent { - source: source.clone(), - raw: event.clone(), - normalized: decode_stream_event(state, source, event), + source, + raw: event, + normalized, } } diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rs index 01eaa9a46..78f2b7e4b 100644 --- a/crates/switchyard-translation/src/engine.rs +++ b/crates/switchyard-translation/src/engine.rs @@ -247,19 +247,23 @@ impl TranslationEngine { } /// Decodes one provider event while retaining its parsed source JSON value. + /// + /// Takes ownership of `event` so preservation does not deep-copy provider JSON + /// on the per-event streaming path. pub fn decode_stream_event( &self, state: &mut StreamTranslationState, source: impl Into, - event: &Value, + event: Value, ) -> Result { let source = source.into(); let source_codec = self.stream_registry.codec(source.clone())?; state.source = Some(source.clone()); + let normalized = source_codec.decode_event(state, &event); Ok(LlmResponseChunk::ProviderEvent { source, - raw: event.clone(), - normalized: source_codec.decode_event(state, event), + raw: event, + normalized, }) } diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs index 7dfaf6901..4d4c25b19 100644 --- a/crates/switchyard-translation/tests/stream_translation.rs +++ b/crates/switchyard-translation/tests/stream_translation.rs @@ -56,12 +56,12 @@ fn preserved_same_format_events_replay_unknown_fields_exactly() -> TestResult { for (format, event) in cases { let engine = TranslationEngine::default(); let mut state = StreamTranslationState::new(format, format); - let preserved = engine.decode_stream_event(&mut state, format, &event)?; + let preserved = engine.decode_stream_event(&mut state, format, event.clone())?; let replayed = engine.encode_stream_event(&mut state, format, preserved)?; assert_eq!(replayed, vec![event.clone()]); let mut state = StreamTranslationState::new(format, format); - let preserved = decode_stream_event_preserving(&mut state, format, &event); + let preserved = decode_stream_event_preserving(&mut state, format, event.clone()); let replayed = encode_stream_event(&mut state, format, preserved); assert_eq!(replayed, vec![event]); } @@ -112,7 +112,7 @@ fn replayed_terminal_event_suppresses_synthesized_finish() -> TestResult { let mut state = StreamTranslationState::new(format, format); let mut replayed = Vec::new(); for event in &events { - let preserved = engine.decode_stream_event(&mut state, format, event)?; + let preserved = engine.decode_stream_event(&mut state, format, event.clone())?; replayed.extend(engine.encode_stream_event(&mut state, format, preserved)?); } assert_eq!(replayed, events, "{format:?} engine replay"); @@ -124,7 +124,7 @@ fn replayed_terminal_event_suppresses_synthesized_finish() -> TestResult { let mut state = StreamTranslationState::new(format, format); let mut replayed = Vec::new(); for event in &events { - let preserved = decode_stream_event_preserving(&mut state, format, event); + let preserved = decode_stream_event_preserving(&mut state, format, event.clone()); replayed.extend(encode_stream_event(&mut state, format, preserved)); } assert_eq!(replayed, events, "{format:?} codec replay"); @@ -153,7 +153,7 @@ fn preserved_cross_format_event_uses_normalized_content() -> TestResult { }] }); - let preserved = engine.decode_stream_event(&mut state, WireFormat::OpenAiChat, &event)?; + let preserved = engine.decode_stream_event(&mut state, WireFormat::OpenAiChat, event)?; let translated = engine.encode_stream_event(&mut state, WireFormat::AnthropicMessages, preserved)?; From 1c160a2664d73de30f82ed22aa3eb5b7fba8b660 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 30 Jul 2026 10:43:07 -0600 Subject: [PATCH 05/51] refactor(translation): centralize preserved stream dispatch Signed-off-by: Bryan Bednarski --- .../src/codecs/anthropic/stream.rs | 15 ++-------- .../src/codecs/openai_chat/stream.rs | 17 ++--------- .../src/codecs/responses/stream.rs | 15 ++-------- .../src/codecs/stream.rs | 28 +++++++++++++++++-- crates/switchyard-translation/src/engine.rs | 25 +---------------- .../tests/extension_points.rs | 20 +++++++++++++ 6 files changed, 54 insertions(+), 66 deletions(-) diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rs index ab0681820..7f4f45a93 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs @@ -7,7 +7,7 @@ use serde_json::{Map, Value, json}; use crate::LlmResponseChunk; use crate::codecs::stream::{ - StreamCodec, StreamTranslationState, mark_replayed_terminal, record_source_identity, + StreamCodec, StreamTranslationState, record_source_identity, target_message_id_or_source_message_id, target_model_or_source_model, }; use crate::format::{FormatId, WireFormat}; @@ -126,18 +126,6 @@ fn encode_anthropic_stream( event: LlmResponseChunk, ) -> Vec { match event { - LlmResponseChunk::ProviderEvent { - source, - raw, - normalized, - } if source == WireFormat::AnthropicMessages.into() => { - mark_replayed_terminal(state, &normalized); - vec![raw] - } - LlmResponseChunk::ProviderEvent { normalized, .. } => normalized - .into_iter() - .flat_map(|event| encode_anthropic_stream(state, event)) - .collect(), LlmResponseChunk::MessageStart { id, model } => { record_source_identity(state, id, model); if state.emitted_message_start { @@ -196,6 +184,7 @@ fn encode_anthropic_stream( LlmResponseChunk::StreamError { message } | LlmResponseChunk::DecodeError { message } => { vec![json!({"type": "error", "error": {"message": message}})] } + LlmResponseChunk::ProviderEvent { .. } => Vec::new(), } } diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs index 361435c1f..059f72a33 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs @@ -7,8 +7,8 @@ use serde_json::{Map, Value, json}; use crate::LlmResponseChunk; use crate::codecs::stream::{ - StreamCodec, StreamTranslationState, mark_replayed_terminal, record_source_identity, - state_source_is, string_field, target_model_or_source_model, + StreamCodec, StreamTranslationState, record_source_identity, state_source_is, string_field, + target_model_or_source_model, }; use crate::format::{FormatId, WireFormat}; use crate::llm::Usage; @@ -154,18 +154,6 @@ fn encode_openai_chat_stream( event: LlmResponseChunk, ) -> Vec { match event { - LlmResponseChunk::ProviderEvent { - source, - raw, - normalized, - } if source == WireFormat::OpenAiChat.into() => { - mark_replayed_terminal(state, &normalized); - vec![raw] - } - LlmResponseChunk::ProviderEvent { normalized, .. } => normalized - .into_iter() - .flat_map(|event| encode_openai_chat_stream(state, event)) - .collect(), LlmResponseChunk::MessageStart { id, model } => { record_source_identity(state, id, model); if state.emitted_message_start @@ -235,6 +223,7 @@ fn encode_openai_chat_stream( LlmResponseChunk::DecodeError { message } | LlmResponseChunk::StreamError { message } => { vec![json!({"error": {"message": message}})] } + LlmResponseChunk::ProviderEvent { .. } => Vec::new(), } } diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs index 6197c8116..a44ae6d1d 100644 --- a/crates/switchyard-translation/src/codecs/responses/stream.rs +++ b/crates/switchyard-translation/src/codecs/responses/stream.rs @@ -7,7 +7,7 @@ use serde_json::{Value, json}; use crate::LlmResponseChunk; use crate::codecs::stream::{ - StreamCodec, StreamTranslationState, mark_replayed_terminal, record_source_identity, + StreamCodec, StreamTranslationState, record_source_identity, target_message_id_or_source_message_id, target_model_or_source_model, }; use crate::format::{FormatId, WireFormat}; @@ -154,18 +154,6 @@ fn encode_responses_stream( event: LlmResponseChunk, ) -> Vec { match event { - LlmResponseChunk::ProviderEvent { - source, - raw, - normalized, - } if source == WireFormat::OpenAiResponses.into() => { - mark_replayed_terminal(state, &normalized); - vec![raw] - } - LlmResponseChunk::ProviderEvent { normalized, .. } => normalized - .into_iter() - .flat_map(|event| encode_responses_stream(state, event)) - .collect(), LlmResponseChunk::MessageStart { id, model } => { record_source_identity(state, id, model); ensure_responses_created(state) @@ -192,6 +180,7 @@ fn encode_responses_stream( LlmResponseChunk::DecodeError { message } | LlmResponseChunk::StreamError { message } => { vec![json!({"type": "error", "message": message})] } + LlmResponseChunk::ProviderEvent { .. } => Vec::new(), } } diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs index c3e19bc6b..2785a9620 100644 --- a/crates/switchyard-translation/src/codecs/stream.rs +++ b/crates/switchyard-translation/src/codecs/stream.rs @@ -230,6 +230,29 @@ pub(crate) fn mark_replayed_terminal( } } +pub(crate) fn encode_stream_chunk( + state: &mut StreamTranslationState, + target_codec: &dyn StreamCodec, + target: &FormatId, + event: LlmResponseChunk, +) -> Vec { + match event { + LlmResponseChunk::ProviderEvent { + source, + raw, + normalized, + } if &source == target => { + mark_replayed_terminal(state, &normalized); + vec![raw] + } + LlmResponseChunk::ProviderEvent { normalized, .. } => normalized + .into_iter() + .flat_map(|event| encode_stream_chunk(state, target_codec, target, event)) + .collect(), + event => target_codec.encode_event(state, event), + } +} + /// Decodes one provider stream event and retains its parsed source JSON value. /// /// Takes ownership of `event` so preservation does not deep-copy provider JSON @@ -255,9 +278,10 @@ pub fn encode_stream_event( target: impl Into, event: LlmResponseChunk, ) -> Vec { + let target = target.into(); StreamCodecRegistry::with_builtins() - .codec(target) - .map(|codec| codec.encode_event(state, event)) + .codec(target.clone()) + .map(|codec| encode_stream_chunk(state, codec.as_ref(), &target, event)) .unwrap_or_else(|error| vec![json!({"error": {"message": error.to_string()}})]) } diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rs index 78f2b7e4b..0fd97867b 100644 --- a/crates/switchyard-translation/src/engine.rs +++ b/crates/switchyard-translation/src/engine.rs @@ -13,7 +13,7 @@ use crate::codecs::anthropic::AnthropicMessagesCodec; use crate::codecs::openai_chat::OpenAiChatCodec; use crate::codecs::responses::OpenAiResponsesCodec; use crate::codecs::stream::{ - StreamCodecRegistry, StreamTranslationState, mark_replayed_terminal, + StreamCodecRegistry, StreamTranslationState, encode_stream_chunk, }; use crate::diagnostic::TranslationDiagnostic; use crate::error::{Result, TranslationError}; @@ -301,29 +301,6 @@ impl TranslationEngine { } } -fn encode_stream_chunk( - state: &mut StreamTranslationState, - target_codec: &dyn crate::codecs::stream::StreamCodec, - target: &FormatId, - event: LlmResponseChunk, -) -> Vec { - match event { - LlmResponseChunk::ProviderEvent { - source, - raw, - normalized, - } if &source == target => { - mark_replayed_terminal(state, &normalized); - vec![raw] - } - LlmResponseChunk::ProviderEvent { normalized, .. } => normalized - .into_iter() - .flat_map(|event| encode_stream_chunk(state, target_codec, target, event)) - .collect(), - event => target_codec.encode_event(state, event), - } -} - // Attaches source and target formats to every diagnostic emitted across both passes. fn with_formats( decoded: Vec, diff --git a/crates/switchyard-translation/tests/extension_points.rs b/crates/switchyard-translation/tests/extension_points.rs index 3ba377fcd..d7db6f41d 100644 --- a/crates/switchyard-translation/tests/extension_points.rs +++ b/crates/switchyard-translation/tests/extension_points.rs @@ -86,6 +86,26 @@ fn custom_stream_codec_can_participate_in_registered_stream_translation() -> Tes Ok(()) } +#[test] +fn custom_stream_codec_replays_preserved_same_format_event() -> TestResult { + let mut registry = StreamCodecRegistry::new(); + registry.register(CustomStreamCodec); + let engine = TranslationEngine::with_registries(FormatRegistry::new(), registry); + let format = FormatId::new("custom_stream"); + let mut state = StreamTranslationState::new(format.clone(), format.clone()); + let event = json!({ + "kind": "delta", + "text": "hello", + "vendor_only": {"must": "survive"} + }); + + let preserved = engine.decode_stream_event(&mut state, format.clone(), event.clone())?; + let replayed = engine.encode_stream_event(&mut state, format, preserved)?; + + assert_eq!(replayed, vec![event]); + Ok(()) +} + // Verifies target capability policy can reject unsupported request features. #[test] fn capability_profile_can_fail_fast_when_target_cannot_accept_request_features() { From d53b4ffa9e9690f6947654ecba329397331bf6a7 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 30 Jul 2026 10:44:20 -0600 Subject: [PATCH 06/51] refactor(translation): trim preserving stream API Keep preservation on TranslationEngine, the API consumed by NVIDIA/NeMo-Relay#586, and remove the unused built-in convenience decoder. Signed-off-by: Bryan Bednarski --- .../src/codecs/stream.rs | 19 ---------------- .../tests/stream_translation.rs | 22 ++----------------- 2 files changed, 2 insertions(+), 39 deletions(-) diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs index 2785a9620..8d8344c6a 100644 --- a/crates/switchyard-translation/src/codecs/stream.rs +++ b/crates/switchyard-translation/src/codecs/stream.rs @@ -253,25 +253,6 @@ pub(crate) fn encode_stream_chunk( } } -/// Decodes one provider stream event and retains its parsed source JSON value. -/// -/// Takes ownership of `event` so preservation does not deep-copy provider JSON -/// on the per-event streaming path. -pub fn decode_stream_event_preserving( - state: &mut StreamTranslationState, - source: impl Into, - event: Value, -) -> LlmResponseChunk { - let source = source.into(); - state.source = Some(source.clone()); - let normalized = decode_stream_event(state, source.clone(), &event); - LlmResponseChunk::ProviderEvent { - source, - raw: event, - normalized, - } -} - /// Encodes one neutral stream event with the built-in codec registry. pub fn encode_stream_event( state: &mut StreamTranslationState, diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs index 4d4c25b19..67b96cd09 100644 --- a/crates/switchyard-translation/tests/stream_translation.rs +++ b/crates/switchyard-translation/tests/stream_translation.rs @@ -8,7 +8,6 @@ use serde_json::json; use switchyard_protocol::{ResponseAccumulator, StopReason}; use switchyard_translation::{ LlmResponseChunk, StreamTranslationState, TranslationEngine, WireFormat, decode_stream_event, - decode_stream_event_preserving, encode_stream_event, }; type TestResult = std::result::Result<(), Box>; @@ -58,19 +57,14 @@ fn preserved_same_format_events_replay_unknown_fields_exactly() -> TestResult { let mut state = StreamTranslationState::new(format, format); let preserved = engine.decode_stream_event(&mut state, format, event.clone())?; let replayed = engine.encode_stream_event(&mut state, format, preserved)?; - assert_eq!(replayed, vec![event.clone()]); - - let mut state = StreamTranslationState::new(format, format); - let preserved = decode_stream_event_preserving(&mut state, format, event.clone()); - let replayed = encode_stream_event(&mut state, format, preserved); assert_eq!(replayed, vec![event]); } Ok(()) } // Replay emits the preserved event without running the encoder, so the encoder never sees the -// stop it would normally record. Both replay paths must still leave the stream marked finished -// or `finish_stream` synthesizes a terminal the client already received. +// stop it would normally record. Replay must still leave the stream marked finished or +// `finish_stream` synthesizes a terminal the client already received. #[test] fn replayed_terminal_event_suppresses_synthesized_finish() -> TestResult { let cases = [ @@ -120,18 +114,6 @@ fn replayed_terminal_event_suppresses_synthesized_finish() -> TestResult { engine.finish_stream(&mut state, format)?.is_empty(), "{format:?} engine replay already delivered a terminal event", ); - - let mut state = StreamTranslationState::new(format, format); - let mut replayed = Vec::new(); - for event in &events { - let preserved = decode_stream_event_preserving(&mut state, format, event.clone()); - replayed.extend(encode_stream_event(&mut state, format, preserved)); - } - assert_eq!(replayed, events, "{format:?} codec replay"); - assert!( - engine.finish_stream(&mut state, format)?.is_empty(), - "{format:?} codec replay already delivered a terminal event", - ); } Ok(()) } From e3f9bc0a66c4f0be3259fc495ea401671e91b806 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 30 Jul 2026 10:45:45 -0600 Subject: [PATCH 07/51] docs(protocol): define provider event boundary Signed-off-by: Bryan Bednarski --- crates/protocol/src/stream.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rs index ceb9569f7..c3e0ae36a 100644 --- a/crates/protocol/src/stream.rs +++ b/crates/protocol/src/stream.rs @@ -162,14 +162,21 @@ fn push_checked_chunk( /// additionally retains the parsed source JSON value, so a same-format round trip replays that /// value rather than the original bytes: SSE framing, whitespace, and object key order are not /// preserved. Normalized children stay available to algorithms and cross-format encoders. +/// +/// `ProviderEvent` is intentionally a transport envelope rather than provider-neutral content. +/// It lives in the protocol crate because [`LlmResponseStream`] crosses the host/algorithm +/// boundary. Algorithms consume its normalized children; only `switchyard-translation` +/// interprets its source format and raw value for replay. This mirrors +/// [`PreservationMetadata`](crate::PreservationMetadata) for buffered bodies without making +/// provider fields part of the semantic IR. /// `switchyard-translation` re-exports this type as `ConversationStreamEvent`. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum LlmResponseChunk { /// One preserved provider event paired with the neutral events decoded from it. ProviderEvent { - /// Provider format that produced `raw`. + /// Opaque source-format identity used by the translation layer. source: FormatId, - /// Source event as a parsed JSON value. + /// Opaque parsed source event retained for translation-layer replay. raw: Value, /// Provider-neutral events decoded from `raw`, in source order. normalized: Vec, From 56012edf1bb014a3d959705cd773790bb2d461c2 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 30 Jul 2026 11:56:46 -0600 Subject: [PATCH 08/51] refactor(protocol): compose preserved stream events Signed-off-by: Bryan Bednarski --- crates/libsy/examples/streaming_agent.rs | 20 +- crates/libsy/src/algorithms/util/llm_judge.rs | 4 +- crates/libsy/src/core/algorithm.rs | 19 +- crates/libsy/src/observability.rs | 44 ++-- crates/libsy/tests/observability.rs | 17 +- crates/protocol/src/lib.rs | 15 +- crates/protocol/src/stream.rs | 203 ++++++++++++------ crates/switchyard-server/src/usage_metrics.rs | 20 +- .../src/codecs/anthropic/stream.rs | 1 - .../src/codecs/openai_chat/stream.rs | 1 - .../src/codecs/responses/stream.rs | 1 - .../src/codecs/stream.rs | 31 ++- crates/switchyard-translation/src/engine.rs | 16 +- crates/switchyard-translation/src/helpers.rs | 107 ++++++--- crates/switchyard-translation/src/lib.rs | 5 +- 15 files changed, 331 insertions(+), 173 deletions(-) diff --git a/crates/libsy/examples/streaming_agent.rs b/crates/libsy/examples/streaming_agent.rs index 1b7e877c5..47a3918aa 100644 --- a/crates/libsy/examples/streaming_agent.rs +++ b/crates/libsy/examples/streaming_agent.rs @@ -7,7 +7,8 @@ //! offloaded as a [`Step::CallLlm`]. The agent serves it with a *streaming* response //! ([`LlmResponse::Stream`]) rather than a buffered one. That stream rides untouched //! through the algorithm and returns as [`Step::ReturnToAgent`], where the agent drives it -//! and prints each [`LlmResponseChunk`] as it arrives — true token streaming end to end. +//! and prints the normalized [`LlmResponseChunk`]s in each event as they arrive — true +//! token streaming end to end. //! Contrast [`Algorithm::run`], which would aggregate the same stream into one buffered //! answer. Run with: //! cargo run -p libsy --example streaming_agent @@ -40,7 +41,8 @@ fn streaming_response(model: &str, tokens: &[&str]) -> Response { reason: Some("stop".to_string()), }); - let stream: LlmResponseStream = futures::stream::iter(chunks.into_iter().map(Ok)).boxed(); + let stream: LlmResponseStream = + futures::stream::iter(chunks.into_iter().map(|chunk| Ok(chunk.into()))).boxed(); Response { llm_response: LlmResponse::Stream(stream), metadata: None, @@ -81,15 +83,17 @@ async fn main() -> Result<()> { } Step::ReturnToAgent(response) => match response.llm_response { // The stream reached the agent untouched: print each token as it arrives. - LlmResponse::Stream(mut chunks) => { + LlmResponse::Stream(mut events) => { print!("agent sees: "); - while let Some(chunk) = chunks.next().await { - let chunk = chunk.map_err(|error| { + while let Some(event) = events.next().await { + let event = event.map_err(|error| { LibsyError::external("reading response stream", error) })?; - if let LlmResponseChunk::TextDelta { text, .. } = chunk { - print!("{text}"); - std::io::stdout().flush().ok(); + for chunk in event.normalized() { + if let LlmResponseChunk::TextDelta { text, .. } = chunk { + print!("{text}"); + std::io::stdout().flush().ok(); + } } } println!(); diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs index c5d81e4bf..4b8fc4c5e 100644 --- a/crates/libsy/src/algorithms/util/llm_judge.rs +++ b/crates/libsy/src/algorithms/util/llm_judge.rs @@ -330,7 +330,7 @@ mod tests { fn streamed(chunks: Vec) -> Response { Response { llm_response: LlmResponse::Stream( - futures::stream::iter(chunks.into_iter().map(Ok)).boxed(), + futures::stream::iter(chunks.into_iter().map(|chunk| Ok(chunk.into()))).boxed(), ), metadata: None, } @@ -338,7 +338,7 @@ mod tests { fn streamed_then_failing(chunk: LlmResponseChunk) -> Response { let items = futures::stream::iter([ - Ok(chunk), + Ok(chunk.into()), Err(LlmClientError::Timeout { source: Box::new(std::io::Error::other("stream died")), }), diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index 1e124ded3..7f8a922f3 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -17,10 +17,13 @@ use futures::{Stream, StreamExt}; use parking_lot::Mutex; use tracing::Instrument; -/// The request/response protocol types, re-exported from [`switchyard_protocol`]. -/// [`LlmRequest`] is the normalized request; [`AggLlmResponse`] is the buffered response; -/// [`LlmResponseChunk`] is one streaming event; [`LlmResponse`] is the streamed response -/// (a live [`LlmResponseStream`] or the terminal aggregate). +/// The request/response protocol types come from [`switchyard_protocol`]. +/// [`switchyard_protocol::LlmRequest`] is the normalized request; +/// [`switchyard_protocol::AggLlmResponse`] is the buffered response; +/// [`switchyard_protocol::LlmResponseChunk`] is normalized streaming content; +/// [`switchyard_protocol::LlmResponseStreamEvent`] is its host/algorithm envelope; and +/// [`switchyard_protocol::LlmResponse`] carries either a live +/// [`switchyard_protocol::LlmResponseStream`] or the terminal aggregate. use switchyard_protocol::{ Context, Decision, LlmClientError, Request, Response, RoutedLlmClient, Signals, Usage, }; @@ -946,7 +949,13 @@ mod tests { _request: Request, _decision: Arc, ) -> std::result::Result { - let stream = futures::stream::iter(self.chunks.clone().into_iter().map(Ok)).boxed(); + let stream = futures::stream::iter( + self.chunks + .clone() + .into_iter() + .map(|chunk| Ok(chunk.into())), + ) + .boxed(); Ok(Response { llm_response: LlmResponse::Stream(stream), metadata: None, diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs index f972bd0c1..ceeed9240 100644 --- a/crates/libsy/src/observability.rs +++ b/crates/libsy/src/observability.rs @@ -48,7 +48,7 @@ use tracing_opentelemetry::OpenTelemetrySpanExt; use crate::{Driver, LibsyError, Result}; use switchyard_protocol::{ AggLlmResponse, Context, Decision, LlmClientError, LlmRequest, LlmResponse, LlmResponseChunk, - LlmResponseStream, Request, Response, Usage, + LlmResponseStream, LlmResponseStreamEvent, Request, Response, Usage, }; const METRICS_SCOPE: &str = "switchyard"; @@ -364,7 +364,7 @@ struct ObservedClientStream { } impl Stream for ObservedClientStream { - type Item = std::result::Result; + type Item = std::result::Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { match self.stream.as_mut().poll_next(cx) { @@ -395,32 +395,48 @@ struct ClientStreamObserver { } impl ClientStreamObserver { - fn observe(&mut self, item: &std::result::Result) -> bool { + fn observe( + &mut self, + item: &std::result::Result, + ) -> bool { match item { - Ok(LlmResponseChunk::MessageStart { id, model }) => { + Ok(event) => { + for chunk in event.normalized() { + self.observe_chunk(chunk); + if self.terminal { + break; + } + } + } + Err(error) => { + let error_type = llm_client_error_type(error); + record_client_error(&self.span, &error_type, error); + self.terminal = true; + } + } + self.terminal + } + + fn observe_chunk(&mut self, chunk: &LlmResponseChunk) { + match chunk { + LlmResponseChunk::MessageStart { id, model } => { record_optional(&self.span, "gen_ai.response.id", id.as_deref()); record_optional(&self.span, "gen_ai.response.model", model.as_deref()); } - Ok(LlmResponseChunk::Usage(usage)) => record_gen_ai_usage(&self.span, usage), - Ok(LlmResponseChunk::MessageStop { reason }) => { + LlmResponseChunk::Usage(usage) => record_gen_ai_usage(&self.span, usage), + LlmResponseChunk::MessageStop { reason } => { record_finish_reasons(&self.span, reason.iter().cloned()); } - Ok(LlmResponseChunk::DecodeError { message }) => { + LlmResponseChunk::DecodeError { message } => { record_client_error(&self.span, "response_translation", message); self.terminal = true; } - Ok(LlmResponseChunk::StreamError { message }) => { + LlmResponseChunk::StreamError { message } => { record_client_error(&self.span, "502", message); self.terminal = true; } - Err(error) => { - let error_type = llm_client_error_type(error); - record_client_error(&self.span, &error_type, error); - self.terminal = true; - } _ => {} } - self.terminal } fn complete(&mut self) { diff --git a/crates/libsy/tests/observability.rs b/crates/libsy/tests/observability.rs index f03fee898..ca361ff63 100644 --- a/crates/libsy/tests/observability.rs +++ b/crates/libsy/tests/observability.rs @@ -40,7 +40,8 @@ use switchyard_protocol::{ Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient, Usage, }; use switchyard_protocol::{ - LlmClientError, LlmResponseChunk, StopReason, text_request, text_response, + LlmClientError, LlmResponseChunk, LlmResponseStreamEvent, StopReason, text_request, + text_response, }; #[derive(Debug, thiserror::Error)] @@ -758,16 +759,16 @@ impl RoutedLlmClient for StreamingUsageClient { cache: Usage::cache_details(Some(8), None), ..Usage::default() }; - let chunks = vec![ - Ok(LlmResponseChunk::MessageStart { + let chunks = vec![Ok(LlmResponseStreamEvent::new(vec![ + LlmResponseChunk::MessageStart { id: Some("obs-stream-response".to_string()), model: Some(decision.selected_model().to_string()), - }), - Ok(LlmResponseChunk::Usage(usage)), - Ok(LlmResponseChunk::MessageStop { + }, + LlmResponseChunk::Usage(usage), + LlmResponseChunk::MessageStop { reason: Some("end_turn".to_string()), - }), - ]; + }, + ]))]; Ok(Response { llm_response: LlmResponse::Stream(Box::pin(futures::stream::iter(chunks))), metadata: None, diff --git a/crates/protocol/src/lib.rs b/crates/protocol/src/lib.rs index 543f2d0bb..b299e0451 100644 --- a/crates/protocol/src/lib.rs +++ b/crates/protocol/src/lib.rs @@ -6,15 +6,14 @@ //! //! This crate owns Switchyard's neutral conversation IR: [`LlmRequest`] (model, messages, //! tools, sampling, …), the buffered [`AggLlmResponse`] (outputs, usage, …), and its -//! streaming counterpart [`LlmResponseChunk`]; the [`Request`]/[`Response`] envelope that -//! pairs them with correlation [`Metadata`]; plus the wire-[`format`] identifiers -//! translation keys off. `switchyard-translation` re-exports the IR types under its own -//! `ConversationRequest` / `ConversationResponse` / `ConversationStreamEvent` names. The IR -//! carries no bare `prompt`/`completion`; the [`text_request`] / [`prompt_text`] / -//! [`text_response`] / [`completion_text`] helpers bridge to and from plain text for the -//! common single-turn case. +//! provider-neutral streaming counterpart [`LlmResponseChunk`]. [`LlmResponseStreamEvent`] +//! composes those chunks with optional provider preservation while crossing the +//! host/algorithm boundary. The [`Request`]/[`Response`] envelope pairs calls with +//! correlation [`Metadata`]; wire-[`mod@format`] identifiers are translation keys. The IR carries +//! no bare `prompt`/`completion`; the [`text_request`] / [`prompt_text`] / [`text_response`] / +//! [`completion_text`] helpers bridge to and from plain text for the common single-turn case. //! -//! The streamed-response type itself — a live stream of chunks *or* the terminal +//! The streamed-response type itself — a live stream of events *or* the terminal //! aggregate — is the [`LlmResponse`] enum; it owns a `futures::Stream`, so it is the one //! non-`Clone`, non-data type in this crate. //! diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rs index c3e0ae36a..0173f2863 100644 --- a/crates/protocol/src/stream.rs +++ b/crates/protocol/src/stream.rs @@ -1,9 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Streaming half of the neutral IR: incremental response chunks ([`LlmResponseChunk`]) -//! and the streamed response ([`LlmResponse`]) that carries either a live stream of them -//! or the terminal [`AggLlmResponse`]. +//! Streaming half of the neutral IR: incremental response chunks ([`LlmResponseChunk`]), +//! their stream envelope ([`LlmResponseStreamEvent`]), and the streamed response +//! ([`LlmResponse`]) that carries either a live stream or the terminal [`AggLlmResponse`]. use std::collections::BTreeMap; use std::pin::Pin; @@ -23,12 +23,96 @@ use crate::{ /// code to propagate; 502 matches how a failed upstream call surfaces elsewhere. const MID_STREAM_UPSTREAM_STATUS: u16 = 502; -/// A boxed, `Send` stream of [`LlmResponseChunk`]s — the token-by-token output of a -/// streaming backend. Each item may fail independently mid-stream. +/// A boxed, `Send` stream of response events. Each item may fail independently mid-stream. pub type LlmResponseStream = - Pin> + Send>>; + Pin> + Send>>; -/// A model response: either a live [`Stream`](LlmResponse::Stream) of chunks or the +/// Parsed provider event retained for same-format replay. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ProviderStreamEvent { + source: FormatId, + raw: Value, +} + +impl ProviderStreamEvent { + /// Source wire format of the retained event. + pub fn source(&self) -> &FormatId { + &self.source + } + + /// Parsed provider JSON retained for replay. + pub fn raw(&self) -> &Value { + &self.raw + } + + /// Consumes the preservation value into its source format and parsed JSON. + pub fn into_parts(self) -> (FormatId, Value) { + (self.source, self.raw) + } +} + +/// One streaming item crossing the host/algorithm boundary. +/// +/// `normalized` contains only provider-neutral chunks. `preservation` is opaque to +/// algorithms and is interpreted by `switchyard-translation` for same-format replay. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct LlmResponseStreamEvent { + preservation: Option, + normalized: Vec, +} + +impl LlmResponseStreamEvent { + /// Creates an event containing only normalized chunks. + pub fn new(normalized: Vec) -> Self { + Self { + preservation: None, + normalized, + } + } + + /// Creates an event with parsed provider JSON retained for replay. + pub fn preserved( + source: impl Into, + raw: Value, + normalized: Vec, + ) -> Self { + Self { + preservation: Some(ProviderStreamEvent { + source: source.into(), + raw, + }), + normalized, + } + } + + /// Retained provider event, when this event came directly from a provider. + pub fn preservation(&self) -> Option<&ProviderStreamEvent> { + self.preservation.as_ref() + } + + /// Provider-neutral chunks carried by this event. + pub fn normalized(&self) -> &[LlmResponseChunk] { + &self.normalized + } + + /// Consumes the event into its preservation and normalized content. + pub fn into_parts(self) -> (Option, Vec) { + (self.preservation, self.normalized) + } + + /// Replaces semantic content and drops raw replay data that no longer describes it. + pub fn replace_normalized(self, normalized: Vec) -> Self { + Self::new(normalized) + } +} + +impl From for LlmResponseStreamEvent { + fn from(chunk: LlmResponseChunk) -> Self { + Self::new(vec![chunk]) + } +} + +/// A model response: either a live [`Stream`](LlmResponse::Stream) of events or the /// terminal buffered [`Agg`](LlmResponse::Agg)regate. /// /// Not `Clone` — the `Stream` variant owns a single-consumption stream. A buffered @@ -49,7 +133,7 @@ impl LlmResponse { } /// Reduce to the buffered aggregate: return an `Agg` unchanged, or drive a `Stream` - /// to completion, folding its chunks into an [`AggLlmResponse`] via + /// to completion, folding its normalized chunks into an [`AggLlmResponse`] via /// [`ResponseAccumulator`]. A stream item error aborts with `Err`, as does an /// in-band [`LlmResponseChunk::DecodeError`] (as `ResponseTranslation`) or /// [`LlmResponseChunk::StreamError`] (as `UpstreamHttp`). @@ -59,7 +143,9 @@ impl LlmResponse { LlmResponse::Stream(mut stream) => { let mut accumulator = ResponseAccumulator::new(); while let Some(item) = stream.next().await { - push_checked_chunk(&mut accumulator, item?)?; + for chunk in item?.normalized { + push_checked_chunk(&mut accumulator, chunk)?; + } } Ok(accumulator.finish()) } @@ -127,7 +213,9 @@ impl AggLlmResponse { }); } chunks.push(LlmResponseChunk::Usage(self.usage)); - Box::pin(futures::stream::iter(chunks.into_iter().map(Ok))) + Box::pin(futures::stream::iter( + chunks.into_iter().map(|chunk| Ok(chunk.into())), + )) } } @@ -136,12 +224,6 @@ fn push_checked_chunk( chunk: LlmResponseChunk, ) -> Result<(), LlmClientError> { match chunk { - LlmResponseChunk::ProviderEvent { normalized, .. } => { - for chunk in normalized { - push_checked_chunk(accumulator, chunk)?; - } - Ok(()) - } LlmResponseChunk::DecodeError { message } => { Err(LlmClientError::ResponseTranslation(message)) } @@ -156,31 +238,9 @@ fn push_checked_chunk( } } -/// One streaming event carried between a host and an algorithm. -/// -/// Normalized variants expose provider-neutral meaning. [`ProviderEvent`](Self::ProviderEvent) -/// additionally retains the parsed source JSON value, so a same-format round trip replays that -/// value rather than the original bytes: SSE framing, whitespace, and object key order are not -/// preserved. Normalized children stay available to algorithms and cross-format encoders. -/// -/// `ProviderEvent` is intentionally a transport envelope rather than provider-neutral content. -/// It lives in the protocol crate because [`LlmResponseStream`] crosses the host/algorithm -/// boundary. Algorithms consume its normalized children; only `switchyard-translation` -/// interprets its source format and raw value for replay. This mirrors -/// [`PreservationMetadata`](crate::PreservationMetadata) for buffered bodies without making -/// provider fields part of the semantic IR. -/// `switchyard-translation` re-exports this type as `ConversationStreamEvent`. +/// One provider-neutral streaming response chunk. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum LlmResponseChunk { - /// One preserved provider event paired with the neutral events decoded from it. - ProviderEvent { - /// Opaque source-format identity used by the translation layer. - source: FormatId, - /// Opaque parsed source event retained for translation-layer replay. - raw: Value, - /// Provider-neutral events decoded from `raw`, in source order. - normalized: Vec, - }, MessageStart { id: Option, model: Option, @@ -248,11 +308,6 @@ impl ResponseAccumulator { /// earlier ones; text, reasoning, and tool-call arguments append. pub fn push(&mut self, chunk: LlmResponseChunk) { match chunk { - LlmResponseChunk::ProviderEvent { normalized, .. } => { - for chunk in normalized { - self.push(chunk); - } - } LlmResponseChunk::MessageStart { id, model } => { if id.is_some() { self.id = id; @@ -399,18 +454,21 @@ mod tests { } #[test] - fn folds_normalized_chunks_inside_provider_event() { - let aggregate = fold(vec![LlmResponseChunk::ProviderEvent { - source: crate::WireFormat::OpenAiChat.into(), - raw: json!({ + fn aggregates_normalized_chunks_inside_stream_event() { + let response = LlmResponse::Stream(Box::pin(stream::iter([Ok( + LlmResponseStreamEvent::preserved( + crate::WireFormat::OpenAiChat, + json!({ "choices": [{"delta": {"content": "hello"}}], "system_fingerprint": "fp_exact" - }), - normalized: vec![LlmResponseChunk::TextDelta { - index: 0, - text: "hello".to_string(), - }], - }]); + }), + vec![LlmResponseChunk::TextDelta { + index: 0, + text: "hello".to_string(), + }], + ), + )]))); + let aggregate = block_on(response.into_agg()).expect("stream event should aggregate"); assert_eq!( aggregate.outputs[0].content, @@ -421,15 +479,40 @@ mod tests { } #[test] - fn stream_errors_inside_provider_events_remain_typed() { + fn replacing_normalized_content_drops_preservation() { + let event = LlmResponseStreamEvent::preserved( + crate::WireFormat::OpenAiChat, + json!({"choices": [{"delta": {"content": "old"}}]}), + vec![LlmResponseChunk::TextDelta { + index: 0, + text: "old".to_string(), + }], + ) + .replace_normalized(vec![LlmResponseChunk::TextDelta { + index: 0, + text: "new".to_string(), + }]); + + assert!(event.preservation().is_none()); + assert_eq!( + event.normalized(), + &[LlmResponseChunk::TextDelta { + index: 0, + text: "new".to_string(), + }] + ); + } + + #[test] + fn stream_errors_inside_preserved_events_remain_typed() { let response = LlmResponse::Stream(Box::pin(stream::iter([Ok( - LlmResponseChunk::ProviderEvent { - source: crate::WireFormat::OpenAiChat.into(), - raw: json!({"error": {"message": "provider failed"}}), - normalized: vec![LlmResponseChunk::StreamError { + LlmResponseStreamEvent::preserved( + crate::WireFormat::OpenAiChat, + json!({"error": {"message": "provider failed"}}), + vec![LlmResponseChunk::StreamError { message: "provider failed".to_string(), }], - }, + ), )]))); let error = block_on(response.into_agg()).err(); diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rs index 56bc44745..8632f2dea 100644 --- a/crates/switchyard-server/src/usage_metrics.rs +++ b/crates/switchyard-server/src/usage_metrics.rs @@ -52,16 +52,22 @@ pub(crate) fn observe( let wrapped = async_stream::stream! { let mut latest_usage = None; while let Some(item) = stream.next().await { - let failed = matches!( - &item, - Err(_) - | Ok( + let failed = match &item { + Err(_) => true, + Ok(event) => event.normalized().iter().any(|chunk| { + matches!( + chunk, LlmResponseChunk::StreamError { .. } | LlmResponseChunk::DecodeError { .. } ) - ); - if let Ok(LlmResponseChunk::Usage(usage)) = &item { - latest_usage = Some(usage.clone()); + }), + }; + if let Ok(event) = &item { + for chunk in event.normalized() { + if let LlmResponseChunk::Usage(usage) = chunk { + latest_usage = Some(usage.clone()); + } + } } if failed { record_stream_error(&stats, &model, tier.as_deref()); diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rs index 7f4f45a93..75ae2b06d 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs @@ -184,7 +184,6 @@ fn encode_anthropic_stream( LlmResponseChunk::StreamError { message } | LlmResponseChunk::DecodeError { message } => { vec![json!({"type": "error", "error": {"message": message}})] } - LlmResponseChunk::ProviderEvent { .. } => Vec::new(), } } diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs index 059f72a33..b11922e07 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs @@ -223,7 +223,6 @@ fn encode_openai_chat_stream( LlmResponseChunk::DecodeError { message } | LlmResponseChunk::StreamError { message } => { vec![json!({"error": {"message": message}})] } - LlmResponseChunk::ProviderEvent { .. } => Vec::new(), } } diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs index a44ae6d1d..10bf323f3 100644 --- a/crates/switchyard-translation/src/codecs/responses/stream.rs +++ b/crates/switchyard-translation/src/codecs/responses/stream.rs @@ -180,7 +180,6 @@ fn encode_responses_stream( LlmResponseChunk::DecodeError { message } | LlmResponseChunk::StreamError { message } => { vec![json!({"type": "error", "message": message})] } - LlmResponseChunk::ProviderEvent { .. } => Vec::new(), } } diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs index 8d8344c6a..e04d0be4e 100644 --- a/crates/switchyard-translation/src/codecs/stream.rs +++ b/crates/switchyard-translation/src/codecs/stream.rs @@ -230,27 +230,25 @@ pub(crate) fn mark_replayed_terminal( } } -pub(crate) fn encode_stream_chunk( +pub(crate) fn encode_response_stream_event( state: &mut StreamTranslationState, target_codec: &dyn StreamCodec, target: &FormatId, - event: LlmResponseChunk, + event: crate::LlmResponseStreamEvent, ) -> Vec { - match event { - LlmResponseChunk::ProviderEvent { - source, - raw, - normalized, - } if &source == target => { + let (preservation, normalized) = event.into_parts(); + if let Some(preservation) = preservation { + let (source, raw) = preservation.into_parts(); + if &source == target { mark_replayed_terminal(state, &normalized); - vec![raw] + return vec![raw]; } - LlmResponseChunk::ProviderEvent { normalized, .. } => normalized - .into_iter() - .flat_map(|event| encode_stream_chunk(state, target_codec, target, event)) - .collect(), - event => target_codec.encode_event(state, event), } + + normalized + .into_iter() + .flat_map(|chunk| target_codec.encode_event(state, chunk)) + .collect() } /// Encodes one neutral stream event with the built-in codec registry. @@ -259,10 +257,9 @@ pub fn encode_stream_event( target: impl Into, event: LlmResponseChunk, ) -> Vec { - let target = target.into(); StreamCodecRegistry::with_builtins() - .codec(target.clone()) - .map(|codec| encode_stream_chunk(state, codec.as_ref(), &target, event)) + .codec(target) + .map(|codec| codec.encode_event(state, event)) .unwrap_or_else(|error| vec![json!({"error": {"message": error.to_string()}})]) } diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rs index 0fd97867b..5cc9a3179 100644 --- a/crates/switchyard-translation/src/engine.rs +++ b/crates/switchyard-translation/src/engine.rs @@ -13,14 +13,14 @@ use crate::codecs::anthropic::AnthropicMessagesCodec; use crate::codecs::openai_chat::OpenAiChatCodec; use crate::codecs::responses::OpenAiResponsesCodec; use crate::codecs::stream::{ - StreamCodecRegistry, StreamTranslationState, encode_stream_chunk, + StreamCodecRegistry, StreamTranslationState, encode_response_stream_event, }; use crate::diagnostic::TranslationDiagnostic; use crate::error::{Result, TranslationError}; use crate::format::FormatId; use crate::llm::{AggLlmResponse, LlmRequest}; use crate::policy::TranslationPolicy; -use crate::LlmResponseChunk; +use crate::LlmResponseStreamEvent; /// Encoded translation result with any diagnostics emitted along the way. #[derive(Debug)] @@ -255,16 +255,12 @@ impl TranslationEngine { state: &mut StreamTranslationState, source: impl Into, event: Value, - ) -> Result { + ) -> Result { let source = source.into(); let source_codec = self.stream_registry.codec(source.clone())?; state.source = Some(source.clone()); let normalized = source_codec.decode_event(state, &event); - Ok(LlmResponseChunk::ProviderEvent { - source, - raw: event, - normalized, - }) + Ok(LlmResponseStreamEvent::preserved(source, event, normalized)) } /// Encodes one neutral or preserved stream event for a target provider. @@ -276,12 +272,12 @@ impl TranslationEngine { &self, state: &mut StreamTranslationState, target: impl Into, - event: LlmResponseChunk, + event: LlmResponseStreamEvent, ) -> Result> { let target = target.into(); let target_codec = self.stream_registry.codec(target.clone())?; state.target = Some(target.clone()); - Ok(encode_stream_chunk( + Ok(encode_response_stream_event( state, target_codec.as_ref(), &target, diff --git a/crates/switchyard-translation/src/helpers.rs b/crates/switchyard-translation/src/helpers.rs index dbb047514..1b72f1bbe 100644 --- a/crates/switchyard-translation/src/helpers.rs +++ b/crates/switchyard-translation/src/helpers.rs @@ -15,10 +15,11 @@ use futures::{Stream, StreamExt, TryStreamExt}; use serde_json::Value; use switchyard_protocol::LlmClientError; +use crate::codecs::stream::encode_response_stream_event; use crate::sse; use crate::{ - AggLlmResponse, LlmRequest, LlmResponseStream, Result, StreamCodecRegistry, - StreamTranslationState, TranslationEngine, TranslationPolicy, WireFormat, + AggLlmResponse, FormatId, LlmRequest, LlmResponseStream, LlmResponseStreamEvent, Result, + StreamCodecRegistry, StreamTranslationState, TranslationEngine, TranslationPolicy, WireFormat, }; static DEFAULT_TRANSLATION_POLICY: LazyLock = @@ -85,17 +86,18 @@ pub fn encode_stream( target: WireFormat, served_model: Option, ) -> std::result::Result { + let target_format: FormatId = target.into(); // The target is always a built-in wire format, so this lookup cannot fail; a // failure returns as an `Err` rather than a panic. let codec = StreamCodecRegistry::with_builtins() - .codec(target) + .codec(target_format.clone()) // Currently the only error is that the codec is missing, which is Configuration .map_err(|err| LlmClientError::Configuration { message: err.to_string(), })?; let mut state = StreamTranslationState { - target: Some(target.into()), + target: Some(target_format.clone()), target_model: served_model, ..Default::default() }; @@ -103,8 +105,10 @@ pub fn encode_stream( let events = try_stream! { while let Some(item) = chunks.next().await { - let chunk = item?; - for value in codec.encode_event(&mut state, chunk) { + let event = item?; + for value in + encode_response_stream_event(&mut state, codec.as_ref(), &target_format, event) + { yield value; } } @@ -130,10 +134,11 @@ where S: Stream, LlmClientError>> + Send + 'static, { let marker = sse::done_marker(source); + let source_format: FormatId = source.into(); // The source is always a built-in wire format, so this lookup cannot fail; a // failure returns as an `Err` rather than a panic. let codec = StreamCodecRegistry::with_builtins() - .codec(source) + .codec(source_format.clone()) .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?; // Adapt the byte-chunk stream into an async line reader. The BufReader // reassembles data split across network chunks (including multi-byte UTF-8), @@ -145,7 +150,10 @@ where Box::pin(bytes.map(|item| item.map_err(std::io::Error::other))); let lines = futures::io::BufReader::new(io_bytes.into_async_read()).lines(); - let mut state = StreamTranslationState::default(); + let mut state = StreamTranslationState { + source: Some(source_format.clone()), + ..StreamTranslationState::default() + }; let mut frame = String::new(); let stream = Box::pin(try_stream! { futures::pin_mut!(lines); @@ -160,9 +168,12 @@ where sse::SseFrame::Empty => {} sse::SseFrame::Done => break, sse::SseFrame::Data(value) => { - for event in codec.decode_event(&mut state, &value) { - yield event; - } + let normalized = codec.decode_event(&mut state, &value); + yield LlmResponseStreamEvent::preserved( + source_format.clone(), + value, + normalized, + ); } } } else { @@ -178,9 +189,8 @@ where let parsed = sse::parse_json_sse_frame(&frame, marker) .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?; if let sse::SseFrame::Data(value) = parsed { - for event in codec.decode_event(&mut state, &value) { - yield event; - } + let normalized = codec.decode_event(&mut state, &value); + yield LlmResponseStreamEvent::preserved(source_format, value, normalized); } } }); @@ -208,7 +218,9 @@ mod tests { use futures::executor::block_on; use futures::{Stream, StreamExt, stream}; use serde_json::{Value, json}; - use switchyard_protocol::{LlmClientError, LlmResponseChunk, completion_text}; + use switchyard_protocol::{ + LlmClientError, LlmResponseChunk, LlmResponseStreamEvent, completion_text, + }; use super::{ decode_aggregated_response, decode_request, decode_stream, encode_aggregated_response, @@ -223,19 +235,23 @@ mod tests { fn decode_all( bytes: impl Stream, LlmClientError>> + Send + 'static, source: WireFormat, - ) -> Result, LlmClientError> { + ) -> Result, LlmClientError> { block_on(decode_stream(bytes, source)?.collect::>()) .into_iter() .collect() } // Concatenates the text of every `TextDelta` chunk. - fn text_of(chunks: &[LlmResponseChunk]) -> String { - chunks + fn text_of(events: &[LlmResponseStreamEvent]) -> String { + events .iter() - .filter_map(|chunk| match chunk { - LlmResponseChunk::TextDelta { text, .. } => Some(text.as_str()), - _ => None, + .flat_map(LlmResponseStreamEvent::normalized) + .filter_map(|chunk| { + if let LlmResponseChunk::TextDelta { text, .. } = chunk { + Some(text.as_str()) + } else { + None + } }) .collect() } @@ -281,14 +297,17 @@ mod tests { Ok(LlmResponseChunk::TextDelta { index: 0, text: "Hello".to_string(), - }), + } + .into()), Ok(LlmResponseChunk::TextDelta { index: 0, text: " world".to_string(), - }), + } + .into()), Ok(LlmResponseChunk::MessageStop { reason: Some("stop".to_string()), - }), + } + .into()), ]) .boxed(); @@ -321,11 +340,13 @@ mod tests { Ok(LlmResponseChunk::MessageStart { id: Some("msg_1".to_string()), model: Some("upstream/model".to_string()), - }), + } + .into()), Ok(LlmResponseChunk::TextDelta { index: 0, text: "hi".to_string(), - }), + } + .into()), ]) .boxed(); @@ -353,11 +374,13 @@ mod tests { Ok(LlmResponseChunk::MessageStart { id: Some("msg_1".to_string()), model: Some("upstream/model".to_string()), - }), + } + .into()), Ok(LlmResponseChunk::TextDelta { index: 0, text: "hi".to_string(), - }), + } + .into()), ]) .boxed(); @@ -374,7 +397,7 @@ mod tests { #[test] fn encode_stream_propagates_chunk_errors() -> Result<(), BoxError> { let chunks: LlmResponseStream = - stream::iter(vec![Err::( + stream::iter(vec![Err::( LlmClientError::General("chunk exploded".to_string()), )]) .boxed(); @@ -397,6 +420,32 @@ mod tests { Ok(()) } + #[test] + fn stream_helpers_replay_same_format_provider_fields() -> Result<(), BoxError> { + let provider_event = json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_provider_specific", + "choices": [{ + "index": 0, + "delta": {"content": "Hello"}, + "finish_reason": "stop" + }] + }); + let bytes = stream::once({ + let frame = format!("data: {provider_event}\n\n").into_bytes(); + async move { Ok::, LlmClientError>(frame) } + }); + let decoded = decode_stream(bytes, WireFormat::OpenAiChat)?; + let replayed = + block_on(encode_stream(decoded, WireFormat::OpenAiChat, None)?.collect::>()) + .into_iter() + .collect::, BoxError>>()?; + + assert_eq!(replayed, vec![provider_event]); + Ok(()) + } + #[test] fn decode_stream_reassembles_frames_split_across_chunks() -> Result<(), BoxError> { // A multi-byte codepoint and the frame boundaries are split across diff --git a/crates/switchyard-translation/src/lib.rs b/crates/switchyard-translation/src/lib.rs index 5cdc51d98..d07a3497d 100644 --- a/crates/switchyard-translation/src/lib.rs +++ b/crates/switchyard-translation/src/lib.rs @@ -17,8 +17,9 @@ mod sse; pub mod stream; pub mod util; -pub use switchyard_protocol::stream::LlmResponseChunk; -pub use switchyard_protocol::stream::LlmResponseStream; +pub use switchyard_protocol::stream::{ + LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, ProviderStreamEvent, +}; pub use switchyard_protocol::{format, llm}; pub use diagnostic::*; From 3ccca40cf98cebfbfea4ba091cd040894c0d1d54 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 30 Jul 2026 18:06:23 -0600 Subject: [PATCH 09/51] fix(translation): retain served model on replay Signed-off-by: Bryan Bednarski --- crates/switchyard-translation/src/helpers.rs | 103 ++++++++++++++++++- 1 file changed, 100 insertions(+), 3 deletions(-) diff --git a/crates/switchyard-translation/src/helpers.rs b/crates/switchyard-translation/src/helpers.rs index 1b72f1bbe..5136825c8 100644 --- a/crates/switchyard-translation/src/helpers.rs +++ b/crates/switchyard-translation/src/helpers.rs @@ -96,6 +96,7 @@ pub fn encode_stream( message: err.to_string(), })?; + let served_model_for_events = served_model.clone(); let mut state = StreamTranslationState { target: Some(target_format.clone()), target_model: served_model, @@ -106,13 +107,23 @@ pub fn encode_stream( let events = try_stream! { while let Some(item) = chunks.next().await { let event = item?; - for value in + for mut value in encode_response_stream_event(&mut state, codec.as_ref(), &target_format, event) { + stamp_streamed_response_model( + &mut value, + target, + served_model_for_events.as_deref(), + ); yield value; } } - for value in codec.finish(&mut state) { + for mut value in codec.finish(&mut state) { + stamp_streamed_response_model( + &mut value, + target, + served_model_for_events.as_deref(), + ); yield value; } }; @@ -120,6 +131,37 @@ pub fn encode_stream( Ok(Box::pin(events)) } +// The raw-response helper promises that the caller sees the model that served the +// request. Same-format preservation bypasses provider codecs, so apply that +// helper-specific override after replay without disturbing any other raw fields. +fn stamp_streamed_response_model( + event: &mut Value, + target: WireFormat, + served_model: Option<&str>, +) { + let Some(served_model) = served_model else { + return; + }; + + match target { + WireFormat::OpenAiChat => { + if let Some(event) = event.as_object_mut() { + event.insert("model".to_string(), Value::String(served_model.to_string())); + } + } + WireFormat::OpenAiResponses => { + if let Some(response) = event.get_mut("response").and_then(Value::as_object_mut) { + response.insert("model".to_string(), Value::String(served_model.to_string())); + } + } + WireFormat::AnthropicMessages => { + if let Some(message) = event.get_mut("message").and_then(Value::as_object_mut) { + message.insert("model".to_string(), Value::String(served_model.to_string())); + } + } + } +} + /// Decodes a byte stream of `source`-format SSE frames into neutral IR chunks. /// /// Operates on raw bytes, not any HTTP client type: the caller adapts its @@ -224,7 +266,7 @@ mod tests { use super::{ decode_aggregated_response, decode_request, decode_stream, encode_aggregated_response, - encode_request, encode_stream, + encode_request, encode_stream, stamp_streamed_response_model, }; use crate::{LlmResponseStream, WireFormat}; @@ -446,6 +488,61 @@ mod tests { Ok(()) } + #[test] + fn openai_chat_replay_stamps_the_served_model_without_losing_extensions() { + let mut event = json!({ + "choices": [{"delta": {"content": "Hello"}}], + "system_fingerprint": "fp_provider_specific", + }); + + stamp_streamed_response_model(&mut event, WireFormat::OpenAiChat, Some("served/model")); + + assert_eq!(event["model"], "served/model"); + assert_eq!(event["system_fingerprint"], "fp_provider_specific"); + } + + #[test] + fn responses_replay_stamps_the_served_model_inside_response() { + let mut event = json!({ + "type": "response.created", + "response": { + "id": "resp_1", + "model": "provider/model", + "provider_extension": true, + }, + }); + + stamp_streamed_response_model( + &mut event, + WireFormat::OpenAiResponses, + Some("served/model"), + ); + + assert_eq!(event["response"]["model"], "served/model"); + assert_eq!(event["response"]["provider_extension"], true); + } + + #[test] + fn anthropic_replay_stamps_the_served_model_inside_message() { + let mut event = json!({ + "type": "message_start", + "message": { + "id": "msg_1", + "model": "provider/model", + "provider_extension": true, + }, + }); + + stamp_streamed_response_model( + &mut event, + WireFormat::AnthropicMessages, + Some("served/model"), + ); + + assert_eq!(event["message"]["model"], "served/model"); + assert_eq!(event["message"]["provider_extension"], true); + } + #[test] fn decode_stream_reassembles_frames_split_across_chunks() -> Result<(), BoxError> { // A multi-byte codepoint and the frame boundaries are split across From a485afcb64855372d3062f535883aed5e9d68946 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 21:44:54 -0600 Subject: [PATCH 10/51] docs(translation): clarify stream preservation contracts Signed-off-by: Bryan Bednarski --- crates/switchyard-translation/src/engine.rs | 2 +- .../switchyard-translation/tests/stream_translation.rs | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rs index 5cc9a3179..9761a46af 100644 --- a/crates/switchyard-translation/src/engine.rs +++ b/crates/switchyard-translation/src/engine.rs @@ -8,6 +8,7 @@ use std::sync::Arc; use serde_json::Value; +use crate::LlmResponseStreamEvent; use crate::codecs::FormatCodec; use crate::codecs::anthropic::AnthropicMessagesCodec; use crate::codecs::openai_chat::OpenAiChatCodec; @@ -20,7 +21,6 @@ use crate::error::{Result, TranslationError}; use crate::format::FormatId; use crate::llm::{AggLlmResponse, LlmRequest}; use crate::policy::TranslationPolicy; -use crate::LlmResponseStreamEvent; /// Encoded translation result with any diagnostics emitted along the way. #[derive(Debug)] diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs index 67b96cd09..ef6c3903c 100644 --- a/crates/switchyard-translation/tests/stream_translation.rs +++ b/crates/switchyard-translation/tests/stream_translation.rs @@ -12,6 +12,7 @@ use switchyard_translation::{ type TestResult = std::result::Result<(), Box>; +// Same-format replay returns the same parsed JSON value, including provider-specific fields. #[test] fn preserved_same_format_events_replay_unknown_fields_exactly() -> TestResult { let cases = [ @@ -118,6 +119,7 @@ fn replayed_terminal_event_suppresses_synthesized_finish() -> TestResult { Ok(()) } +// Cross-format encoding discards provider-specific fields and uses normalized chunks. #[test] fn preserved_cross_format_event_uses_normalized_content() -> TestResult { let engine = TranslationEngine::default(); @@ -140,9 +142,11 @@ fn preserved_cross_format_event_uses_normalized_content() -> TestResult { engine.encode_stream_event(&mut state, WireFormat::AnthropicMessages, preserved)?; assert_eq!(translated[2]["delta"]["text"], "Hi"); - assert!(translated - .iter() - .all(|event| event.get("system_fingerprint").is_none())); + assert!( + translated + .iter() + .all(|event| event.get("system_fingerprint").is_none()) + ); Ok(()) } From 784837c1b16537f3140bcd994b888829f33ddab9 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 22:41:10 -0600 Subject: [PATCH 11/51] fix(translation): retain replay encoder state Signed-off-by: Bryan Bednarski --- .../src/codecs/anthropic/stream.rs | 41 ++++++++-- .../src/codecs/openai_chat/stream.rs | 3 + .../src/codecs/stream.rs | 46 +++++++---- .../tests/stream_translation.rs | 78 +++++++++++++++++++ 4 files changed, 144 insertions(+), 24 deletions(-) diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rs index 75ae2b06d..6cb3d1868 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs @@ -37,6 +37,28 @@ impl StreamCodec for AnthropicMessagesStreamCodec { encode_anthropic_stream(state, event) } + fn observe_replayed_event( + &self, + state: &mut StreamTranslationState, + raw: &Value, + normalized: Vec, + ) { + let normalized_stop = normalized + .iter() + .any(|chunk| matches!(chunk, LlmResponseChunk::MessageStop { .. })); + for chunk in normalized { + drop(encode_anthropic_stream(state, chunk)); + } + + match raw.get("type").and_then(Value::as_str) { + // Anthropic carries the stop reason and usage in `message_delta`, but the separate + // `message_stop` event is what actually terminates the stream. + Some("message_delta") if normalized_stop => state.emitted_message_delta = true, + Some("message_stop") => state.finished = true, + _ => {} + } + } + fn finish(&self, state: &mut StreamTranslationState) -> Vec { finish_anthropic_stream(state) } @@ -230,14 +252,17 @@ fn finish_anthropic_stream(state: &mut StreamTranslationState) -> Vec { out.push(json!({"type": "content_block_stop", "index": 0})); } - out.push(json!({ - "type": "message_delta", - "delta": { - "stop_reason": anthropic_stop_reason(state.stop_reason.as_deref()), - "stop_sequence": Value::Null, - }, - "usage": anthropic_stream_usage(state), - })); + if !state.emitted_message_delta { + out.push(json!({ + "type": "message_delta", + "delta": { + "stop_reason": anthropic_stop_reason(state.stop_reason.as_deref()), + "stop_sequence": Value::Null, + }, + "usage": anthropic_stream_usage(state), + })); + state.emitted_message_delta = true; + } out.push(json!({"type": "message_stop"})); state.finished = true; out diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs index b11922e07..5ea2f10fd 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs @@ -156,6 +156,9 @@ fn encode_openai_chat_stream( match event { LlmResponseChunk::MessageStart { id, model } => { record_source_identity(state, id, model); + // `finish_openai_chat_stream` uses this flag to decide whether a clean EOF needs a + // synthesized terminal chunk. Set it on the encode path as well as the decode path. + state.saw_message_start = true; if state.emitted_message_start || (!state_source_is(state, WireFormat::AnthropicMessages) && !state_source_is(state, WireFormat::OpenAiResponses)) diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs index e04d0be4e..79782907c 100644 --- a/crates/switchyard-translation/src/codecs/stream.rs +++ b/crates/switchyard-translation/src/codecs/stream.rs @@ -40,6 +40,7 @@ pub struct StreamTranslationState { pub(crate) output_tokens_seen: u64, pub(crate) saw_backend_usage: bool, pub(crate) stop_reason: Option, + pub(crate) emitted_message_delta: bool, pub(crate) next_content_index: usize, pub(crate) text_block_index: Option, @@ -109,6 +110,30 @@ pub trait StreamCodec: Send + Sync { event: LlmResponseChunk, ) -> Vec; + /// Advances encoder state after an exact same-format event replay. + /// + /// Exact replay returns the preserved provider JSON instead of the JSON emitted by + /// [`Self::encode_event`]. The encoder must nevertheless observe the normalized chunks so + /// [`Self::finish`] can close an incomplete stream without duplicating an already replayed + /// terminal event. Codecs whose terminal state cannot be inferred from `MessageStop` alone + /// may override this hook and inspect `raw`. + fn observe_replayed_event( + &self, + state: &mut StreamTranslationState, + _raw: &Value, + normalized: Vec, + ) { + let replayed_terminal = normalized + .iter() + .any(|chunk| matches!(chunk, LlmResponseChunk::MessageStop { .. })); + for chunk in normalized { + drop(self.encode_event(state, chunk)); + } + if replayed_terminal { + state.finished = true; + } + } + /// Emits any terminal provider events needed after the source stream ends. /// /// This is intentionally required on every codec. Some target formats @@ -215,21 +240,6 @@ pub fn decode_stream_event( }) } -// Replaying a preserved event emits its retained JSON without running the encoder, so the -// terminal bookkeeping the encoder would have done has to happen here. Without it `finish` -// still believes the stream is unterminated and synthesizes a second terminal sequence. -pub(crate) fn mark_replayed_terminal( - state: &mut StreamTranslationState, - normalized: &[LlmResponseChunk], -) { - if normalized - .iter() - .any(|event| matches!(event, LlmResponseChunk::MessageStop { .. })) - { - state.finished = true; - } -} - pub(crate) fn encode_response_stream_event( state: &mut StreamTranslationState, target_codec: &dyn StreamCodec, @@ -240,7 +250,11 @@ pub(crate) fn encode_response_stream_event( if let Some(preservation) = preservation { let (source, raw) = preservation.into_parts(); if &source == target { - mark_replayed_terminal(state, &normalized); + // Exact replay bypasses the target encoder's emitted JSON, but the encoder must + // still observe every normalized chunk. Otherwise `finish` starts from empty state: + // a clean EOF after a nonterminal provider event can omit or synthesize malformed + // terminal events, while a replayed terminal can be emitted twice. + target_codec.observe_replayed_event(state, &raw, normalized); return vec![raw]; } } diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs index ef6c3903c..d97eee149 100644 --- a/crates/switchyard-translation/tests/stream_translation.rs +++ b/crates/switchyard-translation/tests/stream_translation.rs @@ -119,6 +119,84 @@ fn replayed_terminal_event_suppresses_synthesized_finish() -> TestResult { Ok(()) } +#[test] +fn replayed_nonterminal_event_advances_encoder_state_before_finish() -> TestResult { + let engine = TranslationEngine::default(); + let format = WireFormat::OpenAiChat; + let event = json!({ + "id": "chatcmpl-clean-eof", + "object": "chat.completion.chunk", + "model": "gpt-4o", + "choices": [{ + "index": 0, + "delta": {"role": "assistant", "content": "Hi"}, + "finish_reason": null + }] + }); + let mut decode_state = StreamTranslationState::new(format, format); + let preserved = engine.decode_stream_event(&mut decode_state, format, event.clone())?; + + // Provider decoding and caller encoding are independent stream boundaries in a host such as + // NeMo Relay. Replay must therefore advance a fresh encoder state using the normalized view. + let mut encode_state = StreamTranslationState::new(format, format); + assert_eq!( + engine.encode_stream_event(&mut encode_state, format, preserved)?, + vec![event] + ); + let finish = engine.finish_stream(&mut encode_state, format)?; + + assert_eq!(finish.len(), 1); + assert_eq!(finish[0]["id"], "chatcmpl-clean-eof"); + assert_eq!(finish[0]["model"], "gpt-4o"); + assert_eq!(finish[0]["choices"][0]["finish_reason"], "stop"); + Ok(()) +} + +#[test] +fn replayed_anthropic_terminal_delta_finishes_with_message_stop_only() -> TestResult { + let engine = TranslationEngine::default(); + let format = WireFormat::AnthropicMessages; + let events = [ + json!({ + "type": "message_start", + "message": { + "id": "msg_clean_eof", + "model": "claude", + "usage": {"input_tokens": 3, "output_tokens": 0} + } + }), + json!({ + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": null}, + "usage": {"output_tokens": 2} + }), + ]; + let mut decode_state = StreamTranslationState::new(format, format); + let mut encode_state = StreamTranslationState::new(format, format); + + for event in events { + let preserved = engine.decode_stream_event(&mut decode_state, format, event.clone())?; + assert_eq!( + engine.encode_stream_event(&mut encode_state, format, preserved)?, + vec![event] + ); + } + + let finish = engine.finish_stream(&mut encode_state, format)?; + assert_eq!( + finish + .iter() + .filter(|event| event["type"] == "message_stop") + .count(), + 1 + ); + assert!( + finish.iter().all(|event| event["type"] != "message_delta"), + "the replayed terminal delta must not be synthesized a second time" + ); + Ok(()) +} + // Cross-format encoding discards provider-specific fields and uses normalized chunks. #[test] fn preserved_cross_format_event_uses_normalized_content() -> TestResult { From ae354a0de6c5718b5e94810eab2f7aa799e2428e Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 30 Jul 2026 16:59:38 -0600 Subject: [PATCH 12/51] feat(plugin): add NeMo Relay dynamic integration Signed-off-by: Bryan Bednarski --- Cargo.lock | 189 ++++ Cargo.toml | 3 + crates/libsy/Cargo.toml | 1 + crates/libsy/src/core/algorithm.rs | 84 +- crates/libsy/src/core/driver.rs | 30 +- .../switchyard-nemo-relay-plugin/Cargo.toml | 27 + .../config.schema.json | 109 +++ .../relay-plugin.toml | 31 + .../src/config.rs | 413 +++++++++ .../switchyard-nemo-relay-plugin/src/ffi.rs | 471 ++++++++++ .../switchyard-nemo-relay-plugin/src/lib.rs | 199 +++++ .../src/runtime.rs | 837 ++++++++++++++++++ .../src/translation.rs | 230 +++++ 13 files changed, 2586 insertions(+), 38 deletions(-) create mode 100644 crates/switchyard-nemo-relay-plugin/Cargo.toml create mode 100644 crates/switchyard-nemo-relay-plugin/config.schema.json create mode 100644 crates/switchyard-nemo-relay-plugin/relay-plugin.toml create mode 100644 crates/switchyard-nemo-relay-plugin/src/config.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/ffi.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/lib.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/runtime.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/translation.rs diff --git a/Cargo.lock b/Cargo.lock index 93cdbf64e..83db53250 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "1.0.0" @@ -239,6 +248,9 @@ name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +dependencies = [ + "serde_core", +] [[package]] name = "bumpalo" @@ -287,6 +299,20 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + [[package]] name = "clap" version = "4.6.2" @@ -554,6 +580,12 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + [[package]] name = "futures-util" version = "0.3.32" @@ -758,6 +790,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -1051,6 +1107,29 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nemo-relay-plugin" +version = "0.7.0" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=4590e7b8828689fdf8632f7625ea0db61ed18e6a#4590e7b8828689fdf8632f7625ea0db61ed18e6a" +dependencies = [ + "nemo-relay-types", + "serde", + "serde_json", +] + +[[package]] +name = "nemo-relay-types" +version = "0.7.0" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=4590e7b8828689fdf8632f7625ea0db61ed18e6a#4590e7b8828689fdf8632f7625ea0db61ed18e6a" +dependencies = [ + "bitflags", + "chrono", + "serde", + "serde_json", + "typed-builder", + "uuid", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1060,6 +1139,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.17.0" @@ -1973,6 +2061,7 @@ version = "0.2.0" dependencies = [ "async-trait", "futures", + "futures-timer", "opentelemetry", "opentelemetry_sdk", "parking_lot", @@ -2007,6 +2096,21 @@ dependencies = [ "wiremock", ] +[[package]] +name = "switchyard-nemo-relay-plugin" +version = "0.1.0" +dependencies = [ + "futures", + "futures-util", + "http", + "nemo-relay-plugin", + "serde", + "serde_json", + "switchyard-libsy", + "switchyard-protocol", + "switchyard-translation", +] + [[package]] name = "switchyard-protocol" version = "0.2.0" @@ -2452,6 +2556,26 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typed-builder" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "unicode-ident" version = "1.0.24" @@ -2488,6 +2612,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -2634,12 +2770,65 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index 2325e1ac2..3da8e96b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ members = [ "crates/switchyard-server", "crates/switchyard-skill-distillation", "crates/switchyard-translation", + "crates/switchyard-nemo-relay-plugin", ] [workspace.package] @@ -26,7 +27,9 @@ rust-version = "1.96.1" async-stream = "0.3" async-trait = "0.1" futures = "0.3" +futures-timer = "3" futures-util = "0.3" +http = "1" httpdate = "1" parking_lot = "0.12" rand = "0.10" diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.toml index dc2be5249..2e0dd0864 100644 --- a/crates/libsy/Cargo.toml +++ b/crates/libsy/Cargo.toml @@ -16,6 +16,7 @@ async-trait.workspace = true serde.workspace = true serde_json.workspace = true futures.workspace = true +futures-timer.workspace = true # Metrics-only OTel API: instruments record through the host-installed global # meter provider. Pinned to the 0.32 line used across the workspace. opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] } diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index 7f8a922f3..7c34e97e3 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -7,13 +7,14 @@ use std::{ collections::{HashMap, HashSet}, + panic::AssertUnwindSafe, pin::Pin, sync::Arc, time::{Duration, Instant}, }; use async_trait::async_trait; -use futures::{Stream, StreamExt}; +use futures::{FutureExt, Stream, StreamExt}; use parking_lot::Mutex; use tracing::Instrument; @@ -332,15 +333,6 @@ pub enum Step { ReturnToAgent(Box), } -/// Abort guard -struct AbortOnDrop(tokio::task::AbortHandle); - -impl Drop for AbortOnDrop { - fn drop(&mut self) { - self.0.abort(); - } -} - /// A named routing target: a `semantic_name` an algorithm routes by, and an optional /// [`RoutedLlmClient`] to serve its calls. An algorithm hands a target to /// [`Driver::call_llm_target`]; the client rides along as @@ -602,6 +594,10 @@ pub trait Algorithm: Send + Sync + 'static { /// Each [`Step::CallLlm`] is an offloaded model call the consumer must serve. /// The stream ends with a [`Step::ReturnToAgent`] on success, or an `Err` item on failure. /// Report each model call to `observer`. + /// + /// The returned stream is lazy and executor-neutral: polling it drives the + /// algorithm future on the consumer's executor. Dropping it cancels that + /// future, including algorithms waiting without an outstanding driver call. fn run_stream( self: Arc, ctx: Context, @@ -624,7 +620,7 @@ pub trait Algorithm: Send + Sync + 'static { // contextual parenting. let span = observability::run_span(self.name(), &request); let observed_driver = task_driver.clone(); - let handle = tokio::spawn( + let task = AssertUnwindSafe( async move { observability::observe_run( task_ctx.clone(), @@ -634,17 +630,18 @@ pub trait Algorithm: Send + Sync + 'static { .await } .instrument(span), - ); - // Dropping the stream aborts the algorithm task, so it doesn't keep running after the - let abort_guard = AbortOnDrop(handle.abort_handle()); + ) + .catch_unwind(); let finish_driver = driver.clone(); let finish_ctx = ctx; let tail: StepStream = Box::pin( futures::stream::once(async move { - let result = match handle.await { + let result = match task.await { Ok(response) => response, - Err(source) => Err(LibsyError::AlgorithmTask { source }), + Err(_) => Err(LibsyError::AlgorithmError { + message: "algorithm task panicked".to_string(), + }), }; finish_driver.finish(finish_ctx, result).await }) @@ -652,11 +649,7 @@ pub trait Algorithm: Send + Sync + 'static { ); let stream: StepStream = Box::pin(stream); - Box::pin(futures::stream::select(stream, tail).map(move |step| { - // link abort guard to stream - let _keep_alive = &abort_guard; - step - })) + Box::pin(futures::stream::select(stream, tail)) } /// Process a request to completion, returning the final [`Response`] and the trace of @@ -1080,6 +1073,34 @@ mod tests { Ok(()) } + #[test] + fn run_stream_is_executor_neutral_for_embedded_hosts() -> Result<()> { + futures::executor::block_on(async { + let mut stream = orch(target_set(&[("embedded/model", false)])) + .run_stream(Context::default(), request()); + let mut returned = false; + while let Some(step) = stream.next().await { + match step? { + Step::CallLlm(call) => { + call.respond(Ok(Response { + llm_response: LlmResponse::Agg(text_response(None, "embedded")), + metadata: None, + }))?; + } + Step::Decision(_) => {} + Step::ReturnToAgent(response) => { + returned = response + .llm_response + .as_agg() + .is_some_and(|response| completion_text(response) == "embedded"); + } + } + } + assert!(returned, "embedded executor did not receive ReturnToAgent"); + Ok(()) + }) + } + #[tokio::test] async fn client_backed_target_offloads_with_a_default_client() -> Result<()> { // Every call now offloads to the stream; a client-backed target rides its @@ -1319,7 +1340,10 @@ mod tests { dropped: dropped.clone(), }); - let stream = algo.run_stream(Context::default(), request(), None); + let mut stream = algo.run_stream(Context::default(), request(), None); + // `run_stream` is a lazy stream: poll it once to start the algorithm + // before checking that dropping the stream cancels the in-flight task. + assert!(futures::poll!(stream.as_mut().next()).is_pending()); started_rx .recv() .await @@ -1336,8 +1360,8 @@ mod tests { #[tokio::test] async fn create_run_task_panic_surfaces_as_a_stream_error() -> Result<()> { - // An algorithm whose task panics must surface an `Err` step to the stream - // consumer, not abort the process from an unobserved detached task. + // An algorithm panic must surface as an `Err` step to the stream + // consumer, not abort the embedding host. struct Panicky; #[async_trait] @@ -1364,7 +1388,11 @@ mod tests { while let Some(step) = stream.next().await { match step { Err(err) => { - assert!(matches!(err, LibsyError::AlgorithmTask { .. })); + assert!(matches!( + err, + LibsyError::AlgorithmError { message } + if message == "algorithm task panicked" + )); saw_error = true; } Ok(_) => return Err(test_error("expected the panic to surface as an error step")), @@ -1403,7 +1431,11 @@ mod tests { "expected run to surface the algorithm panic as an error", )), Err(err) => { - assert!(matches!(err, LibsyError::AlgorithmTask { .. })); + assert!(matches!( + err, + LibsyError::AlgorithmError { message } + if message == "algorithm task panicked" + )); Ok(()) } } diff --git a/crates/libsy/src/core/driver.rs b/crates/libsy/src/core/driver.rs index 62f5e9763..04eb9e3eb 100644 --- a/crates/libsy/src/core/driver.rs +++ b/crates/libsy/src/core/driver.rs @@ -42,20 +42,21 @@ //! There is no explicit stop method — the consumer terminates by **dropping the //! stream** (and any [`DriverRequest`] it is holding). The producer's next publish //! (`fulfill_request`/`info`/`done`/`fail`) then resolves to `Err`, and a producer -//! awaiting a response sees `Err` once the promise it handed out is dropped. Either -//! way the algorithm unwinds cooperatively at its next driver interaction. Because the -//! producer runs on a task the driver does not own, hard cancellation (e.g. mid-compute -//! that never touches the driver) is the caller's concern — abort the producer task. +//! awaiting a response sees `Err` once the promise it handed out is dropped. In +//! [`Algorithm::run_stream`](super::algorithm::Algorithm::run_stream), the producer +//! future is owned by the returned stream, so dropping the stream also cancels an +//! algorithm that is between driver interactions. use std::{any::Any, sync::Arc}; use crate::{DriverError, LibsyError, Result}; use parking_lot::Mutex; -use futures::{Stream, StreamExt}; +use futures::{future::Either, pin_mut, Stream, StreamExt}; +use futures_timer::Delay; use switchyard_protocol::Context; use tokio::sync::{mpsc, oneshot}; -use tokio::time::{Duration, timeout}; +use tokio::time::Duration; use tokio_stream::wrappers::ReceiverStream; type BoxAny = Box; @@ -175,12 +176,17 @@ impl TypeErasedDriver { // Outer error: the promise was dropped without a response. Inner error: the // consumer fulfilled it with an explicit `Err` — propagate it as-is. - let response = timeout(FULFILL_REQUEST_TIMEOUT, rx) - .await - .map_err(|_| DriverError::ResponseTimedOut { - timeout: FULFILL_REQUEST_TIMEOUT, - })? - .map_err(|_| DriverError::ResponseDropped)??; + let timeout = Delay::new(FULFILL_REQUEST_TIMEOUT); + pin_mut!(rx, timeout); + let response = match futures::future::select(rx, timeout).await { + Either::Left((response, _)) => response.map_err(|_| DriverError::ResponseDropped)??, + Either::Right(((), _)) => { + return Err(DriverError::ResponseTimedOut { + timeout: FULFILL_REQUEST_TIMEOUT, + } + .into()); + } + }; response.downcast::().map(|boxed| *boxed).map_err(|_| { DriverError::TypeMismatch { expected: std::any::type_name::(), diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml new file mode 100644 index 000000000..83ae635dc --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "switchyard-nemo-relay-plugin" +version = "0.1.0" +description = "Switchyard libsy native dynamic plugin for NeMo Relay" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +futures.workspace = true +futures-util.workspace = true +http.workspace = true +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "4590e7b8828689fdf8632f7625ea0db61ed18e6a" } +serde.workspace = true +serde_json.workspace = true +switchyard-libsy.workspace = true +switchyard-protocol.workspace = true +switchyard-translation.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/config.schema.json b/crates/switchyard-nemo-relay-plugin/config.schema.json new file mode 100644 index 000000000..bb41c1d4c --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/config.schema.json @@ -0,0 +1,109 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Switchyard NeMo Relay Plugin", + "type": "object", + "additionalProperties": false, + "required": ["version", "algorithm", "targets", "default_targets"], + "properties": { + "version": { + "const": 2, + "description": "Library-only Switchyard configuration version." + }, + "priority": { + "type": "integer", + "default": 0 + }, + "max_retries": { + "type": "integer", + "minimum": 0, + "maximum": 10, + "default": 3 + }, + "enabled_inbound_profiles": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "enum": ["openai_chat", "openai_responses", "anthropic_messages"] + } + }, + "algorithm": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["kind"], + "properties": { + "kind": { "const": "random" }, + "seed": { "type": ["integer", "null"], "minimum": 0 } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "classifier_target", + "weak_target", + "strong_target", + "base_threshold" + ], + "properties": { + "kind": { "const": "llm_classifier" }, + "classifier_target": { "type": "string", "minLength": 1 }, + "weak_target": { "type": "string", "minLength": 1 }, + "strong_target": { "type": "string", "minLength": 1 }, + "base_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "min_confidence": { "type": "number", "minimum": 0, "maximum": 1 }, + "capability_elevated_floor": { + "type": ["number", "null"], + "minimum": 0, + "maximum": 1 + }, + "session_affinity": { "type": "boolean", "default": false }, + "message_hash_fallback": { "type": "boolean", "default": false } + } + } + ] + }, + "targets": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["model", "protocol", "base_url"], + "properties": { + "model": { "type": "string", "minLength": 1 }, + "protocol": { + "enum": ["openai_chat", "openai_responses", "anthropic_messages"] + }, + "endpoint": { "type": "string" }, + "base_url": { + "type": "string", + "pattern": "^https?://" + }, + "weight": { "type": "number", "minimum": 0, "default": 1 }, + "headers": { + "type": "object", + "additionalProperties": { "type": "string" } + }, + "header_env": { + "type": "object", + "description": "Maps provider header names to environment-variable names.", + "additionalProperties": { "type": "string", "minLength": 1 } + } + } + } + }, + "default_targets": { + "type": "object", + "additionalProperties": false, + "properties": { + "openai_chat": { "type": "string" }, + "openai_responses": { "type": "string" }, + "anthropic_messages": { "type": "string" } + } + } + } +} diff --git a/crates/switchyard-nemo-relay-plugin/relay-plugin.toml b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml new file mode 100644 index 000000000..38f38d848 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +manifest_version = 1 + +[plugin] +id = "nvidia.switchyard" +kind = "rust_dynamic" + +[compat] +relay = ">=0.7,<1.0" +native_api = "2" + +[defaults] +enabled = false + +[capabilities] +items = ["plugin_native", "config_schema"] + +[config_schema] +path = "config.schema.json" + +[source] +artifact = "" + +[integrity] +sha256 = "sha256:" + +[load] +library = "" +symbol = "nemo_relay_register_plugin" diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs new file mode 100644 index 000000000..dcc224652 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -0,0 +1,413 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use http::header::{HeaderName, HeaderValue}; +use nemo_relay_plugin::LlmDispatchRouteV2; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value as Json}; +use switchyard_libsy::algorithms::{LlmTaskClassifier, Random, TaskClassifierConfig}; +use switchyard_libsy::{Algorithm, LlmTarget, LlmTargetSet}; +use switchyard_protocol::WireFormat; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum WireProtocol { + OpenaiChat, + OpenaiResponses, + AnthropicMessages, +} + +impl WireProtocol { + pub const fn label(self) -> &'static str { + match self { + Self::OpenaiChat => "openai_chat", + Self::OpenaiResponses => "openai_responses", + Self::AnthropicMessages => "anthropic_messages", + } + } + + pub const fn endpoint(self) -> &'static str { + match self { + Self::OpenaiChat => "/v1/chat/completions", + Self::OpenaiResponses => "/v1/responses", + Self::AnthropicMessages => "/v1/messages", + } + } + + pub fn from_call(name: &str) -> Option { + match name { + "openai.chat_completions" | "openai_chat" | "openai_chat_completions" => { + Some(Self::OpenaiChat) + } + "openai.responses" | "openai_responses" => Some(Self::OpenaiResponses), + "anthropic.messages" | "anthropic" | "anthropic_messages" => { + Some(Self::AnthropicMessages) + } + _ => None, + } + } + + pub const fn relay_route(self) -> LlmDispatchRouteV2 { + match self { + Self::OpenaiChat => LlmDispatchRouteV2::OpenaiChat, + Self::OpenaiResponses => LlmDispatchRouteV2::OpenaiResponses, + Self::AnthropicMessages => LlmDispatchRouteV2::AnthropicMessages, + } + } + + pub const fn wire_format(self) -> WireFormat { + match self { + Self::OpenaiChat => WireFormat::OpenAiChat, + Self::OpenaiResponses => WireFormat::OpenAiResponses, + Self::AnthropicMessages => WireFormat::AnthropicMessages, + } + } + + pub fn from_wire_format(format: &WireFormat) -> Option { + match format { + WireFormat::OpenAiChat => Some(Self::OpenaiChat), + WireFormat::OpenAiResponses => Some(Self::OpenaiResponses), + WireFormat::AnthropicMessages => Some(Self::AnthropicMessages), + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct TargetBinding { + pub model: String, + pub protocol: WireProtocol, + #[serde(default)] + pub endpoint: String, + pub base_url: String, + #[serde(default = "default_weight")] + pub weight: f64, + #[serde(default)] + pub headers: BTreeMap, + #[serde(default)] + pub header_env: BTreeMap, +} + +impl TargetBinding { + pub fn dispatch_url(&self) -> String { + let base = self.base_url.trim_end_matches('/'); + let endpoint = if self.endpoint.is_empty() { + self.protocol.endpoint() + } else { + &self.endpoint + }; + let endpoint = if base.ends_with("/v1") && endpoint.starts_with("/v1/") { + &endpoint[3..] + } else { + endpoint + }; + format!("{base}{endpoint}") + } + + pub fn resolved_headers(&self) -> Result, String> { + let mut headers = Map::new(); + for (name, value) in &self.headers { + validate_header(name, value)?; + headers.insert(name.clone(), Json::String(value.clone())); + } + for (name, variable) in &self.header_env { + if self + .headers + .keys() + .any(|configured| configured.eq_ignore_ascii_case(name)) + { + return Err(format!( + "target header {name:?} cannot appear in both headers and header_env" + )); + } + let value = std::env::var(variable) + .map_err(|_| format!("environment variable {variable:?} is not set"))?; + validate_header(name, &value)?; + headers.insert(name.clone(), Json::String(value)); + } + Ok(headers) + } +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub struct ProtocolDefaults { + #[serde(default)] + pub openai_chat: String, + #[serde(default)] + pub openai_responses: String, + #[serde(default)] + pub anthropic_messages: String, +} + +impl ProtocolDefaults { + pub fn target(&self, protocol: WireProtocol) -> &str { + match protocol { + WireProtocol::OpenaiChat => &self.openai_chat, + WireProtocol::OpenaiResponses => &self.openai_responses, + WireProtocol::AnthropicMessages => &self.anthropic_messages, + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum AlgorithmConfig { + Random { + #[serde(default)] + seed: Option, + }, + LlmClassifier { + classifier_target: String, + weak_target: String, + strong_target: String, + base_threshold: f64, + #[serde(default)] + min_confidence: f64, + #[serde(default)] + capability_elevated_floor: Option, + #[serde(default)] + session_affinity: bool, + #[serde(default)] + message_hash_fallback: bool, + }, +} + +impl Default for AlgorithmConfig { + fn default() -> Self { + Self::Random { seed: None } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct SwitchyardConfig { + #[serde(default = "default_version")] + pub version: u32, + #[serde(default)] + pub priority: i32, + #[serde(default = "default_max_retries")] + pub max_retries: u32, + #[serde(default)] + pub algorithm: AlgorithmConfig, + pub targets: BTreeMap, + #[serde(default)] + pub default_targets: ProtocolDefaults, + #[serde(default = "default_enabled_protocols")] + pub enabled_inbound_profiles: BTreeSet, +} + +impl SwitchyardConfig { + pub fn validate(&self) -> Result<(), String> { + if self.version != 2 { + return Err(format!( + "unsupported Switchyard config version {}; version 1 used switchyard-server; migrate to version = 2", + self.version + )); + } + if self.max_retries > 10 { + return Err("max_retries must not exceed 10".into()); + } + if self.targets.is_empty() { + return Err("targets must not be empty".into()); + } + if self.enabled_inbound_profiles.is_empty() { + return Err("enabled_inbound_profiles must not be empty".into()); + } + for (name, target) in &self.targets { + if name.trim().is_empty() || target.model.trim().is_empty() { + return Err("target names and models must be non-empty".into()); + } + if !target.base_url.starts_with("http://") && !target.base_url.starts_with("https://") { + return Err(format!("target {name:?} base_url must use http or https")); + } + if !target.weight.is_finite() || target.weight < 0.0 { + return Err(format!( + "target {name:?} weight must be finite and nonnegative" + )); + } + target.resolved_headers()?; + } + for protocol in &self.enabled_inbound_profiles { + let fallback = self.default_targets.target(*protocol); + let target = self + .targets + .get(fallback) + .ok_or_else(|| format!("default target {fallback:?} is not configured"))?; + if target.protocol != *protocol { + return Err(format!( + "default target {fallback:?} must use protocol {}", + protocol.label() + )); + } + } + self.build_algorithm().map(|_| ()) + } + + pub fn build_algorithm(&self) -> Result, String> { + let target = |name: &str| { + self.targets + .contains_key(name) + .then(|| LlmTarget { + semantic_name: name.to_string(), + llm_client: None, + }) + .ok_or_else(|| format!("algorithm target {name:?} is not configured")) + }; + match &self.algorithm { + AlgorithmConfig::Random { seed } => { + let targets = self + .targets + .keys() + .map(|name| target(name)) + .collect::, _>>()?; + let weights = self + .targets + .values() + .map(|target| target.weight) + .collect::>(); + Random::new(LlmTargetSet::new(targets), Some(weights), *seed) + .map(|algorithm| Arc::new(algorithm) as Arc) + .map_err(|error| error.to_string()) + } + AlgorithmConfig::LlmClassifier { + classifier_target, + weak_target, + strong_target, + base_threshold, + min_confidence, + capability_elevated_floor, + session_affinity, + message_hash_fallback, + } => LlmTaskClassifier::new( + target(classifier_target)?, + target(weak_target)?, + target(strong_target)?, + TaskClassifierConfig { + base_threshold: *base_threshold, + min_confidence: *min_confidence, + capability_elevated_floor: *capability_elevated_floor, + session_affinity: *session_affinity, + message_hash_fallback: *message_hash_fallback, + recent_turn_window: None, + }, + ) + .map(|algorithm| Arc::new(algorithm) as Arc) + .map_err(|error| error.to_string()), + } + } +} + +fn validate_header(name: &str, value: &str) -> Result<(), String> { + HeaderName::from_bytes(name.as_bytes()) + .map_err(|error| format!("invalid target header name {name:?}: {error}"))?; + HeaderValue::from_str(value) + .map_err(|error| format!("invalid target header value for {name:?}: {error}"))?; + Ok(()) +} + +const fn default_version() -> u32 { + 2 +} + +const fn default_max_retries() -> u32 { + 3 +} + +const fn default_weight() -> f64 { + 1.0 +} + +fn default_enabled_protocols() -> BTreeSet { + BTreeSet::from([ + WireProtocol::OpenaiChat, + WireProtocol::OpenaiResponses, + WireProtocol::AnthropicMessages, + ]) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn binding(protocol: WireProtocol, model: &str) -> TargetBinding { + TargetBinding { + model: model.into(), + protocol, + endpoint: String::new(), + base_url: "https://provider.example/v1".into(), + weight: 1.0, + headers: BTreeMap::new(), + header_env: BTreeMap::new(), + } + } + + fn config() -> SwitchyardConfig { + SwitchyardConfig { + version: 2, + priority: 0, + max_retries: 3, + algorithm: AlgorithmConfig::Random { seed: Some(42) }, + targets: BTreeMap::from([ + ( + "chat".into(), + binding(WireProtocol::OpenaiChat, "provider/chat"), + ), + ( + "responses".into(), + binding(WireProtocol::OpenaiResponses, "provider/responses"), + ), + ( + "anthropic".into(), + binding(WireProtocol::AnthropicMessages, "provider/anthropic"), + ), + ]), + default_targets: ProtocolDefaults { + openai_chat: "chat".into(), + openai_responses: "responses".into(), + anthropic_messages: "anthropic".into(), + }, + enabled_inbound_profiles: default_enabled_protocols(), + } + } + + #[test] + fn version_two_random_configuration_builds_without_a_service() { + let config = config(); + config.validate().unwrap(); + assert_eq!(config.build_algorithm().unwrap().name(), "random"); + assert_eq!( + config.targets["chat"].dispatch_url(), + "https://provider.example/v1/chat/completions" + ); + } + + #[test] + fn version_one_reports_the_service_to_library_migration() { + let mut config = config(); + config.version = 1; + let error = config.validate().unwrap_err(); + assert!(error.contains("version 1 used switchyard-server")); + assert!(error.contains("version = 2")); + } + + #[test] + fn classifier_targets_are_semantic_names_not_provider_models() { + let mut config = config(); + config.algorithm = AlgorithmConfig::LlmClassifier { + classifier_target: "chat".into(), + weak_target: "responses".into(), + strong_target: "anthropic".into(), + base_threshold: 0.5, + min_confidence: 0.0, + capability_elevated_floor: None, + session_affinity: false, + message_hash_fallback: false, + }; + config.validate().unwrap(); + assert_eq!( + config.build_algorithm().unwrap().name(), + "llm_task_classifier" + ); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/ffi.rs b/crates/switchyard-nemo-relay-plugin/src/ffi.rs new file mode 100644 index 000000000..834e49baf --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/ffi.rs @@ -0,0 +1,471 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::ffi::c_void; +use std::future::Future; +use std::pin::Pin; +use std::ptr; +use std::task::{Context, Poll}; + +use futures::channel::oneshot; +use futures::Stream; +use nemo_relay_plugin::{ + LlmCallErrorV2, LlmCallOutcomeV2, LlmDispatchRequestV2, LlmRequest, LlmStreamEventV2, + NemoRelayNativeAsyncCompletion, NemoRelayNativeAsyncNext, NemoRelayNativeAsyncNextStreamCb, + NemoRelayNativeAsyncStream, NemoRelayNativeHostApiV1, NemoRelayNativeHostApiV4, + NemoRelayNativeLlmStreamV2, NemoRelayNativeString, NemoRelayStatus, +}; +use serde::Serialize; +use serde_json::Value as Json; + +pub struct HostString { + host: NemoRelayNativeHostApiV1, + ptr: *mut NemoRelayNativeString, +} + +impl HostString { + pub fn json(host: &NemoRelayNativeHostApiV1, value: &impl Serialize) -> Result { + let value = serde_json::to_string(value).map_err(|error| error.to_string())?; + Self::text(host, &value) + } + + pub fn text(host: &NemoRelayNativeHostApiV1, value: &str) -> Result { + let mut ptr: *mut NemoRelayNativeString = ptr::null_mut(); + let status = unsafe { (host.string_new)(value.as_ptr(), value.len(), &mut ptr as *mut _) }; + if status == NemoRelayStatus::Ok && !ptr.is_null() { + Ok(Self { host: *host, ptr }) + } else { + Err(format!("Relay host string allocation failed: {status:?}")) + } + } + + pub fn as_ptr(&self) -> *const NemoRelayNativeString { + self.ptr + } +} + +impl Drop for HostString { + fn drop(&mut self) { + unsafe { (self.host.string_free)(self.ptr) }; + } +} + +pub fn read_string( + host: &NemoRelayNativeHostApiV1, + value: *const NemoRelayNativeString, +) -> Result { + if value.is_null() { + return Err("Relay passed a null native string".into()); + } + let len = unsafe { (host.string_len)(value) }; + let data = unsafe { (host.string_data)(value) }; + if data.is_null() && len != 0 { + return Err("Relay passed an invalid native string".into()); + } + let bytes = if len == 0 { + &[][..] + } else { + unsafe { std::slice::from_raw_parts(data, len) } + }; + std::str::from_utf8(bytes) + .map(str::to_owned) + .map_err(|error| error.to_string()) +} + +pub fn read_json( + host: &NemoRelayNativeHostApiV1, + value: *const NemoRelayNativeString, +) -> Result { + serde_json::from_str(&read_string(host, value)?).map_err(|error| error.to_string()) +} + +pub async fn dispatch_buffered( + host: &NemoRelayNativeHostApiV4, + next: *const NemoRelayNativeAsyncNext, + dispatch: &LlmDispatchRequestV2, +) -> Result { + let dispatch = HostString::json(&host.v3.v1, dispatch).map_err(internal_error)?; + let (sender, receiver) = oneshot::channel::(); + let sender = Box::into_raw(Box::new(sender)).cast::(); + let status = unsafe { + (host.async_llm_next_invoke_result_v2)(next, dispatch.as_ptr(), buffered_result, sender) + }; + if status != NemoRelayStatus::Ok { + unsafe { + drop(Box::from_raw( + sender.cast::>(), + )) + }; + return Err(internal_error(format!( + "Relay rejected buffered dispatch: {status:?}" + ))); + } + match receiver.await { + Ok(LlmCallOutcomeV2::Success { response }) => Ok(response), + Ok(LlmCallOutcomeV2::Failure { error }) => Err(error), + Err(_) => Err(internal_error( + "Relay dropped the buffered dispatch callback".into(), + )), + } +} + +pub async fn dispatch_passthrough_buffered( + host: &NemoRelayNativeHostApiV4, + next: *const NemoRelayNativeAsyncNext, + request: &LlmRequest, +) -> Result { + let request = HostString::json(&host.v3.v1, request)?; + let (sender, receiver) = oneshot::channel::>(); + let sender = Box::into_raw(Box::new(sender)).cast::(); + let status = unsafe { + (host.v3.async_next_invoke_result)( + next, + request.as_ptr(), + passthrough_buffered_result, + sender, + ) + }; + if status != NemoRelayStatus::Ok { + unsafe { + drop(Box::from_raw( + sender.cast::>>(), + )) + }; + return Err(format!("Relay rejected passthrough dispatch: {status:?}")); + } + receiver + .await + .map_err(|_| "Relay dropped the passthrough callback".to_string())? +} + +pub async fn dispatch_passthrough_stream( + host: &NemoRelayNativeHostApiV4, + next: *const NemoRelayNativeAsyncNext, + output: *const NemoRelayNativeAsyncStream, + request: &LlmRequest, +) -> Result<(), String> { + let request = HostString::json(&host.v3.v1, request)?; + let (sender, receiver) = oneshot::channel(); + let state = Box::into_raw(Box::new(PassthroughStreamState { + output: output as usize, + sender: Some(sender), + })) + .cast::(); + let status = unsafe { + (host.v3.async_next_invoke_stream)( + next, + request.as_ptr(), + output, + passthrough_stream_result as NemoRelayNativeAsyncNextStreamCb, + state, + ) + }; + if status != NemoRelayStatus::Ok { + unsafe { drop(Box::from_raw(state.cast::())) }; + return Err(format!("Relay rejected passthrough stream: {status:?}")); + } + receiver + .await + .map_err(|_| "Relay dropped the passthrough stream callback".to_string())? +} + +struct PassthroughStreamState { + output: usize, + sender: Option>>, +} + +unsafe extern "C" fn passthrough_stream_result( + user_data: *mut c_void, + chunk_json: *const NemoRelayNativeString, + error: *const NemoRelayNativeString, + done: bool, +) -> bool { + let host = crate::host(); + let state = unsafe { &mut *user_data.cast::() }; + let output = state.output as *const NemoRelayNativeAsyncStream; + let result = if !error.is_null() { + let message = read_string(&host.v3.v1, error) + .unwrap_or_else(|_| "Relay passthrough stream failed".into()); + Some(Err(message)) + } else if done { + let status = finish_stream(host, output); + Some(if status == NemoRelayStatus::Ok { + Ok(()) + } else { + Err(format!( + "Relay rejected passthrough stream finish: {status:?}" + )) + }) + } else { + let result = + read_json(&host.v3.v1, chunk_json).and_then(|chunk| push_stream(host, output, &chunk)); + if result.is_err() { + Some(result) + } else { + None + } + }; + if let Some(result) = result { + let mut state = unsafe { Box::from_raw(user_data.cast::()) }; + if let Some(sender) = state.sender.take() { + let _ = sender.send(result); + } + false + } else { + true + } +} + +unsafe extern "C" fn passthrough_buffered_result( + user_data: *mut c_void, + value_json: *const NemoRelayNativeString, + error: *const NemoRelayNativeString, +) { + let sender = + unsafe { Box::from_raw(user_data.cast::>>()) }; + let host = crate::host(); + let result = if error.is_null() { + read_json(&host.v3.v1, value_json) + } else { + Err(read_string(&host.v3.v1, error) + .unwrap_or_else(|_| "Relay passthrough dispatch failed".into())) + }; + let _ = sender.send(result); +} + +unsafe extern "C" fn buffered_result( + user_data: *mut c_void, + outcome_json: *const NemoRelayNativeString, +) { + let sender = unsafe { Box::from_raw(user_data.cast::>()) }; + let host = crate::host(); + let outcome = read_json(&host.v3.v1, outcome_json) + .and_then(|value| serde_json::from_value(value).map_err(|error| error.to_string())) + .unwrap_or_else(|error| LlmCallOutcomeV2::Failure { + error: internal_error(format!("invalid Relay buffered outcome: {error}")), + }); + let _ = sender.send(outcome); +} + +pub async fn dispatch_stream( + host: &NemoRelayNativeHostApiV4, + next: *const NemoRelayNativeAsyncNext, + output_stream: *const NemoRelayNativeAsyncStream, + dispatch: &LlmDispatchRequestV2, +) -> Result { + let dispatch = HostString::json(&host.v3.v1, dispatch).map_err(internal_error)?; + let (sender, receiver) = oneshot::channel::>(); + let sender = Box::into_raw(Box::new(sender)).cast::(); + let status = unsafe { + (host.async_llm_next_open_stream_v2)( + next, + dispatch.as_ptr(), + output_stream, + provider_stream_open, + sender, + ) + }; + if status != NemoRelayStatus::Ok { + unsafe { + drop(Box::from_raw( + sender.cast::>>(), + )) + }; + return Err(internal_error(format!( + "Relay rejected streaming dispatch: {status:?}" + ))); + } + let stream = receiver + .await + .map_err(|_| internal_error("Relay dropped the stream-open callback".into()))??; + Ok(ProviderJsonStream { + host: *host, + stream: stream as *const NemoRelayNativeLlmStreamV2, + pending: None, + done: false, + }) +} + +unsafe extern "C" fn provider_stream_open( + user_data: *mut c_void, + stream: *const NemoRelayNativeLlmStreamV2, + error_json: *const NemoRelayNativeString, +) { + let sender = unsafe { + Box::from_raw(user_data.cast::>>()) + }; + let host = crate::host(); + let result = if error_json.is_null() { + if stream.is_null() { + Err(internal_error( + "Relay returned neither a provider stream nor an error".into(), + )) + } else { + Ok(stream as usize) + } + } else { + read_json(&host.v3.v1, error_json) + .and_then(|value| serde_json::from_value(value).map_err(|error| error.to_string())) + .map_err(|error| internal_error(format!("invalid Relay stream-open error: {error}"))) + }; + let _ = sender.send(result); +} + +pub struct ProviderJsonStream { + host: NemoRelayNativeHostApiV4, + stream: *const NemoRelayNativeLlmStreamV2, + pending: Option>, + done: bool, +} + +unsafe impl Send for ProviderJsonStream {} + +impl Stream for ProviderJsonStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if self.done { + return Poll::Ready(None); + } + if self.pending.is_none() { + let (sender, receiver) = oneshot::channel(); + let sender = Box::into_raw(Box::new(sender)).cast::(); + let status = unsafe { + (self.host.async_llm_stream_next_v2)(self.stream, provider_stream_next, sender) + }; + if status != NemoRelayStatus::Ok { + unsafe { + drop(Box::from_raw( + sender.cast::>(), + )) + }; + self.done = true; + return Poll::Ready(Some(Err(internal_error(format!( + "Relay rejected provider stream next: {status:?}" + ))))); + } + self.pending = Some(receiver); + } + let receiver = self.pending.as_mut().expect("pending receiver was set"); + match Pin::new(receiver).poll(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(Ok(LlmStreamEventV2::Chunk { chunk })) => { + self.pending = None; + Poll::Ready(Some(Ok(chunk))) + } + Poll::Ready(Ok(LlmStreamEventV2::Failure { error })) => { + self.pending = None; + self.done = true; + Poll::Ready(Some(Err(error))) + } + Poll::Ready(Ok(LlmStreamEventV2::Done)) => { + self.pending = None; + self.done = true; + Poll::Ready(None) + } + Poll::Ready(Err(_)) => { + self.pending = None; + self.done = true; + Poll::Ready(Some(Err(internal_error( + "Relay dropped the provider stream next callback".into(), + )))) + } + } + } +} + +impl Drop for ProviderJsonStream { + fn drop(&mut self) { + if !self.done { + unsafe { (self.host.async_llm_stream_cancel_v2)(self.stream) }; + } + unsafe { (self.host.async_llm_stream_release_v2)(self.stream) }; + } +} + +unsafe extern "C" fn provider_stream_next( + user_data: *mut c_void, + event_json: *const NemoRelayNativeString, +) { + let sender = unsafe { Box::from_raw(user_data.cast::>()) }; + let host = crate::host(); + let event = read_json(&host.v3.v1, event_json) + .and_then(|value| serde_json::from_value(value).map_err(|error| error.to_string())) + .unwrap_or_else(|error| LlmStreamEventV2::Failure { + error: internal_error(format!("invalid Relay stream event: {error}")), + }); + let _ = sender.send(event); +} + +pub fn resolve_completion( + host: &NemoRelayNativeHostApiV4, + completion: *const NemoRelayNativeAsyncCompletion, + value: &Json, +) -> NemoRelayStatus { + match HostString::json(&host.v3.v1, value) { + Ok(value) => unsafe { (host.v3.async_completion_resolve_json)(completion, value.as_ptr()) }, + Err(_) => NemoRelayStatus::Internal, + } +} + +pub fn reject_completion( + host: &NemoRelayNativeHostApiV4, + completion: *const NemoRelayNativeAsyncCompletion, + message: &str, +) -> NemoRelayStatus { + match HostString::text(&host.v3.v1, message) { + Ok(message) => unsafe { (host.v3.async_completion_reject)(completion, message.as_ptr()) }, + Err(_) => NemoRelayStatus::Internal, + } +} + +pub fn push_stream( + host: &NemoRelayNativeHostApiV4, + stream: *const NemoRelayNativeAsyncStream, + value: &Json, +) -> Result<(), String> { + let value = HostString::json(&host.v3.v1, value)?; + loop { + if unsafe { (host.v3.async_stream_is_cancelled)(stream) } { + return Err("Relay caller cancelled the output stream".into()); + } + match unsafe { (host.v3.async_stream_push_json)(stream, value.as_ptr()) } { + NemoRelayStatus::Ok => return Ok(()), + NemoRelayStatus::Internal => std::thread::yield_now(), + status => return Err(format!("Relay rejected output stream event: {status:?}")), + } + } +} + +pub fn finish_stream( + host: &NemoRelayNativeHostApiV4, + stream: *const NemoRelayNativeAsyncStream, +) -> NemoRelayStatus { + unsafe { (host.v3.async_stream_finish)(stream) } +} + +pub fn reject_stream( + host: &NemoRelayNativeHostApiV4, + stream: *const NemoRelayNativeAsyncStream, + message: &str, +) -> NemoRelayStatus { + match HostString::text(&host.v3.v1, message) { + Ok(message) => unsafe { (host.v3.async_stream_reject)(stream, message.as_ptr()) }, + Err(_) => NemoRelayStatus::Internal, + } +} + +pub unsafe fn release_next(host: &NemoRelayNativeHostApiV4, next: *const NemoRelayNativeAsyncNext) { + unsafe { (host.v3.async_next_release)(next) }; +} + +pub unsafe fn release_stream( + host: &NemoRelayNativeHostApiV4, + stream: *const NemoRelayNativeAsyncStream, +) { + unsafe { (host.v3.async_stream_release)(stream) }; +} + +pub fn internal_error(message: String) -> LlmCallErrorV2 { + LlmCallErrorV2::Internal { message } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/lib.rs b/crates/switchyard-nemo-relay-plugin/src/lib.rs new file mode 100644 index 000000000..4234fb9bd --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/lib.rs @@ -0,0 +1,199 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +mod config; +mod ffi; +mod runtime; +mod translation; + +use std::ffi::c_void; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::{Arc, OnceLock}; + +use nemo_relay_plugin::{ + ConfigDiagnostic, DiagnosticLevel, Json, NativePlugin, NemoRelayNativeAsyncCallbackState, + NemoRelayNativeAsyncCompletion, NemoRelayNativeAsyncNext, NemoRelayNativeAsyncStream, + NemoRelayNativeHostApiV4, NemoRelayNativeString, NemoRelayStatus, PluginContext, +}; +use serde_json::Map; + +use crate::config::SwitchyardConfig; +use crate::runtime::{Invocation, SwitchyardRuntime}; + +static HOST: OnceLock = OnceLock::new(); + +pub(crate) fn host() -> &'static NemoRelayNativeHostApiV4 { + HOST.get() + .expect("Switchyard callback invoked before plugin registration") +} + +#[derive(Default)] +struct SwitchyardPlugin; + +impl NativePlugin for SwitchyardPlugin { + fn plugin_kind(&self) -> &str { + "nvidia.switchyard" + } + + fn allows_multiple_components(&self) -> bool { + false + } + + fn validate(&self, plugin_config: &Map) -> Vec { + match parse_config(plugin_config).and_then(|config| config.validate()) { + Ok(()) => Vec::new(), + Err(message) => vec![ConfigDiagnostic { + level: DiagnosticLevel::Error, + code: "switchyard.invalid_config".into(), + component: Some("nvidia.switchyard".into()), + field: Some("config".into()), + message, + }], + } + } + + fn register( + &mut self, + plugin_config: &Map, + ctx: &mut PluginContext<'_>, + ) -> nemo_relay_plugin::Result<()> { + let host = *ctx + .host_api_v4() + .ok_or_else(|| "Switchyard requires Relay native plugin C API v2".to_string())?; + if let Some(registered) = HOST.get() { + if registered.v3.v1.abi_version != host.v3.v1.abi_version { + return Err("Switchyard was initialized with a different Relay host ABI".into()); + } + } else { + HOST.set(host) + .map_err(|_| "failed to retain the Relay native API v2 host table".to_string())?; + } + + let config = parse_config(plugin_config)?; + let priority = config.priority; + let runtime = Arc::new(SwitchyardRuntime::new(config, ctx.runtime())?); + + let buffered_state = Box::into_raw(Box::new(Arc::clone(&runtime))).cast::(); + let status = unsafe { + ctx.register_async_llm_execution_v2_raw( + "switchyard.run_stream.buffered", + priority, + buffered_callback, + buffered_state, + Some(free_runtime), + ) + }; + if status != NemoRelayStatus::Ok { + return Err(format!( + "failed to register Switchyard buffered execution: {status:?}" + )); + } + + let stream_state = Box::into_raw(Box::new(runtime)).cast::(); + let status = unsafe { + ctx.register_async_llm_stream_execution_v2_raw( + "switchyard.run_stream.streaming", + priority, + stream_callback, + stream_state, + Some(free_runtime), + ) + }; + if status != NemoRelayStatus::Ok { + return Err(format!( + "failed to register Switchyard streaming execution: {status:?}" + )); + } + Ok(()) + } +} + +fn parse_config(plugin_config: &Map) -> Result { + serde_json::from_value(Json::Object(plugin_config.clone())) + .map_err(|error| format!("invalid Switchyard configuration: {error}")) +} + +unsafe extern "C" fn free_runtime(user_data: *mut c_void) { + if !user_data.is_null() { + unsafe { drop(Box::from_raw(user_data.cast::>())) }; + } +} + +unsafe extern "C" fn buffered_callback( + user_data: *mut c_void, + invocation_json: *const NemoRelayNativeString, + next: *const NemoRelayNativeAsyncNext, + completion: *const NemoRelayNativeAsyncCompletion, +) -> u32 { + let result = catch_unwind(AssertUnwindSafe(|| { + let runtime = unsafe { &*user_data.cast::>() }; + let invocation = ffi::read_json(&host().v3.v1, invocation_json).and_then(|value| { + serde_json::from_value::(value).map_err(|e| e.to_string()) + }); + match invocation { + Ok(invocation) => { + match futures::executor::block_on(runtime.execute_buffered( + invocation, + host(), + next, + )) { + Ok(response) => { + let _ = ffi::resolve_completion(host(), completion, &response); + } + Err(error) => { + let _ = ffi::reject_completion(host(), completion, &error); + } + } + } + Err(error) => { + let _ = ffi::reject_completion( + host(), + completion, + &format!("invalid Relay LLM invocation: {error}"), + ); + } + } + })); + if result.is_err() { + let _ = + ffi::reject_completion(host(), completion, "Switchyard buffered execution panicked"); + } + unsafe { ffi::release_next(host(), next) }; + NemoRelayNativeAsyncCallbackState::Complete as u32 +} + +unsafe extern "C" fn stream_callback( + user_data: *mut c_void, + invocation_json: *const NemoRelayNativeString, + next: *const NemoRelayNativeAsyncNext, + output: *const NemoRelayNativeAsyncStream, +) -> u32 { + let result = catch_unwind(AssertUnwindSafe(|| { + let runtime = unsafe { &*user_data.cast::>() }; + let invocation = ffi::read_json(&host().v3.v1, invocation_json).and_then(|value| { + serde_json::from_value::(value).map_err(|e| e.to_string()) + }); + let result = match invocation { + Ok(invocation) => futures::executor::block_on(runtime.execute_stream( + invocation, + host(), + next, + output, + )), + Err(error) => Err(format!("invalid Relay LLM stream invocation: {error}")), + }; + if let Err(error) = result { + let _ = ffi::reject_stream(host(), output, &error); + } + })); + if result.is_err() { + let _ = ffi::reject_stream(host(), output, "Switchyard streaming execution panicked"); + } + unsafe { + ffi::release_next(host(), next); + ffi::release_stream(host(), output); + } + NemoRelayNativeAsyncCallbackState::Complete as u32 +} + +nemo_relay_plugin::nemo_relay_plugin_v2!(nemo_relay_register_plugin, SwitchyardPlugin::default); diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs new file mode 100644 index 000000000..6e9e3d731 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -0,0 +1,837 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeMap; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::{Context as TaskContext, Poll}; + +use futures::Stream; +use futures_util::StreamExt; +use nemo_relay_plugin::{ + Json, LlmCallErrorV2, LlmDispatchRequestV2, LlmDispatchTargetV2, LlmRequest as RelayRequest, + NemoRelayNativeAsyncNext, NemoRelayNativeAsyncStream, NemoRelayNativeHostApiV4, PluginRuntime, +}; +use serde::Deserialize; +use serde_json::{json, Map}; +use switchyard_libsy::{ + Algorithm, CallLlmRequest, Context, LibsyError, LlmResponse, Request, Response, Step, +}; +use switchyard_protocol::{ + LlmClientError, LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, Metadata, +}; +use switchyard_translation::{StreamTranslationState, TranslationEngine}; + +use crate::config::{SwitchyardConfig, TargetBinding, WireProtocol}; +use crate::{ffi, translation}; + +#[derive(Deserialize)] +pub struct Invocation { + pub name: String, + pub request: RelayRequest, +} + +pub struct SwitchyardRuntime { + config: SwitchyardConfig, + algorithm: Arc, + target_headers: BTreeMap>, + translation: TranslationEngine, + relay: PluginRuntime, +} + +impl SwitchyardRuntime { + pub fn new(config: SwitchyardConfig, relay: PluginRuntime) -> Result { + config.validate()?; + let algorithm = config.build_algorithm()?; + let target_headers = config + .targets + .iter() + .map(|(name, target)| { + target + .resolved_headers() + .map(|headers| (name.clone(), headers)) + }) + .collect::>()?; + Ok(Self { + config, + algorithm, + target_headers, + translation: TranslationEngine::default(), + relay, + }) + } + + pub async fn execute_buffered( + &self, + invocation: Invocation, + host: &NemoRelayNativeHostApiV4, + next: *const NemoRelayNativeAsyncNext, + ) -> Result { + let Some(inbound) = WireProtocol::from_call(&invocation.name) else { + return ffi::dispatch_passthrough_buffered(host, next, &invocation.request).await; + }; + if !self.config.enabled_inbound_profiles.contains(&inbound) { + return ffi::dispatch_passthrough_buffered(host, next, &invocation.request).await; + } + let request = self.libsy_request(inbound, &invocation.request, false)?; + let max_attempts = self.config.max_retries.saturating_add(1); + for attempt in 1..=max_attempts { + self.mark( + "switchyard.routing.requested", + json!({"algorithm": self.algorithm.name(), "attempt": attempt}), + identity_metadata(&invocation.request), + ); + match self + .drive_buffered( + request.clone(), + host, + next, + attempt, + identity_metadata(&invocation.request), + ) + .await + { + Ok(response) => { + return match response.llm_response { + LlmResponse::Agg(response) => { + translation::encode_response(&self.translation, inbound, &response) + } + LlmResponse::Stream(_) => { + Err("libsy returned a stream for a buffered request".into()) + } + }; + } + Err(failure) if failure.retryable() && attempt < max_attempts => { + self.mark( + "switchyard.routing.retry", + json!({"attempt": attempt, "error": failure.error.to_string()}), + identity_metadata(&invocation.request), + ); + } + Err(failure) => { + self.mark( + "switchyard.routing.error", + json!({ + "attempt": attempt, + "retryable": failure.retryable(), + "error": failure.error.to_string(), + }), + identity_metadata(&invocation.request), + ); + return self + .fallback_buffered( + inbound, + request, + host, + next, + identity_metadata(&invocation.request), + ) + .await; + } + } + } + Err("Switchyard retry loop ended without a result".into()) + } + + async fn drive_buffered( + &self, + request: Request, + host: &NemoRelayNativeHostApiV4, + next: *const NemoRelayNativeAsyncNext, + attempt: u32, + mark_metadata: Json, + ) -> Result { + let mut context = Context::default(); + context + .values + .insert("relay.routing_attempt".into(), attempt.to_string()); + let mut steps = self.algorithm.clone().run_stream(context, request); + let provider_error = Arc::new(Mutex::new(None)); + while let Some(step) = steps.next().await { + match step { + Ok(Step::Decision(decision)) => { + self.mark( + "switchyard.routing.decision", + json!({ + "algorithm": self.algorithm.name(), + "attempt": attempt, + "selected_target": decision.selected_model(), + "reasoning": decision.reasoning(), + "routing_tier": decision.routing_tier(), + "is_routed_call": decision.is_routed_call(), + }), + mark_metadata.clone(), + ); + } + Ok(Step::CallLlm(call)) => { + self.serve_buffered_call(*call, host, next, Arc::clone(&provider_error)) + .await + .map_err(|error| RunFailure::new(error, &provider_error))?; + } + Ok(Step::ReturnToAgent(response)) => return Ok(*response), + Err(error) => return Err(RunFailure::new(error, &provider_error)), + } + } + Err(RunFailure::new( + LibsyError::MissingFinalResponse, + &provider_error, + )) + } + + async fn serve_buffered_call( + &self, + call: CallLlmRequest, + host: &NemoRelayNativeHostApiV4, + next: *const NemoRelayNativeAsyncNext, + provider_error: Arc>>, + ) -> switchyard_libsy::Result<()> { + let routed = call.get_routed().clone(); + let target_name = routed.decision.selected_model().to_string(); + let result = async { + let target = self.target(&target_name)?; + let request = self.dispatch_request(&target_name, target, routed.request, false)?; + match ffi::dispatch_buffered(host, next, &request).await { + Ok(response) => { + let response = + translation::decode_response(&self.translation, target.protocol, &response) + .map_err(LlmClientError::ResponseTranslation)?; + Ok(Response { + llm_response: LlmResponse::Agg(response), + metadata: Some(Metadata { + wire_format: Some(target.protocol.wire_format()), + ..Metadata::default() + }), + }) + } + Err(error) => { + if let Ok(mut stored) = provider_error.lock() { + *stored = Some(error.clone()); + } + Err(client_error(error, &target_name)) + } + } + } + .await + .map_err(|source| LibsyError::client_call(target_name, source)); + call.respond(result) + } + + async fn fallback_buffered( + &self, + inbound: WireProtocol, + request: Request, + host: &NemoRelayNativeHostApiV4, + next: *const NemoRelayNativeAsyncNext, + metadata: Json, + ) -> Result { + let target_name = self.config.default_targets.target(inbound); + let target = self + .target(target_name) + .map_err(|error| error.to_string())?; + self.mark( + "switchyard.routing.fallback", + json!({"selected_target": target_name}), + metadata, + ); + let dispatch = self + .dispatch_request(target_name, target, request, false) + .map_err(|error| error.to_string())?; + let response = ffi::dispatch_buffered(host, next, &dispatch) + .await + .map_err(|error| format!("trusted fallback failed: {error:?}"))?; + let response = translation::decode_response(&self.translation, target.protocol, &response)?; + translation::encode_response(&self.translation, inbound, &response) + } + + fn libsy_request( + &self, + inbound: WireProtocol, + original: &RelayRequest, + streaming: bool, + ) -> Result { + let mut request = translation::decode_request(&self.translation, inbound, original)?; + request.stream = streaming; + let headers = string_headers(&original.headers); + let mut metadata = Metadata::from_headers(&headers); + metadata.wire_format = Some(inbound.wire_format()); + Ok(Request { + llm_request: request, + raw_request: Some(original.content.clone()), + metadata: Some(metadata), + }) + } + + fn dispatch_request( + &self, + target_name: &str, + target: &TargetBinding, + mut request: Request, + streaming: bool, + ) -> Result { + request.llm_request.stream = streaming; + let headers = self + .target_headers + .get(target_name) + .cloned() + .unwrap_or_default(); + let mut request = translation::encode_request( + &self.translation, + target.protocol, + &request.llm_request, + headers, + ) + .map_err(LlmClientError::RequestEncoding)?; + let body = request.content.as_object_mut().ok_or_else(|| { + LlmClientError::RequestEncoding("translated provider request is not an object".into()) + })?; + body.insert("model".into(), Json::String(target.model.clone())); + body.insert("stream".into(), Json::Bool(streaming)); + Ok(LlmDispatchRequestV2 { + request, + target: LlmDispatchTargetV2 { + url: target.dispatch_url(), + route: target.protocol.relay_route(), + }, + }) + } + + fn target(&self, name: &str) -> Result<&TargetBinding, LlmClientError> { + self.config + .targets + .get(name) + .ok_or_else(|| LlmClientError::Configuration { + message: format!("libsy selected unknown target {name:?}"), + }) + } + + fn mark(&self, name: &str, data: Json, metadata: Json) { + if let Err(error) = self.relay.emit_mark(name, Some(&data), Some(&metadata)) { + eprintln!("Switchyard could not emit routing mark {name:?}: {error}"); + } + } + + pub async fn execute_stream( + &self, + invocation: Invocation, + host: &NemoRelayNativeHostApiV4, + next: *const NemoRelayNativeAsyncNext, + output: *const NemoRelayNativeAsyncStream, + ) -> Result<(), String> { + let Some(inbound) = WireProtocol::from_call(&invocation.name) else { + return ffi::dispatch_passthrough_stream(host, next, output, &invocation.request).await; + }; + if !self.config.enabled_inbound_profiles.contains(&inbound) { + return ffi::dispatch_passthrough_stream(host, next, output, &invocation.request).await; + } + let request = self.libsy_request(inbound, &invocation.request, true)?; + let metadata = identity_metadata(&invocation.request); + let max_attempts = self.config.max_retries.saturating_add(1); + for attempt in 1..=max_attempts { + self.mark( + "switchyard.routing.requested", + json!({"algorithm": self.algorithm.name(), "attempt": attempt}), + metadata.clone(), + ); + let run = match self + .drive_stream( + request.clone(), + host, + next, + output, + attempt, + metadata.clone(), + ) + .await + { + Ok(response) => response, + Err(failure) if failure.retryable() && attempt < max_attempts => { + self.emit_stream_retry(attempt, &failure, &metadata); + continue; + } + Err(failure) => { + self.emit_stream_error(attempt, &failure, &metadata); + return self + .fallback_stream(inbound, request, host, next, output, metadata.clone()) + .await; + } + }; + match self + .emit_returned_stream(run.response, inbound, host, output, &run.provider_error) + .await + { + Ok(()) => { + let status = ffi::finish_stream(host, output); + return if status == nemo_relay_plugin::NemoRelayStatus::Ok { + Ok(()) + } else { + Err(format!("Relay rejected output stream finish: {status:?}")) + }; + } + Err(failure) + if !failure.committed + && failure.failure.retryable() + && attempt < max_attempts => + { + self.emit_stream_retry(attempt, &failure.failure, &metadata); + } + Err(failure) if !failure.committed => { + self.emit_stream_error(attempt, &failure.failure, &metadata); + return self + .fallback_stream(inbound, request, host, next, output, metadata.clone()) + .await; + } + Err(failure) => { + self.emit_stream_error(attempt, &failure.failure, &metadata); + return Err(format!( + "Switchyard stream failed after response commitment: {}", + failure.failure.error + )); + } + } + } + Err("Switchyard stream retry loop ended without a result".into()) + } + + async fn drive_stream( + &self, + request: Request, + host: &NemoRelayNativeHostApiV4, + next: *const NemoRelayNativeAsyncNext, + output: *const NemoRelayNativeAsyncStream, + attempt: u32, + mark_metadata: Json, + ) -> Result { + let mut context = Context::default(); + context + .values + .insert("relay.routing_attempt".into(), attempt.to_string()); + let mut steps = self.algorithm.clone().run_stream(context, request); + let provider_error = Arc::new(Mutex::new(None)); + while let Some(step) = steps.next().await { + match step { + Ok(Step::Decision(decision)) => { + self.mark( + "switchyard.routing.decision", + json!({ + "algorithm": self.algorithm.name(), + "attempt": attempt, + "selected_target": decision.selected_model(), + "reasoning": decision.reasoning(), + "routing_tier": decision.routing_tier(), + "is_routed_call": decision.is_routed_call(), + }), + mark_metadata.clone(), + ); + } + Ok(Step::CallLlm(call)) => { + self.serve_stream_call(*call, host, next, output, Arc::clone(&provider_error)) + .await + .map_err(|error| RunFailure::new(error, &provider_error))?; + } + Ok(Step::ReturnToAgent(response)) => { + return Ok(StreamRun { + response: *response, + provider_error, + }); + } + Err(error) => return Err(RunFailure::new(error, &provider_error)), + } + } + Err(RunFailure::new( + LibsyError::MissingFinalResponse, + &provider_error, + )) + } + + async fn serve_stream_call( + &self, + call: CallLlmRequest, + host: &NemoRelayNativeHostApiV4, + next: *const NemoRelayNativeAsyncNext, + output: *const NemoRelayNativeAsyncStream, + provider_error: Arc>>, + ) -> switchyard_libsy::Result<()> { + let routed = call.get_routed().clone(); + let target_name = routed.decision.selected_model().to_string(); + let result = self + .provider_stream_response( + &target_name, + routed.request, + host, + next, + output, + Arc::clone(&provider_error), + ) + .await + .map_err(|source| LibsyError::client_call(target_name, source)); + call.respond(result) + } + + async fn provider_stream_response( + &self, + target_name: &str, + request: Request, + host: &NemoRelayNativeHostApiV4, + next: *const NemoRelayNativeAsyncNext, + output: *const NemoRelayNativeAsyncStream, + provider_error: Arc>>, + ) -> Result { + let target = self.target(target_name)?; + let metadata = request.metadata.clone(); + let dispatch = self.dispatch_request(target_name, target, request, true)?; + let mut upstream = match ffi::dispatch_stream(host, next, output, &dispatch).await { + Ok(upstream) => upstream, + Err(error) => { + remember_provider_error(&provider_error, &error); + return Err(client_error(error, target_name)); + } + }; + let first_raw = match upstream.next().await { + Some(Ok(first)) => first, + Some(Err(error)) => { + remember_provider_error(&provider_error, &error); + return Err(client_error(error, target_name)); + } + None => { + return Err(LlmClientError::InvalidResponse { + source: Box::new(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "provider returned an empty stream", + )), + }); + } + }; + let mut state = StreamTranslationState::new( + target.protocol.wire_format(), + target.protocol.wire_format(), + ); + let first = + decode_provider_event(&self.translation, &mut state, target.protocol, first_raw)?; + let stream: LlmResponseStream = Box::pin(TranslatedProviderStream { + upstream, + first: Some(first), + protocol: target.protocol, + state, + target_name: target_name.to_string(), + provider_error, + }); + Ok(Response { + llm_response: LlmResponse::Stream(stream), + metadata: Some(Metadata { + wire_format: Some(target.protocol.wire_format()), + ..metadata.unwrap_or_default() + }), + }) + } + + async fn emit_returned_stream( + &self, + response: Response, + inbound: WireProtocol, + host: &NemoRelayNativeHostApiV4, + output: *const NemoRelayNativeAsyncStream, + provider_error: &Arc>>, + ) -> Result<(), StreamAttemptFailure> { + let source = response + .metadata + .as_ref() + .and_then(|metadata| metadata.wire_format.as_ref()) + .and_then(WireProtocol::from_wire_format) + .ok_or_else(|| { + StreamAttemptFailure::translation( + "libsy returned a stream without a supported source wire format", + false, + provider_error, + ) + })?; + let LlmResponse::Stream(mut stream) = response.llm_response else { + return Err(StreamAttemptFailure::translation( + "libsy returned a buffered response for a streaming request", + false, + provider_error, + )); + }; + let mut state = StreamTranslationState::new(source.wire_format(), inbound.wire_format()); + let mut committed = false; + while let Some(item) = stream.next().await { + let event = item + .map_err(|error| StreamAttemptFailure::client(error, committed, provider_error))?; + let events = + translation::encode_stream_event(&self.translation, &mut state, inbound, event) + .map_err(|error| { + StreamAttemptFailure::translation(&error, committed, provider_error) + })?; + for event in events { + ffi::push_stream(host, output, &event).map_err(|error| { + StreamAttemptFailure::translation(&error, committed, provider_error) + })?; + committed = true; + } + } + if source != inbound { + let events = translation::finish_stream(&self.translation, &mut state, inbound) + .map_err(|error| { + StreamAttemptFailure::translation(&error, committed, provider_error) + })?; + for event in events { + ffi::push_stream(host, output, &event).map_err(|error| { + StreamAttemptFailure::translation(&error, committed, provider_error) + })?; + committed = true; + } + } + if !committed { + return Err(StreamAttemptFailure::translation( + "Switchyard produced an empty output stream", + false, + provider_error, + )); + } + Ok(()) + } + + async fn fallback_stream( + &self, + inbound: WireProtocol, + request: Request, + host: &NemoRelayNativeHostApiV4, + next: *const NemoRelayNativeAsyncNext, + output: *const NemoRelayNativeAsyncStream, + metadata: Json, + ) -> Result<(), String> { + let target_name = self.config.default_targets.target(inbound).to_string(); + self.mark( + "switchyard.routing.fallback", + json!({"selected_target": target_name}), + metadata, + ); + let provider_error = Arc::new(Mutex::new(None)); + let response = self + .provider_stream_response( + &target_name, + request, + host, + next, + output, + Arc::clone(&provider_error), + ) + .await + .map_err(|error| format!("trusted fallback stream failed: {error}"))?; + self.emit_returned_stream(response, inbound, host, output, &provider_error) + .await + .map_err(|failure| { + format!("trusted fallback stream failed: {}", failure.failure.error) + })?; + let status = ffi::finish_stream(host, output); + if status == nemo_relay_plugin::NemoRelayStatus::Ok { + Ok(()) + } else { + Err(format!("Relay rejected fallback stream finish: {status:?}")) + } + } + + fn emit_stream_retry(&self, attempt: u32, failure: &RunFailure, metadata: &Json) { + self.mark( + "switchyard.routing.retry", + json!({"attempt": attempt, "error": failure.error.to_string()}), + metadata.clone(), + ); + } + + fn emit_stream_error(&self, attempt: u32, failure: &RunFailure, metadata: &Json) { + self.mark( + "switchyard.routing.error", + json!({ + "attempt": attempt, + "retryable": failure.retryable(), + "error": failure.error.to_string(), + }), + metadata.clone(), + ); + } +} + +struct StreamRun { + response: Response, + provider_error: Arc>>, +} + +struct TranslatedProviderStream { + upstream: ffi::ProviderJsonStream, + first: Option, + protocol: WireProtocol, + state: StreamTranslationState, + target_name: String, + provider_error: Arc>>, +} + +impl Stream for TranslatedProviderStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { + if let Some(first) = self.first.take() { + return Poll::Ready(Some(Ok(first))); + } + match Pin::new(&mut self.upstream).poll_next(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(Some(Ok(raw))) => { + let translation = TranslationEngine::default(); + let protocol = self.protocol; + Poll::Ready(Some(decode_provider_event( + &translation, + &mut self.state, + protocol, + raw, + ))) + } + Poll::Ready(Some(Err(error))) => { + remember_provider_error(&self.provider_error, &error); + Poll::Ready(Some(Err(client_error(error, &self.target_name)))) + } + Poll::Ready(None) => Poll::Ready(None), + } + } +} + +fn decode_provider_event( + translation: &TranslationEngine, + state: &mut StreamTranslationState, + protocol: WireProtocol, + raw: Json, +) -> Result { + let event = translation::decode_stream_event(translation, state, protocol, raw) + .map_err(LlmClientError::ResponseTranslation)?; + for chunk in event.normalized() { + match chunk { + LlmResponseChunk::DecodeError { message } => { + return Err(LlmClientError::ResponseTranslation(message.clone())); + } + LlmResponseChunk::StreamError { message } => { + return Err(LlmClientError::UpstreamHttp { + status: 502, + body: message.clone(), + }); + } + _ => {} + } + } + Ok(event) +} + +fn remember_provider_error( + provider_error: &Arc>>, + error: &LlmCallErrorV2, +) { + if let Ok(mut stored) = provider_error.lock() { + *stored = Some(error.clone()); + } +} + +struct StreamAttemptFailure { + failure: RunFailure, + committed: bool, +} + +impl StreamAttemptFailure { + fn client( + error: LlmClientError, + committed: bool, + provider_error: &Arc>>, + ) -> Self { + Self { + failure: RunFailure::new( + LibsyError::client_call("return_to_agent", error), + provider_error, + ), + committed, + } + } + + fn translation( + error: &str, + committed: bool, + provider_error: &Arc>>, + ) -> Self { + Self::client( + LlmClientError::ResponseTranslation(error.to_string()), + committed, + provider_error, + ) + } +} + +struct RunFailure { + error: LibsyError, + provider_error: Option, +} + +impl RunFailure { + fn new(error: LibsyError, provider_error: &Arc>>) -> Self { + Self { + error, + provider_error: provider_error.lock().ok().and_then(|error| error.clone()), + } + } + + fn retryable(&self) -> bool { + self.provider_error + .as_ref() + .is_some_and(LlmCallErrorV2::is_retryable) + } +} + +fn client_error(error: LlmCallErrorV2, model: &str) -> LlmClientError { + match error { + LlmCallErrorV2::Upstream { + class, + status, + body, + .. + } => match class { + nemo_relay_plugin::LlmUpstreamFailureClassV2::Connection => LlmClientError::Transport { + source: Box::new(std::io::Error::other(body)), + }, + nemo_relay_plugin::LlmUpstreamFailureClassV2::Timeout => LlmClientError::Timeout { + source: Box::new(std::io::Error::new(std::io::ErrorKind::TimedOut, body)), + }, + nemo_relay_plugin::LlmUpstreamFailureClassV2::ContextWindow => { + LlmClientError::ContextWindowExceeded { + model: model.into(), + message: body, + } + } + nemo_relay_plugin::LlmUpstreamFailureClassV2::InvalidRequest + | nemo_relay_plugin::LlmUpstreamFailureClassV2::Authentication => { + LlmClientError::InvalidRequest { message: body } + } + _ => match status { + Some(status) => LlmClientError::UpstreamHttp { status, body }, + None => LlmClientError::General(body), + }, + }, + LlmCallErrorV2::InvalidRequest { message } + | LlmCallErrorV2::GuardrailRejected { message } => { + LlmClientError::InvalidRequest { message } + } + LlmCallErrorV2::Cancelled { message } | LlmCallErrorV2::Internal { message } => { + LlmClientError::General(message) + } + } +} + +fn string_headers(headers: &Map) -> BTreeMap { + headers + .iter() + .filter_map(|(name, value)| value.as_str().map(|value| (name.clone(), value.into()))) + .collect() +} + +fn identity_metadata(request: &RelayRequest) -> Json { + let metadata = Metadata::from_headers(&string_headers(&request.headers)); + json!({ + "session_id": metadata.session_id, + "agent_id": metadata.agent_id, + "turn_id": metadata.turn_id, + "request_id": metadata.correlation_id, + }) +} diff --git a/crates/switchyard-nemo-relay-plugin/src/translation.rs b/crates/switchyard-nemo-relay-plugin/src/translation.rs new file mode 100644 index 000000000..aa04d8b21 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/translation.rs @@ -0,0 +1,230 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use nemo_relay_plugin::LlmRequest as RelayRequest; +use serde_json::{Map, Value as Json}; +use switchyard_protocol::{AggLlmResponse, LlmRequest}; +use switchyard_translation::{ + DeterministicIdPolicy, DiagnosticSeverity, LossyConversionPolicy, PreservationPolicy, + StreamTranslationState, TargetCapabilities, TranslationDiagnostic, TranslationEngine, + TranslationPolicy, UnknownFieldPolicy, +}; + +use crate::config::WireProtocol; + +pub fn decode_request( + engine: &TranslationEngine, + protocol: WireProtocol, + request: &RelayRequest, +) -> Result { + let output = engine + .decode_request(protocol.wire_format(), &request.content, &policy()) + .map_err(error)?; + safe(&output.diagnostics)?; + Ok(output.request) +} + +pub fn encode_request( + engine: &TranslationEngine, + protocol: WireProtocol, + request: &LlmRequest, + headers: Map, +) -> Result { + let output = engine + .encode_request(protocol.wire_format(), request, &request_policy(protocol)) + .map_err(error)?; + safe(&output.diagnostics)?; + Ok(RelayRequest { + headers, + content: output.body, + }) +} + +pub fn decode_response( + engine: &TranslationEngine, + protocol: WireProtocol, + response: &Json, +) -> Result { + let output = engine + .decode_response(protocol.wire_format(), response, &policy()) + .map_err(error)?; + safe(&output.diagnostics)?; + Ok(output.response) +} + +pub fn encode_response( + engine: &TranslationEngine, + protocol: WireProtocol, + response: &AggLlmResponse, +) -> Result { + let output = engine + .encode_response(protocol.wire_format(), response, &policy()) + .map_err(error)?; + safe(&output.diagnostics)?; + Ok(output.body) +} + +pub fn decode_stream_event( + engine: &TranslationEngine, + state: &mut StreamTranslationState, + protocol: WireProtocol, + event: Json, +) -> Result { + engine + .decode_stream_event(state, protocol.wire_format(), event) + .map_err(error) +} + +pub fn encode_stream_event( + engine: &TranslationEngine, + state: &mut StreamTranslationState, + protocol: WireProtocol, + event: switchyard_protocol::LlmResponseStreamEvent, +) -> Result, String> { + engine + .encode_stream_event(state, protocol.wire_format(), event) + .map_err(error) +} + +pub fn finish_stream( + engine: &TranslationEngine, + state: &mut StreamTranslationState, + protocol: WireProtocol, +) -> Result, String> { + engine + .finish_stream(state, protocol.wire_format()) + .map_err(error) +} + +fn policy() -> TranslationPolicy { + TranslationPolicy { + unknown_field_policy: UnknownFieldPolicy::Preserve, + lossy_conversion_policy: LossyConversionPolicy::Reject, + deterministic_ids: DeterministicIdPolicy::GenerateStable { + prefix: "relay".into(), + }, + preservation: PreservationPolicy::InMemory, + target_capabilities: TargetCapabilities::default(), + } +} + +fn request_policy(protocol: WireProtocol) -> TranslationPolicy { + let mut policy = policy(); + if protocol == WireProtocol::AnthropicMessages { + policy + .target_capabilities + .supports_json_schema_response_format = Some(false); + } + policy +} + +fn safe(diagnostics: &[TranslationDiagnostic]) -> Result<(), String> { + let unsafe_diagnostics = diagnostics + .iter() + .filter(|diagnostic| diagnostic.severity != DiagnosticSeverity::Info) + .collect::>(); + if unsafe_diagnostics.is_empty() { + Ok(()) + } else { + Err(format!( + "Switchyard translation was not lossless: {unsafe_diagnostics:?}" + )) + } +} + +fn error(error: switchyard_translation::TranslationError) -> String { + format!("Switchyard translation failed: {error}") +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn plugin_replays_same_protocol_provider_extensions_exactly() { + let cases = [ + ( + WireProtocol::OpenaiChat, + json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "gpt-test", + "system_fingerprint": "fp_provider_specific", + "choices": [{ + "index": 0, + "delta": {"content": "Hi"}, + "finish_reason": null + }] + }), + ), + ( + WireProtocol::OpenaiResponses, + json!({ + "type": "response.output_text.delta", + "item_id": "item-1", + "output_index": 0, + "content_index": 0, + "delta": "Hi", + "sequence_number": 2, + "provider_extension": {"exact": true} + }), + ), + ( + WireProtocol::AnthropicMessages, + json!({ + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "Hi"}, + "provider_extension": {"exact": true} + }), + ), + ]; + + let engine = TranslationEngine::default(); + for (protocol, raw) in cases { + let mut state = + StreamTranslationState::new(protocol.wire_format(), protocol.wire_format()); + let event = decode_stream_event(&engine, &mut state, protocol, raw.clone()).unwrap(); + assert_eq!( + encode_stream_event(&engine, &mut state, protocol, event).unwrap(), + vec![raw] + ); + } + } + + #[test] + fn cross_protocol_streams_use_normalized_content_not_raw_extensions() { + let engine = TranslationEngine::default(); + let mut state = StreamTranslationState::new( + WireProtocol::OpenaiChat.wire_format(), + WireProtocol::AnthropicMessages.wire_format(), + ); + let event = decode_stream_event( + &engine, + &mut state, + WireProtocol::OpenaiChat, + json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "gpt-test", + "system_fingerprint": "not-portable", + "choices": [{ + "index": 0, + "delta": {"content": "Hi"}, + "finish_reason": null + }] + }), + ) + .unwrap(); + let translated = + encode_stream_event(&engine, &mut state, WireProtocol::AnthropicMessages, event) + .unwrap(); + assert!(translated + .iter() + .any(|event| { event.pointer("/delta/text").and_then(Json::as_str) == Some("Hi") })); + assert!(translated + .iter() + .all(|event| event.get("system_fingerprint").is_none())); + } +} From 62afc8cd0b5dbdbfa1b81b12fa88c289f2f17e17 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 30 Jul 2026 16:59:58 -0600 Subject: [PATCH 13/51] test(plugin): add Relay process and bundle validation Signed-off-by: Bryan Bednarski --- README.md | 7 + crates/switchyard-nemo-relay-plugin/README.md | 146 ++++ .../scripts/package_bundle.py | 59 ++ .../tests/e2e/fake_provider.py | 285 ++++++++ .../tests/e2e/run_e2e.py | 669 ++++++++++++++++++ docs/index.md | 1 + 6 files changed, 1167 insertions(+) create mode 100644 crates/switchyard-nemo-relay-plugin/README.md create mode 100644 crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py create mode 100644 crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py create mode 100644 crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py diff --git a/README.md b/README.md index 751ad11c6..d5df69b97 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,12 @@ Verify the proxy in another terminal: curl http://localhost:4000/health ``` +NeMo Relay can instead load libsy in process through the native API v2 +`nvidia.switchyard` plugin. Relay performs provider I/O while the plugin drives +`run_stream` and uses `switchyard-translation` for every request and response. +See the +[`switchyard-nemo-relay-plugin` guide](crates/switchyard-nemo-relay-plugin/README.md). + For a complete configuration and a test request, follow [Getting Started](docs/getting_started.md). @@ -119,6 +125,7 @@ configured LLM client selects one upstream format. - **[`switchyard-libsy`](crates/libsy/README.md)**: embed routing algorithms in a Rust application - **[`switchyard-protocol`](crates/protocol/README.md)**: provider-neutral request, response, and streaming types - **[`switchyard-translation`](crates/switchyard-translation/README.md)**: request, response, and stream translation +- **[`switchyard-nemo-relay-plugin`](crates/switchyard-nemo-relay-plugin/README.md)**: in-process routing for NeMo Relay ## Community diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md new file mode 100644 index 000000000..4195311c8 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -0,0 +1,146 @@ + + +# Switchyard NeMo Relay Dynamic Plugin + +This crate builds the external `nvidia.switchyard` native plugin. It embeds +`switchyard-libsy` and drives `Algorithm::run_stream`; NeMo Relay remains +responsible for provider transport, authentication, retries, fallback, and +observability. + +The plugin requires NeMo Relay native plugin C API v2. It does not link the +Relay runtime, use `switchyard-llm-client`, or start `switchyard-server`. +`switchyard-translation` is the only request, response, and stream translation +layer. + +The crate is a source/build unit and is not published to crates.io. Operators +install a release bundle containing the compiled shared library, materialized +`relay-plugin.toml`, `config.schema.json`, licensing files, and checksum. + +During development, `nemo-relay-plugin` is pinned to the Relay ABI v2 feature +commit. Replace that Git dependency with the first published compatible SDK +version before releasing a bundle. + +## Runtime contract + +For every managed LLM call, the plugin: + +1. decodes the caller body with `switchyard-translation`; +2. drives the configured libsy algorithm through `Algorithm::run_stream`; +3. records each real `Decision`; +4. translates every `CallLlm` request to the selected target protocol; +5. asks Relay to dispatch the translated request through native API v2; +6. passes the actual response, stream, or typed provider failure back through + `CallLlmRequest::respond`; and +7. translates `ReturnToAgent` back to the caller protocol. + +Relay owns URLs, credentials, provider transport, retries, fallback, stream +commitment, and event export. Switchyard owns routing and translation. The +plugin contains no Relay provider codecs and does not use private dispatch +headers. + +`switchyard-translation` is used for same-protocol routes as well as +cross-protocol routes. Same-protocol response events replay their preserved +provider JSON, including unknown fields. Cross-protocol routes encode the +normalized fields shared by the source and destination protocols and reject +lossy conversions. + +## Configuration + +The manifest requires `compat.native_api = "2"`. A Relay project config can +register the bundle and configure a seeded weighted-random router as follows: + +```toml +version = 1 + +[[plugins.dynamic]] +manifest = "/opt/switchyard-relay-plugin/relay-plugin.toml" + +[plugins.dynamic.config] +version = 2 +priority = 0 +max_retries = 3 +enabled_inbound_profiles = [ + "openai_chat", + "openai_responses", + "anthropic_messages", +] + +[plugins.dynamic.config.algorithm] +kind = "random" +seed = 42 + +[plugins.dynamic.config.default_targets] +openai_chat = "chat-default" +openai_responses = "responses-default" +anthropic_messages = "anthropic-default" + +[plugins.dynamic.config.targets.fast] +model = "provider/model" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://provider.example.com" +weight = 1 + +[plugins.dynamic.config.targets.fast.header_env] +authorization = "PROVIDER_AUTHORIZATION" +``` + +Target map keys such as `fast` are the semantic model names exposed to libsy. +The target binding remains authoritative for the provider model, protocol, URL, +and headers. `header_env` resolves credentials in the plugin process without +putting them in configuration or libsy metadata. + +Version-1 service configuration is rejected with a migration error. The plugin +does not provide decision-only or observe-only execution. + +## Build and bundle + +Build the source crate normally, then materialize an operator bundle from the +platform library: + +```bash +cargo build --release -p switchyard-nemo-relay-plugin +python3 crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py \ + --library target/release/libswitchyard_nemo_relay_plugin.so \ + --output dist/switchyard-nemo-relay-plugin-linux-x86_64 +``` + +On macOS the library suffix is `.dylib`. The bundle builder copies the shared +library, manifest, JSON schema, `LICENSE`, and `NOTICE`, materializes the +artifact digest in `relay-plugin.toml`, and writes `SHA256SUMS`. + +Operators install the binary bundle rather than this Rust crate: + +```bash +nemo-relay plugins validate /opt/switchyard-relay-plugin/relay-plugin.toml +nemo-relay plugins add --project /opt/switchyard-relay-plugin/relay-plugin.toml +nemo-relay plugins enable nvidia.switchyard +nemo-relay plugins inspect nvidia.switchyard +``` + +## Validation + +The process E2E requires a Relay binary that implements native API v2 and a +compiled plugin library: + +```bash +python3 crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py \ + --relay-bin /path/to/nemo-relay \ + --plugin-library /path/to/libswitchyard_nemo_relay_plugin.so +``` + +It launches a local three-protocol fake provider and real Relay process. The +test covers: + +- exact same-protocol unknown-field and raw-stream replay; +- buffered and streaming OpenAI Chat, OpenAI Responses, and Anthropic + Messages routes; +- cross-protocol request/response translation; +- 12 concurrent independent random-router calls; +- genuine requested and decision marks; +- an LLM-classifier call followed by its selected provider call; +- a retryable provider failure with a fresh run; and +- non-retryable failure with exactly-once trusted fallback. diff --git a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py new file mode 100644 index 000000000..04c9347c6 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build an operator-facing NeMo Relay plugin bundle from a compiled cdylib.""" + +from __future__ import annotations + +import argparse +import hashlib +import shutil +from pathlib import Path + +CRATE_ROOT = Path(__file__).resolve().parents[1] +REPOSITORY_ROOT = CRATE_ROOT.parents[1] + + +def digest(path: Path) -> str: + value = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--library", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + + library = args.library.resolve() + if not library.is_file(): + parser.error(f"compiled plugin library does not exist: {library}") + + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + artifact = output / library.name + shutil.copy2(library, artifact) + shutil.copy2(CRATE_ROOT / "config.schema.json", output / "config.schema.json") + shutil.copy2(REPOSITORY_ROOT / "LICENSE", output / "LICENSE") + shutil.copy2(REPOSITORY_ROOT / "NOTICE", output / "NOTICE") + + artifact_digest = digest(artifact) + manifest = (CRATE_ROOT / "relay-plugin.toml").read_text(encoding="utf-8") + manifest = manifest.replace("", artifact.name) + manifest = manifest.replace("", artifact_digest) + (output / "relay-plugin.toml").write_text(manifest, encoding="utf-8") + + checksums = [] + for path in sorted(output.iterdir(), key=lambda item: item.name): + if path.name != "SHA256SUMS" and path.is_file(): + checksums.append(f"{digest(path)} {path.name}") + (output / "SHA256SUMS").write_text("\n".join(checksums) + "\n", encoding="utf-8") + + print(output) + + +if __name__ == "__main__": + main() diff --git a/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py b/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py new file mode 100644 index 000000000..a6966e319 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py @@ -0,0 +1,285 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Three-protocol fake provider for the external-plugin process E2E.""" + +from __future__ import annotations + +import argparse +import json +import threading +from collections import Counter +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +CALLS: Counter[str] = Counter() +CALLS_LOCK = threading.Lock() + + +def call_number(model: str) -> int: + with CALLS_LOCK: + CALLS[model] += 1 + return CALLS[model] + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, format: str, *args: object) -> None: + print(format % args, flush=True) + + def do_GET(self) -> None: + if self.path == "/healthz": + self._json(200, {"ok": True}) + elif self.path == "/calls": + with CALLS_LOCK: + self._json(200, dict(CALLS)) + else: + self._json(404, {"error": "not found"}) + + def do_POST(self) -> None: + size = int(self.headers.get("content-length", "0")) + request = json.loads(self.rfile.read(size) or b"{}") + model = request.get("model", "unknown") + attempt = call_number(model) + if model == "fake/retry-once" and attempt == 1: + self._json(503, {"error": {"message": "retry this request"}}) + return + if model == "fake/always-fail": + self._json(400, {"error": {"message": "invalid routed request"}}) + return + if self.path == "/v1/chat/completions": + self._chat(request) + elif self.path == "/v1/responses": + self._responses(request) + elif self.path == "/v1/messages": + self._anthropic(request) + else: + self._json(404, {"error": {"message": f"unknown path {self.path}"}}) + + def _chat(self, request: dict[str, object]) -> None: + model = str(request.get("model", "unknown")) + classifier = model == "fake/classifier" + answer = ( + '{"recommended_route":"efficient","p_solve":0.9,' + '"confidence":0.95,"abstain":false,' + '"capability_boundary":"supported","primary_rule":"SUP-1",' + '"crux":"bounded task"}' + if classifier + else f"chat from {model}" + ) + if request.get("stream"): + events: list[dict[str, object]] = [ + { + "id": "chatcmpl-dynamic", + "object": "chat.completion.chunk", + "model": model, + "system_fingerprint": "fp_dynamic_plugin", + "provider_extension": {"preserved": True}, + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": answer}, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-dynamic", + "object": "chat.completion.chunk", + "model": model, + "system_fingerprint": "fp_dynamic_plugin", + "provider_extension": {"preserved": True}, + "choices": [ + {"index": 0, "delta": {}, "finish_reason": "stop"} + ], + }, + ] + self._sse(events) + return + self._json( + 200, + { + "id": "chatcmpl-dynamic", + "object": "chat.completion", + "model": model, + "system_fingerprint": "fp_dynamic_plugin", + "provider_extension": {"preserved": True}, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": answer}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 4, + "completion_tokens": 2, + "total_tokens": 6, + }, + }, + ) + + def _responses(self, request: dict[str, object]) -> None: + model = str(request.get("model", "unknown")) + text = f"responses from {model}" + if request.get("stream"): + self._sse( + [ + { + "type": "response.created", + "response": {"id": "resp-dynamic", "model": model}, + "provider_extension": {"preserved": True}, + }, + { + "type": "response.output_text.delta", + "item_id": "item-dynamic", + "output_index": 0, + "content_index": 0, + "delta": text, + "provider_extension": {"preserved": True}, + }, + { + "type": "response.completed", + "response": { + "id": "resp-dynamic", + "model": model, + "usage": { + "input_tokens": 4, + "output_tokens": 3, + "total_tokens": 7, + }, + }, + "provider_extension": {"preserved": True}, + }, + ], + named=True, + ) + return + self._json( + 200, + { + "id": "resp-dynamic", + "object": "response", + "created_at": 1, + "status": "completed", + "model": model, + "output": [ + { + "id": "msg-dynamic", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [ + {"type": "output_text", "text": text, "annotations": []} + ], + } + ], + "usage": { + "input_tokens": 4, + "output_tokens": 3, + "total_tokens": 7, + }, + "provider_extension": {"preserved": True}, + }, + ) + + def _anthropic(self, request: dict[str, object]) -> None: + model = str(request.get("model", "unknown")) + text = f"anthropic from {model}" + if request.get("stream"): + self._sse( + [ + { + "type": "message_start", + "message": { + "id": "msg-dynamic", + "type": "message", + "role": "assistant", + "model": model, + "content": [], + "usage": {"input_tokens": 4, "output_tokens": 0}, + }, + "provider_extension": {"preserved": True}, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + "provider_extension": {"preserved": True}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": text}, + "provider_extension": {"preserved": True}, + }, + { + "type": "content_block_stop", + "index": 0, + "provider_extension": {"preserved": True}, + }, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 3}, + "provider_extension": {"preserved": True}, + }, + { + "type": "message_stop", + "provider_extension": {"preserved": True}, + }, + ], + named=True, + ) + return + self._json( + 200, + { + "id": "msg-dynamic", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": text}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 4, "output_tokens": 3}, + "provider_extension": {"preserved": True}, + }, + ) + + def _sse(self, events: list[dict[str, object]], named: bool = False) -> None: + parts = [] + for event in events: + if named: + parts.append(f"event: {event['type']}\n") + parts.append(f"data: {json.dumps(event)}\n\n") + if not named: + parts.append("data: [DONE]\n\n") + data = "".join(parts).encode() + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.send_header("cache-control", "no-cache") + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + self.wfile.flush() + + def _json(self, status: int, value: object) -> None: + data = json.dumps(value).encode() + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + self.wfile.flush() + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--port", required=True, type=int) + args = parser.parse_args() + ThreadingHTTPServer(("127.0.0.1", args.port), Handler).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py b/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py new file mode 100644 index 000000000..4f6108fcc --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py @@ -0,0 +1,669 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Process E2E for the Switchyard plugin against a real NeMo Relay host.""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import json +import os +import signal +import socket +import subprocess +import sys +import tempfile +import threading +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any, cast + +HERE = Path(__file__).resolve().parent +CRATE_ROOT = HERE.parents[1] + + +def free_port() -> int: + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return int(listener.getsockname()[1]) + + +def capture(process: subprocess.Popen[str], sink: list[str]) -> None: + assert process.stdout is not None + for line in process.stdout: + sink.append(line.rstrip()) + + +def http_json(base: str, path: str) -> dict[str, int]: + with urllib.request.urlopen(f"{base}{path}", timeout=5) as response: + return cast(dict[str, int], json.loads(response.read())) + + +def request( + relay_url: str, path: str, body: dict[str, Any] +) -> tuple[int, bytes]: + request = urllib.request.Request( + f"{relay_url}{path}", + data=json.dumps(body).encode(), + headers={ + "content-type": "application/json", + "authorization": "Bearer e2e", + }, + method="POST", + ) + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, response.read() + + +def stream_events(raw: bytes) -> list[dict[str, object]]: + return [ + json.loads(line[6:]) + for line in raw.decode().splitlines() + if line.startswith("data: {") + ] + + +def response_text(protocol: str, response: dict[str, Any]) -> str: + if protocol == "openai_chat": + return str(response["choices"][0]["message"]["content"]) + if protocol == "openai_responses": + return str(response["output"][0]["content"][0]["text"]) + return str(response["content"][0]["text"]) + + +def stream_text(protocol: str, events: list[dict[str, Any]]) -> str: + if protocol == "openai_chat": + return "".join( + str(event["choices"][0]["delta"].get("content", "")) + for event in events + if event.get("choices") + ) + if protocol == "openai_responses": + return "".join( + str(event.get("delta", "")) + for event in events + if event.get("type") == "response.output_text.delta" + ) + return "".join( + str(event.get("delta", {}).get("text", "")) + for event in events + if event.get("type") == "content_block_delta" + ) + + +CASES: tuple[tuple[str, str, dict[str, Any]], ...] = ( + ( + "openai_chat", + "/v1/chat/completions", + { + "model": "caller/chat", + "messages": [{"role": "user", "content": "hello"}], + "caller_extension": {"preserve": True}, + }, + ), + ( + "openai_responses", + "/v1/responses", + { + "model": "caller/responses", + "input": "hello", + "caller_extension": {"preserve": True}, + }, + ), + ( + "anthropic_messages", + "/v1/messages", + { + "model": "caller/anthropic", + "max_tokens": 64, + "messages": [{"role": "user", "content": "hello"}], + "caller_extension": {"preserve": True}, + }, + ), +) + + +def plugin_config( + manifest: Path, + provider_url: str, + atof_directory: Path, + algorithm: str, + targets: str, + defaults: str, + profiles: str, + *, + max_retries: int = 1, +) -> str: + return f"""\ +version = 1 + +[[plugins.dynamic]] +manifest = {json.dumps(str(manifest))} + +[plugins.dynamic.config] +version = 2 +priority = 0 +max_retries = {max_retries} +enabled_inbound_profiles = [{profiles}] + +{algorithm} + +[plugins.dynamic.config.default_targets] +{defaults} + +{targets.format(provider_url=provider_url)} + +[[components]] +kind = "observability" +enabled = true + +[components.config] +version = 3 + +[components.config.atof] +enabled = true + +[[components.config.atof.sinks]] +type = "file" +mode = "overwrite" +output_directory = {json.dumps(str(atof_directory))} +filename = "events.jsonl" +""" + + +class RelayScenario: + def __init__( + self, + relay_bin: Path, + root: Path, + provider_url: str, + name: str, + config: str, + ) -> None: + self.relay_bin = relay_bin + self.root = root / name + self.provider_url = provider_url + self.config = config + self.port = free_port() + self.process: subprocess.Popen[str] | None = None + self.log: list[str] = [] + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.port}" + + @property + def atof_path(self) -> Path: + return self.root / "atof" / "events.jsonl" + + def __enter__(self) -> RelayScenario: + (self.root / ".nemo-relay").mkdir(parents=True) + (self.root / ".nemo-relay" / "plugins.toml").write_text( + self.config, encoding="utf-8" + ) + subprocess.run( + [str(self.relay_bin), "plugins", "enable", "nvidia.switchyard"], + cwd=self.root, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + self.process = subprocess.Popen( + [ + str(self.relay_bin), + "--bind", + f"127.0.0.1:{self.port}", + "--openai-base-url", + f"{self.provider_url}/v1", + "--log-level", + "warn", + ], + cwd=self.root, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + threading.Thread( + target=capture, args=(self.process, self.log), daemon=True + ).start() + deadline = time.time() + 20 + while True: + if self.process.poll() is not None: + raise RuntimeError( + f"Relay exited early ({self.process.returncode}):\n" + + "\n".join(self.log[-40:]) + ) + try: + with urllib.request.urlopen(f"{self.url}/healthz", timeout=1) as response: + if response.status == 200: + return self + except (OSError, urllib.error.URLError): + pass + if time.time() > deadline: + raise TimeoutError("Relay did not become healthy") + time.sleep(0.1) + + def __exit__(self, *_: object) -> None: + assert self.process is not None + self.process.send_signal(signal.SIGINT) + try: + self.process.wait(timeout=10) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait() + (self.root / "relay.log").write_text( + "\n".join(self.log) + "\n", encoding="utf-8" + ) + if self.process.returncode: + raise RuntimeError( + f"Relay exited with {self.process.returncode}:\n" + + "\n".join(self.log[-40:]) + ) + + def marks( + self, name: str, expected: int = 1, timeout: float = 5 + ) -> list[dict[str, object]]: + deadline = time.time() + timeout + while True: + try: + events = [ + json.loads(line) + for line in self.atof_path.read_text(encoding="utf-8").splitlines() + if line + ] + except FileNotFoundError: + events = [] + matches = [event for event in events if event.get("name") == name] + if len(matches) >= expected or time.time() > deadline: + return matches + time.sleep(0.05) + + +def run_same_protocol( + relay_bin: Path, root: Path, manifest: Path, provider_url: str +) -> dict[str, object]: + config = plugin_config( + manifest, + provider_url, + root / "same" / "atof", + '[plugins.dynamic.config.algorithm]\nkind = "random"\nseed = 7', + """\ +[plugins.dynamic.config.targets.chat] +model = "fake/chat" +protocol = "openai_chat" +base_url = "{provider_url}/v1" +weight = 1 +""", + 'openai_chat = "chat"', + '"openai_chat"', + ) + with RelayScenario(relay_bin, root, provider_url, "same", config) as relay: + template = dict(CASES[0][2]) + template["stream"] = False + status, raw = request(relay.url, CASES[0][1], template) + buffered = json.loads(raw) + assert status == 200 + assert buffered["provider_extension"] == {"preserved": True} + assert buffered["system_fingerprint"] == "fp_dynamic_plugin" + + template["stream"] = True + status, raw = request(relay.url, CASES[0][1], template) + events = stream_events(raw) + assert status == 200 + assert len(events) == 2 + assert all(event["provider_extension"] == {"preserved": True} for event in events) + assert all(event["system_fingerprint"] == "fp_dynamic_plugin" for event in events) + assert len(relay.marks("switchyard.routing.decision", 2)) == 2 + return {"buffered_unknown_fields": True, "stream_events_replayed": len(events)} + + +def run_random( + relay_bin: Path, root: Path, manifest: Path, provider_url: str +) -> dict[str, object]: + config = plugin_config( + manifest, + provider_url, + root / "random" / "atof", + '[plugins.dynamic.config.algorithm]\nkind = "random"\nseed = 42', + """\ +[plugins.dynamic.config.targets.chat] +model = "fake/chat" +protocol = "openai_chat" +base_url = "{provider_url}/v1" +weight = 1 + +[plugins.dynamic.config.targets.responses] +model = "fake/responses" +protocol = "openai_responses" +base_url = "{provider_url}/v1" +weight = 1 + +[plugins.dynamic.config.targets.anthropic] +model = "fake/anthropic" +protocol = "anthropic_messages" +base_url = "{provider_url}/v1" +weight = 1 +""", + ( + 'openai_chat = "chat"\n' + 'openai_responses = "responses"\n' + 'anthropic_messages = "anthropic"' + ), + '"openai_chat", "openai_responses", "anthropic_messages"', + ) + with RelayScenario(relay_bin, root, provider_url, "random", config) as relay: + models: set[str] = set() + for protocol, path, template in CASES: + for _ in range(4): + body = dict(template) + body["stream"] = False + status, raw = request(relay.url, path, body) + response = json.loads(raw) + assert status == 200 + assert response_text(protocol, response) + models.add(response["model"]) + + streams = {} + for protocol, path, template in CASES: + body = dict(template) + body["stream"] = True + status, raw = request(relay.url, path, body) + events = stream_events(raw) + assert status == 200 + streams[protocol] = stream_text(protocol, events) + assert streams[protocol] + + def concurrent_call(index: int) -> str: + body = dict(CASES[0][2]) + body["stream"] = False + body["messages"] = [{"role": "user", "content": f"concurrent {index}"}] + status, raw = request(relay.url, CASES[0][1], body) + assert status == 200 + return response_text("openai_chat", json.loads(raw)) + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + concurrent_results = list(executor.map(concurrent_call, range(12))) + assert all(concurrent_results) + + assert models == {"fake/chat", "fake/responses", "fake/anthropic"} + decisions = relay.marks("switchyard.routing.decision", 27) + requested = relay.marks("switchyard.routing.requested", 27) + assert len(decisions) == 27 + assert len(requested) == 27 + assert all(event["parent_uuid"] for event in decisions) + assert { + event["data"]["selected_target"] # type: ignore[index] + for event in decisions + } == {"chat", "responses", "anthropic"} + return { + "models": sorted(models), + "stream_text": streams, + "concurrent_calls": len(concurrent_results), + "routing_decisions": len(decisions), + } + + +def run_classifier( + relay_bin: Path, root: Path, manifest: Path, provider_url: str +) -> dict[str, object]: + config = plugin_config( + manifest, + provider_url, + root / "classifier" / "atof", + """\ +[plugins.dynamic.config.algorithm] +kind = "llm_classifier" +classifier_target = "classifier" +weak_target = "weak" +strong_target = "strong" +base_threshold = 0.5 +min_confidence = 0.5 +session_affinity = false +message_hash_fallback = false +""", + """\ +[plugins.dynamic.config.targets.classifier] +model = "fake/classifier" +protocol = "openai_chat" +base_url = "{provider_url}/v1" + +[plugins.dynamic.config.targets.weak] +model = "fake/weak" +protocol = "openai_responses" +base_url = "{provider_url}/v1" + +[plugins.dynamic.config.targets.strong] +model = "fake/strong" +protocol = "anthropic_messages" +base_url = "{provider_url}/v1" + +[plugins.dynamic.config.targets.fallback] +model = "fake/fallback" +protocol = "openai_chat" +base_url = "{provider_url}/v1" +""", + 'openai_chat = "fallback"', + '"openai_chat"', + ) + with RelayScenario(relay_bin, root, provider_url, "classifier", config) as relay: + body = dict(CASES[0][2]) + body["stream"] = False + status, raw = request(relay.url, CASES[0][1], body) + response = json.loads(raw) + assert status == 200 + assert response["model"] == "fake/weak" + + body["stream"] = True + status, raw = request(relay.url, CASES[0][1], body) + events = stream_events(raw) + assert status == 200 + assert "responses from fake/weak" == stream_text("openai_chat", events) + + decisions = relay.marks("switchyard.routing.decision", 2) + assert len(decisions) == 2 + assert all( + event["data"]["algorithm"] == "llm_task_classifier" # type: ignore[index] + and event["data"]["selected_target"] == "weak" # type: ignore[index] + and event["data"]["routing_tier"] == "weak" # type: ignore[index] + for event in decisions + ) + return {"selected_target": "weak", "decisions": len(decisions)} + + +def single_target_config( + manifest: Path, + provider_url: str, + atof: Path, + selected_model: str, + fallback_model: str, +) -> str: + return plugin_config( + manifest, + provider_url, + atof, + '[plugins.dynamic.config.algorithm]\nkind = "random"\nseed = 1', + f"""\ +[plugins.dynamic.config.targets.selected] +model = "{selected_model}" +protocol = "openai_chat" +base_url = "{{provider_url}}/v1" + +[plugins.dynamic.config.targets.fallback] +model = "{fallback_model}" +protocol = "openai_chat" +base_url = "{{provider_url}}/v1" +""", + 'openai_chat = "fallback"', + '"openai_chat"', + ) + + +def run_retry_and_fallback( + relay_bin: Path, root: Path, manifest: Path, provider_url: str +) -> dict[str, object]: + retry_config = single_target_config( + manifest, + provider_url, + root / "retry" / "atof", + "fake/retry-once", + "fake/retry-fallback", + ) + with RelayScenario(relay_bin, root, provider_url, "retry", retry_config) as relay: + body = dict(CASES[0][2]) + body["stream"] = False + status, raw = request(relay.url, CASES[0][1], body) + assert status == 200 + assert json.loads(raw)["model"] == "fake/retry-once" + assert len(relay.marks("switchyard.routing.retry")) == 1 + assert not relay.marks("switchyard.routing.fallback", expected=0) + + fallback_config = single_target_config( + manifest, + provider_url, + root / "fallback" / "atof", + "fake/always-fail", + "fake/trusted-fallback", + ) + with RelayScenario( + relay_bin, root, provider_url, "fallback", fallback_config + ) as relay: + body = dict(CASES[0][2]) + body["stream"] = False + status, raw = request(relay.url, CASES[0][1], body) + assert status == 200 + assert json.loads(raw)["model"] == "fake/trusted-fallback" + assert len(relay.marks("switchyard.routing.error")) == 1 + assert len(relay.marks("switchyard.routing.fallback")) == 1 + + calls = http_json(provider_url, "/calls") + assert calls["fake/retry-once"] == 2 + assert calls.get("fake/retry-fallback", 0) == 0 + assert calls["fake/always-fail"] == 1 + assert calls["fake/trusted-fallback"] == 1 + return { + "retry_attempts": calls["fake/retry-once"], + "fallback_calls": calls["fake/trusted-fallback"], + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--relay-bin", + type=Path, + default=os.environ.get("NEMO_RELAY_BIN"), + required="NEMO_RELAY_BIN" not in os.environ, + ) + parser.add_argument( + "--plugin-library", + type=Path, + default=os.environ.get("SWITCHYARD_PLUGIN_LIBRARY"), + required="SWITCHYARD_PLUGIN_LIBRARY" not in os.environ, + ) + parser.add_argument("--keep-temp", action="store_true") + args = parser.parse_args() + + relay_bin = args.relay_bin.resolve() + plugin_library = args.plugin_library.resolve() + if not relay_bin.is_file(): + parser.error(f"Relay binary does not exist: {relay_bin}") + if not plugin_library.is_file(): + parser.error(f"plugin library does not exist: {plugin_library}") + + temporary = None + if args.keep_temp: + root = Path(tempfile.mkdtemp(prefix="switchyard-relay-plugin-e2e-")) + else: + temporary = tempfile.TemporaryDirectory(prefix="switchyard-relay-plugin-e2e-") + root = Path(temporary.name) + bundle = root / "bundle" + subprocess.run( + [ + sys.executable, + str(CRATE_ROOT / "scripts" / "package_bundle.py"), + "--library", + str(plugin_library), + "--output", + str(bundle), + ], + check=True, + ) + subprocess.run( + [ + str(relay_bin), + "plugins", + "validate", + str(bundle / "relay-plugin.toml"), + ], + cwd=root, + check=True, + stdout=subprocess.PIPE, + text=True, + ) + + provider_port = free_port() + provider_url = f"http://127.0.0.1:{provider_port}" + provider_log: list[str] = [] + provider = subprocess.Popen( + [ + sys.executable, + "-u", + str(HERE / "fake_provider.py"), + "--port", + str(provider_port), + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + threading.Thread(target=capture, args=(provider, provider_log), daemon=True).start() + try: + deadline = time.time() + 10 + while True: + try: + if http_json(provider_url, "/healthz")["ok"]: + break + except (OSError, urllib.error.URLError): + pass + if provider.poll() is not None: + raise RuntimeError( + "fake provider exited early:\n" + "\n".join(provider_log[-40:]) + ) + if time.time() > deadline: + raise TimeoutError("fake provider did not become healthy") + time.sleep(0.05) + + summary = { + "same_protocol_preservation": run_same_protocol( + relay_bin, root, bundle / "relay-plugin.toml", provider_url + ), + "random": run_random( + relay_bin, root, bundle / "relay-plugin.toml", provider_url + ), + "llm_classifier": run_classifier( + relay_bin, root, bundle / "relay-plugin.toml", provider_url + ), + "reliability": run_retry_and_fallback( + relay_bin, root, bundle / "relay-plugin.toml", provider_url + ), + } + print(json.dumps(summary, indent=2, sort_keys=True)) + finally: + provider.terminate() + try: + provider.wait(timeout=5) + except subprocess.TimeoutExpired: + provider.kill() + provider.wait() + if args.keep_temp: + print(f"preserved E2E directory: {root}") + elif temporary is not None: + temporary.cleanup() + + +if __name__ == "__main__": + main() diff --git a/docs/index.md b/docs/index.md index 2ada13bc0..fdc34af3b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,3 +29,4 @@ standalone `switchyard-server` binary. - [`switchyard-libsy`](../crates/libsy/README.md): embeddable routing algorithms - [`switchyard-protocol`](../crates/protocol/README.md): provider-neutral API types - [`switchyard-translation`](../crates/switchyard-translation/README.md): protocol translation +- [`switchyard-nemo-relay-plugin`](../crates/switchyard-nemo-relay-plugin/README.md): in-process NeMo Relay routing From 47b9d725a7d46f9514e87a33b5e5c1810281c9da Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 31 Jul 2026 14:38:17 -0600 Subject: [PATCH 14/51] refactor(plugin): use targeted LLM continuation contract Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 +- .../switchyard-nemo-relay-plugin/Cargo.toml | 2 +- crates/switchyard-nemo-relay-plugin/README.md | 20 +- .../src/config.rs | 19 +- .../switchyard-nemo-relay-plugin/src/ffi.rs | 56 +++-- .../src/runtime.rs | 221 ++++++++++++------ .../src/translation.rs | 3 +- .../tests/e2e/fake_provider.py | 6 + .../tests/e2e/run_e2e.py | 41 +++- 9 files changed, 254 insertions(+), 118 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 83db53250..ff9537c0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.7.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=4590e7b8828689fdf8632f7625ea0db61ed18e6a#4590e7b8828689fdf8632f7625ea0db61ed18e6a" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=8cb270ebd1bb9f70bf58a10e314320d4683b65a6#8cb270ebd1bb9f70bf58a10e314320d4683b65a6" dependencies = [ "nemo-relay-types", "serde", @@ -1120,7 +1120,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.7.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=4590e7b8828689fdf8632f7625ea0db61ed18e6a#4590e7b8828689fdf8632f7625ea0db61ed18e6a" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=8cb270ebd1bb9f70bf58a10e314320d4683b65a6#8cb270ebd1bb9f70bf58a10e314320d4683b65a6" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 83ae635dc..a757724c0 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib", "rlib"] futures.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "4590e7b8828689fdf8632f7625ea0db61ed18e6a" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "8cb270ebd1bb9f70bf58a10e314320d4683b65a6" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 4195311c8..665055eb7 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -7,8 +7,8 @@ SPDX-License-Identifier: Apache-2.0 This crate builds the external `nvidia.switchyard` native plugin. It embeds `switchyard-libsy` and drives `Algorithm::run_stream`; NeMo Relay remains -responsible for provider transport, authentication, retries, fallback, and -observability. +responsible for provider transport, retries, fallback, and observability. The +plugin resolves each target's provider headers and credentials. The plugin requires NeMo Relay native plugin C API v2. It does not link the Relay runtime, use `switchyard-llm-client`, or start `switchyard-server`. @@ -36,11 +36,20 @@ For every managed LLM call, the plugin: `CallLlmRequest::respond`; and 7. translates `ReturnToAgent` back to the caller protocol. -Relay owns URLs, credentials, provider transport, retries, fallback, stream -commitment, and event export. Switchyard owns routing and translation. The -plugin contains no Relay provider codecs and does not use private dispatch +Switchyard owns routing, translation, target URLs, and target credentials. +Relay validates and transports the selected HTTP target, runs it through the +captured LLM continuation, and owns retries, fallback, stream commitment, and +event export. Target data never enters `LlmRequest.headers`, marks, or spans. +The plugin contains no Relay provider codecs and does not use private dispatch headers. +Provider failures use HTTP semantics: status, a bounded body, and safe response +headers when Relay received an HTTP response; otherwise a transport, timeout, +cancelled, invalid-request, guardrail, or internal kind. Retry policy is derived +from those values rather than serialized. HTTP 408, 425, 429, 500, 502, 503, +and 504 plus transport and timeout failures retry. The plugin does not inspect +provider bodies to reclassify HTTP 400 context-window or HTTP 404 model errors. + `switchyard-translation` is used for same-protocol routes as well as cross-protocol routes. Same-protocol response events replay their preserved provider JSON, including unknown fields. Cross-protocol routes encode the @@ -136,6 +145,7 @@ It launches a local three-protocol fake provider and real Relay process. The test covers: - exact same-protocol unknown-field and raw-stream replay; +- isolated target credentials and headers without source-header inheritance; - buffered and streaming OpenAI Chat, OpenAI Responses, and Anthropic Messages routes; - cross-protocol request/response translation; diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index dcc224652..acd6bedae 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -5,9 +5,8 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use http::header::{HeaderName, HeaderValue}; -use nemo_relay_plugin::LlmDispatchRouteV2; +use nemo_relay_plugin::LlmContinuationRouteV2; use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value as Json}; use switchyard_libsy::algorithms::{LlmTaskClassifier, Random, TaskClassifierConfig}; use switchyard_libsy::{Algorithm, LlmTarget, LlmTargetSet}; use switchyard_protocol::WireFormat; @@ -50,11 +49,11 @@ impl WireProtocol { } } - pub const fn relay_route(self) -> LlmDispatchRouteV2 { + pub const fn relay_route(self) -> LlmContinuationRouteV2 { match self { - Self::OpenaiChat => LlmDispatchRouteV2::OpenaiChat, - Self::OpenaiResponses => LlmDispatchRouteV2::OpenaiResponses, - Self::AnthropicMessages => LlmDispatchRouteV2::AnthropicMessages, + Self::OpenaiChat => LlmContinuationRouteV2::OpenaiChat, + Self::OpenaiResponses => LlmContinuationRouteV2::OpenaiResponses, + Self::AnthropicMessages => LlmContinuationRouteV2::AnthropicMessages, } } @@ -106,11 +105,11 @@ impl TargetBinding { format!("{base}{endpoint}") } - pub fn resolved_headers(&self) -> Result, String> { - let mut headers = Map::new(); + pub fn resolved_headers(&self) -> Result, String> { + let mut headers = BTreeMap::new(); for (name, value) in &self.headers { validate_header(name, value)?; - headers.insert(name.clone(), Json::String(value.clone())); + headers.insert(name.clone(), value.clone()); } for (name, variable) in &self.header_env { if self @@ -125,7 +124,7 @@ impl TargetBinding { let value = std::env::var(variable) .map_err(|_| format!("environment variable {variable:?} is not set"))?; validate_header(name, &value)?; - headers.insert(name.clone(), Json::String(value)); + headers.insert(name.clone(), value); } Ok(headers) } diff --git a/crates/switchyard-nemo-relay-plugin/src/ffi.rs b/crates/switchyard-nemo-relay-plugin/src/ffi.rs index 834e49baf..4d3c990de 100644 --- a/crates/switchyard-nemo-relay-plugin/src/ffi.rs +++ b/crates/switchyard-nemo-relay-plugin/src/ffi.rs @@ -10,7 +10,8 @@ use std::task::{Context, Poll}; use futures::channel::oneshot; use futures::Stream; use nemo_relay_plugin::{ - LlmCallErrorV2, LlmCallOutcomeV2, LlmDispatchRequestV2, LlmRequest, LlmStreamEventV2, + LlmContinuationFailureV2, LlmContinuationInvocationV2, LlmContinuationOutcomeV2, + LlmContinuationStreamEventV2, LlmNonHttpFailureKindV2, LlmNonHttpFailureV2, LlmRequest, NemoRelayNativeAsyncCompletion, NemoRelayNativeAsyncNext, NemoRelayNativeAsyncNextStreamCb, NemoRelayNativeAsyncStream, NemoRelayNativeHostApiV1, NemoRelayNativeHostApiV4, NemoRelayNativeLlmStreamV2, NemoRelayNativeString, NemoRelayStatus, @@ -82,10 +83,10 @@ pub fn read_json( pub async fn dispatch_buffered( host: &NemoRelayNativeHostApiV4, next: *const NemoRelayNativeAsyncNext, - dispatch: &LlmDispatchRequestV2, -) -> Result { + dispatch: &LlmContinuationInvocationV2, +) -> Result { let dispatch = HostString::json(&host.v3.v1, dispatch).map_err(internal_error)?; - let (sender, receiver) = oneshot::channel::(); + let (sender, receiver) = oneshot::channel::(); let sender = Box::into_raw(Box::new(sender)).cast::(); let status = unsafe { (host.async_llm_next_invoke_result_v2)(next, dispatch.as_ptr(), buffered_result, sender) @@ -93,7 +94,7 @@ pub async fn dispatch_buffered( if status != NemoRelayStatus::Ok { unsafe { drop(Box::from_raw( - sender.cast::>(), + sender.cast::>(), )) }; return Err(internal_error(format!( @@ -101,8 +102,8 @@ pub async fn dispatch_buffered( ))); } match receiver.await { - Ok(LlmCallOutcomeV2::Success { response }) => Ok(response), - Ok(LlmCallOutcomeV2::Failure { error }) => Err(error), + Ok(LlmContinuationOutcomeV2::Success { response }) => Ok(response), + Ok(LlmContinuationOutcomeV2::Failure { error }) => Err(error), Err(_) => Err(internal_error( "Relay dropped the buffered dispatch callback".into(), )), @@ -237,11 +238,12 @@ unsafe extern "C" fn buffered_result( user_data: *mut c_void, outcome_json: *const NemoRelayNativeString, ) { - let sender = unsafe { Box::from_raw(user_data.cast::>()) }; + let sender = + unsafe { Box::from_raw(user_data.cast::>()) }; let host = crate::host(); let outcome = read_json(&host.v3.v1, outcome_json) .and_then(|value| serde_json::from_value(value).map_err(|error| error.to_string())) - .unwrap_or_else(|error| LlmCallOutcomeV2::Failure { + .unwrap_or_else(|error| LlmContinuationOutcomeV2::Failure { error: internal_error(format!("invalid Relay buffered outcome: {error}")), }); let _ = sender.send(outcome); @@ -251,10 +253,10 @@ pub async fn dispatch_stream( host: &NemoRelayNativeHostApiV4, next: *const NemoRelayNativeAsyncNext, output_stream: *const NemoRelayNativeAsyncStream, - dispatch: &LlmDispatchRequestV2, -) -> Result { + dispatch: &LlmContinuationInvocationV2, +) -> Result { let dispatch = HostString::json(&host.v3.v1, dispatch).map_err(internal_error)?; - let (sender, receiver) = oneshot::channel::>(); + let (sender, receiver) = oneshot::channel::>(); let sender = Box::into_raw(Box::new(sender)).cast::(); let status = unsafe { (host.async_llm_next_open_stream_v2)( @@ -268,7 +270,7 @@ pub async fn dispatch_stream( if status != NemoRelayStatus::Ok { unsafe { drop(Box::from_raw( - sender.cast::>>(), + sender.cast::>>(), )) }; return Err(internal_error(format!( @@ -292,7 +294,7 @@ unsafe extern "C" fn provider_stream_open( error_json: *const NemoRelayNativeString, ) { let sender = unsafe { - Box::from_raw(user_data.cast::>>()) + Box::from_raw(user_data.cast::>>()) }; let host = crate::host(); let result = if error_json.is_null() { @@ -314,14 +316,14 @@ unsafe extern "C" fn provider_stream_open( pub struct ProviderJsonStream { host: NemoRelayNativeHostApiV4, stream: *const NemoRelayNativeLlmStreamV2, - pending: Option>, + pending: Option>, done: bool, } unsafe impl Send for ProviderJsonStream {} impl Stream for ProviderJsonStream { - type Item = Result; + type Item = Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { if self.done { @@ -336,7 +338,7 @@ impl Stream for ProviderJsonStream { if status != NemoRelayStatus::Ok { unsafe { drop(Box::from_raw( - sender.cast::>(), + sender.cast::>(), )) }; self.done = true; @@ -349,16 +351,16 @@ impl Stream for ProviderJsonStream { let receiver = self.pending.as_mut().expect("pending receiver was set"); match Pin::new(receiver).poll(cx) { Poll::Pending => Poll::Pending, - Poll::Ready(Ok(LlmStreamEventV2::Chunk { chunk })) => { + Poll::Ready(Ok(LlmContinuationStreamEventV2::Chunk { chunk })) => { self.pending = None; Poll::Ready(Some(Ok(chunk))) } - Poll::Ready(Ok(LlmStreamEventV2::Failure { error })) => { + Poll::Ready(Ok(LlmContinuationStreamEventV2::Failure { error })) => { self.pending = None; self.done = true; Poll::Ready(Some(Err(error))) } - Poll::Ready(Ok(LlmStreamEventV2::Done)) => { + Poll::Ready(Ok(LlmContinuationStreamEventV2::Done)) => { self.pending = None; self.done = true; Poll::Ready(None) @@ -387,11 +389,12 @@ unsafe extern "C" fn provider_stream_next( user_data: *mut c_void, event_json: *const NemoRelayNativeString, ) { - let sender = unsafe { Box::from_raw(user_data.cast::>()) }; + let sender = + unsafe { Box::from_raw(user_data.cast::>()) }; let host = crate::host(); let event = read_json(&host.v3.v1, event_json) .and_then(|value| serde_json::from_value(value).map_err(|error| error.to_string())) - .unwrap_or_else(|error| LlmStreamEventV2::Failure { + .unwrap_or_else(|error| LlmContinuationStreamEventV2::Failure { error: internal_error(format!("invalid Relay stream event: {error}")), }); let _ = sender.send(event); @@ -466,6 +469,11 @@ pub unsafe fn release_stream( unsafe { (host.v3.async_stream_release)(stream) }; } -pub fn internal_error(message: String) -> LlmCallErrorV2 { - LlmCallErrorV2::Internal { message } +pub fn internal_error(message: String) -> LlmContinuationFailureV2 { + LlmContinuationFailureV2::NonHttp { + failure: LlmNonHttpFailureV2 { + kind: LlmNonHttpFailureKindV2::Internal, + message, + }, + } } diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 6e9e3d731..2e00c9b87 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -9,8 +9,9 @@ use std::task::{Context as TaskContext, Poll}; use futures::Stream; use futures_util::StreamExt; use nemo_relay_plugin::{ - Json, LlmCallErrorV2, LlmDispatchRequestV2, LlmDispatchTargetV2, LlmRequest as RelayRequest, - NemoRelayNativeAsyncNext, NemoRelayNativeAsyncStream, NemoRelayNativeHostApiV4, PluginRuntime, + Json, LlmContinuationFailureV2, LlmContinuationInvocationV2, LlmContinuationTargetV2, + LlmNonHttpFailureKindV2, LlmRequest as RelayRequest, NemoRelayNativeAsyncNext, + NemoRelayNativeAsyncStream, NemoRelayNativeHostApiV4, PluginRuntime, }; use serde::Deserialize; use serde_json::{json, Map}; @@ -34,7 +35,7 @@ pub struct Invocation { pub struct SwitchyardRuntime { config: SwitchyardConfig, algorithm: Arc, - target_headers: BTreeMap>, + target_headers: BTreeMap>, translation: TranslationEngine, relay: PluginRuntime, } @@ -104,18 +105,14 @@ impl SwitchyardRuntime { Err(failure) if failure.retryable() && attempt < max_attempts => { self.mark( "switchyard.routing.retry", - json!({"attempt": attempt, "error": failure.error.to_string()}), + failure_mark_data(attempt, &failure), identity_metadata(&invocation.request), ); } Err(failure) => { self.mark( "switchyard.routing.error", - json!({ - "attempt": attempt, - "retryable": failure.retryable(), - "error": failure.error.to_string(), - }), + failure_mark_data(attempt, &failure), identity_metadata(&invocation.request), ); return self @@ -183,7 +180,7 @@ impl SwitchyardRuntime { call: CallLlmRequest, host: &NemoRelayNativeHostApiV4, next: *const NemoRelayNativeAsyncNext, - provider_error: Arc>>, + provider_error: Arc>>, ) -> switchyard_libsy::Result<()> { let routed = call.get_routed().clone(); let target_name = routed.decision.selected_model().to_string(); @@ -207,7 +204,7 @@ impl SwitchyardRuntime { if let Ok(mut stored) = provider_error.lock() { *stored = Some(error.clone()); } - Err(client_error(error, &target_name)) + Err(client_error(error)) } } } @@ -267,30 +264,28 @@ impl SwitchyardRuntime { target: &TargetBinding, mut request: Request, streaming: bool, - ) -> Result { + ) -> Result { request.llm_request.stream = streaming; let headers = self .target_headers .get(target_name) .cloned() .unwrap_or_default(); - let mut request = translation::encode_request( - &self.translation, - target.protocol, - &request.llm_request, - headers, - ) - .map_err(LlmClientError::RequestEncoding)?; + let mut request = + translation::encode_request(&self.translation, target.protocol, &request.llm_request) + .map_err(LlmClientError::RequestEncoding)?; let body = request.content.as_object_mut().ok_or_else(|| { LlmClientError::RequestEncoding("translated provider request is not an object".into()) })?; body.insert("model".into(), Json::String(target.model.clone())); body.insert("stream".into(), Json::Bool(streaming)); - Ok(LlmDispatchRequestV2 { + Ok(LlmContinuationInvocationV2 { request, - target: LlmDispatchTargetV2 { + target: LlmContinuationTargetV2 { + method: "POST".into(), url: target.dispatch_url(), route: target.protocol.relay_route(), + headers, }, }) } @@ -449,7 +444,7 @@ impl SwitchyardRuntime { host: &NemoRelayNativeHostApiV4, next: *const NemoRelayNativeAsyncNext, output: *const NemoRelayNativeAsyncStream, - provider_error: Arc>>, + provider_error: Arc>>, ) -> switchyard_libsy::Result<()> { let routed = call.get_routed().clone(); let target_name = routed.decision.selected_model().to_string(); @@ -474,7 +469,7 @@ impl SwitchyardRuntime { host: &NemoRelayNativeHostApiV4, next: *const NemoRelayNativeAsyncNext, output: *const NemoRelayNativeAsyncStream, - provider_error: Arc>>, + provider_error: Arc>>, ) -> Result { let target = self.target(target_name)?; let metadata = request.metadata.clone(); @@ -483,14 +478,14 @@ impl SwitchyardRuntime { Ok(upstream) => upstream, Err(error) => { remember_provider_error(&provider_error, &error); - return Err(client_error(error, target_name)); + return Err(client_error(error)); } }; let first_raw = match upstream.next().await { Some(Ok(first)) => first, Some(Err(error)) => { remember_provider_error(&provider_error, &error); - return Err(client_error(error, target_name)); + return Err(client_error(error)); } None => { return Err(LlmClientError::InvalidResponse { @@ -512,7 +507,6 @@ impl SwitchyardRuntime { first: Some(first), protocol: target.protocol, state, - target_name: target_name.to_string(), provider_error, }); Ok(Response { @@ -530,7 +524,7 @@ impl SwitchyardRuntime { inbound: WireProtocol, host: &NemoRelayNativeHostApiV4, output: *const NemoRelayNativeAsyncStream, - provider_error: &Arc>>, + provider_error: &Arc>>, ) -> Result<(), StreamAttemptFailure> { let source = response .metadata @@ -633,7 +627,7 @@ impl SwitchyardRuntime { fn emit_stream_retry(&self, attempt: u32, failure: &RunFailure, metadata: &Json) { self.mark( "switchyard.routing.retry", - json!({"attempt": attempt, "error": failure.error.to_string()}), + failure_mark_data(attempt, failure), metadata.clone(), ); } @@ -641,11 +635,7 @@ impl SwitchyardRuntime { fn emit_stream_error(&self, attempt: u32, failure: &RunFailure, metadata: &Json) { self.mark( "switchyard.routing.error", - json!({ - "attempt": attempt, - "retryable": failure.retryable(), - "error": failure.error.to_string(), - }), + failure_mark_data(attempt, failure), metadata.clone(), ); } @@ -653,7 +643,7 @@ impl SwitchyardRuntime { struct StreamRun { response: Response, - provider_error: Arc>>, + provider_error: Arc>>, } struct TranslatedProviderStream { @@ -661,8 +651,7 @@ struct TranslatedProviderStream { first: Option, protocol: WireProtocol, state: StreamTranslationState, - target_name: String, - provider_error: Arc>>, + provider_error: Arc>>, } impl Stream for TranslatedProviderStream { @@ -686,7 +675,7 @@ impl Stream for TranslatedProviderStream { } Poll::Ready(Some(Err(error))) => { remember_provider_error(&self.provider_error, &error); - Poll::Ready(Some(Err(client_error(error, &self.target_name)))) + Poll::Ready(Some(Err(client_error(error)))) } Poll::Ready(None) => Poll::Ready(None), } @@ -719,8 +708,8 @@ fn decode_provider_event( } fn remember_provider_error( - provider_error: &Arc>>, - error: &LlmCallErrorV2, + provider_error: &Arc>>, + error: &LlmContinuationFailureV2, ) { if let Ok(mut stored) = provider_error.lock() { *stored = Some(error.clone()); @@ -736,7 +725,7 @@ impl StreamAttemptFailure { fn client( error: LlmClientError, committed: bool, - provider_error: &Arc>>, + provider_error: &Arc>>, ) -> Self { Self { failure: RunFailure::new( @@ -750,7 +739,7 @@ impl StreamAttemptFailure { fn translation( error: &str, committed: bool, - provider_error: &Arc>>, + provider_error: &Arc>>, ) -> Self { Self::client( LlmClientError::ResponseTranslation(error.to_string()), @@ -762,11 +751,14 @@ impl StreamAttemptFailure { struct RunFailure { error: LibsyError, - provider_error: Option, + provider_error: Option, } impl RunFailure { - fn new(error: LibsyError, provider_error: &Arc>>) -> Self { + fn new( + error: LibsyError, + provider_error: &Arc>>, + ) -> Self { Self { error, provider_error: provider_error.lock().ok().and_then(|error| error.clone()), @@ -776,47 +768,71 @@ impl RunFailure { fn retryable(&self) -> bool { self.provider_error .as_ref() - .is_some_and(LlmCallErrorV2::is_retryable) + .is_some_and(LlmContinuationFailureV2::is_retryable) } } -fn client_error(error: LlmCallErrorV2, model: &str) -> LlmClientError { +fn client_error(error: LlmContinuationFailureV2) -> LlmClientError { match error { - LlmCallErrorV2::Upstream { - class, - status, - body, - .. - } => match class { - nemo_relay_plugin::LlmUpstreamFailureClassV2::Connection => LlmClientError::Transport { - source: Box::new(std::io::Error::other(body)), + LlmContinuationFailureV2::Http { failure } => LlmClientError::UpstreamHttp { + status: failure.status, + body: failure.body, + }, + LlmContinuationFailureV2::NonHttp { failure } => match failure.kind { + LlmNonHttpFailureKindV2::Transport => LlmClientError::Transport { + source: Box::new(std::io::Error::other(failure.message)), }, - nemo_relay_plugin::LlmUpstreamFailureClassV2::Timeout => LlmClientError::Timeout { - source: Box::new(std::io::Error::new(std::io::ErrorKind::TimedOut, body)), + LlmNonHttpFailureKindV2::Timeout => LlmClientError::Timeout { + source: Box::new(std::io::Error::new( + std::io::ErrorKind::TimedOut, + failure.message, + )), }, - nemo_relay_plugin::LlmUpstreamFailureClassV2::ContextWindow => { - LlmClientError::ContextWindowExceeded { - model: model.into(), - message: body, + LlmNonHttpFailureKindV2::InvalidRequest | LlmNonHttpFailureKindV2::Guardrail => { + LlmClientError::InvalidRequest { + message: failure.message, } } - nemo_relay_plugin::LlmUpstreamFailureClassV2::InvalidRequest - | nemo_relay_plugin::LlmUpstreamFailureClassV2::Authentication => { - LlmClientError::InvalidRequest { message: body } + LlmNonHttpFailureKindV2::Cancelled | LlmNonHttpFailureKindV2::Internal => { + LlmClientError::General(failure.message) } - _ => match status { - Some(status) => LlmClientError::UpstreamHttp { status, body }, - None => LlmClientError::General(body), - }, }, - LlmCallErrorV2::InvalidRequest { message } - | LlmCallErrorV2::GuardrailRejected { message } => { - LlmClientError::InvalidRequest { message } + } +} + +fn failure_mark_data(attempt: u32, failure: &RunFailure) -> Json { + let mut data = Map::from_iter([ + ("attempt".into(), Json::from(attempt)), + ("retryable".into(), Json::from(failure.retryable())), + ]); + match &failure.provider_error { + Some(LlmContinuationFailureV2::Http { failure }) => { + data.insert("failure_kind".into(), Json::from("http")); + data.insert("http_status".into(), Json::from(failure.status)); + } + Some(LlmContinuationFailureV2::NonHttp { failure }) => { + data.insert("failure_kind".into(), Json::from("non_http")); + data.insert( + "non_http_kind".into(), + Json::from(non_http_failure_label(failure.kind)), + ); } - LlmCallErrorV2::Cancelled { message } | LlmCallErrorV2::Internal { message } => { - LlmClientError::General(message) + None => { + data.insert("failure_kind".into(), Json::from("algorithm")); } } + Json::Object(data) +} + +const fn non_http_failure_label(kind: LlmNonHttpFailureKindV2) -> &'static str { + match kind { + LlmNonHttpFailureKindV2::Transport => "transport", + LlmNonHttpFailureKindV2::Timeout => "timeout", + LlmNonHttpFailureKindV2::Cancelled => "cancelled", + LlmNonHttpFailureKindV2::InvalidRequest => "invalid_request", + LlmNonHttpFailureKindV2::Guardrail => "guardrail", + LlmNonHttpFailureKindV2::Internal => "internal", + } } fn string_headers(headers: &Map) -> BTreeMap { @@ -835,3 +851,66 @@ fn identity_metadata(request: &RelayRequest) -> Json { "request_id": metadata.correlation_id, }) } + +#[cfg(test)] +mod tests { + use super::*; + use nemo_relay_plugin::{LlmHttpFailureV2, LlmNonHttpFailureV2}; + + #[test] + fn http_failures_keep_status_semantics_without_provider_classification() { + let error = client_error(LlmContinuationFailureV2::Http { + failure: LlmHttpFailureV2 { + status: 400, + body: "context length exceeded".into(), + headers: BTreeMap::new(), + }, + }); + assert!(matches!( + error, + LlmClientError::UpstreamHttp { status: 400, .. } + )); + } + + #[test] + fn routing_failure_marks_exclude_provider_payloads() { + let failure = RunFailure { + error: LibsyError::MissingFinalResponse, + provider_error: Some(LlmContinuationFailureV2::Http { + failure: LlmHttpFailureV2 { + status: 429, + body: "provider body must not be recorded".into(), + headers: BTreeMap::from([("retry-after".into(), "secret".into())]), + }, + }), + }; + assert_eq!( + failure_mark_data(2, &failure), + json!({ + "attempt": 2, + "retryable": true, + "failure_kind": "http", + "http_status": 429, + }) + ); + + let failure = RunFailure { + error: LibsyError::MissingFinalResponse, + provider_error: Some(LlmContinuationFailureV2::NonHttp { + failure: LlmNonHttpFailureV2 { + kind: LlmNonHttpFailureKindV2::Timeout, + message: "timeout detail must not be recorded".into(), + }, + }), + }; + assert_eq!( + failure_mark_data(3, &failure), + json!({ + "attempt": 3, + "retryable": true, + "failure_kind": "non_http", + "non_http_kind": "timeout", + }) + ); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/translation.rs b/crates/switchyard-nemo-relay-plugin/src/translation.rs index aa04d8b21..ae8f83fc6 100644 --- a/crates/switchyard-nemo-relay-plugin/src/translation.rs +++ b/crates/switchyard-nemo-relay-plugin/src/translation.rs @@ -28,14 +28,13 @@ pub fn encode_request( engine: &TranslationEngine, protocol: WireProtocol, request: &LlmRequest, - headers: Map, ) -> Result { let output = engine .encode_request(protocol.wire_format(), request, &request_policy(protocol)) .map_err(error)?; safe(&output.diagnostics)?; Ok(RelayRequest { - headers, + headers: Map::new(), content: output.body, }) } diff --git a/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py b/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py index a6966e319..3a1b2c4da 100644 --- a/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py +++ b/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py @@ -41,6 +41,12 @@ def do_POST(self) -> None: request = json.loads(self.rfile.read(size) or b"{}") model = request.get("model", "unknown") attempt = call_number(model) + if model == "fake/header-target" and ( + self.headers.get("authorization") != "Bearer target-e2e" + or self.headers.get("x-switchyard-target") != "same" + ): + self._json(401, {"error": {"message": "target headers were not isolated"}}) + return if model == "fake/retry-once" and attempt == 1: self._json(503, {"error": {"message": "retry this request"}}) return diff --git a/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py b/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py index 4f6108fcc..092e23160 100644 --- a/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py +++ b/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py @@ -23,6 +23,8 @@ HERE = Path(__file__).resolve().parent CRATE_ROOT = HERE.parents[1] +TARGET_AUTHORIZATION = "Bearer target-e2e" +TARGET_AUTHORIZATION_ENV = "SWITCHYARD_E2E_TARGET_AUTHORIZATION" def free_port() -> int: @@ -207,6 +209,7 @@ def __enter__(self) -> RelayScenario: subprocess.run( [str(self.relay_bin), "plugins", "enable", "nvidia.switchyard"], cwd=self.root, + env={**os.environ, TARGET_AUTHORIZATION_ENV: TARGET_AUTHORIZATION}, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -223,6 +226,7 @@ def __enter__(self) -> RelayScenario: "warn", ], cwd=self.root, + env={**os.environ, TARGET_AUTHORIZATION_ENV: TARGET_AUTHORIZATION}, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, @@ -293,10 +297,16 @@ def run_same_protocol( '[plugins.dynamic.config.algorithm]\nkind = "random"\nseed = 7', """\ [plugins.dynamic.config.targets.chat] -model = "fake/chat" +model = "fake/header-target" protocol = "openai_chat" base_url = "{provider_url}/v1" weight = 1 + +[plugins.dynamic.config.targets.chat.header_env] +authorization = "SWITCHYARD_E2E_TARGET_AUTHORIZATION" + +[plugins.dynamic.config.targets.chat.headers] +x-switchyard-target = "same" """, 'openai_chat = "chat"', '"openai_chat"', @@ -518,7 +528,14 @@ def run_retry_and_fallback( status, raw = request(relay.url, CASES[0][1], body) assert status == 200 assert json.loads(raw)["model"] == "fake/retry-once" - assert len(relay.marks("switchyard.routing.retry")) == 1 + retry_marks = relay.marks("switchyard.routing.retry") + assert len(retry_marks) == 1 + assert retry_marks[0]["data"] == { + "attempt": 1, + "retryable": True, + "failure_kind": "http", + "http_status": 503, + } assert not relay.marks("switchyard.routing.fallback", expected=0) fallback_config = single_target_config( @@ -536,7 +553,14 @@ def run_retry_and_fallback( status, raw = request(relay.url, CASES[0][1], body) assert status == 200 assert json.loads(raw)["model"] == "fake/trusted-fallback" - assert len(relay.marks("switchyard.routing.error")) == 1 + error_marks = relay.marks("switchyard.routing.error") + assert len(error_marks) == 1 + assert error_marks[0]["data"] == { + "attempt": 1, + "retryable": False, + "failure_kind": "http", + "http_status": 400, + } assert len(relay.marks("switchyard.routing.fallback")) == 1 calls = http_json(provider_url, "/calls") @@ -651,6 +675,17 @@ def main() -> None: relay_bin, root, bundle / "relay-plugin.toml", provider_url ), } + recorded = [ + path + for path in root.rglob("*") + if path.suffix in {".jsonl", ".log", ".toml"} + and TARGET_AUTHORIZATION in path.read_text(encoding="utf-8") + ] + assert not recorded, f"target credential was recorded in {recorded}" + summary["target_headers"] = { + "source_credentials_replaced": True, + "credential_recorded": False, + } print(json.dumps(summary, indent=2, sort_keys=True)) finally: provider.terminate() From 19b3bfd407c91e528b30233af0a683f6724821bd Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 31 Jul 2026 14:53:48 -0600 Subject: [PATCH 15/51] chore(plugin): update Relay SDK pin Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 ++-- crates/switchyard-nemo-relay-plugin/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff9537c0b..015e0deaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.7.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=8cb270ebd1bb9f70bf58a10e314320d4683b65a6#8cb270ebd1bb9f70bf58a10e314320d4683b65a6" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=a3e78ac79d9f319a3df669a1557f0006be23d5cf#a3e78ac79d9f319a3df669a1557f0006be23d5cf" dependencies = [ "nemo-relay-types", "serde", @@ -1120,7 +1120,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.7.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=8cb270ebd1bb9f70bf58a10e314320d4683b65a6#8cb270ebd1bb9f70bf58a10e314320d4683b65a6" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=a3e78ac79d9f319a3df669a1557f0006be23d5cf#a3e78ac79d9f319a3df669a1557f0006be23d5cf" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index a757724c0..04d3b9344 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib", "rlib"] futures.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "8cb270ebd1bb9f70bf58a10e314320d4683b65a6" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "a3e78ac79d9f319a3df669a1557f0006be23d5cf" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true From 919340593960effaaad9d808de02b9ccab2f1c9a Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 31 Jul 2026 15:04:28 -0600 Subject: [PATCH 16/51] chore(plugin): refresh Relay SDK pin Signed-off-by: Bryan Bednarski --- Cargo.lock | 8 ++++---- crates/switchyard-nemo-relay-plugin/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 015e0deaf..f1db3cbff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1109,8 +1109,8 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" -version = "0.7.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=a3e78ac79d9f319a3df669a1557f0006be23d5cf#a3e78ac79d9f319a3df669a1557f0006be23d5cf" +version = "0.8.0" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=a11ff5e105850a4b0f07a261546648bacb64d771#a11ff5e105850a4b0f07a261546648bacb64d771" dependencies = [ "nemo-relay-types", "serde", @@ -1119,8 +1119,8 @@ dependencies = [ [[package]] name = "nemo-relay-types" -version = "0.7.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=a3e78ac79d9f319a3df669a1557f0006be23d5cf#a3e78ac79d9f319a3df669a1557f0006be23d5cf" +version = "0.8.0" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=a11ff5e105850a4b0f07a261546648bacb64d771#a11ff5e105850a4b0f07a261546648bacb64d771" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 04d3b9344..692a64a11 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib", "rlib"] futures.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "a3e78ac79d9f319a3df669a1557f0006be23d5cf" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "a11ff5e105850a4b0f07a261546648bacb64d771" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true From 6f9dbfccf78708327f4743826c29a660c0af459b Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sat, 1 Aug 2026 01:15:14 -0600 Subject: [PATCH 17/51] refactor(plugin): use Relay safe native v2 SDK Signed-off-by: Bryan Bednarski --- Cargo.lock | 7 +- .../switchyard-nemo-relay-plugin/Cargo.toml | 4 +- crates/switchyard-nemo-relay-plugin/README.md | 31 +- .../switchyard-nemo-relay-plugin/src/ffi.rs | 479 ----------------- .../switchyard-nemo-relay-plugin/src/lib.rs | 164 +----- .../src/runtime.rs | 504 +++++++++++------- 6 files changed, 364 insertions(+), 825 deletions(-) delete mode 100644 crates/switchyard-nemo-relay-plugin/src/ffi.rs diff --git a/Cargo.lock b/Cargo.lock index f1db3cbff..3a37b3708 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,8 +1110,9 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=a11ff5e105850a4b0f07a261546648bacb64d771#a11ff5e105850a4b0f07a261546648bacb64d771" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=ed397cd598d0a5c3bde94b47db7b796077ab3d9a#ed397cd598d0a5c3bde94b47db7b796077ab3d9a" dependencies = [ + "futures", "nemo-relay-types", "serde", "serde_json", @@ -1120,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=a11ff5e105850a4b0f07a261546648bacb64d771#a11ff5e105850a4b0f07a261546648bacb64d771" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=ed397cd598d0a5c3bde94b47db7b796077ab3d9a#ed397cd598d0a5c3bde94b47db7b796077ab3d9a" dependencies = [ "bitflags", "chrono", @@ -2100,7 +2101,7 @@ dependencies = [ name = "switchyard-nemo-relay-plugin" version = "0.1.0" dependencies = [ - "futures", + "async-stream", "futures-util", "http", "nemo-relay-plugin", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 692a64a11..da43ceaae 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -16,10 +16,10 @@ publish = false crate-type = ["cdylib", "rlib"] [dependencies] -futures.workspace = true +async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "a11ff5e105850a4b0f07a261546648bacb64d771" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "ed397cd598d0a5c3bde94b47db7b796077ab3d9a" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 665055eb7..3573e85a4 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -7,13 +7,16 @@ SPDX-License-Identifier: Apache-2.0 This crate builds the external `nvidia.switchyard` native plugin. It embeds `switchyard-libsy` and drives `Algorithm::run_stream`; NeMo Relay remains -responsible for provider transport, retries, fallback, and observability. The -plugin resolves each target's provider headers and credentials. +responsible for provider transport and the observability substrate. The plugin +resolves each target's provider headers and credentials and coordinates the +Switchyard retry and trusted-fallback policy. -The plugin requires NeMo Relay native plugin C API v2. It does not link the -Relay runtime, use `switchyard-llm-client`, or start `switchyard-server`. -`switchyard-translation` is the only request, response, and stream translation -layer. +The plugin requires NeMo Relay native API v2. It uses the safe Rust continuation +facade from `nemo-relay-plugin`; the C ABI remains the binary boundary, but no +raw callbacks, handles, host tables, or unsafe FFI glue appear in this crate. It +does not link the Relay runtime, use `switchyard-llm-client`, or start +`switchyard-server`. `switchyard-translation` is the only request, response, and +stream translation layer. The crate is a source/build unit and is not published to crates.io. Operators install a release bundle containing the compiled shared library, materialized @@ -31,17 +34,23 @@ For every managed LLM call, the plugin: 2. drives the configured libsy algorithm through `Algorithm::run_stream`; 3. records each real `Decision`; 4. translates every `CallLlm` request to the selected target protocol; -5. asks Relay to dispatch the translated request through native API v2; +5. asks Relay's safe native API v2 continuation to dispatch the translated + request; 6. passes the actual response, stream, or typed provider failure back through `CallLlmRequest::respond`; and 7. translates `ReturnToAgent` back to the caller protocol. Switchyard owns routing, translation, target URLs, and target credentials. Relay validates and transports the selected HTTP target, runs it through the -captured LLM continuation, and owns retries, fallback, stream commitment, and -event export. Target data never enters `LlmRequest.headers`, marks, or spans. -The plugin contains no Relay provider codecs and does not use private dispatch -headers. +captured LLM continuation, and owns stream transport and event export. +Switchyard retries or falls back only before the first caller event; after +commitment, a late provider failure is returned without retry. Target data never +enters `LlmRequest.headers`, marks, or spans. The plugin contains no Relay +provider codecs and does not use private dispatch headers. + +Calls outside the enabled profiles return the SDK's explicit `Passthrough` +outcome. Relay then forwards the downstream provider stream through its bounded +host queue; unmanaged provider events do not cross the plugin ABI. Provider failures use HTTP semantics: status, a bounded body, and safe response headers when Relay received an HTTP response; otherwise a transport, timeout, diff --git a/crates/switchyard-nemo-relay-plugin/src/ffi.rs b/crates/switchyard-nemo-relay-plugin/src/ffi.rs deleted file mode 100644 index 4d3c990de..000000000 --- a/crates/switchyard-nemo-relay-plugin/src/ffi.rs +++ /dev/null @@ -1,479 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use std::ffi::c_void; -use std::future::Future; -use std::pin::Pin; -use std::ptr; -use std::task::{Context, Poll}; - -use futures::channel::oneshot; -use futures::Stream; -use nemo_relay_plugin::{ - LlmContinuationFailureV2, LlmContinuationInvocationV2, LlmContinuationOutcomeV2, - LlmContinuationStreamEventV2, LlmNonHttpFailureKindV2, LlmNonHttpFailureV2, LlmRequest, - NemoRelayNativeAsyncCompletion, NemoRelayNativeAsyncNext, NemoRelayNativeAsyncNextStreamCb, - NemoRelayNativeAsyncStream, NemoRelayNativeHostApiV1, NemoRelayNativeHostApiV4, - NemoRelayNativeLlmStreamV2, NemoRelayNativeString, NemoRelayStatus, -}; -use serde::Serialize; -use serde_json::Value as Json; - -pub struct HostString { - host: NemoRelayNativeHostApiV1, - ptr: *mut NemoRelayNativeString, -} - -impl HostString { - pub fn json(host: &NemoRelayNativeHostApiV1, value: &impl Serialize) -> Result { - let value = serde_json::to_string(value).map_err(|error| error.to_string())?; - Self::text(host, &value) - } - - pub fn text(host: &NemoRelayNativeHostApiV1, value: &str) -> Result { - let mut ptr: *mut NemoRelayNativeString = ptr::null_mut(); - let status = unsafe { (host.string_new)(value.as_ptr(), value.len(), &mut ptr as *mut _) }; - if status == NemoRelayStatus::Ok && !ptr.is_null() { - Ok(Self { host: *host, ptr }) - } else { - Err(format!("Relay host string allocation failed: {status:?}")) - } - } - - pub fn as_ptr(&self) -> *const NemoRelayNativeString { - self.ptr - } -} - -impl Drop for HostString { - fn drop(&mut self) { - unsafe { (self.host.string_free)(self.ptr) }; - } -} - -pub fn read_string( - host: &NemoRelayNativeHostApiV1, - value: *const NemoRelayNativeString, -) -> Result { - if value.is_null() { - return Err("Relay passed a null native string".into()); - } - let len = unsafe { (host.string_len)(value) }; - let data = unsafe { (host.string_data)(value) }; - if data.is_null() && len != 0 { - return Err("Relay passed an invalid native string".into()); - } - let bytes = if len == 0 { - &[][..] - } else { - unsafe { std::slice::from_raw_parts(data, len) } - }; - std::str::from_utf8(bytes) - .map(str::to_owned) - .map_err(|error| error.to_string()) -} - -pub fn read_json( - host: &NemoRelayNativeHostApiV1, - value: *const NemoRelayNativeString, -) -> Result { - serde_json::from_str(&read_string(host, value)?).map_err(|error| error.to_string()) -} - -pub async fn dispatch_buffered( - host: &NemoRelayNativeHostApiV4, - next: *const NemoRelayNativeAsyncNext, - dispatch: &LlmContinuationInvocationV2, -) -> Result { - let dispatch = HostString::json(&host.v3.v1, dispatch).map_err(internal_error)?; - let (sender, receiver) = oneshot::channel::(); - let sender = Box::into_raw(Box::new(sender)).cast::(); - let status = unsafe { - (host.async_llm_next_invoke_result_v2)(next, dispatch.as_ptr(), buffered_result, sender) - }; - if status != NemoRelayStatus::Ok { - unsafe { - drop(Box::from_raw( - sender.cast::>(), - )) - }; - return Err(internal_error(format!( - "Relay rejected buffered dispatch: {status:?}" - ))); - } - match receiver.await { - Ok(LlmContinuationOutcomeV2::Success { response }) => Ok(response), - Ok(LlmContinuationOutcomeV2::Failure { error }) => Err(error), - Err(_) => Err(internal_error( - "Relay dropped the buffered dispatch callback".into(), - )), - } -} - -pub async fn dispatch_passthrough_buffered( - host: &NemoRelayNativeHostApiV4, - next: *const NemoRelayNativeAsyncNext, - request: &LlmRequest, -) -> Result { - let request = HostString::json(&host.v3.v1, request)?; - let (sender, receiver) = oneshot::channel::>(); - let sender = Box::into_raw(Box::new(sender)).cast::(); - let status = unsafe { - (host.v3.async_next_invoke_result)( - next, - request.as_ptr(), - passthrough_buffered_result, - sender, - ) - }; - if status != NemoRelayStatus::Ok { - unsafe { - drop(Box::from_raw( - sender.cast::>>(), - )) - }; - return Err(format!("Relay rejected passthrough dispatch: {status:?}")); - } - receiver - .await - .map_err(|_| "Relay dropped the passthrough callback".to_string())? -} - -pub async fn dispatch_passthrough_stream( - host: &NemoRelayNativeHostApiV4, - next: *const NemoRelayNativeAsyncNext, - output: *const NemoRelayNativeAsyncStream, - request: &LlmRequest, -) -> Result<(), String> { - let request = HostString::json(&host.v3.v1, request)?; - let (sender, receiver) = oneshot::channel(); - let state = Box::into_raw(Box::new(PassthroughStreamState { - output: output as usize, - sender: Some(sender), - })) - .cast::(); - let status = unsafe { - (host.v3.async_next_invoke_stream)( - next, - request.as_ptr(), - output, - passthrough_stream_result as NemoRelayNativeAsyncNextStreamCb, - state, - ) - }; - if status != NemoRelayStatus::Ok { - unsafe { drop(Box::from_raw(state.cast::())) }; - return Err(format!("Relay rejected passthrough stream: {status:?}")); - } - receiver - .await - .map_err(|_| "Relay dropped the passthrough stream callback".to_string())? -} - -struct PassthroughStreamState { - output: usize, - sender: Option>>, -} - -unsafe extern "C" fn passthrough_stream_result( - user_data: *mut c_void, - chunk_json: *const NemoRelayNativeString, - error: *const NemoRelayNativeString, - done: bool, -) -> bool { - let host = crate::host(); - let state = unsafe { &mut *user_data.cast::() }; - let output = state.output as *const NemoRelayNativeAsyncStream; - let result = if !error.is_null() { - let message = read_string(&host.v3.v1, error) - .unwrap_or_else(|_| "Relay passthrough stream failed".into()); - Some(Err(message)) - } else if done { - let status = finish_stream(host, output); - Some(if status == NemoRelayStatus::Ok { - Ok(()) - } else { - Err(format!( - "Relay rejected passthrough stream finish: {status:?}" - )) - }) - } else { - let result = - read_json(&host.v3.v1, chunk_json).and_then(|chunk| push_stream(host, output, &chunk)); - if result.is_err() { - Some(result) - } else { - None - } - }; - if let Some(result) = result { - let mut state = unsafe { Box::from_raw(user_data.cast::()) }; - if let Some(sender) = state.sender.take() { - let _ = sender.send(result); - } - false - } else { - true - } -} - -unsafe extern "C" fn passthrough_buffered_result( - user_data: *mut c_void, - value_json: *const NemoRelayNativeString, - error: *const NemoRelayNativeString, -) { - let sender = - unsafe { Box::from_raw(user_data.cast::>>()) }; - let host = crate::host(); - let result = if error.is_null() { - read_json(&host.v3.v1, value_json) - } else { - Err(read_string(&host.v3.v1, error) - .unwrap_or_else(|_| "Relay passthrough dispatch failed".into())) - }; - let _ = sender.send(result); -} - -unsafe extern "C" fn buffered_result( - user_data: *mut c_void, - outcome_json: *const NemoRelayNativeString, -) { - let sender = - unsafe { Box::from_raw(user_data.cast::>()) }; - let host = crate::host(); - let outcome = read_json(&host.v3.v1, outcome_json) - .and_then(|value| serde_json::from_value(value).map_err(|error| error.to_string())) - .unwrap_or_else(|error| LlmContinuationOutcomeV2::Failure { - error: internal_error(format!("invalid Relay buffered outcome: {error}")), - }); - let _ = sender.send(outcome); -} - -pub async fn dispatch_stream( - host: &NemoRelayNativeHostApiV4, - next: *const NemoRelayNativeAsyncNext, - output_stream: *const NemoRelayNativeAsyncStream, - dispatch: &LlmContinuationInvocationV2, -) -> Result { - let dispatch = HostString::json(&host.v3.v1, dispatch).map_err(internal_error)?; - let (sender, receiver) = oneshot::channel::>(); - let sender = Box::into_raw(Box::new(sender)).cast::(); - let status = unsafe { - (host.async_llm_next_open_stream_v2)( - next, - dispatch.as_ptr(), - output_stream, - provider_stream_open, - sender, - ) - }; - if status != NemoRelayStatus::Ok { - unsafe { - drop(Box::from_raw( - sender.cast::>>(), - )) - }; - return Err(internal_error(format!( - "Relay rejected streaming dispatch: {status:?}" - ))); - } - let stream = receiver - .await - .map_err(|_| internal_error("Relay dropped the stream-open callback".into()))??; - Ok(ProviderJsonStream { - host: *host, - stream: stream as *const NemoRelayNativeLlmStreamV2, - pending: None, - done: false, - }) -} - -unsafe extern "C" fn provider_stream_open( - user_data: *mut c_void, - stream: *const NemoRelayNativeLlmStreamV2, - error_json: *const NemoRelayNativeString, -) { - let sender = unsafe { - Box::from_raw(user_data.cast::>>()) - }; - let host = crate::host(); - let result = if error_json.is_null() { - if stream.is_null() { - Err(internal_error( - "Relay returned neither a provider stream nor an error".into(), - )) - } else { - Ok(stream as usize) - } - } else { - read_json(&host.v3.v1, error_json) - .and_then(|value| serde_json::from_value(value).map_err(|error| error.to_string())) - .map_err(|error| internal_error(format!("invalid Relay stream-open error: {error}"))) - }; - let _ = sender.send(result); -} - -pub struct ProviderJsonStream { - host: NemoRelayNativeHostApiV4, - stream: *const NemoRelayNativeLlmStreamV2, - pending: Option>, - done: bool, -} - -unsafe impl Send for ProviderJsonStream {} - -impl Stream for ProviderJsonStream { - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if self.done { - return Poll::Ready(None); - } - if self.pending.is_none() { - let (sender, receiver) = oneshot::channel(); - let sender = Box::into_raw(Box::new(sender)).cast::(); - let status = unsafe { - (self.host.async_llm_stream_next_v2)(self.stream, provider_stream_next, sender) - }; - if status != NemoRelayStatus::Ok { - unsafe { - drop(Box::from_raw( - sender.cast::>(), - )) - }; - self.done = true; - return Poll::Ready(Some(Err(internal_error(format!( - "Relay rejected provider stream next: {status:?}" - ))))); - } - self.pending = Some(receiver); - } - let receiver = self.pending.as_mut().expect("pending receiver was set"); - match Pin::new(receiver).poll(cx) { - Poll::Pending => Poll::Pending, - Poll::Ready(Ok(LlmContinuationStreamEventV2::Chunk { chunk })) => { - self.pending = None; - Poll::Ready(Some(Ok(chunk))) - } - Poll::Ready(Ok(LlmContinuationStreamEventV2::Failure { error })) => { - self.pending = None; - self.done = true; - Poll::Ready(Some(Err(error))) - } - Poll::Ready(Ok(LlmContinuationStreamEventV2::Done)) => { - self.pending = None; - self.done = true; - Poll::Ready(None) - } - Poll::Ready(Err(_)) => { - self.pending = None; - self.done = true; - Poll::Ready(Some(Err(internal_error( - "Relay dropped the provider stream next callback".into(), - )))) - } - } - } -} - -impl Drop for ProviderJsonStream { - fn drop(&mut self) { - if !self.done { - unsafe { (self.host.async_llm_stream_cancel_v2)(self.stream) }; - } - unsafe { (self.host.async_llm_stream_release_v2)(self.stream) }; - } -} - -unsafe extern "C" fn provider_stream_next( - user_data: *mut c_void, - event_json: *const NemoRelayNativeString, -) { - let sender = - unsafe { Box::from_raw(user_data.cast::>()) }; - let host = crate::host(); - let event = read_json(&host.v3.v1, event_json) - .and_then(|value| serde_json::from_value(value).map_err(|error| error.to_string())) - .unwrap_or_else(|error| LlmContinuationStreamEventV2::Failure { - error: internal_error(format!("invalid Relay stream event: {error}")), - }); - let _ = sender.send(event); -} - -pub fn resolve_completion( - host: &NemoRelayNativeHostApiV4, - completion: *const NemoRelayNativeAsyncCompletion, - value: &Json, -) -> NemoRelayStatus { - match HostString::json(&host.v3.v1, value) { - Ok(value) => unsafe { (host.v3.async_completion_resolve_json)(completion, value.as_ptr()) }, - Err(_) => NemoRelayStatus::Internal, - } -} - -pub fn reject_completion( - host: &NemoRelayNativeHostApiV4, - completion: *const NemoRelayNativeAsyncCompletion, - message: &str, -) -> NemoRelayStatus { - match HostString::text(&host.v3.v1, message) { - Ok(message) => unsafe { (host.v3.async_completion_reject)(completion, message.as_ptr()) }, - Err(_) => NemoRelayStatus::Internal, - } -} - -pub fn push_stream( - host: &NemoRelayNativeHostApiV4, - stream: *const NemoRelayNativeAsyncStream, - value: &Json, -) -> Result<(), String> { - let value = HostString::json(&host.v3.v1, value)?; - loop { - if unsafe { (host.v3.async_stream_is_cancelled)(stream) } { - return Err("Relay caller cancelled the output stream".into()); - } - match unsafe { (host.v3.async_stream_push_json)(stream, value.as_ptr()) } { - NemoRelayStatus::Ok => return Ok(()), - NemoRelayStatus::Internal => std::thread::yield_now(), - status => return Err(format!("Relay rejected output stream event: {status:?}")), - } - } -} - -pub fn finish_stream( - host: &NemoRelayNativeHostApiV4, - stream: *const NemoRelayNativeAsyncStream, -) -> NemoRelayStatus { - unsafe { (host.v3.async_stream_finish)(stream) } -} - -pub fn reject_stream( - host: &NemoRelayNativeHostApiV4, - stream: *const NemoRelayNativeAsyncStream, - message: &str, -) -> NemoRelayStatus { - match HostString::text(&host.v3.v1, message) { - Ok(message) => unsafe { (host.v3.async_stream_reject)(stream, message.as_ptr()) }, - Err(_) => NemoRelayStatus::Internal, - } -} - -pub unsafe fn release_next(host: &NemoRelayNativeHostApiV4, next: *const NemoRelayNativeAsyncNext) { - unsafe { (host.v3.async_next_release)(next) }; -} - -pub unsafe fn release_stream( - host: &NemoRelayNativeHostApiV4, - stream: *const NemoRelayNativeAsyncStream, -) { - unsafe { (host.v3.async_stream_release)(stream) }; -} - -pub fn internal_error(message: String) -> LlmContinuationFailureV2 { - LlmContinuationFailureV2::NonHttp { - failure: LlmNonHttpFailureV2 { - kind: LlmNonHttpFailureKindV2::Internal, - message, - }, - } -} diff --git a/crates/switchyard-nemo-relay-plugin/src/lib.rs b/crates/switchyard-nemo-relay-plugin/src/lib.rs index 4234fb9bd..54ffdef8f 100644 --- a/crates/switchyard-nemo-relay-plugin/src/lib.rs +++ b/crates/switchyard-nemo-relay-plugin/src/lib.rs @@ -2,30 +2,16 @@ // SPDX-License-Identifier: Apache-2.0 mod config; -mod ffi; mod runtime; mod translation; -use std::ffi::c_void; -use std::panic::{catch_unwind, AssertUnwindSafe}; -use std::sync::{Arc, OnceLock}; +use std::sync::Arc; -use nemo_relay_plugin::{ - ConfigDiagnostic, DiagnosticLevel, Json, NativePlugin, NemoRelayNativeAsyncCallbackState, - NemoRelayNativeAsyncCompletion, NemoRelayNativeAsyncNext, NemoRelayNativeAsyncStream, - NemoRelayNativeHostApiV4, NemoRelayNativeString, NemoRelayStatus, PluginContext, -}; +use nemo_relay_plugin::{ConfigDiagnostic, DiagnosticLevel, Json, NativePlugin, PluginContext}; use serde_json::Map; use crate::config::SwitchyardConfig; -use crate::runtime::{Invocation, SwitchyardRuntime}; - -static HOST: OnceLock = OnceLock::new(); - -pub(crate) fn host() -> &'static NemoRelayNativeHostApiV4 { - HOST.get() - .expect("Switchyard callback invoked before plugin registration") -} +use crate::runtime::SwitchyardRuntime; #[derive(Default)] struct SwitchyardPlugin; @@ -57,53 +43,28 @@ impl NativePlugin for SwitchyardPlugin { plugin_config: &Map, ctx: &mut PluginContext<'_>, ) -> nemo_relay_plugin::Result<()> { - let host = *ctx - .host_api_v4() - .ok_or_else(|| "Switchyard requires Relay native plugin C API v2".to_string())?; - if let Some(registered) = HOST.get() { - if registered.v3.v1.abi_version != host.v3.v1.abi_version { - return Err("Switchyard was initialized with a different Relay host ABI".into()); - } - } else { - HOST.set(host) - .map_err(|_| "failed to retain the Relay native API v2 host table".to_string())?; - } - let config = parse_config(plugin_config)?; let priority = config.priority; let runtime = Arc::new(SwitchyardRuntime::new(config, ctx.runtime())?); - let buffered_state = Box::into_raw(Box::new(Arc::clone(&runtime))).cast::(); - let status = unsafe { - ctx.register_async_llm_execution_v2_raw( - "switchyard.run_stream.buffered", - priority, - buffered_callback, - buffered_state, - Some(free_runtime), - ) - }; - if status != NemoRelayStatus::Ok { - return Err(format!( - "failed to register Switchyard buffered execution: {status:?}" - )); - } - - let stream_state = Box::into_raw(Box::new(runtime)).cast::(); - let status = unsafe { - ctx.register_async_llm_stream_execution_v2_raw( - "switchyard.run_stream.streaming", - priority, - stream_callback, - stream_state, - Some(free_runtime), - ) - }; - if status != NemoRelayStatus::Ok { - return Err(format!( - "failed to register Switchyard streaming execution: {status:?}" - )); - } + let buffered_runtime = Arc::clone(&runtime); + ctx.register_async_llm_execution_v2( + "switchyard.run_stream.buffered", + priority, + move |name, request, continuation| { + let runtime = Arc::clone(&buffered_runtime); + async move { runtime.execute_buffered(name, request, continuation).await } + }, + )?; + + ctx.register_async_llm_stream_execution_v2( + "switchyard.run_stream.streaming", + priority, + move |name, request, continuation| { + let runtime = Arc::clone(&runtime); + async move { runtime.execute_stream(name, request, continuation).await } + }, + )?; Ok(()) } } @@ -113,87 +74,4 @@ fn parse_config(plugin_config: &Map) -> Result>())) }; - } -} - -unsafe extern "C" fn buffered_callback( - user_data: *mut c_void, - invocation_json: *const NemoRelayNativeString, - next: *const NemoRelayNativeAsyncNext, - completion: *const NemoRelayNativeAsyncCompletion, -) -> u32 { - let result = catch_unwind(AssertUnwindSafe(|| { - let runtime = unsafe { &*user_data.cast::>() }; - let invocation = ffi::read_json(&host().v3.v1, invocation_json).and_then(|value| { - serde_json::from_value::(value).map_err(|e| e.to_string()) - }); - match invocation { - Ok(invocation) => { - match futures::executor::block_on(runtime.execute_buffered( - invocation, - host(), - next, - )) { - Ok(response) => { - let _ = ffi::resolve_completion(host(), completion, &response); - } - Err(error) => { - let _ = ffi::reject_completion(host(), completion, &error); - } - } - } - Err(error) => { - let _ = ffi::reject_completion( - host(), - completion, - &format!("invalid Relay LLM invocation: {error}"), - ); - } - } - })); - if result.is_err() { - let _ = - ffi::reject_completion(host(), completion, "Switchyard buffered execution panicked"); - } - unsafe { ffi::release_next(host(), next) }; - NemoRelayNativeAsyncCallbackState::Complete as u32 -} - -unsafe extern "C" fn stream_callback( - user_data: *mut c_void, - invocation_json: *const NemoRelayNativeString, - next: *const NemoRelayNativeAsyncNext, - output: *const NemoRelayNativeAsyncStream, -) -> u32 { - let result = catch_unwind(AssertUnwindSafe(|| { - let runtime = unsafe { &*user_data.cast::>() }; - let invocation = ffi::read_json(&host().v3.v1, invocation_json).and_then(|value| { - serde_json::from_value::(value).map_err(|e| e.to_string()) - }); - let result = match invocation { - Ok(invocation) => futures::executor::block_on(runtime.execute_stream( - invocation, - host(), - next, - output, - )), - Err(error) => Err(format!("invalid Relay LLM stream invocation: {error}")), - }; - if let Err(error) = result { - let _ = ffi::reject_stream(host(), output, &error); - } - })); - if result.is_err() { - let _ = ffi::reject_stream(host(), output, "Switchyard streaming execution panicked"); - } - unsafe { - ffi::release_next(host(), next); - ffi::release_stream(host(), output); - } - NemoRelayNativeAsyncCallbackState::Complete as u32 -} - nemo_relay_plugin::nemo_relay_plugin_v2!(nemo_relay_register_plugin, SwitchyardPlugin::default); diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 2e00c9b87..84b76dd69 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -6,14 +6,13 @@ use std::pin::Pin; use std::sync::{Arc, Mutex}; use std::task::{Context as TaskContext, Poll}; -use futures::Stream; -use futures_util::StreamExt; +use futures_util::{Stream, StreamExt}; use nemo_relay_plugin::{ Json, LlmContinuationFailureV2, LlmContinuationInvocationV2, LlmContinuationTargetV2, - LlmNonHttpFailureKindV2, LlmRequest as RelayRequest, NemoRelayNativeAsyncNext, - NemoRelayNativeAsyncStream, NemoRelayNativeHostApiV4, PluginRuntime, + LlmContinuationV2, LlmJsonAsyncStreamV2, LlmNonHttpFailureKindV2, LlmProviderStreamV2, + LlmRequest as RelayRequest, LlmStreamContinuationV2, LlmStreamExecutionOutcomeV2, + PluginRuntime, }; -use serde::Deserialize; use serde_json::{json, Map}; use switchyard_libsy::{ Algorithm, CallLlmRequest, Context, LibsyError, LlmResponse, Request, Response, Step, @@ -24,19 +23,13 @@ use switchyard_protocol::{ use switchyard_translation::{StreamTranslationState, TranslationEngine}; use crate::config::{SwitchyardConfig, TargetBinding, WireProtocol}; -use crate::{ffi, translation}; - -#[derive(Deserialize)] -pub struct Invocation { - pub name: String, - pub request: RelayRequest, -} +use crate::translation; pub struct SwitchyardRuntime { config: SwitchyardConfig, algorithm: Arc, target_headers: BTreeMap>, - translation: TranslationEngine, + translation: Arc, relay: PluginRuntime, } @@ -57,38 +50,38 @@ impl SwitchyardRuntime { config, algorithm, target_headers, - translation: TranslationEngine::default(), + translation: Arc::new(TranslationEngine::default()), relay, }) } pub async fn execute_buffered( &self, - invocation: Invocation, - host: &NemoRelayNativeHostApiV4, - next: *const NemoRelayNativeAsyncNext, + name: String, + request: RelayRequest, + continuation: LlmContinuationV2, ) -> Result { - let Some(inbound) = WireProtocol::from_call(&invocation.name) else { - return ffi::dispatch_passthrough_buffered(host, next, &invocation.request).await; + let Some(inbound) = WireProtocol::from_call(&name) else { + return continuation.call_passthrough(request).await; }; if !self.config.enabled_inbound_profiles.contains(&inbound) { - return ffi::dispatch_passthrough_buffered(host, next, &invocation.request).await; + return continuation.call_passthrough(request).await; } - let request = self.libsy_request(inbound, &invocation.request, false)?; + let libsy_request = self.libsy_request(inbound, &request, false)?; + let metadata = identity_metadata(&request); let max_attempts = self.config.max_retries.saturating_add(1); for attempt in 1..=max_attempts { self.mark( "switchyard.routing.requested", json!({"algorithm": self.algorithm.name(), "attempt": attempt}), - identity_metadata(&invocation.request), + metadata.clone(), ); match self .drive_buffered( - request.clone(), - host, - next, + libsy_request.clone(), + &continuation, attempt, - identity_metadata(&invocation.request), + metadata.clone(), ) .await { @@ -106,23 +99,17 @@ impl SwitchyardRuntime { self.mark( "switchyard.routing.retry", failure_mark_data(attempt, &failure), - identity_metadata(&invocation.request), + metadata.clone(), ); } Err(failure) => { self.mark( "switchyard.routing.error", failure_mark_data(attempt, &failure), - identity_metadata(&invocation.request), + metadata.clone(), ); return self - .fallback_buffered( - inbound, - request, - host, - next, - identity_metadata(&invocation.request), - ) + .fallback_buffered(inbound, libsy_request, &continuation, metadata) .await; } } @@ -133,8 +120,7 @@ impl SwitchyardRuntime { async fn drive_buffered( &self, request: Request, - host: &NemoRelayNativeHostApiV4, - next: *const NemoRelayNativeAsyncNext, + continuation: &LlmContinuationV2, attempt: u32, mark_metadata: Json, ) -> Result { @@ -161,7 +147,7 @@ impl SwitchyardRuntime { ); } Ok(Step::CallLlm(call)) => { - self.serve_buffered_call(*call, host, next, Arc::clone(&provider_error)) + self.serve_buffered_call(*call, continuation, Arc::clone(&provider_error)) .await .map_err(|error| RunFailure::new(error, &provider_error))?; } @@ -178,8 +164,7 @@ impl SwitchyardRuntime { async fn serve_buffered_call( &self, call: CallLlmRequest, - host: &NemoRelayNativeHostApiV4, - next: *const NemoRelayNativeAsyncNext, + continuation: &LlmContinuationV2, provider_error: Arc>>, ) -> switchyard_libsy::Result<()> { let routed = call.get_routed().clone(); @@ -187,7 +172,7 @@ impl SwitchyardRuntime { let result = async { let target = self.target(&target_name)?; let request = self.dispatch_request(&target_name, target, routed.request, false)?; - match ffi::dispatch_buffered(host, next, &request).await { + match continuation.call(request).await { Ok(response) => { let response = translation::decode_response(&self.translation, target.protocol, &response) @@ -217,8 +202,7 @@ impl SwitchyardRuntime { &self, inbound: WireProtocol, request: Request, - host: &NemoRelayNativeHostApiV4, - next: *const NemoRelayNativeAsyncNext, + continuation: &LlmContinuationV2, metadata: Json, ) -> Result { let target_name = self.config.default_targets.target(inbound); @@ -233,7 +217,8 @@ impl SwitchyardRuntime { let dispatch = self .dispatch_request(target_name, target, request, false) .map_err(|error| error.to_string())?; - let response = ffi::dispatch_buffered(host, next, &dispatch) + let response = continuation + .call(dispatch) .await .map_err(|error| format!("trusted fallback failed: {error:?}"))?; let response = translation::decode_response(&self.translation, target.protocol, &response)?; @@ -306,22 +291,33 @@ impl SwitchyardRuntime { } pub async fn execute_stream( - &self, - invocation: Invocation, - host: &NemoRelayNativeHostApiV4, - next: *const NemoRelayNativeAsyncNext, - output: *const NemoRelayNativeAsyncStream, - ) -> Result<(), String> { - let Some(inbound) = WireProtocol::from_call(&invocation.name) else { - return ffi::dispatch_passthrough_stream(host, next, output, &invocation.request).await; + self: Arc, + name: String, + request: RelayRequest, + continuation: LlmStreamContinuationV2, + ) -> Result { + let Some(inbound) = WireProtocol::from_call(&name) else { + return Ok(LlmStreamExecutionOutcomeV2::Passthrough(request)); }; if !self.config.enabled_inbound_profiles.contains(&inbound) { - return ffi::dispatch_passthrough_stream(host, next, output, &invocation.request).await; + return Ok(LlmStreamExecutionOutcomeV2::Passthrough(request)); } - let request = self.libsy_request(inbound, &invocation.request, true)?; - let metadata = identity_metadata(&invocation.request); + let libsy_request = self.libsy_request(inbound, &request, true)?; + let metadata = identity_metadata(&request); + let stream = self.routed_stream(inbound, libsy_request, continuation, metadata); + Ok(LlmStreamExecutionOutcomeV2::Stream(stream)) + } + + fn routed_stream( + self: Arc, + inbound: WireProtocol, + request: Request, + continuation: LlmStreamContinuationV2, + metadata: Json, + ) -> LlmJsonAsyncStreamV2 { + Box::pin(async_stream::try_stream! { let max_attempts = self.config.max_retries.saturating_add(1); - for attempt in 1..=max_attempts { + 'attempts: for attempt in 1..=max_attempts { self.mark( "switchyard.routing.requested", json!({"algorithm": self.algorithm.name(), "attempt": attempt}), @@ -330,9 +326,7 @@ impl SwitchyardRuntime { let run = match self .drive_stream( request.clone(), - host, - next, - output, + &continuation, attempt, metadata.clone(), ) @@ -345,54 +339,81 @@ impl SwitchyardRuntime { } Err(failure) => { self.emit_stream_error(attempt, &failure, &metadata); - return self - .fallback_stream(inbound, request, host, next, output, metadata.clone()) - .await; + let mut fallback = self + .fallback_stream( + inbound, + request.clone(), + &continuation, + metadata.clone(), + ) + .await?; + while let Some(item) = fallback.next().await { + yield item.map_err(|failure| { + format!( + "trusted fallback stream failed: {}", + failure.failure.error + ) + })?; + } + return; } }; - match self - .emit_returned_stream(run.response, inbound, host, output, &run.provider_error) - .await - { - Ok(()) => { - let status = ffi::finish_stream(host, output); - return if status == nemo_relay_plugin::NemoRelayStatus::Ok { - Ok(()) - } else { - Err(format!("Relay rejected output stream finish: {status:?}")) - }; - } - Err(failure) - if !failure.committed - && failure.failure.retryable() - && attempt < max_attempts => - { - self.emit_stream_retry(attempt, &failure.failure, &metadata); - } - Err(failure) if !failure.committed => { - self.emit_stream_error(attempt, &failure.failure, &metadata); - return self - .fallback_stream(inbound, request, host, next, output, metadata.clone()) - .await; - } - Err(failure) => { - self.emit_stream_error(attempt, &failure.failure, &metadata); - return Err(format!( - "Switchyard stream failed after response commitment: {}", - failure.failure.error - )); + let mut returned = Self::returned_stream( + run.response, + inbound, + Arc::clone(&self.translation), + Arc::clone(&run.provider_error), + ); + while let Some(item) = returned.next().await { + match item { + Ok(event) => yield event, + Err(failure) + if !failure.committed + && failure.failure.retryable() + && attempt < max_attempts => + { + self.emit_stream_retry(attempt, &failure.failure, &metadata); + continue 'attempts; + } + Err(failure) if !failure.committed => { + self.emit_stream_error(attempt, &failure.failure, &metadata); + let mut fallback = self + .fallback_stream( + inbound, + request.clone(), + &continuation, + metadata.clone(), + ) + .await?; + while let Some(item) = fallback.next().await { + yield item.map_err(|failure| { + format!( + "trusted fallback stream failed: {}", + failure.failure.error + ) + })?; + } + return; + } + Err(failure) => { + self.emit_stream_error(attempt, &failure.failure, &metadata); + Err(format!( + "Switchyard stream failed after response commitment: {}", + failure.failure.error + ))?; + } } } + return; } - Err("Switchyard stream retry loop ended without a result".into()) + Err("Switchyard stream retry loop ended without a result".to_string())?; + }) } async fn drive_stream( &self, request: Request, - host: &NemoRelayNativeHostApiV4, - next: *const NemoRelayNativeAsyncNext, - output: *const NemoRelayNativeAsyncStream, + continuation: &LlmStreamContinuationV2, attempt: u32, mark_metadata: Json, ) -> Result { @@ -419,7 +440,7 @@ impl SwitchyardRuntime { ); } Ok(Step::CallLlm(call)) => { - self.serve_stream_call(*call, host, next, output, Arc::clone(&provider_error)) + self.serve_stream_call(*call, continuation, Arc::clone(&provider_error)) .await .map_err(|error| RunFailure::new(error, &provider_error))?; } @@ -441,9 +462,7 @@ impl SwitchyardRuntime { async fn serve_stream_call( &self, call: CallLlmRequest, - host: &NemoRelayNativeHostApiV4, - next: *const NemoRelayNativeAsyncNext, - output: *const NemoRelayNativeAsyncStream, + continuation: &LlmStreamContinuationV2, provider_error: Arc>>, ) -> switchyard_libsy::Result<()> { let routed = call.get_routed().clone(); @@ -452,9 +471,7 @@ impl SwitchyardRuntime { .provider_stream_response( &target_name, routed.request, - host, - next, - output, + continuation, Arc::clone(&provider_error), ) .await @@ -466,15 +483,13 @@ impl SwitchyardRuntime { &self, target_name: &str, request: Request, - host: &NemoRelayNativeHostApiV4, - next: *const NemoRelayNativeAsyncNext, - output: *const NemoRelayNativeAsyncStream, + continuation: &LlmStreamContinuationV2, provider_error: Arc>>, ) -> Result { let target = self.target(target_name)?; let metadata = request.metadata.clone(); let dispatch = self.dispatch_request(target_name, target, request, true)?; - let mut upstream = match ffi::dispatch_stream(host, next, output, &dispatch).await { + let mut upstream = match continuation.open_stream(dispatch).await { Ok(upstream) => upstream, Err(error) => { remember_provider_error(&provider_error, &error); @@ -507,6 +522,7 @@ impl SwitchyardRuntime { first: Some(first), protocol: target.protocol, state, + translation: Arc::clone(&self.translation), provider_error, }); Ok(Response { @@ -518,81 +534,111 @@ impl SwitchyardRuntime { }) } - async fn emit_returned_stream( - &self, + fn returned_stream( response: Response, inbound: WireProtocol, - host: &NemoRelayNativeHostApiV4, - output: *const NemoRelayNativeAsyncStream, - provider_error: &Arc>>, - ) -> Result<(), StreamAttemptFailure> { - let source = response - .metadata - .as_ref() - .and_then(|metadata| metadata.wire_format.as_ref()) - .and_then(WireProtocol::from_wire_format) - .ok_or_else(|| { - StreamAttemptFailure::translation( - "libsy returned a stream without a supported source wire format", + translation: Arc, + provider_error: Arc>>, + ) -> ReturnedJsonStream { + Box::pin(async_stream::stream! { + let source = match response + .metadata + .as_ref() + .and_then(|metadata| metadata.wire_format.as_ref()) + .and_then(WireProtocol::from_wire_format) + { + Some(source) => source, + None => { + yield Err(StreamAttemptFailure::translation( + "libsy returned a stream without a supported source wire format", + false, + &provider_error, + )); + return; + } + }; + let LlmResponse::Stream(mut stream) = response.llm_response else { + yield Err(StreamAttemptFailure::translation( + "libsy returned a buffered response for a streaming request", false, - provider_error, - ) - })?; - let LlmResponse::Stream(mut stream) = response.llm_response else { - return Err(StreamAttemptFailure::translation( - "libsy returned a buffered response for a streaming request", - false, - provider_error, - )); - }; - let mut state = StreamTranslationState::new(source.wire_format(), inbound.wire_format()); - let mut committed = false; - while let Some(item) = stream.next().await { - let event = item - .map_err(|error| StreamAttemptFailure::client(error, committed, provider_error))?; - let events = - translation::encode_stream_event(&self.translation, &mut state, inbound, event) - .map_err(|error| { - StreamAttemptFailure::translation(&error, committed, provider_error) - })?; - for event in events { - ffi::push_stream(host, output, &event).map_err(|error| { - StreamAttemptFailure::translation(&error, committed, provider_error) - })?; - committed = true; + &provider_error, + )); + return; + }; + let mut state = + StreamTranslationState::new(source.wire_format(), inbound.wire_format()); + let mut committed = false; + while let Some(item) = stream.next().await { + let event = match item { + Ok(event) => event, + Err(error) => { + yield Err(StreamAttemptFailure::client( + error, + committed, + &provider_error, + )); + return; + } + }; + let events = match translation::encode_stream_event( + &translation, + &mut state, + inbound, + event, + ) { + Ok(events) => events, + Err(error) => { + yield Err(StreamAttemptFailure::translation( + &error, + committed, + &provider_error, + )); + return; + } + }; + for event in events { + committed = true; + yield Ok(event); + } } - } - if source != inbound { - let events = translation::finish_stream(&self.translation, &mut state, inbound) - .map_err(|error| { - StreamAttemptFailure::translation(&error, committed, provider_error) - })?; - for event in events { - ffi::push_stream(host, output, &event).map_err(|error| { - StreamAttemptFailure::translation(&error, committed, provider_error) - })?; - committed = true; + if source != inbound { + let events = match translation::finish_stream( + &translation, + &mut state, + inbound, + ) { + Ok(events) => events, + Err(error) => { + yield Err(StreamAttemptFailure::translation( + &error, + committed, + &provider_error, + )); + return; + } + }; + for event in events { + committed = true; + yield Ok(event); + } } - } - if !committed { - return Err(StreamAttemptFailure::translation( - "Switchyard produced an empty output stream", - false, - provider_error, - )); - } - Ok(()) + if !committed { + yield Err(StreamAttemptFailure::translation( + "Switchyard produced an empty output stream", + false, + &provider_error, + )); + } + }) } async fn fallback_stream( &self, inbound: WireProtocol, request: Request, - host: &NemoRelayNativeHostApiV4, - next: *const NemoRelayNativeAsyncNext, - output: *const NemoRelayNativeAsyncStream, + continuation: &LlmStreamContinuationV2, metadata: Json, - ) -> Result<(), String> { + ) -> Result { let target_name = self.config.default_targets.target(inbound).to_string(); self.mark( "switchyard.routing.fallback", @@ -604,24 +650,17 @@ impl SwitchyardRuntime { .provider_stream_response( &target_name, request, - host, - next, - output, + continuation, Arc::clone(&provider_error), ) .await .map_err(|error| format!("trusted fallback stream failed: {error}"))?; - self.emit_returned_stream(response, inbound, host, output, &provider_error) - .await - .map_err(|failure| { - format!("trusted fallback stream failed: {}", failure.failure.error) - })?; - let status = ffi::finish_stream(host, output); - if status == nemo_relay_plugin::NemoRelayStatus::Ok { - Ok(()) - } else { - Err(format!("Relay rejected fallback stream finish: {status:?}")) - } + Ok(Self::returned_stream( + response, + inbound, + Arc::clone(&self.translation), + provider_error, + )) } fn emit_stream_retry(&self, attempt: u32, failure: &RunFailure, metadata: &Json) { @@ -646,11 +685,14 @@ struct StreamRun { provider_error: Arc>>, } +type ReturnedJsonStream = Pin> + Send>>; + struct TranslatedProviderStream { - upstream: ffi::ProviderJsonStream, + upstream: LlmProviderStreamV2, first: Option, protocol: WireProtocol, state: StreamTranslationState, + translation: Arc, provider_error: Arc>>, } @@ -664,7 +706,7 @@ impl Stream for TranslatedProviderStream { match Pin::new(&mut self.upstream).poll_next(cx) { Poll::Pending => Poll::Pending, Poll::Ready(Some(Ok(raw))) => { - let translation = TranslationEngine::default(); + let translation = Arc::clone(&self.translation); let protocol = self.protocol; Poll::Ready(Some(decode_provider_event( &translation, @@ -855,6 +897,7 @@ fn identity_metadata(request: &RelayRequest) -> Json { #[cfg(test)] mod tests { use super::*; + use futures_util::FutureExt; use nemo_relay_plugin::{LlmHttpFailureV2, LlmNonHttpFailureV2}; #[test] @@ -872,6 +915,33 @@ mod tests { )); } + #[test] + fn stream_open_failures_keep_typed_retry_semantics() { + let retryable = RunFailure { + error: LibsyError::MissingFinalResponse, + provider_error: Some(LlmContinuationFailureV2::Http { + failure: LlmHttpFailureV2 { + status: 503, + body: "temporarily unavailable".into(), + headers: BTreeMap::new(), + }, + }), + }; + assert!(retryable.retryable()); + + let non_retryable = RunFailure { + error: LibsyError::MissingFinalResponse, + provider_error: Some(LlmContinuationFailureV2::Http { + failure: LlmHttpFailureV2 { + status: 400, + body: "invalid request".into(), + headers: BTreeMap::new(), + }, + }), + }; + assert!(!non_retryable.retryable()); + } + #[test] fn routing_failure_marks_exclude_provider_payloads() { let failure = RunFailure { @@ -913,4 +983,64 @@ mod tests { }) ); } + + #[test] + fn late_provider_failure_stays_committed_and_cannot_retry() { + let raw = json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "provider/model", + "choices": [{ + "index": 0, + "delta": {"content": "committed"}, + "finish_reason": null + }] + }); + let provider_failure = LlmContinuationFailureV2::NonHttp { + failure: LlmNonHttpFailureV2 { + kind: LlmNonHttpFailureKindV2::Timeout, + message: "provider failed after its first event".into(), + }, + }; + let provider_error = Arc::new(Mutex::new(Some(provider_failure))); + let response = Response { + llm_response: LlmResponse::Stream(Box::pin(futures_util::stream::iter([ + Ok(LlmResponseStreamEvent::preserved( + WireProtocol::OpenaiChat.wire_format(), + raw.clone(), + vec![LlmResponseChunk::TextDelta { + index: 0, + text: "committed".into(), + }], + )), + Err(LlmClientError::General("late failure".into())), + ]))), + metadata: Some(Metadata { + wire_format: Some(WireProtocol::OpenaiChat.wire_format()), + ..Metadata::default() + }), + }; + + let mut stream = SwitchyardRuntime::returned_stream( + response, + WireProtocol::OpenaiChat, + Arc::new(TranslationEngine::default()), + provider_error, + ); + let first = stream + .next() + .now_or_never() + .expect("first event is ready") + .expect("first event exists"); + assert_eq!(first.ok(), Some(raw)); + + let failure = stream + .next() + .now_or_never() + .expect("late failure is ready") + .expect("late failure exists") + .expect_err("late failure is propagated"); + assert!(failure.committed); + assert!(failure.failure.retryable()); + } } From 128bead26d48d4096006eacee6c2cd67fec80b59 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sat, 1 Aug 2026 01:15:29 -0600 Subject: [PATCH 18/51] test(plugin): cover safe stream outcomes Signed-off-by: Bryan Bednarski --- .../tests/e2e/fake_provider.py | 32 ++- .../tests/e2e/run_e2e.py | 189 ++++++++++++++++++ 2 files changed, 219 insertions(+), 2 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py b/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py index 3a1b2c4da..fc8adaea8 100644 --- a/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py +++ b/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py @@ -47,10 +47,10 @@ def do_POST(self) -> None: ): self._json(401, {"error": {"message": "target headers were not isolated"}}) return - if model == "fake/retry-once" and attempt == 1: + if model in {"fake/retry-once", "fake/retry-stream-once"} and attempt == 1: self._json(503, {"error": {"message": "retry this request"}}) return - if model == "fake/always-fail": + if model in {"fake/always-fail", "fake/always-fail-stream"}: self._json(400, {"error": {"message": "invalid routed request"}}) return if self.path == "/v1/chat/completions": @@ -73,6 +73,22 @@ def _chat(self, request: dict[str, object]) -> None: if classifier else f"chat from {model}" ) + if request.get("stream") and model == "fake/late-stream-failure": + self._sse_then_disconnect( + { + "id": "chatcmpl-late-failure", + "object": "chat.completion.chunk", + "model": model, + "choices": [ + { + "index": 0, + "delta": {"content": "committed before failure"}, + "finish_reason": None, + } + ], + } + ) + return if request.get("stream"): events: list[dict[str, object]] = [ { @@ -270,6 +286,18 @@ def _sse(self, events: list[dict[str, object]], named: bool = False) -> None: self.wfile.write(data) self.wfile.flush() + def _sse_then_disconnect(self, first: dict[str, object]) -> None: + data = f"data: {json.dumps(first)}\n\n".encode() + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.send_header("cache-control", "no-cache") + self.send_header("content-length", str(len(data) + 64)) + self.send_header("connection", "close") + self.end_headers() + self.wfile.write(data) + self.wfile.flush() + self.close_connection = True + def _json(self, status: int, value: object) -> None: data = json.dumps(value).encode() self.send_response(status) diff --git a/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py b/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py index 092e23160..53eaa4f49 100644 --- a/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py +++ b/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py @@ -7,6 +7,7 @@ import argparse import concurrent.futures +import http.client import json import os import signal @@ -60,6 +61,25 @@ def request( return response.status, response.read() +def request_until_stream_error( + relay_url: str, path: str, body: dict[str, Any] +) -> tuple[int, bytes, bool]: + request = urllib.request.Request( + f"{relay_url}{path}", + data=json.dumps(body).encode(), + headers={ + "content-type": "application/json", + "authorization": "Bearer e2e", + }, + method="POST", + ) + with urllib.request.urlopen(request, timeout=15) as response: + try: + return response.status, response.read(), False + except http.client.IncompleteRead as error: + return response.status, error.partial, True + + def stream_events(raw: bytes) -> list[dict[str, object]]: return [ json.loads(line[6:]) @@ -490,6 +510,8 @@ def single_target_config( atof: Path, selected_model: str, fallback_model: str, + *, + max_retries: int = 1, ) -> str: return plugin_config( manifest, @@ -509,6 +531,7 @@ def single_target_config( """, 'openai_chat = "fallback"', '"openai_chat"', + max_retries=max_retries, ) @@ -574,6 +597,166 @@ def run_retry_and_fallback( } +def run_stream_reliability( + relay_bin: Path, root: Path, manifest: Path, provider_url: str +) -> dict[str, object]: + retry_config = single_target_config( + manifest, + provider_url, + root / "stream-retry" / "atof", + "fake/retry-stream-once", + "fake/retry-stream-fallback", + max_retries=1, + ) + with RelayScenario(relay_bin, root, provider_url, "stream-retry", retry_config) as relay: + body = dict(CASES[0][2]) + body["stream"] = True + status, raw = request(relay.url, CASES[0][1], body) + events = stream_events(raw) + assert status == 200 + assert len(events) == 2 + assert raw.decode().count("data: [DONE]") == 1 + assert stream_text("openai_chat", events) == "chat from fake/retry-stream-once" + retry_marks = relay.marks("switchyard.routing.retry") + assert len(retry_marks) == 1 + assert retry_marks[0]["data"] == { + "attempt": 1, + "retryable": True, + "failure_kind": "http", + "http_status": 503, + } + assert not relay.marks("switchyard.routing.fallback", expected=0) + assert len(relay.marks("switchyard.routing.requested", expected=2)) == 2 + assert len(relay.marks("switchyard.routing.decision", expected=2)) == 2 + + fallback_config = single_target_config( + manifest, + provider_url, + root / "stream-fallback" / "atof", + "fake/always-fail-stream", + "fake/trusted-stream-fallback", + max_retries=1, + ) + with RelayScenario(relay_bin, root, provider_url, "stream-fallback", fallback_config) as relay: + body = dict(CASES[0][2]) + body["stream"] = True + status, raw = request(relay.url, CASES[0][1], body) + events = stream_events(raw) + assert status == 200 + assert len(events) == 2 + assert raw.decode().count("data: [DONE]") == 1 + assert stream_text("openai_chat", events) == "chat from fake/trusted-stream-fallback" + error_marks = relay.marks("switchyard.routing.error") + assert len(error_marks) == 1 + assert error_marks[0]["data"] == { + "attempt": 1, + "retryable": False, + "failure_kind": "http", + "http_status": 400, + } + assert len(relay.marks("switchyard.routing.fallback")) == 1 + assert len(relay.marks("switchyard.routing.requested")) == 1 + assert len(relay.marks("switchyard.routing.decision")) == 1 + + late_config = single_target_config( + manifest, + provider_url, + root / "stream-late-failure" / "atof", + "fake/late-stream-failure", + "fake/late-stream-fallback", + max_retries=1, + ) + with RelayScenario(relay_bin, root, provider_url, "stream-late-failure", late_config) as relay: + body = dict(CASES[0][2]) + body["stream"] = True + status, raw, saw_stream_error = request_until_stream_error( + relay.url, CASES[0][1], body + ) + events = stream_events(raw) + assert status == 200 + assert saw_stream_error + assert len(events) == 1 + assert stream_text("openai_chat", events) == "committed before failure" + assert "data: [DONE]" not in raw.decode() + late_error_marks = relay.marks("switchyard.routing.error") + assert len(late_error_marks) == 1 + assert late_error_marks[0]["data"] == { + "attempt": 1, + "retryable": True, + "failure_kind": "non_http", + "non_http_kind": "transport", + } + assert not relay.marks("switchyard.routing.retry", expected=0) + assert not relay.marks("switchyard.routing.fallback", expected=0) + assert len(relay.marks("switchyard.routing.requested")) == 1 + assert len(relay.marks("switchyard.routing.decision")) == 1 + + calls = http_json(provider_url, "/calls") + assert calls["fake/retry-stream-once"] == 2 + assert calls.get("fake/retry-stream-fallback", 0) == 0 + assert calls["fake/always-fail-stream"] == 1 + assert calls["fake/trusted-stream-fallback"] == 1 + assert calls["fake/late-stream-failure"] == 1 + assert calls.get("fake/late-stream-fallback", 0) == 0 + return { + "retry_attempts": calls["fake/retry-stream-once"], + "fallback_calls": calls["fake/trusted-stream-fallback"], + "late_events_before_failure": len(events), + "late_error_marks": len(late_error_marks), + } + + +def run_unmanaged_passthrough( + relay_bin: Path, root: Path, manifest: Path, provider_url: str +) -> dict[str, object]: + config = plugin_config( + manifest, + provider_url, + root / "unmanaged-passthrough" / "atof", + '[plugins.dynamic.config.algorithm]\nkind = "random"\nseed = 5', + """\ +[plugins.dynamic.config.targets.responses] +model = "fake/unused-responses-target" +protocol = "openai_responses" +base_url = "{provider_url}/v1" +weight = 1 +""", + 'openai_responses = "responses"', + '"openai_responses"', + ) + with RelayScenario(relay_bin, root, provider_url, "unmanaged-passthrough", config) as relay: + body = dict(CASES[0][2]) + body["model"] = "fake/unmanaged-passthrough" + body["stream"] = False + status, raw = request(relay.url, CASES[0][1], body) + response = json.loads(raw) + assert status == 200 + assert response["model"] == "fake/unmanaged-passthrough" + + body["stream"] = True + status, raw = request(relay.url, CASES[0][1], body) + events = stream_events(raw) + assert status == 200 + assert len(events) == 2 + assert raw.decode().count("data: [DONE]") == 1 + assert stream_text("openai_chat", events) == "chat from fake/unmanaged-passthrough" + + assert not relay.marks("switchyard.routing.requested", expected=0) + assert not relay.marks("switchyard.routing.decision", expected=0) + assert not relay.marks("switchyard.routing.retry", expected=0) + assert not relay.marks("switchyard.routing.error", expected=0) + assert not relay.marks("switchyard.routing.fallback", expected=0) + calls = http_json(provider_url, "/calls") + assert calls["fake/unmanaged-passthrough"] == 2 + assert calls.get("fake/unused-responses-target", 0) == 0 + return { + "buffered": True, + "streaming": True, + "provider_calls": calls["fake/unmanaged-passthrough"], + "switchyard_marks": 0, + } + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( @@ -674,6 +857,12 @@ def main() -> None: "reliability": run_retry_and_fallback( relay_bin, root, bundle / "relay-plugin.toml", provider_url ), + "stream_reliability": run_stream_reliability( + relay_bin, root, bundle / "relay-plugin.toml", provider_url + ), + "unmanaged_passthrough": run_unmanaged_passthrough( + relay_bin, root, bundle / "relay-plugin.toml", provider_url + ), } recorded = [ path From 5ac0f3ae6d93bd7af71906dfb53be077d2f7e521 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sat, 1 Aug 2026 01:59:02 -0600 Subject: [PATCH 19/51] chore(plugin): update Relay safe v2 SDK pin Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 ++-- crates/switchyard-nemo-relay-plugin/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a37b3708..a0221f378 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=ed397cd598d0a5c3bde94b47db7b796077ab3d9a#ed397cd598d0a5c3bde94b47db7b796077ab3d9a" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=da4376f7b360dbdbeae0b992c20a88cf0acfd6f0#da4376f7b360dbdbeae0b992c20a88cf0acfd6f0" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=ed397cd598d0a5c3bde94b47db7b796077ab3d9a#ed397cd598d0a5c3bde94b47db7b796077ab3d9a" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=da4376f7b360dbdbeae0b992c20a88cf0acfd6f0#da4376f7b360dbdbeae0b992c20a88cf0acfd6f0" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index da43ceaae..9cef2eb2a 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib", "rlib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "ed397cd598d0a5c3bde94b47db7b796077ab3d9a" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "da4376f7b360dbdbeae0b992c20a88cf0acfd6f0" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true From 325ed5f6f2110bd663df8b287ffdc748b911d13b Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sat, 1 Aug 2026 03:29:34 -0600 Subject: [PATCH 20/51] chore(plugin): update Relay safe v2 SDK pin Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 ++-- crates/switchyard-nemo-relay-plugin/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a0221f378..0a18660a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=da4376f7b360dbdbeae0b992c20a88cf0acfd6f0#da4376f7b360dbdbeae0b992c20a88cf0acfd6f0" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=369cbd47cf5db1800578eb007ba0f8442037f470#369cbd47cf5db1800578eb007ba0f8442037f470" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=da4376f7b360dbdbeae0b992c20a88cf0acfd6f0#da4376f7b360dbdbeae0b992c20a88cf0acfd6f0" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=369cbd47cf5db1800578eb007ba0f8442037f470#369cbd47cf5db1800578eb007ba0f8442037f470" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 9cef2eb2a..52dcf9e20 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib", "rlib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "da4376f7b360dbdbeae0b992c20a88cf0acfd6f0" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "369cbd47cf5db1800578eb007ba0f8442037f470" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true From e6b6f49f6d56fe0f791ba093dbd04e2586ae22ef Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sat, 1 Aug 2026 11:34:23 -0600 Subject: [PATCH 21/51] refactor(plugin): own routing retry policy Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 +- .../switchyard-nemo-relay-plugin/Cargo.toml | 2 +- crates/switchyard-nemo-relay-plugin/README.md | 9 +-- .../src/config.rs | 9 --- .../src/runtime.rs | 68 +++++++++++++------ 5 files changed, 56 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a18660a9..970388160 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=369cbd47cf5db1800578eb007ba0f8442037f470#369cbd47cf5db1800578eb007ba0f8442037f470" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=54452966e9c12065cf77ac3426ee81198a6658b0#54452966e9c12065cf77ac3426ee81198a6658b0" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=369cbd47cf5db1800578eb007ba0f8442037f470#369cbd47cf5db1800578eb007ba0f8442037f470" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=54452966e9c12065cf77ac3426ee81198a6658b0#54452966e9c12065cf77ac3426ee81198a6658b0" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 52dcf9e20..af154f631 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib", "rlib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "369cbd47cf5db1800578eb007ba0f8442037f470" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "54452966e9c12065cf77ac3426ee81198a6658b0" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 3573e85a4..eae2a76a0 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -54,10 +54,11 @@ host queue; unmanaged provider events do not cross the plugin ABI. Provider failures use HTTP semantics: status, a bounded body, and safe response headers when Relay received an HTTP response; otherwise a transport, timeout, -cancelled, invalid-request, guardrail, or internal kind. Retry policy is derived -from those values rather than serialized. HTTP 408, 425, 429, 500, 502, 503, -and 504 plus transport and timeout failures retry. The plugin does not inspect -provider bodies to reclassify HTTP 400 context-window or HTTP 404 model errors. +cancelled, invalid-request, guardrail, or internal kind. Relay reports those +neutral failure facts; the Switchyard plugin owns retry and fallback policy. +HTTP 408, 425, 429, 500, 502, 503, and 504 plus transport and timeout failures +retry. The plugin does not inspect provider bodies to reclassify HTTP 400 +context-window or HTTP 404 model errors. `switchyard-translation` is used for same-protocol routes as well as cross-protocol routes. Same-protocol response events replay their preserved diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index acd6bedae..97a23e7d8 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -5,7 +5,6 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use http::header::{HeaderName, HeaderValue}; -use nemo_relay_plugin::LlmContinuationRouteV2; use serde::{Deserialize, Serialize}; use switchyard_libsy::algorithms::{LlmTaskClassifier, Random, TaskClassifierConfig}; use switchyard_libsy::{Algorithm, LlmTarget, LlmTargetSet}; @@ -49,14 +48,6 @@ impl WireProtocol { } } - pub const fn relay_route(self) -> LlmContinuationRouteV2 { - match self { - Self::OpenaiChat => LlmContinuationRouteV2::OpenaiChat, - Self::OpenaiResponses => LlmContinuationRouteV2::OpenaiResponses, - Self::AnthropicMessages => LlmContinuationRouteV2::AnthropicMessages, - } - } - pub const fn wire_format(self) -> WireFormat { match self { Self::OpenaiChat => WireFormat::OpenAiChat, diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 84b76dd69..8ac0d50cf 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -269,7 +269,6 @@ impl SwitchyardRuntime { target: LlmContinuationTargetV2 { method: "POST".into(), url: target.dispatch_url(), - route: target.protocol.relay_route(), headers, }, }) @@ -810,7 +809,19 @@ impl RunFailure { fn retryable(&self) -> bool { self.provider_error .as_ref() - .is_some_and(LlmContinuationFailureV2::is_retryable) + .is_some_and(should_retry_provider_failure) + } +} + +fn should_retry_provider_failure(failure: &LlmContinuationFailureV2) -> bool { + match failure { + LlmContinuationFailureV2::Http { failure } => { + matches!(failure.status, 408 | 425 | 429 | 500 | 502 | 503 | 504) + } + LlmContinuationFailureV2::NonHttp { failure } => matches!( + failure.kind, + LlmNonHttpFailureKindV2::Transport | LlmNonHttpFailureKindV2::Timeout + ), } } @@ -916,30 +927,47 @@ mod tests { } #[test] - fn stream_open_failures_keep_typed_retry_semantics() { - let retryable = RunFailure { - error: LibsyError::MissingFinalResponse, - provider_error: Some(LlmContinuationFailureV2::Http { + fn switchyard_retry_policy_uses_http_status_and_non_http_kind() { + for status in [408, 425, 429, 500, 502, 503, 504] { + let failure = LlmContinuationFailureV2::Http { failure: LlmHttpFailureV2 { - status: 503, - body: "temporarily unavailable".into(), + status, + body: String::new(), headers: BTreeMap::new(), }, - }), - }; - assert!(retryable.retryable()); - - let non_retryable = RunFailure { - error: LibsyError::MissingFinalResponse, - provider_error: Some(LlmContinuationFailureV2::Http { + }; + assert!(should_retry_provider_failure(&failure), "status={status}"); + } + for status in [400, 401, 404, 409, 422, 501] { + let failure = LlmContinuationFailureV2::Http { failure: LlmHttpFailureV2 { - status: 400, - body: "invalid request".into(), + status, + body: String::new(), headers: BTreeMap::new(), }, - }), - }; - assert!(!non_retryable.retryable()); + }; + assert!(!should_retry_provider_failure(&failure), "status={status}"); + } + for (kind, expected) in [ + (LlmNonHttpFailureKindV2::Transport, true), + (LlmNonHttpFailureKindV2::Timeout, true), + (LlmNonHttpFailureKindV2::Cancelled, false), + (LlmNonHttpFailureKindV2::InvalidRequest, false), + (LlmNonHttpFailureKindV2::Guardrail, false), + (LlmNonHttpFailureKindV2::Internal, false), + ] { + let failure = LlmContinuationFailureV2::NonHttp { + failure: LlmNonHttpFailureV2 { + kind, + message: String::new(), + }, + }; + assert_eq!( + should_retry_provider_failure(&failure), + expected, + "{kind:?}" + ); + } } #[test] From b56c907ce89fcc4a4110b45f1c4a042ffb49a289 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 10:27:34 -0600 Subject: [PATCH 22/51] chore(plugin): adopt cooperative Relay SDK Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 ++-- .../switchyard-nemo-relay-plugin/Cargo.toml | 2 +- crates/switchyard-nemo-relay-plugin/README.md | 22 ++++++++++++++----- .../relay-plugin.toml | 2 +- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 970388160..31082b211 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=54452966e9c12065cf77ac3426ee81198a6658b0#54452966e9c12065cf77ac3426ee81198a6658b0" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=5126ff511aa6d655f2075a37973729d30b5dc33c#5126ff511aa6d655f2075a37973729d30b5dc33c" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=54452966e9c12065cf77ac3426ee81198a6658b0#54452966e9c12065cf77ac3426ee81198a6658b0" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=5126ff511aa6d655f2075a37973729d30b5dc33c#5126ff511aa6d655f2075a37973729d30b5dc33c" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index af154f631..f320c9163 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib", "rlib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "54452966e9c12065cf77ac3426ee81198a6658b0" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "5126ff511aa6d655f2075a37973729d30b5dc33c" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index eae2a76a0..6e7ad2d85 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -18,13 +18,21 @@ does not link the Relay runtime, use `switchyard-llm-client`, or start `switchyard-server`. `switchyard-translation` is the only request, response, and stream translation layer. +Relay polls the plugin's pending Rust futures cooperatively on its Tokio +runtime and restores the captured continuation and scope context on every poll. +An active `run_stream` policy therefore does not occupy a blocking worker for +the lifetime of the request. Provider results, stream capacity, and +cancellation wake the policy through the generic native task contract. + The crate is a source/build unit and is not published to crates.io. Operators install a release bundle containing the compiled shared library, materialized `relay-plugin.toml`, `config.schema.json`, licensing files, and checksum. -During development, `nemo-relay-plugin` is pinned to the Relay ABI v2 feature -commit. Replace that Git dependency with the first published compatible SDK -version before releasing a bundle. +During development, `nemo-relay-plugin` is pinned to the Relay native API v2 +feature commit. Native API v2 remains unreleased, so every bundle must be +rebuilt against the exact pinned revision; an older v2 bundle must not be used +with a newer draft host table. Replace the Git dependency with the first +published compatible SDK version before releasing a bundle. ## Runtime contract @@ -44,9 +52,11 @@ Switchyard owns routing, translation, target URLs, and target credentials. Relay validates and transports the selected HTTP target, runs it through the captured LLM continuation, and owns stream transport and event export. Switchyard retries or falls back only before the first caller event; after -commitment, a late provider failure is returned without retry. Target data never -enters `LlmRequest.headers`, marks, or spans. The plugin contains no Relay -provider codecs and does not use private dispatch headers. +commitment, a late provider failure is returned without retry. Target URLs, +transport headers, and credentials never enter `LlmRequest.headers`, marks, or +spans; semantic target names remain visible in genuine routing marks. The +plugin contains no Relay provider codecs and does not use private dispatch +headers. Calls outside the enabled profiles return the SDK's explicit `Passthrough` outcome. Relay then forwards the downstream provider stream through its bounded diff --git a/crates/switchyard-nemo-relay-plugin/relay-plugin.toml b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml index 38f38d848..9f6736630 100644 --- a/crates/switchyard-nemo-relay-plugin/relay-plugin.toml +++ b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml @@ -8,7 +8,7 @@ id = "nvidia.switchyard" kind = "rust_dynamic" [compat] -relay = ">=0.7,<1.0" +relay = ">=0.8,<1.0" native_api = "2" [defaults] From cb2b785d296d944f5b3062d9cc09755943b3b6c3 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 10:38:08 -0600 Subject: [PATCH 23/51] refactor(plugin): rely on host cooperative runtime Signed-off-by: Bryan Bednarski --- Cargo.lock | 7 -- Cargo.toml | 1 - crates/libsy/Cargo.toml | 1 - crates/libsy/src/core/algorithm.rs | 84 ++++++------------- crates/libsy/src/core/driver.rs | 30 +++---- .../src/config.rs | 5 +- .../src/runtime.rs | 11 ++- 7 files changed, 46 insertions(+), 93 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 31082b211..1e7d5873d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -580,12 +580,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" -[[package]] -name = "futures-timer" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" - [[package]] name = "futures-util" version = "0.3.32" @@ -2062,7 +2056,6 @@ version = "0.2.0" dependencies = [ "async-trait", "futures", - "futures-timer", "opentelemetry", "opentelemetry_sdk", "parking_lot", diff --git a/Cargo.toml b/Cargo.toml index 3da8e96b8..d9186bd0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,6 @@ rust-version = "1.96.1" async-stream = "0.3" async-trait = "0.1" futures = "0.3" -futures-timer = "3" futures-util = "0.3" http = "1" httpdate = "1" diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.toml index 2e0dd0864..dc2be5249 100644 --- a/crates/libsy/Cargo.toml +++ b/crates/libsy/Cargo.toml @@ -16,7 +16,6 @@ async-trait.workspace = true serde.workspace = true serde_json.workspace = true futures.workspace = true -futures-timer.workspace = true # Metrics-only OTel API: instruments record through the host-installed global # meter provider. Pinned to the 0.32 line used across the workspace. opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] } diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index 7c34e97e3..7f8a922f3 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -7,14 +7,13 @@ use std::{ collections::{HashMap, HashSet}, - panic::AssertUnwindSafe, pin::Pin, sync::Arc, time::{Duration, Instant}, }; use async_trait::async_trait; -use futures::{FutureExt, Stream, StreamExt}; +use futures::{Stream, StreamExt}; use parking_lot::Mutex; use tracing::Instrument; @@ -333,6 +332,15 @@ pub enum Step { ReturnToAgent(Box), } +/// Abort guard +struct AbortOnDrop(tokio::task::AbortHandle); + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.abort(); + } +} + /// A named routing target: a `semantic_name` an algorithm routes by, and an optional /// [`RoutedLlmClient`] to serve its calls. An algorithm hands a target to /// [`Driver::call_llm_target`]; the client rides along as @@ -594,10 +602,6 @@ pub trait Algorithm: Send + Sync + 'static { /// Each [`Step::CallLlm`] is an offloaded model call the consumer must serve. /// The stream ends with a [`Step::ReturnToAgent`] on success, or an `Err` item on failure. /// Report each model call to `observer`. - /// - /// The returned stream is lazy and executor-neutral: polling it drives the - /// algorithm future on the consumer's executor. Dropping it cancels that - /// future, including algorithms waiting without an outstanding driver call. fn run_stream( self: Arc, ctx: Context, @@ -620,7 +624,7 @@ pub trait Algorithm: Send + Sync + 'static { // contextual parenting. let span = observability::run_span(self.name(), &request); let observed_driver = task_driver.clone(); - let task = AssertUnwindSafe( + let handle = tokio::spawn( async move { observability::observe_run( task_ctx.clone(), @@ -630,18 +634,17 @@ pub trait Algorithm: Send + Sync + 'static { .await } .instrument(span), - ) - .catch_unwind(); + ); + // Dropping the stream aborts the algorithm task, so it doesn't keep running after the + let abort_guard = AbortOnDrop(handle.abort_handle()); let finish_driver = driver.clone(); let finish_ctx = ctx; let tail: StepStream = Box::pin( futures::stream::once(async move { - let result = match task.await { + let result = match handle.await { Ok(response) => response, - Err(_) => Err(LibsyError::AlgorithmError { - message: "algorithm task panicked".to_string(), - }), + Err(source) => Err(LibsyError::AlgorithmTask { source }), }; finish_driver.finish(finish_ctx, result).await }) @@ -649,7 +652,11 @@ pub trait Algorithm: Send + Sync + 'static { ); let stream: StepStream = Box::pin(stream); - Box::pin(futures::stream::select(stream, tail)) + Box::pin(futures::stream::select(stream, tail).map(move |step| { + // link abort guard to stream + let _keep_alive = &abort_guard; + step + })) } /// Process a request to completion, returning the final [`Response`] and the trace of @@ -1073,34 +1080,6 @@ mod tests { Ok(()) } - #[test] - fn run_stream_is_executor_neutral_for_embedded_hosts() -> Result<()> { - futures::executor::block_on(async { - let mut stream = orch(target_set(&[("embedded/model", false)])) - .run_stream(Context::default(), request()); - let mut returned = false; - while let Some(step) = stream.next().await { - match step? { - Step::CallLlm(call) => { - call.respond(Ok(Response { - llm_response: LlmResponse::Agg(text_response(None, "embedded")), - metadata: None, - }))?; - } - Step::Decision(_) => {} - Step::ReturnToAgent(response) => { - returned = response - .llm_response - .as_agg() - .is_some_and(|response| completion_text(response) == "embedded"); - } - } - } - assert!(returned, "embedded executor did not receive ReturnToAgent"); - Ok(()) - }) - } - #[tokio::test] async fn client_backed_target_offloads_with_a_default_client() -> Result<()> { // Every call now offloads to the stream; a client-backed target rides its @@ -1340,10 +1319,7 @@ mod tests { dropped: dropped.clone(), }); - let mut stream = algo.run_stream(Context::default(), request(), None); - // `run_stream` is a lazy stream: poll it once to start the algorithm - // before checking that dropping the stream cancels the in-flight task. - assert!(futures::poll!(stream.as_mut().next()).is_pending()); + let stream = algo.run_stream(Context::default(), request(), None); started_rx .recv() .await @@ -1360,8 +1336,8 @@ mod tests { #[tokio::test] async fn create_run_task_panic_surfaces_as_a_stream_error() -> Result<()> { - // An algorithm panic must surface as an `Err` step to the stream - // consumer, not abort the embedding host. + // An algorithm whose task panics must surface an `Err` step to the stream + // consumer, not abort the process from an unobserved detached task. struct Panicky; #[async_trait] @@ -1388,11 +1364,7 @@ mod tests { while let Some(step) = stream.next().await { match step { Err(err) => { - assert!(matches!( - err, - LibsyError::AlgorithmError { message } - if message == "algorithm task panicked" - )); + assert!(matches!(err, LibsyError::AlgorithmTask { .. })); saw_error = true; } Ok(_) => return Err(test_error("expected the panic to surface as an error step")), @@ -1431,11 +1403,7 @@ mod tests { "expected run to surface the algorithm panic as an error", )), Err(err) => { - assert!(matches!( - err, - LibsyError::AlgorithmError { message } - if message == "algorithm task panicked" - )); + assert!(matches!(err, LibsyError::AlgorithmTask { .. })); Ok(()) } } diff --git a/crates/libsy/src/core/driver.rs b/crates/libsy/src/core/driver.rs index 04eb9e3eb..c9e233e2c 100644 --- a/crates/libsy/src/core/driver.rs +++ b/crates/libsy/src/core/driver.rs @@ -42,21 +42,20 @@ //! There is no explicit stop method — the consumer terminates by **dropping the //! stream** (and any [`DriverRequest`] it is holding). The producer's next publish //! (`fulfill_request`/`info`/`done`/`fail`) then resolves to `Err`, and a producer -//! awaiting a response sees `Err` once the promise it handed out is dropped. In -//! [`Algorithm::run_stream`](super::algorithm::Algorithm::run_stream), the producer -//! future is owned by the returned stream, so dropping the stream also cancels an -//! algorithm that is between driver interactions. +//! awaiting a response sees `Err` once the promise it handed out is dropped. Either +//! way the algorithm unwinds cooperatively at its next driver interaction. Because the +//! producer runs on a task the driver does not own, hard cancellation (e.g. mid-compute +//! that never touches the driver) is the caller's concern — abort the producer task. use std::{any::Any, sync::Arc}; use crate::{DriverError, LibsyError, Result}; use parking_lot::Mutex; -use futures::{future::Either, pin_mut, Stream, StreamExt}; -use futures_timer::Delay; +use futures::{Stream, StreamExt}; use switchyard_protocol::Context; use tokio::sync::{mpsc, oneshot}; -use tokio::time::Duration; +use tokio::time::{timeout, Duration}; use tokio_stream::wrappers::ReceiverStream; type BoxAny = Box; @@ -176,17 +175,12 @@ impl TypeErasedDriver { // Outer error: the promise was dropped without a response. Inner error: the // consumer fulfilled it with an explicit `Err` — propagate it as-is. - let timeout = Delay::new(FULFILL_REQUEST_TIMEOUT); - pin_mut!(rx, timeout); - let response = match futures::future::select(rx, timeout).await { - Either::Left((response, _)) => response.map_err(|_| DriverError::ResponseDropped)??, - Either::Right(((), _)) => { - return Err(DriverError::ResponseTimedOut { - timeout: FULFILL_REQUEST_TIMEOUT, - } - .into()); - } - }; + let response = timeout(FULFILL_REQUEST_TIMEOUT, rx) + .await + .map_err(|_| DriverError::ResponseTimedOut { + timeout: FULFILL_REQUEST_TIMEOUT, + })? + .map_err(|_| DriverError::ResponseDropped)??; response.downcast::().map(|boxed| *boxed).map_err(|_| { DriverError::TypeMismatch { expected: std::any::type_name::(), diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index 97a23e7d8..828b56c81 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -6,8 +6,9 @@ use std::sync::Arc; use http::header::{HeaderName, HeaderValue}; use serde::{Deserialize, Serialize}; -use switchyard_libsy::algorithms::{LlmTaskClassifier, Random, TaskClassifierConfig}; -use switchyard_libsy::{Algorithm, LlmTarget, LlmTargetSet}; +use switchyard_libsy::{ + Algorithm, LlmTarget, LlmTargetSet, LlmTaskClassifier, Random, TaskClassifierConfig, +}; use switchyard_protocol::WireFormat; #[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 8ac0d50cf..85fd4653f 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -14,11 +14,10 @@ use nemo_relay_plugin::{ PluginRuntime, }; use serde_json::{json, Map}; -use switchyard_libsy::{ - Algorithm, CallLlmRequest, Context, LibsyError, LlmResponse, Request, Response, Step, -}; +use switchyard_libsy::{Algorithm, CallLlmRequest, LibsyError, Step}; use switchyard_protocol::{ - LlmClientError, LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, Metadata, + Context, LlmClientError, LlmResponse, LlmResponseChunk, LlmResponseStream, + LlmResponseStreamEvent, Metadata, Request, Response, }; use switchyard_translation::{StreamTranslationState, TranslationEngine}; @@ -128,7 +127,7 @@ impl SwitchyardRuntime { context .values .insert("relay.routing_attempt".into(), attempt.to_string()); - let mut steps = self.algorithm.clone().run_stream(context, request); + let mut steps = self.algorithm.clone().run_stream(context, request, None); let provider_error = Arc::new(Mutex::new(None)); while let Some(step) = steps.next().await { match step { @@ -420,7 +419,7 @@ impl SwitchyardRuntime { context .values .insert("relay.routing_attempt".into(), attempt.to_string()); - let mut steps = self.algorithm.clone().run_stream(context, request); + let mut steps = self.algorithm.clone().run_stream(context, request, None); let provider_error = Arc::new(Mutex::new(None)); while let Some(step) = steps.next().await { match step { From 434e27f041ae010c8a858a2a7a5ad102667d31fa Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 10:52:04 -0600 Subject: [PATCH 24/51] fix(plugin): keep libsy futures executor neutral Signed-off-by: Bryan Bednarski --- Cargo.lock | 11 ++- Cargo.toml | 1 + crates/libsy/Cargo.toml | 1 + crates/libsy/src/core/algorithm.rs | 87 +++++++++++++------ crates/libsy/src/core/driver.rs | 30 ++++--- .../switchyard-nemo-relay-plugin/Cargo.toml | 2 +- crates/switchyard-nemo-relay-plugin/README.md | 6 ++ 7 files changed, 97 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1e7d5873d..e4d31eb6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -580,6 +580,12 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + [[package]] name = "futures-util" version = "0.3.32" @@ -1104,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=5126ff511aa6d655f2075a37973729d30b5dc33c#5126ff511aa6d655f2075a37973729d30b5dc33c" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=6a55a82d8d2b30f384027a1d4d086552d76a341b#6a55a82d8d2b30f384027a1d4d086552d76a341b" dependencies = [ "futures", "nemo-relay-types", @@ -1115,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=5126ff511aa6d655f2075a37973729d30b5dc33c#5126ff511aa6d655f2075a37973729d30b5dc33c" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=6a55a82d8d2b30f384027a1d4d086552d76a341b#6a55a82d8d2b30f384027a1d4d086552d76a341b" dependencies = [ "bitflags", "chrono", @@ -2056,6 +2062,7 @@ version = "0.2.0" dependencies = [ "async-trait", "futures", + "futures-timer", "opentelemetry", "opentelemetry_sdk", "parking_lot", diff --git a/Cargo.toml b/Cargo.toml index d9186bd0c..3da8e96b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ rust-version = "1.96.1" async-stream = "0.3" async-trait = "0.1" futures = "0.3" +futures-timer = "3" futures-util = "0.3" http = "1" httpdate = "1" diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.toml index dc2be5249..2e0dd0864 100644 --- a/crates/libsy/Cargo.toml +++ b/crates/libsy/Cargo.toml @@ -16,6 +16,7 @@ async-trait.workspace = true serde.workspace = true serde_json.workspace = true futures.workspace = true +futures-timer.workspace = true # Metrics-only OTel API: instruments record through the host-installed global # meter provider. Pinned to the 0.32 line used across the workspace. opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] } diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index 7f8a922f3..17eceac00 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -7,13 +7,14 @@ use std::{ collections::{HashMap, HashSet}, + panic::AssertUnwindSafe, pin::Pin, sync::Arc, time::{Duration, Instant}, }; use async_trait::async_trait; -use futures::{Stream, StreamExt}; +use futures::{FutureExt, Stream, StreamExt}; use parking_lot::Mutex; use tracing::Instrument; @@ -332,15 +333,6 @@ pub enum Step { ReturnToAgent(Box), } -/// Abort guard -struct AbortOnDrop(tokio::task::AbortHandle); - -impl Drop for AbortOnDrop { - fn drop(&mut self) { - self.0.abort(); - } -} - /// A named routing target: a `semantic_name` an algorithm routes by, and an optional /// [`RoutedLlmClient`] to serve its calls. An algorithm hands a target to /// [`Driver::call_llm_target`]; the client rides along as @@ -602,6 +594,10 @@ pub trait Algorithm: Send + Sync + 'static { /// Each [`Step::CallLlm`] is an offloaded model call the consumer must serve. /// The stream ends with a [`Step::ReturnToAgent`] on success, or an `Err` item on failure. /// Report each model call to `observer`. + /// + /// The returned stream is lazy and executor-neutral: polling it drives the + /// algorithm future on the consumer's executor. Dropping it cancels that + /// future, including algorithms waiting without an outstanding driver call. fn run_stream( self: Arc, ctx: Context, @@ -624,7 +620,7 @@ pub trait Algorithm: Send + Sync + 'static { // contextual parenting. let span = observability::run_span(self.name(), &request); let observed_driver = task_driver.clone(); - let handle = tokio::spawn( + let task = AssertUnwindSafe( async move { observability::observe_run( task_ctx.clone(), @@ -634,17 +630,18 @@ pub trait Algorithm: Send + Sync + 'static { .await } .instrument(span), - ); - // Dropping the stream aborts the algorithm task, so it doesn't keep running after the - let abort_guard = AbortOnDrop(handle.abort_handle()); + ) + .catch_unwind(); let finish_driver = driver.clone(); let finish_ctx = ctx; let tail: StepStream = Box::pin( futures::stream::once(async move { - let result = match handle.await { + let result = match task.await { Ok(response) => response, - Err(source) => Err(LibsyError::AlgorithmTask { source }), + Err(_) => Err(LibsyError::AlgorithmError { + message: "algorithm task panicked".to_string(), + }), }; finish_driver.finish(finish_ctx, result).await }) @@ -652,11 +649,7 @@ pub trait Algorithm: Send + Sync + 'static { ); let stream: StepStream = Box::pin(stream); - Box::pin(futures::stream::select(stream, tail).map(move |step| { - // link abort guard to stream - let _keep_alive = &abort_guard; - step - })) + Box::pin(futures::stream::select(stream, tail)) } /// Process a request to completion, returning the final [`Response`] and the trace of @@ -1080,6 +1073,37 @@ mod tests { Ok(()) } + #[test] + fn run_stream_is_executor_neutral_for_embedded_hosts() -> Result<()> { + futures::executor::block_on(async { + let mut stream = orch(target_set(&[("embedded/model", false)])).run_stream( + Context::default(), + request(), + None, + ); + let mut returned = false; + while let Some(step) = stream.next().await { + match step? { + Step::CallLlm(call) => { + call.respond(Ok(Response { + llm_response: LlmResponse::Agg(text_response(None, "embedded")), + metadata: None, + }))?; + } + Step::Decision(_) => {} + Step::ReturnToAgent(response) => { + returned = response + .llm_response + .as_agg() + .is_some_and(|response| completion_text(response) == "embedded"); + } + } + } + assert!(returned, "embedded executor did not receive ReturnToAgent"); + Ok(()) + }) + } + #[tokio::test] async fn client_backed_target_offloads_with_a_default_client() -> Result<()> { // Every call now offloads to the stream; a client-backed target rides its @@ -1319,7 +1343,10 @@ mod tests { dropped: dropped.clone(), }); - let stream = algo.run_stream(Context::default(), request(), None); + let mut stream = algo.run_stream(Context::default(), request(), None); + // `run_stream` is a lazy stream: poll it once to start the algorithm + // before checking that dropping the stream cancels the in-flight task. + assert!(futures::poll!(stream.as_mut().next()).is_pending()); started_rx .recv() .await @@ -1336,8 +1363,8 @@ mod tests { #[tokio::test] async fn create_run_task_panic_surfaces_as_a_stream_error() -> Result<()> { - // An algorithm whose task panics must surface an `Err` step to the stream - // consumer, not abort the process from an unobserved detached task. + // An algorithm panic must surface as an `Err` step to the stream + // consumer, not abort the embedding host. struct Panicky; #[async_trait] @@ -1364,7 +1391,11 @@ mod tests { while let Some(step) = stream.next().await { match step { Err(err) => { - assert!(matches!(err, LibsyError::AlgorithmTask { .. })); + assert!(matches!( + err, + LibsyError::AlgorithmError { message } + if message == "algorithm task panicked" + )); saw_error = true; } Ok(_) => return Err(test_error("expected the panic to surface as an error step")), @@ -1403,7 +1434,11 @@ mod tests { "expected run to surface the algorithm panic as an error", )), Err(err) => { - assert!(matches!(err, LibsyError::AlgorithmTask { .. })); + assert!(matches!( + err, + LibsyError::AlgorithmError { message } + if message == "algorithm task panicked" + )); Ok(()) } } diff --git a/crates/libsy/src/core/driver.rs b/crates/libsy/src/core/driver.rs index c9e233e2c..04eb9e3eb 100644 --- a/crates/libsy/src/core/driver.rs +++ b/crates/libsy/src/core/driver.rs @@ -42,20 +42,21 @@ //! There is no explicit stop method — the consumer terminates by **dropping the //! stream** (and any [`DriverRequest`] it is holding). The producer's next publish //! (`fulfill_request`/`info`/`done`/`fail`) then resolves to `Err`, and a producer -//! awaiting a response sees `Err` once the promise it handed out is dropped. Either -//! way the algorithm unwinds cooperatively at its next driver interaction. Because the -//! producer runs on a task the driver does not own, hard cancellation (e.g. mid-compute -//! that never touches the driver) is the caller's concern — abort the producer task. +//! awaiting a response sees `Err` once the promise it handed out is dropped. In +//! [`Algorithm::run_stream`](super::algorithm::Algorithm::run_stream), the producer +//! future is owned by the returned stream, so dropping the stream also cancels an +//! algorithm that is between driver interactions. use std::{any::Any, sync::Arc}; use crate::{DriverError, LibsyError, Result}; use parking_lot::Mutex; -use futures::{Stream, StreamExt}; +use futures::{future::Either, pin_mut, Stream, StreamExt}; +use futures_timer::Delay; use switchyard_protocol::Context; use tokio::sync::{mpsc, oneshot}; -use tokio::time::{timeout, Duration}; +use tokio::time::Duration; use tokio_stream::wrappers::ReceiverStream; type BoxAny = Box; @@ -175,12 +176,17 @@ impl TypeErasedDriver { // Outer error: the promise was dropped without a response. Inner error: the // consumer fulfilled it with an explicit `Err` — propagate it as-is. - let response = timeout(FULFILL_REQUEST_TIMEOUT, rx) - .await - .map_err(|_| DriverError::ResponseTimedOut { - timeout: FULFILL_REQUEST_TIMEOUT, - })? - .map_err(|_| DriverError::ResponseDropped)??; + let timeout = Delay::new(FULFILL_REQUEST_TIMEOUT); + pin_mut!(rx, timeout); + let response = match futures::future::select(rx, timeout).await { + Either::Left((response, _)) => response.map_err(|_| DriverError::ResponseDropped)??, + Either::Right(((), _)) => { + return Err(DriverError::ResponseTimedOut { + timeout: FULFILL_REQUEST_TIMEOUT, + } + .into()); + } + }; response.downcast::().map(|boxed| *boxed).map_err(|_| { DriverError::TypeMismatch { expected: std::any::type_name::(), diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index f320c9163..fea81319e 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib", "rlib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "5126ff511aa6d655f2075a37973729d30b5dc33c" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "6a55a82d8d2b30f384027a1d4d086552d76a341b" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 6e7ad2d85..d4a89bd32 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -24,6 +24,12 @@ An active `run_stream` policy therefore does not occupy a blocking worker for the lifetime of the request. Provider results, stream capacity, and cancellation wake the policy through the generic native task contract. +The plugin future itself remains executor-neutral. Relay and a native plugin +can link distinct copies of Tokio, so polling from Relay's runtime does not +enter plugin-local Tokio state across the dynamic-library boundary. libsy's +`run_stream` and driver response timeout are therefore poll-driven rather than +calling `tokio::spawn` or `tokio::time::timeout`. + The crate is a source/build unit and is not published to crates.io. Operators install a release bundle containing the compiled shared library, materialized `relay-plugin.toml`, `config.schema.json`, licensing files, and checksum. From 61a4c3fd3ce4f0f6399e247f0002e706b926d43b Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 10:56:40 -0600 Subject: [PATCH 25/51] docs(libsy): describe poll-driven run streams Signed-off-by: Bryan Bednarski --- crates/libsy/src/error.rs | 2 +- crates/libsy/src/lib.rs | 4 ++-- crates/libsy/src/observability.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/libsy/src/error.rs b/crates/libsy/src/error.rs index 834ceff33..15eac3989 100644 --- a/crates/libsy/src/error.rs +++ b/crates/libsy/src/error.rs @@ -43,7 +43,7 @@ pub enum LibsyError { #[error(transparent)] Driver(#[from] DriverError), - /// The spawned algorithm task failed before returning normally. + /// A host-spawned algorithm task failed before returning normally. #[error("algorithm task failed: {source}")] AlgorithmTask { /// Tokio task failure, including panic and unexpected cancellation details. diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index bff81c6d7..6f4a53168 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -17,8 +17,8 @@ //! and makes as many model calls as it needs — via [`Driver::call_llm_target`], which look //! like ordinary calls — publishes its [`Decision`](switchyard_protocol::Decision)s with [`Driver::info`], and //! returns the final [`Response`](switchyard_protocol::Response). The provided -//! [`run_stream`](Algorithm::run_stream) drives that on its own task and hands -//! back a stream of [`Step`]s; [`run`](Algorithm::run) runs +//! [`run_stream`](Algorithm::run_stream) drives that as its returned stream is +//! polled and hands back [`Step`]s; [`run`](Algorithm::run) runs //! it to completion with the targets' default clients. //! - An [`LlmTarget`] names a routing target by its [`semantic_name`](LlmTarget::semantic_name). //! Every call is *offloaded* to the request's stream as a [`Step::CallLlm`]; the diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs index ceeed9240..7a9c98080 100644 --- a/crates/libsy/src/observability.rs +++ b/crates/libsy/src/observability.rs @@ -13,7 +13,7 @@ //! OpenTelemetry ecosystem bridges with `tracing-opentelemetry` / //! `opentelemetry-appender-tracing`), so the host's subscriber decides where //! they go. Method spans use `#[tracing::instrument]`; the `libsy.run` span is -//! attached to the spawned run task with [`tracing::Instrument`]. Neither holds +//! attached to the poll-driven run future with [`tracing::Instrument`]. Neither holds //! a [`Span::enter`] guard across an `.await` — a suspended task would leave //! the span entered on its executor thread, mis-parenting every span other //! tasks create there (see the `tracing` docs on spans in asynchronous code). From 8edc9b571eb9c21e3e6e4becf7d968725aad0820 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 14:11:47 -0600 Subject: [PATCH 26/51] refactor(plugin): simplify Relay run stream driver Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 +- .../switchyard-nemo-relay-plugin/Cargo.toml | 2 +- .../src/config.rs | 149 +++- .../src/runtime.rs | 671 +++++++----------- 4 files changed, 403 insertions(+), 423 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e4d31eb6c..255966534 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=6a55a82d8d2b30f384027a1d4d086552d76a341b#6a55a82d8d2b30f384027a1d4d086552d76a341b" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=648fb185d20cb8286a16cc2a7759a4039ceade21#648fb185d20cb8286a16cc2a7759a4039ceade21" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=6a55a82d8d2b30f384027a1d4d086552d76a341b#6a55a82d8d2b30f384027a1d4d086552d76a341b" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=648fb185d20cb8286a16cc2a7759a4039ceade21#648fb185d20cb8286a16cc2a7759a4039ceade21" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index fea81319e..74b2bcd03 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib", "rlib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "6a55a82d8d2b30f384027a1d4d086552d76a341b" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "648fb185d20cb8286a16cc2a7759a4039ceade21" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index 828b56c81..a8a397625 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -5,13 +5,13 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use http::header::{HeaderName, HeaderValue}; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use switchyard_libsy::{ Algorithm, LlmTarget, LlmTargetSet, LlmTaskClassifier, Random, TaskClassifierConfig, }; use switchyard_protocol::WireFormat; -#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd)] #[serde(rename_all = "snake_case")] pub enum WireProtocol { OpenaiChat, @@ -66,7 +66,7 @@ impl WireProtocol { } } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Deserialize)] pub struct TargetBinding { pub model: String, pub protocol: WireProtocol, @@ -97,13 +97,12 @@ impl TargetBinding { format!("{base}{endpoint}") } - pub fn resolved_headers(&self) -> Result, String> { - let mut headers = BTreeMap::new(); + fn validate_headers(&self) -> Result<(), String> { for (name, value) in &self.headers { validate_header(name, value)?; - headers.insert(name.clone(), value.clone()); } for (name, variable) in &self.header_env { + validate_header_name(name)?; if self .headers .keys() @@ -113,16 +112,47 @@ impl TargetBinding { "target header {name:?} cannot appear in both headers and header_env" )); } - let value = std::env::var(variable) + if variable.is_empty() { + return Err(format!( + "environment variable name for target header {name:?} must not be empty" + )); + } + } + Ok(()) + } + + fn into_prepared(self) -> Result { + let dispatch_url = self.dispatch_url(); + let mut headers = self.headers; + for (name, variable) in self.header_env { + let value = std::env::var(&variable) .map_err(|_| format!("environment variable {variable:?} is not set"))?; - validate_header(name, &value)?; - headers.insert(name.clone(), value); + validate_header(&name, &value)?; + headers.insert(name, value); } - Ok(headers) + Ok(PreparedTargetBinding { + model: self.model, + protocol: self.protocol, + dispatch_url, + headers, + }) } } -#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub struct PreparedTargetBinding { + pub model: String, + pub protocol: WireProtocol, + dispatch_url: String, + pub headers: BTreeMap, +} + +impl PreparedTargetBinding { + pub fn dispatch_url(&self) -> &str { + &self.dispatch_url + } +} + +#[derive(Default, Deserialize)] pub struct ProtocolDefaults { #[serde(default)] pub openai_chat: String, @@ -142,7 +172,7 @@ impl ProtocolDefaults { } } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum AlgorithmConfig { Random { @@ -171,7 +201,7 @@ impl Default for AlgorithmConfig { } } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Deserialize)] pub struct SwitchyardConfig { #[serde(default = "default_version")] pub version: u32, @@ -188,8 +218,21 @@ pub struct SwitchyardConfig { pub enabled_inbound_profiles: BTreeSet, } +pub(crate) struct PreparedConfig { + pub max_retries: u32, + pub algorithm: Arc, + pub targets: BTreeMap, + pub default_targets: ProtocolDefaults, + pub enabled_inbound_profiles: BTreeSet, +} + impl SwitchyardConfig { pub fn validate(&self) -> Result<(), String> { + self.validate_structure()?; + self.build_algorithm().map(drop) + } + + fn validate_structure(&self) -> Result<(), String> { if self.version != 2 { return Err(format!( "unsupported Switchyard config version {}; version 1 used switchyard-server; migrate to version = 2", @@ -217,7 +260,7 @@ impl SwitchyardConfig { "target {name:?} weight must be finite and nonnegative" )); } - target.resolved_headers()?; + target.validate_headers()?; } for protocol in &self.enabled_inbound_profiles { let fallback = self.default_targets.target(*protocol); @@ -232,10 +275,27 @@ impl SwitchyardConfig { )); } } - self.build_algorithm().map(|_| ()) + Ok(()) + } + + pub(crate) fn prepare(self) -> Result { + self.validate_structure()?; + let algorithm = self.build_algorithm()?; + let targets = self + .targets + .into_iter() + .map(|(name, target)| target.into_prepared().map(|prepared| (name, prepared))) + .collect::>()?; + Ok(PreparedConfig { + max_retries: self.max_retries, + algorithm, + targets, + default_targets: self.default_targets, + enabled_inbound_profiles: self.enabled_inbound_profiles, + }) } - pub fn build_algorithm(&self) -> Result, String> { + fn build_algorithm(&self) -> Result, String> { let target = |name: &str| { self.targets .contains_key(name) @@ -289,9 +349,14 @@ impl SwitchyardConfig { } } -fn validate_header(name: &str, value: &str) -> Result<(), String> { +fn validate_header_name(name: &str) -> Result<(), String> { HeaderName::from_bytes(name.as_bytes()) - .map_err(|error| format!("invalid target header name {name:?}: {error}"))?; + .map(|_| ()) + .map_err(|error| format!("invalid target header name {name:?}: {error}")) +} + +fn validate_header(name: &str, value: &str) -> Result<(), String> { + validate_header_name(name)?; HeaderValue::from_str(value) .map_err(|error| format!("invalid target header value for {name:?}: {error}"))?; Ok(()) @@ -366,11 +431,11 @@ mod tests { fn version_two_random_configuration_builds_without_a_service() { let config = config(); config.validate().unwrap(); - assert_eq!(config.build_algorithm().unwrap().name(), "random"); assert_eq!( config.targets["chat"].dispatch_url(), "https://provider.example/v1/chat/completions" ); + assert_eq!(config.prepare().unwrap().algorithm.name(), "random"); } #[test] @@ -397,8 +462,52 @@ mod tests { }; config.validate().unwrap(); assert_eq!( - config.build_algorithm().unwrap().name(), + config.prepare().unwrap().algorithm.name(), "llm_task_classifier" ); } + + #[test] + fn validation_does_not_resolve_environment_backed_headers() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().header_env = BTreeMap::from([( + "authorization".into(), + "SWITCHYARD_TEST_ENVIRONMENT_VARIABLE_THAT_IS_NOT_SET".into(), + )]); + + config.validate().unwrap(); + let error = config + .prepare() + .err() + .expect("preparation must resolve headers"); + assert!(error.contains("SWITCHYARD_TEST_ENVIRONMENT_VARIABLE_THAT_IS_NOT_SET")); + } + + #[test] + fn static_validation_preserves_algorithm_constructor_checks() { + let mut random = config(); + for target in random.targets.values_mut() { + target.weight = 0.0; + } + assert!(random + .validate() + .unwrap_err() + .contains("at least one weight must be positive")); + + let mut classifier = config(); + classifier.algorithm = AlgorithmConfig::LlmClassifier { + classifier_target: "chat".into(), + weak_target: "responses".into(), + strong_target: "anthropic".into(), + base_threshold: 1.1, + min_confidence: 0.0, + capability_elevated_floor: None, + session_affinity: false, + message_hash_fallback: false, + }; + assert!(classifier + .validate() + .unwrap_err() + .contains("base_threshold must be between 0 and 1")); + } } diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 85fd4653f..a430e306d 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -1,54 +1,46 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::pin::Pin; -use std::sync::{Arc, Mutex}; -use std::task::{Context as TaskContext, Poll}; +use std::sync::Arc; use futures_util::{Stream, StreamExt}; use nemo_relay_plugin::{ Json, LlmContinuationFailureV2, LlmContinuationInvocationV2, LlmContinuationTargetV2, - LlmContinuationV2, LlmJsonAsyncStreamV2, LlmNonHttpFailureKindV2, LlmProviderStreamV2, - LlmRequest as RelayRequest, LlmStreamContinuationV2, LlmStreamExecutionOutcomeV2, - PluginRuntime, + LlmContinuationV2, LlmJsonAsyncStreamV2, LlmNonHttpFailureKindV2, LlmRequest as RelayRequest, + LlmStreamContinuationV2, LlmStreamExecutionOutcomeV2, PluginRuntime, }; use serde_json::{json, Map}; use switchyard_libsy::{Algorithm, CallLlmRequest, LibsyError, Step}; use switchyard_protocol::{ - Context, LlmClientError, LlmResponse, LlmResponseChunk, LlmResponseStream, - LlmResponseStreamEvent, Metadata, Request, Response, + Context, LlmClientError, LlmRequest as SwitchyardLlmRequest, LlmResponse, LlmResponseChunk, + LlmResponseStream, LlmResponseStreamEvent, Metadata, Request, Response, }; use switchyard_translation::{StreamTranslationState, TranslationEngine}; -use crate::config::{SwitchyardConfig, TargetBinding, WireProtocol}; +use crate::config::{PreparedTargetBinding, ProtocolDefaults, SwitchyardConfig, WireProtocol}; use crate::translation; pub struct SwitchyardRuntime { - config: SwitchyardConfig, + max_retries: u32, algorithm: Arc, - target_headers: BTreeMap>, + targets: BTreeMap, + default_targets: ProtocolDefaults, + enabled_inbound_profiles: BTreeSet, translation: Arc, relay: PluginRuntime, } impl SwitchyardRuntime { pub fn new(config: SwitchyardConfig, relay: PluginRuntime) -> Result { - config.validate()?; - let algorithm = config.build_algorithm()?; - let target_headers = config - .targets - .iter() - .map(|(name, target)| { - target - .resolved_headers() - .map(|headers| (name.clone(), headers)) - }) - .collect::>()?; + let prepared = config.prepare()?; Ok(Self { - config, - algorithm, - target_headers, + max_retries: prepared.max_retries, + algorithm: prepared.algorithm, + targets: prepared.targets, + default_targets: prepared.default_targets, + enabled_inbound_profiles: prepared.enabled_inbound_profiles, translation: Arc::new(TranslationEngine::default()), relay, }) @@ -63,25 +55,20 @@ impl SwitchyardRuntime { let Some(inbound) = WireProtocol::from_call(&name) else { return continuation.call_passthrough(request).await; }; - if !self.config.enabled_inbound_profiles.contains(&inbound) { + if !self.enabled_inbound_profiles.contains(&inbound) { return continuation.call_passthrough(request).await; } let libsy_request = self.libsy_request(inbound, &request, false)?; - let metadata = identity_metadata(&request); - let max_attempts = self.config.max_retries.saturating_add(1); + let metadata = identity_metadata(libsy_request.metadata.as_ref()); + let max_attempts = self.max_retries + 1; for attempt in 1..=max_attempts { self.mark( "switchyard.routing.requested", json!({"algorithm": self.algorithm.name(), "attempt": attempt}), - metadata.clone(), + &metadata, ); match self - .drive_buffered( - libsy_request.clone(), - &continuation, - attempt, - metadata.clone(), - ) + .drive_buffered(libsy_request.clone(), &continuation, attempt, &metadata) .await { Ok(response) => { @@ -94,21 +81,21 @@ impl SwitchyardRuntime { } }; } - Err(failure) if failure.retryable() && attempt < max_attempts => { + Err(failure) if libsy_error_retryable(&failure) && attempt < max_attempts => { self.mark( "switchyard.routing.retry", failure_mark_data(attempt, &failure), - metadata.clone(), + &metadata, ); } Err(failure) => { self.mark( "switchyard.routing.error", failure_mark_data(attempt, &failure), - metadata.clone(), + &metadata, ); return self - .fallback_buffered(inbound, libsy_request, &continuation, metadata) + .fallback_buffered(inbound, libsy_request, &continuation, &metadata) .await; } } @@ -121,14 +108,12 @@ impl SwitchyardRuntime { request: Request, continuation: &LlmContinuationV2, attempt: u32, - mark_metadata: Json, - ) -> Result { - let mut context = Context::default(); - context - .values - .insert("relay.routing_attempt".into(), attempt.to_string()); - let mut steps = self.algorithm.clone().run_stream(context, request, None); - let provider_error = Arc::new(Mutex::new(None)); + mark_metadata: &Json, + ) -> Result { + let mut steps = self + .algorithm + .clone() + .run_stream(Context::default(), request, None); while let Some(step) = steps.next().await { match step { Ok(Step::Decision(decision)) => { @@ -142,55 +127,40 @@ impl SwitchyardRuntime { "routing_tier": decision.routing_tier(), "is_routed_call": decision.is_routed_call(), }), - mark_metadata.clone(), + mark_metadata, ); } Ok(Step::CallLlm(call)) => { - self.serve_buffered_call(*call, continuation, Arc::clone(&provider_error)) - .await - .map_err(|error| RunFailure::new(error, &provider_error))?; + self.serve_buffered_call(*call, continuation).await?; } Ok(Step::ReturnToAgent(response)) => return Ok(*response), - Err(error) => return Err(RunFailure::new(error, &provider_error)), + Err(error) => return Err(error), } } - Err(RunFailure::new( - LibsyError::MissingFinalResponse, - &provider_error, - )) + Err(LibsyError::MissingFinalResponse) } async fn serve_buffered_call( &self, call: CallLlmRequest, continuation: &LlmContinuationV2, - provider_error: Arc>>, ) -> switchyard_libsy::Result<()> { - let routed = call.get_routed().clone(); - let target_name = routed.decision.selected_model().to_string(); + let target_name = call.get_decision().selected_model().to_string(); + let request = call.get_request().llm_request.clone(); let result = async { let target = self.target(&target_name)?; - let request = self.dispatch_request(&target_name, target, routed.request, false)?; - match continuation.call(request).await { - Ok(response) => { - let response = - translation::decode_response(&self.translation, target.protocol, &response) - .map_err(LlmClientError::ResponseTranslation)?; - Ok(Response { - llm_response: LlmResponse::Agg(response), - metadata: Some(Metadata { - wire_format: Some(target.protocol.wire_format()), - ..Metadata::default() - }), - }) - } - Err(error) => { - if let Ok(mut stored) = provider_error.lock() { - *stored = Some(error.clone()); - } - Err(client_error(error)) - } - } + let request = self.dispatch_request(target, request, false)?; + let response = continuation.call(request).await.map_err(client_error)?; + let response = + translation::decode_response(&self.translation, target.protocol, &response) + .map_err(LlmClientError::ResponseTranslation)?; + Ok(Response { + llm_response: LlmResponse::Agg(response), + metadata: Some(Metadata { + wire_format: Some(target.protocol.wire_format()), + ..Metadata::default() + }), + }) } .await .map_err(|source| LibsyError::client_call(target_name, source)); @@ -202,9 +172,9 @@ impl SwitchyardRuntime { inbound: WireProtocol, request: Request, continuation: &LlmContinuationV2, - metadata: Json, + metadata: &Json, ) -> Result { - let target_name = self.config.default_targets.target(inbound); + let target_name = self.default_targets.target(inbound); let target = self .target(target_name) .map_err(|error| error.to_string())?; @@ -214,7 +184,7 @@ impl SwitchyardRuntime { metadata, ); let dispatch = self - .dispatch_request(target_name, target, request, false) + .dispatch_request(target, request.llm_request, false) .map_err(|error| error.to_string())?; let response = continuation .call(dispatch) @@ -237,27 +207,21 @@ impl SwitchyardRuntime { metadata.wire_format = Some(inbound.wire_format()); Ok(Request { llm_request: request, - raw_request: Some(original.content.clone()), + raw_request: None, metadata: Some(metadata), }) } fn dispatch_request( &self, - target_name: &str, - target: &TargetBinding, - mut request: Request, + target: &PreparedTargetBinding, + mut request: SwitchyardLlmRequest, streaming: bool, ) -> Result { - request.llm_request.stream = streaming; - let headers = self - .target_headers - .get(target_name) - .cloned() - .unwrap_or_default(); - let mut request = - translation::encode_request(&self.translation, target.protocol, &request.llm_request) - .map_err(LlmClientError::RequestEncoding)?; + request.stream = streaming; + let headers = target.headers.clone(); + let mut request = translation::encode_request(&self.translation, target.protocol, &request) + .map_err(LlmClientError::RequestEncoding)?; let body = request.content.as_object_mut().ok_or_else(|| { LlmClientError::RequestEncoding("translated provider request is not an object".into()) })?; @@ -266,24 +230,22 @@ impl SwitchyardRuntime { Ok(LlmContinuationInvocationV2 { request, target: LlmContinuationTargetV2 { - method: "POST".into(), - url: target.dispatch_url(), + url: target.dispatch_url().to_string(), headers, }, }) } - fn target(&self, name: &str) -> Result<&TargetBinding, LlmClientError> { - self.config - .targets + fn target(&self, name: &str) -> Result<&PreparedTargetBinding, LlmClientError> { + self.targets .get(name) .ok_or_else(|| LlmClientError::Configuration { message: format!("libsy selected unknown target {name:?}"), }) } - fn mark(&self, name: &str, data: Json, metadata: Json) { - if let Err(error) = self.relay.emit_mark(name, Some(&data), Some(&metadata)) { + fn mark(&self, name: &str, data: Json, metadata: &Json) { + if let Err(error) = self.relay.emit_mark(name, Some(&data), Some(metadata)) { eprintln!("Switchyard could not emit routing mark {name:?}: {error}"); } } @@ -297,11 +259,11 @@ impl SwitchyardRuntime { let Some(inbound) = WireProtocol::from_call(&name) else { return Ok(LlmStreamExecutionOutcomeV2::Passthrough(request)); }; - if !self.config.enabled_inbound_profiles.contains(&inbound) { + if !self.enabled_inbound_profiles.contains(&inbound) { return Ok(LlmStreamExecutionOutcomeV2::Passthrough(request)); } let libsy_request = self.libsy_request(inbound, &request, true)?; - let metadata = identity_metadata(&request); + let metadata = identity_metadata(libsy_request.metadata.as_ref()); let stream = self.routed_stream(inbound, libsy_request, continuation, metadata); Ok(LlmStreamExecutionOutcomeV2::Stream(stream)) } @@ -314,24 +276,24 @@ impl SwitchyardRuntime { metadata: Json, ) -> LlmJsonAsyncStreamV2 { Box::pin(async_stream::try_stream! { - let max_attempts = self.config.max_retries.saturating_add(1); + let max_attempts = self.max_retries + 1; 'attempts: for attempt in 1..=max_attempts { self.mark( "switchyard.routing.requested", json!({"algorithm": self.algorithm.name(), "attempt": attempt}), - metadata.clone(), + &metadata, ); - let run = match self + let response = match self .drive_stream( request.clone(), &continuation, attempt, - metadata.clone(), + &metadata, ) .await { Ok(response) => response, - Err(failure) if failure.retryable() && attempt < max_attempts => { + Err(failure) if libsy_error_retryable(&failure) && attempt < max_attempts => { self.emit_stream_retry(attempt, &failure, &metadata); continue; } @@ -342,62 +304,58 @@ impl SwitchyardRuntime { inbound, request.clone(), &continuation, - metadata.clone(), + &metadata, ) .await?; while let Some(item) = fallback.next().await { yield item.map_err(|failure| { format!( "trusted fallback stream failed: {}", - failure.failure.error + failure.error ) })?; } return; } }; - let mut returned = Self::returned_stream( - run.response, - inbound, - Arc::clone(&self.translation), - Arc::clone(&run.provider_error), - ); + let mut returned = + Self::returned_stream(response, inbound, Arc::clone(&self.translation)); while let Some(item) = returned.next().await { match item { Ok(event) => yield event, Err(failure) if !failure.committed - && failure.failure.retryable() + && libsy_error_retryable(&failure.error) && attempt < max_attempts => { - self.emit_stream_retry(attempt, &failure.failure, &metadata); + self.emit_stream_retry(attempt, &failure.error, &metadata); continue 'attempts; } Err(failure) if !failure.committed => { - self.emit_stream_error(attempt, &failure.failure, &metadata); + self.emit_stream_error(attempt, &failure.error, &metadata); let mut fallback = self .fallback_stream( inbound, request.clone(), &continuation, - metadata.clone(), + &metadata, ) .await?; while let Some(item) = fallback.next().await { yield item.map_err(|failure| { format!( "trusted fallback stream failed: {}", - failure.failure.error + failure.error ) })?; } return; } Err(failure) => { - self.emit_stream_error(attempt, &failure.failure, &metadata); + self.emit_stream_error(attempt, &failure.error, &metadata); Err(format!( "Switchyard stream failed after response commitment: {}", - failure.failure.error + failure.error ))?; } } @@ -413,14 +371,12 @@ impl SwitchyardRuntime { request: Request, continuation: &LlmStreamContinuationV2, attempt: u32, - mark_metadata: Json, - ) -> Result { - let mut context = Context::default(); - context - .values - .insert("relay.routing_attempt".into(), attempt.to_string()); - let mut steps = self.algorithm.clone().run_stream(context, request, None); - let provider_error = Arc::new(Mutex::new(None)); + mark_metadata: &Json, + ) -> Result { + let mut steps = self + .algorithm + .clone() + .run_stream(Context::default(), request, None); while let Some(step) = steps.next().await { match step { Ok(Step::Decision(decision)) => { @@ -434,44 +390,30 @@ impl SwitchyardRuntime { "routing_tier": decision.routing_tier(), "is_routed_call": decision.is_routed_call(), }), - mark_metadata.clone(), + mark_metadata, ); } Ok(Step::CallLlm(call)) => { - self.serve_stream_call(*call, continuation, Arc::clone(&provider_error)) - .await - .map_err(|error| RunFailure::new(error, &provider_error))?; - } - Ok(Step::ReturnToAgent(response)) => { - return Ok(StreamRun { - response: *response, - provider_error, - }); + self.serve_stream_call(*call, continuation).await?; } - Err(error) => return Err(RunFailure::new(error, &provider_error)), + Ok(Step::ReturnToAgent(response)) => return Ok(*response), + Err(error) => return Err(error), } } - Err(RunFailure::new( - LibsyError::MissingFinalResponse, - &provider_error, - )) + Err(LibsyError::MissingFinalResponse) } async fn serve_stream_call( &self, call: CallLlmRequest, continuation: &LlmStreamContinuationV2, - provider_error: Arc>>, ) -> switchyard_libsy::Result<()> { - let routed = call.get_routed().clone(); - let target_name = routed.decision.selected_model().to_string(); + let target_name = call.get_decision().selected_model().to_string(); + let request = call.get_request(); + let llm_request = request.llm_request.clone(); + let metadata = request.metadata.clone(); let result = self - .provider_stream_response( - &target_name, - routed.request, - continuation, - Arc::clone(&provider_error), - ) + .provider_stream_response(&target_name, llm_request, metadata, continuation) .await .map_err(|source| LibsyError::client_call(target_name, source)); call.respond(result) @@ -480,26 +422,19 @@ impl SwitchyardRuntime { async fn provider_stream_response( &self, target_name: &str, - request: Request, + request: SwitchyardLlmRequest, + metadata: Option, continuation: &LlmStreamContinuationV2, - provider_error: Arc>>, ) -> Result { let target = self.target(target_name)?; - let metadata = request.metadata.clone(); - let dispatch = self.dispatch_request(target_name, target, request, true)?; - let mut upstream = match continuation.open_stream(dispatch).await { - Ok(upstream) => upstream, - Err(error) => { - remember_provider_error(&provider_error, &error); - return Err(client_error(error)); - } - }; + let dispatch = self.dispatch_request(target, request, true)?; + let mut upstream = continuation + .open_stream(dispatch) + .await + .map_err(client_error)?; let first_raw = match upstream.next().await { Some(Ok(first)) => first, - Some(Err(error)) => { - remember_provider_error(&provider_error, &error); - return Err(client_error(error)); - } + Some(Err(error)) => return Err(client_error(error)), None => { return Err(LlmClientError::InvalidResponse { source: Box::new(std::io::Error::new( @@ -515,13 +450,14 @@ impl SwitchyardRuntime { ); let first = decode_provider_event(&self.translation, &mut state, target.protocol, first_raw)?; - let stream: LlmResponseStream = Box::pin(TranslatedProviderStream { - upstream, - first: Some(first), - protocol: target.protocol, - state, - translation: Arc::clone(&self.translation), - provider_error, + let protocol = target.protocol; + let translation = Arc::clone(&self.translation); + let stream: LlmResponseStream = Box::pin(async_stream::try_stream! { + yield first; + while let Some(item) = upstream.next().await { + let raw = item.map_err(client_error)?; + yield decode_provider_event(&translation, &mut state, protocol, raw)?; + } }); Ok(Response { llm_response: LlmResponse::Stream(stream), @@ -536,7 +472,6 @@ impl SwitchyardRuntime { response: Response, inbound: WireProtocol, translation: Arc, - provider_error: Arc>>, ) -> ReturnedJsonStream { Box::pin(async_stream::stream! { let source = match response @@ -550,7 +485,6 @@ impl SwitchyardRuntime { yield Err(StreamAttemptFailure::translation( "libsy returned a stream without a supported source wire format", false, - &provider_error, )); return; } @@ -559,7 +493,6 @@ impl SwitchyardRuntime { yield Err(StreamAttemptFailure::translation( "libsy returned a buffered response for a streaming request", false, - &provider_error, )); return; }; @@ -570,11 +503,7 @@ impl SwitchyardRuntime { let event = match item { Ok(event) => event, Err(error) => { - yield Err(StreamAttemptFailure::client( - error, - committed, - &provider_error, - )); + yield Err(StreamAttemptFailure::client(error, committed)); return; } }; @@ -589,7 +518,6 @@ impl SwitchyardRuntime { yield Err(StreamAttemptFailure::translation( &error, committed, - &provider_error, )); return; } @@ -610,7 +538,6 @@ impl SwitchyardRuntime { yield Err(StreamAttemptFailure::translation( &error, committed, - &provider_error, )); return; } @@ -624,7 +551,6 @@ impl SwitchyardRuntime { yield Err(StreamAttemptFailure::translation( "Switchyard produced an empty output stream", false, - &provider_error, )); } }) @@ -635,21 +561,20 @@ impl SwitchyardRuntime { inbound: WireProtocol, request: Request, continuation: &LlmStreamContinuationV2, - metadata: Json, + metadata: &Json, ) -> Result { - let target_name = self.config.default_targets.target(inbound).to_string(); + let target_name = self.default_targets.target(inbound); self.mark( "switchyard.routing.fallback", json!({"selected_target": target_name}), metadata, ); - let provider_error = Arc::new(Mutex::new(None)); let response = self .provider_stream_response( - &target_name, - request, + target_name, + request.llm_request, + request.metadata, continuation, - Arc::clone(&provider_error), ) .await .map_err(|error| format!("trusted fallback stream failed: {error}"))?; @@ -657,71 +582,28 @@ impl SwitchyardRuntime { response, inbound, Arc::clone(&self.translation), - provider_error, )) } - fn emit_stream_retry(&self, attempt: u32, failure: &RunFailure, metadata: &Json) { + fn emit_stream_retry(&self, attempt: u32, failure: &LibsyError, metadata: &Json) { self.mark( "switchyard.routing.retry", failure_mark_data(attempt, failure), - metadata.clone(), + metadata, ); } - fn emit_stream_error(&self, attempt: u32, failure: &RunFailure, metadata: &Json) { + fn emit_stream_error(&self, attempt: u32, failure: &LibsyError, metadata: &Json) { self.mark( "switchyard.routing.error", failure_mark_data(attempt, failure), - metadata.clone(), + metadata, ); } } -struct StreamRun { - response: Response, - provider_error: Arc>>, -} - type ReturnedJsonStream = Pin> + Send>>; -struct TranslatedProviderStream { - upstream: LlmProviderStreamV2, - first: Option, - protocol: WireProtocol, - state: StreamTranslationState, - translation: Arc, - provider_error: Arc>>, -} - -impl Stream for TranslatedProviderStream { - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { - if let Some(first) = self.first.take() { - return Poll::Ready(Some(Ok(first))); - } - match Pin::new(&mut self.upstream).poll_next(cx) { - Poll::Pending => Poll::Pending, - Poll::Ready(Some(Ok(raw))) => { - let translation = Arc::clone(&self.translation); - let protocol = self.protocol; - Poll::Ready(Some(decode_provider_event( - &translation, - &mut self.state, - protocol, - raw, - ))) - } - Poll::Ready(Some(Err(error))) => { - remember_provider_error(&self.provider_error, &error); - Poll::Ready(Some(Err(client_error(error)))) - } - Poll::Ready(None) => Poll::Ready(None), - } - } -} - fn decode_provider_event( translation: &TranslationEngine, state: &mut StreamTranslationState, @@ -747,143 +629,110 @@ fn decode_provider_event( Ok(event) } -fn remember_provider_error( - provider_error: &Arc>>, - error: &LlmContinuationFailureV2, -) { - if let Ok(mut stored) = provider_error.lock() { - *stored = Some(error.clone()); - } -} - struct StreamAttemptFailure { - failure: RunFailure, + error: LibsyError, committed: bool, } impl StreamAttemptFailure { - fn client( - error: LlmClientError, - committed: bool, - provider_error: &Arc>>, - ) -> Self { + fn client(error: LlmClientError, committed: bool) -> Self { Self { - failure: RunFailure::new( - LibsyError::client_call("return_to_agent", error), - provider_error, - ), + error: LibsyError::client_call("return_to_agent", error), committed, } } - fn translation( - error: &str, - committed: bool, - provider_error: &Arc>>, - ) -> Self { + fn translation(error: &str, committed: bool) -> Self { Self::client( LlmClientError::ResponseTranslation(error.to_string()), committed, - provider_error, ) } } -struct RunFailure { - error: LibsyError, - provider_error: Option, -} - -impl RunFailure { - fn new( - error: LibsyError, - provider_error: &Arc>>, - ) -> Self { - Self { - error, - provider_error: provider_error.lock().ok().and_then(|error| error.clone()), - } - } - - fn retryable(&self) -> bool { - self.provider_error - .as_ref() - .is_some_and(should_retry_provider_failure) - } -} - -fn should_retry_provider_failure(failure: &LlmContinuationFailureV2) -> bool { - match failure { - LlmContinuationFailureV2::Http { failure } => { - matches!(failure.status, 408 | 425 | 429 | 500 | 502 | 503 | 504) +fn libsy_error_retryable(error: &LibsyError) -> bool { + matches!( + error, + LibsyError::ClientCall { + source: LlmClientError::UpstreamHttp { status, .. }, + .. + } if matches!(*status, 408 | 425 | 429 | 500 | 502 | 503 | 504) + ) || matches!( + error, + LibsyError::ClientCall { + source: LlmClientError::Transport { .. } | LlmClientError::Timeout { .. }, + .. } - LlmContinuationFailureV2::NonHttp { failure } => matches!( - failure.kind, - LlmNonHttpFailureKindV2::Transport | LlmNonHttpFailureKindV2::Timeout - ), - } + ) } fn client_error(error: LlmContinuationFailureV2) -> LlmClientError { match error { - LlmContinuationFailureV2::Http { failure } => LlmClientError::UpstreamHttp { - status: failure.status, - body: failure.body, - }, - LlmContinuationFailureV2::NonHttp { failure } => match failure.kind { + LlmContinuationFailureV2::Http { status, body, .. } => { + LlmClientError::UpstreamHttp { status, body } + } + LlmContinuationFailureV2::NonHttp { kind, message } => match kind { LlmNonHttpFailureKindV2::Transport => LlmClientError::Transport { - source: Box::new(std::io::Error::other(failure.message)), + source: Box::new(std::io::Error::other(message)), }, LlmNonHttpFailureKindV2::Timeout => LlmClientError::Timeout { - source: Box::new(std::io::Error::new( - std::io::ErrorKind::TimedOut, - failure.message, - )), + source: Box::new(std::io::Error::new(std::io::ErrorKind::TimedOut, message)), }, LlmNonHttpFailureKindV2::InvalidRequest | LlmNonHttpFailureKindV2::Guardrail => { - LlmClientError::InvalidRequest { - message: failure.message, - } + LlmClientError::InvalidRequest { message } } LlmNonHttpFailureKindV2::Cancelled | LlmNonHttpFailureKindV2::Internal => { - LlmClientError::General(failure.message) + LlmClientError::General(message) } }, } } -fn failure_mark_data(attempt: u32, failure: &RunFailure) -> Json { +fn failure_mark_data(attempt: u32, failure: &LibsyError) -> Json { let mut data = Map::from_iter([ ("attempt".into(), Json::from(attempt)), - ("retryable".into(), Json::from(failure.retryable())), + ( + "retryable".into(), + Json::from(libsy_error_retryable(failure)), + ), ]); - match &failure.provider_error { - Some(LlmContinuationFailureV2::Http { failure }) => { + match failure { + LibsyError::ClientCall { + source: LlmClientError::UpstreamHttp { status, .. }, + .. + } => { data.insert("failure_kind".into(), Json::from("http")); - data.insert("http_status".into(), Json::from(failure.status)); + data.insert("http_status".into(), Json::from(*status)); } - Some(LlmContinuationFailureV2::NonHttp { failure }) => { + LibsyError::ClientCall { source, .. } => { data.insert("failure_kind".into(), Json::from("non_http")); data.insert( "non_http_kind".into(), - Json::from(non_http_failure_label(failure.kind)), + Json::from(client_error_label(source)), ); } - None => { + _ => { data.insert("failure_kind".into(), Json::from("algorithm")); } } Json::Object(data) } -const fn non_http_failure_label(kind: LlmNonHttpFailureKindV2) -> &'static str { - match kind { - LlmNonHttpFailureKindV2::Transport => "transport", - LlmNonHttpFailureKindV2::Timeout => "timeout", - LlmNonHttpFailureKindV2::Cancelled => "cancelled", - LlmNonHttpFailureKindV2::InvalidRequest => "invalid_request", - LlmNonHttpFailureKindV2::Guardrail => "guardrail", - LlmNonHttpFailureKindV2::Internal => "internal", +fn client_error_label(error: &LlmClientError) -> &'static str { + match error { + LlmClientError::InvalidRequest { .. } => "invalid_request", + LlmClientError::RequestTranslation(_) => "request_translation", + LlmClientError::RequestEncoding(_) => "request_encoding", + LlmClientError::ResponseTranslation(_) => "response_translation", + LlmClientError::Configuration { .. } => "configuration", + LlmClientError::Transport { .. } => "transport", + LlmClientError::Timeout { .. } => "timeout", + LlmClientError::ContextWindowExceeded { .. } => "context_window_exceeded", + LlmClientError::UpstreamHttp { .. } => "http", + LlmClientError::InvalidResponse { .. } => "invalid_response", + LlmClientError::Ffi { .. } => "ffi", + LlmClientError::General(_) => "general", + _ => "unknown", } } @@ -894,13 +743,12 @@ fn string_headers(headers: &Map) -> BTreeMap { .collect() } -fn identity_metadata(request: &RelayRequest) -> Json { - let metadata = Metadata::from_headers(&string_headers(&request.headers)); +fn identity_metadata(metadata: Option<&Metadata>) -> Json { json!({ - "session_id": metadata.session_id, - "agent_id": metadata.agent_id, - "turn_id": metadata.turn_id, - "request_id": metadata.correlation_id, + "session_id": metadata.and_then(|value| value.session_id.as_deref()), + "agent_id": metadata.and_then(|value| value.agent_id.as_deref()), + "turn_id": metadata.and_then(|value| value.turn_id.as_deref()), + "request_id": metadata.and_then(|value| value.correlation_id.as_deref()), }) } @@ -908,79 +756,78 @@ fn identity_metadata(request: &RelayRequest) -> Json { mod tests { use super::*; use futures_util::FutureExt; - use nemo_relay_plugin::{LlmHttpFailureV2, LlmNonHttpFailureV2}; #[test] fn http_failures_keep_status_semantics_without_provider_classification() { let error = client_error(LlmContinuationFailureV2::Http { - failure: LlmHttpFailureV2 { - status: 400, - body: "context length exceeded".into(), - headers: BTreeMap::new(), - }, + status: 400, + body: "context length exceeded".into(), + headers: BTreeMap::new(), }); assert!(matches!( error, LlmClientError::UpstreamHttp { status: 400, .. } )); + + let error = client_error(LlmContinuationFailureV2::NonHttp { + kind: LlmNonHttpFailureKindV2::Guardrail, + message: "blocked".into(), + }); + assert!(matches!(error, LlmClientError::InvalidRequest { .. })); } #[test] - fn switchyard_retry_policy_uses_http_status_and_non_http_kind() { + fn switchyard_retry_policy_uses_libsy_client_error() { for status in [408, 425, 429, 500, 502, 503, 504] { - let failure = LlmContinuationFailureV2::Http { - failure: LlmHttpFailureV2 { + let failure = LibsyError::client_call( + "provider", + LlmClientError::UpstreamHttp { status, body: String::new(), - headers: BTreeMap::new(), }, - }; - assert!(should_retry_provider_failure(&failure), "status={status}"); + ); + assert!(libsy_error_retryable(&failure), "status={status}"); } for status in [400, 401, 404, 409, 422, 501] { - let failure = LlmContinuationFailureV2::Http { - failure: LlmHttpFailureV2 { + let failure = LibsyError::client_call( + "provider", + LlmClientError::UpstreamHttp { status, body: String::new(), - headers: BTreeMap::new(), }, - }; - assert!(!should_retry_provider_failure(&failure), "status={status}"); + ); + assert!(!libsy_error_retryable(&failure), "status={status}"); } - for (kind, expected) in [ - (LlmNonHttpFailureKindV2::Transport, true), - (LlmNonHttpFailureKindV2::Timeout, true), - (LlmNonHttpFailureKindV2::Cancelled, false), - (LlmNonHttpFailureKindV2::InvalidRequest, false), - (LlmNonHttpFailureKindV2::Guardrail, false), - (LlmNonHttpFailureKindV2::Internal, false), + for source in [ + LlmClientError::Transport { + source: Box::new(std::io::Error::other("transport")), + }, + LlmClientError::Timeout { + source: Box::new(std::io::Error::new(std::io::ErrorKind::TimedOut, "timeout")), + }, ] { - let failure = LlmContinuationFailureV2::NonHttp { - failure: LlmNonHttpFailureV2 { - kind, - message: String::new(), - }, - }; - assert_eq!( - should_retry_provider_failure(&failure), - expected, - "{kind:?}" - ); + assert!(libsy_error_retryable(&LibsyError::client_call( + "provider", source + ))); } + assert!(!libsy_error_retryable(&LibsyError::client_call( + "provider", + LlmClientError::InvalidRequest { + message: "invalid".into(), + }, + ))); + assert!(!libsy_error_retryable(&LibsyError::MissingFinalResponse)); } #[test] fn routing_failure_marks_exclude_provider_payloads() { - let failure = RunFailure { - error: LibsyError::MissingFinalResponse, - provider_error: Some(LlmContinuationFailureV2::Http { - failure: LlmHttpFailureV2 { - status: 429, - body: "provider body must not be recorded".into(), - headers: BTreeMap::from([("retry-after".into(), "secret".into())]), - }, - }), - }; + let failure = LibsyError::client_call( + "provider", + LlmClientError::UpstreamHttp { + status: 429, + body: "provider body must not be recorded".into(), + }, + ); assert_eq!( failure_mark_data(2, &failure), json!({ @@ -991,15 +838,15 @@ mod tests { }) ); - let failure = RunFailure { - error: LibsyError::MissingFinalResponse, - provider_error: Some(LlmContinuationFailureV2::NonHttp { - failure: LlmNonHttpFailureV2 { - kind: LlmNonHttpFailureKindV2::Timeout, - message: "timeout detail must not be recorded".into(), - }, - }), - }; + let failure = LibsyError::client_call( + "provider", + LlmClientError::Timeout { + source: Box::new(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "timeout detail must not be recorded", + )), + }, + ); assert_eq!( failure_mark_data(3, &failure), json!({ @@ -1009,6 +856,33 @@ mod tests { "non_http_kind": "timeout", }) ); + + let failure = LibsyError::client_call( + "provider", + client_error(LlmContinuationFailureV2::NonHttp { + kind: LlmNonHttpFailureKindV2::Guardrail, + message: "guardrail detail must not be recorded".into(), + }), + ); + assert_eq!( + failure_mark_data(4, &failure), + json!({ + "attempt": 4, + "retryable": false, + "failure_kind": "non_http", + "non_http_kind": "invalid_request", + }) + ); + + let failure = LibsyError::MissingFinalResponse; + assert_eq!( + failure_mark_data(5, &failure), + json!({ + "attempt": 5, + "retryable": false, + "failure_kind": "algorithm", + }) + ); } #[test] @@ -1023,13 +897,6 @@ mod tests { "finish_reason": null }] }); - let provider_failure = LlmContinuationFailureV2::NonHttp { - failure: LlmNonHttpFailureV2 { - kind: LlmNonHttpFailureKindV2::Timeout, - message: "provider failed after its first event".into(), - }, - }; - let provider_error = Arc::new(Mutex::new(Some(provider_failure))); let response = Response { llm_response: LlmResponse::Stream(Box::pin(futures_util::stream::iter([ Ok(LlmResponseStreamEvent::preserved( @@ -1040,7 +907,12 @@ mod tests { text: "committed".into(), }], )), - Err(LlmClientError::General("late failure".into())), + Err(LlmClientError::Timeout { + source: Box::new(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "provider failed after its first event", + )), + }), ]))), metadata: Some(Metadata { wire_format: Some(WireProtocol::OpenaiChat.wire_format()), @@ -1052,7 +924,6 @@ mod tests { response, WireProtocol::OpenaiChat, Arc::new(TranslationEngine::default()), - provider_error, ); let first = stream .next() @@ -1068,6 +939,6 @@ mod tests { .expect("late failure exists") .expect_err("late failure is propagated"); assert!(failure.committed); - assert!(failure.failure.retryable()); + assert!(libsy_error_retryable(&failure.error)); } } From 7226c038c8437102fbf0dfa3ed36a69c5c75c705 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 15:05:47 -0600 Subject: [PATCH 27/51] refactor(plugin): simplify run stream routing Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 +- .../switchyard-nemo-relay-plugin/Cargo.toml | 2 +- .../config.schema.json | 10 +- .../src/config.rs | 229 ++++++------ .../src/runtime.rs | 346 +++++++----------- .../src/translation.rs | 58 ++- .../tests/e2e/run_e2e.py | 7 - 7 files changed, 279 insertions(+), 377 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 255966534..2b6517de7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=648fb185d20cb8286a16cc2a7759a4039ceade21#648fb185d20cb8286a16cc2a7759a4039ceade21" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=1580e22b33dc727710cf0a42e23ce6eeb7936dfe#1580e22b33dc727710cf0a42e23ce6eeb7936dfe" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=648fb185d20cb8286a16cc2a7759a4039ceade21#648fb185d20cb8286a16cc2a7759a4039ceade21" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=1580e22b33dc727710cf0a42e23ce6eeb7936dfe#1580e22b33dc727710cf0a42e23ce6eeb7936dfe" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 74b2bcd03..c61cf2602 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib", "rlib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "648fb185d20cb8286a16cc2a7759a4039ceade21" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "1580e22b33dc727710cf0a42e23ce6eeb7936dfe" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/config.schema.json b/crates/switchyard-nemo-relay-plugin/config.schema.json index bb41c1d4c..14d2a84ef 100644 --- a/crates/switchyard-nemo-relay-plugin/config.schema.json +++ b/crates/switchyard-nemo-relay-plugin/config.schema.json @@ -19,14 +19,6 @@ "maximum": 10, "default": 3 }, - "enabled_inbound_profiles": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "enum": ["openai_chat", "openai_responses", "anthropic_messages"] - } - }, "algorithm": { "oneOf": [ { @@ -98,6 +90,8 @@ }, "default_targets": { "type": "object", + "description": "Maps each managed inbound protocol to its trusted fallback target.", + "minProperties": 1, "additionalProperties": false, "properties": { "openai_chat": { "type": "string" }, diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index a8a397625..f247cca81 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::sync::Arc; use http::header::{HeaderName, HeaderValue}; @@ -11,65 +11,27 @@ use switchyard_libsy::{ }; use switchyard_protocol::WireFormat; -#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd)] -#[serde(rename_all = "snake_case")] -pub enum WireProtocol { - OpenaiChat, - OpenaiResponses, - AnthropicMessages, -} - -impl WireProtocol { - pub const fn label(self) -> &'static str { - match self { - Self::OpenaiChat => "openai_chat", - Self::OpenaiResponses => "openai_responses", - Self::AnthropicMessages => "anthropic_messages", - } - } - - pub const fn endpoint(self) -> &'static str { - match self { - Self::OpenaiChat => "/v1/chat/completions", - Self::OpenaiResponses => "/v1/responses", - Self::AnthropicMessages => "/v1/messages", - } - } - - pub fn from_call(name: &str) -> Option { - match name { - "openai.chat_completions" | "openai_chat" | "openai_chat_completions" => { - Some(Self::OpenaiChat) - } - "openai.responses" | "openai_responses" => Some(Self::OpenaiResponses), - "anthropic.messages" | "anthropic" | "anthropic_messages" => { - Some(Self::AnthropicMessages) - } - _ => None, - } - } - - pub const fn wire_format(self) -> WireFormat { - match self { - Self::OpenaiChat => WireFormat::OpenAiChat, - Self::OpenaiResponses => WireFormat::OpenAiResponses, - Self::AnthropicMessages => WireFormat::AnthropicMessages, - } +pub(crate) fn protocol_from_call(name: &str) -> Option { + match name { + "openai.chat_completions" => Some(WireFormat::OpenAiChat), + "openai.responses" => Some(WireFormat::OpenAiResponses), + "anthropic.messages" => Some(WireFormat::AnthropicMessages), + _ => None, } +} - pub fn from_wire_format(format: &WireFormat) -> Option { - match format { - WireFormat::OpenAiChat => Some(Self::OpenaiChat), - WireFormat::OpenAiResponses => Some(Self::OpenaiResponses), - WireFormat::AnthropicMessages => Some(Self::AnthropicMessages), - } +const fn default_endpoint(protocol: WireFormat) -> &'static str { + match protocol { + WireFormat::OpenAiChat => "/v1/chat/completions", + WireFormat::OpenAiResponses => "/v1/responses", + WireFormat::AnthropicMessages => "/v1/messages", } } #[derive(Deserialize)] pub struct TargetBinding { pub model: String, - pub protocol: WireProtocol, + pub protocol: WireFormat, #[serde(default)] pub endpoint: String, pub base_url: String, @@ -85,7 +47,7 @@ impl TargetBinding { pub fn dispatch_url(&self) -> String { let base = self.base_url.trim_end_matches('/'); let endpoint = if self.endpoint.is_empty() { - self.protocol.endpoint() + default_endpoint(self.protocol) } else { &self.endpoint }; @@ -141,7 +103,7 @@ impl TargetBinding { pub struct PreparedTargetBinding { pub model: String, - pub protocol: WireProtocol, + pub protocol: WireFormat, dispatch_url: String, pub headers: BTreeMap, } @@ -152,26 +114,6 @@ impl PreparedTargetBinding { } } -#[derive(Default, Deserialize)] -pub struct ProtocolDefaults { - #[serde(default)] - pub openai_chat: String, - #[serde(default)] - pub openai_responses: String, - #[serde(default)] - pub anthropic_messages: String, -} - -impl ProtocolDefaults { - pub fn target(&self, protocol: WireProtocol) -> &str { - match protocol { - WireProtocol::OpenaiChat => &self.openai_chat, - WireProtocol::OpenaiResponses => &self.openai_responses, - WireProtocol::AnthropicMessages => &self.anthropic_messages, - } - } -} - #[derive(Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum AlgorithmConfig { @@ -195,35 +137,24 @@ pub enum AlgorithmConfig { }, } -impl Default for AlgorithmConfig { - fn default() -> Self { - Self::Random { seed: None } - } -} - #[derive(Deserialize)] +#[serde(deny_unknown_fields)] pub struct SwitchyardConfig { - #[serde(default = "default_version")] pub version: u32, #[serde(default)] pub priority: i32, #[serde(default = "default_max_retries")] pub max_retries: u32, - #[serde(default)] pub algorithm: AlgorithmConfig, pub targets: BTreeMap, - #[serde(default)] - pub default_targets: ProtocolDefaults, - #[serde(default = "default_enabled_protocols")] - pub enabled_inbound_profiles: BTreeSet, + pub default_targets: BTreeMap, } pub(crate) struct PreparedConfig { pub max_retries: u32, pub algorithm: Arc, pub targets: BTreeMap, - pub default_targets: ProtocolDefaults, - pub enabled_inbound_profiles: BTreeSet, + pub default_targets: BTreeMap, } impl SwitchyardConfig { @@ -245,8 +176,8 @@ impl SwitchyardConfig { if self.targets.is_empty() { return Err("targets must not be empty".into()); } - if self.enabled_inbound_profiles.is_empty() { - return Err("enabled_inbound_profiles must not be empty".into()); + if self.default_targets.is_empty() { + return Err("default_targets must not be empty".into()); } for (name, target) in &self.targets { if name.trim().is_empty() || target.model.trim().is_empty() { @@ -262,8 +193,7 @@ impl SwitchyardConfig { } target.validate_headers()?; } - for protocol in &self.enabled_inbound_profiles { - let fallback = self.default_targets.target(*protocol); + for (protocol, fallback) in &self.default_targets { let target = self .targets .get(fallback) @@ -271,7 +201,7 @@ impl SwitchyardConfig { if target.protocol != *protocol { return Err(format!( "default target {fallback:?} must use protocol {}", - protocol.label() + protocol.as_str() )); } } @@ -291,7 +221,6 @@ impl SwitchyardConfig { algorithm, targets, default_targets: self.default_targets, - enabled_inbound_profiles: self.enabled_inbound_profiles, }) } @@ -362,10 +291,6 @@ fn validate_header(name: &str, value: &str) -> Result<(), String> { Ok(()) } -const fn default_version() -> u32 { - 2 -} - const fn default_max_retries() -> u32 { 3 } @@ -374,19 +299,12 @@ const fn default_weight() -> f64 { 1.0 } -fn default_enabled_protocols() -> BTreeSet { - BTreeSet::from([ - WireProtocol::OpenaiChat, - WireProtocol::OpenaiResponses, - WireProtocol::AnthropicMessages, - ]) -} - #[cfg(test)] mod tests { use super::*; + use serde_json::json; - fn binding(protocol: WireProtocol, model: &str) -> TargetBinding { + fn binding(protocol: WireFormat, model: &str) -> TargetBinding { TargetBinding { model: model.into(), protocol, @@ -407,23 +325,22 @@ mod tests { targets: BTreeMap::from([ ( "chat".into(), - binding(WireProtocol::OpenaiChat, "provider/chat"), + binding(WireFormat::OpenAiChat, "provider/chat"), ), ( "responses".into(), - binding(WireProtocol::OpenaiResponses, "provider/responses"), + binding(WireFormat::OpenAiResponses, "provider/responses"), ), ( "anthropic".into(), - binding(WireProtocol::AnthropicMessages, "provider/anthropic"), + binding(WireFormat::AnthropicMessages, "provider/anthropic"), ), ]), - default_targets: ProtocolDefaults { - openai_chat: "chat".into(), - openai_responses: "responses".into(), - anthropic_messages: "anthropic".into(), - }, - enabled_inbound_profiles: default_enabled_protocols(), + default_targets: BTreeMap::from([ + (WireFormat::OpenAiChat, "chat".into()), + (WireFormat::OpenAiResponses, "responses".into()), + (WireFormat::AnthropicMessages, "anthropic".into()), + ]), } } @@ -447,6 +364,86 @@ mod tests { assert!(error.contains("version = 2")); } + #[test] + fn default_target_keys_define_the_managed_protocols() { + let mut config = config(); + config + .default_targets + .retain(|protocol, _| *protocol == WireFormat::OpenAiChat); + config.validate().unwrap(); + assert_eq!(config.default_targets.len(), 1); + } + + #[test] + fn only_canonical_relay_execution_names_resolve_protocols() { + assert_eq!( + protocol_from_call("openai.chat_completions"), + Some(WireFormat::OpenAiChat) + ); + assert_eq!( + protocol_from_call("openai.responses"), + Some(WireFormat::OpenAiResponses) + ); + assert_eq!( + protocol_from_call("anthropic.messages"), + Some(WireFormat::AnthropicMessages) + ); + for alias in [ + "openai_chat", + "openai_chat_completions", + "openai_responses", + "anthropic", + "anthropic_messages", + ] { + assert_eq!(protocol_from_call(alias), None, "alias={alias}"); + } + } + + #[test] + fn schema_required_contract_fields_do_not_default_during_deserialization() { + let base = json!({ + "version": 2, + "algorithm": {"kind": "random"}, + "targets": { + "chat": { + "model": "provider/chat", + "protocol": "openai_chat", + "base_url": "https://provider.example/v1" + } + }, + "default_targets": {"openai_chat": "chat"} + }); + for field in ["version", "algorithm", "default_targets"] { + let mut value = base.clone(); + value.as_object_mut().unwrap().remove(field); + let error = serde_json::from_value::(value) + .err() + .expect("required field must not default"); + assert!(error.to_string().contains(field), "field={field}: {error}"); + } + } + + #[test] + fn removed_enabled_profile_list_is_not_silently_ignored() { + let value = json!({ + "version": 2, + "algorithm": {"kind": "random"}, + "targets": { + "chat": { + "model": "provider/chat", + "protocol": "openai_chat", + "base_url": "https://provider.example/v1" + } + }, + "default_targets": {"openai_chat": "chat"}, + "enabled_inbound_profiles": ["openai_chat"] + }); + let error = serde_json::from_value::(value) + .err() + .expect("removed field must produce a migration error"); + assert!(error.to_string().contains("enabled_inbound_profiles")); + } + #[test] fn classifier_targets_are_semantic_names_not_provider_models() { let mut config = config(); diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index a430e306d..bdfdda4f9 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::pin::Pin; use std::sync::Arc; @@ -14,20 +14,20 @@ use nemo_relay_plugin::{ use serde_json::{json, Map}; use switchyard_libsy::{Algorithm, CallLlmRequest, LibsyError, Step}; use switchyard_protocol::{ - Context, LlmClientError, LlmRequest as SwitchyardLlmRequest, LlmResponse, LlmResponseChunk, - LlmResponseStream, LlmResponseStreamEvent, Metadata, Request, Response, + Context, Decision, LlmClientError, LlmRequest as SwitchyardLlmRequest, LlmResponse, + LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, Metadata, Request, Response, + WireFormat, }; use switchyard_translation::{StreamTranslationState, TranslationEngine}; -use crate::config::{PreparedTargetBinding, ProtocolDefaults, SwitchyardConfig, WireProtocol}; +use crate::config::{protocol_from_call, PreparedTargetBinding, SwitchyardConfig}; use crate::translation; pub struct SwitchyardRuntime { max_retries: u32, algorithm: Arc, targets: BTreeMap, - default_targets: ProtocolDefaults, - enabled_inbound_profiles: BTreeSet, + default_targets: BTreeMap, translation: Arc, relay: PluginRuntime, } @@ -40,7 +40,6 @@ impl SwitchyardRuntime { algorithm: prepared.algorithm, targets: prepared.targets, default_targets: prepared.default_targets, - enabled_inbound_profiles: prepared.enabled_inbound_profiles, translation: Arc::new(TranslationEngine::default()), relay, }) @@ -52,12 +51,9 @@ impl SwitchyardRuntime { request: RelayRequest, continuation: LlmContinuationV2, ) -> Result { - let Some(inbound) = WireProtocol::from_call(&name) else { + let Some(inbound) = self.managed_protocol(&name) else { return continuation.call_passthrough(request).await; }; - if !self.enabled_inbound_profiles.contains(&inbound) { - return continuation.call_passthrough(request).await; - } let libsy_request = self.libsy_request(inbound, &request, false)?; let metadata = identity_metadata(libsy_request.metadata.as_ref()); let max_attempts = self.max_retries + 1; @@ -117,18 +113,7 @@ impl SwitchyardRuntime { while let Some(step) = steps.next().await { match step { Ok(Step::Decision(decision)) => { - self.mark( - "switchyard.routing.decision", - json!({ - "algorithm": self.algorithm.name(), - "attempt": attempt, - "selected_target": decision.selected_model(), - "reasoning": decision.reasoning(), - "routing_tier": decision.routing_tier(), - "is_routed_call": decision.is_routed_call(), - }), - mark_metadata, - ); + self.emit_decision(decision.as_ref(), attempt, mark_metadata); } Ok(Step::CallLlm(call)) => { self.serve_buffered_call(*call, continuation).await?; @@ -157,7 +142,7 @@ impl SwitchyardRuntime { Ok(Response { llm_response: LlmResponse::Agg(response), metadata: Some(Metadata { - wire_format: Some(target.protocol.wire_format()), + wire_format: Some(target.protocol), ..Metadata::default() }), }) @@ -169,12 +154,12 @@ impl SwitchyardRuntime { async fn fallback_buffered( &self, - inbound: WireProtocol, + inbound: WireFormat, request: Request, continuation: &LlmContinuationV2, metadata: &Json, ) -> Result { - let target_name = self.default_targets.target(inbound); + let target_name = self.default_target(inbound); let target = self .target(target_name) .map_err(|error| error.to_string())?; @@ -196,7 +181,7 @@ impl SwitchyardRuntime { fn libsy_request( &self, - inbound: WireProtocol, + inbound: WireFormat, original: &RelayRequest, streaming: bool, ) -> Result { @@ -204,7 +189,7 @@ impl SwitchyardRuntime { request.stream = streaming; let headers = string_headers(&original.headers); let mut metadata = Metadata::from_headers(&headers); - metadata.wire_format = Some(inbound.wire_format()); + metadata.wire_format = Some(inbound); Ok(Request { llm_request: request, raw_request: None, @@ -244,24 +229,46 @@ impl SwitchyardRuntime { }) } + fn managed_protocol(&self, name: &str) -> Option { + protocol_from_call(name).filter(|protocol| self.default_targets.contains_key(protocol)) + } + + fn default_target(&self, protocol: WireFormat) -> &str { + self.default_targets + .get(&protocol) + .expect("managed protocol must have a default target") + } + fn mark(&self, name: &str, data: Json, metadata: &Json) { if let Err(error) = self.relay.emit_mark(name, Some(&data), Some(metadata)) { eprintln!("Switchyard could not emit routing mark {name:?}: {error}"); } } + fn emit_decision(&self, decision: &dyn Decision, attempt: u32, metadata: &Json) { + self.mark( + "switchyard.routing.decision", + json!({ + "algorithm": self.algorithm.name(), + "attempt": attempt, + "selected_target": decision.selected_model(), + "reasoning": decision.reasoning(), + "routing_tier": decision.routing_tier(), + "is_routed_call": decision.is_routed_call(), + }), + metadata, + ); + } + pub async fn execute_stream( self: Arc, name: String, request: RelayRequest, continuation: LlmStreamContinuationV2, ) -> Result { - let Some(inbound) = WireProtocol::from_call(&name) else { + let Some(inbound) = self.managed_protocol(&name) else { return Ok(LlmStreamExecutionOutcomeV2::Passthrough(request)); }; - if !self.enabled_inbound_profiles.contains(&inbound) { - return Ok(LlmStreamExecutionOutcomeV2::Passthrough(request)); - } let libsy_request = self.libsy_request(inbound, &request, true)?; let metadata = identity_metadata(libsy_request.metadata.as_ref()); let stream = self.routed_stream(inbound, libsy_request, continuation, metadata); @@ -270,20 +277,20 @@ impl SwitchyardRuntime { fn routed_stream( self: Arc, - inbound: WireProtocol, + inbound: WireFormat, request: Request, continuation: LlmStreamContinuationV2, metadata: Json, ) -> LlmJsonAsyncStreamV2 { Box::pin(async_stream::try_stream! { let max_attempts = self.max_retries + 1; - 'attempts: for attempt in 1..=max_attempts { + for attempt in 1..=max_attempts { self.mark( "switchyard.routing.requested", json!({"algorithm": self.algorithm.name(), "attempt": attempt}), &metadata, ); - let response = match self + let failure = match self .drive_stream( request.clone(), &continuation, @@ -292,73 +299,56 @@ impl SwitchyardRuntime { ) .await { - Ok(response) => response, - Err(failure) if libsy_error_retryable(&failure) && attempt < max_attempts => { - self.emit_stream_retry(attempt, &failure, &metadata); - continue; - } - Err(failure) => { - self.emit_stream_error(attempt, &failure, &metadata); - let mut fallback = self - .fallback_stream( - inbound, - request.clone(), - &continuation, - &metadata, - ) - .await?; - while let Some(item) = fallback.next().await { - yield item.map_err(|failure| { - format!( - "trusted fallback stream failed: {}", - failure.error - ) - })?; - } - return; - } - }; - let mut returned = - Self::returned_stream(response, inbound, Arc::clone(&self.translation)); - while let Some(item) = returned.next().await { - match item { - Ok(event) => yield event, - Err(failure) - if !failure.committed - && libsy_error_retryable(&failure.error) - && attempt < max_attempts => - { - self.emit_stream_retry(attempt, &failure.error, &metadata); - continue 'attempts; - } - Err(failure) if !failure.committed => { - self.emit_stream_error(attempt, &failure.error, &metadata); - let mut fallback = self - .fallback_stream( - inbound, - request.clone(), - &continuation, - &metadata, - ) - .await?; - while let Some(item) = fallback.next().await { - yield item.map_err(|failure| { - format!( - "trusted fallback stream failed: {}", - failure.error - ) - })?; + Ok(response) => { + let mut returned = Self::returned_stream( + response, + inbound, + Arc::clone(&self.translation), + ); + let mut committed = false; + loop { + match returned.next().await { + Some(Ok(event)) => { + committed = true; + yield event; + } + Some(Err(failure)) if committed => { + self.mark( + "switchyard.routing.error", + failure_mark_data(attempt, &failure), + &metadata, + ); + Err(format!( + "Switchyard stream failed after response commitment: {failure}" + ))?; + } + Some(Err(failure)) => break failure, + None => return, } - return; - } - Err(failure) => { - self.emit_stream_error(attempt, &failure.error, &metadata); - Err(format!( - "Switchyard stream failed after response commitment: {}", - failure.error - ))?; } } + Err(failure) => failure, + }; + if libsy_error_retryable(&failure) && attempt < max_attempts { + self.mark( + "switchyard.routing.retry", + failure_mark_data(attempt, &failure), + &metadata, + ); + continue; + } + self.mark( + "switchyard.routing.error", + failure_mark_data(attempt, &failure), + &metadata, + ); + let mut fallback = self + .fallback_stream(inbound, request.clone(), &continuation, &metadata) + .await?; + while let Some(item) = fallback.next().await { + yield item.map_err(|failure| { + format!("trusted fallback stream failed: {failure}") + })?; } return; } @@ -380,18 +370,7 @@ impl SwitchyardRuntime { while let Some(step) = steps.next().await { match step { Ok(Step::Decision(decision)) => { - self.mark( - "switchyard.routing.decision", - json!({ - "algorithm": self.algorithm.name(), - "attempt": attempt, - "selected_target": decision.selected_model(), - "reasoning": decision.reasoning(), - "routing_tier": decision.routing_tier(), - "is_routed_call": decision.is_routed_call(), - }), - mark_metadata, - ); + self.emit_decision(decision.as_ref(), attempt, mark_metadata); } Ok(Step::CallLlm(call)) => { self.serve_stream_call(*call, continuation).await?; @@ -444,10 +423,7 @@ impl SwitchyardRuntime { }); } }; - let mut state = StreamTranslationState::new( - target.protocol.wire_format(), - target.protocol.wire_format(), - ); + let mut state = StreamTranslationState::new(target.protocol, target.protocol); let first = decode_provider_event(&self.translation, &mut state, target.protocol, first_raw)?; let protocol = target.protocol; @@ -462,7 +438,7 @@ impl SwitchyardRuntime { Ok(Response { llm_response: LlmResponse::Stream(stream), metadata: Some(Metadata { - wire_format: Some(target.protocol.wire_format()), + wire_format: Some(target.protocol), ..metadata.unwrap_or_default() }), }) @@ -470,40 +446,36 @@ impl SwitchyardRuntime { fn returned_stream( response: Response, - inbound: WireProtocol, + inbound: WireFormat, translation: Arc, ) -> ReturnedJsonStream { Box::pin(async_stream::stream! { let source = match response .metadata .as_ref() - .and_then(|metadata| metadata.wire_format.as_ref()) - .and_then(WireProtocol::from_wire_format) + .and_then(|metadata| metadata.wire_format) { Some(source) => source, None => { - yield Err(StreamAttemptFailure::translation( + yield Err(returned_stream_translation_error( "libsy returned a stream without a supported source wire format", - false, )); return; } }; let LlmResponse::Stream(mut stream) = response.llm_response else { - yield Err(StreamAttemptFailure::translation( + yield Err(returned_stream_translation_error( "libsy returned a buffered response for a streaming request", - false, )); return; }; - let mut state = - StreamTranslationState::new(source.wire_format(), inbound.wire_format()); - let mut committed = false; + let mut state = StreamTranslationState::new(source, inbound); + let mut emitted = false; while let Some(item) = stream.next().await { let event = match item { Ok(event) => event, Err(error) => { - yield Err(StreamAttemptFailure::client(error, committed)); + yield Err(LibsyError::client_call("return_to_agent", error)); return; } }; @@ -515,42 +487,29 @@ impl SwitchyardRuntime { ) { Ok(events) => events, Err(error) => { - yield Err(StreamAttemptFailure::translation( - &error, - committed, - )); + yield Err(returned_stream_translation_error(error)); return; } }; for event in events { - committed = true; + emitted = true; yield Ok(event); } } - if source != inbound { - let events = match translation::finish_stream( - &translation, - &mut state, - inbound, - ) { - Ok(events) => events, - Err(error) => { - yield Err(StreamAttemptFailure::translation( - &error, - committed, - )); - return; - } - }; - for event in events { - committed = true; - yield Ok(event); + let events = match translation::finish_stream(&translation, &mut state, inbound) { + Ok(events) => events, + Err(error) => { + yield Err(returned_stream_translation_error(error)); + return; } + }; + for event in events { + emitted = true; + yield Ok(event); } - if !committed { - yield Err(StreamAttemptFailure::translation( + if !emitted { + yield Err(returned_stream_translation_error( "Switchyard produced an empty output stream", - false, )); } }) @@ -558,12 +517,12 @@ impl SwitchyardRuntime { async fn fallback_stream( &self, - inbound: WireProtocol, + inbound: WireFormat, request: Request, continuation: &LlmStreamContinuationV2, metadata: &Json, ) -> Result { - let target_name = self.default_targets.target(inbound); + let target_name = self.default_target(inbound); self.mark( "switchyard.routing.fallback", json!({"selected_target": target_name}), @@ -584,30 +543,14 @@ impl SwitchyardRuntime { Arc::clone(&self.translation), )) } - - fn emit_stream_retry(&self, attempt: u32, failure: &LibsyError, metadata: &Json) { - self.mark( - "switchyard.routing.retry", - failure_mark_data(attempt, failure), - metadata, - ); - } - - fn emit_stream_error(&self, attempt: u32, failure: &LibsyError, metadata: &Json) { - self.mark( - "switchyard.routing.error", - failure_mark_data(attempt, failure), - metadata, - ); - } } -type ReturnedJsonStream = Pin> + Send>>; +type ReturnedJsonStream = Pin> + Send>>; fn decode_provider_event( translation: &TranslationEngine, state: &mut StreamTranslationState, - protocol: WireProtocol, + protocol: WireFormat, raw: Json, ) -> Result { let event = translation::decode_stream_event(translation, state, protocol, raw) @@ -629,41 +572,25 @@ fn decode_provider_event( Ok(event) } -struct StreamAttemptFailure { - error: LibsyError, - committed: bool, -} - -impl StreamAttemptFailure { - fn client(error: LlmClientError, committed: bool) -> Self { - Self { - error: LibsyError::client_call("return_to_agent", error), - committed, - } - } - - fn translation(error: &str, committed: bool) -> Self { - Self::client( - LlmClientError::ResponseTranslation(error.to_string()), - committed, - ) - } +fn returned_stream_translation_error(error: impl Into) -> LibsyError { + LibsyError::client_call( + "return_to_agent", + LlmClientError::ResponseTranslation(error.into()), + ) } fn libsy_error_retryable(error: &LibsyError) -> bool { - matches!( - error, - LibsyError::ClientCall { - source: LlmClientError::UpstreamHttp { status, .. }, - .. - } if matches!(*status, 408 | 425 | 429 | 500 | 502 | 503 | 504) - ) || matches!( - error, - LibsyError::ClientCall { - source: LlmClientError::Transport { .. } | LlmClientError::Timeout { .. }, - .. + let LibsyError::ClientCall { source, .. } = error else { + return false; + }; + match source { + LlmClientError::UpstreamHttp { status, .. } => { + LlmContinuationFailureV2::http_status_is_retryable(*status) } - ) + LlmClientError::Transport { .. } => LlmNonHttpFailureKindV2::Transport.is_retryable(), + LlmClientError::Timeout { .. } => LlmNonHttpFailureKindV2::Timeout.is_retryable(), + _ => false, + } } fn client_error(error: LlmContinuationFailureV2) -> LlmClientError { @@ -886,7 +813,7 @@ mod tests { } #[test] - fn late_provider_failure_stays_committed_and_cannot_retry() { + fn returned_stream_preserves_late_provider_failure() { let raw = json!({ "id": "chatcmpl-test", "object": "chat.completion.chunk", @@ -900,7 +827,7 @@ mod tests { let response = Response { llm_response: LlmResponse::Stream(Box::pin(futures_util::stream::iter([ Ok(LlmResponseStreamEvent::preserved( - WireProtocol::OpenaiChat.wire_format(), + WireFormat::OpenAiChat, raw.clone(), vec![LlmResponseChunk::TextDelta { index: 0, @@ -915,14 +842,14 @@ mod tests { }), ]))), metadata: Some(Metadata { - wire_format: Some(WireProtocol::OpenaiChat.wire_format()), + wire_format: Some(WireFormat::OpenAiChat), ..Metadata::default() }), }; let mut stream = SwitchyardRuntime::returned_stream( response, - WireProtocol::OpenaiChat, + WireFormat::OpenAiChat, Arc::new(TranslationEngine::default()), ); let first = stream @@ -938,7 +865,6 @@ mod tests { .expect("late failure is ready") .expect("late failure exists") .expect_err("late failure is propagated"); - assert!(failure.committed); - assert!(libsy_error_retryable(&failure.error)); + assert!(libsy_error_retryable(&failure)); } } diff --git a/crates/switchyard-nemo-relay-plugin/src/translation.rs b/crates/switchyard-nemo-relay-plugin/src/translation.rs index ae8f83fc6..3123c507d 100644 --- a/crates/switchyard-nemo-relay-plugin/src/translation.rs +++ b/crates/switchyard-nemo-relay-plugin/src/translation.rs @@ -3,22 +3,20 @@ use nemo_relay_plugin::LlmRequest as RelayRequest; use serde_json::{Map, Value as Json}; -use switchyard_protocol::{AggLlmResponse, LlmRequest}; +use switchyard_protocol::{AggLlmResponse, LlmRequest, WireFormat}; use switchyard_translation::{ DeterministicIdPolicy, DiagnosticSeverity, LossyConversionPolicy, PreservationPolicy, StreamTranslationState, TargetCapabilities, TranslationDiagnostic, TranslationEngine, TranslationPolicy, UnknownFieldPolicy, }; -use crate::config::WireProtocol; - pub fn decode_request( engine: &TranslationEngine, - protocol: WireProtocol, + protocol: WireFormat, request: &RelayRequest, ) -> Result { let output = engine - .decode_request(protocol.wire_format(), &request.content, &policy()) + .decode_request(protocol, &request.content, &policy()) .map_err(error)?; safe(&output.diagnostics)?; Ok(output.request) @@ -26,11 +24,11 @@ pub fn decode_request( pub fn encode_request( engine: &TranslationEngine, - protocol: WireProtocol, + protocol: WireFormat, request: &LlmRequest, ) -> Result { let output = engine - .encode_request(protocol.wire_format(), request, &request_policy(protocol)) + .encode_request(protocol, request, &request_policy(protocol)) .map_err(error)?; safe(&output.diagnostics)?; Ok(RelayRequest { @@ -41,11 +39,11 @@ pub fn encode_request( pub fn decode_response( engine: &TranslationEngine, - protocol: WireProtocol, + protocol: WireFormat, response: &Json, ) -> Result { let output = engine - .decode_response(protocol.wire_format(), response, &policy()) + .decode_response(protocol, response, &policy()) .map_err(error)?; safe(&output.diagnostics)?; Ok(output.response) @@ -53,11 +51,11 @@ pub fn decode_response( pub fn encode_response( engine: &TranslationEngine, - protocol: WireProtocol, + protocol: WireFormat, response: &AggLlmResponse, ) -> Result { let output = engine - .encode_response(protocol.wire_format(), response, &policy()) + .encode_response(protocol, response, &policy()) .map_err(error)?; safe(&output.diagnostics)?; Ok(output.body) @@ -66,33 +64,31 @@ pub fn encode_response( pub fn decode_stream_event( engine: &TranslationEngine, state: &mut StreamTranslationState, - protocol: WireProtocol, + protocol: WireFormat, event: Json, ) -> Result { engine - .decode_stream_event(state, protocol.wire_format(), event) + .decode_stream_event(state, protocol, event) .map_err(error) } pub fn encode_stream_event( engine: &TranslationEngine, state: &mut StreamTranslationState, - protocol: WireProtocol, + protocol: WireFormat, event: switchyard_protocol::LlmResponseStreamEvent, ) -> Result, String> { engine - .encode_stream_event(state, protocol.wire_format(), event) + .encode_stream_event(state, protocol, event) .map_err(error) } pub fn finish_stream( engine: &TranslationEngine, state: &mut StreamTranslationState, - protocol: WireProtocol, + protocol: WireFormat, ) -> Result, String> { - engine - .finish_stream(state, protocol.wire_format()) - .map_err(error) + engine.finish_stream(state, protocol).map_err(error) } fn policy() -> TranslationPolicy { @@ -107,9 +103,9 @@ fn policy() -> TranslationPolicy { } } -fn request_policy(protocol: WireProtocol) -> TranslationPolicy { +fn request_policy(protocol: WireFormat) -> TranslationPolicy { let mut policy = policy(); - if protocol == WireProtocol::AnthropicMessages { + if protocol == WireFormat::AnthropicMessages { policy .target_capabilities .supports_json_schema_response_format = Some(false); @@ -144,7 +140,7 @@ mod tests { fn plugin_replays_same_protocol_provider_extensions_exactly() { let cases = [ ( - WireProtocol::OpenaiChat, + WireFormat::OpenAiChat, json!({ "id": "chatcmpl-test", "object": "chat.completion.chunk", @@ -158,7 +154,7 @@ mod tests { }), ), ( - WireProtocol::OpenaiResponses, + WireFormat::OpenAiResponses, json!({ "type": "response.output_text.delta", "item_id": "item-1", @@ -170,7 +166,7 @@ mod tests { }), ), ( - WireProtocol::AnthropicMessages, + WireFormat::AnthropicMessages, json!({ "type": "content_block_delta", "index": 0, @@ -182,8 +178,7 @@ mod tests { let engine = TranslationEngine::default(); for (protocol, raw) in cases { - let mut state = - StreamTranslationState::new(protocol.wire_format(), protocol.wire_format()); + let mut state = StreamTranslationState::new(protocol, protocol); let event = decode_stream_event(&engine, &mut state, protocol, raw.clone()).unwrap(); assert_eq!( encode_stream_event(&engine, &mut state, protocol, event).unwrap(), @@ -195,14 +190,12 @@ mod tests { #[test] fn cross_protocol_streams_use_normalized_content_not_raw_extensions() { let engine = TranslationEngine::default(); - let mut state = StreamTranslationState::new( - WireProtocol::OpenaiChat.wire_format(), - WireProtocol::AnthropicMessages.wire_format(), - ); + let mut state = + StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::AnthropicMessages); let event = decode_stream_event( &engine, &mut state, - WireProtocol::OpenaiChat, + WireFormat::OpenAiChat, json!({ "id": "chatcmpl-test", "object": "chat.completion.chunk", @@ -217,8 +210,7 @@ mod tests { ) .unwrap(); let translated = - encode_stream_event(&engine, &mut state, WireProtocol::AnthropicMessages, event) - .unwrap(); + encode_stream_event(&engine, &mut state, WireFormat::AnthropicMessages, event).unwrap(); assert!(translated .iter() .any(|event| { event.pointer("/delta/text").and_then(Json::as_str) == Some("Hi") })); diff --git a/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py b/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py index 53eaa4f49..761cf43f4 100644 --- a/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py +++ b/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py @@ -155,7 +155,6 @@ def plugin_config( algorithm: str, targets: str, defaults: str, - profiles: str, *, max_retries: int = 1, ) -> str: @@ -169,7 +168,6 @@ def plugin_config( version = 2 priority = 0 max_retries = {max_retries} -enabled_inbound_profiles = [{profiles}] {algorithm} @@ -329,7 +327,6 @@ def run_same_protocol( x-switchyard-target = "same" """, 'openai_chat = "chat"', - '"openai_chat"', ) with RelayScenario(relay_bin, root, provider_url, "same", config) as relay: template = dict(CASES[0][2]) @@ -383,7 +380,6 @@ def run_random( 'openai_responses = "responses"\n' 'anthropic_messages = "anthropic"' ), - '"openai_chat", "openai_responses", "anthropic_messages"', ) with RelayScenario(relay_bin, root, provider_url, "random", config) as relay: models: set[str] = set() @@ -477,7 +473,6 @@ def run_classifier( base_url = "{provider_url}/v1" """, 'openai_chat = "fallback"', - '"openai_chat"', ) with RelayScenario(relay_bin, root, provider_url, "classifier", config) as relay: body = dict(CASES[0][2]) @@ -530,7 +525,6 @@ def single_target_config( base_url = "{{provider_url}}/v1" """, 'openai_chat = "fallback"', - '"openai_chat"', max_retries=max_retries, ) @@ -722,7 +716,6 @@ def run_unmanaged_passthrough( weight = 1 """, 'openai_responses = "responses"', - '"openai_responses"', ) with RelayScenario(relay_bin, root, provider_url, "unmanaged-passthrough", config) as relay: body = dict(CASES[0][2]) From 8a15966862ed5864c6e50fb74c10dbe742591593 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 15:19:14 -0600 Subject: [PATCH 28/51] test(plugin): expand dynamic routing end-to-end coverage Signed-off-by: Bryan Bednarski --- .../tests/e2e/fake_provider.py | 27 +- .../tests/e2e/run_e2e.py | 334 ++++++++++++++---- 2 files changed, 283 insertions(+), 78 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py b/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py index fc8adaea8..d37fe2108 100644 --- a/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py +++ b/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py @@ -50,6 +50,9 @@ def do_POST(self) -> None: if model in {"fake/retry-once", "fake/retry-stream-once"} and attempt == 1: self._json(503, {"error": {"message": "retry this request"}}) return + if model in {"fake/reselect-fail", "fake/reselect-stream-fail"}: + self._json(503, {"error": {"message": "reselect this target"}}) + return if model in {"fake/always-fail", "fake/always-fail-stream"}: self._json(400, {"error": {"message": "invalid routed request"}}) return @@ -64,15 +67,20 @@ def do_POST(self) -> None: def _chat(self, request: dict[str, object]) -> None: model = str(request.get("model", "unknown")) - classifier = model == "fake/classifier" + classifier = model in {"fake/classifier", "fake/classifier-strong"} + p_solve = 0.1 if model == "fake/classifier-strong" else 0.9 + recommended_route = "capable" if model == "fake/classifier-strong" else "efficient" answer = ( - '{"recommended_route":"efficient","p_solve":0.9,' + f'{{"recommended_route":"{recommended_route}","p_solve":{p_solve},' '"confidence":0.95,"abstain":false,' '"capability_boundary":"supported","primary_rule":"SUP-1",' '"crux":"bounded task"}' if classifier else f"chat from {model}" ) + if request.get("stream") and model == "fake/empty-stream": + self._empty_sse() + return if request.get("stream") and model == "fake/late-stream-failure": self._sse_then_disconnect( { @@ -111,9 +119,7 @@ def _chat(self, request: dict[str, object]) -> None: "model": model, "system_fingerprint": "fp_dynamic_plugin", "provider_extension": {"preserved": True}, - "choices": [ - {"index": 0, "delta": {}, "finish_reason": "stop"} - ], + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], }, ] self._sse(events) @@ -191,9 +197,7 @@ def _responses(self, request: dict[str, object]) -> None: "type": "message", "status": "completed", "role": "assistant", - "content": [ - {"type": "output_text", "text": text, "annotations": []} - ], + "content": [{"type": "output_text", "text": text, "annotations": []}], } ], "usage": { @@ -298,6 +302,13 @@ def _sse_then_disconnect(self, first: dict[str, object]) -> None: self.wfile.flush() self.close_connection = True + def _empty_sse(self) -> None: + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.send_header("cache-control", "no-cache") + self.send_header("content-length", "0") + self.end_headers() + def _json(self, status: int, value: object) -> None: data = json.dumps(value).encode() self.send_response(status) diff --git a/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py b/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py index 761cf43f4..db0adb05c 100644 --- a/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py +++ b/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py @@ -45,9 +45,7 @@ def http_json(base: str, path: str) -> dict[str, int]: return cast(dict[str, int], json.loads(response.read())) -def request( - relay_url: str, path: str, body: dict[str, Any] -) -> tuple[int, bytes]: +def request(relay_url: str, path: str, body: dict[str, Any]) -> tuple[int, bytes]: request = urllib.request.Request( f"{relay_url}{path}", data=json.dumps(body).encode(), @@ -82,9 +80,7 @@ def request_until_stream_error( def stream_events(raw: bytes) -> list[dict[str, object]]: return [ - json.loads(line[6:]) - for line in raw.decode().splitlines() - if line.startswith("data: {") + json.loads(line[6:]) for line in raw.decode().splitlines() if line.startswith("data: {") ] @@ -221,9 +217,7 @@ def atof_path(self) -> Path: def __enter__(self) -> RelayScenario: (self.root / ".nemo-relay").mkdir(parents=True) - (self.root / ".nemo-relay" / "plugins.toml").write_text( - self.config, encoding="utf-8" - ) + (self.root / ".nemo-relay" / "plugins.toml").write_text(self.config, encoding="utf-8") subprocess.run( [str(self.relay_bin), "plugins", "enable", "nvidia.switchyard"], cwd=self.root, @@ -249,15 +243,12 @@ def __enter__(self) -> RelayScenario: stderr=subprocess.STDOUT, text=True, ) - threading.Thread( - target=capture, args=(self.process, self.log), daemon=True - ).start() + threading.Thread(target=capture, args=(self.process, self.log), daemon=True).start() deadline = time.time() + 20 while True: if self.process.poll() is not None: raise RuntimeError( - f"Relay exited early ({self.process.returncode}):\n" - + "\n".join(self.log[-40:]) + f"Relay exited early ({self.process.returncode}):\n" + "\n".join(self.log[-40:]) ) try: with urllib.request.urlopen(f"{self.url}/healthz", timeout=1) as response: @@ -277,18 +268,13 @@ def __exit__(self, *_: object) -> None: except subprocess.TimeoutExpired: self.process.kill() self.process.wait() - (self.root / "relay.log").write_text( - "\n".join(self.log) + "\n", encoding="utf-8" - ) + (self.root / "relay.log").write_text("\n".join(self.log) + "\n", encoding="utf-8") if self.process.returncode: raise RuntimeError( - f"Relay exited with {self.process.returncode}:\n" - + "\n".join(self.log[-40:]) + f"Relay exited with {self.process.returncode}:\n" + "\n".join(self.log[-40:]) ) - def marks( - self, name: str, expected: int = 1, timeout: float = 5 - ) -> list[dict[str, object]]: + def marks(self, name: str, expected: int = 1, timeout: float = 5) -> list[dict[str, object]]: deadline = time.time() + timeout while True: try: @@ -345,12 +331,44 @@ def run_same_protocol( assert all(event["provider_extension"] == {"preserved": True} for event in events) assert all(event["system_fingerprint"] == "fp_dynamic_plugin" for event in events) assert len(relay.marks("switchyard.routing.decision", 2)) == 2 - return {"buffered_unknown_fields": True, "stream_events_replayed": len(events)} + replayed = {"openai_chat": len(events)} + for protocol, path, template in CASES[1:]: + config = plugin_config( + manifest, + provider_url, + root / f"same-{protocol}" / "atof", + '[plugins.dynamic.config.algorithm]\nkind = "random"\nseed = 7', + f"""\ +[plugins.dynamic.config.targets.target] +model = "fake/preserve-{protocol}" +protocol = "{protocol}" +base_url = "{{provider_url}}/v1" +weight = 1 +""", + f'{protocol} = "target"', + ) + with RelayScenario(relay_bin, root, provider_url, f"same-{protocol}", config) as scenario: + body = dict(template) + body["stream"] = False + status, raw = request(scenario.url, path, body) + response = json.loads(raw) + assert status == 200 + assert response["provider_extension"] == {"preserved": True} + + body["stream"] = True + status, raw = request(scenario.url, path, body) + events = stream_events(raw) + assert status == 200 + assert events + assert all(event["provider_extension"] == {"preserved": True} for event in events) + assert len(scenario.marks("switchyard.routing.decision", 2)) == 2 + replayed[protocol] = len(events) -def run_random( - relay_bin: Path, root: Path, manifest: Path, provider_url: str -) -> dict[str, object]: + return {"buffered_unknown_fields": True, "stream_events_replayed": replayed} + + +def run_random(relay_bin: Path, root: Path, manifest: Path, provider_url: str) -> dict[str, object]: config = plugin_config( manifest, provider_url, @@ -375,11 +393,7 @@ def run_random( base_url = "{provider_url}/v1" weight = 1 """, - ( - 'openai_chat = "chat"\n' - 'openai_responses = "responses"\n' - 'anthropic_messages = "anthropic"' - ), + ('openai_chat = "chat"\nopenai_responses = "responses"\nanthropic_messages = "anthropic"'), ) with RelayScenario(relay_bin, root, provider_url, "random", config) as relay: models: set[str] = set() @@ -433,13 +447,16 @@ def concurrent_call(index: int) -> str: } -def run_classifier( - relay_bin: Path, root: Path, manifest: Path, provider_url: str -) -> dict[str, object]: - config = plugin_config( +def classifier_config( + manifest: Path, + provider_url: str, + atof: Path, + classifier_model: str, +) -> str: + return plugin_config( manifest, provider_url, - root / "classifier" / "atof", + atof, """\ [plugins.dynamic.config.algorithm] kind = "llm_classifier" @@ -451,52 +468,116 @@ def run_classifier( session_affinity = false message_hash_fallback = false """, - """\ + f"""\ [plugins.dynamic.config.targets.classifier] -model = "fake/classifier" +model = "{classifier_model}" protocol = "openai_chat" -base_url = "{provider_url}/v1" +base_url = "{{provider_url}}/v1" [plugins.dynamic.config.targets.weak] model = "fake/weak" protocol = "openai_responses" -base_url = "{provider_url}/v1" +base_url = "{{provider_url}}/v1" [plugins.dynamic.config.targets.strong] model = "fake/strong" protocol = "anthropic_messages" -base_url = "{provider_url}/v1" +base_url = "{{provider_url}}/v1" [plugins.dynamic.config.targets.fallback] model = "fake/fallback" protocol = "openai_chat" -base_url = "{provider_url}/v1" +base_url = "{{provider_url}}/v1" """, - 'openai_chat = "fallback"', + ('openai_chat = "fallback"\nopenai_responses = "weak"\nanthropic_messages = "strong"'), ) - with RelayScenario(relay_bin, root, provider_url, "classifier", config) as relay: + + +def run_classifier( + relay_bin: Path, root: Path, manifest: Path, provider_url: str +) -> dict[str, object]: + weak_config = classifier_config( + manifest, + provider_url, + root / "classifier-weak" / "atof", + "fake/classifier", + ) + with RelayScenario(relay_bin, root, provider_url, "classifier-weak", weak_config) as weak_relay: + for protocol, path, template in CASES: + body = dict(template) + body["stream"] = False + status, raw = request(weak_relay.url, path, body) + response = json.loads(raw) + assert status == 200 + assert response["model"] == "fake/weak" + assert response_text(protocol, response) == "responses from fake/weak" + + body["stream"] = True + status, raw = request(weak_relay.url, path, body) + events = stream_events(raw) + assert status == 200 + assert stream_text(protocol, events) == "responses from fake/weak" + + def concurrent_classifier_call(index: int) -> str: + protocol, path, template = CASES[index % len(CASES)] + body = dict(template) + body["stream"] = False + status, raw = request(weak_relay.url, path, body) + response = json.loads(raw) + assert status == 200 + assert response["model"] == "fake/weak" + return response_text(protocol, response) + + with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor: + concurrent_results = list(executor.map(concurrent_classifier_call, range(6))) + assert all(result == "responses from fake/weak" for result in concurrent_results) + + weak_decisions = weak_relay.marks("switchyard.routing.decision", 12) + assert len(weak_decisions) == 12 + assert all( + event["data"]["algorithm"] == "llm_task_classifier" # type: ignore[index] + and event["data"]["selected_target"] == "weak" # type: ignore[index] + and event["data"]["routing_tier"] == "weak" # type: ignore[index] + for event in weak_decisions + ) + + strong_config = classifier_config( + manifest, + provider_url, + root / "classifier-strong" / "atof", + "fake/classifier-strong", + ) + with RelayScenario( + relay_bin, root, provider_url, "classifier-strong", strong_config + ) as strong_relay: body = dict(CASES[0][2]) body["stream"] = False - status, raw = request(relay.url, CASES[0][1], body) + status, raw = request(strong_relay.url, CASES[0][1], body) response = json.loads(raw) assert status == 200 - assert response["model"] == "fake/weak" + assert response["model"] == "fake/strong" + assert response_text("openai_chat", response) == "anthropic from fake/strong" body["stream"] = True - status, raw = request(relay.url, CASES[0][1], body) + status, raw = request(strong_relay.url, CASES[0][1], body) events = stream_events(raw) assert status == 200 - assert "responses from fake/weak" == stream_text("openai_chat", events) + assert stream_text("openai_chat", events) == "anthropic from fake/strong" - decisions = relay.marks("switchyard.routing.decision", 2) - assert len(decisions) == 2 + strong_decisions = strong_relay.marks("switchyard.routing.decision", 2) + assert len(strong_decisions) == 2 assert all( event["data"]["algorithm"] == "llm_task_classifier" # type: ignore[index] - and event["data"]["selected_target"] == "weak" # type: ignore[index] - and event["data"]["routing_tier"] == "weak" # type: ignore[index] - for event in decisions + and event["data"]["selected_target"] == "strong" # type: ignore[index] + and event["data"]["routing_tier"] == "strong" # type: ignore[index] + for event in strong_decisions ) - return {"selected_target": "weak", "decisions": len(decisions)} + return { + "weak_decisions": len(weak_decisions), + "strong_decisions": len(strong_decisions), + "protocols": [protocol for protocol, _, _ in CASES], + "concurrent_calls": len(concurrent_results), + } def single_target_config( @@ -529,6 +610,43 @@ def single_target_config( ) +def reselection_config( + manifest: Path, + provider_url: str, + atof: Path, + failing_model: str, + succeeding_model: str, + fallback_model: str, +) -> str: + return plugin_config( + manifest, + provider_url, + atof, + '[plugins.dynamic.config.algorithm]\nkind = "random"\nseed = 6', + f"""\ +[plugins.dynamic.config.targets.a_fail] +model = "{failing_model}" +protocol = "openai_chat" +base_url = "{{provider_url}}/v1" +weight = 1 + +[plugins.dynamic.config.targets.b_success] +model = "{succeeding_model}" +protocol = "openai_chat" +base_url = "{{provider_url}}/v1" +weight = 1 + +[plugins.dynamic.config.targets.z_fallback] +model = "{fallback_model}" +protocol = "openai_chat" +base_url = "{{provider_url}}/v1" +weight = 0 +""", + 'openai_chat = "z_fallback"', + max_retries=1, + ) + + def run_retry_and_fallback( relay_bin: Path, root: Path, manifest: Path, provider_url: str ) -> dict[str, object]: @@ -562,9 +680,7 @@ def run_retry_and_fallback( "fake/always-fail", "fake/trusted-fallback", ) - with RelayScenario( - relay_bin, root, provider_url, "fallback", fallback_config - ) as relay: + with RelayScenario(relay_bin, root, provider_url, "fallback", fallback_config) as relay: body = dict(CASES[0][2]) body["stream"] = False status, raw = request(relay.url, CASES[0][1], body) @@ -580,14 +696,40 @@ def run_retry_and_fallback( } assert len(relay.marks("switchyard.routing.fallback")) == 1 + reselection = reselection_config( + manifest, + provider_url, + root / "retry-reselection" / "atof", + "fake/reselect-fail", + "fake/reselect-success", + "fake/reselect-fallback", + ) + with RelayScenario(relay_bin, root, provider_url, "retry-reselection", reselection) as relay: + body = dict(CASES[0][2]) + body["stream"] = False + status, raw = request(relay.url, CASES[0][1], body) + assert status == 200 + assert json.loads(raw)["model"] == "fake/reselect-success" + decisions = relay.marks("switchyard.routing.decision", 2) + assert [event["data"]["selected_target"] for event in decisions] == [ # type: ignore[index] + "a_fail", + "b_success", + ] + assert len(relay.marks("switchyard.routing.retry")) == 1 + assert not relay.marks("switchyard.routing.fallback", expected=0) + calls = http_json(provider_url, "/calls") assert calls["fake/retry-once"] == 2 assert calls.get("fake/retry-fallback", 0) == 0 assert calls["fake/always-fail"] == 1 assert calls["fake/trusted-fallback"] == 1 + assert calls["fake/reselect-fail"] == 1 + assert calls["fake/reselect-success"] == 1 + assert calls.get("fake/reselect-fallback", 0) == 0 return { "retry_attempts": calls["fake/retry-once"], "fallback_calls": calls["fake/trusted-fallback"], + "retry_reselected": ["a_fail", "b_success"], } @@ -663,14 +805,12 @@ def run_stream_reliability( with RelayScenario(relay_bin, root, provider_url, "stream-late-failure", late_config) as relay: body = dict(CASES[0][2]) body["stream"] = True - status, raw, saw_stream_error = request_until_stream_error( - relay.url, CASES[0][1], body - ) - events = stream_events(raw) + status, raw, saw_stream_error = request_until_stream_error(relay.url, CASES[0][1], body) + late_events = stream_events(raw) assert status == 200 assert saw_stream_error - assert len(events) == 1 - assert stream_text("openai_chat", events) == "committed before failure" + assert len(late_events) == 1 + assert stream_text("openai_chat", late_events) == "committed before failure" assert "data: [DONE]" not in raw.decode() late_error_marks = relay.marks("switchyard.routing.error") assert len(late_error_marks) == 1 @@ -685,6 +825,57 @@ def run_stream_reliability( assert len(relay.marks("switchyard.routing.requested")) == 1 assert len(relay.marks("switchyard.routing.decision")) == 1 + empty_config = single_target_config( + manifest, + provider_url, + root / "stream-empty" / "atof", + "fake/empty-stream", + "fake/empty-stream-fallback", + max_retries=1, + ) + with RelayScenario(relay_bin, root, provider_url, "stream-empty", empty_config) as relay: + body = dict(CASES[0][2]) + body["stream"] = True + status, raw = request(relay.url, CASES[0][1], body) + events = stream_events(raw) + assert status == 200 + assert stream_text("openai_chat", events) == "chat from fake/empty-stream-fallback" + empty_error_marks = relay.marks("switchyard.routing.error") + assert len(empty_error_marks) == 1 + assert empty_error_marks[0]["data"] == { + "attempt": 1, + "retryable": False, + "failure_kind": "non_http", + "non_http_kind": "invalid_response", + } + assert len(relay.marks("switchyard.routing.fallback")) == 1 + assert not relay.marks("switchyard.routing.retry", expected=0) + + stream_reselection = reselection_config( + manifest, + provider_url, + root / "stream-reselection" / "atof", + "fake/reselect-stream-fail", + "fake/reselect-stream-success", + "fake/reselect-stream-fallback", + ) + with RelayScenario( + relay_bin, root, provider_url, "stream-reselection", stream_reselection + ) as relay: + body = dict(CASES[0][2]) + body["stream"] = True + status, raw = request(relay.url, CASES[0][1], body) + events = stream_events(raw) + assert status == 200 + assert stream_text("openai_chat", events) == "chat from fake/reselect-stream-success" + decisions = relay.marks("switchyard.routing.decision", 2) + assert [event["data"]["selected_target"] for event in decisions] == [ # type: ignore[index] + "a_fail", + "b_success", + ] + assert len(relay.marks("switchyard.routing.retry")) == 1 + assert not relay.marks("switchyard.routing.fallback", expected=0) + calls = http_json(provider_url, "/calls") assert calls["fake/retry-stream-once"] == 2 assert calls.get("fake/retry-stream-fallback", 0) == 0 @@ -692,11 +883,18 @@ def run_stream_reliability( assert calls["fake/trusted-stream-fallback"] == 1 assert calls["fake/late-stream-failure"] == 1 assert calls.get("fake/late-stream-fallback", 0) == 0 + assert calls["fake/empty-stream"] == 1 + assert calls["fake/empty-stream-fallback"] == 1 + assert calls["fake/reselect-stream-fail"] == 1 + assert calls["fake/reselect-stream-success"] == 1 + assert calls.get("fake/reselect-stream-fallback", 0) == 0 return { "retry_attempts": calls["fake/retry-stream-once"], "fallback_calls": calls["fake/trusted-stream-fallback"], - "late_events_before_failure": len(events), + "late_events_before_failure": len(late_events), "late_error_marks": len(late_error_marks), + "empty_stream_fallback_calls": calls["fake/empty-stream-fallback"], + "retry_reselected": ["a_fail", "b_success"], } @@ -830,9 +1028,7 @@ def main() -> None: except (OSError, urllib.error.URLError): pass if provider.poll() is not None: - raise RuntimeError( - "fake provider exited early:\n" + "\n".join(provider_log[-40:]) - ) + raise RuntimeError("fake provider exited early:\n" + "\n".join(provider_log[-40:])) if time.time() > deadline: raise TimeoutError("fake provider did not become healthy") time.sleep(0.05) @@ -841,9 +1037,7 @@ def main() -> None: "same_protocol_preservation": run_same_protocol( relay_bin, root, bundle / "relay-plugin.toml", provider_url ), - "random": run_random( - relay_bin, root, bundle / "relay-plugin.toml", provider_url - ), + "random": run_random(relay_bin, root, bundle / "relay-plugin.toml", provider_url), "llm_classifier": run_classifier( relay_bin, root, bundle / "relay-plugin.toml", provider_url ), From cd8b448549fd75d902666fb5a2eaedede5dae388 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 15:58:25 -0600 Subject: [PATCH 29/51] refactor(plugin): trim dynamic plugin surface Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 +- .../switchyard-nemo-relay-plugin/Cargo.toml | 4 +- .../src/config.rs | 61 ++++++++++--------- .../src/runtime.rs | 13 ++-- 4 files changed, 42 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b6517de7..81a8750d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=1580e22b33dc727710cf0a42e23ce6eeb7936dfe#1580e22b33dc727710cf0a42e23ce6eeb7936dfe" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=8c1d724b17be278d5506ab550b154836577aad72#8c1d724b17be278d5506ab550b154836577aad72" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=1580e22b33dc727710cf0a42e23ce6eeb7936dfe#1580e22b33dc727710cf0a42e23ce6eeb7936dfe" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=8c1d724b17be278d5506ab550b154836577aad72#8c1d724b17be278d5506ab550b154836577aad72" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index c61cf2602..488e411b4 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -13,13 +13,13 @@ rust-version.workspace = true publish = false [lib] -crate-type = ["cdylib", "rlib"] +crate-type = ["cdylib"] [dependencies] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "1580e22b33dc727710cf0a42e23ce6eeb7936dfe" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "8c1d724b17be278d5506ab550b154836577aad72" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index f247cca81..358f13ffe 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -29,22 +29,22 @@ const fn default_endpoint(protocol: WireFormat) -> &'static str { } #[derive(Deserialize)] -pub struct TargetBinding { - pub model: String, - pub protocol: WireFormat, +struct TargetBinding { + model: String, + protocol: WireFormat, #[serde(default)] - pub endpoint: String, - pub base_url: String, + endpoint: String, + base_url: String, #[serde(default = "default_weight")] - pub weight: f64, + weight: f64, #[serde(default)] - pub headers: BTreeMap, + headers: BTreeMap, #[serde(default)] - pub header_env: BTreeMap, + header_env: BTreeMap, } impl TargetBinding { - pub fn dispatch_url(&self) -> String { + fn dispatch_url(&self) -> String { let base = self.base_url.trim_end_matches('/'); let endpoint = if self.endpoint.is_empty() { default_endpoint(self.protocol) @@ -101,22 +101,22 @@ impl TargetBinding { } } -pub struct PreparedTargetBinding { - pub model: String, - pub protocol: WireFormat, +pub(crate) struct PreparedTargetBinding { + pub(crate) model: String, + pub(crate) protocol: WireFormat, dispatch_url: String, - pub headers: BTreeMap, + pub(crate) headers: BTreeMap, } impl PreparedTargetBinding { - pub fn dispatch_url(&self) -> &str { + pub(crate) fn dispatch_url(&self) -> &str { &self.dispatch_url } } #[derive(Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] -pub enum AlgorithmConfig { +enum AlgorithmConfig { Random { #[serde(default)] seed: Option, @@ -139,26 +139,26 @@ pub enum AlgorithmConfig { #[derive(Deserialize)] #[serde(deny_unknown_fields)] -pub struct SwitchyardConfig { - pub version: u32, +pub(crate) struct SwitchyardConfig { + version: u32, #[serde(default)] - pub priority: i32, + pub(crate) priority: i32, #[serde(default = "default_max_retries")] - pub max_retries: u32, - pub algorithm: AlgorithmConfig, - pub targets: BTreeMap, - pub default_targets: BTreeMap, + max_retries: u32, + algorithm: AlgorithmConfig, + targets: BTreeMap, + default_targets: BTreeMap, } pub(crate) struct PreparedConfig { - pub max_retries: u32, - pub algorithm: Arc, - pub targets: BTreeMap, - pub default_targets: BTreeMap, + pub(crate) max_retries: u32, + pub(crate) algorithm: Arc, + pub(crate) targets: BTreeMap, + pub(crate) default_targets: BTreeMap, } impl SwitchyardConfig { - pub fn validate(&self) -> Result<(), String> { + pub(crate) fn validate(&self) -> Result<(), String> { self.validate_structure()?; self.build_algorithm().map(drop) } @@ -239,8 +239,11 @@ impl SwitchyardConfig { let targets = self .targets .keys() - .map(|name| target(name)) - .collect::, _>>()?; + .map(|name| LlmTarget { + semantic_name: name.clone(), + llm_client: None, + }) + .collect(); let weights = self .targets .values() diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index bdfdda4f9..ef3ff9e2e 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -23,7 +23,7 @@ use switchyard_translation::{StreamTranslationState, TranslationEngine}; use crate::config::{protocol_from_call, PreparedTargetBinding, SwitchyardConfig}; use crate::translation; -pub struct SwitchyardRuntime { +pub(crate) struct SwitchyardRuntime { max_retries: u32, algorithm: Arc, targets: BTreeMap, @@ -33,7 +33,7 @@ pub struct SwitchyardRuntime { } impl SwitchyardRuntime { - pub fn new(config: SwitchyardConfig, relay: PluginRuntime) -> Result { + pub(crate) fn new(config: SwitchyardConfig, relay: PluginRuntime) -> Result { let prepared = config.prepare()?; Ok(Self { max_retries: prepared.max_retries, @@ -45,7 +45,7 @@ impl SwitchyardRuntime { }) } - pub async fn execute_buffered( + pub(crate) async fn execute_buffered( &self, name: String, request: RelayRequest, @@ -260,7 +260,7 @@ impl SwitchyardRuntime { ); } - pub async fn execute_stream( + pub(crate) async fn execute_stream( self: Arc, name: String, request: RelayRequest, @@ -585,10 +585,9 @@ fn libsy_error_retryable(error: &LibsyError) -> bool { }; match source { LlmClientError::UpstreamHttp { status, .. } => { - LlmContinuationFailureV2::http_status_is_retryable(*status) + matches!(*status, 408 | 425 | 429 | 500 | 502 | 503 | 504) } - LlmClientError::Transport { .. } => LlmNonHttpFailureKindV2::Transport.is_retryable(), - LlmClientError::Timeout { .. } => LlmNonHttpFailureKindV2::Timeout.is_retryable(), + LlmClientError::Transport { .. } | LlmClientError::Timeout { .. } => true, _ => false, } } From 642599f175fcc0a5e7e34e1a242dc82c708dec69 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 16:23:37 -0600 Subject: [PATCH 30/51] refactor(plugin): tighten dynamic plugin contract Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 ++-- .../switchyard-nemo-relay-plugin/Cargo.toml | 2 +- .../config.schema.json | 8 ++++---- .../scripts/package_bundle.py | 4 ++++ .../src/config.rs | 20 +++++++++++++++++++ .../src/translation.rs | 14 ++++++------- 6 files changed, 38 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 81a8750d2..b740a47f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=8c1d724b17be278d5506ab550b154836577aad72#8c1d724b17be278d5506ab550b154836577aad72" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=9efde25b0894721852a10c3c6e127bab1bf87e77#9efde25b0894721852a10c3c6e127bab1bf87e77" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=8c1d724b17be278d5506ab550b154836577aad72#8c1d724b17be278d5506ab550b154836577aad72" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=9efde25b0894721852a10c3c6e127bab1bf87e77#9efde25b0894721852a10c3c6e127bab1bf87e77" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 488e411b4..b89583595 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "8c1d724b17be278d5506ab550b154836577aad72" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "9efde25b0894721852a10c3c6e127bab1bf87e77" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/config.schema.json b/crates/switchyard-nemo-relay-plugin/config.schema.json index 14d2a84ef..242a7b127 100644 --- a/crates/switchyard-nemo-relay-plugin/config.schema.json +++ b/crates/switchyard-nemo-relay-plugin/config.schema.json @@ -70,7 +70,7 @@ "protocol": { "enum": ["openai_chat", "openai_responses", "anthropic_messages"] }, - "endpoint": { "type": "string" }, + "endpoint": { "type": "string", "pattern": "^$|^/" }, "base_url": { "type": "string", "pattern": "^https?://" @@ -94,9 +94,9 @@ "minProperties": 1, "additionalProperties": false, "properties": { - "openai_chat": { "type": "string" }, - "openai_responses": { "type": "string" }, - "anthropic_messages": { "type": "string" } + "openai_chat": { "type": "string", "minLength": 1 }, + "openai_responses": { "type": "string", "minLength": 1 }, + "anthropic_messages": { "type": "string", "minLength": 1 } } } } diff --git a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py index 04c9347c6..2a64aad6f 100644 --- a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py +++ b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py @@ -33,6 +33,10 @@ def main() -> None: parser.error(f"compiled plugin library does not exist: {library}") output = args.output.resolve() + if output.exists() and not output.is_dir(): + parser.error(f"bundle output exists and is not a directory: {output}") + if output.is_dir() and any(output.iterdir()): + parser.error(f"bundle output directory must be empty: {output}") output.mkdir(parents=True, exist_ok=True) artifact = output / library.name shutil.copy2(library, artifact) diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index 358f13ffe..29aa93754 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -186,6 +186,11 @@ impl SwitchyardConfig { if !target.base_url.starts_with("http://") && !target.base_url.starts_with("https://") { return Err(format!("target {name:?} base_url must use http or https")); } + if !target.endpoint.is_empty() && !target.endpoint.starts_with('/') { + return Err(format!( + "target {name:?} endpoint must be empty or begin with '/'" + )); + } if !target.weight.is_finite() || target.weight < 0.0 { return Err(format!( "target {name:?} weight must be finite and nonnegative" @@ -367,6 +372,21 @@ mod tests { assert!(error.contains("version = 2")); } + #[test] + fn target_endpoints_are_empty_or_begin_with_a_slash() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().endpoint = "/custom/chat".into(); + config.validate().unwrap(); + assert_eq!( + config.targets["chat"].dispatch_url(), + "https://provider.example/v1/custom/chat" + ); + + config.targets.get_mut("chat").unwrap().endpoint = "v1/chat/completions".into(); + let error = config.validate().unwrap_err(); + assert!(error.contains("endpoint must be empty or begin with '/'")); + } + #[test] fn default_target_keys_define_the_managed_protocols() { let mut config = config(); diff --git a/crates/switchyard-nemo-relay-plugin/src/translation.rs b/crates/switchyard-nemo-relay-plugin/src/translation.rs index 3123c507d..94d0eb888 100644 --- a/crates/switchyard-nemo-relay-plugin/src/translation.rs +++ b/crates/switchyard-nemo-relay-plugin/src/translation.rs @@ -10,7 +10,7 @@ use switchyard_translation::{ TranslationPolicy, UnknownFieldPolicy, }; -pub fn decode_request( +pub(crate) fn decode_request( engine: &TranslationEngine, protocol: WireFormat, request: &RelayRequest, @@ -22,7 +22,7 @@ pub fn decode_request( Ok(output.request) } -pub fn encode_request( +pub(crate) fn encode_request( engine: &TranslationEngine, protocol: WireFormat, request: &LlmRequest, @@ -37,7 +37,7 @@ pub fn encode_request( }) } -pub fn decode_response( +pub(crate) fn decode_response( engine: &TranslationEngine, protocol: WireFormat, response: &Json, @@ -49,7 +49,7 @@ pub fn decode_response( Ok(output.response) } -pub fn encode_response( +pub(crate) fn encode_response( engine: &TranslationEngine, protocol: WireFormat, response: &AggLlmResponse, @@ -61,7 +61,7 @@ pub fn encode_response( Ok(output.body) } -pub fn decode_stream_event( +pub(crate) fn decode_stream_event( engine: &TranslationEngine, state: &mut StreamTranslationState, protocol: WireFormat, @@ -72,7 +72,7 @@ pub fn decode_stream_event( .map_err(error) } -pub fn encode_stream_event( +pub(crate) fn encode_stream_event( engine: &TranslationEngine, state: &mut StreamTranslationState, protocol: WireFormat, @@ -83,7 +83,7 @@ pub fn encode_stream_event( .map_err(error) } -pub fn finish_stream( +pub(crate) fn finish_stream( engine: &TranslationEngine, state: &mut StreamTranslationState, protocol: WireFormat, From 55456f9001124829c9a9d28336fba44edeaeb1bd Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 16:23:45 -0600 Subject: [PATCH 31/51] docs(plugin): clarify dynamic routing contract Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/README.md | 61 +++++++++++-------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index d4a89bd32..faf770f6b 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -30,9 +30,10 @@ enter plugin-local Tokio state across the dynamic-library boundary. libsy's `run_stream` and driver response timeout are therefore poll-driven rather than calling `tokio::spawn` or `tokio::time::timeout`. -The crate is a source/build unit and is not published to crates.io. Operators -install a release bundle containing the compiled shared library, materialized -`relay-plugin.toml`, `config.schema.json`, licensing files, and checksum. +The crate is a `cdylib`-only source/build unit and is not published to +crates.io. Operators install a release bundle containing the compiled shared +library, materialized `relay-plugin.toml`, `config.schema.json`, licensing +files, and checksum. During development, `nemo-relay-plugin` is pinned to the Relay native API v2 feature commit. Native API v2 remains unreleased, so every bundle must be @@ -55,8 +56,9 @@ For every managed LLM call, the plugin: 7. translates `ReturnToAgent` back to the caller protocol. Switchyard owns routing, translation, target URLs, and target credentials. -Relay validates and transports the selected HTTP target, runs it through the -captured LLM continuation, and owns stream transport and event export. +Each continuation target is an absolute HTTP(S) URL plus headers, and Relay +dispatches it with HTTP `POST` through the captured LLM continuation. Relay +validates the target and owns stream transport and event export. Switchyard retries or falls back only before the first caller event; after commitment, a late provider failure is returned without retry. Target URLs, transport headers, and credentials never enter `LlmRequest.headers`, marks, or @@ -64,14 +66,19 @@ spans; semantic target names remain visible in genuine routing marks. The plugin contains no Relay provider codecs and does not use private dispatch headers. -Calls outside the enabled profiles return the SDK's explicit `Passthrough` -outcome. Relay then forwards the downstream provider stream through its bounded -host queue; unmanaged provider events do not cross the plugin ABI. - -Provider failures use HTTP semantics: status, a bounded body, and safe response -headers when Relay received an HTTP response; otherwise a transport, timeout, -cancelled, invalid-request, guardrail, or internal kind. Relay reports those -neutral failure facts; the Switchyard plugin owns retry and fallback policy. +Calls whose protocol is absent from `default_targets` return the SDK's explicit +`Passthrough` outcome. Relay then forwards the downstream provider stream +through its bounded host queue. These ordinary untargeted calls remain inside +Relay's managed LLM lifecycle, but their provider events do not cross the +plugin ABI. Targeted provider streams permit at most one pending pull; plugin +output and direct pass-through use bounded 32-event host queues. + +Provider failures use HTTP semantics. Relay supplies status, a bounded body, +and safe response headers when it received an HTTP response; Switchyard passes +the status and body to libsy but does not currently use the response headers. +Failures without an HTTP response use a transport, timeout, cancelled, +invalid-request, guardrail, or internal kind. The Switchyard plugin alone owns +retry and fallback policy. HTTP 408, 425, 429, 500, 502, 503, and 504 plus transport and timeout failures retry. The plugin does not inspect provider bodies to reclassify HTTP 400 context-window or HTTP 404 model errors. @@ -97,20 +104,13 @@ manifest = "/opt/switchyard-relay-plugin/relay-plugin.toml" version = 2 priority = 0 max_retries = 3 -enabled_inbound_profiles = [ - "openai_chat", - "openai_responses", - "anthropic_messages", -] [plugins.dynamic.config.algorithm] kind = "random" seed = 42 [plugins.dynamic.config.default_targets] -openai_chat = "chat-default" -openai_responses = "responses-default" -anthropic_messages = "anthropic-default" +openai_chat = "fast" [plugins.dynamic.config.targets.fast] model = "provider/model" @@ -125,8 +125,9 @@ authorization = "PROVIDER_AUTHORIZATION" Target map keys such as `fast` are the semantic model names exposed to libsy. The target binding remains authoritative for the provider model, protocol, URL, -and headers. `header_env` resolves credentials in the plugin process without -putting them in configuration or libsy metadata. +and headers. Each `default_targets` key both enables that inbound protocol and +names its trusted fallback. `header_env` resolves credentials in the plugin +process without putting them in configuration or libsy metadata. Version-1 service configuration is rejected with a migration error. The plugin does not provide decision-only or observe-only execution. @@ -170,13 +171,19 @@ python3 crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py \ It launches a local three-protocol fake provider and real Relay process. The test covers: -- exact same-protocol unknown-field and raw-stream replay; +- same-protocol preservation of unknown buffered and raw stream-event fields + across all three protocols; - isolated target credentials and headers without source-header inheritance; - buffered and streaming OpenAI Chat, OpenAI Responses, and Anthropic Messages routes; - cross-protocol request/response translation; - 12 concurrent independent random-router calls; - genuine requested and decision marks; -- an LLM-classifier call followed by its selected provider call; -- a retryable provider failure with a fresh run; and -- non-retryable failure with exactly-once trusted fallback. +- LLM-classifier weak and strong selections followed by their provider calls; +- buffered and streaming retry reselection plus exactly-once trusted fallback; +- empty-stream fallback and a committed late error with no retry; +- untargeted buffered and streaming pass-through with no Switchyard marks; and +- target credential replacement without recording credential values. + +Translation unit tests separately require exact parsed-JSON replay for +same-protocol raw stream events. From 6d2a155d7388e4c62d1ef93d14f0821a766582ae Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 16:44:02 -0600 Subject: [PATCH 32/51] docs(plugin): define managed call boundary Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index faf770f6b..b1850cee9 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -66,12 +66,12 @@ spans; semantic target names remain visible in genuine routing marks. The plugin contains no Relay provider codecs and does not use private dispatch headers. -Calls whose protocol is absent from `default_targets` return the SDK's explicit -`Passthrough` outcome. Relay then forwards the downstream provider stream -through its bounded host queue. These ordinary untargeted calls remain inside -Relay's managed LLM lifecycle, but their provider events do not cross the -plugin ABI. Targeted provider streams permit at most one pending pull; plugin -output and direct pass-through use bounded 32-event host queues. +Only supported LLM execution names whose mapped protocol appears in +`default_targets` are managed. Every other buffered or streaming call uses the +SDK's explicit `Passthrough` path. These ordinary untargeted calls remain +inside Relay's managed LLM lifecycle, but their provider events do not cross +the plugin ABI. Targeted provider streams permit at most one pending pull; +plugin output and direct pass-through use bounded 32-event host queues. Provider failures use HTTP semantics. Relay supplies status, a bounded body, and safe response headers when it received an HTTP response; Switchyard passes From 716cee46046a400a1a03bcd5214f7a11c86a95c2 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 16:50:32 -0600 Subject: [PATCH 33/51] chore(plugin): pin final Relay v2 revision Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 ++-- crates/switchyard-nemo-relay-plugin/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b740a47f8..676d6ed71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=9efde25b0894721852a10c3c6e127bab1bf87e77#9efde25b0894721852a10c3c6e127bab1bf87e77" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=4bf99d15cf83d6bf015ee31def8891a6543ace12#4bf99d15cf83d6bf015ee31def8891a6543ace12" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=9efde25b0894721852a10c3c6e127bab1bf87e77#9efde25b0894721852a10c3c6e127bab1bf87e77" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=4bf99d15cf83d6bf015ee31def8891a6543ace12#4bf99d15cf83d6bf015ee31def8891a6543ace12" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index b89583595..db8550825 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "9efde25b0894721852a10c3c6e127bab1bf87e77" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "4bf99d15cf83d6bf015ee31def8891a6543ace12" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true From 6ae2ae1349c68e81a1b00090a023def7d83beb98 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 17:08:02 -0600 Subject: [PATCH 34/51] chore(plugin): pin bounded Relay v2 transport Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 ++-- crates/switchyard-nemo-relay-plugin/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 676d6ed71..578b56154 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=4bf99d15cf83d6bf015ee31def8891a6543ace12#4bf99d15cf83d6bf015ee31def8891a6543ace12" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=bfb06e06cc8ab5d5e277748c1446ec016c2bd5c1#bfb06e06cc8ab5d5e277748c1446ec016c2bd5c1" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=4bf99d15cf83d6bf015ee31def8891a6543ace12#4bf99d15cf83d6bf015ee31def8891a6543ace12" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=bfb06e06cc8ab5d5e277748c1446ec016c2bd5c1#bfb06e06cc8ab5d5e277748c1446ec016c2bd5c1" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index db8550825..56a6333ff 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "4bf99d15cf83d6bf015ee31def8891a6543ace12" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "bfb06e06cc8ab5d5e277748c1446ec016c2bd5c1" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true From 3568275f396e54c7946ae51680831add079334a6 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 17:36:25 -0600 Subject: [PATCH 35/51] refactor(plugin): reuse libsy classifier config Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 +- .../switchyard-nemo-relay-plugin/Cargo.toml | 2 +- crates/switchyard-nemo-relay-plugin/README.md | 5 ++ .../config.schema.json | 4 ++ .../src/config.rs | 61 ++++++++----------- 5 files changed, 36 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 578b56154..915ccba66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=bfb06e06cc8ab5d5e277748c1446ec016c2bd5c1#bfb06e06cc8ab5d5e277748c1446ec016c2bd5c1" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=cec9de823047344d722d7d53ce4722e0d2aa22f9#cec9de823047344d722d7d53ce4722e0d2aa22f9" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=bfb06e06cc8ab5d5e277748c1446ec016c2bd5c1#bfb06e06cc8ab5d5e277748c1446ec016c2bd5c1" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=cec9de823047344d722d7d53ce4722e0d2aa22f9#cec9de823047344d722d7d53ce4722e0d2aa22f9" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 56a6333ff..084c49eb1 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "bfb06e06cc8ab5d5e277748c1446ec016c2bd5c1" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "cec9de823047344d722d7d53ce4722e0d2aa22f9" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index b1850cee9..dd555e22d 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -129,6 +129,11 @@ and headers. Each `default_targets` key both enables that inbound protocol and names its trusted fallback. `header_env` resolves credentials in the plugin process without putting them in configuration or libsy metadata. +For `kind = "llm_classifier"`, the classifier thresholds, affinity options, and +`recent_turn_window` use libsy's `TaskClassifierConfig` directly. The plugin +adds only the semantic `classifier_target`, `weak_target`, and `strong_target` +bindings required to resolve Relay continuations. + Version-1 service configuration is rejected with a migration error. The plugin does not provide decision-only or observe-only execution. diff --git a/crates/switchyard-nemo-relay-plugin/config.schema.json b/crates/switchyard-nemo-relay-plugin/config.schema.json index 242a7b127..5a35d7614 100644 --- a/crates/switchyard-nemo-relay-plugin/config.schema.json +++ b/crates/switchyard-nemo-relay-plugin/config.schema.json @@ -52,6 +52,10 @@ "minimum": 0, "maximum": 1 }, + "recent_turn_window": { + "type": ["integer", "null"], + "minimum": 0 + }, "session_affinity": { "type": "boolean", "default": false }, "message_hash_fallback": { "type": "boolean", "default": false } } diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index 29aa93754..5688ced98 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -125,15 +125,8 @@ enum AlgorithmConfig { classifier_target: String, weak_target: String, strong_target: String, - base_threshold: f64, - #[serde(default)] - min_confidence: f64, - #[serde(default)] - capability_elevated_floor: Option, - #[serde(default)] - session_affinity: bool, - #[serde(default)] - message_hash_fallback: bool, + #[serde(flatten)] + config: TaskClassifierConfig, }, } @@ -262,23 +255,12 @@ impl SwitchyardConfig { classifier_target, weak_target, strong_target, - base_threshold, - min_confidence, - capability_elevated_floor, - session_affinity, - message_hash_fallback, + config, } => LlmTaskClassifier::new( target(classifier_target)?, target(weak_target)?, target(strong_target)?, - TaskClassifierConfig { - base_threshold: *base_threshold, - min_confidence: *min_confidence, - capability_elevated_floor: *capability_elevated_floor, - session_affinity: *session_affinity, - message_hash_fallback: *message_hash_fallback, - recent_turn_window: None, - }, + config.clone(), ) .map(|algorithm| Arc::new(algorithm) as Arc) .map_err(|error| error.to_string()), @@ -468,18 +450,24 @@ mod tests { } #[test] - fn classifier_targets_are_semantic_names_not_provider_models() { + fn classifier_reuses_libsy_configuration_with_semantic_targets() { let mut config = config(); - config.algorithm = AlgorithmConfig::LlmClassifier { - classifier_target: "chat".into(), - weak_target: "responses".into(), - strong_target: "anthropic".into(), - base_threshold: 0.5, - min_confidence: 0.0, - capability_elevated_floor: None, - session_affinity: false, - message_hash_fallback: false, + config.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "base_threshold": 0.5, + "recent_turn_window": 4 + })) + .unwrap(); + let AlgorithmConfig::LlmClassifier { + config: classifier, .. + } = &config.algorithm + else { + panic!("expected classifier configuration"); }; + assert_eq!(classifier.recent_turn_window, Some(4)); config.validate().unwrap(); assert_eq!( config.prepare().unwrap().algorithm.name(), @@ -519,11 +507,10 @@ mod tests { classifier_target: "chat".into(), weak_target: "responses".into(), strong_target: "anthropic".into(), - base_threshold: 1.1, - min_confidence: 0.0, - capability_elevated_floor: None, - session_affinity: false, - message_hash_fallback: false, + config: TaskClassifierConfig { + base_threshold: 1.1, + ..Default::default() + }, }; assert!(classifier .validate() From a3867655736c2dec276deaa572d7ab5735848b3e Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 17:53:38 -0600 Subject: [PATCH 36/51] refactor(plugin): remove draft retry leftovers Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/src/config.rs | 8 ++++---- crates/switchyard-nemo-relay-plugin/src/runtime.rs | 10 ++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index 5688ced98..b9914b8ef 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -429,7 +429,7 @@ mod tests { } #[test] - fn removed_enabled_profile_list_is_not_silently_ignored() { + fn unknown_configuration_fields_are_rejected() { let value = json!({ "version": 2, "algorithm": {"kind": "random"}, @@ -441,12 +441,12 @@ mod tests { } }, "default_targets": {"openai_chat": "chat"}, - "enabled_inbound_profiles": ["openai_chat"] + "unexpected_setting": true }); let error = serde_json::from_value::(value) .err() - .expect("removed field must produce a migration error"); - assert!(error.to_string().contains("enabled_inbound_profiles")); + .expect("unknown field must be rejected"); + assert!(error.to_string().contains("unexpected_setting")); } #[test] diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index ef3ff9e2e..f51df36ab 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -57,7 +57,8 @@ impl SwitchyardRuntime { let libsy_request = self.libsy_request(inbound, &request, false)?; let metadata = identity_metadata(libsy_request.metadata.as_ref()); let max_attempts = self.max_retries + 1; - for attempt in 1..=max_attempts { + let mut attempt = 1; + loop { self.mark( "switchyard.routing.requested", json!({"algorithm": self.algorithm.name(), "attempt": attempt}), @@ -83,6 +84,7 @@ impl SwitchyardRuntime { failure_mark_data(attempt, &failure), &metadata, ); + attempt += 1; } Err(failure) => { self.mark( @@ -96,7 +98,6 @@ impl SwitchyardRuntime { } } } - Err("Switchyard retry loop ended without a result".into()) } async fn drive_buffered( @@ -284,7 +285,8 @@ impl SwitchyardRuntime { ) -> LlmJsonAsyncStreamV2 { Box::pin(async_stream::try_stream! { let max_attempts = self.max_retries + 1; - for attempt in 1..=max_attempts { + let mut attempt = 1; + loop { self.mark( "switchyard.routing.requested", json!({"algorithm": self.algorithm.name(), "attempt": attempt}), @@ -335,6 +337,7 @@ impl SwitchyardRuntime { failure_mark_data(attempt, &failure), &metadata, ); + attempt += 1; continue; } self.mark( @@ -352,7 +355,6 @@ impl SwitchyardRuntime { } return; } - Err("Switchyard stream retry loop ended without a result".to_string())?; }) } From 90ce083d4184234172c385a6d8ecce09fd897a5b Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 18:01:00 -0600 Subject: [PATCH 37/51] docs(plugin): describe shared bounded output Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index dd555e22d..f2a929501 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -71,7 +71,7 @@ Only supported LLM execution names whose mapped protocol appears in SDK's explicit `Passthrough` path. These ordinary untargeted calls remain inside Relay's managed LLM lifecycle, but their provider events do not cross the plugin ABI. Targeted provider streams permit at most one pending pull; -plugin output and direct pass-through use bounded 32-event host queues. +plugin output and direct pass-through use Relay's bounded host queue. Provider failures use HTTP semantics. Relay supplies status, a bounded body, and safe response headers when it received an HTTP response; Switchyard passes From 10d204bd401cc2ad754d7e7d5a271417b64be52d Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 22:05:13 -0600 Subject: [PATCH 38/51] chore(plugin): pin reviewed Relay native API v2 Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 ++-- crates/libsy/src/core/driver.rs | 2 +- .../switchyard-nemo-relay-plugin/Cargo.toml | 2 +- .../src/config.rs | 20 +++++++++++-------- .../src/runtime.rs | 4 ++-- .../src/translation.rs | 16 +++++++++------ 6 files changed, 28 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 915ccba66..ec5d60c76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=cec9de823047344d722d7d53ce4722e0d2aa22f9#cec9de823047344d722d7d53ce4722e0d2aa22f9" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=f8cebd0dc22b1b77b6d20ec3490b84186542a2d7#f8cebd0dc22b1b77b6d20ec3490b84186542a2d7" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=cec9de823047344d722d7d53ce4722e0d2aa22f9#cec9de823047344d722d7d53ce4722e0d2aa22f9" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=f8cebd0dc22b1b77b6d20ec3490b84186542a2d7#f8cebd0dc22b1b77b6d20ec3490b84186542a2d7" dependencies = [ "bitflags", "chrono", diff --git a/crates/libsy/src/core/driver.rs b/crates/libsy/src/core/driver.rs index 04eb9e3eb..581ac647a 100644 --- a/crates/libsy/src/core/driver.rs +++ b/crates/libsy/src/core/driver.rs @@ -52,7 +52,7 @@ use std::{any::Any, sync::Arc}; use crate::{DriverError, LibsyError, Result}; use parking_lot::Mutex; -use futures::{future::Either, pin_mut, Stream, StreamExt}; +use futures::{Stream, StreamExt, future::Either, pin_mut}; use futures_timer::Delay; use switchyard_protocol::Context; use tokio::sync::{mpsc, oneshot}; diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 084c49eb1..7c82251e0 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "cec9de823047344d722d7d53ce4722e0d2aa22f9" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "f8cebd0dc22b1b77b6d20ec3490b84186542a2d7" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index b9914b8ef..e477ad34e 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -497,10 +497,12 @@ mod tests { for target in random.targets.values_mut() { target.weight = 0.0; } - assert!(random - .validate() - .unwrap_err() - .contains("at least one weight must be positive")); + assert!( + random + .validate() + .unwrap_err() + .contains("at least one weight must be positive") + ); let mut classifier = config(); classifier.algorithm = AlgorithmConfig::LlmClassifier { @@ -512,9 +514,11 @@ mod tests { ..Default::default() }, }; - assert!(classifier - .validate() - .unwrap_err() - .contains("base_threshold must be between 0 and 1")); + assert!( + classifier + .validate() + .unwrap_err() + .contains("base_threshold must be between 0 and 1") + ); } } diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index f51df36ab..a1a0e8f14 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -11,7 +11,7 @@ use nemo_relay_plugin::{ LlmContinuationV2, LlmJsonAsyncStreamV2, LlmNonHttpFailureKindV2, LlmRequest as RelayRequest, LlmStreamContinuationV2, LlmStreamExecutionOutcomeV2, PluginRuntime, }; -use serde_json::{json, Map}; +use serde_json::{Map, json}; use switchyard_libsy::{Algorithm, CallLlmRequest, LibsyError, Step}; use switchyard_protocol::{ Context, Decision, LlmClientError, LlmRequest as SwitchyardLlmRequest, LlmResponse, @@ -20,7 +20,7 @@ use switchyard_protocol::{ }; use switchyard_translation::{StreamTranslationState, TranslationEngine}; -use crate::config::{protocol_from_call, PreparedTargetBinding, SwitchyardConfig}; +use crate::config::{PreparedTargetBinding, SwitchyardConfig, protocol_from_call}; use crate::translation; pub(crate) struct SwitchyardRuntime { diff --git a/crates/switchyard-nemo-relay-plugin/src/translation.rs b/crates/switchyard-nemo-relay-plugin/src/translation.rs index 94d0eb888..2065c9d26 100644 --- a/crates/switchyard-nemo-relay-plugin/src/translation.rs +++ b/crates/switchyard-nemo-relay-plugin/src/translation.rs @@ -211,11 +211,15 @@ mod tests { .unwrap(); let translated = encode_stream_event(&engine, &mut state, WireFormat::AnthropicMessages, event).unwrap(); - assert!(translated - .iter() - .any(|event| { event.pointer("/delta/text").and_then(Json::as_str) == Some("Hi") })); - assert!(translated - .iter() - .all(|event| event.get("system_fingerprint").is_none())); + assert!( + translated + .iter() + .any(|event| { event.pointer("/delta/text").and_then(Json::as_str) == Some("Hi") }) + ); + assert!( + translated + .iter() + .all(|event| event.get("system_fingerprint").is_none()) + ); } } From f92e6e2271d8c88e734c697d6af62f6a9e4b71e3 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 2 Aug 2026 22:49:10 -0600 Subject: [PATCH 39/51] fix(plugin): stop routing after terminal controls Signed-off-by: Bryan Bednarski --- .../src/runtime.rs | 257 ++++++++++++++++-- 1 file changed, 231 insertions(+), 26 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index a1a0e8f14..818f39863 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -2,8 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use std::collections::BTreeMap; +use std::fmt; use std::pin::Pin; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use futures_util::{Stream, StreamExt}; use nemo_relay_plugin::{ @@ -59,13 +60,20 @@ impl SwitchyardRuntime { let max_attempts = self.max_retries + 1; let mut attempt = 1; loop { + let terminal_control = TerminalContinuationControl::default(); self.mark( "switchyard.routing.requested", json!({"algorithm": self.algorithm.name(), "attempt": attempt}), &metadata, ); match self - .drive_buffered(libsy_request.clone(), &continuation, attempt, &metadata) + .drive_buffered( + libsy_request.clone(), + &continuation, + &terminal_control, + attempt, + &metadata, + ) .await { Ok(response) => { @@ -92,6 +100,9 @@ impl SwitchyardRuntime { failure_mark_data(attempt, &failure), &metadata, ); + if !fallback_allowed(&failure) { + return Err(failure.to_string()); + } return self .fallback_buffered(inbound, libsy_request, &continuation, &metadata) .await; @@ -104,6 +115,7 @@ impl SwitchyardRuntime { &self, request: Request, continuation: &LlmContinuationV2, + terminal_control: &TerminalContinuationControl, attempt: u32, mark_metadata: &Json, ) -> Result { @@ -112,12 +124,21 @@ impl SwitchyardRuntime { .clone() .run_stream(Context::default(), request, None); while let Some(step) = steps.next().await { + if let Some(failure) = terminal_control.failure() { + return Err(failure); + } match step { Ok(Step::Decision(decision)) => { self.emit_decision(decision.as_ref(), attempt, mark_metadata); } Ok(Step::CallLlm(call)) => { - self.serve_buffered_call(*call, continuation).await?; + let result = self + .serve_buffered_call(*call, continuation, terminal_control) + .await; + if let Some(failure) = terminal_control.failure() { + return Err(failure); + } + result?; } Ok(Step::ReturnToAgent(response)) => return Ok(*response), Err(error) => return Err(error), @@ -130,13 +151,17 @@ impl SwitchyardRuntime { &self, call: CallLlmRequest, continuation: &LlmContinuationV2, + terminal_control: &TerminalContinuationControl, ) -> switchyard_libsy::Result<()> { let target_name = call.get_decision().selected_model().to_string(); let request = call.get_request().llm_request.clone(); let result = async { let target = self.target(&target_name)?; let request = self.dispatch_request(target, request, false)?; - let response = continuation.call(request).await.map_err(client_error)?; + let response = continuation + .call(request) + .await + .map_err(|error| client_error_with_control(error, terminal_control))?; let response = translation::decode_response(&self.translation, target.protocol, &response) .map_err(LlmClientError::ResponseTranslation)?; @@ -287,6 +312,7 @@ impl SwitchyardRuntime { let max_attempts = self.max_retries + 1; let mut attempt = 1; loop { + let terminal_control = TerminalContinuationControl::default(); self.mark( "switchyard.routing.requested", json!({"algorithm": self.algorithm.name(), "attempt": attempt}), @@ -296,6 +322,7 @@ impl SwitchyardRuntime { .drive_stream( request.clone(), &continuation, + &terminal_control, attempt, &metadata, ) @@ -315,6 +342,7 @@ impl SwitchyardRuntime { yield event; } Some(Err(failure)) if committed => { + let failure = terminal_control.failure().unwrap_or(failure); self.mark( "switchyard.routing.error", failure_mark_data(attempt, &failure), @@ -331,6 +359,7 @@ impl SwitchyardRuntime { } Err(failure) => failure, }; + let failure = terminal_control.failure().unwrap_or(failure); if libsy_error_retryable(&failure) && attempt < max_attempts { self.mark( "switchyard.routing.retry", @@ -345,6 +374,9 @@ impl SwitchyardRuntime { failure_mark_data(attempt, &failure), &metadata, ); + if !fallback_allowed(&failure) { + Err(format!("Switchyard routing stopped by downstream policy: {failure}"))?; + } let mut fallback = self .fallback_stream(inbound, request.clone(), &continuation, &metadata) .await?; @@ -362,6 +394,7 @@ impl SwitchyardRuntime { &self, request: Request, continuation: &LlmStreamContinuationV2, + terminal_control: &TerminalContinuationControl, attempt: u32, mark_metadata: &Json, ) -> Result { @@ -370,12 +403,21 @@ impl SwitchyardRuntime { .clone() .run_stream(Context::default(), request, None); while let Some(step) = steps.next().await { + if let Some(failure) = terminal_control.failure() { + return Err(failure); + } match step { Ok(Step::Decision(decision)) => { self.emit_decision(decision.as_ref(), attempt, mark_metadata); } Ok(Step::CallLlm(call)) => { - self.serve_stream_call(*call, continuation).await?; + let result = self + .serve_stream_call(*call, continuation, terminal_control) + .await; + if let Some(failure) = terminal_control.failure() { + return Err(failure); + } + result?; } Ok(Step::ReturnToAgent(response)) => return Ok(*response), Err(error) => return Err(error), @@ -388,13 +430,20 @@ impl SwitchyardRuntime { &self, call: CallLlmRequest, continuation: &LlmStreamContinuationV2, + terminal_control: &TerminalContinuationControl, ) -> switchyard_libsy::Result<()> { let target_name = call.get_decision().selected_model().to_string(); let request = call.get_request(); let llm_request = request.llm_request.clone(); let metadata = request.metadata.clone(); let result = self - .provider_stream_response(&target_name, llm_request, metadata, continuation) + .provider_stream_response( + &target_name, + llm_request, + metadata, + continuation, + terminal_control, + ) .await .map_err(|source| LibsyError::client_call(target_name, source)); call.respond(result) @@ -406,16 +455,19 @@ impl SwitchyardRuntime { request: SwitchyardLlmRequest, metadata: Option, continuation: &LlmStreamContinuationV2, + terminal_control: &TerminalContinuationControl, ) -> Result { let target = self.target(target_name)?; let dispatch = self.dispatch_request(target, request, true)?; let mut upstream = continuation .open_stream(dispatch) .await - .map_err(client_error)?; + .map_err(|error| client_error_with_control(error, terminal_control))?; let first_raw = match upstream.next().await { Some(Ok(first)) => first, - Some(Err(error)) => return Err(client_error(error)), + Some(Err(error)) => { + return Err(client_error_with_control(error, terminal_control)); + } None => { return Err(LlmClientError::InvalidResponse { source: Box::new(std::io::Error::new( @@ -430,10 +482,13 @@ impl SwitchyardRuntime { decode_provider_event(&self.translation, &mut state, target.protocol, first_raw)?; let protocol = target.protocol; let translation = Arc::clone(&self.translation); + let terminal_control = terminal_control.clone(); let stream: LlmResponseStream = Box::pin(async_stream::try_stream! { yield first; while let Some(item) = upstream.next().await { - let raw = item.map_err(client_error)?; + let raw = item.map_err(|error| { + client_error_with_control(error, &terminal_control) + })?; yield decode_provider_event(&translation, &mut state, protocol, raw)?; } }); @@ -530,12 +585,14 @@ impl SwitchyardRuntime { json!({"selected_target": target_name}), metadata, ); + let terminal_control = TerminalContinuationControl::default(); let response = self .provider_stream_response( target_name, request.llm_request, request.metadata, continuation, + &terminal_control, ) .await .map_err(|error| format!("trusted fallback stream failed: {error}"))?; @@ -549,6 +606,80 @@ impl SwitchyardRuntime { type ReturnedJsonStream = Pin> + Send>>; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum TerminalContinuationKind { + Guardrail, + Cancelled, +} + +impl TerminalContinuationKind { + fn label(self) -> &'static str { + match self { + Self::Guardrail => "guardrail", + Self::Cancelled => "cancelled", + } + } +} + +#[derive(Clone, Debug)] +struct TerminalContinuationFailure { + kind: TerminalContinuationKind, +} + +impl fmt::Display for TerminalContinuationFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.kind { + TerminalContinuationKind::Guardrail => { + formatter.write_str("downstream LLM continuation was blocked by a guardrail") + } + TerminalContinuationKind::Cancelled => { + formatter.write_str("downstream LLM continuation was cancelled") + } + } + } +} + +impl std::error::Error for TerminalContinuationFailure {} + +#[derive(Clone, Default)] +struct TerminalContinuationControl { + failure: Arc>>, +} + +impl TerminalContinuationControl { + fn record(&self, error: &LlmContinuationFailureV2) { + let kind = match error { + LlmContinuationFailureV2::NonHttp { + kind: LlmNonHttpFailureKindV2::Guardrail, + .. + } => TerminalContinuationKind::Guardrail, + LlmContinuationFailureV2::NonHttp { + kind: LlmNonHttpFailureKindV2::Cancelled, + .. + } => TerminalContinuationKind::Cancelled, + _ => return, + }; + let mut failure = self + .failure + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + failure.get_or_insert(kind); + } + + fn failure(&self) -> Option { + let kind = *self + .failure + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + kind.map(|kind| { + LibsyError::external( + "downstream LLM continuation", + TerminalContinuationFailure { kind }, + ) + }) + } +} + fn decode_provider_event( translation: &TranslationEngine, state: &mut StreamTranslationState, @@ -594,6 +725,27 @@ fn libsy_error_retryable(error: &LibsyError) -> bool { } } +fn terminal_continuation_failure(error: &LibsyError) -> Option<&TerminalContinuationFailure> { + let LibsyError::External { source, .. } = error else { + return None; + }; + source + .as_ref() + .downcast_ref::() +} + +fn fallback_allowed(error: &LibsyError) -> bool { + terminal_continuation_failure(error).is_none() +} + +fn client_error_with_control( + error: LlmContinuationFailureV2, + terminal_control: &TerminalContinuationControl, +) -> LlmClientError { + terminal_control.record(&error); + client_error(error) +} + fn client_error(error: LlmContinuationFailureV2) -> LlmClientError { match error { LlmContinuationFailureV2::Http { status, body, .. } => { @@ -624,23 +776,28 @@ fn failure_mark_data(attempt: u32, failure: &LibsyError) -> Json { Json::from(libsy_error_retryable(failure)), ), ]); - match failure { - LibsyError::ClientCall { - source: LlmClientError::UpstreamHttp { status, .. }, - .. - } => { - data.insert("failure_kind".into(), Json::from("http")); - data.insert("http_status".into(), Json::from(*status)); - } - LibsyError::ClientCall { source, .. } => { - data.insert("failure_kind".into(), Json::from("non_http")); - data.insert( - "non_http_kind".into(), - Json::from(client_error_label(source)), - ); - } - _ => { - data.insert("failure_kind".into(), Json::from("algorithm")); + if let Some(terminal) = terminal_continuation_failure(failure) { + data.insert("failure_kind".into(), Json::from("non_http")); + data.insert("non_http_kind".into(), Json::from(terminal.kind.label())); + } else { + match failure { + LibsyError::ClientCall { + source: LlmClientError::UpstreamHttp { status, .. }, + .. + } => { + data.insert("failure_kind".into(), Json::from("http")); + data.insert("http_status".into(), Json::from(*status)); + } + LibsyError::ClientCall { source, .. } => { + data.insert("failure_kind".into(), Json::from("non_http")); + data.insert( + "non_http_kind".into(), + Json::from(client_error_label(source)), + ); + } + _ => { + data.insert("failure_kind".into(), Json::from("algorithm")); + } } } Json::Object(data) @@ -747,6 +904,54 @@ mod tests { assert!(!libsy_error_retryable(&LibsyError::MissingFinalResponse)); } + #[test] + fn terminal_downstream_controls_never_retry_or_fallback() { + for (kind, label) in [ + (LlmNonHttpFailureKindV2::Guardrail, "guardrail"), + (LlmNonHttpFailureKindV2::Cancelled, "cancelled"), + ] { + let control = TerminalContinuationControl::default(); + let _ = client_error_with_control( + LlmContinuationFailureV2::NonHttp { + kind, + message: "detail must not appear in routing marks".into(), + }, + &control, + ); + let failure = control.failure().expect("terminal failure is retained"); + + assert!(!libsy_error_retryable(&failure)); + assert!(!fallback_allowed(&failure)); + assert_eq!( + failure_mark_data(2, &failure), + json!({ + "attempt": 2, + "retryable": false, + "failure_kind": "non_http", + "non_http_kind": label, + }) + ); + } + + for kind in [ + LlmNonHttpFailureKindV2::InvalidRequest, + LlmNonHttpFailureKindV2::Internal, + ] { + let control = TerminalContinuationControl::default(); + let mapped = client_error_with_control( + LlmContinuationFailureV2::NonHttp { + kind, + message: "ordinary failure".into(), + }, + &control, + ); + assert!(control.failure().is_none()); + assert!(fallback_allowed(&LibsyError::client_call( + "provider", mapped + ))); + } + } + #[test] fn routing_failure_marks_exclude_provider_payloads() { let failure = LibsyError::client_call( From 4e87155c377b023c8bfd6926a315f7e361323313 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 08:47:23 -0600 Subject: [PATCH 40/51] fix(libsy): avoid unused classifier session state Signed-off-by: Bryan Bednarski --- crates/libsy/src/algorithms/fall_through.rs | 23 +++++++++++++++++++++ crates/libsy/src/algorithms/llm_class.rs | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs index 1b41f08e5..f61aa33eb 100644 --- a/crates/libsy/src/algorithms/fall_through.rs +++ b/crates/libsy/src/algorithms/fall_through.rs @@ -127,6 +127,19 @@ impl FallThrough where S: Default + Send + 'static, { + /// Creates a router whose state exists only for one run. + pub fn new_stateless(targets: LlmTargetSet) -> Self { + Self { + name: "fall_through".to_string(), + decision_reason: default_decision_reason, + processors: Vec::new(), + classifiers: Vec::new(), + targets, + session_states: None, + session_evictions: SessionEvictions::default(), + } + } + /// Creates a router that retains one private `S` per session. pub fn new_with_state(targets: LlmTargetSet) -> Self { Self { @@ -1181,6 +1194,16 @@ mod tests { assert_eq!(second_session, "weak"); assert_eq!(anonymous1, "weak"); assert_eq!(anonymous2, "weak"); + + let stateless = Arc::new( + FallThrough::::new_stateless(target_set(&["strong", "weak"])) + .with_processor(Arc::new(CountingProcessor)) + .with_classifier(Arc::new(ThresholdClassifier)), + ); + let (stateless_first, _) = run_turn(&stateless).await?; + let (stateless_second, _) = run_turn(&stateless).await?; + assert_eq!(stateless_first, "weak"); + assert_eq!(stateless_second, "weak"); Ok(()) } } diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 57726d2bc..c7041e578 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -471,7 +471,7 @@ impl LlmTaskClassifier { // Affinity comes first so a retained assignment short-circuits the judge call. // Note: when this classifier is embedded inside another cascade (e.g. StageRouter) // the affinity processor never fires — only the inner score() is called. - let mut route = FallThrough::::new_with_state(targets).with_name(ALGORITHM_NAME); + let mut route = FallThrough::::new_stateless(targets).with_name(ALGORITHM_NAME); if session_affinity { let affinity = if message_hash_fallback { AffinityRouter::new().with_message_hash_fallback() From fc40daeddf51ef9b56df44def32e8c9b7a65a314 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 08:47:39 -0600 Subject: [PATCH 41/51] docs(plugin): define initial router scope Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/README.md | 17 +++++- .../config.schema.json | 1 + .../src/config.rs | 57 ++++++++++++++++--- 3 files changed, 66 insertions(+), 9 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index f2a929501..db40b2be3 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -35,6 +35,20 @@ crates.io. Operators install a release bundle containing the compiled shared library, materialized `relay-plugin.toml`, `config.schema.json`, licensing files, and checksum. +## Supported routers + +This initial plugin release supports exactly two libsy algorithms: + +- seeded, weighted `random` routing; and +- capability-based `llm_classifier` routing, where a judge selects the weak or + strong target before the final provider call. + +`stage_router` and the response-judging escalation mode are intentionally not +part of this release. Their configuration is rejected rather than silently +mapped onto one of the supported algorithms. They are being developed as +separate follow-ups so their request-mutation and streaming-response contracts +can be reviewed independently. + During development, `nemo-relay-plugin` is pinned to the Relay native API v2 feature commit. Native API v2 remains unreleased, so every bundle must be rebuilt against the exact pinned revision; an older v2 bundle must not be used @@ -132,7 +146,8 @@ process without putting them in configuration or libsy metadata. For `kind = "llm_classifier"`, the classifier thresholds, affinity options, and `recent_turn_window` use libsy's `TaskClassifierConfig` directly. The plugin adds only the semantic `classifier_target`, `weak_target`, and `strong_target` -bindings required to resolve Relay continuations. +bindings required to resolve Relay continuations. An `escalation` table is not +accepted by this release. Version-1 service configuration is rejected with a migration error. The plugin does not provide decision-only or observe-only execution. diff --git a/crates/switchyard-nemo-relay-plugin/config.schema.json b/crates/switchyard-nemo-relay-plugin/config.schema.json index 5a35d7614..49a815c5b 100644 --- a/crates/switchyard-nemo-relay-plugin/config.schema.json +++ b/crates/switchyard-nemo-relay-plugin/config.schema.json @@ -20,6 +20,7 @@ "default": 3 }, "algorithm": { + "description": "Router configuration. This release supports random and capability-based llm_classifier only.", "oneOf": [ { "type": "object", diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index e477ad34e..00ed5c909 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -125,6 +125,8 @@ enum AlgorithmConfig { classifier_target: String, weak_target: String, strong_target: String, + #[serde(default)] + escalation: Option, #[serde(flatten)] config: TaskClassifierConfig, }, @@ -255,15 +257,24 @@ impl SwitchyardConfig { classifier_target, weak_target, strong_target, + escalation, config, - } => LlmTaskClassifier::new( - target(classifier_target)?, - target(weak_target)?, - target(strong_target)?, - config.clone(), - ) - .map(|algorithm| Arc::new(algorithm) as Arc) - .map_err(|error| error.to_string()), + } => { + if escalation.is_some() { + return Err( + "llm_classifier escalation mode is not supported by this plugin version" + .into(), + ); + } + LlmTaskClassifier::new( + target(classifier_target)?, + target(weak_target)?, + target(strong_target)?, + config.clone(), + ) + .map(|algorithm| Arc::new(algorithm) as Arc) + .map_err(|error| error.to_string()) + } } } } @@ -475,6 +486,35 @@ mod tests { ); } + #[test] + fn unsupported_router_modes_are_rejected_explicitly() { + let mut classifier = config(); + classifier.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "base_threshold": 0.5, + "escalation": {"confirmations": 1} + })) + .unwrap(); + assert_eq!( + classifier.validate().unwrap_err(), + "llm_classifier escalation mode is not supported by this plugin version" + ); + + let error = serde_json::from_value::(json!({ + "kind": "stage_router", + "capable_target": "anthropic", + "efficient_target": "responses", + "picker": "efficient_first", + "confidence_threshold": 0.5 + })) + .err() + .expect("stage_router must remain outside the base plugin scope"); + assert!(error.to_string().contains("unknown variant `stage_router`")); + } + #[test] fn validation_does_not_resolve_environment_backed_headers() { let mut config = config(); @@ -509,6 +549,7 @@ mod tests { classifier_target: "chat".into(), weak_target: "responses".into(), strong_target: "anthropic".into(), + escalation: None, config: TaskClassifierConfig { base_threshold: 1.1, ..Default::default() From f44666d3ec2f6f4f6103ed046199eca8f76789ac Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 08:48:02 -0600 Subject: [PATCH 42/51] chore(plugin): pin rebased Relay native API v2 Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 ++-- crates/switchyard-nemo-relay-plugin/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ec5d60c76..3a15de9ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=f8cebd0dc22b1b77b6d20ec3490b84186542a2d7#f8cebd0dc22b1b77b6d20ec3490b84186542a2d7" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=3a41736e38abfb1c4bf875868a555a83ccf1622f#3a41736e38abfb1c4bf875868a555a83ccf1622f" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=f8cebd0dc22b1b77b6d20ec3490b84186542a2d7#f8cebd0dc22b1b77b6d20ec3490b84186542a2d7" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=3a41736e38abfb1c4bf875868a555a83ccf1622f#3a41736e38abfb1c4bf875868a555a83ccf1622f" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 7c82251e0..415fd360c 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "f8cebd0dc22b1b77b6d20ec3490b84186542a2d7" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "3a41736e38abfb1c4bf875868a555a83ccf1622f" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true From 77be543e603e2cf110c7c372a92723ace4a9202f Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 09:35:31 -0600 Subject: [PATCH 43/51] chore(plugin): refresh Relay native API v2 pin Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 ++-- crates/switchyard-nemo-relay-plugin/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a15de9ca..ac4f6efd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=3a41736e38abfb1c4bf875868a555a83ccf1622f#3a41736e38abfb1c4bf875868a555a83ccf1622f" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=59ad84df40cd2088b9c389495b7789c4802fb5ce#59ad84df40cd2088b9c389495b7789c4802fb5ce" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=3a41736e38abfb1c4bf875868a555a83ccf1622f#3a41736e38abfb1c4bf875868a555a83ccf1622f" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=59ad84df40cd2088b9c389495b7789c4802fb5ce#59ad84df40cd2088b9c389495b7789c4802fb5ce" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 415fd360c..fd40f4878 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "3a41736e38abfb1c4bf875868a555a83ccf1622f" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "59ad84df40cd2088b9c389495b7789c4802fb5ce" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true From 34d48dfdb3d565ad4e8a845a9e9570c19133bbed Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 09:42:42 -0600 Subject: [PATCH 44/51] fix(plugin): address review safety findings Signed-off-by: Bryan Bednarski --- .../scripts/package_bundle.py | 7 ++++++- .../src/runtime.rs | 15 +++++++++++---- .../tests/e2e/fake_provider.py | 18 ++++++++++++++++-- .../tests/e2e/run_e2e.py | 2 +- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py index 2a64aad6f..bb56a22d1 100644 --- a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py +++ b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py @@ -32,6 +32,12 @@ def main() -> None: if not library.is_file(): parser.error(f"compiled plugin library does not exist: {library}") + manifest = (CRATE_ROOT / "relay-plugin.toml").read_text(encoding="utf-8") + placeholders = ("", "") + missing = [placeholder for placeholder in placeholders if placeholder not in manifest] + if missing: + parser.error(f"plugin manifest is missing placeholders: {', '.join(missing)}") + output = args.output.resolve() if output.exists() and not output.is_dir(): parser.error(f"bundle output exists and is not a directory: {output}") @@ -45,7 +51,6 @@ def main() -> None: shutil.copy2(REPOSITORY_ROOT / "NOTICE", output / "NOTICE") artifact_digest = digest(artifact) - manifest = (CRATE_ROOT / "relay-plugin.toml").read_text(encoding="utf-8") manifest = manifest.replace("", artifact.name) manifest = manifest.replace("", artifact_digest) (output / "relay-plugin.toml").write_text(manifest, encoding="utf-8") diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 818f39863..5767a5ed4 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -185,7 +185,9 @@ impl SwitchyardRuntime { continuation: &LlmContinuationV2, metadata: &Json, ) -> Result { - let target_name = self.default_target(inbound); + let target_name = self + .default_target(inbound) + .map_err(|error| error.to_string())?; let target = self .target(target_name) .map_err(|error| error.to_string())?; @@ -259,10 +261,13 @@ impl SwitchyardRuntime { protocol_from_call(name).filter(|protocol| self.default_targets.contains_key(protocol)) } - fn default_target(&self, protocol: WireFormat) -> &str { + fn default_target(&self, protocol: WireFormat) -> Result<&str, LlmClientError> { self.default_targets .get(&protocol) - .expect("managed protocol must have a default target") + .map(String::as_str) + .ok_or_else(|| LlmClientError::Configuration { + message: format!("managed protocol {protocol} has no default target"), + }) } fn mark(&self, name: &str, data: Json, metadata: &Json) { @@ -579,7 +584,9 @@ impl SwitchyardRuntime { continuation: &LlmStreamContinuationV2, metadata: &Json, ) -> Result { - let target_name = self.default_target(inbound); + let target_name = self + .default_target(inbound) + .map_err(|error| error.to_string())?; self.mark( "switchyard.routing.fallback", json!({"selected_target": target_name}), diff --git a/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py b/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py index d37fe2108..1494eeb43 100644 --- a/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py +++ b/crates/switchyard-nemo-relay-plugin/tests/e2e/fake_provider.py @@ -37,8 +37,22 @@ def do_GET(self) -> None: self._json(404, {"error": "not found"}) def do_POST(self) -> None: - size = int(self.headers.get("content-length", "0")) - request = json.loads(self.rfile.read(size) or b"{}") + if self.headers.get("transfer-encoding") is not None: + self._json(411, {"error": {"message": "chunked requests are unsupported"}}) + return + content_length = self.headers.get("content-length") + if content_length is None: + self._json(411, {"error": {"message": "content-length is required"}}) + return + try: + size = int(content_length) + except ValueError: + self._json(400, {"error": {"message": "content-length is invalid"}}) + return + if size <= 0: + self._json(400, {"error": {"message": "request body is required"}}) + return + request = json.loads(self.rfile.read(size)) model = request.get("model", "unknown") attempt = call_number(model) if model == "fake/header-target" and ( diff --git a/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py b/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py index db0adb05c..fb60abac8 100644 --- a/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py +++ b/crates/switchyard-nemo-relay-plugin/tests/e2e/run_e2e.py @@ -1055,7 +1055,7 @@ def main() -> None: path for path in root.rglob("*") if path.suffix in {".jsonl", ".log", ".toml"} - and TARGET_AUTHORIZATION in path.read_text(encoding="utf-8") + and TARGET_AUTHORIZATION in path.read_text(encoding="utf-8", errors="replace") ] assert not recorded, f"target credential was recorded in {recorded}" summary["target_headers"] = { From 63da58627d433b502ba995e6065b95ae6ae81d01 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 09:49:31 -0600 Subject: [PATCH 45/51] fix(plugin): reject lossy Anthropic classifier targets Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/README.md | 6 +++-- .../config.schema.json | 6 ++++- .../src/config.rs | 27 +++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index db40b2be3..3b3dccf28 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -146,8 +146,10 @@ process without putting them in configuration or libsy metadata. For `kind = "llm_classifier"`, the classifier thresholds, affinity options, and `recent_turn_window` use libsy's `TaskClassifierConfig` directly. The plugin adds only the semantic `classifier_target`, `weak_target`, and `strong_target` -bindings required to resolve Relay continuations. An `escalation` table is not -accepted by this release. +bindings required to resolve Relay continuations. The classifier target must +use `openai_chat` or `openai_responses`: libsy's judge request requires a JSON +schema response format that cannot be encoded losslessly for Anthropic +Messages. An `escalation` table is not accepted by this release. Version-1 service configuration is rejected with a migration error. The plugin does not provide decision-only or observe-only execution. diff --git a/crates/switchyard-nemo-relay-plugin/config.schema.json b/crates/switchyard-nemo-relay-plugin/config.schema.json index 49a815c5b..17422ceba 100644 --- a/crates/switchyard-nemo-relay-plugin/config.schema.json +++ b/crates/switchyard-nemo-relay-plugin/config.schema.json @@ -43,7 +43,11 @@ ], "properties": { "kind": { "const": "llm_classifier" }, - "classifier_target": { "type": "string", "minLength": 1 }, + "classifier_target": { + "type": "string", + "minLength": 1, + "description": "Semantic target name for the judge. The referenced target must use openai_chat or openai_responses because the judge requires a JSON-schema response format." + }, "weak_target": { "type": "string", "minLength": 1 }, "strong_target": { "type": "string", "minLength": 1 }, "base_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index 00ed5c909..02eb8622d 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -266,6 +266,14 @@ impl SwitchyardConfig { .into(), ); } + let classifier_binding = self.targets.get(classifier_target).ok_or_else(|| { + format!("algorithm target {classifier_target:?} is not configured") + })?; + if classifier_binding.protocol == WireFormat::AnthropicMessages { + return Err(format!( + "classifier target {classifier_target:?} uses anthropic_messages, which cannot encode the required JSON-schema response format without loss; use an openai_chat or openai_responses target" + )); + } LlmTaskClassifier::new( target(classifier_target)?, target(weak_target)?, @@ -486,6 +494,25 @@ mod tests { ); } + #[test] + fn classifier_rejects_anthropic_judge_targets_before_dispatch() { + let mut config = config(); + config.algorithm = AlgorithmConfig::LlmClassifier { + classifier_target: "anthropic".into(), + weak_target: "responses".into(), + strong_target: "chat".into(), + escalation: None, + config: TaskClassifierConfig { + base_threshold: 0.5, + ..Default::default() + }, + }; + + let error = config.validate().unwrap_err(); + assert!(error.contains("classifier target \"anthropic\" uses anthropic_messages")); + assert!(error.contains("use an openai_chat or openai_responses target")); + } + #[test] fn unsupported_router_modes_are_rejected_explicitly() { let mut classifier = config(); From c9d3808f9fa6bdcf0d963b80447452fa74b36d4e Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 10:31:37 -0600 Subject: [PATCH 46/51] chore(plugin): refresh Relay SDK pin Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 ++-- crates/switchyard-nemo-relay-plugin/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac4f6efd7..50a034c28 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=59ad84df40cd2088b9c389495b7789c4802fb5ce#59ad84df40cd2088b9c389495b7789c4802fb5ce" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=f2d1c1b22a4f54622464e1bb8c07bddbf2411a16#f2d1c1b22a4f54622464e1bb8c07bddbf2411a16" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=59ad84df40cd2088b9c389495b7789c4802fb5ce#59ad84df40cd2088b9c389495b7789c4802fb5ce" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=f2d1c1b22a4f54622464e1bb8c07bddbf2411a16#f2d1c1b22a4f54622464e1bb8c07bddbf2411a16" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index fd40f4878..dd1a2865d 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "59ad84df40cd2088b9c389495b7789c4802fb5ce" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "f2d1c1b22a4f54622464e1bb8c07bddbf2411a16" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true From dd1bd848efc0a04dfd2bead89a5d8d702e047d7e Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 10:41:24 -0600 Subject: [PATCH 47/51] chore(plugin): refresh Relay SDK pin Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 ++-- crates/switchyard-nemo-relay-plugin/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 50a034c28..95e4ad3f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=f2d1c1b22a4f54622464e1bb8c07bddbf2411a16#f2d1c1b22a4f54622464e1bb8c07bddbf2411a16" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=cb9b843e6e9e37c16d504038d13cd7d94328e2d2#cb9b843e6e9e37c16d504038d13cd7d94328e2d2" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=f2d1c1b22a4f54622464e1bb8c07bddbf2411a16#f2d1c1b22a4f54622464e1bb8c07bddbf2411a16" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=cb9b843e6e9e37c16d504038d13cd7d94328e2d2#cb9b843e6e9e37c16d504038d13cd7d94328e2d2" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index dd1a2865d..84896cdeb 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "f2d1c1b22a4f54622464e1bb8c07bddbf2411a16" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "cb9b843e6e9e37c16d504038d13cd7d94328e2d2" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true From 57254df8bf59eee287cefcbfbc320c80406d40d5 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 10:55:22 -0600 Subject: [PATCH 48/51] chore(plugin): refresh Relay SDK pin Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 ++-- crates/switchyard-nemo-relay-plugin/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95e4ad3f0..9dd5506d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=cb9b843e6e9e37c16d504038d13cd7d94328e2d2#cb9b843e6e9e37c16d504038d13cd7d94328e2d2" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=18b23aaa931b996ad5b9c0de91354b0886118b60#18b23aaa931b996ad5b9c0de91354b0886118b60" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=cb9b843e6e9e37c16d504038d13cd7d94328e2d2#cb9b843e6e9e37c16d504038d13cd7d94328e2d2" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=18b23aaa931b996ad5b9c0de91354b0886118b60#18b23aaa931b996ad5b9c0de91354b0886118b60" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 84896cdeb..656ec48fe 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "cb9b843e6e9e37c16d504038d13cd7d94328e2d2" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "18b23aaa931b996ad5b9c0de91354b0886118b60" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true From 881e0d3cdcbf4aaa18aa5a8ebe5ac9eebe02c197 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 11:23:49 -0600 Subject: [PATCH 49/51] chore(plugin): refresh Relay SDK pin Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 ++-- crates/switchyard-nemo-relay-plugin/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9dd5506d7..9e6bc0205 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=18b23aaa931b996ad5b9c0de91354b0886118b60#18b23aaa931b996ad5b9c0de91354b0886118b60" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=070b6d3c7d5b681beebc1aa6f12bfcfafc5e1c0d#070b6d3c7d5b681beebc1aa6f12bfcfafc5e1c0d" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=18b23aaa931b996ad5b9c0de91354b0886118b60#18b23aaa931b996ad5b9c0de91354b0886118b60" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=070b6d3c7d5b681beebc1aa6f12bfcfafc5e1c0d#070b6d3c7d5b681beebc1aa6f12bfcfafc5e1c0d" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 656ec48fe..ba906ac4e 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "18b23aaa931b996ad5b9c0de91354b0886118b60" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "070b6d3c7d5b681beebc1aa6f12bfcfafc5e1c0d" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true From 0b3b243e65209862599e94881ea738cc255988e3 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 11:29:10 -0600 Subject: [PATCH 50/51] chore(plugin): refresh Relay SDK pin Signed-off-by: Bryan Bednarski --- Cargo.lock | 4 ++-- crates/switchyard-nemo-relay-plugin/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e6bc0205..067e1cf1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1110,7 +1110,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=070b6d3c7d5b681beebc1aa6f12bfcfafc5e1c0d#070b6d3c7d5b681beebc1aa6f12bfcfafc5e1c0d" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=3a8f8f0745fc6545a9162bd585da2273fa052785#3a8f8f0745fc6545a9162bd585da2273fa052785" dependencies = [ "futures", "nemo-relay-types", @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=070b6d3c7d5b681beebc1aa6f12bfcfafc5e1c0d#070b6d3c7d5b681beebc1aa6f12bfcfafc5e1c0d" +source = "git+https://github.com/bbednarski9/NeMo-Relay?rev=3a8f8f0745fc6545a9162bd585da2273fa052785#3a8f8f0745fc6545a9162bd585da2273fa052785" dependencies = [ "bitflags", "chrono", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index ba906ac4e..15772ccd3 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["cdylib"] async-stream.workspace = true futures-util.workspace = true http.workspace = true -nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "070b6d3c7d5b681beebc1aa6f12bfcfafc5e1c0d" } +nemo-relay-plugin = { git = "https://github.com/bbednarski9/NeMo-Relay", rev = "3a8f8f0745fc6545a9162bd585da2273fa052785" } serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true From cda935c5360f85e5b87bb825fd51137c27183e0c Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 11:38:39 -0600 Subject: [PATCH 51/51] chore(plugin): track classifier token limit Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/README.md | 15 ++++++++------- .../config.schema.json | 5 +++++ crates/switchyard-nemo-relay-plugin/src/config.rs | 4 +++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 3b3dccf28..fdb96052b 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -143,13 +143,14 @@ and headers. Each `default_targets` key both enables that inbound protocol and names its trusted fallback. `header_env` resolves credentials in the plugin process without putting them in configuration or libsy metadata. -For `kind = "llm_classifier"`, the classifier thresholds, affinity options, and -`recent_turn_window` use libsy's `TaskClassifierConfig` directly. The plugin -adds only the semantic `classifier_target`, `weak_target`, and `strong_target` -bindings required to resolve Relay continuations. The classifier target must -use `openai_chat` or `openai_responses`: libsy's judge request requires a JSON -schema response format that cannot be encoded losslessly for Anthropic -Messages. An `escalation` table is not accepted by this release. +For `kind = "llm_classifier"`, the classifier thresholds, affinity options, +`recent_turn_window`, and `max_output_tokens` use libsy's +`TaskClassifierConfig` directly. The plugin adds only the semantic +`classifier_target`, `weak_target`, and `strong_target` bindings required to +resolve Relay continuations. The classifier target must use `openai_chat` or +`openai_responses`: libsy's judge request requires a JSON schema response +format that cannot be encoded losslessly for Anthropic Messages. An +`escalation` table is not accepted by this release. Version-1 service configuration is rejected with a migration error. The plugin does not provide decision-only or observe-only execution. diff --git a/crates/switchyard-nemo-relay-plugin/config.schema.json b/crates/switchyard-nemo-relay-plugin/config.schema.json index 17422ceba..21ba5300d 100644 --- a/crates/switchyard-nemo-relay-plugin/config.schema.json +++ b/crates/switchyard-nemo-relay-plugin/config.schema.json @@ -61,6 +61,11 @@ "type": ["integer", "null"], "minimum": 0 }, + "max_output_tokens": { + "type": "integer", + "minimum": 1, + "default": 4096 + }, "session_affinity": { "type": "boolean", "default": false }, "message_hash_fallback": { "type": "boolean", "default": false } } diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index 02eb8622d..1378a6c53 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -477,7 +477,8 @@ mod tests { "weak_target": "responses", "strong_target": "anthropic", "base_threshold": 0.5, - "recent_turn_window": 4 + "recent_turn_window": 4, + "max_output_tokens": 512 })) .unwrap(); let AlgorithmConfig::LlmClassifier { @@ -487,6 +488,7 @@ mod tests { panic!("expected classifier configuration"); }; assert_eq!(classifier.recent_turn_window, Some(4)); + assert_eq!(classifier.max_output_tokens, 512); config.validate().unwrap(); assert_eq!( config.prepare().unwrap().algorithm.name(),