From d754fb871e7e956e4bf39ac1b9d52c897efe3ff0 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Mon, 26 Oct 2026 10:00:32 +0300 Subject: [PATCH] fix: recover reasoning-only model responses --- Cargo.lock | 126 ++----- docs/how-to/serve-a-model-on-spark.md | 10 + src/spark/agent.rs | 478 ++++++++++++++++++++++---- src/spark/gateway.rs | 151 +++++++- src/spark/upstream.rs | 7 + 5 files changed, 593 insertions(+), 179 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 472cd96..6047b80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,7 +68,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -79,7 +79,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -826,7 +826,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2016,7 +2016,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2343,7 +2343,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -2690,9 +2690,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "once_cell", @@ -2754,9 +2754,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -3058,7 +3058,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3377,9 +3377,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.1" +version = "1.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" dependencies = [ "bytes", "libc", @@ -3961,18 +3961,18 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] [[package]] name = "webpki-roots" -version = "1.0.7" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] @@ -4151,7 +4151,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -4160,16 +4160,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -4187,31 +4178,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -4229,96 +4203,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "0.7.15" diff --git a/docs/how-to/serve-a-model-on-spark.md b/docs/how-to/serve-a-model-on-spark.md index 5dbfc32..831c524 100644 --- a/docs/how-to/serve-a-model-on-spark.md +++ b/docs/how-to/serve-a-model-on-spark.md @@ -119,6 +119,16 @@ model-identity probe pass, OpenAI-compatible routes are at routes are under `/anthropic//v1`. Warming or recovering generations return protocol-native `503` with `Retry-After`. +For translated Responses requests, an engine can occasionally stop after +reasoning without producing an answer or tool call. The gateway makes at most +one assistant-prefill continuation, retaining the request history, tools and +sampling settings. The engine's chat template closes the retained reasoning +block so generation can continue into the answer. Both attempts share the +remaining output budget, and reported usage includes both generations. This +does not reset or compact the coding session. Unsupported continuations and +repeated reasoning-only stops fail explicitly; malformed tool arguments and +length-limited generations are not retried by this recovery path. + For Ornith, reasoning streams independently from final answer text as OpenAI `reasoning_content`, Responses reasoning-summary events, or Anthropic thinking blocks. `sparkplane ... launch codex` advertises that capability in its private diff --git a/src/spark/agent.rs b/src/spark/agent.rs index 2ea55fd..43e6524 100644 --- a/src/spark/agent.rs +++ b/src/spark/agent.rs @@ -1665,101 +1665,178 @@ async fn gateway_responses_stream( Ok(upstream) => upstream, Err(error) => return gateway_generation_error(error), }; - let encoder = gateway::ResponsesEncoder::new(route.public_model.clone(), request.custom_tools); - if request.stream { - responses_sse(upstream, encoder, permit, token_permit) + let stream = request.stream; + let generation = ResponsesGeneration::new(route, request, upstream); + if stream { + responses_sse(generation, permit, token_permit) } else { - responses_json(upstream, encoder, permit, token_permit).await + responses_json(generation, permit, token_permit).await } } -async fn responses_json( - mut upstream: super::upstream::CompletionStream, - mut encoder: gateway::ResponsesEncoder, - _permit: tokio::sync::OwnedSemaphorePermit, - _token_permit: tokio::sync::OwnedSemaphorePermit, -) -> Response { - let mut done = false; - while let Some(event) = upstream.next().await { - match event { - Ok(GenerationEvent::Done) => { - if let Err(error) = encoder.accept(GenerationEvent::Done) { - upstream.response_failed(); - tracing::warn!( - event_code = "spark.inference.response-invalid", - reason = error.message - ); - return gateway_upstream_unavailable(); +struct ResponsesGeneration { + route: Arc, + upstream: Option, + observer: Option>, + original_body: Vec, + encoder: gateway::ResponsesEncoder, + prior_usage: (u64, u64), + attempt_usage: Option<(u64, u64)>, + recovered: bool, + terminal: bool, + succeeded: bool, +} + +impl ResponsesGeneration { + fn new( + route: Arc, + request: gateway::GenerationRequest, + mut upstream: super::upstream::CompletionStream, + ) -> Self { + let observer = upstream.take_usage_observer(); + let encoder = + gateway::ResponsesEncoder::new(route.public_model.clone(), request.custom_tools); + Self { + route, + upstream: Some(upstream), + observer, + original_body: request.body, + encoder, + prior_usage: (0, 0), + attempt_usage: None, + recovered: false, + terminal: false, + succeeded: false, + } + } + + fn fail(&mut self, message: &'static str) { + if let Some(observer) = &self.observer { + observer.response_failed(); + } + tracing::warn!( + event_code = "spark.inference.response-invalid", + reason = message + ); + self.encoder.fail_with_message(message); + self.terminal = true; + self.upstream.take(); + } + + async fn next(&mut self) -> Option { + loop { + if let Some(event) = self.encoder.pop() { + return Some(event); + } + if self.terminal { + return None; + } + let event = match self.upstream.as_mut()?.next().await { + Some(Ok(event)) => event, + _ => { + if let Some(observer) = &self.observer { + observer.incomplete(); + } + self.fail("upstream stream failed"); + continue; } - done = true; - break; + }; + if matches!(event, GenerationEvent::Done) + && !self.recovered + && let Some(body) = self + .encoder + .reasoning_only_recovery_request(&self.original_body) + { + self.recovered = true; + self.prior_usage = self.attempt_usage.unwrap_or_default(); + self.attempt_usage = None; + // Release the old connection before opening the continuation. + // Dropping this response also cancels whichever stream is live. + self.upstream.take(); + tracing::warn!(event_code = "spark.inference.reasoning-only-recovery"); + match self + .route + .upstream + .chat_stream_with_idle_timeout(&body, self.route.profile.stream_idle_timeout()) + .await + { + Ok(mut upstream) => { + upstream.take_usage_observer(); + self.upstream = Some(upstream); + } + Err(_) => self.fail("upstream generation produced no actionable output"), + } + continue; } - Ok(event) => { - if let Err(error) = encoder.accept(event) { - upstream.response_failed(); - tracing::warn!( - event_code = "spark.inference.response-invalid", - reason = error.message - ); - return gateway_upstream_unavailable(); + if self.recovered + && matches!(event, GenerationEvent::Done) + && self.attempt_usage.is_none() + { + self.fail("upstream recovery usage is missing"); + continue; + } + let event = match event { + GenerationEvent::Usage { + prompt_tokens, + completion_tokens, + } => { + self.attempt_usage = Some((prompt_tokens, completion_tokens)); + GenerationEvent::Usage { + prompt_tokens: self.prior_usage.0.saturating_add(prompt_tokens), + completion_tokens: self.prior_usage.1.saturating_add(completion_tokens), + } } + event => event, + }; + let done = matches!(event, GenerationEvent::Done); + // Only the validated public terminal marks this logical request + // successful. Usage counts include the input and output of both passes. + if !done && let Some(observer) = &self.observer { + observer.observe(&event); } - _ => { - upstream.response_failed(); - return gateway_upstream_unavailable(); + if let Err(error) = self.encoder.accept(event) { + self.fail(error.message); + } else if done { + if let Some(observer) = &self.observer { + observer.observe(&GenerationEvent::Done); + } + self.terminal = true; + self.succeeded = true; + self.upstream.take(); } } } - if done { - Json(encoder.final_document()).into_response() +} + +async fn responses_json( + mut generation: ResponsesGeneration, + _permit: tokio::sync::OwnedSemaphorePermit, + _token_permit: tokio::sync::OwnedSemaphorePermit, +) -> Response { + while generation.next().await.is_some() {} + if generation.succeeded { + Json(generation.encoder.final_document()).into_response() } else { - upstream.response_failed(); gateway_upstream_unavailable() } } fn responses_sse( - upstream: super::upstream::CompletionStream, - encoder: gateway::ResponsesEncoder, + generation: ResponsesGeneration, permit: tokio::sync::OwnedSemaphorePermit, token_permit: tokio::sync::OwnedSemaphorePermit, ) -> Response { let stream = futures_util::stream::unfold( - (upstream, encoder, permit, token_permit, false), - |(mut upstream, mut encoder, permit, token_permit, mut terminal)| async move { - loop { - if let Some(event) = encoder.pop() { - let frame = Event::default() - .event(event.name) - .data(event.data.to_string()); - return Some(( - Ok::<_, std::convert::Infallible>(frame), - (upstream, encoder, permit, token_permit, terminal), - )); - } - if terminal { - return None; - } - match upstream.next().await { - Some(Ok(event)) => { - terminal = matches!(event, GenerationEvent::Done); - if let Err(error) = encoder.accept(event) { - upstream.response_failed(); - tracing::warn!( - event_code = "spark.inference.response-invalid", - reason = error.message - ); - encoder.fail_with_message(error.message); - terminal = true; - } - } - _ => { - upstream.response_failed(); - encoder.fail(); - terminal = true; - } - } - } + (generation, permit, token_permit), + |(mut generation, permit, token_permit)| async move { + let event = generation.next().await?; + let frame = Event::default() + .event(event.name) + .data(event.data.to_string()); + Some(( + Ok::<_, std::convert::Infallible>(frame), + (generation, permit, token_permit), + )) }, ); Sse::new(stream).into_response() @@ -5864,6 +5941,259 @@ mod tests { llama_chat_server_with(llama_chat_sse()).await } + async fn chat_sequence_server( + responses: Vec, + ) -> (SocketAddr, tokio::task::JoinHandle>) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let task = tokio::spawn(async move { + let mut requests = Vec::new(); + for body in responses { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut bytes = Vec::new(); + let (start, length) = loop { + let mut buffer = [0; 4096]; + let count = socket.read(&mut buffer).await.unwrap(); + assert_ne!(count, 0); + bytes.extend_from_slice(&buffer[..count]); + if let Some(end) = bytes.windows(4).position(|part| part == b"\r\n\r\n") { + let headers = String::from_utf8_lossy(&bytes[..end]); + assert!(headers.starts_with("POST /v1/chat/completions ")); + let length = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().unwrap()) + }) + .unwrap(); + break (end + 4, length); + } + }; + while bytes.len() < start + length { + let mut buffer = [0; 4096]; + let count = socket.read(&mut buffer).await.unwrap(); + assert_ne!(count, 0); + bytes.extend_from_slice(&buffer[..count]); + } + requests.push(serde_json::from_slice(&bytes[start..start + length]).unwrap()); + socket.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",body.len()).as_bytes()).await.unwrap(); + for chunk in body.as_bytes().chunks(17) { + socket.write_all(chunk).await.unwrap(); + } + } + assert!( + tokio::time::timeout(Duration::from_millis(100), listener.accept()) + .await + .is_err(), + "recovery attempted an additional generation" + ); + requests + }); + (address, task) + } + + #[tokio::test] + async fn responses_recovery_preserves_history_tools_and_accounts_both_attempts() { + use crate::spark::upstream::ObservedRoute; + use axum::http::StatusCode; + for stream in [false, true] { + for succeeds in [false, true] { + let second = if succeeds { + llama_chat_sse() + } else { + reasoning_only_chat_sse() + }; + let (address, server) = + chat_sequence_server(vec![reasoning_only_chat_sse(), second]).await; + let state = AgentState::new(TOKEN, Vec::new(), Vec::new()); + let upstream = ObservedRoute::new( + "i_11111111111111111111111111111111", + 1, + address.ip(), + address.port(), + [("POST", "/v1/chat/completions")], + ) + .unwrap(); + state + .routes + .publish("fixture", "public".into(), "served".into(), upstream); + let input = serde_json::json!({ + "input":[{"type":"message","role":"user","content":"original task"},{"type":"message","role":"assistant","content":"earlier answer"},{"type":"message","role":"user","content":"continue"}], + "tools":[{"type":"function","name":"lookup","parameters":{"type":"object"}}], + "reasoning":{"effort":"high"}, "temperature":0.7, "max_output_tokens":100, "stream":stream + }); + let request = Request::post("/openai/fixture/v1/responses") + .header(header::AUTHORIZATION, format!("Bearer {TOKEN}")) + .body(Body::from(input.to_string())) + .unwrap(); + let response = router(state).oneshot(request).await.unwrap(); + assert_eq!( + response.status(), + if stream || succeeds { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + } + ); + let bytes = axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(); + let body = String::from_utf8(bytes.to_vec()).unwrap(); + if stream { + let events = body + .lines() + .filter_map(|line| line.strip_prefix("data: ")) + .map(|line| serde_json::from_str::(line).unwrap()) + .collect::>(); + assert_eq!( + events + .iter() + .filter(|e| e["type"] == "response.created") + .count(), + 1 + ); + assert!( + events + .windows(2) + .all(|pair| pair[0]["sequence_number"].as_u64() + < pair[1]["sequence_number"].as_u64()) + ); + let expected = if succeeds { + "response.completed" + } else { + "response.failed" + }; + assert_eq!(events.last().unwrap()["type"], expected); + assert_eq!(events.iter().filter(|e| e["type"] == expected).count(), 1); + if succeeds { + assert_eq!( + events.last().unwrap()["response"]["usage"]["input_tokens"], + 14 + ); + assert_eq!( + events.last().unwrap()["response"]["usage"]["output_tokens"], + 6 + ); + } + } else if succeeds { + let response: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(response["status"], "completed"); + assert_eq!(response["usage"]["input_tokens"], 14); + assert_eq!(response["usage"]["output_tokens"], 6); + } + let requests = server.await.unwrap(); + assert_eq!(requests.len(), 2); + let first = &requests[0]; + let second = &requests[1]; + assert_eq!( + &second["messages"].as_array().unwrap()[..3], + first["messages"].as_array().unwrap() + ); + assert_eq!( + second["messages"][3], + serde_json::json!({"role":"assistant","content":"","reasoning_content":"inspect"}) + ); + for key in [ + "model", + "tools", + "tool_choice", + "reasoning_effort", + "temperature", + ] { + assert_eq!(second[key], first[key]); + } + assert_eq!(second["max_tokens"], 97); + assert_eq!(second["continue_final_message"], true); + assert_eq!(second["add_generation_prompt"], false); + } + } + } + + #[tokio::test] + async fn responses_recovery_cancellation_closes_connection_and_releases_admission() { + use crate::spark::{gateway, upstream::ObservedRoute}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + for headers_received in [false, true] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (recovering, started) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + for index in 0..2 { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut buffer = [0; 16384]; + assert!(socket.read(&mut buffer).await.unwrap() > 0); + if index == 0 { + let body = reasoning_only_chat_sse(); + socket.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",body.len()).as_bytes()).await.unwrap(); + } else { + if headers_received { + socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n").await.unwrap(); + } + recovering.send(()).unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(2), socket.read(&mut buffer)) + .await + .unwrap() + .unwrap(), + 0 + ); + break; + } + } + }); + let routes = gateway::RouteRegistry::default(); + routes.publish( + "fixture", + "public".into(), + "served".into(), + ObservedRoute::new( + "i_11111111111111111111111111111111", + 1, + address.ip(), + address.port(), + [("POST", "/v1/chat/completions")], + ) + .unwrap(), + ); + let gateway::RouteLookup::Healthy(route) = routes.lookup("fixture") else { + panic!("missing route"); + }; + let admission = Arc::new(tokio::sync::Semaphore::new(1)); + let token_permit = Arc::clone(&admission).acquire_owned().await.unwrap(); + let request = gateway::rewrite_responses_request( + br#"{"input":"continue","stream":true}"#, + "served", + ) + .unwrap(); + let response = + super::gateway_responses_stream(Arc::clone(&route), request, token_permit).await; + let client = tokio::spawn(async move { + let _ = axum::body::to_bytes(response.into_body(), 65536).await; + }); + tokio::time::timeout(Duration::from_secs(2), started) + .await + .unwrap() + .unwrap(); + assert_eq!(admission.available_permits(), 0); + client.abort(); + let _ = client.await; + server.await.unwrap(); + assert_eq!(admission.available_permits(), 1); + let mut permits = Vec::new(); + for _ in 0..4 { + permits.push( + tokio::time::timeout(Duration::from_secs(1), route.acquire()) + .await + .unwrap() + .unwrap(), + ); + } + } + } + #[test] fn legacy_recipe_benchmark_and_tuning_paths_are_not_public() { let document = serde_json::to_value(ApiDoc::openapi()).unwrap(); diff --git a/src/spark/gateway.rs b/src/spark/gateway.rs index 2ccc0fe..46958f8 100644 --- a/src/spark/gateway.rs +++ b/src/spark/gateway.rs @@ -1211,11 +1211,7 @@ impl ResponsesEncoder { }) { return Err(invalid_request("upstream tool arguments are invalid")); } - if self.finish_reason.as_deref() == Some("stop") - && !self.reasoning.trim().is_empty() - && self.text.trim().is_empty() - && self.tools.is_empty() - { + if self.reasoning_only_stop() { return Err(OpenAiError { code: "server_error", message: "upstream generation produced no actionable output", @@ -1239,6 +1235,38 @@ impl ResponsesEncoder { ); Ok(()) } + + fn reasoning_only_stop(&self) -> bool { + self.finish_reason.as_deref() == Some("stop") + && !self.reasoning.trim().is_empty() + && self.text.trim().is_empty() + && self.tools.is_empty() + } + + /// Resume an assistant that emitted EOS inside its reasoning block. The + /// engine's own template closes that block before continuing the answer; + /// history, tools and the requested sampling policy remain intact. + pub fn reasoning_only_recovery_request(&self, original: &[u8]) -> Option> { + if !self.reasoning_only_stop() || self.usage.0 == 0 || self.usage.1 == 0 { + return None; + } + let mut body: Value = serde_json::from_slice(original).ok()?; + if body["continue_final_message"] == true { + return None; + } + let budget = body["max_tokens"].as_u64().unwrap_or(MAX_OUTPUT_TOKENS); + let remaining = budget + .checked_sub(self.usage.1) + .filter(|value| *value > 0)?; + body["messages"].as_array_mut()?.push(serde_json::json!({ + "role":"assistant","content":"","reasoning_content":self.reasoning + })); + body["continue_final_message"] = true.into(); + body["add_generation_prompt"] = false.into(); + body["max_tokens"] = remaining.into(); + let bytes = serde_json::to_vec(&body).ok()?; + (bytes.len() <= MAX_COMPLETION_BODY_BYTES).then_some(bytes) + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -3222,6 +3250,119 @@ mod tests { assert_eq!(body["chat_template_kwargs"]["enable_thinking"], false); } + #[test] + fn reasoning_only_recovery_continues_the_assistant_with_remaining_budget() { + let original = serde_json::json!({ + "model":"served", "messages":[{"role":"user","content":"continue"}], + "tools":[{"type":"function","function":{"name":"inspect","parameters":{"type":"object"}}}], + "reasoning_effort":"xhigh", "temperature":1.0, "max_tokens":100, + "stream":true,"stream_options":{"include_usage":true} + }); + let bytes = serde_json::to_vec(&original).unwrap(); + let mut encoder = ResponsesEncoder::new("public".into(), BTreeSet::new()); + for event in [ + GenerationEvent::ReasoningDelta { + text: "Inspect the existing state.".into(), + }, + GenerationEvent::Usage { + prompt_tokens: 50, + completion_tokens: 20, + }, + GenerationEvent::Finished { + finish_reason: Some("stop".into()), + }, + ] { + encoder.accept(event).unwrap(); + } + let recovery: Value = + serde_json::from_slice(&encoder.reasoning_only_recovery_request(&bytes).unwrap()) + .unwrap(); + assert_eq!(recovery["messages"][0], original["messages"][0]); + assert_eq!( + recovery["messages"][1], + serde_json::json!({ + "role":"assistant","content":"","reasoning_content":"Inspect the existing state." + }) + ); + for key in ["tools", "reasoning_effort", "temperature", "stream_options"] { + assert_eq!(recovery[key], original[key]); + } + assert_eq!(recovery["continue_final_message"], true); + assert_eq!(recovery["add_generation_prompt"], false); + assert_eq!(recovery["max_tokens"], 80); + assert_eq!(serde_json::from_slice::(&bytes).unwrap(), original); + encoder + .accept(GenerationEvent::Finished { + finish_reason: Some("length".into()), + }) + .unwrap(); + assert!(encoder.reasoning_only_recovery_request(&bytes).is_none()); + } + + #[test] + fn reasoning_only_recovery_never_retries_actions_or_exhausted_or_unknown_usage() { + let body = br#"{"messages":[{"role":"user","content":"work"}],"max_tokens":20}"#; + let encoder = || { + let mut encoder = ResponsesEncoder::new("public".into(), BTreeSet::new()); + for event in [ + GenerationEvent::ReasoningDelta { + text: "inspect".into(), + }, + GenerationEvent::Finished { + finish_reason: Some("stop".into()), + }, + GenerationEvent::Usage { + prompt_tokens: 50, + completion_tokens: 20, + }, + ] { + encoder.accept(event).unwrap(); + } + encoder + }; + assert!(encoder().reasoning_only_recovery_request(body).is_none()); + let body = br#"{"messages":[{"role":"user","content":"work"}]}"#; + let recovery: Value = + serde_json::from_slice(&encoder().reasoning_only_recovery_request(body).unwrap()) + .unwrap(); + assert_eq!(recovery["max_tokens"], MAX_OUTPUT_TOKENS - 20); + for event in [ + GenerationEvent::TextDelta { + text: "visible answer".into(), + }, + GenerationEvent::ToolCallDelta { + index: 0, + call_id: Some("call_1".into()), + name: Some("inspect".into()), + arguments: "{".into(), + }, + GenerationEvent::Finished { + finish_reason: Some("length".into()), + }, + GenerationEvent::Usage { + prompt_tokens: 50, + completion_tokens: 0, + }, + ] { + let mut encoder = encoder(); + encoder.accept(event).unwrap(); + assert!(encoder.reasoning_only_recovery_request(body).is_none()); + } + assert!( + encoder() + .reasoning_only_recovery_request( + br#"{"messages":[],"continue_final_message":true}"# + ) + .is_none() + ); + let oversized = serde_json::to_vec(&serde_json::json!({"messages":[{"role":"user","content":"x".repeat(MAX_COMPLETION_BODY_BYTES)}]})).unwrap(); + assert!( + encoder() + .reasoning_only_recovery_request(&oversized) + .is_none() + ); + } + #[test] fn codex_compaction_requests_produce_visible_summaries_without_changing_coding_effort() { let mut request = serde_json::json!({ diff --git a/src/spark/upstream.rs b/src/spark/upstream.rs index dc7ffc3..bfac4b3 100644 --- a/src/spark/upstream.rs +++ b/src/spark/upstream.rs @@ -283,6 +283,13 @@ pub struct CompletionStream { } impl CompletionStream { + /// A composing stream owns accounting when it joins multiple generations. + /// In particular, an intermediate upstream Done must not mark the logical + /// response successful before the public encoder validates it. + pub fn take_usage_observer(&mut self) -> Option> { + self.meter.take() + } + pub fn response_failed(&self) { if let Some(meter) = &self.meter { meter.response_failed();