From d527e9e629f0b4651c34922e2afe9bd9dd2fbae8 Mon Sep 17 00:00:00 2001 From: sunnysab Date: Wed, 28 Oct 2026 14:43:32 +0800 Subject: [PATCH 1/2] fix(gateway): classify "insufficient credits" as a missing balance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A relay that answered `400 {"code":"BAD_REQUEST"}` with "You have insufficient credits to make this request." matched none of the `NO_BALANCE` phrases: the list knew "insufficient balance" and "credit balance is too low", not "insufficient credits". The verdict fell through to `ClientError`, so the request went back to the client, the upstream was never swapped and never paused — the client retried the same dead upstream until the operator disabled it by hand. Add the credit phrasings, including "don't have enough credits" (the contracting variant a phrase starting with "not enough" misses). --- crates/tw-gateway/src/failure.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/tw-gateway/src/failure.rs b/crates/tw-gateway/src/failure.rs index 6d33d0d1..1446476a 100644 --- a/crates/tw-gateway/src/failure.rs +++ b/crates/tw-gateway/src/failure.rs @@ -85,6 +85,9 @@ const NO_BALANCE: &[&str] = &[ "insufficient account balance", "credit balance is too low", "balance is too low", + "insufficient credit", + "not enough credit", + "have enough credit", "payment required", "billing_hard_limit", "arrearage", @@ -236,6 +239,14 @@ mod tests { r#"{"error":{"code":"insufficient_quota","message":"You exceeded your current quota"}}"#, ), (402, r#"{"error":{"message":"Insufficient Balance"}}"#), + ( + 400, + r#"{"message":"You have insufficient credits to make this request. Please purchase more credits to continue using the service.","type":"invalid_request_error","code":"BAD_REQUEST"}"#, + ), + ( + 400, + r#"{"error":{"message":"You don't have enough credits for this model."}}"#, + ), (403, r#"{"error":{"code":"Arrearage"}}"#), ( 429, From 6b3fa2e5e82ef70aeec1d153a63ec364577985b8 Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Sat, 31 Oct 2026 20:37:14 +0800 Subject: [PATCH 2/2] fix(gateway): classify decoded error fields for balance failover Matching the whole JSON response can mistake echoed request text for a balance failure, while "have enough credit" also matches affirmative messages. Inspect decoded error fields from flat, nested and Responses error envelopes, preserving text fallback for incomplete or unfamiliar errors and preventing phrases from crossing field boundaries. Use explicit negative credit phrases and cover escaped messages, request echoes and sufficient-credit errors. Exercise the reported 400 on an ordered Chat Completions route in both streaming modes, including skipping the paused provider on subsequent requests and handling an error at the start of an SSE response. --- crates/tw-gateway/src/failure.rs | 119 ++++++++++++++++++++++++- crates/tw-gateway/tests/passthrough.rs | 114 +++++++++++++++++++++++ 2 files changed, 231 insertions(+), 2 deletions(-) diff --git a/crates/tw-gateway/src/failure.rs b/crates/tw-gateway/src/failure.rs index 1446476a..fbeb2fca 100644 --- a/crates/tw-gateway/src/failure.rs +++ b/crates/tw-gateway/src/failure.rs @@ -50,7 +50,7 @@ pub const BODY_PEEK: usize = 64 * 1024; /// 看一个非 2xx 的回答。`body` 是正文的开头（最多 [`BODY_PEEK`]）。 pub fn classify(status: u16, headers: &HeaderMap, body: &[u8], now_ms: u64) -> Verdict { - let text = String::from_utf8_lossy(body).to_lowercase(); + let text = error_text(body); // **正文说的原因优先于状态码**：没钱了有人回 402，有人回 429、400、403， // 甚至 500；额度用完有人回 429，有人回 403 if says(&text, NO_BALANCE) { @@ -87,7 +87,8 @@ const NO_BALANCE: &[&str] = &[ "balance is too low", "insufficient credit", "not enough credit", - "have enough credit", + "don't have enough credit", + "do not have enough credit", "payment required", "billing_hard_limit", "arrearage", @@ -127,6 +128,42 @@ fn says(text: &str, words: &[&str]) -> bool { words.iter().any(|w| text.contains(w)) } +/// 只看错误字段，不把响应里回显的请求、元数据当成失败原因。先解码 JSON， +/// `don\u0027t` 和中文的 Unicode 转义才和原文同义。扁平的中转错误、嵌套的 +/// API 错误、Responses 的失败事件都看；纯文本、截断的 JSON 或没有这些字段的 +/// 格式保留原来的全文回退。 +fn error_text(body: &[u8]) -> String { + let raw = || String::from_utf8_lossy(body).to_lowercase(); + let Ok(value) = serde_json::from_slice::(body) else { + return raw(); + }; + let mut texts = Vec::new(); + for scope in [ + value.pointer("/response/error"), + value.get("error"), + Some(&value), + ] + .into_iter() + .flatten() + { + if let Some(s) = scope.as_str() { + texts.push(s); + } else { + texts.extend( + ["code", "type", "status", "message", "detail"] + .into_iter() + .filter_map(|k| scope.get(k).and_then(|v| v.as_str())), + ); + } + } + if texts.is_empty() { + raw() + } else { + // 不同字段的尾和头不能拼成一条短语。 + texts.join("\0").to_lowercase() + } +} + /// 额度头说用完了没有：`None` 是没说用完，`Some(None)` 是说用完了、没说什么时候 /// 重置，`Some(Some(t))` 是在 `t` 重置。 /// @@ -261,6 +298,84 @@ mod tests { } } + #[test] + fn sufficient_credit_is_not_a_missing_balance() { + let body = r#"{"error":{"type":"invalid_request_error","message":"You have enough credits, but max_tokens must be positive."}}"#; + assert_eq!(verdict(400, body), Verdict::ClientError); + assert_eq!( + verdict(429, body), + Verdict::Failed(Cause::RateLimited { retry_after: None }) + ); + } + + #[test] + fn request_echoes_do_not_override_the_error_reason() { + for (status, message, expected) in [ + (400, "max_tokens must be positive", Verdict::ClientError), + ( + 429, + "too many requests", + Verdict::Failed(Cause::RateLimited { retry_after: None }), + ), + ( + 400, + "unknown model: foo", + Verdict::Failed(Cause::ModelUnavailable), + ), + ] { + let body = serde_json::json!({ + "error": {"message": message}, + "request": {"messages": [{"content": "I don't have enough credits"}]}, + "metadata": {"note": "insufficient balance"} + }) + .to_string(); + assert_eq!(verdict(status, &body), expected, "{body}"); + } + } + + #[test] + fn json_error_fields_are_decoded_across_upstream_envelopes() { + for body in [ + r#"{"message":"You have insufficient credit","code":"BAD_REQUEST"}"#, + r#"{"error":{"message":"You do not have enough credits"}}"#, + r#"{"error":{"message":"Not enough credit"}}"#, + r#"{"error":{"message":"You don\u0027t have enough credits"}}"#, + r#"{"error":{"message":"\u4f59\u989d\u4e0d\u8db3"}}"#, + r#"{"error":"You don\u0027t have enough credits"}"#, + r#"{"type":"response.failed","response":{"error":{"code":"BAD_REQUEST","message":"You don\u0027t have enough credits"}}}"#, + r#"{"error":"BAD_REQUEST","message":"You have insufficient credits"}"#, + ] { + assert_eq!( + verdict(400, body), + Verdict::Failed(Cause::NoBalance), + "{body}" + ); + } + } + + #[test] + fn unfamiliar_or_incomplete_errors_keep_the_text_fallback() { + for body in [ + "You don't have enough credits", + r#"{"errors":[{"reason":"insufficient credits"}]}"#, + r#"{"error":{"message":"insufficient credits""#, + ] { + assert_eq!( + verdict(400, body), + Verdict::Failed(Cause::NoBalance), + "{body}" + ); + } + } + + #[test] + fn separate_error_fields_cannot_form_a_balance_signal() { + assert_eq!( + verdict(400, r#"{"code":"insufficient","message":"credit"}"#), + Verdict::ClientError + ); + } + #[test] fn a_used_up_quota_waits_for_the_reset_the_upstream_names() { let body = format!( diff --git a/crates/tw-gateway/tests/passthrough.rs b/crates/tw-gateway/tests/passthrough.rs index 44d2c114..0998401b 100644 --- a/crates/tw-gateway/tests/passthrough.rs +++ b/crates/tw-gateway/tests/passthrough.rs @@ -837,6 +837,120 @@ async fn a_missing_balance_moves_on_and_sets_the_upstream_aside() { assert_eq!(broke_hits.load(Ordering::SeqCst), 1, "停用着的不再去试"); } +#[tokio::test] +async fn an_ordered_chat_route_moves_on_from_insufficient_credits_and_skips_it_next_time() { + for streaming in [false, true] { + let (broke, broke_hits) = start_answering_upstream( + 400, + "application/json", + r#"{"message":"You have insufficient credits to make this request. Please purchase more credits to continue using the service.","type":"invalid_request_error","code":"BAD_REQUEST"}"#, + ) + .await; + let (content_type, body) = if streaming { + ( + "text/event-stream", + "data: {\"id\":\"chatcmpl-test\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"answer\"},\"finish_reason\":null}]}\n\n\ + data: {\"id\":\"chatcmpl-test\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n\ + data: [DONE]\n\n", + ) + } else { + ( + "application/json", + r#"{"id":"chatcmpl-test","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","content":"answer"},"finish_reason":"stop"}]}"#, + ) + }; + let (good, good_hits) = start_answering_upstream(200, content_type, body).await; + let mut cfg = two(broke, good); + for provider in &mut cfg.providers { + provider.protocol = Some(tw_config::Protocol::OpenaiChat); + } + cfg.groups = vec![ + serde_yaml_ng::from_str("name: ordered\ntype: fallback\nproviders: [first, second]\n") + .unwrap(), + ]; + cfg.routes = vec![tw_engine::RouteSet::default_with(vec![ + serde_yaml_ng::from_str("name: catch-all\nto: ordered\n").unwrap(), + ])]; + let state = tw_gateway::AppState::new(cfg).unwrap(); + let health = state.health.clone(); + let gw = tw_gateway::serve(state, ([127, 0, 0, 1], 0).into()) + .await + .unwrap(); + for _ in 0..2 { + let response = reqwest::Client::new() + .post(format!("http://{gw}/v1/chat/completions")) + .bearer_auth("tw-k") + .json(&serde_json::json!({ + "model": "deepseek-v4.1-flash", + "messages": [{"role": "user", "content": "hello"}], + "stream": streaming + })) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 200); + let text = response.text().await.unwrap(); + assert!(text.contains("answer"), "{text}"); + } + assert!(!health.is_available("first")); + assert_eq!(broke_hits.load(Ordering::SeqCst), 1); + assert_eq!(good_hits.load(Ordering::SeqCst), 2); + } +} + +#[tokio::test] +async fn other_credit_mentions_do_not_pause_or_fail_over_a_client_error() { + for body in [ + r#"{"error":{"type":"invalid_request_error","message":"You have enough credits, but max_tokens must be positive."}}"#, + r#"{"error":{"type":"invalid_request_error","message":"max_tokens must be positive"},"request":{"messages":[{"content":"I don't have enough credits"}]}}"#, + ] { + let (first, _) = start_answering_upstream(400, "application/json", body).await; + let (second, second_hits) = start_answering_upstream(200, "application/json", "{}").await; + let state = tw_gateway::AppState::new(two(first, second)).unwrap(); + let health = state.health.clone(); + let gw = tw_gateway::serve(state, ([127, 0, 0, 1], 0).into()) + .await + .unwrap(); + let response = send_to(gw).await; + assert_eq!(response.status(), 400); + assert_eq!(response.text().await.unwrap(), body); + assert_eq!(second_hits.load(Ordering::SeqCst), 0); + assert!(health.is_available("first")); + } +} + +#[tokio::test] +async fn insufficient_credits_at_the_start_of_a_stream_moves_on_and_pauses() { + let (broke, broke_hits) = start_answering_upstream( + 200, + "text/event-stream", + "event: error\ndata: {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You don\\u0027t have enough credits\"}}\n\n", + ) + .await; + let (good, _) = start_upstream(true).await; + let state = tw_gateway::AppState::new(two(broke, good)).unwrap(); + let health = state.health.clone(); + let gw = tw_gateway::serve(state, ([127, 0, 0, 1], 0).into()) + .await + .unwrap(); + for _ in 0..2 { + let text = reqwest::Client::new() + .post(format!("http://{gw}/v1/messages")) + .header("x-api-key", "tw-k") + .body(r#"{"model":"claude-sonnet-4-5","stream":true}"#) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + assert!(text.contains("message_stop"), "{text}"); + assert!(!text.contains("have enough credits"), "{text}"); + } + assert!(!health.is_available("first")); + assert_eq!(broke_hits.load(Ordering::SeqCst), 1); +} + #[tokio::test] async fn the_last_upstreams_own_refusal_reaches_the_client() { // 没有下一家可换了：上游自己的原话比「回了 401」有用