diff --git a/crates/tw-gateway/src/failure.rs b/crates/tw-gateway/src/failure.rs index 6d33d0d1..fbeb2fca 100644 --- a/crates/tw-gateway/src/failure.rs +++ b/crates/tw-gateway/src/failure.rs @@ -50,7 +50,7 @@ pub const BODY_PEEK: usize = 64 * 1024; /// 看一个非 2xx 的回答。`body` 是正文的开头(最多 [`BODY_PEEK`])。 pub fn classify(status: u16, headers: &HeaderMap, body: &[u8], now_ms: u64) -> Verdict { - let text = String::from_utf8_lossy(body).to_lowercase(); + let text = error_text(body); // **正文说的原因优先于状态码**:没钱了有人回 402,有人回 429、400、403, // 甚至 500;额度用完有人回 429,有人回 403 if says(&text, NO_BALANCE) { @@ -85,6 +85,10 @@ const NO_BALANCE: &[&str] = &[ "insufficient account balance", "credit balance is too low", "balance is too low", + "insufficient credit", + "not enough credit", + "don't have enough credit", + "do not have enough credit", "payment required", "billing_hard_limit", "arrearage", @@ -124,6 +128,42 @@ fn says(text: &str, words: &[&str]) -> bool { words.iter().any(|w| text.contains(w)) } +/// 只看错误字段,不把响应里回显的请求、元数据当成失败原因。先解码 JSON, +/// `don\u0027t` 和中文的 Unicode 转义才和原文同义。扁平的中转错误、嵌套的 +/// API 错误、Responses 的失败事件都看;纯文本、截断的 JSON 或没有这些字段的 +/// 格式保留原来的全文回退。 +fn error_text(body: &[u8]) -> String { + let raw = || String::from_utf8_lossy(body).to_lowercase(); + let Ok(value) = serde_json::from_slice::(body) else { + return raw(); + }; + let mut texts = Vec::new(); + for scope in [ + value.pointer("/response/error"), + value.get("error"), + Some(&value), + ] + .into_iter() + .flatten() + { + if let Some(s) = scope.as_str() { + texts.push(s); + } else { + texts.extend( + ["code", "type", "status", "message", "detail"] + .into_iter() + .filter_map(|k| scope.get(k).and_then(|v| v.as_str())), + ); + } + } + if texts.is_empty() { + raw() + } else { + // 不同字段的尾和头不能拼成一条短语。 + texts.join("\0").to_lowercase() + } +} + /// 额度头说用完了没有:`None` 是没说用完,`Some(None)` 是说用完了、没说什么时候 /// 重置,`Some(Some(t))` 是在 `t` 重置。 /// @@ -236,6 +276,14 @@ mod tests { r#"{"error":{"code":"insufficient_quota","message":"You exceeded your current quota"}}"#, ), (402, r#"{"error":{"message":"Insufficient Balance"}}"#), + ( + 400, + r#"{"message":"You have insufficient credits to make this request. Please purchase more credits to continue using the service.","type":"invalid_request_error","code":"BAD_REQUEST"}"#, + ), + ( + 400, + r#"{"error":{"message":"You don't have enough credits for this model."}}"#, + ), (403, r#"{"error":{"code":"Arrearage"}}"#), ( 429, @@ -250,6 +298,84 @@ mod tests { } } + #[test] + fn sufficient_credit_is_not_a_missing_balance() { + let body = r#"{"error":{"type":"invalid_request_error","message":"You have enough credits, but max_tokens must be positive."}}"#; + assert_eq!(verdict(400, body), Verdict::ClientError); + assert_eq!( + verdict(429, body), + Verdict::Failed(Cause::RateLimited { retry_after: None }) + ); + } + + #[test] + fn request_echoes_do_not_override_the_error_reason() { + for (status, message, expected) in [ + (400, "max_tokens must be positive", Verdict::ClientError), + ( + 429, + "too many requests", + Verdict::Failed(Cause::RateLimited { retry_after: None }), + ), + ( + 400, + "unknown model: foo", + Verdict::Failed(Cause::ModelUnavailable), + ), + ] { + let body = serde_json::json!({ + "error": {"message": message}, + "request": {"messages": [{"content": "I don't have enough credits"}]}, + "metadata": {"note": "insufficient balance"} + }) + .to_string(); + assert_eq!(verdict(status, &body), expected, "{body}"); + } + } + + #[test] + fn json_error_fields_are_decoded_across_upstream_envelopes() { + for body in [ + r#"{"message":"You have insufficient credit","code":"BAD_REQUEST"}"#, + r#"{"error":{"message":"You do not have enough credits"}}"#, + r#"{"error":{"message":"Not enough credit"}}"#, + r#"{"error":{"message":"You don\u0027t have enough credits"}}"#, + r#"{"error":{"message":"\u4f59\u989d\u4e0d\u8db3"}}"#, + r#"{"error":"You don\u0027t have enough credits"}"#, + r#"{"type":"response.failed","response":{"error":{"code":"BAD_REQUEST","message":"You don\u0027t have enough credits"}}}"#, + r#"{"error":"BAD_REQUEST","message":"You have insufficient credits"}"#, + ] { + assert_eq!( + verdict(400, body), + Verdict::Failed(Cause::NoBalance), + "{body}" + ); + } + } + + #[test] + fn unfamiliar_or_incomplete_errors_keep_the_text_fallback() { + for body in [ + "You don't have enough credits", + r#"{"errors":[{"reason":"insufficient credits"}]}"#, + r#"{"error":{"message":"insufficient credits""#, + ] { + assert_eq!( + verdict(400, body), + Verdict::Failed(Cause::NoBalance), + "{body}" + ); + } + } + + #[test] + fn separate_error_fields_cannot_form_a_balance_signal() { + assert_eq!( + verdict(400, r#"{"code":"insufficient","message":"credit"}"#), + Verdict::ClientError + ); + } + #[test] fn a_used_up_quota_waits_for_the_reset_the_upstream_names() { let body = format!( diff --git a/crates/tw-gateway/tests/passthrough.rs b/crates/tw-gateway/tests/passthrough.rs index 44d2c114..0998401b 100644 --- a/crates/tw-gateway/tests/passthrough.rs +++ b/crates/tw-gateway/tests/passthrough.rs @@ -837,6 +837,120 @@ async fn a_missing_balance_moves_on_and_sets_the_upstream_aside() { assert_eq!(broke_hits.load(Ordering::SeqCst), 1, "停用着的不再去试"); } +#[tokio::test] +async fn an_ordered_chat_route_moves_on_from_insufficient_credits_and_skips_it_next_time() { + for streaming in [false, true] { + let (broke, broke_hits) = start_answering_upstream( + 400, + "application/json", + r#"{"message":"You have insufficient credits to make this request. Please purchase more credits to continue using the service.","type":"invalid_request_error","code":"BAD_REQUEST"}"#, + ) + .await; + let (content_type, body) = if streaming { + ( + "text/event-stream", + "data: {\"id\":\"chatcmpl-test\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"answer\"},\"finish_reason\":null}]}\n\n\ + data: {\"id\":\"chatcmpl-test\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n\ + data: [DONE]\n\n", + ) + } else { + ( + "application/json", + r#"{"id":"chatcmpl-test","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","content":"answer"},"finish_reason":"stop"}]}"#, + ) + }; + let (good, good_hits) = start_answering_upstream(200, content_type, body).await; + let mut cfg = two(broke, good); + for provider in &mut cfg.providers { + provider.protocol = Some(tw_config::Protocol::OpenaiChat); + } + cfg.groups = vec![ + serde_yaml_ng::from_str("name: ordered\ntype: fallback\nproviders: [first, second]\n") + .unwrap(), + ]; + cfg.routes = vec![tw_engine::RouteSet::default_with(vec![ + serde_yaml_ng::from_str("name: catch-all\nto: ordered\n").unwrap(), + ])]; + let state = tw_gateway::AppState::new(cfg).unwrap(); + let health = state.health.clone(); + let gw = tw_gateway::serve(state, ([127, 0, 0, 1], 0).into()) + .await + .unwrap(); + for _ in 0..2 { + let response = reqwest::Client::new() + .post(format!("http://{gw}/v1/chat/completions")) + .bearer_auth("tw-k") + .json(&serde_json::json!({ + "model": "deepseek-v4.1-flash", + "messages": [{"role": "user", "content": "hello"}], + "stream": streaming + })) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 200); + let text = response.text().await.unwrap(); + assert!(text.contains("answer"), "{text}"); + } + assert!(!health.is_available("first")); + assert_eq!(broke_hits.load(Ordering::SeqCst), 1); + assert_eq!(good_hits.load(Ordering::SeqCst), 2); + } +} + +#[tokio::test] +async fn other_credit_mentions_do_not_pause_or_fail_over_a_client_error() { + for body in [ + r#"{"error":{"type":"invalid_request_error","message":"You have enough credits, but max_tokens must be positive."}}"#, + r#"{"error":{"type":"invalid_request_error","message":"max_tokens must be positive"},"request":{"messages":[{"content":"I don't have enough credits"}]}}"#, + ] { + let (first, _) = start_answering_upstream(400, "application/json", body).await; + let (second, second_hits) = start_answering_upstream(200, "application/json", "{}").await; + let state = tw_gateway::AppState::new(two(first, second)).unwrap(); + let health = state.health.clone(); + let gw = tw_gateway::serve(state, ([127, 0, 0, 1], 0).into()) + .await + .unwrap(); + let response = send_to(gw).await; + assert_eq!(response.status(), 400); + assert_eq!(response.text().await.unwrap(), body); + assert_eq!(second_hits.load(Ordering::SeqCst), 0); + assert!(health.is_available("first")); + } +} + +#[tokio::test] +async fn insufficient_credits_at_the_start_of_a_stream_moves_on_and_pauses() { + let (broke, broke_hits) = start_answering_upstream( + 200, + "text/event-stream", + "event: error\ndata: {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"You don\\u0027t have enough credits\"}}\n\n", + ) + .await; + let (good, _) = start_upstream(true).await; + let state = tw_gateway::AppState::new(two(broke, good)).unwrap(); + let health = state.health.clone(); + let gw = tw_gateway::serve(state, ([127, 0, 0, 1], 0).into()) + .await + .unwrap(); + for _ in 0..2 { + let text = reqwest::Client::new() + .post(format!("http://{gw}/v1/messages")) + .header("x-api-key", "tw-k") + .body(r#"{"model":"claude-sonnet-4-5","stream":true}"#) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + assert!(text.contains("message_stop"), "{text}"); + assert!(!text.contains("have enough credits"), "{text}"); + } + assert!(!health.is_available("first")); + assert_eq!(broke_hits.load(Ordering::SeqCst), 1); +} + #[tokio::test] async fn the_last_upstreams_own_refusal_reaches_the_client() { // 没有下一家可换了:上游自己的原话比「回了 401」有用