diff --git a/Cargo.lock b/Cargo.lock index c66cd1d..27605e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,15 +58,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -dependencies = [ - "derive_arbitrary", -] - [[package]] name = "asn1-rs" version = "0.6.2" @@ -167,6 +158,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "clap" version = "4.6.7" @@ -313,17 +310,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "derive_arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "digest" version = "0.10.7" @@ -654,7 +640,7 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jaynshare" -version = "2.3.3" +version = "2.4.0" dependencies = [ "base64", "bytes", @@ -780,6 +766,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -1001,9 +999,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "log", "once_cell", @@ -1054,9 +1052,9 @@ checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ruzstd" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a252f5e20f038fe7b4ea53e073e65398d652c864cc162fc77c56c2f13717b888" +checksum = "69e4ce4a039f42bf25920b95667e4d789e98749ab11bbb8d2c2936f788421bb5" [[package]] name = "schannel" @@ -1493,6 +1491,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -1755,15 +1759,16 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zip" -version = "4.6.1" +version = "9.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa8cd6af31c3b31c6631b8f483848b91589021b28fffe50adada48d4f4d2ed1" +checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e" dependencies = [ - "arbitrary", "crc32fast", "flate2", "indexmap", "memchr", + "nix", + "typed-path", "zopfli", ] diff --git a/Cargo.toml b/Cargo.toml index 8e72336..1dc5b17 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jaynshare" -version = "2.3.3" +version = "2.4.0" edition = "2024" rust-version = "1.95" publish = false @@ -39,7 +39,7 @@ x509-parser = { version = "0.16", features = ["verify"] } uuid = { version = "1", features = ["v4", "serde"] } # The enrollment bundle and client kit are ZIP archives; stored plus deflate # covers what the release tooling produces. -zip = { version = "4", default-features = false, features = ["deflate"] } +zip = { version = "9", default-features = false, features = ["deflate"] } # The Linux platform archive is a `.tar.gz`; native install reads it. flate2 = "1" tar = { version = "0.4", default-features = false } diff --git a/NOTICE.md b/NOTICE.md index 5d1580e..5fbd8b9 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -14,7 +14,6 @@ set is enforced in CI by `cargo-deny` (`deny.toml`). | `anstyle-parse` | 1.0.0 | MIT OR Apache-2.0 | https://github.com/rust-cli/anstyle.git | | `anstyle-query` | 1.1.5 | MIT OR Apache-2.0 | https://github.com/rust-cli/anstyle.git | | `anstyle-wincon` | 3.0.11 | MIT OR Apache-2.0 | https://github.com/rust-cli/anstyle.git | -| `arbitrary` | 1.4.2 | MIT OR Apache-2.0 | https://github.com/rust-fuzz/arbitrary/ | | `asn1-rs` | 0.6.2 | MIT OR Apache-2.0 | https://github.com/rusticata/asn1-rs.git | | `asn1-rs-derive` | 0.5.1 | MIT OR Apache-2.0 | https://github.com/rusticata/asn1-rs.git | | `asn1-rs-impl` | 0.2.0 | MIT/Apache-2.0 | https://github.com/rusticata/asn1-rs.git | @@ -27,6 +26,7 @@ set is enforced in CI by `cargo-deny` (`deny.toml`). | `bytes` | 1.12.1 | MIT | https://github.com/tokio-rs/bytes | | `cc` | 1.4.6 | MIT OR Apache-2.0 | https://github.com/rust-lang/cc-rs | | `cfg-if` | 1.0.4 | MIT OR Apache-2.0 | https://github.com/rust-lang/cfg-if | +| `cfg_aliases` | 0.2.2 | MIT | https://github.com/katharostech/cfg_aliases | | `clap` | 4.6.7 | MIT OR Apache-2.0 | https://github.com/clap-rs/clap | | `clap_builder` | 4.6.7 | MIT OR Apache-2.0 | https://github.com/clap-rs/clap | | `clap_derive` | 4.6.7 | MIT OR Apache-2.0 | https://github.com/clap-rs/clap | @@ -42,7 +42,6 @@ set is enforced in CI by `cargo-deny` (`deny.toml`). | `data-encoding` | 2.11.1 | MIT | https://github.com/ia0/data-encoding | | `der-parser` | 9.0.0 | MIT/Apache-2.0 | https://github.com/rusticata/der-parser.git | | `deranged` | 0.5.8 | MIT OR Apache-2.0 | https://github.com/jhpratt/deranged | -| `derive_arbitrary` | 1.4.2 | MIT OR Apache-2.0 | https://github.com/rust-fuzz/arbitrary | | `digest` | 0.10.7 | MIT OR Apache-2.0 | https://github.com/RustCrypto/traits | | `displaydoc` | 0.2.7 | MIT OR Apache-2.0 | https://github.com/yaahc/displaydoc | | `document-features` | 0.2.12 | MIT OR Apache-2.0 | https://github.com/slint-ui/document-features | @@ -87,6 +86,7 @@ set is enforced in CI by `cargo-deny` (`deny.toml`). | `minimal-lexical` | 0.2.1 | MIT/Apache-2.0 | https://github.com/Alexhuszagh/minimal-lexical | | `miniz_oxide` | 0.9.1 | MIT OR Zlib OR Apache-2.0 | https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide | | `mio` | 1.2.3 | MIT | https://github.com/tokio-rs/mio | +| `nix` | 0.31.3 | MIT | https://github.com/nix-rust/nix | | `nom` | 7.1.3 | MIT | https://github.com/Geal/nom | | `num-bigint` | 0.4.8 | MIT OR Apache-2.0 | https://github.com/rust-num/num-bigint | | `num-conv` | 0.2.2 | MIT OR Apache-2.0 | https://github.com/jhpratt/num-conv | @@ -111,12 +111,12 @@ set is enforced in CI by `cargo-deny` (`deny.toml`). | `rtoolbox` | 0.0.6 | Apache-2.0 | https://github.com/conradkleinespel/rtoolbox | | `rusticata-macros` | 4.1.0 | MIT/Apache-2.0 | https://github.com/rusticata/rusticata-macros.git | | `rustix` | 1.1.5 | Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT | https://github.com/bytecodealliance/rustix | -| `rustls` | 0.23.45 | Apache-2.0 OR ISC OR MIT | https://github.com/rustls/rustls | +| `rustls` | 0.23.46 | Apache-2.0 OR ISC OR MIT | https://github.com/rustls/rustls | | `rustls-native-certs` | 0.8.4 | Apache-2.0 OR ISC OR MIT | https://github.com/rustls/rustls-native-certs | | `rustls-pki-types` | 1.15.1 | MIT OR Apache-2.0 | https://github.com/rustls/pki-types | | `rustls-webpki` | 0.103.15 | ISC | https://github.com/rustls/webpki | | `rustversion` | 1.0.23 | MIT OR Apache-2.0 | https://github.com/dtolnay/rustversion | -| `ruzstd` | 0.9.0 | MIT | https://github.com/KillingSpark/zstd-rs | +| `ruzstd` | 0.9.1 | MIT | https://github.com/KillingSpark/zstd-rs | | `schannel` | 0.1.29 | MIT | https://github.com/steffengy/schannel-rs | | `scopeguard` | 1.2.0 | MIT OR Apache-2.0 | https://github.com/bluss/scopeguard | | `security-framework` | 3.7.0 | MIT OR Apache-2.0 | https://github.com/kornelski/rust-security-framework | @@ -162,6 +162,7 @@ set is enforced in CI by `cargo-deny` (`deny.toml`). | `tracing-core` | 0.1.36 | MIT | https://github.com/tokio-rs/tracing | | `tracing-subscriber` | 0.3.23 | MIT | https://github.com/tokio-rs/tracing | | `try-lock` | 0.2.5 | MIT | https://github.com/seanmonstar/try-lock | +| `typed-path` | 0.12.3 | MIT OR Apache-2.0 | https://github.com/chipsenkbeil/typed-path | | `typenum` | 1.20.1 | MIT OR Apache-2.0 | https://github.com/paholg/typenum | | `unicase` | 2.9.0 | MIT OR Apache-2.0 | https://github.com/seanmonstar/unicase | | `unicode-ident` | 1.0.24 | (MIT OR Apache-2.0) AND Unicode-3.0 | https://github.com/dtolnay/unicode-ident | @@ -194,7 +195,7 @@ set is enforced in CI by `cargo-deny` (`deny.toml`). | `x509-parser` | 0.16.0 | MIT OR Apache-2.0 | https://github.com/rusticata/x509-parser.git | | `yasna` | 0.5.2 | MIT OR Apache-2.0 | https://github.com/qnighy/yasna.rs | | `zeroize` | 1.9.0 | Apache-2.0 OR MIT | https://github.com/RustCrypto/utils | -| `zip` | 4.6.1 | MIT | https://github.com/zip-rs/zip2.git | +| `zip` | 9.0.5 | MIT | https://github.com/zip-rs/zip2 | | `zlib-rs` | 0.6.8 | Zlib | https://github.com/trifectatechfoundation/zlib-rs | | `zmij` | 1.0.23 | MIT | https://github.com/dtolnay/zmij | | `zopfli` | 0.8.3 | Apache-2.0 | https://github.com/zopfli-rs/zopfli | diff --git a/README.md b/README.md index 5e651c8..3c4d77b 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,8 @@ updates itself whenever the server does. `jaynshare claude` and `jaynshare codex` open a picker when the pool cannot choose for you; `--account ` names the account, `--auto` lets the pool -pick, and `--direct` runs the tool outside the pool under your own login. +pick and move the session to another account when its own runs out, and +`--direct` runs the tool outside the pool under your own login. `jaynshare` alone opens one picker over every pooled account, grouped by tool with each account's five-hour and weekly usage, and launches the tool of the account you pick. @@ -211,8 +212,8 @@ included, from its next message, without quitting Desktop; a moved conversation starts without its prompt cache. With `--log`, or when standard error is not a terminal, it prints one line per request instead. A second command with the same account and enrollment reuses the running one. With -`--auto`, existing sessions keep their account; an exhausted account returns -an error instead of switching by itself. +`--auto`, a conversation whose account runs out moves to another one by +itself, without its prompt cache; a pinned account returns an error instead. Chat and Code go through the pool, with the models its accounts offer. Attachments, Cowork, Chrome, connectors, projects, memory and artifacts are diff --git a/src/bundle.rs b/src/bundle.rs index 2991cc8..1cd2c3b 100644 --- a/src/bundle.rs +++ b/src/bundle.rs @@ -367,7 +367,10 @@ pub fn read_zip(path: &Path) -> Result>, String> { let mut entry = archive .by_index(index) .map_err(|e| format!("{}: damaged member: {e}", path.display()))?; - let name = entry.name().to_string(); + let name = entry + .name() + .map_err(|e| format!("{}: unreadable member name: {e}", path.display()))? + .into_owned(); if name.starts_with('/') || name.contains("..") || name.contains('\\') { return Err(format!( "archive member {name:?} is not a relative kit path" diff --git a/src/cli/edit.rs b/src/cli/edit.rs index 0c81e07..37886c3 100644 --- a/src/cli/edit.rs +++ b/src/cli/edit.rs @@ -105,6 +105,7 @@ tls = "identity" # first_byte_timeout_seconds = 120 # [selection] # switch_threshold = 0.98 +# failover = true # [quota] # probe_enabled = false # [logging] diff --git a/src/config/mod.rs b/src/config/mod.rs index 8b21f31..8cd6446 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -66,6 +66,8 @@ pub struct QuotaSettings { pub struct SelectionSettings { pub switch_threshold: f64, pub distribute_sessions: bool, + /// An unpinned session moves off an account that cannot serve it. + pub failover: bool, pub blocked_models: Vec, pub priorities: Vec, pub routes: Vec, @@ -517,6 +519,7 @@ mod tests { assert_eq!(c.data_plane.listen.to_string(), DEFAULT_LISTEN); assert_eq!(c.data_plane.first_byte_timeout_seconds, 120); assert_eq!(c.selection.switch_threshold, 0.98); + assert!(c.selection.failover); assert_eq!(c.data_plane.telemetry_policy, TelemetryPolicy::Block); assert!(c.data_plane.upstream_origin.is_none()); assert_eq!(c.audit.retained_files, 7); diff --git a/src/config/validate.rs b/src/config/validate.rs index 11e4b8c..6d51548 100644 --- a/src/config/validate.rs +++ b/src/config/validate.rs @@ -302,6 +302,7 @@ impl Validator<'_> { &[ "switch_threshold", "distribute_sessions", + "failover", "blocked_models", "priorities", "routes", @@ -342,6 +343,7 @@ impl Validator<'_> { SelectionSettings { switch_threshold, distribute_sessions: self.boolean(node, "distribute_sessions", false), + failover: self.boolean(node, "failover", true), blocked_models: self .string_array(node, "blocked_models") .unwrap_or_default(), diff --git a/src/data_plane/exchange.rs b/src/data_plane/exchange.rs index deeb056..04c8a90 100644 --- a/src/data_plane/exchange.rs +++ b/src/data_plane/exchange.rs @@ -459,17 +459,20 @@ pub async fn run( headers.insert(HOST, server.upstream.host_header(provider)); let uri = server.upstream.uri_for(provider, &path_and_query); - // A failure that is neither a network failure nor a status is the one - // path that may try another account, and only for an exchange that is - // neither pinned nor a session's: a session's first attempt is its - // binding, so its exchange has one candidate. The exclusion set exists - // for the sessionless case. + // A failure that is neither a network failure nor a status may try + // another account, and only for an exchange that is neither pinned nor a + // session's: a session's first attempt is its binding, so its exchange + // has one candidate. The exclusion set exists for the sessionless case, + // and for failover past an exhausted account. let one_candidate = pin.is_some() || session.is_some(); + let fails_over = pin.is_none() && settings.selection.failover; let mut exclusion: Vec = Vec::new(); let mut first_account: Option = None; let mut other_failure: Option = None; + // The last exhaustion 429 passed over, relayed when nobody else serves. + let mut passed_over = None; - loop { + let (handle, mut response_parts, incoming, permit, collected) = loop { let facts = RequestFacts { exclusion: &exclusion, pin, @@ -487,6 +490,9 @@ pub async fn run( { Ok(choice) => choice, Err(nobody) => { + if let Some(relayed) = passed_over { + break relayed; + } exchange.record.no_service_reason = Some(reason_name(&nobody.reason)); // The refusal names the rule that produced it — // the pin, or the session's binding. @@ -585,7 +591,7 @@ pub async fn run( }) .await; - let (mut response_parts, incoming, permit, collected) = match attempt { + let (response_parts, incoming, permit, collected) = match attempt { Attempt::Served { response, permit, @@ -641,12 +647,17 @@ pub async fn run( parts, collected, permit, - } => ( - parts, - Either::Right(Full::new(collected.clone().unwrap_or_default())), - permit, - collected, - ), + exhausted, + } => { + let body = Either::Right(Full::new(collected.clone().unwrap_or_default())); + if exhausted && fails_over { + tracing::info!(event = "exhausted_passed_over", account = %display_name, "exhaustion 429; trying another account"); + exclusion.push(handle); + passed_over = Some((handle, parts, body, permit, collected)); + continue; + } + (parts, body, permit, collected) + } Attempt::Network(message) => { tracing::warn!(event = "upstream_network_failure", account = %display_name, error = %message, "second failure; closing the client connection"); exchange.write(None, Some(ErrorClass::Upstream)); @@ -718,63 +729,65 @@ pub async fn run( } }; - // Relayed as is; a 429 reaches the client classified and - // byte-identical. - strip_response_headers(&mut response_parts.headers); - let status = response_parts.status; - if let Some(c) = &mut capture { - c.response(status, &response_parts.headers); - } - let content_type = response_parts - .headers - .get(CONTENT_TYPE) - .and_then(|v| v.to_str().ok()); - let usage = UsageExtractor::for_content_type(content_type); - let idle = Duration::from_secs(settings.data_plane.body_idle_timeout_seconds); - exchange.record.status = Some(status.as_u16()); - let error_class = match status.as_u16() { - 429 => Some(ErrorClass::RateLimit), - s if s >= 500 => Some(ErrorClass::Upstream), - _ => None, + break (handle, response_parts, incoming, permit, collected); + }; + + // Relayed as is; a 429 reaches the client classified and + // byte-identical. + strip_response_headers(&mut response_parts.headers); + let status = response_parts.status; + if let Some(c) = &mut capture { + c.response(status, &response_parts.headers); + } + let content_type = response_parts + .headers + .get(CONTENT_TYPE) + .and_then(|v| v.to_str().ok()); + let usage = UsageExtractor::for_content_type(content_type); + let idle = Duration::from_secs(settings.data_plane.body_idle_timeout_seconds); + exchange.record.status = Some(status.as_u16()); + let error_class = match status.as_u16() { + 429 => Some(ErrorClass::RateLimit), + s if s >= 500 => Some(ErrorClass::Upstream), + _ => None, + }; + let server_for_end = Arc::clone(&server); + let mut exchange_at_end = exchange; + let on_end = Box::new(move |end: BodyEnd| { + let (tokens, class) = match end { + BodyEnd::Complete(t) | BodyEnd::Dropped(t) => (t, error_class), + BodyEnd::Failed(t) => (t, Some(ErrorClass::Upstream)), }; - let server_for_end = Arc::clone(&server); - let mut exchange_at_end = exchange; - let on_end = Box::new(move |end: BodyEnd| { - let (tokens, class) = match end { - BodyEnd::Complete(t) | BodyEnd::Dropped(t) => (t, error_class), - BodyEnd::Failed(t) => (t, Some(ErrorClass::Upstream)), + if tokens.input > 0 || tokens.output > 0 { + // Usage totals are runtime facts; the + // traffic counters alone never schedule a quota-state write. + server_for_end.pool.lock().expect("pool lock").add_usage( + handle, + tokens.input, + tokens.output, + ); + } + exchange_at_end.write(Some(status.as_u16()), class); + drop(permit); + }); + let body = match collected { + // A classified 429 the proxy already read: relayed as one frame. + Some(bytes) => { + let tokens = { + let mut usage = UsageExtractor::for_content_type(content_type); + usage.feed(&bytes); + usage.finish() }; - if tokens.input > 0 || tokens.output > 0 { - // Usage totals are runtime facts; the - // traffic counters alone never schedule a quota-state write. - server_for_end.pool.lock().expect("pool lock").add_usage( - handle, - tokens.input, - tokens.output, - ); - } - exchange_at_end.write(Some(status.as_u16()), class); - drop(permit); - }); - let body = match collected { - // A classified 429 the proxy already read: relayed as one frame. - Some(bytes) => { - let tokens = { - let mut usage = UsageExtractor::for_content_type(content_type); - usage.feed(&bytes); - usage.finish() - }; - relay::FullBody::new(bytes, tokens, on_end).boxed() - } - None => { - let Either::Left(incoming) = incoming else { - unreachable!("an unclassified body streams, it is not pre-read") - }; - RelayBody::new(incoming, idle, usage, capture, on_end).boxed() - } - }; - return Ok(Response::from_parts(response_parts, body)); - } + relay::FullBody::new(bytes, tokens, on_end).boxed() + } + None => { + let Either::Left(incoming) = incoming else { + unreachable!("an unclassified body streams, it is not pre-read") + }; + RelayBody::new(incoming, idle, usage, capture, on_end).boxed() + } + }; + Ok(Response::from_parts(response_parts, body)) } fn build_request(method: &Method, uri: &Uri, headers: &HeaderMap, body: Bytes) -> Request { @@ -826,6 +839,7 @@ enum Attempt { parts: http::response::Parts, collected: Option, permit: OwnedSemaphorePermit, + exhausted: bool, }, /// The second network failure — close the client connection. Network(String), @@ -1033,11 +1047,12 @@ async fn attempt_candidate(input: AttemptInput<'_>) -> Attempt { pool.quota_hold_end(handle, OffsetDateTime::now_utc()) }; let hold_end = hold.map(crate::timestamp::rfc3339); - tracing::info!(event = "exhausted_relayed", account = %display_name, hold_end = hold_end.as_deref().unwrap_or("none"), "exhaustion 429 relayed with its retry-after"); + tracing::info!(event = "exhausted", account = %display_name, hold_end = hold_end.as_deref().unwrap_or("none"), "exhaustion 429"); break Attempt::Classified429 { parts: response_parts, collected, permit, + exhausted: true, }; } quota::Classification::Throttle => { @@ -1053,6 +1068,7 @@ async fn attempt_candidate(input: AttemptInput<'_>) -> Attempt { parts: response_parts, collected, permit, + exhausted: false, }; } } diff --git a/src/desktop/app.rs b/src/desktop/app.rs index 60c60fd..0fccea8 100644 --- a/src/desktop/app.rs +++ b/src/desktop/app.rs @@ -8,6 +8,10 @@ use tokio::sync::OnceCell; use super::{bundle, compatibility, config}; +/// Generous: the first run of a freshly installed binary waits for macOS's +/// security scan. +const COMMAND_TIMEOUT: Duration = Duration::from_secs(15); + type Stamp = (u64, SystemTime); pub struct App { @@ -175,7 +179,7 @@ fn stamp(path: &Path) -> Result { pub async fn running(bundle: &Path) -> Result { let path = bundle.join("Contents/MacOS/Claude"); let result = tokio::time::timeout( - Duration::from_secs(2), + COMMAND_TIMEOUT, Command::new("pgrep") .args(["-f", "-l"]) .arg(path) @@ -214,7 +218,7 @@ async fn policy() -> Result<(), String> { } async fn output(command: &mut Command) -> Result { - let result = tokio::time::timeout(Duration::from_secs(3), command.kill_on_drop(true).output()) + let result = tokio::time::timeout(COMMAND_TIMEOUT, command.kill_on_drop(true).output()) .await .map_err(|_| "Desktop compatibility command timed out")? .map_err(|e| e.to_string())?; diff --git a/src/pool/mod.rs b/src/pool/mod.rs index 0d59e31..ada4ce3 100644 --- a/src/pool/mod.rs +++ b/src/pool/mod.rs @@ -388,8 +388,9 @@ impl Pool { } if choice.binds && let Some(key) = session + && let Some(from) = self.sessions.bind(key, choice.handle) { - self.sessions.bind(key, choice.handle); + tracing::info!(event = "session_moved", from = %from, to = %choice.handle, "failover moved the session to another account"); } Ok(choice) } diff --git a/src/pool/selection.rs b/src/pool/selection.rs index 938b3ed..e66ca74 100644 --- a/src/pool/selection.rs +++ b/src/pool/selection.rs @@ -111,7 +111,7 @@ pub struct Choice { pub default_moved_from: Option, /// The old default's reason, beside `default_moved_from`. pub default_moved_cause: Option, - /// The attempt is a session's first, so this answer binds it. + /// The answer binds the session: its first attempt, or a failover. pub binds: bool, /// The answer came from the request-model-only pass. pub advisor_fallback: bool, @@ -502,7 +502,9 @@ impl<'a> Pass<'a> { } fn run(&self) -> Result { - let binds = self.facts.session && self.facts.binding.is_none(); + let bound = self.facts.binding.and_then(|handle| self.bound(handle)); + let moves = self.settings.failover && matches!(bound, Some(Err(_))); + let binds = self.facts.session && (self.facts.binding.is_none() || moves); let choice = |handle, cause| Choice { handle, cause, @@ -539,9 +541,10 @@ impl<'a> Pass<'a> { { return Ok(choice(preferred, Cause::RoutePreference)); } - // Rule 4: a bound session is served by its binding and nothing else runs. - if let Some(bound) = self.facts.binding - && let Some(answer) = self.bound(bound) + // Rule 4: a bound session is served by its binding and nothing else + // runs, unless the binding cannot serve and failover moves it. + if let Some(answer) = bound + && !moves { return answer; } @@ -1113,7 +1116,11 @@ mod tests { (bound, Cause::Session, false) ); - // Under a hold, or disabled, the bound account ends the exchange naming itself. + // Without failover, a held or disabled binding ends the exchange naming itself. + let s = SelectionSettings { + failover: false, + ..s + }; let mut held = oauth("a", 0.5); held.quota[0].exhaustion_hold_until = Some(NOW + time::Duration::minutes(5)); let f = Fixture::new(vec![held.clone(), b.clone()], Some(b.handle)); @@ -1140,6 +1147,64 @@ mod tests { ); } + #[test] + fn failover_rebinds_a_session_whose_binding_cannot_serve() { + let s = settings(); + let b = oauth("b", 0.1); + let mut held = oauth("a", 0.5); + held.quota[0].exhaustion_hold_until = Some(NOW + time::Duration::minutes(5)); + let mut off = oauth("c", 0.5); + off.enabled = false; + let f = Fixture::new(vec![held.clone(), off.clone(), b.clone()], Some(b.handle)); + for bound in [held.handle, off.handle] { + let facts = RequestFacts { + session: true, + binding: Some(bound), + ..RequestFacts::default() + }; + let choice = select(&facts, &f.snapshot(), &s).unwrap(); + assert_eq!( + (choice.handle, choice.cause, choice.binds), + (b.handle, Cause::Default, true) + ); + } + // An exhaustion 429 in this exchange excludes the binding. + let a = oauth("a", 0.5); + let f = Fixture::new(vec![a.clone(), b.clone()], Some(a.handle)); + let facts = RequestFacts { + session: true, + binding: Some(a.handle), + exclusion: &[a.handle], + ..RequestFacts::default() + }; + let choice = select(&facts, &f.snapshot(), &s).unwrap(); + assert_eq!((choice.handle, choice.binds), (b.handle, true)); + // With nobody else, nobody is named. + let f = Fixture::new(vec![held.clone()], Some(held.handle)); + let facts = RequestFacts { + session: true, + binding: Some(held.handle), + ..RequestFacts::default() + }; + let nobody = select(&facts, &f.snapshot(), &s).unwrap_err(); + assert_eq!( + (nobody.reason, nobody.named), + (NoService::AllHeldOrOverThreshold, None) + ); + // A pin never fails over. + let facts = RequestFacts { + pin: Some(held.handle), + session: true, + binding: Some(held.handle), + ..RequestFacts::default() + }; + let f = Fixture::new(vec![held.clone(), b.clone()], Some(b.handle)); + assert_eq!( + select(&facts, &f.snapshot(), &s).unwrap_err().reason, + NoService::PinnedUnavailable + ); + } + #[test] fn pin_and_preference_precede_everything_and_a_pin_never_falls_over() { let s = settings(); diff --git a/src/pool/sessions.rs b/src/pool/sessions.rs index 18d65e5..96fee73 100644 --- a/src/pool/sessions.rs +++ b/src/pool/sessions.rs @@ -1,8 +1,8 @@ //! Sessions and their bindings. //! //! A session is the pair (principal, `x-claude-code-session-id`). The pool -//! remembers, per session, the account it is bound to — set once at its first -//! attempt and never moved — the account that served it last, when it +//! remembers, per session, the account it is bound to — set at its first +//! attempt, moved only by failover — the account that served it last, when it //! was last seen and how many of its exchanges are in flight. Every //! time-dependent decision takes `now` from the caller so the windows are //! provable without a clock seam. @@ -26,7 +26,7 @@ pub struct SessionKey { #[derive(Debug, Clone, PartialEq, Eq)] pub struct SessionRecord { - /// Set at the first attempt, never moved. + /// Set at the first attempt, moved only by failover. pub bound: Option, pub last_served: Option, pub last_seen: OffsetDateTime, @@ -83,13 +83,10 @@ impl Sessions { record.last_seen = now; } - /// The first attempt binds; a session that already has a binding keeps it. - pub fn bind(&mut self, key: &SessionKey, handle: Uuid) { - if let Some(record) = self.records.get_mut(key) - && record.bound.is_none() - { - record.bound = Some(handle); - } + /// Bind the session to `handle`; the other account it was bound to, if any. + pub fn bind(&mut self, key: &SessionKey, handle: Uuid) -> Option { + let record = self.records.get_mut(key)?; + record.bound.replace(handle).filter(|old| *old != handle) } /// The exchange ended; `served` is the account that made its last attempt, if any. @@ -158,16 +155,17 @@ mod tests { } #[test] - fn binding_is_set_once_and_never_moved() { + fn a_bind_names_the_binding_it_moves() { let (a, b) = (Uuid::new_v4(), Uuid::new_v4()); let mut s = Sessions::default(); s.begin(key("s1"), T0); assert_eq!(s.binding(&key("s1")), None); - s.bind(&key("s1"), a); - s.bind(&key("s1"), b); - assert_eq!(s.binding(&key("s1")), Some(a)); - s.end(&key("s1"), Some(a), T0); - assert_eq!(s.records.get(&key("s1")).unwrap().last_served, Some(a)); + assert_eq!(s.bind(&key("s1"), a), None); + assert_eq!(s.bind(&key("s1"), a), None); + assert_eq!(s.bind(&key("s1"), b), Some(a)); + assert_eq!(s.binding(&key("s1")), Some(b)); + s.end(&key("s1"), Some(b), T0); + assert_eq!(s.records.get(&key("s1")).unwrap().last_served, Some(b)); } #[test] diff --git a/tests/acceptance/acp.rs b/tests/acceptance/acp.rs index 09d388d..08955ae 100644 --- a/tests/acceptance/acp.rs +++ b/tests/acceptance/acp.rs @@ -324,13 +324,16 @@ async fn the_refused_candidate_ends_502_and_the_sibling_is_never_tried() { "FSUB2 untouched" ); - // The session bound before the failure keeps its binding: it - // gets the synthetic 429 naming its errored account, no attempt made. + // Failover moves the session bound before the failure to FSUB2. let calls = instance.upstream.calls(); let answer = send(instance.addr, in_session(messages(haiku_prompt()), "oscar")).await; - assert_eq!(answer.status, StatusCode::TOO_MANY_REQUESTS); - assert!(answer.text().contains("FSUB"), "{}", answer.text()); - assert_eq!(instance.upstream.calls(), calls, "no attempt for the 429"); + assert_eq!(answer.status, StatusCode::OK, "{}", answer.text()); + assert_eq!(instance.upstream.calls(), calls + 1); + assert_eq!( + instance.last_record(3)["serving_account"]["display_name"], + "FSUB2" + ); + assert_eq!(instance.events("session_moved").len(), 1); // (b) a bare 403: the 502 ends the exchange, the account not errored. let instance = Instance::start_with_accounts( diff --git a/tests/acceptance/cdx_mitm.rs b/tests/acceptance/cdx_mitm.rs index 7a772ed..d469d19 100644 --- a/tests/acceptance/cdx_mitm.rs +++ b/tests/acceptance/cdx_mitm.rs @@ -10,8 +10,8 @@ use ruzstd::encoding::{CompressionLevel, compress_to_vec}; use crate::harness::{ Bytes, CODEX_RESPONSES_PATH, Command, Duration, Full, Instance, Instant, Method, - OffsetDateTime, Request, Setup, Stdio, Value, fs, haiku_prompt, json, make_private, - private_dir, + OffsetDateTime, Reply, Request, Setup, Stdio, Value, add_two, fs, haiku_prompt, json, + make_private, private_dir, }; use crate::proxy::{Intercepted, Offer, ca_file, intercept}; @@ -353,6 +353,72 @@ async fn codex_s_usage_read_rides_the_pooled_login() { ); } +/// Failover on Codex: an exhausted login's 429 is answered by the next +/// one, and the thread stays there. +#[tokio::test(flavor = "multi_thread")] +async fn an_exhausted_codex_login_hands_the_thread_to_the_next() { + let _leak_sweep = crate::leaks::LeakGuard::default(); + let mut instance = Instance::start_with( + "codex-failover", + Setup { + mitm: true, + ..Setup::default() + }, + ) + .await; + add_two(&instance); + instance.restart_with_state(|state| { + for record in state["accounts"].as_array_mut().expect("accounts") { + let name = record["display_name"].as_str().expect("name"); + record["chatgpt_account_id"] = json!(format!("ws-{name}")); + record["provider"] = json!("codex"); + } + }); + let mut tunnel = codex_tunnel(&instance, &ca_file(&instance)) + .await + .expect("the handshake succeeds"); + let turn = || responses_request(&zstd(&json!({ "model": "gpt-5-codex", "input": [] }))); + let workspaces = || -> Vec { + instance + .upstream + .seen() + .into_iter() + .filter(|seen| seen.path == CODEX_RESPONSES_PATH) + .map(|seen| { + seen.header("chatgpt-account-id") + .expect("workspace") + .to_string() + }) + .collect() + }; + + assert_eq!(tunnel.send(turn()).await.status, 200); + instance.upstream.script([Reply::Raw { + status: 429, + headers: [ + ("content-type", "application/json"), + ("x-codex-primary-used-percent", "100"), + ("x-codex-primary-window-minutes", "300"), + ("x-codex-primary-reset-at", "1900000000"), + ] + .map(|(name, value)| (name.to_string(), value.to_string())) + .to_vec(), + body: json!({ "error": { "type": "usage_limit_reached", "message": "limit" } }).to_string(), + }]); + let answer = tunnel.send(turn()).await; + assert_eq!(answer.status, 200, "{}", answer.text()); + assert!(answer.text().contains("hello from the fake chatgpt")); + assert_eq!(tunnel.send(turn()).await.status, 200); + + let seen = workspaces(); + let [first, exhausted, next, after] = &seen[..] else { + panic!("four turns upstream: {seen:?}"); + }; + assert_eq!(exhausted, first, "the bound login answered the 429"); + assert_ne!(next, first, "the next login took the turn"); + assert_eq!(after, next, "and keeps the thread"); +} + #[tokio::test(flavor = "multi_thread")] async fn the_proxy_s_own_429_is_codex_s_usage_limit_with_its_reset() { let _leak_sweep = crate::leaks::LeakGuard::default(); diff --git a/tests/acceptance/dep_release.rs b/tests/acceptance/dep_release.rs index bdaa31f..471101a 100644 --- a/tests/acceptance/dep_release.rs +++ b/tests/acceptance/dep_release.rs @@ -458,7 +458,14 @@ async fn one_release_set_one_version_only_the_allowed_files() { let file = std::fs::File::open(&path).expect("open the archive"); let mut archive = zip::ZipArchive::new(file).expect("read the archive"); (0..archive.len()) - .map(|i| archive.by_index(i).expect("member").name().to_string()) + .map(|i| { + archive + .by_index(i) + .expect("member") + .name() + .expect("name") + .into_owned() + }) .collect() } else { let listed = std::process::Command::new("tar") diff --git a/tests/acceptance/desktop.rs b/tests/acceptance/desktop.rs index 2f6c23c..7b4ba04 100644 --- a/tests/acceptance/desktop.rs +++ b/tests/acceptance/desktop.rs @@ -1625,7 +1625,7 @@ esac } #[tokio::test(flavor = "multi_thread")] - async fn pinned_and_auto_sessions_resume_without_mid_chat_failover() { + async fn an_automatic_conversation_fails_over_mid_chat_and_stays() { let _leak_sweep = crate::leaks::LeakGuard::default(); let instance = Instance::start_client("desktop-session-selection").await; add_two(&instance); @@ -1671,23 +1671,31 @@ esac StatusCode::OK ); instance.last_record(3); + // FSUB2 runs out mid-chat: the message is answered by FSUB, and the + // conversation stays there. instance.upstream.script([reply_exhausted_429(60)]); + let calls = instance.upstream.calls(); assert_eq!( send(gateway.addr, gateway.engine("pinned-conversation")) .await .status, - StatusCode::TOO_MANY_REQUESTS + StatusCode::OK ); - instance.last_record(4); + assert_eq!(instance.upstream.calls(), calls + 2); + let record = instance.last_record(4); + assert_eq!(record["serving_account"]["display_name"], "FSUB"); + assert_eq!(record["failed_over"], true); let calls = instance.upstream.calls(); assert_eq!( send(gateway.addr, gateway.engine("pinned-conversation")) .await .status, - StatusCode::TOO_MANY_REQUESTS + StatusCode::OK ); - assert_eq!(instance.upstream.calls(), calls); - assert_eq!(instance.last_record(5)["session_id"], "pinned-conversation"); + assert_eq!(instance.upstream.calls(), calls + 1); + let record = instance.last_record(5); + assert_eq!(record["session_id"], "pinned-conversation"); + assert_eq!(record["serving_account"]["display_name"], "FSUB"); assert_eq!( send(gateway.addr, gateway.engine("new-conversation")) .await diff --git a/tests/acceptance/dpl.rs b/tests/acceptance/dpl.rs index 09fee03..0cbdd25 100644 --- a/tests/acceptance/dpl.rs +++ b/tests/acceptance/dpl.rs @@ -933,6 +933,7 @@ async fn every_exchange_leaves_one_record() { "exchange-leaves-record", Setup { blocked_models: vec!["*opus*".into()], + selection: "failover = false\n".into(), ..Setup::default() }, ) @@ -1360,7 +1361,14 @@ async fn http_403_on_the_bound_account_ends_with_502_naming_it() { #[tokio::test(flavor = "multi_thread")] async fn exhaustion_429_on_the_bound_account_is_relayed() { let _leak_sweep = crate::leaks::LeakGuard::default(); - let instance = Instance::start("exhaustion-429-bound").await; + let instance = Instance::start_with( + "exhaustion-429-bound", + Setup { + selection: "failover = false\n".into(), + ..Setup::default() + }, + ) + .await; add_two(&instance); send( instance.addr, @@ -1479,9 +1487,15 @@ async fn throttle_retry_absorbed_then_relayed() { async fn inline_wait_then_reattempt_of_the_bound_account() { let _leak_sweep = crate::leaks::LeakGuard::default(); let faults = crate::faults::Faults::new(); - let instance = - Instance::start_with_faults("inline-wait-reattempt", Setup::default(), faults.clone()) - .await; + let instance = Instance::start_with_faults( + "inline-wait-reattempt", + Setup { + selection: "failover = false\n".into(), + ..Setup::default() + }, + faults.clone(), + ) + .await; add_two(&instance); send(instance.addr, in_session(messages(haiku_prompt()), "golf")).await; diff --git a/tests/acceptance/qta.rs b/tests/acceptance/qta.rs index cc55ef2..d51c105 100644 --- a/tests/acceptance/qta.rs +++ b/tests/acceptance/qta.rs @@ -219,7 +219,14 @@ async fn usage_401_refreshes_once_and_timeout_retains_quota() { #[tokio::test(flavor = "multi_thread")] async fn spend_cap_holds_the_organisation_and_not_others() { let _leak_sweep = crate::leaks::LeakGuard::default(); - let instance = Instance::start("spend-cap-holds-organisation").await; + let instance = Instance::start_with( + "spend-cap-holds-organisation", + Setup { + selection: "failover = false\n".into(), + ..Setup::default() + }, + ) + .await; add_two(&instance); instance.add_other_org( "FOTHER", diff --git a/tests/acceptance/sel.rs b/tests/acceptance/sel.rs index a5494cd..dc9e78d 100644 --- a/tests/acceptance/sel.rs +++ b/tests/acceptance/sel.rs @@ -415,7 +415,10 @@ async fn a_bound_session_gets_its_accounts_real_429() { let faults = Faults::new(); let instance = Instance::start_with_faults( "bound-session-gets-accounts", - Setup::default(), + Setup { + selection: "failover = false\n".into(), + ..Setup::default() + }, faults.clone(), ) .await; @@ -476,6 +479,7 @@ async fn disabled_errored_held_never_chosen() { "disabled-errored-held", Setup { quota: "revalidation_floor_seconds = 2\nrevalidation_interval_seconds = 2\n".into(), + selection: "failover = false\n".into(), ..Setup::default() }, ) @@ -791,7 +795,7 @@ async fn held_assignment_keeps_the_session_and_binds_nothing() { let instance = Instance::start_with( "held-assignment-keeps-session", Setup { - selection: "distribute_sessions = true\n".into(), + selection: "distribute_sessions = true\nfailover = false\n".into(), ..Setup::default() }, ) @@ -848,7 +852,14 @@ async fn held_assignment_keeps_the_session_and_binds_nothing() { #[tokio::test(flavor = "multi_thread")] async fn bound_and_held_gets_the_synthetic_429() { let _leak_sweep = crate::leaks::LeakGuard::default(); - let instance = Instance::start("bound-held-gets-synthetic").await; + let instance = Instance::start_with( + "bound-held-gets-synthetic", + Setup { + selection: "failover = false\n".into(), + ..Setup::default() + }, + ) + .await; add_two(&instance); send( instance.addr, @@ -896,6 +907,65 @@ async fn bound_and_held_gets_the_synthetic_429() { assert!(instance.events("default_moved").is_empty()); } +/// Failover: the bound account's exhaustion 429 is answered by the next +/// account, and the session stays there; a throttle never moves it; with +/// nobody left, the last account's own 429 is relayed. +#[tokio::test(flavor = "multi_thread")] +async fn failover_moves_an_exhausted_session_and_keeps_it_there() { + let _leak_sweep = crate::leaks::LeakGuard::default(); + let instance = Instance::start("failover-moves-session").await; + add_two(&instance); + let session = || in_session(messages(haiku_prompt()), "tango"); + send(instance.addr, session()).await; + assert_eq!( + instance.last_record(1)["serving_account"]["display_name"], + "FSUB" + ); + + instance.upstream.script([reply_exhausted_429(60)]); + let calls = instance.upstream.calls(); + let answer = send(instance.addr, session()).await; + assert_eq!(answer.status, StatusCode::OK, "{}", answer.text()); + assert_eq!(instance.upstream.calls(), calls + 2); + let record = instance.last_record(2); + assert_eq!(record["serving_account"]["display_name"], "FSUB2"); + assert_eq!(record["failed_over"], true); + assert_eq!(record["attempts"], 2); + assert_eq!(instance.events("session_moved").len(), 1); + + let calls = instance.upstream.calls(); + send(instance.addr, session()).await; + assert_eq!(instance.upstream.calls(), calls + 1, "straight to FSUB2"); + assert_eq!( + instance.last_record(3)["serving_account"]["display_name"], + "FSUB2" + ); + + instance.upstream.script([reply_throttle_429(Some(1))]); + let answer = send(instance.addr, session()).await; + assert_eq!(answer.status, StatusCode::OK, "{}", answer.text()); + assert_eq!( + instance.last_record(4)["serving_account"]["display_name"], + "FSUB2" + ); + assert_eq!( + instance.events("session_moved").len(), + 1, + "a throttle stays" + ); + + instance.upstream.script([reply_exhausted_429(45)]); + let calls = instance.upstream.calls(); + let answer = send(instance.addr, session()).await; + assert_eq!(answer.status, StatusCode::TOO_MANY_REQUESTS); + assert_eq!(answer.header("retry-after"), Some("45")); + assert_eq!(answer.header("request-id"), Some("req_fixture_0429")); + assert_eq!(instance.upstream.calls(), calls + 1, "FSUB is still held"); + let record = instance.last_record(5); + assert_eq!(record["serving_account"]["display_name"], "FSUB2"); + assert_eq!(record["error_class"], "rate_limit"); +} + // ---- step 5: the operator's steer, the route view and the advisor /// A haiku prompt whose advisor tool names opus.