diff --git a/CHANGELOG.md b/CHANGELOG.md index 43e84d1..26700bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Provider failover chain, core (#209): `failover::FailoverProvider` retries a step on the next provider of an ordered chain when the active one is unavailable (connection failure, timeout, dropped stream, HTTP 5xx or 429); request errors (401/403/404/400, malformed responses) fail loud and never fail over. The answering provider stays active and the primary is retried after a cooldown. Each step records the route under `tole_route` in its usage row (durable, auditable); rows holding only tole's own keys are not counted as provider steps. The env/config wiring follows separately. +- Provider failover chain (#209): `failover::FailoverProvider` retries a step on the next provider of an ordered chain when the active one is unavailable (connection failure, timeout, dropped stream, HTTP 5xx or 429); request errors (401/403/404/400, malformed responses) fail loud and never fail over. The answering provider stays active and the primary is retried after a cooldown. Each step records the route under `tole_route` in its usage row (durable, auditable); rows holding only tole's own keys are not counted as provider steps. Wired into every session face through `TOLE_FALLBACK_BASE_URL` / `TOLE_FALLBACK_API_KEY` / `TOLE_FALLBACK_MODEL` (a reserve without its own key is refused; the primary's key is never sent to another host). A stream that breaks after part of the answer was already shown live does not fail over (no duplicated output); a reserve that fails before the cooldown ends wraps back to the primary; and after a failover only the last provider's error decides the turn loop's transient retry, so an earlier timeout no longer makes a reserve's 401 look retryable. - Child-agent handoff schema and verify gate (#383): `tole_core::handoff` defines the typed handoff document (role `explore` / `implement` / `verify` / `fix`, artifact references by path and rev, verification evidence), strict validation of the model-authored input (workspace-relative paths, option-safe refs, bounded fields), the close gate (an implement/fix child without passing verification is `Unverified` / `VerifyFailed`, never closed silently) and the prompt for the follow-up `fix` child. `agent_start` takes `role` and `consumes` (artifact references the child reads itself), a role child gets a `handoff_submit` tool, and `agent_poll` reports the gate and a ready-made `fix_prompt` (the parent decides whether to start the fix child; nothing is spawned automatically). Design and anti-goals (no DAG engine) in `docs/child-agents-handoff.md`. - Sibling change notices, core (#380): `sibling_watch` tracks which files each child agent reads and writes in one shared append-only log (owner-only, bounded) and tells a child, once per stale path, when a sibling changed a file it had read or written; the `Tracked` tool decorator records `read_file` / `write_file` / `edit_file` / `delete_file` on success. Interrupts gain a `source`: a notice is delivered at the next safe point as an attributed entry (`source: notice`) and never withdraws output or supersedes a final answer. Wiring into `tole run` children follows separately. - Sibling change notices (#380): `sibling_watch` tracks which files each child agent reads and writes in one shared append-only log (owner-only, bounded) and tells a child, once per stale path, when a sibling changed a file it had read or written; the `Tracked` tool decorator records `read_file` / `write_file` / `edit_file` / `delete_file` on success. Interrupts gain a `source`: a notice is delivered at the next safe point as an attributed entry (`source: notice`) and never withdraws output or supersedes a final answer. Children spawned by `agent_start` get `TOLE_AGENT_ID` and a shared `TOLE_AGENT_TOUCH_LOG`; in a child, `tole run` wraps its file tools and runs the watcher, so a sibling's change reaches the model at its next safe point. diff --git a/README.md b/README.md index 2f3d123..df79ed2 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,7 @@ trust: `internal` / `read_only`), `--skill ` (load a SKILL.md), | Variable | Purpose | |----------|---------| | `TOLE_BASE_URL`, `TOLE_MODEL`, `TOLE_API_KEY` | LLM provider — any OpenAI-compatible endpoint (`OPENAI_*` equivalents read as fallback) | +| `TOLE_FALLBACK_BASE_URL`, `TOLE_FALLBACK_API_KEY`, `TOLE_FALLBACK_MODEL` | optional reserve provider (#209): a step that fails because the primary is unavailable (connection failure, timeout, 5xx, 429) is retried on the reserve, and the switch is recorded in the session log (`tole_route`). The reserve needs its own key (the primary's is never sent to another host); the model defaults to the primary's. Request errors (401/403/404/400) never fail over | | `TOLE_SYSTEM_PROMPT` | default persona when `--system` is absent | | `TOLE_MEMORY` | memory loop backend (`uteke`) — same as `--memory uteke` | | `TOLE_NO_UPDATE_CHECK` | `1` disables the startup update-check banner (issue #220) | diff --git a/crates/tole-cli/src/lib.rs b/crates/tole-cli/src/lib.rs index c64996b..ee749ca 100644 --- a/crates/tole-cli/src/lib.rs +++ b/crates/tole-cli/src/lib.rs @@ -10,6 +10,7 @@ pub mod config_apply; pub mod config_trust; pub mod doctor; pub mod handoff; +pub mod provider_chain; pub mod session_host; pub mod session_tools; pub mod sibling; diff --git a/crates/tole-cli/src/main.rs b/crates/tole-cli/src/main.rs index 49a8871..69c7d31 100644 --- a/crates/tole-cli/src/main.rs +++ b/crates/tole-cli/src/main.rs @@ -28,7 +28,6 @@ use tole_core::gh::GhTool; use tole_core::git::GitTool; #[cfg(feature = "shell-tools")] use tole_core::jobs::{JobPollTool, JobStartTool}; -use tole_core::openai::OpenAiProvider; use tole_core::read_file::ReadFileTool; #[cfg(feature = "shell-tools")] use tole_core::run_command::RunCommandTool; @@ -1961,10 +1960,16 @@ fn run_command( .map_err(|e| anyhow::anyhow!("registering todo_write: {e}"))?; } - let mut provider = OpenAiProvider::new(cfg).with_tool_specs(registry.specs()); - if let Some(sys) = system_prompt.as_deref() { - provider = provider.with_system_prompt(sys); - } + let specs = registry.specs(); + let mut provider = + tole_cli::provider_chain::build(cfg, &tole_cli::config_apply::real_env, |p| { + let p = p.with_tool_specs(specs.clone()); + match system_prompt.as_deref() { + Some(sys) => p.with_system_prompt(sys), + None => p, + } + }) + .map_err(|e| anyhow::anyhow!(e))?; // Memory loop, pre-turn: recalled context rides inside the first // user message — the durable log stores exactly what was sent. The // summary remembers the PRE-injection prompt: storing the injected @@ -2130,13 +2135,20 @@ fn resume_command( .collect(); registry.set_hooks(hooks); } - let mut provider = OpenAiProvider::new(cfg).with_tool_specs(registry.specs()); // B2: the system prompt is pinned in the session header — resume // re-applies exactly what the session was created with (never the // ambient env, which may have changed since). - if let Some(sys) = storage.system_prompt() { - provider = provider.with_system_prompt(sys); - } + let specs = registry.specs(); + let pinned = storage.system_prompt().map(str::to_string); + let mut provider = + tole_cli::provider_chain::build(cfg, &tole_cli::config_apply::real_env, |p| { + let p = p.with_tool_specs(specs.clone()); + match pinned.as_deref() { + Some(sys) => p.with_system_prompt(sys), + None => p, + } + }) + .map_err(|e| anyhow::anyhow!(e))?; let outcome = match prompt { Some(text) if !text.trim().is_empty() => { // New instructions on a settled session (issue #55): the @@ -2506,12 +2518,19 @@ fn chat_command( "tole chat — session {session_id} (Ctrl-D exits, resume: tole chat --resume {session_id})" ); - let mut provider = OpenAiProvider::new(cfg).with_tool_specs(registry.specs()); // B2: fresh sessions pin the resolved prompt; resumed sessions re-apply // the header-pinned one (see create_with above / JsonlStorage::open). - if let Some(sys) = storage.system_prompt() { - provider = provider.with_system_prompt(sys); - } + let specs = registry.specs(); + let pinned = storage.system_prompt().map(str::to_string); + let mut provider = + tole_cli::provider_chain::build(cfg, &tole_cli::config_apply::real_env, |p| { + let p = p.with_tool_specs(specs.clone()); + match pinned.as_deref() { + Some(sys) => p.with_system_prompt(sys), + None => p, + } + }) + .map_err(|e| anyhow::anyhow!(e))?; // Memory loop state: recall rides into the first message of a FRESH // session only (a resumed session already carries its context); the diff --git a/crates/tole-cli/src/mission.rs b/crates/tole-cli/src/mission.rs index c869eb0..b2f7556 100644 --- a/crates/tole-cli/src/mission.rs +++ b/crates/tole-cli/src/mission.rs @@ -213,13 +213,13 @@ fn tokens_used(storage: &JsonlStorage) -> u64 { /// storage, so `--resume` re-applies exactly the MISSION_COMPLETE rules /// the mission was created with (cora 0.19 scan MAJOR: the header was /// written but never sent, so the model never saw the marker contract). -fn mission_provider( - cfg: tole_core::openai::OpenAiConfig, +fn configure_mission_provider( + provider: OpenAiProvider, specs: Vec, - storage: &JsonlStorage, + system_prompt: Option<&str>, ) -> OpenAiProvider { - let provider = OpenAiProvider::new(cfg).with_tool_specs(specs); - match storage.system_prompt() { + let provider = provider.with_tool_specs(specs); + match system_prompt { Some(sys) => provider.with_system_prompt(sys), None => provider, } @@ -286,7 +286,13 @@ pub fn run_mission( .map_err(|e| anyhow::anyhow!("registering todo_write: {e}"))?; } - let mut provider = mission_provider(provider_cfg, registry.specs(), &storage); + let specs = registry.specs(); + let system_prompt = storage.system_prompt().map(str::to_owned); + let mut provider = + tole_cli::provider_chain::build(provider_cfg, &tole_cli::config_apply::real_env, |p| { + configure_mission_provider(p, specs.clone(), system_prompt.as_deref()) + }) + .map_err(|e| anyhow::anyhow!(e))?; let mut turns: u64 = 0; let mut verify_failures: u32 = 0; @@ -621,7 +627,12 @@ mod verify_error_tests { let fresh = JsonlStorage::create_with(&d, "m1", None, Some(&sys)).unwrap(); let cfg = || OpenAiConfig::new("https://x/v1", "m", "k"); for storage in [fresh, JsonlStorage::open(d.join("m1.jsonl")).unwrap()] { - let body = mission_provider(cfg(), vec![], &storage).request_body(&[]); + let body = configure_mission_provider( + OpenAiProvider::new(cfg()), + vec![], + storage.system_prompt(), + ) + .request_body(&[]); let first = &body["messages"][0]; assert_eq!(first["role"], json!("system"), "{body}"); assert!( diff --git a/crates/tole-cli/src/provider_chain.rs b/crates/tole-cli/src/provider_chain.rs new file mode 100644 index 0000000..f405caa --- /dev/null +++ b/crates/tole-cli/src/provider_chain.rs @@ -0,0 +1,161 @@ +//! Provider construction with an optional reserve (#209). +//! +//! Every session-building site configures an `OpenAiProvider` (tool specs, +//! system prompt, live-token sinks, ...). `build` applies that SAME +//! configuration to the primary and, when `TOLE_FALLBACK_BASE_URL` is set, +//! to a reserve, and returns a `FailoverProvider` over the two. Without a +//! reserve the plain provider is returned: no behaviour change. +//! +//! The reserve needs its own `TOLE_FALLBACK_API_KEY`: the primary's key is +//! never sent to another host. `TOLE_FALLBACK_MODEL` defaults to the +//! primary's model. + +use tole_core::failover::{FailoverProvider, Link}; +use tole_core::openai::{OpenAiConfig, OpenAiProvider}; +use tole_core::provider::Provider; + +pub const ENV_FALLBACK_BASE_URL: &str = "TOLE_FALLBACK_BASE_URL"; +pub const ENV_FALLBACK_MODEL: &str = "TOLE_FALLBACK_MODEL"; +// The env var NAME, not a secret value. +// cora-ignore: sec-hardcoded-secret +pub const ENV_FALLBACK_API_KEY: &str = "TOLE_FALLBACK_API_KEY"; + +fn nonblank(env: &dyn Fn(&str) -> Option, name: &str) -> Option { + env(name) + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) +} + +/// The reserve's config, if one is configured. A base URL without a key is +/// an error (fail loud rather than run without the reserve the operator +/// asked for). +pub fn fallback_config( + primary: &OpenAiConfig, + env: &dyn Fn(&str) -> Option, +) -> Result, String> { + let Some(base_url) = nonblank(env, ENV_FALLBACK_BASE_URL) else { + return Ok(None); + }; + let api_key = nonblank(env, ENV_FALLBACK_API_KEY).ok_or_else(|| { + format!( + "{ENV_FALLBACK_BASE_URL} is set but {ENV_FALLBACK_API_KEY} is not: the reserve needs \ + its own key (the primary's key is never sent to another host)" + ) + })?; + let model = nonblank(env, ENV_FALLBACK_MODEL).unwrap_or_else(|| primary.model.clone()); + Ok(Some(OpenAiConfig::new(base_url, model, api_key))) +} + +/// `role: model @ host` with URL credentials stripped (recorded in +/// `tole_route`, so it must never carry a secret). +pub fn label(role: &str, cfg: &OpenAiConfig) -> String { + format!( + "{role}: {} @ {}", + cfg.model, + crate::doctor::redact_url(&cfg.base_url) + ) +} + +/// Build the session's provider: `configure` is applied to the primary and, +/// when configured, to the reserve. +pub fn build( + primary: OpenAiConfig, + env: &dyn Fn(&str) -> Option, + configure: impl Fn(OpenAiProvider) -> OpenAiProvider, +) -> Result, String> { + let Some(reserve) = fallback_config(&primary, env)? else { + return Ok(Box::new(configure(OpenAiProvider::new(primary)))); + }; + let chain = vec![ + Link { + label: label("primary", &primary), + provider: Box::new(configure(OpenAiProvider::new(primary))), + }, + Link { + label: label("fallback", &reserve), + provider: Box::new(configure(OpenAiProvider::new(reserve))), + }, + ]; + Ok(Box::new(FailoverProvider::new(chain))) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn env<'a>(pairs: &'a [(&'a str, &'a str)]) -> impl Fn(&str) -> Option + 'a { + move |k| { + pairs + .iter() + .find(|(n, _)| *n == k) + .map(|(_, v)| v.to_string()) + } + } + + fn primary() -> OpenAiConfig { + OpenAiConfig::new("https://a.example/v1", "model-a", "sk-primary") + } + + #[test] + fn no_reserve_unless_a_base_url_is_set() { + assert_eq!(fallback_config(&primary(), &env(&[])), Ok(None)); + assert_eq!( + fallback_config(&primary(), &env(&[(ENV_FALLBACK_BASE_URL, " ")])), + Ok(None) + ); + } + + #[test] + fn the_reserve_needs_its_own_key_and_inherits_the_model() { + let e = fallback_config( + &primary(), + &env(&[(ENV_FALLBACK_BASE_URL, "https://b.example/v1")]), + ) + .unwrap_err(); + assert!(e.contains(ENV_FALLBACK_API_KEY), "{e}"); + assert!(!e.contains("sk-primary")); + let r = fallback_config( + &primary(), + &env(&[ + (ENV_FALLBACK_BASE_URL, "https://b.example/v1"), + (ENV_FALLBACK_API_KEY, "sk-b"), + ]), + ) + .unwrap() + .unwrap(); + assert_eq!( + (r.base_url.as_str(), r.model.as_str(), r.api_key.as_str()), + ("https://b.example/v1", "model-a", "sk-b") + ); + let r = fallback_config( + &primary(), + &env(&[ + (ENV_FALLBACK_BASE_URL, "https://b.example/v1"), + (ENV_FALLBACK_API_KEY, "sk-b"), + (ENV_FALLBACK_MODEL, "model-b"), + ]), + ) + .unwrap() + .unwrap(); + assert_eq!(r.model, "model-b"); + } + + #[test] + fn labels_never_carry_credentials() { + let cfg = OpenAiConfig::new("https://user:pw@b.example/v1?key=k", "m", "sk-x"); + let l = label("fallback", &cfg); + assert_eq!(l, "fallback: m @ https://b.example/v1"); + assert!(!l.contains("pw") && !l.contains("sk-x")); + } + + #[test] + fn build_fails_loud_on_a_half_configured_reserve() { + let r = build( + primary(), + &env(&[(ENV_FALLBACK_BASE_URL, "https://b.example/v1")]), + |p| p, + ); + assert!(r.is_err()); + assert!(build(primary(), &env(&[]), |p| p).is_ok()); + } +} diff --git a/crates/tole-cli/src/session_host.rs b/crates/tole-cli/src/session_host.rs index 2befdc2..3c9c238 100644 --- a/crates/tole-cli/src/session_host.rs +++ b/crates/tole-cli/src/session_host.rs @@ -598,22 +598,26 @@ pub fn run_session_turn( { cfg.model = model.to_string(); } - let mut provider = - tole_core::openai::OpenAiProvider::new(cfg).with_tool_specs(registry.specs()); - if let Some(sys) = system_prompt.as_deref() { - provider = provider.with_system_prompt(sys); - } // Live-token sinks (issue #196 phase 3): when an observer is wired // (ACP), streamed reasoning/content deltas flow to it DURING the - // provider call; the decisive output semantics are unchanged. - if let Some(o) = observer.as_ref() { - let ot = StdArc::clone(o); - let or = StdArc::clone(o); - provider = provider.with_delta_sinks( - Some(StdArc::new(move |t: &str| ot.text_delta(t))), - Some(StdArc::new(move |r: &str| or.reasoning(r))), - ); - } + // provider call; the decisive output semantics are unchanged. The same + // configuration applies to a reserve provider (#209). + let specs = registry.specs(); + let mut provider = crate::provider_chain::build(cfg, &crate::config_apply::real_env, |p| { + let mut p = p.with_tool_specs(specs.clone()); + if let Some(sys) = system_prompt.as_deref() { + p = p.with_system_prompt(sys); + } + if let Some(o) = observer.as_ref() { + let ot = StdArc::clone(o); + let or = StdArc::clone(o); + p = p.with_delta_sinks( + Some(StdArc::new(move |t: &str| ot.text_delta(t))), + Some(StdArc::new(move |r: &str| or.reasoning(r))), + ); + } + p + })?; let outcome = tole_core::turn::run_turn_with_observer( &mut *storage, @@ -687,11 +691,14 @@ pub fn resume_session_turn( { cfg.model = model.to_string(); } - let mut provider = - tole_core::openai::OpenAiProvider::new(cfg).with_tool_specs(registry.specs()); - if let Some(sys) = system_prompt.as_deref() { - provider = provider.with_system_prompt(sys); - } + let specs = registry.specs(); + let mut provider = crate::provider_chain::build(cfg, &crate::config_apply::real_env, |p| { + let p = p.with_tool_specs(specs.clone()); + match system_prompt.as_deref() { + Some(sys) => p.with_system_prompt(sys), + None => p, + } + })?; // #318: the session token is wired into the replan loop, so // `/cancel` during an approval-resume settles the turn as cancelled. // The `reset()` above stays: like `run_session_turn`, it runs inside diff --git a/crates/tole-cli/tests/provider_failover_e2e.rs b/crates/tole-cli/tests/provider_failover_e2e.rs new file mode 100644 index 0000000..f488be6 --- /dev/null +++ b/crates/tole-cli/tests/provider_failover_e2e.rs @@ -0,0 +1,167 @@ +//! #209 provider failover through the REAL `tole run` binary: the primary +//! is unreachable, the reserve configured with `TOLE_FALLBACK_*` answers, +//! and the switch is recorded in the durable session log. + +use serde_json::json; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::process::Command; +use std::time::Duration; + +/// A provider that answers every chat completion with `text`. +fn serve_final(text: &'static str) -> String { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + for conn in listener.incoming() { + let Ok(mut s) = conn else { continue }; + let _ = s.set_read_timeout(Some(Duration::from_secs(5))); + let mut buf = Vec::new(); + let mut chunk = [0u8; 16384]; + loop { + let n = s.read(&mut chunk).unwrap_or(0); + if n == 0 { + break; + } + buf.extend_from_slice(&chunk[..n]); + if let Some(h) = buf.windows(4).position(|w| w == b"\r\n\r\n") { + let head = String::from_utf8_lossy(&buf[..h]).to_ascii_lowercase(); + let len: usize = head + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if buf.len() >= h + 4 + len { + break; + } + } + } + let body = json!({"choices":[{"message":{"role":"assistant","content":text}}], + "usage":{"prompt_tokens":2,"completion_tokens":1}}) + .to_string(); + let _ = s.write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .as_bytes(), + ); + } + }); + format!("http://{addr}/v1") +} + +fn closed_port_url() -> String { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = l.local_addr().unwrap().port(); + drop(l); + format!("http://127.0.0.1:{port}/v1") +} + +fn tmpdir(tag: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("tole-failover-e2e-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +fn tole(dir: &std::path::Path, primary: &str, fallback: &[(&str, &str)]) -> std::process::Output { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_tole")); + cmd.args(["--sessions-dir"]) + .arg(dir.join("sessions")) + .args(["run", "say hi"]) + .current_dir(dir) + .env("TOLE_BASE_URL", primary) + .env("TOLE_MODEL", "m-primary") + .env("TOLE_API_KEY", "sk-primary") + .env("TOLE_STREAM", "0") + .env("TOLE_NO_UPDATE_CHECK", "1") + .env_remove("TOLE_MEMORY") + .env_remove("TOLE_TRUST") + .env_remove("OPENAI_BASE_URL") + .env_remove("OPENAI_API_KEY") + .env_remove("OPENAI_MODEL"); + for k in [ + "TOLE_FALLBACK_BASE_URL", + "TOLE_FALLBACK_API_KEY", + "TOLE_FALLBACK_MODEL", + ] { + cmd.env_remove(k); + } + for (k, v) in fallback { + cmd.env(k, v); + } + cmd.output().unwrap() +} + +#[test] +fn a_dead_primary_fails_over_to_the_reserve_and_the_switch_is_durable() { + let dir = tmpdir("ok"); + let reserve = serve_final("FROM_RESERVE"); + let out = tole( + &dir, + &closed_port_url(), + &[ + ("TOLE_FALLBACK_BASE_URL", &reserve), + ("TOLE_FALLBACK_API_KEY", "sk-reserve"), + ], + ); + let stdout = String::from_utf8_lossy(&out.stdout); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(out.status.success(), "stdout={stdout}\nstderr={stderr}"); + assert!(stdout.contains("FROM_RESERVE"), "stdout={stdout}"); + // The session log records which provider answered and why. + let log = std::fs::read_dir(dir.join("sessions")) + .unwrap() + .flatten() + .find(|e| e.path().extension().is_some_and(|x| x == "jsonl")) + .map(|e| std::fs::read_to_string(e.path()).unwrap()) + .expect("a session log"); + assert!(log.contains("\"tole_route\""), "{log}"); + assert!( + log.contains("fallback: m-primary @ http://127.0.0.1"), + "{log}" + ); + assert!(log.contains("\"from\":\"primary: m-primary"), "{log}"); + // Neither key ends up in the durable log. + assert!(!log.contains("sk-reserve") && !log.contains("sk-primary")); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_reserve_without_its_own_key_is_refused_loudly() { + let dir = tmpdir("nokey"); + let out = tole( + &dir, + &closed_port_url(), + &[("TOLE_FALLBACK_BASE_URL", "http://127.0.0.1:9/v1")], + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(!out.status.success()); + assert!(stderr.contains("TOLE_FALLBACK_API_KEY"), "{stderr}"); + assert!(!stderr.contains("sk-primary"), "{stderr}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn without_a_reserve_nothing_changes() { + let dir = tmpdir("none"); + let primary = serve_final("FROM_PRIMARY"); + let out = tole(&dir, &primary, &[]); + let stdout = String::from_utf8_lossy(&out.stdout); + assert!( + out.status.success() && stdout.contains("FROM_PRIMARY"), + "{stdout}" + ); + let log = std::fs::read_dir(dir.join("sessions")) + .unwrap() + .flatten() + .find(|e| e.path().extension().is_some_and(|x| x == "jsonl")) + .map(|e| std::fs::read_to_string(e.path()).unwrap()) + .unwrap(); + assert!( + !log.contains("tole_route"), + "a single provider records no route" + ); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/tole-core/src/failover.rs b/crates/tole-core/src/failover.rs index 21b84c9..a53f81c 100644 --- a/crates/tole-core/src/failover.rs +++ b/crates/tole-core/src/failover.rs @@ -26,10 +26,32 @@ pub const DEFAULT_RETRY_PRIMARY_AFTER: Duration = Duration::from_secs(300); /// Longest error text kept per failed provider in the route record. const MAX_ERROR_CHARS: usize = 200; +/// Separates the last provider's error from the earlier ones in a +/// combined error: `" [after failover from