From ab8d98768ac16d8cc0b33610344061dc542a99a3 Mon Sep 17 00:00:00 2001 From: ajianaz Date: Fri, 16 Oct 2026 16:43:09 +0700 Subject: [PATCH 1/3] feat(core): provider failover chain with a durable route record (#209, 1/3) --- CHANGELOG.md | 1 + crates/tole-cli/src/usage_report.rs | 29 ++- crates/tole-core/src/failover.rs | 354 ++++++++++++++++++++++++++++ crates/tole-core/src/lib.rs | 1 + crates/tole-core/src/provider.rs | 8 + crates/tole-core/src/turn.rs | 9 +- 6 files changed, 394 insertions(+), 8 deletions(-) create mode 100644 crates/tole-core/src/failover.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index dd2f468..43e84d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Provider failover chain, core (#209): `failover::FailoverProvider` retries a step on the next provider of an ordered chain when the active one is unavailable (connection failure, timeout, dropped stream, HTTP 5xx or 429); request errors (401/403/404/400, malformed responses) fail loud and never fail over. The answering provider stays active and the primary is retried after a cooldown. Each step records the route under `tole_route` in its usage row (durable, auditable); rows holding only tole's own keys are not counted as provider steps. The env/config wiring follows separately. - Child-agent handoff schema and verify gate (#383): `tole_core::handoff` defines the typed handoff document (role `explore` / `implement` / `verify` / `fix`, artifact references by path and rev, verification evidence), strict validation of the model-authored input (workspace-relative paths, option-safe refs, bounded fields), the close gate (an implement/fix child without passing verification is `Unverified` / `VerifyFailed`, never closed silently) and the prompt for the follow-up `fix` child. `agent_start` takes `role` and `consumes` (artifact references the child reads itself), a role child gets a `handoff_submit` tool, and `agent_poll` reports the gate and a ready-made `fix_prompt` (the parent decides whether to start the fix child; nothing is spawned automatically). Design and anti-goals (no DAG engine) in `docs/child-agents-handoff.md`. - Sibling change notices, core (#380): `sibling_watch` tracks which files each child agent reads and writes in one shared append-only log (owner-only, bounded) and tells a child, once per stale path, when a sibling changed a file it had read or written; the `Tracked` tool decorator records `read_file` / `write_file` / `edit_file` / `delete_file` on success. Interrupts gain a `source`: a notice is delivered at the next safe point as an attributed entry (`source: notice`) and never withdraws output or supersedes a final answer. Wiring into `tole run` children follows separately. - Sibling change notices (#380): `sibling_watch` tracks which files each child agent reads and writes in one shared append-only log (owner-only, bounded) and tells a child, once per stale path, when a sibling changed a file it had read or written; the `Tracked` tool decorator records `read_file` / `write_file` / `edit_file` / `delete_file` on success. Interrupts gain a `source`: a notice is delivered at the next safe point as an attributed entry (`source: notice`) and never withdraws output or supersedes a final answer. Children spawned by `agent_start` get `TOLE_AGENT_ID` and a shared `TOLE_AGENT_TOUCH_LOG`; in a child, `tole run` wraps its file tools and runs the watcher, so a sibling's change reaches the model at its next safe point. diff --git a/crates/tole-cli/src/usage_report.rs b/crates/tole-cli/src/usage_report.rs index 2ca7053..1420666 100644 --- a/crates/tole-cli/src/usage_report.rs +++ b/crates/tole-cli/src/usage_report.rs @@ -55,15 +55,18 @@ pub struct UsageReport { pub history_growth_per_step: Option, } -/// True when the ledger row carries a provider-reported usage object, -/// i.e. it is NOT a wire-only row (`{"tole_wire": ...}` and nothing -/// else, stored when a provider reports request sizes but no usage). -/// The single definition of "provider step" for the mission step budget, -/// `fact/mission.steps` and the report, so wire-only rows never change -/// step accounting (#211). +/// True when the ledger row carries a provider-reported usage object, i.e. +/// it is NOT a row holding only tole's own namespaced keys (`tole_wire`, +/// stored when a provider reports request sizes but no usage, #211; +/// `tole_route`, the failover route, #209). The single definition of +/// "provider step" for the mission step budget, `fact/mission.steps` and +/// the report, so such rows never change step accounting. pub fn reports_provider_usage(rec: &UsageRecord) -> bool { match rec.usage.as_object() { - Some(o) => !(o.len() == 1 && o.contains_key("tole_wire")), + // Only tole's own namespaced keys (`tole_wire`, `tole_route`, ...): + // the provider reported nothing for this step. + // An empty object still counts (the pre-#211 contract). + Some(o) => o.is_empty() || o.keys().any(|k| !k.starts_with("tole_")), None => true, } } @@ -675,6 +678,18 @@ mod tests { assert_eq!(t[0].input_tokens, Some(4)); } + #[test] + fn rows_with_only_tole_keys_are_not_provider_steps() { + let route_only = rec(json!({"tole_route": {"provider": "reserve"}})); + let wire_and_route = + rec(json!({"tole_wire": wire(1, 2, 3), "tole_route": {"provider": "p"}})); + let real = rec(json!({"prompt_tokens": 4, "tole_route": {"provider": "p"}})); + assert!(!reports_provider_usage(&route_only)); + assert!(!reports_provider_usage(&wire_and_route)); + assert!(reports_provider_usage(&real)); + assert_eq!(provider_steps(&[route_only, wire_and_route, real]), 1); + } + #[test] fn unknown_stays_unknown_never_zero() { let entries = vec![user(1), assistant(2), user(3), assistant(4)]; diff --git a/crates/tole-core/src/failover.rs b/crates/tole-core/src/failover.rs new file mode 100644 index 0000000..21b84c9 --- /dev/null +++ b/crates/tole-core/src/failover.rs @@ -0,0 +1,354 @@ +//! Provider failover chain (#209). +//! +//! `FailoverProvider` wraps an ordered chain of providers (primary first). +//! When the active provider fails with an error that means "this provider +//! is unavailable" — connection failure, timeout, a dropped stream, HTTP +//! 5xx or 429 — the same step is retried on the next provider in the chain. +//! Errors that mean "this request is wrong" (401/403/404/400, a malformed +//! response) fail loud and never fail over: a reserve would only hide a +//! misconfiguration. +//! +//! The provider that answered stays active (no flapping, no repeated slow +//! failures on a dead primary); after `retry_primary_after` the primary is +//! tried again. Each answered step reports `last_route()` — the provider +//! that answered and, when it took a failover, where from and why — which +//! the turn loop stores under `tole_route` in the step's usage row, so a +//! switch is durable and auditable. + +use crate::entry::Entry; +use crate::provider::{Provider, ProviderError, ProviderOutput}; +use serde_json::{json, Value}; +use std::time::{Duration, Instant}; + +/// Default time on a reserve before the primary is tried again. +pub const DEFAULT_RETRY_PRIMARY_AFTER: Duration = Duration::from_secs(300); + +/// Longest error text kept per failed provider in the route record. +const MAX_ERROR_CHARS: usize = 200; + +/// Does this provider error mean "provider unavailable" (fail over) rather +/// than "request wrong" (fail loud)? +pub fn is_failover_error(message: &str) -> bool { + let m = message.to_ascii_lowercase(); + if let Some(rest) = m.strip_prefix("http status: ") { + let code: u16 = rest + .chars() + .take_while(char::is_ascii_digit) + .collect::() + .parse() + .unwrap_or(0); + return code == 429 || (500..600).contains(&code); + } + if m.starts_with("malformed") { + return false; + } + [ + "timeout", + "timed out", + "io:", + "connection", + "host not found", + "dns", + "stream read", + "reset", + "broken pipe", + "unexpected eof", + ] + .iter() + .any(|k| m.contains(k)) +} + +/// One provider in the chain, with the label recorded in `tole_route`. +pub struct Link { + pub label: String, + pub provider: Box, +} + +pub struct FailoverProvider { + chain: Vec, + active: usize, + left_primary_at: Option, + retry_primary_after: Duration, + last_answered: Option, + /// `[(label, error)]` of the providers that failed before the last + /// answer of the current step. + last_failed: Vec<(String, String)>, +} + +impl FailoverProvider { + /// `chain[0]` is the primary. Panics on an empty chain (a host bug). + pub fn new(chain: Vec) -> Self { + assert!(!chain.is_empty(), "a failover chain needs a provider"); + Self { + chain, + active: 0, + left_primary_at: None, + retry_primary_after: DEFAULT_RETRY_PRIMARY_AFTER, + last_answered: None, + last_failed: Vec::new(), + } + } + + pub fn with_retry_primary_after(mut self, after: Duration) -> Self { + self.retry_primary_after = after; + self + } + + fn answered(&self) -> Option<&dyn Provider> { + self.last_answered.map(|i| self.chain[i].provider.as_ref()) + } +} + +fn short(s: &str) -> String { + let mut out: String = s.chars().take(MAX_ERROR_CHARS).collect(); + if s.chars().count() > MAX_ERROR_CHARS { + out.push('…'); + } + out +} + +impl Provider for FailoverProvider { + fn complete(&mut self, transcript: &[Entry]) -> Result { + // Back to the primary once the cooldown has passed. + if self.active > 0 + && self + .left_primary_at + .is_some_and(|t| t.elapsed() >= self.retry_primary_after) + { + self.active = 0; + self.left_primary_at = None; + } + self.last_answered = None; + self.last_failed.clear(); + let start = self.active; + for i in start..self.chain.len() { + match self.chain[i].provider.complete(transcript) { + Ok(out) => { + if i != self.active { + if self.active == 0 { + self.left_primary_at = Some(Instant::now()); + } + self.active = i; + } + self.last_answered = Some(i); + return Ok(out); + } + Err(e) if i + 1 < self.chain.len() && is_failover_error(&e.0) => { + self.last_failed + .push((self.chain[i].label.clone(), short(&e.0))); + } + Err(e) => { + if self.last_failed.is_empty() { + return Err(e); + } + // Keep the last error first: the turn loop's transient + // retry classification reads the start of the message. + let earlier: Vec = self + .last_failed + .iter() + .map(|(l, m)| format!("{l}: {m}")) + .collect(); + return Err(ProviderError(format!( + "{} [after failover from {}]", + e.0, + earlier.join("; ") + ))); + } + } + } + unreachable!("the loop returns on the last provider") + } + + fn last_usage(&self) -> Option { + self.answered().and_then(Provider::last_usage) + } + + fn last_reasoning(&self) -> Option { + self.answered().and_then(Provider::last_reasoning) + } + + fn last_wire_stats(&self) -> Option { + self.answered().and_then(Provider::last_wire_stats) + } + + fn last_route(&self) -> Option { + let i = self.last_answered?; + let mut route = json!({ "provider": self.chain[i].label }); + if !self.last_failed.is_empty() { + route["failover"] = json!({ + "from": self.last_failed[0].0, + "errors": self + .last_failed + .iter() + .map(|(l, m)| json!({ "provider": l, "error": m })) + .collect::>(), + }); + } + Some(route) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + use std::rc::Rc; + + /// Scripted provider that counts its calls. + struct Scripted { + replies: Vec>, + calls: Rc>, + } + + impl Provider for Scripted { + fn complete(&mut self, _t: &[Entry]) -> Result { + let n = self.calls.get(); + self.calls.set(n + 1); + match self + .replies + .get(n) + .copied() + .unwrap_or(Err("script exhausted")) + { + Ok(text) => Ok(ProviderOutput::Final { text: text.into() }), + Err(e) => Err(ProviderError(e.into())), + } + } + fn last_usage(&self) -> Option { + Some(json!({ "prompt_tokens": self.calls.get() })) + } + } + + fn link( + label: &str, + replies: Vec>, + ) -> (Link, Rc>) { + let calls = Rc::new(Cell::new(0)); + ( + Link { + label: label.into(), + provider: Box::new(Scripted { + replies, + calls: Rc::clone(&calls), + }), + }, + calls, + ) + } + + fn text(o: ProviderOutput) -> String { + match o { + ProviderOutput::Final { text } => text, + other => panic!("{other:?}"), + } + } + + #[test] + fn classification() { + for yes in [ + "http status: 503 Service Unavailable — upstream down", + "http status: 500 Internal Server Error — x", + "http status: 429 Too Many Requests — slow down", + "io: Connection refused (os error 61)", + "timeout: global", + "host not found", + "stream read: connection reset by peer", + ] { + assert!(is_failover_error(yes), "{yes}"); + } + for no in [ + "http status: 401 Unauthorized — bad key", + "http status: 403 Forbidden — x", + "http status: 404 Not Found — no such model", + "http status: 400 Bad Request — context too long", + "malformed response: no choices[0].message", + "malformed tool_call: missing function.name", + ] { + assert!(!is_failover_error(no), "{no}"); + } + } + + #[test] + fn a_down_primary_fails_over_and_the_route_is_reported() { + let (p, pc) = link( + "primary", + vec![Err("http status: 503 Service Unavailable — down")], + ); + let (r, rc) = link("reserve", vec![Ok("from reserve"), Ok("again")]); + let mut f = FailoverProvider::new(vec![p, r]); + assert_eq!(text(f.complete(&[]).unwrap()), "from reserve"); + let route = f.last_route().unwrap(); + assert_eq!(route["provider"], "reserve"); + assert_eq!(route["failover"]["from"], "primary"); + assert!(route["failover"]["errors"][0]["error"] + .as_str() + .unwrap() + .contains("503")); + assert_eq!( + f.last_usage().unwrap()["prompt_tokens"], + 1, + "usage of the answering provider" + ); + // Sticky: the next step goes straight to the reserve. + assert_eq!(text(f.complete(&[]).unwrap()), "again"); + assert_eq!((pc.get(), rc.get()), (1, 2)); + assert!(f.last_route().unwrap().get("failover").is_none()); + } + + #[test] + fn request_errors_fail_loud_and_never_reach_the_reserve() { + let (p, _) = link( + "primary", + vec![Err("http status: 401 Unauthorized — bad key")], + ); + let (r, rc) = link("reserve", vec![Ok("unused")]); + let mut f = FailoverProvider::new(vec![p, r]); + let e = f.complete(&[]).unwrap_err(); + assert!(e.0.starts_with("http status: 401"), "{}", e.0); + assert_eq!(rc.get(), 0); + assert!(f.last_route().is_none()); + } + + #[test] + fn the_primary_is_retried_after_the_cooldown() { + let (p, pc) = link( + "primary", + vec![ + Err("io: Connection refused (os error 61)"), + Ok("primary is back"), + ], + ); + let (r, _) = link("reserve", vec![Ok("reserve")]); + let mut f = FailoverProvider::new(vec![p, r]).with_retry_primary_after(Duration::ZERO); + assert_eq!(text(f.complete(&[]).unwrap()), "reserve"); + assert_eq!(text(f.complete(&[]).unwrap()), "primary is back"); + assert_eq!(pc.get(), 2); + assert_eq!(f.last_route().unwrap()["provider"], "primary"); + } + + #[test] + fn when_every_provider_fails_the_last_error_leads() { + let (p, _) = link("primary", vec![Err("http status: 502 Bad Gateway — a")]); + let (r, _) = link("reserve", vec![Err("timeout: global")]); + let mut f = FailoverProvider::new(vec![p, r]); + let e = f.complete(&[]).unwrap_err().0; + assert!(e.starts_with("timeout: global"), "{e}"); + assert!( + e.contains("after failover from primary: http status: 502"), + "{e}" + ); + } + + #[test] + fn a_single_provider_chain_is_transparent() { + let (p, _) = link( + "primary", + vec![Err("http status: 503 Service Unavailable — x")], + ); + let mut f = FailoverProvider::new(vec![p]); + assert_eq!( + f.complete(&[]).unwrap_err().0, + "http status: 503 Service Unavailable — x" + ); + } +} diff --git a/crates/tole-core/src/lib.rs b/crates/tole-core/src/lib.rs index 0d76f90..fc1cfa4 100644 --- a/crates/tole-core/src/lib.rs +++ b/crates/tole-core/src/lib.rs @@ -33,6 +33,7 @@ pub mod cancel; #[cfg(feature = "shell-tools")] pub mod cora_search; pub mod entry; +pub mod failover; pub mod file_tools; // Tool-call authorization gate (#303): crate-internal, no public API. pub(crate) mod gate; diff --git a/crates/tole-core/src/provider.rs b/crates/tole-core/src/provider.rs index 85a77d5..cbb57cd 100644 --- a/crates/tole-core/src/provider.rs +++ b/crates/tole-core/src/provider.rs @@ -78,4 +78,12 @@ pub trait Provider { fn last_wire_stats(&self) -> Option { None } + + /// Which provider answered the LAST `complete()` call, and whether it + /// took a failover to get there (#209, `failover::FailoverProvider`). + /// Stored by the turn loop under the namespaced `tole_route` key of the + /// step's usage record. Default None: a single provider reports no route. + fn last_route(&self) -> Option { + None + } } diff --git a/crates/tole-core/src/turn.rs b/crates/tole-core/src/turn.rs index eb6a610..8d9448e 100644 --- a/crates/tole-core/src/turn.rs +++ b/crates/tole-core/src/turn.rs @@ -536,7 +536,7 @@ fn drive( // reported wire stats but no usage still leaves a record holding // only `tole_wire` (readers sum `prompt_tokens` etc. with a 0 // default). Neither reported: no record, as before. - let usage = match (p.last_usage(), p.last_wire_stats()) { + let mut usage = match (p.last_usage(), p.last_wire_stats()) { (Some(mut u), Some(w)) => { if let Some(obj) = u.as_object_mut() { obj.insert("tole_wire".to_string(), w); @@ -547,6 +547,13 @@ fn drive( (None, Some(w)) => Some(json!({ "tole_wire": w })), (None, None) => None, }; + // Issue #209: the provider that answered (and any failover) rides + // along under the namespaced `tole_route` key — a switch is durable. + if let Some(route) = p.last_route() { + if let Some(obj) = usage.get_or_insert_with(|| json!({})).as_object_mut() { + obj.insert("tole_route".to_string(), route); + } + } if let Some(u) = usage { if let Some(last) = s.entries().last() { s.commit(Commit::new().usage(UsageRecord { From dacda9812cc71ebfacd468654826263e8fd0b676 Mon Sep 17 00:00:00 2001 From: ajianaz Date: Fri, 16 Oct 2026 16:53:12 +0700 Subject: [PATCH 2/3] feat(cli): wire the provider failover chain into every face via TOLE_FALLBACK_* (#209, 3/3) --- CHANGELOG.md | 2 +- README.md | 1 + crates/tole-cli/src/lib.rs | 1 + crates/tole-cli/src/main.rs | 45 +++-- crates/tole-cli/src/mission.rs | 8 +- crates/tole-cli/src/provider_chain.rs | 161 +++++++++++++++++ crates/tole-cli/src/session_host.rs | 45 +++-- .../tole-cli/tests/provider_failover_e2e.rs | 167 ++++++++++++++++++ crates/tole-core/src/provider.rs | 21 +++ 9 files changed, 416 insertions(+), 35 deletions(-) create mode 100644 crates/tole-cli/src/provider_chain.rs create mode 100644 crates/tole-cli/tests/provider_failover_e2e.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 43e84d1..7426266 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Provider failover chain, core (#209): `failover::FailoverProvider` retries a step on the next provider of an ordered chain when the active one is unavailable (connection failure, timeout, dropped stream, HTTP 5xx or 429); request errors (401/403/404/400, malformed responses) fail loud and never fail over. The answering provider stays active and the primary is retried after a cooldown. Each step records the route under `tole_route` in its usage row (durable, auditable); rows holding only tole's own keys are not counted as provider steps. The env/config wiring follows separately. +- Provider failover chain (#209): `failover::FailoverProvider` retries a step on the next provider of an ordered chain when the active one is unavailable (connection failure, timeout, dropped stream, HTTP 5xx or 429); request errors (401/403/404/400, malformed responses) fail loud and never fail over. The answering provider stays active and the primary is retried after a cooldown. Each step records the route under `tole_route` in its usage row (durable, auditable); rows holding only tole's own keys are not counted as provider steps. Wired into every session face through `TOLE_FALLBACK_BASE_URL` / `TOLE_FALLBACK_API_KEY` / `TOLE_FALLBACK_MODEL` (a reserve without its own key is refused; the primary's key is never sent to another host). - Child-agent handoff schema and verify gate (#383): `tole_core::handoff` defines the typed handoff document (role `explore` / `implement` / `verify` / `fix`, artifact references by path and rev, verification evidence), strict validation of the model-authored input (workspace-relative paths, option-safe refs, bounded fields), the close gate (an implement/fix child without passing verification is `Unverified` / `VerifyFailed`, never closed silently) and the prompt for the follow-up `fix` child. `agent_start` takes `role` and `consumes` (artifact references the child reads itself), a role child gets a `handoff_submit` tool, and `agent_poll` reports the gate and a ready-made `fix_prompt` (the parent decides whether to start the fix child; nothing is spawned automatically). Design and anti-goals (no DAG engine) in `docs/child-agents-handoff.md`. - Sibling change notices, core (#380): `sibling_watch` tracks which files each child agent reads and writes in one shared append-only log (owner-only, bounded) and tells a child, once per stale path, when a sibling changed a file it had read or written; the `Tracked` tool decorator records `read_file` / `write_file` / `edit_file` / `delete_file` on success. Interrupts gain a `source`: a notice is delivered at the next safe point as an attributed entry (`source: notice`) and never withdraws output or supersedes a final answer. Wiring into `tole run` children follows separately. - Sibling change notices (#380): `sibling_watch` tracks which files each child agent reads and writes in one shared append-only log (owner-only, bounded) and tells a child, once per stale path, when a sibling changed a file it had read or written; the `Tracked` tool decorator records `read_file` / `write_file` / `edit_file` / `delete_file` on success. Interrupts gain a `source`: a notice is delivered at the next safe point as an attributed entry (`source: notice`) and never withdraws output or supersedes a final answer. Children spawned by `agent_start` get `TOLE_AGENT_ID` and a shared `TOLE_AGENT_TOUCH_LOG`; in a child, `tole run` wraps its file tools and runs the watcher, so a sibling's change reaches the model at its next safe point. diff --git a/README.md b/README.md index 94e0162..e28807e 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,7 @@ trust: `internal` / `read_only`), `--skill ` (load a SKILL.md), | Variable | Purpose | |----------|---------| | `TOLE_BASE_URL`, `TOLE_MODEL`, `TOLE_API_KEY` | LLM provider — any OpenAI-compatible endpoint (`OPENAI_*` equivalents read as fallback) | +| `TOLE_FALLBACK_BASE_URL`, `TOLE_FALLBACK_API_KEY`, `TOLE_FALLBACK_MODEL` | optional reserve provider (#209): a step that fails because the primary is unavailable (connection failure, timeout, 5xx, 429) is retried on the reserve, and the switch is recorded in the session log (`tole_route`). The reserve needs its own key (the primary's is never sent to another host); the model defaults to the primary's. Request errors (401/403/404/400) never fail over | | `TOLE_SYSTEM_PROMPT` | default persona when `--system` is absent | | `TOLE_MEMORY` | memory loop backend (`uteke`) — same as `--memory uteke` | | `TOLE_NO_UPDATE_CHECK` | `1` disables the startup update-check banner (issue #220) | diff --git a/crates/tole-cli/src/lib.rs b/crates/tole-cli/src/lib.rs index c64996b..ee749ca 100644 --- a/crates/tole-cli/src/lib.rs +++ b/crates/tole-cli/src/lib.rs @@ -10,6 +10,7 @@ pub mod config_apply; pub mod config_trust; pub mod doctor; pub mod handoff; +pub mod provider_chain; pub mod session_host; pub mod session_tools; pub mod sibling; diff --git a/crates/tole-cli/src/main.rs b/crates/tole-cli/src/main.rs index 209d22f..8f0bc63 100644 --- a/crates/tole-cli/src/main.rs +++ b/crates/tole-cli/src/main.rs @@ -28,7 +28,6 @@ use tole_core::gh::GhTool; use tole_core::git::GitTool; #[cfg(feature = "shell-tools")] use tole_core::jobs::{JobPollTool, JobStartTool}; -use tole_core::openai::OpenAiProvider; use tole_core::read_file::ReadFileTool; #[cfg(feature = "shell-tools")] use tole_core::run_command::RunCommandTool; @@ -1956,10 +1955,16 @@ fn run_command( .map_err(|e| anyhow::anyhow!("registering todo_write: {e}"))?; } - let mut provider = OpenAiProvider::new(cfg).with_tool_specs(registry.specs()); - if let Some(sys) = system_prompt.as_deref() { - provider = provider.with_system_prompt(sys); - } + let specs = registry.specs(); + let mut provider = + tole_cli::provider_chain::build(cfg, &tole_cli::config_apply::real_env, |p| { + let p = p.with_tool_specs(specs.clone()); + match system_prompt.as_deref() { + Some(sys) => p.with_system_prompt(sys), + None => p, + } + }) + .map_err(|e| anyhow::anyhow!(e))?; // Memory loop, pre-turn: recalled context rides inside the first // user message — the durable log stores exactly what was sent. The // summary remembers the PRE-injection prompt: storing the injected @@ -2125,13 +2130,20 @@ fn resume_command( .collect(); registry.set_hooks(hooks); } - let mut provider = OpenAiProvider::new(cfg).with_tool_specs(registry.specs()); // B2: the system prompt is pinned in the session header — resume // re-applies exactly what the session was created with (never the // ambient env, which may have changed since). - if let Some(sys) = storage.system_prompt() { - provider = provider.with_system_prompt(sys); - } + let specs = registry.specs(); + let pinned = storage.system_prompt().map(str::to_string); + let mut provider = + tole_cli::provider_chain::build(cfg, &tole_cli::config_apply::real_env, |p| { + let p = p.with_tool_specs(specs.clone()); + match pinned.as_deref() { + Some(sys) => p.with_system_prompt(sys), + None => p, + } + }) + .map_err(|e| anyhow::anyhow!(e))?; let outcome = match prompt { Some(text) if !text.trim().is_empty() => { // New instructions on a settled session (issue #55): the @@ -2501,12 +2513,19 @@ fn chat_command( "tole chat — session {session_id} (Ctrl-D exits, resume: tole chat --resume {session_id})" ); - let mut provider = OpenAiProvider::new(cfg).with_tool_specs(registry.specs()); // B2: fresh sessions pin the resolved prompt; resumed sessions re-apply // the header-pinned one (see create_with above / JsonlStorage::open). - if let Some(sys) = storage.system_prompt() { - provider = provider.with_system_prompt(sys); - } + let specs = registry.specs(); + let pinned = storage.system_prompt().map(str::to_string); + let mut provider = + tole_cli::provider_chain::build(cfg, &tole_cli::config_apply::real_env, |p| { + let p = p.with_tool_specs(specs.clone()); + match pinned.as_deref() { + Some(sys) => p.with_system_prompt(sys), + None => p, + } + }) + .map_err(|e| anyhow::anyhow!(e))?; // Memory loop state: recall rides into the first message of a FRESH // session only (a resumed session already carries its context); the diff --git a/crates/tole-cli/src/mission.rs b/crates/tole-cli/src/mission.rs index c1bd4f1..810765b 100644 --- a/crates/tole-cli/src/mission.rs +++ b/crates/tole-cli/src/mission.rs @@ -24,7 +24,6 @@ use anyhow::{Context, Result}; use serde_json::{json, Value}; -use tole_core::openai::OpenAiProvider; use tole_core::storage::{JsonlStorage, Storage}; use tole_core::subprocess::run_with_timeout; use tole_core::tool::ToolRegistry; @@ -269,7 +268,12 @@ pub fn run_mission( .map_err(|e| anyhow::anyhow!("registering todo_write: {e}"))?; } - let mut provider = OpenAiProvider::new(provider_cfg).with_tool_specs(registry.specs()); + let specs = registry.specs(); + let mut provider = + tole_cli::provider_chain::build(provider_cfg, &tole_cli::config_apply::real_env, |p| { + p.with_tool_specs(specs.clone()) + }) + .map_err(|e| anyhow::anyhow!(e))?; let mut turns: u64 = 0; let mut verify_failures: u32 = 0; diff --git a/crates/tole-cli/src/provider_chain.rs b/crates/tole-cli/src/provider_chain.rs new file mode 100644 index 0000000..f405caa --- /dev/null +++ b/crates/tole-cli/src/provider_chain.rs @@ -0,0 +1,161 @@ +//! Provider construction with an optional reserve (#209). +//! +//! Every session-building site configures an `OpenAiProvider` (tool specs, +//! system prompt, live-token sinks, ...). `build` applies that SAME +//! configuration to the primary and, when `TOLE_FALLBACK_BASE_URL` is set, +//! to a reserve, and returns a `FailoverProvider` over the two. Without a +//! reserve the plain provider is returned: no behaviour change. +//! +//! The reserve needs its own `TOLE_FALLBACK_API_KEY`: the primary's key is +//! never sent to another host. `TOLE_FALLBACK_MODEL` defaults to the +//! primary's model. + +use tole_core::failover::{FailoverProvider, Link}; +use tole_core::openai::{OpenAiConfig, OpenAiProvider}; +use tole_core::provider::Provider; + +pub const ENV_FALLBACK_BASE_URL: &str = "TOLE_FALLBACK_BASE_URL"; +pub const ENV_FALLBACK_MODEL: &str = "TOLE_FALLBACK_MODEL"; +// The env var NAME, not a secret value. +// cora-ignore: sec-hardcoded-secret +pub const ENV_FALLBACK_API_KEY: &str = "TOLE_FALLBACK_API_KEY"; + +fn nonblank(env: &dyn Fn(&str) -> Option, name: &str) -> Option { + env(name) + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) +} + +/// The reserve's config, if one is configured. A base URL without a key is +/// an error (fail loud rather than run without the reserve the operator +/// asked for). +pub fn fallback_config( + primary: &OpenAiConfig, + env: &dyn Fn(&str) -> Option, +) -> Result, String> { + let Some(base_url) = nonblank(env, ENV_FALLBACK_BASE_URL) else { + return Ok(None); + }; + let api_key = nonblank(env, ENV_FALLBACK_API_KEY).ok_or_else(|| { + format!( + "{ENV_FALLBACK_BASE_URL} is set but {ENV_FALLBACK_API_KEY} is not: the reserve needs \ + its own key (the primary's key is never sent to another host)" + ) + })?; + let model = nonblank(env, ENV_FALLBACK_MODEL).unwrap_or_else(|| primary.model.clone()); + Ok(Some(OpenAiConfig::new(base_url, model, api_key))) +} + +/// `role: model @ host` with URL credentials stripped (recorded in +/// `tole_route`, so it must never carry a secret). +pub fn label(role: &str, cfg: &OpenAiConfig) -> String { + format!( + "{role}: {} @ {}", + cfg.model, + crate::doctor::redact_url(&cfg.base_url) + ) +} + +/// Build the session's provider: `configure` is applied to the primary and, +/// when configured, to the reserve. +pub fn build( + primary: OpenAiConfig, + env: &dyn Fn(&str) -> Option, + configure: impl Fn(OpenAiProvider) -> OpenAiProvider, +) -> Result, String> { + let Some(reserve) = fallback_config(&primary, env)? else { + return Ok(Box::new(configure(OpenAiProvider::new(primary)))); + }; + let chain = vec![ + Link { + label: label("primary", &primary), + provider: Box::new(configure(OpenAiProvider::new(primary))), + }, + Link { + label: label("fallback", &reserve), + provider: Box::new(configure(OpenAiProvider::new(reserve))), + }, + ]; + Ok(Box::new(FailoverProvider::new(chain))) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn env<'a>(pairs: &'a [(&'a str, &'a str)]) -> impl Fn(&str) -> Option + 'a { + move |k| { + pairs + .iter() + .find(|(n, _)| *n == k) + .map(|(_, v)| v.to_string()) + } + } + + fn primary() -> OpenAiConfig { + OpenAiConfig::new("https://a.example/v1", "model-a", "sk-primary") + } + + #[test] + fn no_reserve_unless_a_base_url_is_set() { + assert_eq!(fallback_config(&primary(), &env(&[])), Ok(None)); + assert_eq!( + fallback_config(&primary(), &env(&[(ENV_FALLBACK_BASE_URL, " ")])), + Ok(None) + ); + } + + #[test] + fn the_reserve_needs_its_own_key_and_inherits_the_model() { + let e = fallback_config( + &primary(), + &env(&[(ENV_FALLBACK_BASE_URL, "https://b.example/v1")]), + ) + .unwrap_err(); + assert!(e.contains(ENV_FALLBACK_API_KEY), "{e}"); + assert!(!e.contains("sk-primary")); + let r = fallback_config( + &primary(), + &env(&[ + (ENV_FALLBACK_BASE_URL, "https://b.example/v1"), + (ENV_FALLBACK_API_KEY, "sk-b"), + ]), + ) + .unwrap() + .unwrap(); + assert_eq!( + (r.base_url.as_str(), r.model.as_str(), r.api_key.as_str()), + ("https://b.example/v1", "model-a", "sk-b") + ); + let r = fallback_config( + &primary(), + &env(&[ + (ENV_FALLBACK_BASE_URL, "https://b.example/v1"), + (ENV_FALLBACK_API_KEY, "sk-b"), + (ENV_FALLBACK_MODEL, "model-b"), + ]), + ) + .unwrap() + .unwrap(); + assert_eq!(r.model, "model-b"); + } + + #[test] + fn labels_never_carry_credentials() { + let cfg = OpenAiConfig::new("https://user:pw@b.example/v1?key=k", "m", "sk-x"); + let l = label("fallback", &cfg); + assert_eq!(l, "fallback: m @ https://b.example/v1"); + assert!(!l.contains("pw") && !l.contains("sk-x")); + } + + #[test] + fn build_fails_loud_on_a_half_configured_reserve() { + let r = build( + primary(), + &env(&[(ENV_FALLBACK_BASE_URL, "https://b.example/v1")]), + |p| p, + ); + assert!(r.is_err()); + assert!(build(primary(), &env(&[]), |p| p).is_ok()); + } +} diff --git a/crates/tole-cli/src/session_host.rs b/crates/tole-cli/src/session_host.rs index 2befdc2..3c9c238 100644 --- a/crates/tole-cli/src/session_host.rs +++ b/crates/tole-cli/src/session_host.rs @@ -598,22 +598,26 @@ pub fn run_session_turn( { cfg.model = model.to_string(); } - let mut provider = - tole_core::openai::OpenAiProvider::new(cfg).with_tool_specs(registry.specs()); - if let Some(sys) = system_prompt.as_deref() { - provider = provider.with_system_prompt(sys); - } // Live-token sinks (issue #196 phase 3): when an observer is wired // (ACP), streamed reasoning/content deltas flow to it DURING the - // provider call; the decisive output semantics are unchanged. - if let Some(o) = observer.as_ref() { - let ot = StdArc::clone(o); - let or = StdArc::clone(o); - provider = provider.with_delta_sinks( - Some(StdArc::new(move |t: &str| ot.text_delta(t))), - Some(StdArc::new(move |r: &str| or.reasoning(r))), - ); - } + // provider call; the decisive output semantics are unchanged. The same + // configuration applies to a reserve provider (#209). + let specs = registry.specs(); + let mut provider = crate::provider_chain::build(cfg, &crate::config_apply::real_env, |p| { + let mut p = p.with_tool_specs(specs.clone()); + if let Some(sys) = system_prompt.as_deref() { + p = p.with_system_prompt(sys); + } + if let Some(o) = observer.as_ref() { + let ot = StdArc::clone(o); + let or = StdArc::clone(o); + p = p.with_delta_sinks( + Some(StdArc::new(move |t: &str| ot.text_delta(t))), + Some(StdArc::new(move |r: &str| or.reasoning(r))), + ); + } + p + })?; let outcome = tole_core::turn::run_turn_with_observer( &mut *storage, @@ -687,11 +691,14 @@ pub fn resume_session_turn( { cfg.model = model.to_string(); } - let mut provider = - tole_core::openai::OpenAiProvider::new(cfg).with_tool_specs(registry.specs()); - if let Some(sys) = system_prompt.as_deref() { - provider = provider.with_system_prompt(sys); - } + let specs = registry.specs(); + let mut provider = crate::provider_chain::build(cfg, &crate::config_apply::real_env, |p| { + let p = p.with_tool_specs(specs.clone()); + match system_prompt.as_deref() { + Some(sys) => p.with_system_prompt(sys), + None => p, + } + })?; // #318: the session token is wired into the replan loop, so // `/cancel` during an approval-resume settles the turn as cancelled. // The `reset()` above stays: like `run_session_turn`, it runs inside diff --git a/crates/tole-cli/tests/provider_failover_e2e.rs b/crates/tole-cli/tests/provider_failover_e2e.rs new file mode 100644 index 0000000..f488be6 --- /dev/null +++ b/crates/tole-cli/tests/provider_failover_e2e.rs @@ -0,0 +1,167 @@ +//! #209 provider failover through the REAL `tole run` binary: the primary +//! is unreachable, the reserve configured with `TOLE_FALLBACK_*` answers, +//! and the switch is recorded in the durable session log. + +use serde_json::json; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::process::Command; +use std::time::Duration; + +/// A provider that answers every chat completion with `text`. +fn serve_final(text: &'static str) -> String { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + for conn in listener.incoming() { + let Ok(mut s) = conn else { continue }; + let _ = s.set_read_timeout(Some(Duration::from_secs(5))); + let mut buf = Vec::new(); + let mut chunk = [0u8; 16384]; + loop { + let n = s.read(&mut chunk).unwrap_or(0); + if n == 0 { + break; + } + buf.extend_from_slice(&chunk[..n]); + if let Some(h) = buf.windows(4).position(|w| w == b"\r\n\r\n") { + let head = String::from_utf8_lossy(&buf[..h]).to_ascii_lowercase(); + let len: usize = head + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if buf.len() >= h + 4 + len { + break; + } + } + } + let body = json!({"choices":[{"message":{"role":"assistant","content":text}}], + "usage":{"prompt_tokens":2,"completion_tokens":1}}) + .to_string(); + let _ = s.write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .as_bytes(), + ); + } + }); + format!("http://{addr}/v1") +} + +fn closed_port_url() -> String { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = l.local_addr().unwrap().port(); + drop(l); + format!("http://127.0.0.1:{port}/v1") +} + +fn tmpdir(tag: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("tole-failover-e2e-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +fn tole(dir: &std::path::Path, primary: &str, fallback: &[(&str, &str)]) -> std::process::Output { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_tole")); + cmd.args(["--sessions-dir"]) + .arg(dir.join("sessions")) + .args(["run", "say hi"]) + .current_dir(dir) + .env("TOLE_BASE_URL", primary) + .env("TOLE_MODEL", "m-primary") + .env("TOLE_API_KEY", "sk-primary") + .env("TOLE_STREAM", "0") + .env("TOLE_NO_UPDATE_CHECK", "1") + .env_remove("TOLE_MEMORY") + .env_remove("TOLE_TRUST") + .env_remove("OPENAI_BASE_URL") + .env_remove("OPENAI_API_KEY") + .env_remove("OPENAI_MODEL"); + for k in [ + "TOLE_FALLBACK_BASE_URL", + "TOLE_FALLBACK_API_KEY", + "TOLE_FALLBACK_MODEL", + ] { + cmd.env_remove(k); + } + for (k, v) in fallback { + cmd.env(k, v); + } + cmd.output().unwrap() +} + +#[test] +fn a_dead_primary_fails_over_to_the_reserve_and_the_switch_is_durable() { + let dir = tmpdir("ok"); + let reserve = serve_final("FROM_RESERVE"); + let out = tole( + &dir, + &closed_port_url(), + &[ + ("TOLE_FALLBACK_BASE_URL", &reserve), + ("TOLE_FALLBACK_API_KEY", "sk-reserve"), + ], + ); + let stdout = String::from_utf8_lossy(&out.stdout); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(out.status.success(), "stdout={stdout}\nstderr={stderr}"); + assert!(stdout.contains("FROM_RESERVE"), "stdout={stdout}"); + // The session log records which provider answered and why. + let log = std::fs::read_dir(dir.join("sessions")) + .unwrap() + .flatten() + .find(|e| e.path().extension().is_some_and(|x| x == "jsonl")) + .map(|e| std::fs::read_to_string(e.path()).unwrap()) + .expect("a session log"); + assert!(log.contains("\"tole_route\""), "{log}"); + assert!( + log.contains("fallback: m-primary @ http://127.0.0.1"), + "{log}" + ); + assert!(log.contains("\"from\":\"primary: m-primary"), "{log}"); + // Neither key ends up in the durable log. + assert!(!log.contains("sk-reserve") && !log.contains("sk-primary")); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_reserve_without_its_own_key_is_refused_loudly() { + let dir = tmpdir("nokey"); + let out = tole( + &dir, + &closed_port_url(), + &[("TOLE_FALLBACK_BASE_URL", "http://127.0.0.1:9/v1")], + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(!out.status.success()); + assert!(stderr.contains("TOLE_FALLBACK_API_KEY"), "{stderr}"); + assert!(!stderr.contains("sk-primary"), "{stderr}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn without_a_reserve_nothing_changes() { + let dir = tmpdir("none"); + let primary = serve_final("FROM_PRIMARY"); + let out = tole(&dir, &primary, &[]); + let stdout = String::from_utf8_lossy(&out.stdout); + assert!( + out.status.success() && stdout.contains("FROM_PRIMARY"), + "{stdout}" + ); + let log = std::fs::read_dir(dir.join("sessions")) + .unwrap() + .flatten() + .find(|e| e.path().extension().is_some_and(|x| x == "jsonl")) + .map(|e| std::fs::read_to_string(e.path()).unwrap()) + .unwrap(); + assert!( + !log.contains("tole_route"), + "a single provider records no route" + ); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/tole-core/src/provider.rs b/crates/tole-core/src/provider.rs index cbb57cd..8670abc 100644 --- a/crates/tole-core/src/provider.rs +++ b/crates/tole-core/src/provider.rs @@ -87,3 +87,24 @@ pub trait Provider { None } } + +/// A boxed provider is a provider (hosts pick the concrete type at runtime, +/// e.g. a plain provider or a `failover::FailoverProvider`). Every method is +/// forwarded, including the defaulted ones. +impl Provider for Box

{ + fn complete(&mut self, transcript: &[Entry]) -> Result { + (**self).complete(transcript) + } + fn last_usage(&self) -> Option { + (**self).last_usage() + } + fn last_reasoning(&self) -> Option { + (**self).last_reasoning() + } + fn last_wire_stats(&self) -> Option { + (**self).last_wire_stats() + } + fn last_route(&self) -> Option { + (**self).last_route() + } +} From a3a5457b1889cf9fb54f0364afe27d952d48ec70 Mon Sep 17 00:00:00 2001 From: Aji Anaz Date: Sun, 18 Oct 2026 10:53:48 +0700 Subject: [PATCH 3/3] fix(core): close three failover edge cases found by the cora scan (#209) - A stream cut after a live sink showed output now reports "stream cut after output" and never fails over, so the reserve cannot re-stream a full answer after the partial one. - The chain walks from the active link and wraps round, so a sticky reserve that fails during the cooldown falls back to the primary. - The turn loop's transient retry reads only the last provider's error (failover::leading_error), so an earlier "timeout"/"429" no longer makes a fail-loud 401 from the reserve look retryable. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- crates/tole-core/src/failover.rs | 97 +++++++++++++++++++++++++++++--- crates/tole-core/src/openai.rs | 72 +++++++++++++++++++++++- crates/tole-core/src/turn.rs | 31 +++++++++- 4 files changed, 191 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7426266..26700bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Provider failover chain (#209): `failover::FailoverProvider` retries a step on the next provider of an ordered chain when the active one is unavailable (connection failure, timeout, dropped stream, HTTP 5xx or 429); request errors (401/403/404/400, malformed responses) fail loud and never fail over. The answering provider stays active and the primary is retried after a cooldown. Each step records the route under `tole_route` in its usage row (durable, auditable); rows holding only tole's own keys are not counted as provider steps. Wired into every session face through `TOLE_FALLBACK_BASE_URL` / `TOLE_FALLBACK_API_KEY` / `TOLE_FALLBACK_MODEL` (a reserve without its own key is refused; the primary's key is never sent to another host). +- Provider failover chain (#209): `failover::FailoverProvider` retries a step on the next provider of an ordered chain when the active one is unavailable (connection failure, timeout, dropped stream, HTTP 5xx or 429); request errors (401/403/404/400, malformed responses) fail loud and never fail over. The answering provider stays active and the primary is retried after a cooldown. Each step records the route under `tole_route` in its usage row (durable, auditable); rows holding only tole's own keys are not counted as provider steps. Wired into every session face through `TOLE_FALLBACK_BASE_URL` / `TOLE_FALLBACK_API_KEY` / `TOLE_FALLBACK_MODEL` (a reserve without its own key is refused; the primary's key is never sent to another host). A stream that breaks after part of the answer was already shown live does not fail over (no duplicated output); a reserve that fails before the cooldown ends wraps back to the primary; and after a failover only the last provider's error decides the turn loop's transient retry, so an earlier timeout no longer makes a reserve's 401 look retryable. - Child-agent handoff schema and verify gate (#383): `tole_core::handoff` defines the typed handoff document (role `explore` / `implement` / `verify` / `fix`, artifact references by path and rev, verification evidence), strict validation of the model-authored input (workspace-relative paths, option-safe refs, bounded fields), the close gate (an implement/fix child without passing verification is `Unverified` / `VerifyFailed`, never closed silently) and the prompt for the follow-up `fix` child. `agent_start` takes `role` and `consumes` (artifact references the child reads itself), a role child gets a `handoff_submit` tool, and `agent_poll` reports the gate and a ready-made `fix_prompt` (the parent decides whether to start the fix child; nothing is spawned automatically). Design and anti-goals (no DAG engine) in `docs/child-agents-handoff.md`. - Sibling change notices, core (#380): `sibling_watch` tracks which files each child agent reads and writes in one shared append-only log (owner-only, bounded) and tells a child, once per stale path, when a sibling changed a file it had read or written; the `Tracked` tool decorator records `read_file` / `write_file` / `edit_file` / `delete_file` on success. Interrupts gain a `source`: a notice is delivered at the next safe point as an attributed entry (`source: notice`) and never withdraws output or supersedes a final answer. Wiring into `tole run` children follows separately. - Sibling change notices (#380): `sibling_watch` tracks which files each child agent reads and writes in one shared append-only log (owner-only, bounded) and tells a child, once per stale path, when a sibling changed a file it had read or written; the `Tracked` tool decorator records `read_file` / `write_file` / `edit_file` / `delete_file` on success. Interrupts gain a `source`: a notice is delivered at the next safe point as an attributed entry (`source: notice`) and never withdraws output or supersedes a final answer. Children spawned by `agent_start` get `TOLE_AGENT_ID` and a shared `TOLE_AGENT_TOUCH_LOG`; in a child, `tole run` wraps its file tools and runs the watcher, so a sibling's change reaches the model at its next safe point. diff --git a/crates/tole-core/src/failover.rs b/crates/tole-core/src/failover.rs index 21b84c9..a53f81c 100644 --- a/crates/tole-core/src/failover.rs +++ b/crates/tole-core/src/failover.rs @@ -26,10 +26,32 @@ pub const DEFAULT_RETRY_PRIMARY_AFTER: Duration = Duration::from_secs(300); /// Longest error text kept per failed provider in the route record. const MAX_ERROR_CHARS: usize = 200; +/// Separates the last provider's error from the earlier ones in a +/// combined error: `" [after failover from