From d47cda726409ef06a91e998325ad6555b67577c3 Mon Sep 17 00:00:00 2001 From: Kyle Tse Date: Mon, 16 Nov 2026 02:19:47 +0000 Subject: [PATCH] fix(supervisor): commit a proven line before an ended turn's custody is read A harness that answers `Deferred` can take and finish a turn before the pass returns. The idle pass then read the ended turn while the store held no open turn for its proven line: an errored turn was followed up with no turn to carry (the recovery note was skipped, and the turn stayed closed through its whole backoff), and a completed turn was left open behind its own proof. `record_usage_with` now commits what the harness proved before it reads or closes an ended turn's custody, and retries the pass when the store cannot record the proof. Test clocks no longer race the supervisor: time moves only to a timer a worker is blocked on, and mail that arrives during a backoff does so with time already at its deadline. --- crates/anyagents-supervisor/src/lib.rs | 25 +- .../tests/errored_turns.rs | 226 +++++++++++++----- .../anyagents-supervisor/tests/support/mod.rs | 67 ++++++ crates/anyagents/src/vendor/spiron_tests.rs | 46 +++- 4 files changed, 292 insertions(+), 72 deletions(-) diff --git a/crates/anyagents-supervisor/src/lib.rs b/crates/anyagents-supervisor/src/lib.rs index f488b57f..c1ab7a8f 100644 --- a/crates/anyagents-supervisor/src/lib.rs +++ b/crates/anyagents-supervisor/src/lib.rs @@ -1694,18 +1694,23 @@ impl Session { /// skips the delivery pass: a quota wait, a switch drain, or replacing /// the process, alive or dead. A proof outlives the process that gave it /// but not the adapter that holds it, so an uncommitted one would hand - /// the message to the next process again. Nothing is written. - fn commit_proven(&mut self) { + /// the message to the next process again. Nothing is written. `false` + /// when the store could not record a proof: the commit is tried again. + fn commit_proven(&mut self) -> bool { let Some(delivery) = self.delivery.as_mut() else { - return; + return true; }; match delivery.commit_proven() { Ok(committed) => { self.written = self .written .saturating_add(committed.clamp(0, u64::from(u32::MAX)) as u32); + true + } + Err(err) => { + self.last_error = Some(format!("commit proven mail: {err}")); + false } - Err(err) => self.last_error = Some(format!("commit proven mail: {err}")), } } @@ -2005,6 +2010,18 @@ impl Session { let advanced = usage_advanced(&served, &before); let ended = (served_turn || (advanced && health.process_alive)) && health.turn_state == TurnState::Idle; + // A line the harness took and finished while `deliver` still answered + // `Deferred` is proven, not yet committed: the store holds no open + // turn for it. Commit that proof before the turn's custody is read or + // closed below, or an errored turn would be followed up as one + // nobody holds open, and a completed one left open behind its own + // proof. A store that cannot record it leaves the turn for another pass. + if ended && !self.commit_proven() { + return false; + } + let Some(delivery) = self.delivery.as_ref() else { + return true; + }; let completed = ended && health.sign_in_fault.is_none(); // A turn the harness saw end with an error (not a quota wait) never // holds the mailbox: it is counted now, and `follow_errored_turn` diff --git a/crates/anyagents-supervisor/tests/errored_turns.rs b/crates/anyagents-supervisor/tests/errored_turns.rs index 30150498..57c9f795 100644 --- a/crates/anyagents-supervisor/tests/errored_turns.rs +++ b/crates/anyagents-supervisor/tests/errored_turns.rs @@ -5,15 +5,16 @@ mod support; -use std::sync::{Arc, Mutex, atomic::AtomicBool}; +use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use anyagents_core::{Incoming, MAX_ERRORED_TURNS, NewAgent, SYSTEM_SENDER, Store}; use anyagents_harness::{ DeliverOutcome, Delivery, DeliveryLevel, Harness, HarnessError, HarnessHealth, OutboundMessage, - TurnState, + Proof, TurnState, }; use anyagents_supervisor::{Clock, LaunchRequest, ManualClock, Supervisor, SystemClock}; +use support::RecordingClock; const OVERFLOW: &str = "Sylphx AI failure Some(\"context_length_exceeded\"): The request exceeds the model's context window."; @@ -21,11 +22,15 @@ const OVERFLOW: &str = "Sylphx AI failure Some(\"context_length_exceeded\"): The const UNAVAILABLE: &str = "Sylphx AI failure Some(\"service_unavailable\"): The model response was interrupted. Retry the request."; /// A vendor whose turns end at once: the next `fail` of them with `error` -/// (as Spiron reports a turn the model refused), the rest completed. +/// (as Spiron reports a turn the model refused), the rest completed. With +/// `deferred` each input is written, taken and finished before `deliver` +/// returns, yet answered `Deferred`: the vendor's acceptance and result reach +/// the adapter after the write, so the line is proven rather than reported. #[derive(Default, Clone)] struct Vendor { fail: usize, error: Option<&'static str>, + deferred: bool, served: u32, fault: Option, given: Vec<(String, String)>, @@ -56,10 +61,22 @@ impl Harness for ErroringHarness { } else { None }; - Ok(DeliverOutcome::Accepted { - turn_finished: true, + Ok(if vendor.deferred { + DeliverOutcome::Deferred + } else { + DeliverOutcome::Accepted { + turn_finished: true, + } }) } + fn proven(&self, message_id: &str) -> Option { + let vendor = self.vendor.lock().unwrap(); + (vendor.deferred && vendor.given.iter().any(|(id, _)| id == message_id)).then_some( + Proof::Taken { + turn_finished: true, + }, + ) + } fn background(&mut self) -> Result<(), HarnessError> { Ok(()) } @@ -113,6 +130,17 @@ fn start(fail: usize) -> Fixture { } fn start_with(fail: usize, error: Option<&'static str>, clock: Arc) -> Fixture { + start_vendor( + Vendor { + fail, + error, + ..Vendor::default() + }, + clock, + ) +} + +fn start_vendor(vendor: Vendor, clock: Arc) -> Fixture { let dir = tempfile::tempdir().unwrap(); let store = Arc::new(Store::open(dir.path().join("state.db")).unwrap()); let lead = store.create_agent(NewAgent::new("lead", "spiron")).unwrap(); @@ -129,11 +157,7 @@ fn start_with(fail: usize, error: Option<&'static str>, clock: Arc) - store .set_agent_running(&worker.id, &work.display().to_string()) .unwrap(); - let vendor = Arc::new(Mutex::new(Vendor { - fail, - error, - ..Vendor::default() - })); + let vendor = Arc::new(Mutex::new(vendor)); let supervisor = Supervisor::start( Arc::clone(&store), clock, @@ -304,52 +328,35 @@ fn turns_that_keep_erroring_park_the_agent_and_tell_its_parent() { support::stop(fx.supervisor); } -// Record the actual installed deadline, rather than racing durable error -// persistence (which precedes installation of the in-memory follow-up). -struct RecordingClock { - inner: ManualClock, - waits: Mutex)>>, -} - -impl Clock for RecordingClock { - fn now_ms(&self) -> i64 { - self.inner.now_ms() - } - fn wait(&self, deadline: Option, woken: &Arc) -> bool { - self.waits.lock().unwrap().push((self.now_ms(), deadline)); - self.inner.wait(deadline, woken) - } - fn poke(&self) { - self.inner.poke(); - } -} - // This fixture proves terminal consumption using a new, nonduplicate served // counter and Idle with no outstanding inputs. It does not infer a terminal // result from an arbitrary provider error or authorise replay of partial work. fn terminal_systemic_fault_keeps_backoff_and_queued_mail(error: &'static str) { - let clock = ManualClock::new(1_700_000_000_000); - let recorded = Arc::new(RecordingClock { - inner: clock.clone(), - waits: Mutex::new(Vec::new()), - }); + const START: i64 = 1_700_000_000_000; + let clock = ManualClock::new(START); + let recorded = RecordingClock::new(clock.clone()); let fx = start_with(1, Some(error), recorded.clone()); let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { fx.store .receive(&fx.worker.id, Incoming::new("original task")) .unwrap(); assert!(wait_until(|| fx.vendor.lock().unwrap().given.len() == 1)); - clock.set_ms(clock.now_ms() + 1_000); - assert!(wait_until(|| fx - .store - .turn_errors(&fx.worker.id) - .unwrap() - .is_some_and( - |errors| errors.last_error.as_deref() == Some(error) - ))); + // The pass that records the error runs on the short timer the worker + // installs after the delivery: fire it once it is installed, never + // earlier, and never past the second the backoff window allows for. + let errored = || { + fx.store + .turn_errors(&fx.worker.id) + .unwrap() + .is_some_and(|errors| errors.last_error.as_deref() == Some(error)) + }; + assert!(wait_until(|| errored() || { + recorded.fire_next_timer(START + 1_000); + false + })); assert!(wait_until(|| { - recorded.waits.lock().unwrap().iter().any(|(_, deadline)| { - deadline.is_some_and(|at| (1_700_000_015_000..=1_700_000_016_000).contains(&at)) + recorded.waits().iter().any(|(_, deadline)| { + deadline.is_some_and(|at| (START + 15_000..=START + 16_000).contains(&at)) }) || fx.vendor.lock().unwrap().attempts.len() > 1 })); let (original_id, attempts) = { @@ -358,23 +365,23 @@ fn terminal_systemic_fault_keeps_backoff_and_queued_mail(error: &'static str) { }; assert_eq!(attempts.as_slice(), std::slice::from_ref(&original_id)); let deadline = recorded - .waits - .lock() - .unwrap() + .waits() .iter() .find_map(|(_, deadline)| { - deadline.filter(|at| (1_700_000_015_000..=1_700_000_016_000).contains(at)) + deadline.filter(|at| (START + 15_000..=START + 16_000).contains(at)) }) .unwrap(); - let observed = recorded.waits.lock().unwrap().len(); - fx.store - .receive(&fx.worker.id, Incoming::new("mail during backoff")) - .unwrap(); // A mailbox wake must neither erase the terminal follow-up nor bypass its // deadline. Process replacement must not turn it into an open-turn replay. + // Time stands just before the deadline when the mail arrives, so the + // wake the mail causes re-installs the backoff at that time. clock.set_ms(deadline - 1); + let observed = recorded.waits().len(); + fx.store + .receive(&fx.worker.id, Incoming::new("mail during backoff")) + .unwrap(); assert!(wait_until( - || recorded.waits.lock().unwrap()[observed..].contains(&(deadline - 1, Some(deadline))) + || recorded.waits()[observed..].contains(&(deadline - 1, Some(deadline))) )); { let vendor = fx.vendor.lock().unwrap().clone(); @@ -454,30 +461,123 @@ fn transient_text_cannot_discard_proven_terminal_backoff_and_mail() { terminal_systemic_fault_keeps_backoff_and_queued_mail("HTTP 503: service unavailable"); } +fn received(store: &Store, agent: &anyagents_core::Agent, text: &str) -> anyagents_core::Message { + match store.receive(&agent.id, Incoming::new(text)).unwrap() { + anyagents_core::ReceiveOutcome::New(message) => message, + anyagents_core::ReceiveOutcome::Duplicate(_) => unreachable!(), + } +} + +/// A vendor that has taken and finished a turn before `deliver` returns +/// answers `Deferred`: the line is proven, not reported. The pass that finds +/// the turn ended commits that proof first, so the turn is held open for its +/// follow-up (an error) or closed (a completion), and the line is never +/// handed over again. +#[test] +fn an_errored_turn_proven_while_deferred_stays_open_for_its_follow_up() { + const START: i64 = 1_700_000_000_000; + let clock = ManualClock::new(START); + let fx = start_vendor( + Vendor { + fail: 1, + error: Some(UNAVAILABLE), + deferred: true, + ..Vendor::default() + }, + Arc::new(clock.clone()) as Arc, + ); + let task = received(&fx.store, &fx.worker, "summarise the incident"); + assert!(wait_until(|| { + fx.store + .turn_errors(&fx.worker.id) + .unwrap() + .is_some_and(|errors| errors.last_error.as_deref() == Some(UNAVAILABLE)) + })); + assert!( + fx.store.has_open_turn(&fx.worker.id).unwrap(), + "the errored turn is held open for its follow-up" + ); + assert_eq!( + fx.store.delivered_cursor(&fx.worker.id).unwrap(), + task.seq, + "the proof is committed, not handed over again" + ); + // The systemic backoff ends: the note goes out and closes the turn. + clock.set_ms(START + 15_000); + assert!(wait_until(|| fx.vendor.lock().unwrap().given.len() == 2)); + assert!(wait_until(|| !fx + .store + .has_open_turn(&fx.worker.id) + .unwrap())); + let vendor = fx.vendor.lock().unwrap().clone(); + assert!( + vendor.given[1].1.contains("[Recovery]") && vendor.given[1].1.contains(task.id.as_str()), + "{:?}", + vendor.given + ); + assert_eq!( + vendor + .attempts + .iter() + .filter(|id| **id == vendor.given[0].0) + .count(), + 1, + "the proven task is never offered again: {:?}", + vendor.attempts + ); + support::stop(fx.supervisor); +} + +#[test] +fn a_completed_turn_proven_while_deferred_is_closed() { + let fx = start_vendor( + Vendor { + deferred: true, + ..Vendor::default() + }, + Arc::new(SystemClock::new()), + ); + let task = received(&fx.store, &fx.worker, "summarise the incident"); + assert!(wait_until(|| fx + .store + .delivered_cursor(&fx.worker.id) + .unwrap() + >= task.seq)); + assert!( + wait_until(|| !fx.store.has_open_turn(&fx.worker.id).unwrap()), + "a turn the vendor finished is not left open" + ); + assert_eq!(fx.vendor.lock().unwrap().attempts.len(), 1); + support::stop(fx.supervisor); +} + /// A provider that is unavailable is the fleet's trouble, not the agent's: /// each retry waits out a growing backoff, the turn stays open (a restart /// would recover it) while it waits, and no run of them ever parks it. #[test] fn systemic_errors_back_off_and_never_park() { - let clock = ManualClock::new(1_700_000_000_000); - let fx = start_with( - usize::MAX, - Some(UNAVAILABLE), - Arc::new(clock.clone()) as Arc, - ); + const START: i64 = 1_700_000_000_000; + let clock = ManualClock::new(START); + let recorded = RecordingClock::new(clock.clone()); + let fx = start_with(usize::MAX, Some(UNAVAILABLE), recorded.clone()); let given = || fx.vendor.lock().unwrap().given.len(); let advance = |ms: i64| clock.set_ms(clock.now_ms() + ms); fx.store .receive(&fx.worker.id, Incoming::new("summarise the incident")) .unwrap(); assert!(wait_until(|| given() == 1)); - // The next pass finds the errored turn and starts its backoff. - advance(1_000); - assert!(wait_until(|| { + // The next pass finds the errored turn and starts its backoff. It runs on + // the short timer the worker installs after the delivery: fire it once it + // is installed, as time moved earlier is read by the worker as its own. + let errored = || { fx.store .turn_errors(&fx.worker.id) .unwrap() .is_some_and(|errors| errors.last_error.as_deref() == Some(UNAVAILABLE)) + }; + assert!(wait_until(|| errored() || { + recorded.fire_next_timer(START + 1_000); + false })); advance(10_000); std::thread::sleep(Duration::from_millis(200)); diff --git a/crates/anyagents-supervisor/tests/support/mod.rs b/crates/anyagents-supervisor/tests/support/mod.rs index edd45aad..3bd318ce 100644 --- a/crates/anyagents-supervisor/tests/support/mod.rs +++ b/crates/anyagents-supervisor/tests/support/mod.rs @@ -3,8 +3,11 @@ #![allow(dead_code)] +use std::sync::{Arc, Mutex, atomic::AtomicBool}; use std::time::Duration; +use anyagents_supervisor::{Clock, ManualClock}; + /// How long a supervisor may take to stop. pub const STOP_WITHIN: Duration = Duration::from_secs(30); @@ -31,3 +34,67 @@ pub fn within(what: &str, stop: impl FnOnce() + Send + 'static) { pub fn stop(supervisor: T) { within("the supervisor stops", move || drop(supervisor)); } + +/// A [`ManualClock`] that records every wait it is asked for, and which of +/// them still block a worker. A test reads the deadline a worker actually +/// installed, and moves time only to a timer one waits on: time moved before +/// a worker reads the clock shifts the timer it is about to install past the +/// new time, and nothing wakes it. +pub struct RecordingClock { + inner: ManualClock, + waits: Mutex)>>, + blocked: Mutex>, +} + +impl RecordingClock { + pub fn new(inner: ManualClock) -> Arc { + Arc::new(Self { + inner, + waits: Mutex::new(Vec::new()), + blocked: Mutex::new(Vec::new()), + }) + } + + /// Every wait so far: the time it was asked at and its deadline. + pub fn waits(&self) -> Vec<(i64, Option)> { + self.waits.lock().unwrap().clone() + } + + /// Move time to the nearest timer a worker is blocked on, when it falls + /// after now and no later than `limit`. `true` when time moved. + pub fn fire_next_timer(&self, limit: i64) -> bool { + let now = self.inner.now_ms(); + let next = self + .blocked + .lock() + .unwrap() + .iter() + .copied() + .filter(|at| (now + 1..=limit).contains(at)) + .min(); + next.map(|at| self.inner.set_ms(at)).is_some() + } +} + +impl Clock for RecordingClock { + fn now_ms(&self) -> i64 { + self.inner.now_ms() + } + fn wait(&self, deadline: Option, woken: &Arc) -> bool { + self.waits.lock().unwrap().push((self.now_ms(), deadline)); + if let Some(at) = deadline { + self.blocked.lock().unwrap().push(at); + } + let flagged = self.inner.wait(deadline, woken); + if let Some(at) = deadline { + let mut blocked = self.blocked.lock().unwrap(); + if let Some(index) = blocked.iter().position(|blocked| *blocked == at) { + blocked.swap_remove(index); + } + } + flagged + } + fn poke(&self) { + self.inner.poke(); + } +} diff --git a/crates/anyagents/src/vendor/spiron_tests.rs b/crates/anyagents/src/vendor/spiron_tests.rs index 87849f94..8ddc796a 100644 --- a/crates/anyagents/src/vendor/spiron_tests.rs +++ b/crates/anyagents/src/vendor/spiron_tests.rs @@ -606,9 +606,26 @@ fn spiron_supervisor_recovers_from_transient_mid_turn_server_error_via_recovery( // Observe the installed wait, not error persistence, which precedes // installation of the in-memory follow-up and its not_before deadline. + // Time moves only to a timer a worker is blocked on: moved earlier, the + // timer it is about to install lands past the new time and nothing wakes it. struct RecordingClock { inner: ManualClock, waits: Mutex)>>, + blocked: Mutex>, + } + impl RecordingClock { + fn fire_next_timer(&self, limit: i64) -> bool { + let now = self.inner.now_ms(); + let next = self + .blocked + .lock() + .unwrap() + .iter() + .copied() + .filter(|at| (now + 1..=limit).contains(at)) + .min(); + next.map(|at| self.inner.set_ms(at)).is_some() + } } impl Clock for RecordingClock { fn now_ms(&self) -> i64 { @@ -616,7 +633,17 @@ fn spiron_supervisor_recovers_from_transient_mid_turn_server_error_via_recovery( } fn wait(&self, deadline: Option, woken: &Arc) -> bool { self.waits.lock().unwrap().push((self.now_ms(), deadline)); - self.inner.wait(deadline, woken) + if let Some(at) = deadline { + self.blocked.lock().unwrap().push(at); + } + let flagged = self.inner.wait(deadline, woken); + if let Some(at) = deadline { + let mut blocked = self.blocked.lock().unwrap(); + if let Some(index) = blocked.iter().position(|blocked| *blocked == at) { + blocked.swap_remove(index); + } + } + flagged } fn poke(&self) { self.inner.poke(); @@ -629,6 +656,7 @@ fn spiron_supervisor_recovers_from_transient_mid_turn_server_error_via_recovery( let recorded = Arc::new(RecordingClock { inner: clock.clone(), waits: Mutex::new(Vec::new()), + blocked: Mutex::new(Vec::new()), }); let _lock = crate::server::tests::env_lock(); @@ -739,12 +767,18 @@ for request_num, raw in enumerate(sys.stdin, 1): }; let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { eventually(|| read_wire().len() == 1); - clock.set_ms(START + 1_000); + // The pass that records the error may wait on the short timer the + // worker installs after the delivery: fire it once it is installed, + // never past the second the backoff window allows for. eventually(|| { - store + let errored = store .turn_errors(&agent.id) .unwrap() - .is_some_and(|errors| errors.last_error.as_deref() == Some(ERROR)) + .is_some_and(|errors| errors.last_error.as_deref() == Some(ERROR)); + if !errored { + recorded.fire_next_timer(START + 1_000); + } + errored }); eventually(|| { recorded.waits.lock().unwrap().iter().any(|(_, deadline)| { @@ -765,11 +799,13 @@ for request_num, raw in enumerate(sys.stdin, 1): deadline.filter(|at| (START + 15_000..=START + 16_000).contains(at)) }) .expect("the systemic follow-up must install its backoff"); + // Time stands just before the deadline when the mail arrives, so the + // wake the mail causes re-installs the backoff at that time. + clock.set_ms(deadline - 1); let observed = recorded.waits.lock().unwrap().len(); store .receive(&agent.id, Incoming::new("mail during backoff")) .unwrap(); - clock.set_ms(deadline - 1); eventually(|| { recorded.waits.lock().unwrap()[observed..].contains(&(deadline - 1, Some(deadline))) });