Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 21 additions & 4 deletions crates/anyagents-supervisor/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1694,18 +1694,23 @@ impl<H: Harness + 'static> Session<H> {
/// skips the delivery pass: a quota wait, a switch drain, or replacing
/// the process, alive or dead. A proof outlives the process that gave it
/// but not the adapter that holds it, so an uncommitted one would hand
/// the message to the next process again. Nothing is written.
fn commit_proven(&mut self) {
/// the message to the next process again. Nothing is written. `false`
/// when the store could not record a proof: the commit is tried again.
fn commit_proven(&mut self) -> bool {
let Some(delivery) = self.delivery.as_mut() else {
return;
return true;
};
match delivery.commit_proven() {
Ok(committed) => {
self.written = self
.written
.saturating_add(committed.clamp(0, u64::from(u32::MAX)) as u32);
true
}
Err(err) => {
self.last_error = Some(format!("commit proven mail: {err}"));
false
}
Err(err) => self.last_error = Some(format!("commit proven mail: {err}")),
}
}

Expand Down Expand Up @@ -2005,6 +2010,18 @@ impl<H: Harness + 'static> Session<H> {
let advanced = usage_advanced(&served, &before);
let ended = (served_turn || (advanced && health.process_alive))
&& health.turn_state == TurnState::Idle;
// A line the harness took and finished while `deliver` still answered
// `Deferred` is proven, not yet committed: the store holds no open
// turn for it. Commit that proof before the turn's custody is read or
// closed below, or an errored turn would be followed up as one
// nobody holds open, and a completed one left open behind its own
// proof. A store that cannot record it leaves the turn for another pass.
if ended && !self.commit_proven() {
return false;
}
let Some(delivery) = self.delivery.as_ref() else {
return true;
};
let completed = ended && health.sign_in_fault.is_none();
// A turn the harness saw end with an error (not a quota wait) never
// holds the mailbox: it is counted now, and `follow_errored_turn`
Expand Down
226 changes: 163 additions & 63 deletions crates/anyagents-supervisor/tests/errored_turns.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,27 +5,32 @@

mod support;

use std::sync::{Arc, Mutex, atomic::AtomicBool};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};

use anyagents_core::{Incoming, MAX_ERRORED_TURNS, NewAgent, SYSTEM_SENDER, Store};
use anyagents_harness::{
DeliverOutcome, Delivery, DeliveryLevel, Harness, HarnessError, HarnessHealth, OutboundMessage,
TurnState,
Proof, TurnState,
};
use anyagents_supervisor::{Clock, LaunchRequest, ManualClock, Supervisor, SystemClock};
use support::RecordingClock;

const OVERFLOW: &str = "Sylphx AI failure Some(\"context_length_exceeded\"): The request exceeds the model's context window.";

/// What a busy provider answers: the same request may succeed later.
const UNAVAILABLE: &str = "Sylphx AI failure Some(\"service_unavailable\"): The model response was interrupted. Retry the request.";

/// A vendor whose turns end at once: the next `fail` of them with `error`
/// (as Spiron reports a turn the model refused), the rest completed.
/// (as Spiron reports a turn the model refused), the rest completed. With
/// `deferred` each input is written, taken and finished before `deliver`
/// returns, yet answered `Deferred`: the vendor's acceptance and result reach
/// the adapter after the write, so the line is proven rather than reported.
#[derive(Default, Clone)]
struct Vendor {
fail: usize,
error: Option<&'static str>,
deferred: bool,
served: u32,
fault: Option<String>,
given: Vec<(String, String)>,
Expand Down Expand Up @@ -56,10 +61,22 @@ impl Harness for ErroringHarness {
} else {
None
};
Ok(DeliverOutcome::Accepted {
turn_finished: true,
Ok(if vendor.deferred {
DeliverOutcome::Deferred
} else {
DeliverOutcome::Accepted {
turn_finished: true,
}
})
}
fn proven(&self, message_id: &str) -> Option<Proof> {
let vendor = self.vendor.lock().unwrap();
(vendor.deferred && vendor.given.iter().any(|(id, _)| id == message_id)).then_some(
Proof::Taken {
turn_finished: true,
},
)
}
fn background(&mut self) -> Result<(), HarnessError> {
Ok(())
}
Expand Down Expand Up @@ -113,6 +130,17 @@ fn start(fail: usize) -> Fixture {
}

fn start_with(fail: usize, error: Option<&'static str>, clock: Arc<dyn Clock>) -> Fixture {
start_vendor(
Vendor {
fail,
error,
..Vendor::default()
},
clock,
)
}

fn start_vendor(vendor: Vendor, clock: Arc<dyn Clock>) -> Fixture {
let dir = tempfile::tempdir().unwrap();
let store = Arc::new(Store::open(dir.path().join("state.db")).unwrap());
let lead = store.create_agent(NewAgent::new("lead", "spiron")).unwrap();
Expand All @@ -129,11 +157,7 @@ fn start_with(fail: usize, error: Option<&'static str>, clock: Arc<dyn Clock>) -
store
.set_agent_running(&worker.id, &work.display().to_string())
.unwrap();
let vendor = Arc::new(Mutex::new(Vendor {
fail,
error,
..Vendor::default()
}));
let vendor = Arc::new(Mutex::new(vendor));
let supervisor = Supervisor::start(
Arc::clone(&store),
clock,
Expand Down Expand Up @@ -304,52 +328,35 @@ fn turns_that_keep_erroring_park_the_agent_and_tell_its_parent() {
support::stop(fx.supervisor);
}

// Record the actual installed deadline, rather than racing durable error
// persistence (which precedes installation of the in-memory follow-up).
struct RecordingClock {
inner: ManualClock,
waits: Mutex<Vec<(i64, Option<i64>)>>,
}

impl Clock for RecordingClock {
fn now_ms(&self) -> i64 {
self.inner.now_ms()
}
fn wait(&self, deadline: Option<i64>, woken: &Arc<AtomicBool>) -> bool {
self.waits.lock().unwrap().push((self.now_ms(), deadline));
self.inner.wait(deadline, woken)
}
fn poke(&self) {
self.inner.poke();
}
}

// This fixture proves terminal consumption using a new, nonduplicate served
// counter and Idle with no outstanding inputs. It does not infer a terminal
// result from an arbitrary provider error or authorise replay of partial work.
fn terminal_systemic_fault_keeps_backoff_and_queued_mail(error: &'static str) {
let clock = ManualClock::new(1_700_000_000_000);
let recorded = Arc::new(RecordingClock {
inner: clock.clone(),
waits: Mutex::new(Vec::new()),
});
const START: i64 = 1_700_000_000_000;
let clock = ManualClock::new(START);
let recorded = RecordingClock::new(clock.clone());
let fx = start_with(1, Some(error), recorded.clone());
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
fx.store
.receive(&fx.worker.id, Incoming::new("original task"))
.unwrap();
assert!(wait_until(|| fx.vendor.lock().unwrap().given.len() == 1));
clock.set_ms(clock.now_ms() + 1_000);
assert!(wait_until(|| fx
.store
.turn_errors(&fx.worker.id)
.unwrap()
.is_some_and(
|errors| errors.last_error.as_deref() == Some(error)
)));
// The pass that records the error runs on the short timer the worker
// installs after the delivery: fire it once it is installed, never
// earlier, and never past the second the backoff window allows for.
let errored = || {
fx.store
.turn_errors(&fx.worker.id)
.unwrap()
.is_some_and(|errors| errors.last_error.as_deref() == Some(error))
};
assert!(wait_until(|| errored() || {
recorded.fire_next_timer(START + 1_000);
false
}));
assert!(wait_until(|| {
recorded.waits.lock().unwrap().iter().any(|(_, deadline)| {
deadline.is_some_and(|at| (1_700_000_015_000..=1_700_000_016_000).contains(&at))
recorded.waits().iter().any(|(_, deadline)| {
deadline.is_some_and(|at| (START + 15_000..=START + 16_000).contains(&at))
}) || fx.vendor.lock().unwrap().attempts.len() > 1
}));
let (original_id, attempts) = {
Expand All @@ -358,23 +365,23 @@ fn terminal_systemic_fault_keeps_backoff_and_queued_mail(error: &'static str) {
};
assert_eq!(attempts.as_slice(), std::slice::from_ref(&original_id));
let deadline = recorded
.waits
.lock()
.unwrap()
.waits()
.iter()
.find_map(|(_, deadline)| {
deadline.filter(|at| (1_700_000_015_000..=1_700_000_016_000).contains(at))
deadline.filter(|at| (START + 15_000..=START + 16_000).contains(at))
})
.unwrap();
let observed = recorded.waits.lock().unwrap().len();
fx.store
.receive(&fx.worker.id, Incoming::new("mail during backoff"))
.unwrap();
// A mailbox wake must neither erase the terminal follow-up nor bypass its
// deadline. Process replacement must not turn it into an open-turn replay.
// Time stands just before the deadline when the mail arrives, so the
// wake the mail causes re-installs the backoff at that time.
clock.set_ms(deadline - 1);
let observed = recorded.waits().len();
fx.store
.receive(&fx.worker.id, Incoming::new("mail during backoff"))
.unwrap();
assert!(wait_until(
|| recorded.waits.lock().unwrap()[observed..].contains(&(deadline - 1, Some(deadline)))
|| recorded.waits()[observed..].contains(&(deadline - 1, Some(deadline)))
));
{
let vendor = fx.vendor.lock().unwrap().clone();
Expand Down Expand Up @@ -454,30 +461,123 @@ fn transient_text_cannot_discard_proven_terminal_backoff_and_mail() {
terminal_systemic_fault_keeps_backoff_and_queued_mail("HTTP 503: service unavailable");
}

fn received(store: &Store, agent: &anyagents_core::Agent, text: &str) -> anyagents_core::Message {
match store.receive(&agent.id, Incoming::new(text)).unwrap() {
anyagents_core::ReceiveOutcome::New(message) => message,
anyagents_core::ReceiveOutcome::Duplicate(_) => unreachable!(),
}
}

/// A vendor that has taken and finished a turn before `deliver` returns
/// answers `Deferred`: the line is proven, not reported. The pass that finds
/// the turn ended commits that proof first, so the turn is held open for its
/// follow-up (an error) or closed (a completion), and the line is never
/// handed over again.
#[test]
fn an_errored_turn_proven_while_deferred_stays_open_for_its_follow_up() {
const START: i64 = 1_700_000_000_000;
let clock = ManualClock::new(START);
let fx = start_vendor(
Vendor {
fail: 1,
error: Some(UNAVAILABLE),
deferred: true,
..Vendor::default()
},
Arc::new(clock.clone()) as Arc<dyn Clock>,
);
let task = received(&fx.store, &fx.worker, "summarise the incident");
assert!(wait_until(|| {
fx.store
.turn_errors(&fx.worker.id)
.unwrap()
.is_some_and(|errors| errors.last_error.as_deref() == Some(UNAVAILABLE))
}));
assert!(
fx.store.has_open_turn(&fx.worker.id).unwrap(),
"the errored turn is held open for its follow-up"
);
assert_eq!(
fx.store.delivered_cursor(&fx.worker.id).unwrap(),
task.seq,
"the proof is committed, not handed over again"
);
// The systemic backoff ends: the note goes out and closes the turn.
clock.set_ms(START + 15_000);
assert!(wait_until(|| fx.vendor.lock().unwrap().given.len() == 2));
assert!(wait_until(|| !fx
.store
.has_open_turn(&fx.worker.id)
.unwrap()));
let vendor = fx.vendor.lock().unwrap().clone();
assert!(
vendor.given[1].1.contains("[Recovery]") && vendor.given[1].1.contains(task.id.as_str()),
"{:?}",
vendor.given
);
assert_eq!(
vendor
.attempts
.iter()
.filter(|id| **id == vendor.given[0].0)
.count(),
1,
"the proven task is never offered again: {:?}",
vendor.attempts
);
support::stop(fx.supervisor);
}

#[test]
fn a_completed_turn_proven_while_deferred_is_closed() {
let fx = start_vendor(
Vendor {
deferred: true,
..Vendor::default()
},
Arc::new(SystemClock::new()),
);
let task = received(&fx.store, &fx.worker, "summarise the incident");
assert!(wait_until(|| fx
.store
.delivered_cursor(&fx.worker.id)
.unwrap()
>= task.seq));
assert!(
wait_until(|| !fx.store.has_open_turn(&fx.worker.id).unwrap()),
"a turn the vendor finished is not left open"
);
assert_eq!(fx.vendor.lock().unwrap().attempts.len(), 1);
support::stop(fx.supervisor);
}

/// A provider that is unavailable is the fleet's trouble, not the agent's:
/// each retry waits out a growing backoff, the turn stays open (a restart
/// would recover it) while it waits, and no run of them ever parks it.
#[test]
fn systemic_errors_back_off_and_never_park() {
let clock = ManualClock::new(1_700_000_000_000);
let fx = start_with(
usize::MAX,
Some(UNAVAILABLE),
Arc::new(clock.clone()) as Arc<dyn Clock>,
);
const START: i64 = 1_700_000_000_000;
let clock = ManualClock::new(START);
let recorded = RecordingClock::new(clock.clone());
let fx = start_with(usize::MAX, Some(UNAVAILABLE), recorded.clone());
let given = || fx.vendor.lock().unwrap().given.len();
let advance = |ms: i64| clock.set_ms(clock.now_ms() + ms);
fx.store
.receive(&fx.worker.id, Incoming::new("summarise the incident"))
.unwrap();
assert!(wait_until(|| given() == 1));
// The next pass finds the errored turn and starts its backoff.
advance(1_000);
assert!(wait_until(|| {
// The next pass finds the errored turn and starts its backoff. It runs on
// the short timer the worker installs after the delivery: fire it once it
// is installed, as time moved earlier is read by the worker as its own.
let errored = || {
fx.store
.turn_errors(&fx.worker.id)
.unwrap()
.is_some_and(|errors| errors.last_error.as_deref() == Some(UNAVAILABLE))
};
assert!(wait_until(|| errored() || {
recorded.fire_next_timer(START + 1_000);
false
}));
advance(10_000);
std::thread::sleep(Duration::from_millis(200));
Expand Down
Loading
Loading