diff --git a/src/controller/core/admission.rs b/src/controller/core/admission.rs index fbc60d1..c8b90e9 100644 --- a/src/controller/core/admission.rs +++ b/src/controller/core/admission.rs @@ -11,7 +11,7 @@ use crate::{ admission::AdmissionPolicy, slot::{AdmissionTransition, ReplaceAction, SlotPhase, SlotState}, }, - events::{Event, EventKind}, + events::{Event, EventKind, RejectionKind}, identity::TaskId, reasons, }; @@ -48,9 +48,14 @@ impl Controller { Event::new(EventKind::ControllerRejected) .with_task(spec.slot_name().to_owned()) .with_id(id) + .with_rejection_kind(RejectionKind::ControllerShuttingDown) .with_reason(crate::reasons::CONTROLLER_SHUTTING_DOWN), ); - self.finalize_rejected(id, crate::reasons::CONTROLLER_SHUTTING_DOWN); + self.finalize_rejected( + id, + RejectionKind::ControllerShuttingDown, + crate::reasons::CONTROLLER_SHUTTING_DOWN, + ); return; } @@ -65,9 +70,14 @@ impl Controller { Event::new(EventKind::ControllerRejected) .with_task(Arc::clone(&slot_name)) .with_id(id) + .with_rejection_kind(RejectionKind::ControllerShuttingDown) .with_reason(crate::reasons::CONTROLLER_SHUTTING_DOWN), ); - self.finalize_rejected(id, crate::reasons::CONTROLLER_SHUTTING_DOWN); + self.finalize_rejected( + id, + RejectionKind::ControllerShuttingDown, + crate::reasons::CONTROLLER_SHUTTING_DOWN, + ); return; } @@ -95,9 +105,10 @@ impl Controller { Event::new(EventKind::ControllerRejected) .with_task(Arc::clone(&slot_name)) .with_id(id) + .with_rejection_kind(RejectionKind::AdmissionFailed) .with_reason(reason.clone()), ); - self.finalize_rejected(id, &reason); + self.finalize_rejected(id, RejectionKind::AdmissionFailed, &reason); self.gc_if_idle(&slot_name, slot); } } @@ -185,9 +196,10 @@ impl Controller { Event::new(EventKind::ControllerRejected) .with_task(Arc::clone(&slot_name)) .with_id(id) + .with_rejection_kind(RejectionKind::SlotBusy) .with_reason(reason.clone()), ); - self.finalize_rejected(id, &reason); + self.finalize_rejected(id, RejectionKind::SlotBusy, &reason); } } } @@ -342,9 +354,10 @@ impl Controller { Event::new(EventKind::ControllerRejected) .with_task(Arc::clone(slot_name)) .with_id(next_id) + .with_rejection_kind(RejectionKind::AdmissionFailed) .with_reason(reason.clone()), ); - self.finalize_rejected(next_id, &reason); + self.finalize_rejected(next_id, RejectionKind::AdmissionFailed, &reason); } } } diff --git a/src/controller/core/identity.rs b/src/controller/core/identity.rs index d8f9278..24b471c 100644 --- a/src/controller/core/identity.rs +++ b/src/controller/core/identity.rs @@ -6,7 +6,7 @@ use tokio::{sync::oneshot, task::JoinSet}; use crate::{ RuntimeError, - events::{Event, EventKind}, + events::{Event, EventKind, RejectionKind}, identity::TaskId, }; @@ -108,9 +108,14 @@ impl Controller { Event::new(EventKind::ControllerRejected) .with_task(Arc::clone(&slot_name)) .with_id(id) + .with_rejection_kind(RejectionKind::RemovedFromQueue) .with_reason(crate::reasons::REMOVED_FROM_QUEUE), ); - self.finalize_rejected(id, crate::reasons::REMOVED_FROM_QUEUE); + self.finalize_rejected( + id, + RejectionKind::RemovedFromQueue, + crate::reasons::REMOVED_FROM_QUEUE, + ); self.gc_if_idle(&slot_name, slot); return true; } diff --git a/src/controller/core/lifecycle.rs b/src/controller/core/lifecycle.rs index c022cc7..49c3b64 100644 --- a/src/controller/core/lifecycle.rs +++ b/src/controller/core/lifecycle.rs @@ -5,10 +5,7 @@ use std::{future::Future, sync::Arc}; use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; -use crate::{ - controller::error::ControllerError, - events::{Event, EventKind}, -}; +use crate::{controller::error::ControllerError, events::Event}; use super::{Controller, ControllerCommand, ControllerTask}; @@ -31,18 +28,16 @@ impl Controller { match crate::core::panic_guard::guarded(self.run_inner(token)).await { Ok(Ok(())) => {} Ok(Err(error)) => { - self.bus.publish( - Event::new(EventKind::ControllerRejected) - .with_task("controller") - .with_reason(format!("controller_loop_exited: {error}")), - ); + self.bus.publish(Event::runtime_failure( + "controller", + format!("controller_loop_exited: {error}"), + )); } Err(panic) => { - self.bus.publish( - Event::new(EventKind::ControllerRejected) - .with_task("controller") - .with_reason(format!("controller_loop_panicked: {panic}")), - ); + self.bus.publish(Event::runtime_failure( + "controller", + format!("controller_loop_panicked: {panic}"), + )); } } @@ -117,9 +112,10 @@ impl Controller { } Some(Err(error)) => { self.bus.publish( - Event::new(EventKind::ControllerRejected) - .with_task("controller") - .with_reason(format!("admission_waiter_failed: {error}")), + Event::runtime_failure( + "controller", + format!("admission_waiter_failed: {error}"), + ), ); } None => {} @@ -137,9 +133,10 @@ impl Controller { } Some(Err(error)) => { self.bus.publish( - Event::new(EventKind::ControllerRejected) - .with_task("controller") - .with_reason(format!("completion_waiter_failed: {error}")), + Event::runtime_failure( + "controller", + format!("completion_waiter_failed: {error}"), + ), ); } None => {} @@ -157,9 +154,10 @@ impl Controller { } Some(Err(error)) => { self.bus.publish( - Event::new(EventKind::ControllerRejected) - .with_task("controller") - .with_reason(format!("removal_waiter_failed: {error}")), + Event::runtime_failure( + "controller", + format!("removal_waiter_failed: {error}"), + ), ); } None => {} @@ -170,9 +168,10 @@ impl Controller { Some(Ok(())) => {} Some(Err(error)) => { self.bus.publish( - Event::new(EventKind::ControllerRejected) - .with_task("controller") - .with_reason(format!("identity_operation_failed: {error}")), + Event::runtime_failure( + "controller", + format!("identity_operation_failed: {error}"), + ), ); } None => {} @@ -222,18 +221,17 @@ impl Controller { Self::drain_workers(&mut identity_operations).await; self.finalize_slot_state_on_shutdown().await; if let Err(panic) = loop_result { - self.bus.publish( - Event::new(EventKind::ControllerRejected) - .with_task("controller") - .with_reason(format!("controller_loop_panicked: {panic}")), - ); + self.bus.publish(Event::runtime_failure( + "controller", + format!("controller_loop_panicked: {panic}"), + )); } Ok(()) } /// Runs one controller work unit behind a panic boundary. /// - /// A panic is converted into a diagnostic `ControllerRejected` event and the loop continues. + /// A panic is converted into a diagnostic `RuntimeFailure` event and the loop continues. /// /// This guard does not repair partially updated slot state by itself. /// Callers that park watcher state must still make sure the watcher is resolved or returned on every failure path. @@ -246,11 +244,10 @@ impl Controller { match crate::core::panic_guard::guarded(fut).await { Ok(output) => Some(output), Err(msg) => { - self.bus.publish( - Event::new(EventKind::ControllerRejected) - .with_task("controller") - .with_reason(format!("{who}_panicked: {msg}")), - ); + self.bus.publish(Event::runtime_failure( + "controller", + format!("{who}_panicked: {msg}"), + )); None } } diff --git a/src/controller/core/mod.rs b/src/controller/core/mod.rs index 807254c..7fe4500 100644 --- a/src/controller/core/mod.rs +++ b/src/controller/core/mod.rs @@ -53,7 +53,7 @@ use tokio_util::sync::CancellationToken; use crate::{ core::{OutcomeTx, SupervisorCore, TaskOutcome}, - events::{Bus, Event, EventKind}, + events::{Bus, Event, EventKind, RejectionKind}, identity::TaskId, }; @@ -152,9 +152,10 @@ impl Controller { /// Resolves a parked watched submission as `Rejected`. /// /// This is a no-op for unwatched submissions and for watched submissions already handed to the runtime registry. - fn finalize_rejected(&self, id: TaskId, reason: &str) { + fn finalize_rejected(&self, id: TaskId, kind: RejectionKind, reason: &str) { if let Some((_, tx)) = self.watchers.remove(&id) { let _ = tx.send(TaskOutcome::Rejected { + kind, reason: Arc::from(reason), }); } @@ -184,9 +185,14 @@ impl Controller { self.bus.publish( Event::new(EventKind::ControllerRejected) .with_id(id) + .with_rejection_kind(RejectionKind::ControllerShuttingDown) .with_reason(crate::reasons::CONTROLLER_SHUTTING_DOWN), ); - self.finalize_rejected(id, crate::reasons::CONTROLLER_SHUTTING_DOWN); + self.finalize_rejected( + id, + RejectionKind::ControllerShuttingDown, + crate::reasons::CONTROLLER_SHUTTING_DOWN, + ); } } diff --git a/src/controller/core/queue.rs b/src/controller/core/queue.rs index 4a582d7..ee1d1f0 100644 --- a/src/controller/core/queue.rs +++ b/src/controller/core/queue.rs @@ -6,7 +6,7 @@ use tokio::sync::Mutex; use crate::{ controller::slot::SlotState, - events::{Event, EventKind}, + events::{Event, EventKind, RejectionKind}, identity::TaskId, }; @@ -57,9 +57,10 @@ impl Controller { Event::new(EventKind::ControllerRejected) .with_task(slot_name) .with_id(id) + .with_rejection_kind(RejectionKind::QueueFull) .with_reason(reason.clone()), ); - self.finalize_rejected(id, &reason); + self.finalize_rejected(id, RejectionKind::QueueFull, &reason); true } else { false @@ -86,9 +87,14 @@ impl Controller { Event::new(EventKind::ControllerRejected) .with_task(Arc::clone(slot_name)) .with_id(displaced_id) + .with_rejection_kind(RejectionKind::SupersededByReplace) .with_reason(crate::reasons::SUPERSEDED_BY_REPLACE), ); - self.finalize_rejected(displaced_id, crate::reasons::SUPERSEDED_BY_REPLACE); + self.finalize_rejected( + displaced_id, + RejectionKind::SupersededByReplace, + crate::reasons::SUPERSEDED_BY_REPLACE, + ); } else { slot.queue.push_front((id, task_spec)); } diff --git a/src/controller/core/shutdown.rs b/src/controller/core/shutdown.rs index eeda149..c0a44d5 100644 --- a/src/controller/core/shutdown.rs +++ b/src/controller/core/shutdown.rs @@ -20,7 +20,7 @@ use tokio::{sync::mpsc, task::JoinSet}; use crate::RuntimeError; use crate::core::TaskOutcome; -use crate::events::{Event, EventKind}; +use crate::events::{Event, EventKind, RejectionKind}; use super::{Controller, ControllerCommand}; @@ -39,6 +39,7 @@ impl Controller { ControllerCommand::Submit(sub) => { let mut event = Event::new(EventKind::ControllerRejected) .with_id(sub.id) + .with_rejection_kind(RejectionKind::ControllerShuttingDown) .with_reason(crate::reasons::CONTROLLER_SHUTTING_DOWN); if let Some(slot_name) = sub.spec.slot_override() { event = event.with_task(slot_name.to_owned()); @@ -47,6 +48,7 @@ impl Controller { if let Some(done) = sub.done { let _ = done.send(TaskOutcome::Rejected { + kind: RejectionKind::ControllerShuttingDown, reason: Arc::from(crate::reasons::CONTROLLER_SHUTTING_DOWN), }); } @@ -76,9 +78,14 @@ impl Controller { Event::new(EventKind::ControllerRejected) .with_task(Arc::clone(&slot_name)) .with_id(id) + .with_rejection_kind(RejectionKind::ControllerShuttingDown) .with_reason(crate::reasons::CONTROLLER_SHUTTING_DOWN), ); - self.finalize_rejected(id, crate::reasons::CONTROLLER_SHUTTING_DOWN); + self.finalize_rejected( + id, + RejectionKind::ControllerShuttingDown, + crate::reasons::CONTROLLER_SHUTTING_DOWN, + ); } } diff --git a/src/controller/core/task.rs b/src/controller/core/task.rs index e22532f..b896b6e 100644 --- a/src/controller/core/task.rs +++ b/src/controller/core/task.rs @@ -2,7 +2,7 @@ use tokio::{sync::Mutex, task::JoinHandle}; -use crate::events::{Bus, Event, EventKind}; +use crate::events::{Bus, Event}; pub(super) struct ControllerTask { state: Mutex, @@ -36,11 +36,10 @@ impl ControllerTask { let clean = match handle.await { Ok(()) => true, Err(error) => { - bus.publish( - Event::new(EventKind::ControllerRejected) - .with_task("controller") - .with_reason(format!("controller_join_failed: {error}")), - ); + bus.publish(Event::runtime_failure( + "controller", + format!("controller_join_failed: {error}"), + )); false } }; diff --git a/src/controller/core/tests.rs b/src/controller/core/tests.rs index e729166..6c810d1 100644 --- a/src/controller/core/tests.rs +++ b/src/controller/core/tests.rs @@ -83,6 +83,10 @@ fn replace_head_or_push_replaces_existing_head_and_rejects_displaced() { let ev = rx.try_recv().expect("displaced head must be rejected"); assert_eq!(ev.kind, EventKind::ControllerRejected); + assert_eq!( + ev.rejection_kind, + Some(crate::RejectionKind::SupersededByReplace) + ); assert_eq!(ev.id, Some(displaced)); assert_eq!( ev.reason.as_deref(), @@ -249,11 +253,11 @@ async fn removal_error_preserves_owner_and_queue_and_emits_one_diagnostic() { let event = events .try_recv() .expect("the current owner's removal error must be observable"); - assert_eq!(event.kind, EventKind::ControllerRejected); + assert_eq!(event.kind, EventKind::RuntimeFailure); assert_eq!(event.id, Some(owner)); - assert_eq!(event.task.as_deref(), Some("s")); + assert_eq!(event.task.as_deref(), Some("controller")); assert!(event.reason.as_deref().is_some_and(|reason| { - reason.starts_with("remove_failed:") && reason.contains("queue is full") + reason.starts_with("remove_failed slot=s:") && reason.contains("queue is full") })); assert!( events.try_recv().is_err(), @@ -484,7 +488,11 @@ async fn repeated_replace_while_admitting_is_latest_wins_with_one_removal_after_ assert!(matches!( first_outcome.await, - Ok(TaskOutcome::Rejected { reason }) + Ok(TaskOutcome::Rejected { + kind: crate::RejectionKind::SupersededByReplace, + reason, + .. + }) if reason.as_ref() == crate::reasons::SUPERSEDED_BY_REPLACE )); @@ -607,7 +615,11 @@ async fn shutdown_rejects_slot_queue_and_clears_controller_state() { assert!(matches!( outcome.await, - Ok(TaskOutcome::Rejected { reason }) + Ok(TaskOutcome::Rejected { + kind: crate::RejectionKind::ControllerShuttingDown, + reason, + .. + }) if reason.as_ref() == crate::reasons::CONTROLLER_SHUTTING_DOWN )); assert!(ctrl.watchers.is_empty()); @@ -748,7 +760,7 @@ async fn guarded_converts_panic_to_diagnostic_and_survives() { let ev = rx .try_recv() .expect("a panicking work-unit must publish a diagnostic"); - assert_eq!(ev.kind, EventKind::ControllerRejected); + assert_eq!(ev.kind, EventKind::RuntimeFailure); assert!( ev.reason.as_deref().unwrap_or_default().contains("boom 1"), "diagnostic must carry the panic message, got {:?}", @@ -914,7 +926,11 @@ async fn public_shutdown_waits_for_controller_join_and_survives_a_dropped_waiter .expect("the buffered watched command must resolve before shutdown returns"); assert!(matches!( queued_outcome, - TaskOutcome::Rejected { reason } + TaskOutcome::Rejected { + kind: crate::RejectionKind::ControllerShuttingDown, + reason, + .. + } if reason.as_ref() == crate::reasons::CONTROLLER_SHUTTING_DOWN )); let panicking_outcome = @@ -924,7 +940,11 @@ async fn public_shutdown_waits_for_controller_join_and_survives_a_dropped_waiter .expect("the hostile buffered watcher must resolve as an outcome"); assert!(matches!( panicking_outcome, - TaskOutcome::Rejected { reason } + TaskOutcome::Rejected { + kind: crate::RejectionKind::ControllerShuttingDown, + reason, + .. + } if reason.as_ref() == crate::reasons::CONTROLLER_SHUTTING_DOWN )); assert!(identity.is_finished()); @@ -1519,7 +1539,11 @@ async fn replace_stays_responsive_under_registry_backpressure() { ); assert!(matches!( first_outcome.await, - Ok(TaskOutcome::Rejected { reason }) + Ok(TaskOutcome::Rejected { + kind: crate::RejectionKind::SupersededByReplace, + reason, + .. + }) if reason.as_ref() == crate::reasons::SUPERSEDED_BY_REPLACE )); @@ -1586,9 +1610,11 @@ async fn queued_cancel_is_ordered_without_runtime_bus_events() { "the first cancellation caller must claim the queued submission" ); let outcome = waiter.wait().await.expect("the queued waiter must resolve"); - assert!( - matches!(outcome, TaskOutcome::Rejected { reason } if reason.as_ref() == crate::reasons::REMOVED_FROM_QUEUE) - ); + assert!(matches!(outcome, TaskOutcome::Rejected { + kind: crate::RejectionKind::RemovedFromQueue, + reason, + .. + } if reason.as_ref() == crate::reasons::REMOVED_FROM_QUEUE)); let try_ran = Arc::clone(&victim_ran); let try_victim: TaskRef = TaskFn::arc("try-remove-victim", move |_ctx: TaskContext| { @@ -1613,9 +1639,11 @@ async fn queued_cancel_is_ordered_without_runtime_bus_events() { .wait() .await .expect("the try_remove waiter must resolve"); - assert!( - matches!(try_outcome, TaskOutcome::Rejected { reason } if reason.as_ref() == crate::reasons::REMOVED_FROM_QUEUE) - ); + assert!(matches!(try_outcome, TaskOutcome::Rejected { + kind: crate::RejectionKind::RemovedFromQueue, + reason, + .. + } if reason.as_ref() == crate::reasons::REMOVED_FROM_QUEUE)); let try_cancel_ran = Arc::clone(&victim_ran); let try_cancel_victim: TaskRef = TaskFn::arc("try-cancel-victim", move |_ctx: TaskContext| { @@ -1640,9 +1668,11 @@ async fn queued_cancel_is_ordered_without_runtime_bus_events() { .wait() .await .expect("the try_cancel waiter must resolve"); - assert!( - matches!(try_cancel_outcome, TaskOutcome::Rejected { reason } if reason.as_ref() == crate::reasons::REMOVED_FROM_QUEUE) - ); + assert!(matches!(try_cancel_outcome, TaskOutcome::Rejected { + kind: crate::RejectionKind::RemovedFromQueue, + reason, + .. + } if reason.as_ref() == crate::reasons::REMOVED_FROM_QUEUE)); assert!( handle @@ -1783,9 +1813,11 @@ async fn duplicate_reply_frees_slot_without_task_add_failed() { .await .expect("registry rejection must resolve the watcher") .expect("registry must send a rejected outcome"); - assert!( - matches!(outcome, TaskOutcome::Rejected { reason } if reason.as_ref() == crate::reasons::ALREADY_EXISTS) - ); + assert!(matches!(outcome, TaskOutcome::Rejected { + kind: crate::RejectionKind::AlreadyExists, + reason, + .. + } if reason.as_ref() == crate::reasons::ALREADY_EXISTS)); assert!( poll_until(Duration::from_secs(2), || async { ctrl.slots.get("s").is_none() && !ctrl.watchers.contains_key(&id) @@ -1845,9 +1877,11 @@ async fn queued_admission_skips_registry_rejected_head() { let duplicate_outcome = duplicate_outcome .await .expect("registry must resolve the duplicate watcher"); - assert!( - matches!(duplicate_outcome, TaskOutcome::Rejected { reason } if reason.as_ref() == crate::reasons::ALREADY_EXISTS) - ); + assert!(matches!(duplicate_outcome, TaskOutcome::Rejected { + kind: crate::RejectionKind::AlreadyExists, + reason, + .. + } if reason.as_ref() == crate::reasons::ALREADY_EXISTS)); let slot = slot_arc.lock().await; assert_eq!(slot.owner_id(), Some(accepted_id)); assert!(matches!(slot.phase(), SlotPhase::Running { .. })); diff --git a/src/controller/core/workers.rs b/src/controller/core/workers.rs index f7016f0..c9d267f 100644 --- a/src/controller/core/workers.rs +++ b/src/controller/core/workers.rs @@ -7,7 +7,7 @@ use tokio::task::JoinSet; use crate::{ RuntimeError, core::{AddReplyRx, RemovalCompletion, SupervisorCore}, - events::{Event, EventKind}, + events::Event, identity::TaskId, }; @@ -85,10 +85,11 @@ impl Controller { } if let Err(error) = result.decision { self.bus.publish( - Event::new(EventKind::ControllerRejected) - .with_task(result.slot_name) - .with_id(result.id) - .with_reason(format!("remove_failed: {error}")), + Event::runtime_failure( + "controller", + format!("remove_failed slot={}: {error}", result.slot_name), + ) + .with_id(result.id), ); } } diff --git a/src/core/handle.rs b/src/core/handle.rs index dc1324f..7e5b753 100644 --- a/src/core/handle.rs +++ b/src/core/handle.rs @@ -15,7 +15,24 @@ //! With a controller, identity-based remove and cancel operations are ordered after earlier submissions. //! This lets them find work that is still queued and has not reached the registry. //! -//! [`SupervisorCore`]: crate::core::SupervisorCore +//! ## Confirmation Boundaries +//! +//! | Method family | A successful return confirms | It does not confirm | +//! |--------------------------------------------|-------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------| +//! | `add`, `try_add` | Registry admission and runner creation | First attempt started | +//! | `add_and_watch`, `try_add_and_watch` | Registry admission; the returned waiter later confirms terminal cleanup | Task success at method return | +//! | `submit`, `try_submit` | Controller command-queue acceptance | Slot admission, registry admission, or task start | +//! | `submit_and_watch`, `try_submit_and_watch` | Controller command-queue acceptance; the returned waiter later confirms rejection or terminal cleanup | Slot or registry admission at method return | +//! | `remove`, `try_remove` | The stop claim was decided; queued controller work is removed before return | Terminal cleanup of registered work | +//! | `remove_by_label`, `try_remove_by_label` | The registry name lookup and stop claim were decided | Terminal cleanup | +//! | `cancel`, `try_cancel` | Terminal cleanup for known registered work; queued controller removal is complete | That this caller was the first stop claimant when the result is `false` | +//! | `cancel_by_label*`, `try_cancel_by_label*` | Terminal cleanup when the call returns `Ok`; timeout variants bound only the cleanup wait | Cleanup after `TaskTerminationTimeout` | +//! | `list` | One authoritative registry snapshot | That entries are currently executing an attempt | +//! | `alive_snapshot`, `is_alive` | One best-effort event-derived cache read | Authoritative registry membership | +//! | `controller_snapshot` | One best-effort rolling slot snapshot | An atomic view across every slot | +//! | `shutdown` | Shared runtime cleanup completed with the returned result | - | +//! +//! A `try_*` method changes queue admission only: it fails immediately when the relevant bounded queue is full. use std::{sync::Arc, time::Duration}; diff --git a/src/core/outcome.rs b/src/core/outcome.rs index e164db5..26ae6d9 100644 --- a/src/core/outcome.rs +++ b/src/core/outcome.rs @@ -43,6 +43,7 @@ use std::sync::Arc; use tokio::sync::oneshot; use crate::error::{RuntimeError, SharedError}; +use crate::events::RejectionKind; use crate::identity::TaskId; /// Final result of one watched task or controller submission. @@ -143,7 +144,9 @@ pub enum TaskOutcome { /// - registration failed because the task name already existed. #[non_exhaustive] Rejected { - /// Why the submission was rejected. + /// Stable category for machine-readable handling. + kind: RejectionKind, + /// Readable rejection details. reason: Arc, }, } @@ -200,14 +203,18 @@ impl TaskOutcome { /// ```rust /// use taskvisor::TaskOutcome; /// - /// let outcome = TaskOutcome::rejected_for_tests("queue_full"); + /// let outcome = TaskOutcome::rejected_for_tests( + /// taskvisor::RejectionKind::QueueFull, + /// "queue_full", + /// ); /// assert_eq!(outcome.as_label(), "outcome_rejected"); /// ``` #[cfg(feature = "test-util")] #[cfg_attr(docsrs, doc(cfg(feature = "test-util")))] #[must_use] - pub fn rejected_for_tests(reason: impl Into>) -> Self { + pub fn rejected_for_tests(kind: RejectionKind, reason: impl Into>) -> Self { Self::Rejected { + kind, reason: reason.into(), } } @@ -335,10 +342,11 @@ mod tests { TaskOutcome::Fatal { reason, exit_code: None, .. } if reason.as_ref() == "bad config" )); - let rejected = TaskOutcome::rejected_for_tests("queue_full"); + let rejected = TaskOutcome::rejected_for_tests(RejectionKind::QueueFull, "queue_full"); assert!(matches!( &rejected, - TaskOutcome::Rejected { reason, .. } if reason.as_ref() == "queue_full" + TaskOutcome::Rejected { kind: RejectionKind::QueueFull, reason, .. } + if reason.as_ref() == "queue_full" )); assert!(rejected.source().is_none()); } @@ -370,6 +378,7 @@ mod tests { (TaskOutcome::Panicked, "outcome_panicked", false), ( TaskOutcome::Rejected { + kind: RejectionKind::AdmissionFailed, reason: Arc::from("x"), }, "outcome_rejected", diff --git a/src/core/registry/admission.rs b/src/core/registry/admission.rs index 722b5c9..1f454c1 100644 --- a/src/core/registry/admission.rs +++ b/src/core/registry/admission.rs @@ -17,7 +17,7 @@ use crate::{ core::actor::{ActorExitReason, TaskActor, TaskActorParams}, core::outcome::TaskOutcome, error::RuntimeError, - events::{Event, EventKind}, + events::{Event, EventKind, RejectionKind}, identity::TaskId, reasons, tasks::TaskSpec, @@ -138,10 +138,16 @@ impl Registry { } else { reasons::BATCH_REJECTED }; + let rejection_kind = if conflicting_ids.contains(&item.id) { + RejectionKind::AlreadyExists + } else { + RejectionKind::BatchRejected + }; self.bus.publish( Event::new(EventKind::TaskAddFailed) .with_task(item.label) .with_id(item.id) + .with_rejection_kind(rejection_kind) .with_reason(reason), ); } @@ -203,6 +209,7 @@ impl Registry { })); if let Some(done) = done { let _ = done.send(TaskOutcome::Rejected { + kind: RejectionKind::AlreadyExists, reason: Arc::from(reasons::ALREADY_EXISTS), }); } @@ -210,6 +217,7 @@ impl Registry { Event::new(EventKind::TaskAddFailed) .with_task(label) .with_id(id) + .with_rejection_kind(RejectionKind::AlreadyExists) .with_reason(reasons::ALREADY_EXISTS), ); return; diff --git a/src/core/registry/tests.rs b/src/core/registry/tests.rs index 7ac1424..5ff5989 100644 --- a/src/core/registry/tests.rs +++ b/src/core/registry/tests.rs @@ -664,7 +664,7 @@ async fn duplicate_add_reply_rejects_without_starting_body() { assert_eq!(runs.load(Ordering::SeqCst), 0, "rejected body must not run"); assert!(matches!( receive_reply(outcome_rx, "duplicate outcome").await, - TaskOutcome::Rejected { reason } if reason.as_ref() == reasons::ALREADY_EXISTS + TaskOutcome::Rejected { reason, .. } if reason.as_ref() == reasons::ALREADY_EXISTS )); stop_registry(®istry, &token).await; diff --git a/src/core/runtime/tests.rs b/src/core/runtime/tests.rs index 22956d3..55c2ee9 100644 --- a/src/core/runtime/tests.rs +++ b/src/core/runtime/tests.rs @@ -1203,6 +1203,7 @@ async fn bounded_command_queue_reports_full_and_recovers_capacity() { Err((RuntimeError::CommandQueueFull, Some(returned))) => { returned .send(TaskOutcome::Rejected { + kind: crate::RejectionKind::AdmissionFailed, reason: Arc::from("command_queue_full"), }) .expect("the full command must return its outcome sender"); @@ -1211,7 +1212,7 @@ async fn bounded_command_queue_reports_full_and_recovers_capacity() { } assert!(matches!( outcome_rx.await, - Ok(TaskOutcome::Rejected { reason }) if reason.as_ref() == "command_queue_full" + Ok(TaskOutcome::Rejected { reason, .. }) if reason.as_ref() == "command_queue_full" )); assert_eq!(runs.load(Ordering::SeqCst), 0); assert!(!core.contains_id(rejected_id).await); @@ -1346,6 +1347,7 @@ async fn closed_command_queue_returns_shutting_down_and_watcher() { Err((RuntimeError::ShuttingDown, Some(returned))) => { returned .send(TaskOutcome::Rejected { + kind: crate::RejectionKind::ControllerShuttingDown, reason: Arc::from("shutting_down"), }) .expect("closed queue must return its outcome sender"); @@ -1354,7 +1356,7 @@ async fn closed_command_queue_returns_shutting_down_and_watcher() { } assert!(matches!( outcome_rx.await, - Ok(TaskOutcome::Rejected { reason }) if reason.as_ref() == "shutting_down" + Ok(TaskOutcome::Rejected { reason, .. }) if reason.as_ref() == "shutting_down" )); assert_eq!(runs.load(Ordering::SeqCst), 0); @@ -1377,6 +1379,7 @@ async fn add_task_with_id_watched_returns_watcher_on_failure() { Err((RuntimeError::ShuttingDown, Some(returned))) => { returned .send(crate::TaskOutcome::Rejected { + kind: crate::RejectionKind::AdmissionFailed, reason: Arc::from("rejected"), }) .expect("returned watcher must still be live"); diff --git a/src/events/event.rs b/src/events/event.rs index 69ac297..1d25006 100644 --- a/src/events/event.rs +++ b/src/events/event.rs @@ -9,6 +9,7 @@ //! | [`EventKind`] | Event classification | //! | [`Event`] | Event payload and metadata | //! | [`BackoffSource`] | Why a `BackoffScheduled` event was emitted | +//! | [`RejectionKind`] | Machine-readable submission rejection | //! //! ## Sequence numbers //! @@ -31,11 +32,13 @@ //! - `id`: the stable [`TaskId`] for one submission and run. //! - `attempt`: task attempt number, starting from 1. //! - `task`: usually a task name. Subscriber diagnostics use it for the subscriber name, and controller events use it for the slot name. +//! - `rejection_kind`: machine-readable category for a rejected add or controller submission. //! //! `timeout_ms`, `delay_ms`, and `duration_ms` use whole milliseconds. //! Values above `u32::MAX` milliseconds are stored as `u32::MAX`. //! -//! Treat `reason` a readable text unless the event points to a constant in [`reasons`](crate::reasons). +//! Treat `reason` as readable text unless the event points to a constant in [`reasons`](crate::reasons). +//! Use [`RejectionKind`] instead of parsing rejection text. //! > Use [`EventKind::as_label`] for a stable event label. //! //! ## Example @@ -226,6 +229,7 @@ pub enum EventKind { /// Sets: /// - `id`: task run identity of the rejected add request /// - `task`: task name + /// - `rejection_kind`: [`RejectionKind::AlreadyExists`] or [`RejectionKind::BatchRejected`] /// - `reason`: e.g. "already_exists" or "batch_rejected" /// - `at`: wall-clock timestamp /// - `seq`: process-local sequence @@ -289,16 +293,13 @@ pub enum EventKind { #[cfg(feature = "controller")] #[cfg_attr(docsrs, doc(cfg(feature = "controller")))] - /// The controller rejected a submission or could not complete admission. + /// The controller rejected a submission. /// /// Sets: - /// - `task`: slot name when known, or `controller` for loop-level diagnostics; may be absent - /// when shutdown rejects a buffered submission whose slot defaults to user task metadata - /// - `id`: the rejected submission's [`TaskId`], when the rejection concerns a specific submission. - /// Absent for slot- or loop-level diagnostics that have no submission behind - /// them (e.g. a failed deferred removal or the controller loop exiting). - /// - `reason`: rejection reason. Values listed in [`reasons`](crate::reasons) - /// are stable; other text is diagnostic. + /// - `task`: slot name, when known + /// - `id`: the rejected submission's [`TaskId`] + /// - `rejection_kind`: stable machine-readable rejection category + /// - `reason`: readable rejection details ControllerRejected, #[cfg(feature = "controller")] @@ -407,6 +408,55 @@ impl BackoffSource { } } +/// Reason why a task or controller submission did not start. +/// +/// [`Event::reason`] and the `reason` field on [`TaskOutcome::Rejected`](crate::TaskOutcome::Rejected) retain readable details. +/// Use this enum for branching, metrics, and state transitions. +/// +/// ```rust +/// use taskvisor::RejectionKind; +/// +/// assert_eq!(RejectionKind::QueueFull.as_label(), "queue_full"); +/// assert_eq!(RejectionKind::RemovedFromQueue.as_label(), "removed_from_queue"); +/// ``` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum RejectionKind { + /// The registry already contains a task with the requested name. + AlreadyExists, + /// One conflicting item caused an all-or-nothing batch to reject this non-conflicting item. + BatchRejected, + /// `DropIfRunning` rejected a submission because the controller slot was busy. + SlotBusy, + /// The controller slot queue reached its configured capacity. + QueueFull, + /// A newer `Replace` submission displaced this queued submission. + SupersededByReplace, + /// An explicit remove or cancel operation removed the queued submission. + RemovedFromQueue, + /// Runtime shutdown rejected work that had not reached registry admission. + ControllerShuttingDown, + /// The controller could not commit the submission to the runtime registry. + AdmissionFailed, +} + +impl RejectionKind { + /// Returns a stable machine-readable label for logs and metrics. + #[must_use] + pub fn as_label(&self) -> &'static str { + match self { + Self::AlreadyExists => "already_exists", + Self::BatchRejected => "batch_rejected", + Self::SlotBusy => "slot_busy", + Self::QueueFull => "queue_full", + Self::SupersededByReplace => "superseded_by_replace", + Self::RemovedFromQueue => "removed_from_queue", + Self::ControllerShuttingDown => "controller_shutting_down", + Self::AdmissionFailed => "admission_failed", + } + } +} + /// One runtime event with optional metadata. /// /// - `at`: wall-clock timestamp (for logs) @@ -441,6 +491,10 @@ pub struct Event { pub duration_ms: Option, /// Only values documented in [`reasons`](crate::reasons) are stable. pub reason: Option>, + /// Machine-readable category for `TaskAddFailed` and `ControllerRejected`. + /// + /// Readable details remain available in [`reason`](Self::reason). + pub rejection_kind: Option, /// Attempt count (starting from 1). pub attempt: Option, /// This is normally a task name. Subscriber diagnostics use it for a subscriber name, and controller events use it for a slot name. @@ -476,6 +530,7 @@ impl Event { duration_ms: None, attempt: None, reason: None, + rejection_kind: None, task: None, id: None, exit_code: None, @@ -490,6 +545,14 @@ impl Event { self } + /// Attaches a machine-readable submission rejection category. + #[inline] + #[must_use] + pub fn with_rejection_kind(mut self, kind: RejectionKind) -> Self { + self.rejection_kind = Some(kind); + self + } + /// Attaches a task name. #[inline] #[must_use] @@ -631,6 +694,9 @@ impl std::fmt::Debug for Event { if let Some(ref reason) = self.reason { d.field("reason", reason); } + if let Some(rejection_kind) = self.rejection_kind { + d.field("rejection_kind", &rejection_kind); + } if let Some(timeout_ms) = self.timeout_ms { d.field("timeout_ms", &timeout_ms); } @@ -708,11 +774,33 @@ mod tests { assert_eq!(ev.attempt, None); assert_eq!(ev.exit_code, None); assert_eq!(ev.reason, None); + assert_eq!(ev.rejection_kind, None); assert_eq!(ev.task, None); assert_eq!(ev.id, None); assert_eq!(ev.backoff_source, None); } + #[test] + fn rejection_kind_labels_are_stable() { + let cases = [ + (RejectionKind::AlreadyExists, "already_exists"), + (RejectionKind::BatchRejected, "batch_rejected"), + (RejectionKind::SlotBusy, "slot_busy"), + (RejectionKind::QueueFull, "queue_full"), + (RejectionKind::SupersededByReplace, "superseded_by_replace"), + (RejectionKind::RemovedFromQueue, "removed_from_queue"), + ( + RejectionKind::ControllerShuttingDown, + "controller_shutting_down", + ), + (RejectionKind::AdmissionFailed, "admission_failed"), + ]; + + for (kind, expected) in cases { + assert_eq!(kind.as_label(), expected, "{kind:?}"); + } + } + #[test] fn ms_builders_set_then_clamp_to_u32_max() { let normal = Duration::from_millis(42); diff --git a/src/events/mod.rs b/src/events/mod.rs index 6bc6a16..c1e527f 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -8,6 +8,7 @@ //! | [`EventKind`] | Event classification | //! | [`Event`] | Event payload and metadata | //! | [`BackoffSource`] | Why a `BackoffScheduled` event was emitted | +//! | [`RejectionKind`] | Machine-readable submission rejection | //! //! ## Events and final outcomes are different //! @@ -54,7 +55,7 @@ //! [optional TaskRemoveRequested] ──► TaskRemoved //! //! Queued controller removal (feature `controller`): -//! TaskRemoveRequested ──► ControllerRejected(removed_from_queue) +//! TaskRemoveRequested ──► ControllerRejected(RemovedFromQueue) //! //! Shutdown: //! ShutdownRequested ──► AllStoppedWithinGrace | GraceExceeded @@ -71,6 +72,7 @@ //! It can help sort events and detect gaps, but it is not a causal clock. //! - Use [`EventKind::as_label`] for a stable telemetry label. //! Treat free-form [`Event::reason`] text as diagnostic unless it is documented in [`reasons`](crate::reasons). +//! - Use [`RejectionKind`] for machine-readable handling of `TaskAddFailed` and `ControllerRejected`. //! //! ## Subscribers //! @@ -79,7 +81,7 @@ //! With the `logging` feature, `LogWriter` prints simple development logs. mod event; -pub use event::{BackoffSource, Event, EventKind}; +pub use event::{BackoffSource, Event, EventKind, RejectionKind}; mod bus; pub(crate) use bus::Bus; diff --git a/src/lib.rs b/src/lib.rs index e652011..3ecf151 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -156,7 +156,7 @@ pub use core::{ }; pub mod tasks; -pub use tasks::{BoxTaskFuture, Task, TaskContext, TaskFn, TaskRef, TaskSpec}; +pub use tasks::{BoxTaskFuture, Task, TaskContext, TaskFn, TaskRef, TaskSetting, TaskSpec}; pub mod policies; pub use policies::{BackoffError, BackoffPolicy, JitterPolicy, RestartPolicy}; @@ -165,7 +165,7 @@ pub mod error; pub use error::{BoxError, Error, RuntimeError, SharedError, TaskError}; pub mod events; -pub use events::{BackoffSource, Event, EventKind}; +pub use events::{BackoffSource, Event, EventKind, RejectionKind}; pub mod subscribers; pub use subscribers::Subscribe; diff --git a/src/prelude.rs b/src/prelude.rs index 868b847..573be34 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -20,13 +20,13 @@ pub use crate::core::{ }; /// Task abstractions and task specs. -pub use crate::tasks::{BoxTaskFuture, Task, TaskContext, TaskFn, TaskRef, TaskSpec}; +pub use crate::tasks::{BoxTaskFuture, Task, TaskContext, TaskFn, TaskRef, TaskSetting, TaskSpec}; /// Restart, retry, backoff, and jitter policies. pub use crate::policies::{BackoffError, BackoffPolicy, JitterPolicy, RestartPolicy}; /// Runtime event types. -pub use crate::events::{BackoffSource, Event, EventKind}; +pub use crate::events::{BackoffSource, Event, EventKind, RejectionKind}; /// Runtime and task error types. pub use crate::error::{Error, RuntimeError, TaskError}; diff --git a/src/subscribers/embedded/log.rs b/src/subscribers/embedded/log.rs index 6c34f0a..4d8e418 100644 --- a/src/subscribers/embedded/log.rs +++ b/src/subscribers/embedded/log.rs @@ -167,7 +167,18 @@ impl LogWriter { // Controller: the `task` field carries the slot name. #[cfg(feature = "controller")] - EventKind::ControllerRejected | EventKind::ControllerSlotTransition => { + EventKind::ControllerRejected => { + println!( + "{head} slot={} rejection={} reason=\"{}\"", + or(e.task.as_deref(), "none"), + e.rejection_kind + .map(|kind| kind.as_label()) + .unwrap_or("unknown"), + or(e.reason.as_deref(), "unknown") + ); + } + #[cfg(feature = "controller")] + EventKind::ControllerSlotTransition => { println!( "{head} slot={} reason=\"{}\"", or(e.task.as_deref(), "none"), diff --git a/src/subscribers/embedded/tracing.rs b/src/subscribers/embedded/tracing.rs index 63ac1fc..440f99b 100644 --- a/src/subscribers/embedded/tracing.rs +++ b/src/subscribers/embedded/tracing.rs @@ -100,6 +100,7 @@ impl Subscribe for TracingBridge { duration_ms = e.duration_ms.map(u64::from), exit_code = e.exit_code.map(i64::from), backoff_source = e.backoff_source.map(|s| s.as_label()), + rejection_kind = e.rejection_kind.map(|kind| kind.as_label()), ) }; } diff --git a/src/tasks/mod.rs b/src/tasks/mod.rs index 0959981..f7c4cb6 100644 --- a/src/tasks/mod.rs +++ b/src/tasks/mod.rs @@ -8,6 +8,7 @@ //! | [`TaskFn`] | Closure-based [`Task`] | //! | [`TaskRef`] | Shared task handle: `Arc` | //! | [`TaskSpec`] | Task plus restart, backoff, timeout, and retry settings | +//! | [`TaskSetting`] | Explicit or inherited setting before admission | //! | [`BoxTaskFuture`] | Future returned by [`Task::spawn`] | //! //! ## Create a Task @@ -59,11 +60,11 @@ mod task; pub use task::{BoxTaskFuture, Task, TaskRef}; +mod spec; +pub use spec::{TaskSetting, TaskSpec}; + mod context; pub use context::TaskContext; mod r#impl; pub use r#impl::func::TaskFn; - -mod spec; -pub use spec::TaskSpec; diff --git a/src/tasks/spec.rs b/src/tasks/spec.rs index ec502cb..fd4b3f2 100644 --- a/src/tasks/spec.rs +++ b/src/tasks/spec.rs @@ -71,35 +71,47 @@ fn normalize_timeout(timeout: Option) -> Option { #[must_use] pub struct TaskSpec { /// Restart policy selected explicitly or inherited from [`TaskDefaults`]. - restart: Override, + restart: TaskSetting, /// Backoff policy selected explicitly or inherited from [`TaskDefaults`]. - backoff: Override, - /// Per-attempt timeout; `Set(None)` explicitly disables an inherited timeout. - timeout: Override>, - /// Retry limit; `Set(None)` explicitly selects unlimited retries. - max_retries: Override>, + backoff: TaskSetting, + /// Per-attempt timeout; `Explicit(None)` disables an inherited timeout. + timeout: TaskSetting>, + /// Retry limit; `Explicit(None)` selects unlimited retries. + max_retries: TaskSetting>, /// Task object reused across all attempts started from this spec. task: TaskRef, } -/// Origin of one `TaskSpec` setting before admission resolves defaults. +/// Whether a [`TaskSpec`] setting is inherited or explicitly selected. /// -/// For `Override>`, `Inherit` and `Set(None)` are intentionally distinct. -#[derive(Clone, Copy, Debug)] -enum Override { +/// Optional settings use `TaskSetting>`. This keeps inherited and +/// explicitly disabled values distinct without exposing `Option>`: +/// +/// ```rust +/// use taskvisor::TaskSetting; +/// +/// let inherited: TaskSetting> = TaskSetting::Inherit; +/// let disabled = TaskSetting::Explicit(None); +/// let limited = TaskSetting::Explicit(Some(3)); +/// +/// assert_ne!(inherited, disabled); +/// assert_ne!(disabled, limited); +/// ``` +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TaskSetting { /// Resolve the field from [`TaskDefaults`] at registry admission. Inherit, /// Use this explicit value instead of the corresponding default. - Set(T), + Explicit(T), } -impl Override { +impl TaskSetting { #[inline] fn value(self) -> Option { match self { Self::Inherit => None, - Self::Set(value) => Some(value), + Self::Explicit(value) => Some(value), } } @@ -107,7 +119,7 @@ impl Override { fn resolve(self, default: T) -> T { match self { Self::Inherit => default, - Self::Set(value) => value, + Self::Explicit(value) => value, } } } @@ -154,10 +166,10 @@ impl TaskSpec { /// > A later `with_*` call sets that one field explicitly. pub fn from_defaults(task: TaskRef) -> Self { Self { - restart: Override::Inherit, - backoff: Override::Inherit, - timeout: Override::Inherit, - max_retries: Override::Inherit, + restart: TaskSetting::Inherit, + backoff: TaskSetting::Inherit, + timeout: TaskSetting::Inherit, + max_retries: TaskSetting::Inherit, task, } } @@ -178,10 +190,10 @@ impl TaskSpec { timeout: impl Into>, ) -> Self { Self { - restart: Override::Set(restart), - backoff: Override::Set(backoff), - timeout: Override::Set(normalize_timeout(timeout.into())), - max_retries: Override::Set(None), + restart: TaskSetting::Explicit(restart), + backoff: TaskSetting::Explicit(backoff), + timeout: TaskSetting::Explicit(normalize_timeout(timeout.into())), + max_retries: TaskSetting::Explicit(None), task, } } @@ -192,10 +204,10 @@ impl TaskSpec { /// > Override them with the matching `with_*` methods. pub fn once(task: TaskRef) -> Self { Self { - restart: Override::Set(RestartPolicy::Never), - backoff: Override::Inherit, - timeout: Override::Inherit, - max_retries: Override::Inherit, + restart: TaskSetting::Explicit(RestartPolicy::Never), + backoff: TaskSetting::Inherit, + timeout: TaskSetting::Inherit, + max_retries: TaskSetting::Inherit, task, } } @@ -206,10 +218,10 @@ impl TaskSpec { /// > Backoff, timeout, and retry limit are inherited from [`TaskDefaults`]. pub fn restartable(task: TaskRef) -> Self { Self { - restart: Override::Set(RestartPolicy::OnFailure), - backoff: Override::Inherit, - timeout: Override::Inherit, - max_retries: Override::Inherit, + restart: TaskSetting::Explicit(RestartPolicy::OnFailure), + backoff: TaskSetting::Inherit, + timeout: TaskSetting::Inherit, + max_retries: TaskSetting::Inherit, task, } } @@ -243,12 +255,12 @@ impl TaskSpec { #[doc(alias = "fixed delay")] pub fn periodic(task: TaskRef, every: Duration) -> Self { Self { - restart: Override::Set(RestartPolicy::Always { + restart: TaskSetting::Explicit(RestartPolicy::Always { interval: Some(every).filter(|d| !d.is_zero()), }), - backoff: Override::Inherit, - timeout: Override::Inherit, - max_retries: Override::Inherit, + backoff: TaskSetting::Inherit, + timeout: TaskSetting::Inherit, + max_retries: TaskSetting::Inherit, task, } } @@ -279,22 +291,22 @@ impl TaskSpec { /// Returns how this spec overrides the attempt timeout. /// - /// - `None` means inherit the default. - /// - `Some(None)` means explicitly disable the timeout. - /// - `Some(Some(duration))` means use that timeout. + /// - [`TaskSetting::Inherit`] means inherit the default. + /// - `TaskSetting::Explicit(None)` explicitly disables the timeout. + /// - `TaskSetting::Explicit(Some(duration))` selects that timeout. #[must_use] - pub fn timeout_override(&self) -> Option> { - self.timeout.value() + pub fn timeout_override(&self) -> TaskSetting> { + self.timeout } /// Returns how this spec overrides the retry limit. /// - /// - `None` means inherit the default. - /// - `Some(None)` means explicitly allow unlimited retries. - /// - `Some(Some(limit))` means use that retry limit. + /// - [`TaskSetting::Inherit`] means inherit the default. + /// - `TaskSetting::Explicit(None)` explicitly allows unlimited retries. + /// - `TaskSetting::Explicit(Some(limit))` selects that retry limit. #[must_use] - pub fn max_retries_override(&self) -> Option> { - self.max_retries.value() + pub fn max_retries_override(&self) -> TaskSetting> { + self.max_retries } /// Sets the timeout for each attempt. @@ -304,7 +316,7 @@ impl TaskSpec { #[doc(alias = "watchdog")] #[doc(alias = "attempt deadline")] pub fn with_timeout(mut self, timeout: impl Into>) -> Self { - self.timeout = Override::Set(normalize_timeout(timeout.into())); + self.timeout = TaskSetting::Explicit(normalize_timeout(timeout.into())); self } @@ -312,7 +324,7 @@ impl TaskSpec { /// /// This value overrides the supervisor default. pub fn with_backoff(mut self, backoff: BackoffPolicy) -> Self { - self.backoff = Override::Set(backoff); + self.backoff = TaskSetting::Explicit(backoff); self } @@ -320,7 +332,7 @@ impl TaskSpec { /// /// This value overrides the supervisor default. pub fn with_restart(mut self, restart: RestartPolicy) -> Self { - self.restart = Override::Set(restart); + self.restart = TaskSetting::Explicit(restart); self } @@ -333,7 +345,7 @@ impl TaskSpec { #[doc(alias = "retry limit")] #[doc(alias = "retry budget")] pub fn with_max_retries(mut self, max_retries: impl Into>) -> Self { - self.max_retries = Override::Set(max_retries.into()); + self.max_retries = TaskSetting::Explicit(max_retries.into()); self } @@ -406,12 +418,16 @@ mod tests { fn assert_inherits_non_restart_settings(spec: &TaskSpec) { assert!(spec.backoff_override().is_none()); - assert!(spec.timeout_override().is_none()); - assert!(spec.max_retries_override().is_none()); + assert_eq!(spec.timeout_override(), TaskSetting::Inherit); + assert_eq!(spec.max_retries_override(), TaskSetting::Inherit); } fn assert_explicit_timeout(spec: TaskSpec, expected: Option, case: &str) { - assert_eq!(spec.timeout_override(), Some(expected), "{case}"); + assert_eq!( + spec.timeout_override(), + TaskSetting::Explicit(expected), + "{case}" + ); } #[test] @@ -468,8 +484,11 @@ mod tests { spec.backoff_override().map(|policy| policy.first()), Some(Duration::from_secs(2)) ); - assert_eq!(spec.timeout_override(), Some(Some(timeout))); - assert_eq!(spec.max_retries_override(), Some(None)); + assert_eq!( + spec.timeout_override(), + TaskSetting::Explicit(Some(timeout)) + ); + assert_eq!(spec.max_retries_override(), TaskSetting::Explicit(None)); } #[test] @@ -482,8 +501,8 @@ mod tests { .with_timeout(None) .with_max_retries(None); - assert_eq!(spec.timeout_override(), Some(None)); - assert_eq!(spec.max_retries_override(), Some(None)); + assert_eq!(spec.timeout_override(), TaskSetting::Explicit(None)); + assert_eq!(spec.max_retries_override(), TaskSetting::Explicit(None)); let resolved = spec.resolve(&defaults); assert_eq!(resolved.timeout(), None); @@ -579,10 +598,10 @@ mod tests { let spec = TaskSpec::once(task("limited")) .try_with_max_retries(3) .expect("a positive retry limit must be accepted"); - assert_eq!( - spec.max_retries_override().flatten().map(NonZeroU32::get), - Some(3) - ); + assert!(matches!( + spec.max_retries_override(), + TaskSetting::Explicit(Some(limit)) if limit.get() == 3 + )); assert_eq!( TaskSpec::once(task("zero"))