diff --git a/src/controller/core/admission.rs b/src/controller/core/admission.rs index 9cb92b0..fbc60d1 100644 --- a/src/controller/core/admission.rs +++ b/src/controller/core/admission.rs @@ -13,6 +13,7 @@ use crate::{ }, events::{Event, EventKind}, identity::TaskId, + reasons, }; use super::{AdmissionResult, CompletionResult, Controller, RemovalResult, Submission}; @@ -179,7 +180,7 @@ impl Controller { | SlotPhase::Terminating { .. }, AdmissionPolicy::DropIfRunning, ) => { - let reason = format!("dropped: slot busy ({})", slot.status_label()); + let reason = format!("{} ({})", reasons::DROP_IF_RUNNING, slot.status_label()); self.bus.publish( Event::new(EventKind::ControllerRejected) .with_task(Arc::clone(&slot_name)) diff --git a/src/core/builder.rs b/src/core/builder.rs index dfd1e28..54a913a 100644 --- a/src/core/builder.rs +++ b/src/core/builder.rs @@ -116,12 +116,6 @@ impl SupervisorBuilder { Ok(self) } - /// Sets how many recent events the broadcast bus keeps. - pub fn with_bus_capacity(mut self, bus_capacity: NonZeroUsize) -> Self { - self.runtime = self.runtime.with_bus_capacity(bus_capacity); - self - } - /// Sets the event-bus capacity from a raw integer. /// /// # Errors @@ -155,6 +149,12 @@ impl SupervisorBuilder { Ok(self) } + /// Sets how many recent events the broadcast bus keeps. + pub fn with_bus_capacity(mut self, bus_capacity: NonZeroUsize) -> Self { + self.runtime = self.runtime.with_bus_capacity(bus_capacity); + self + } + /// Replaces the subscribers that receive best-effort lifecycle events. pub fn with_subscribers(mut self, subscribers: Vec>) -> Self { self.subscribers = subscribers; @@ -175,6 +175,22 @@ impl SupervisorBuilder { /// /// It is safe to call outside Tokio. /// The method allocates channels and stores configuration, but does not spawn tasks. + /// + /// ## Example + /// + /// ```rust + /// use std::time::Duration; + /// use taskvisor::{SupervisorBuilder, SupervisorConfig}; + /// + /// let supervisor = SupervisorBuilder::new(SupervisorConfig::default()) + /// .with_grace(Duration::from_secs(15)) + /// .build(); + /// + /// assert_eq!( + /// supervisor.runtime_config().grace(), + /// Duration::from_secs(15) + /// ); + /// ``` #[must_use] pub fn build(self) -> Arc { let bus = Bus::new(self.runtime.bus_capacity().get()); diff --git a/src/core/config.rs b/src/core/config.rs index feacb0a..3d96dd9 100644 --- a/src/core/config.rs +++ b/src/core/config.rs @@ -7,7 +7,10 @@ use std::time::Duration; use thiserror::Error; +/// Default capacity of the event bus and registry command queue. const DEFAULT_CAPACITY: NonZeroUsize = NonZeroUsize::new(1024).unwrap(); + +/// Default deadline for draining subscriber queues during shutdown. const DEFAULT_SUBSCRIBER_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); /// Error from a checked configuration setter. diff --git a/src/core/handle.rs b/src/core/handle.rs index 9f57c76..dc1324f 100644 --- a/src/core/handle.rs +++ b/src/core/handle.rs @@ -1,7 +1,6 @@ //! # Manage a running supervisor //! //! [`Supervisor::serve`](crate::Supervisor::serve) returns a [`SupervisorHandle`]. -//! Use it to add, inspect, stop, and watch tasks while the service is running. //! //! Direct state-changing `add*`, `remove*`, and `cancel*` commands use one bounded management queue. //! A regular method waits for capacity; its `try_*` form fails fast when that queue is full. @@ -34,8 +33,8 @@ use super::outcome::TaskWaiter; /// Dropping the last public `Supervisor` or handle sends best-effort cancellation, but cannot wait for cleanup. /// Call [`shutdown`](Self::shutdown) for a confirmed shutdown result. /// -/// Use [`remove`](Self::remove) to request a stop and return after the request is accepted. -/// Use [`cancel`](Self::cancel) to wait until the task is joined and its name and identity are released. +/// > Use [`remove`](Self::remove) to request a stop and return after the request is accepted. +/// > Use [`cancel`](Self::cancel) to wait until the task is joined and its name and identity are released. /// /// ## Example /// @@ -132,9 +131,10 @@ impl SupervisorHandle { /// Adds a task and returns a waiter for its final outcome. /// - /// Registration is the same as [`add`](Self::add). The [`TaskWaiter`] resolves after all retries end, - /// the registry joins the task actor, and registry membership is removed. - /// It uses a direct completion channel, not best-effort lifecycle events. + /// Registration is the same as [`add`](Self::add). + /// The [`TaskWaiter`] resolves after all retries end, the registry joins the task actor, and registry membership is removed. + /// + /// > It uses a direct completion channel, not best-effort lifecycle events. /// /// # Errors /// @@ -191,8 +191,8 @@ impl SupervisorHandle { /// `Ok(false)` means the identity was unknown, already finished, or already claimed by another stop request. /// /// For a registered task, the method returns before final cleanup. - /// Use [`cancel`](Self::cancel) when you need to wait until termination. /// Removing queued controller work is complete when this method returns. + /// > Use [`cancel`](Self::cancel) when you need to wait until termination. /// /// # Errors /// @@ -251,8 +251,7 @@ impl SupervisorHandle { /// Returns the authoritative registry view as `(id, name)` pairs. /// /// The list comes from the registry and is sorted by [`TaskId`]. - /// It includes every registry entry, whether its actor is running, waiting for a permit, - /// backoff, or restart interval, already finished but not cleaned up, or being removed. + /// It includes every registry entry, whether its actor is running, waiting for a permit, backoff, or restart interval, already finished but not cleaned up, or being removed. /// /// See [`alive_snapshot`](Self::alive_snapshot) for the best-effort list of task names currently marked alive. pub async fn list(&self) -> Vec<(TaskId, Arc)> { @@ -273,7 +272,7 @@ impl SupervisorHandle { /// Returns whether the cache currently marks any run with this name as alive. /// /// This best-effort query does not check a specific [`TaskId`] and can miss state after event loss. - /// Use [`list`](Self::list) for registry membership. + /// > Use [`list`](Self::list) for registry membership. pub async fn is_alive(&self, name: &str) -> bool { self.core().is_alive(name).await } @@ -471,8 +470,8 @@ impl SupervisorHandle { /// `Ok(id)` confirms only that the controller queue accepted the submission. /// Slot admission and runtime registration happen later. /// - /// Use [`try_submit`](Self::try_submit) to fail fast when the controller queue is full. - /// Use [`submit_and_watch`](Self::submit_and_watch) to observe the final outcome, including admission rejection. + /// > Use [`try_submit`](Self::try_submit) to fail fast when the controller queue is full. + /// > Use [`submit_and_watch`](Self::submit_and_watch) to observe the final outcome, including admission rejection. /// /// Requires the `controller` feature. /// diff --git a/src/core/outcome.rs b/src/core/outcome.rs index 7de5a3b..e164db5 100644 --- a/src/core/outcome.rs +++ b/src/core/outcome.rs @@ -1,10 +1,12 @@ //! # Reliable final task results //! -//! Lifecycle events show live progress, but they are best-effort. -//! Use a [`TaskWaiter`] when application logic needs the final [`TaskOutcome`]. +//! Lifecycle events answer "what is happening now?". +//! They may be dropped. Application logic must not reconstruct a final result from them. //! -//! A waiter uses a direct one-shot channel. -//! Event-bus lag does not affect it. +//! A [`TaskWaiter`] answers "how did this task end?" for one [`TaskId`]. +//! It receives one final [`TaskOutcome`] through a direct one-shot channel, outside the event bus. +//! For an admitted task, the registry sends the outcome after joining the actor and removing its membership. +//! Event-bus lag does not affect this path. If the outcome cannot be delivered, [`TaskWaiter::wait`] returns an error instead of guessing the result. //! //! ## Successful Direct-Add Flow //! @@ -30,8 +32,7 @@ //! - One waiter follows one [`TaskId`]. //! - For admitted work, it resolves after all retries end and the registry joins the task actor. //! - Dropping a waiter is safe and does not cancel the task. -//! - If the runtime drops the sender before it creates an outcome, -//! [`TaskWaiter::wait`] returns an error instead of inventing a result. +//! - If the runtime drops the sender before it creates an outcome, [`TaskWaiter::wait`] returns an error instead of inventing a result. //! //! Direct [`SupervisorHandle::add_and_watch`](crate::SupervisorHandle::add_and_watch) //! returns registration errors such as a duplicate name before it gives the @@ -121,8 +122,8 @@ pub enum TaskOutcome { /// The runtime aborted the actor before cooperative stop completed. /// - /// This normally happens after the configured grace period. Last-owner - /// fallback and signal-setup failure cleanup cannot wait for that period. + /// This normally happens after the configured grace period. + /// Last-owner fallback and signal-setup failure cleanup cannot wait for that period. ForceAborted, /// The internal actor panicked. @@ -214,7 +215,8 @@ impl TaskOutcome { /// Returns the original error source for [`Failed`](Self::Failed) or [`Fatal`](Self::Fatal). /// /// Returns `None` when the outcome has no source error. - /// Callers can use `downcast_ref` or pass it to an error-reporting library. + /// + /// > Callers can use `downcast_ref` or pass it to an error-reporting library. #[must_use] pub fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { @@ -305,7 +307,9 @@ impl TaskWaiter { /// /// # Errors /// - /// Returns [`RuntimeError::ShuttingDown`] if the runtime drops its sender before producing an outcome. No final result is available in that case. + /// Returns [`RuntimeError::ShuttingDown`] if the runtime drops its sender before producing an outcome. + /// + /// > No final result is available in that case. pub async fn wait(self) -> Result { self.rx.await.map_err(|_| RuntimeError::ShuttingDown) } diff --git a/src/core/owner.rs b/src/core/owner.rs index e69d706..a3d5bf8 100644 --- a/src/core/owner.rs +++ b/src/core/owner.rs @@ -1,4 +1,21 @@ -//! Tracks the lifetime of public runtime owners. +//! # Public runtime ownership +//! +//! [`RuntimeOwner`] is the shared lease held by [`Supervisor`](crate::Supervisor) and every [`SupervisorHandle`](crate::SupervisorHandle). +//! +//! ```text +//! Supervisor ────────────┐ +//! SupervisorHandle ──────┼── Arc ── Arc +//! cloned handles ────────┘ +//! +//! internal workers ────────────────────────────── Arc +//! ``` +//! +//! Internal workers retain the runtime core directly, but they do not retain `RuntimeOwner`. +//! The last public owner can therefore disappear while cleanup work still holds the core alive. +//! +//! Dropping that last owner cannot wait for graceful cleanup. +//! It invokes [`SupervisorCore::abandon`] to close runtime intake and propagate cancellation. +//! If an explicit shared shutdown operation already exists, that operation keeps ownership of cleanup and the fallback does not replace it. use std::sync::Arc; @@ -9,25 +26,37 @@ use super::SupervisorCore; /// Internal workers do not hold this value. /// Its destructor therefore runs when the last public owner disappears and starts best-effort runtime cancellation. pub(crate) struct RuntimeOwner { + /// Runtime state shared with public owners and internal workers. + /// + /// Keeping the core behind a separate `Arc` lets detached shutdown and cleanup work outlive the public ownership lease when necessary. core: Arc, } impl RuntimeOwner { + /// Creates the first public ownership lease for `core`. + /// + /// The returned `Arc` is cloned into each [`SupervisorHandle`](crate::SupervisorHandle). pub(crate) fn new(core: Arc) -> Arc { Arc::new(Self { core }) } + /// Borrows the shared runtime core without creating another public owner. pub(crate) fn core(&self) -> &Arc { &self.core } } +/// Omits runtime internals from the ownership lease's debug representation. impl std::fmt::Debug for RuntimeOwner { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("RuntimeOwner").finish_non_exhaustive() } } +/// Starts the non-blocking last-owner fallback. +/// +/// `Drop` cannot await joins or return a shutdown result. +/// Confirmed cleanup must go through [`SupervisorHandle::shutdown`](crate::SupervisorHandle::shutdown). impl Drop for RuntimeOwner { fn drop(&mut self) { self.core.abandon(); diff --git a/src/core/registry/completion.rs b/src/core/registry/completion.rs index 50354bc..2df2575 100644 --- a/src/core/registry/completion.rs +++ b/src/core/registry/completion.rs @@ -1,4 +1,15 @@ -//! Lossless completion signals shared by registry clients and cleanup owners. +//! # Registry completion signals +//! +//! Natural completion, an explicit remove or cancel, and shutdown can all start +//! terminal removal. One path claims the actor and becomes its join owner. +//! [`RemovalCompletion`] connects callers waiting for cleanup with that owner. +//! +//! ```text +//! Registered ──► Removing ──► join or force-abort ──► remove registry membership +//! │ +//! ▼ +//! RemovalCompletion complete +//! ``` use tokio::sync::oneshot; use tokio_util::sync::CancellationToken; @@ -8,27 +19,47 @@ use crate::core::outcome::TaskOutcome; /// Sender used to resolve a watched task with its final [`TaskOutcome`]. pub(crate) type OutcomeTx = oneshot::Sender; -/// Shared terminal signal for callers waiting until registry cleanup is committed. +/// Shared one-shot signal for committed terminal registry cleanup. +/// +/// Every clone observes the same completion. The signal becomes complete only +/// after the actor has been joined or force-aborted and its identity and label +/// have been removed from the registry. Watched-outcome delivery and final event +/// publication are attempted before waiters are released. +/// +/// This is not the actor's cancellation token. Creating or waiting on this value +/// does not request task cancellation. #[derive(Clone, Debug)] pub(crate) struct RemovalCompletion { + /// One-shot latch shared by cleanup owners and waiters. + /// + /// The token's cancelled state represents completed registry cleanup. token: CancellationToken, } impl RemovalCompletion { + /// Creates a new incomplete terminal-cleanup signal. pub(crate) fn new() -> Self { Self { token: CancellationToken::new(), } } + /// Waits until terminal registry cleanup has been committed. + /// + /// If cleanup is already complete, this returns immediately. Dropping this + /// wait does not affect other waiters or the cleanup owner. pub(crate) async fn wait(&self) { self.token.cancelled().await; } + /// Returns `true` when terminal registry cleanup has been committed. pub(super) fn is_complete(&self) -> bool { self.token.is_cancelled() } + /// Marks terminal registry cleanup complete and releases all waiters. + /// + /// Repeated calls leave the signal complete. pub(super) fn complete(&self) { self.token.cancel(); } diff --git a/src/core/registry/listener.rs b/src/core/registry/listener.rs index f17ed80..c28e2b0 100644 --- a/src/core/registry/listener.rs +++ b/src/core/registry/listener.rs @@ -208,7 +208,7 @@ impl Registry { match handle.await { Ok(()) => true, Err(error) => { - self.bus.publish(Event::subscriber_panicked( + self.bus.publish(Event::runtime_failure( "registry", format!("listener join failed: {error}"), )); @@ -236,7 +236,7 @@ impl Registry { /// A panic while processing one command or completion is reported as a diagnostic event instead of killing the registry listener. async fn guarded(&self, who: &'static str, fut: impl Future) { if let Err(msg) = crate::core::panic_guard::guarded(fut).await { - self.bus.publish(Event::subscriber_panicked( + self.bus.publish(Event::runtime_failure( who, format!("listener panic: {msg}"), )); diff --git a/src/core/registry/protocol.rs b/src/core/registry/protocol.rs index 1e15f7b..db6f508 100644 --- a/src/core/registry/protocol.rs +++ b/src/core/registry/protocol.rs @@ -23,7 +23,7 @@ pub(crate) struct AddBatchItem { /// Authoritative result of one registry remove command. /// /// `Ok(true)` means the registry claimed the task and sent cancellation. -/// It does not mean the actor has terminated yet. +/// > It does not mean the actor has terminated yet. pub(crate) type RemoveReply = Result; /// Receiver for an authoritative registry remove result. @@ -32,7 +32,7 @@ pub(crate) type RemoveReplyRx = oneshot::Receiver; /// Registry decision returned to one cancellation caller. /// /// `claimed` is true only for the caller that changed `Registered` to `Removing`. -/// Every caller that observes the same removal waits on the same terminal completion. +/// > Every caller that observes the same removal waits on the same terminal completion. pub(crate) struct CancelDecision { pub(crate) id: TaskId, pub(crate) claimed: bool, @@ -53,8 +53,7 @@ impl CancelDecision { /// Authoritative result of one registry cancel command. /// -/// `Ok(None)` means no registry entry exists at this command's ordering point: -/// the identity is unknown or terminal cleanup has already removed it. +/// `Ok(None)` means no registry entry exists at this command's ordering point: the identity is unknown or terminal cleanup has already removed it. pub(crate) type CancelReply = Result, RuntimeError>; /// Receiver for an authoritative registry cancel decision. diff --git a/src/core/registry/removal.rs b/src/core/registry/removal.rs index 6d1fdb7..76fe8d8 100644 --- a/src/core/registry/removal.rs +++ b/src/core/registry/removal.rs @@ -17,6 +17,7 @@ use crate::{ core::{actor::ActorExitReason, outcome::TaskOutcome}, events::{Bus, Event, EventKind}, identity::TaskId, + reasons, }; /// Terminal result passed from the single join owner to registry cleanup. @@ -424,7 +425,7 @@ impl Registry { Event::new(EventKind::TaskRemoved) .with_task(Arc::clone(&entry.label)) .with_id(id) - .with_reason("force_terminated_after_grace"), + .with_reason(reasons::FORCE_TERMINATED_AFTER_GRACE), ); } } diff --git a/src/core/registry/state.rs b/src/core/registry/state.rs index d32bad4..95f3004 100644 --- a/src/core/registry/state.rs +++ b/src/core/registry/state.rs @@ -54,7 +54,7 @@ struct PendingInner { /// Number of in-flight join reporters per task identity. counts: HashMap, - /// Human labels used for shutdown diagnostics when joins do not finish in time. + /// Labels used for shutdown diagnostics when joins do not finish in time. labels: HashMap>, } diff --git a/src/core/runner.rs b/src/core/runner.rs index 5568f81..97085cb 100644 --- a/src/core/runner.rs +++ b/src/core/runner.rs @@ -1,7 +1,6 @@ //! # Run one task attempt //! -//! [`run_once`] calls [`Task::spawn`], applies one attempt timeout, catches -//! panics while calling or polling the task, and publishes attempt events. +//! [`run_once`] calls [`Task::spawn`], applies one attempt timeout, catches panics while calling or polling the task, and publishes attempt events. //! It returns the attempt result to [`TaskActor`](super::actor::TaskActor), which decides whether to restart. //! //! ## Event Flow @@ -17,10 +16,10 @@ //! ## Rules //! //! - Each completed call publishes one final attempt event: `TaskStopped`,`TaskCanceled`, or `TaskFailed`. Force-aborting the actor can drop an in-flight call before that event. -//! - `TimeoutHit` is published only when the configured attempt timer expires. -//! - `TaskError::Canceled` is a cooperative stop, not a failure. //! - Each attempt gets a child cancellation token. Parent cancellation reaches it, but child cancellation does not affect the parent. //! - Panics while calling `spawn()` or polling its future become retryable [`TaskError::Fail`] values. +//! - `TimeoutHit` is published only when the configured attempt timer expires. +//! - `TaskError::Canceled` is a cooperative stop, not a failure. use std::future::Future; use std::panic::AssertUnwindSafe; @@ -310,7 +309,6 @@ mod tests { async fn failure_returns_fail_variant() { let bus = Bus::new(16); let parent = CancellationToken::new(); - let result = run_once(&FailTask, &parent, None, 1, TaskId::next(), &bus).await; assert!( diff --git a/src/core/runtime/event_relay.rs b/src/core/runtime/event_relay.rs index 55b03a5..94a0cc4 100644 --- a/src/core/runtime/event_relay.rs +++ b/src/core/runtime/event_relay.rs @@ -71,7 +71,7 @@ impl SupervisorCore { ) .await { - set.emit_arc(Arc::new(Event::subscriber_panicked( + set.emit_arc(Arc::new(Event::runtime_failure( "subscriber_listener", format!("listener panic: {panic}"), ))); @@ -123,7 +123,7 @@ impl SupervisorCore { match handle.await { Ok(()) => true, Err(error) => { - self.subs.emit_arc(Arc::new(Event::subscriber_panicked( + self.subs.emit_arc(Arc::new(Event::runtime_failure( "subscriber_listener", format!("listener join failed: {error}"), ))); diff --git a/src/core/runtime/lifecycle.rs b/src/core/runtime/lifecycle.rs index bdc215e..f424d55 100644 --- a/src/core/runtime/lifecycle.rs +++ b/src/core/runtime/lifecycle.rs @@ -12,9 +12,6 @@ impl SupervisorCore { /// - subscriber queue workers, /// - the event relay, /// - the registry listener. - /// - /// Safe to call more than once. - /// Concurrent callers wait for the first startup to install every listener before they return. pub(crate) fn start(&self) { if self.started.load(Ordering::Acquire) { return; @@ -101,7 +98,6 @@ impl SupervisorCore { /// If a received signal wins the shutdown race, it starts graceful shutdown and publishes `ShutdownRequested`. /// If a signal-setup error wins, the shared result is [`RuntimeError::SignalSetupFailed`]; /// common cleanup still runs, but `ShutdownRequested` and the graceful task-drain verdict are not emitted. - /// A losing signal result joins the operation already in progress. pub(super) async fn on_shutdown_signal( self: &Arc, res: std::io::Result<()>, diff --git a/src/core/runtime/management.rs b/src/core/runtime/management.rs index f1cde20..3ae746a 100644 --- a/src/core/runtime/management.rs +++ b/src/core/runtime/management.rs @@ -201,8 +201,7 @@ impl SupervisorCore { /// Publishes the request event and makes an already-reserved Add visible. /// - /// Reserving capacity before this call keeps rejected commands silent while - /// preserving `TaskAddRequested` before the registry result event. + /// Reserving capacity before this call keeps rejected commands silent while preserving `TaskAddRequested` before the registry result event. fn commit_add( &self, permit: mpsc::Permit<'_, RegistryCommand>, @@ -591,8 +590,7 @@ impl SupervisorCore { Self::wait_cancel_decision(decision, Some(wait_for)).await } - /// Tries to cancel the task that owns `label` without waiting for command queue capacity and - /// bounds the terminal-completion wait after the registry decision. + /// Tries to cancel the task that owns `label` without waiting for command queue capacity and bounds the terminal-completion wait after the registry decision. pub(crate) async fn try_cancel_by_label_with_timeout( &self, label: Arc, diff --git a/src/core/runtime/shutdown_workflow.rs b/src/core/runtime/shutdown_workflow.rs index 0e93c76..1db0128 100644 --- a/src/core/runtime/shutdown_workflow.rs +++ b/src/core/runtime/shutdown_workflow.rs @@ -295,7 +295,7 @@ impl SupervisorCore { /// Reports an internal shutdown panic without interrupting later cleanup phases. fn report_shutdown_panic(&self, phase: &str, panic: String) { - self.bus.publish(Event::subscriber_panicked( + self.bus.publish(Event::runtime_failure( "shutdown_owner", format!("{phase} panic: {panic}"), )); diff --git a/src/core/supervisor.rs b/src/core/supervisor.rs index e03b2b2..383b10a 100644 --- a/src/core/supervisor.rs +++ b/src/core/supervisor.rs @@ -36,8 +36,8 @@ use crate::{error::RuntimeError, subscribers::Subscribe, tasks::TaskSpec}; /// Owner and entry point for one taskvisor runtime. /// -/// Use [`new`](Self::new) for the standard defaults. -/// Use [`builder`](Self::builder) to set [`TaskDefaults`](crate::TaskDefaults), subscribers, or optional controller admission. +/// > Use [`new`](Self::new) for the standard defaults. +/// > Use [`builder`](Self::builder) to set [`TaskDefaults`](crate::TaskDefaults), subscribers, or optional controller admission. /// /// ## See Also /// @@ -86,7 +86,8 @@ impl Supervisor { /// Task specs use [`TaskDefaults::default`](crate::TaskDefaults::default). /// Use [`builder`](Self::builder) and [`with_task_defaults`](crate::SupervisorBuilder::with_task_defaults) to replace those defaults. /// - /// This method does not start Tokio tasks. Call [`run`](Self::run) or [`serve`](Self::serve) later. + /// This method does not start Tokio tasks. + /// Call [`run`](Self::run) or [`serve`](Self::serve) later. pub fn new(cfg: SupervisorConfig, subscribers: Vec>) -> Arc { Self::builder(cfg).with_subscribers(subscribers).build() } @@ -111,6 +112,26 @@ impl Supervisor { /// This method may be called more than once. /// Runtime workers start once; every call returns another handle to the same runtime. /// + /// ## Example + /// + /// ```rust,no_run + /// use taskvisor::prelude::*; + /// + /// # #[tokio::main] async fn main() -> Result<(), Box> { + /// let supervisor = Supervisor::new(SupervisorConfig::default(), vec![]); + /// let handle = supervisor.serve(); + /// + /// let worker: TaskRef = TaskFn::arc("worker", |ctx| async move { + /// ctx.cancelled().await; + /// Err(TaskError::Canceled) + /// }); + /// + /// let id = handle.add(TaskSpec::once(worker)).await?; + /// handle.cancel(id).await?; + /// handle.shutdown().await?; + /// # Ok(()) } + /// ``` + /// /// # Panics /// /// Panics if the runtime must start and there is no active Tokio runtime. @@ -141,6 +162,22 @@ impl Supervisor { /// It does not mean that every managed task completed successfully. /// Task failures, fatal errors, and panics remain task-level outcomes; use [`SupervisorHandle::add_and_watch`](crate::SupervisorHandle::add_and_watch) when application logic needs a reliable outcome for one task. /// + /// ## Example + /// + /// ```rust,no_run + /// use taskvisor::prelude::*; + /// + /// # #[tokio::main] async fn main() -> Result<(), Box> { + /// let supervisor = Supervisor::new(SupervisorConfig::default(), vec![]); + /// let task: TaskRef = TaskFn::arc("worker", |_ctx| async move { + /// println!("one unit of work"); + /// Ok(()) + /// }); + /// + /// supervisor.run(vec![TaskSpec::once(task)]).await?; + /// # Ok(()) } + /// ``` + /// /// # Panics /// /// Panics if the runtime must start and there is no active Tokio runtime. diff --git a/src/error.rs b/src/error.rs index 0da7077..c8a4e52 100644 --- a/src/error.rs +++ b/src/error.rs @@ -27,6 +27,10 @@ pub type SharedError = Arc; /// /// - [`Supervisor`](crate::Supervisor) - returns `RuntimeError` from [`run`](crate::Supervisor::run) /// - [`SupervisorHandle`](crate::SupervisorHandle) - returns `RuntimeError` from management methods +#[cfg_attr( + feature = "controller", + doc = "- [`ControllerError`](crate::ControllerError) - errors from optional controller configuration and `submit*` methods" +)] #[non_exhaustive] #[derive(Error, Debug)] pub enum RuntimeError { @@ -147,7 +151,7 @@ pub enum TaskError { #[error("fatal error (no retry): {reason}")] #[non_exhaustive] Fatal { - /// Human-readable failure reason. + /// Readable failure reason. reason: String, /// Process-style exit code, when available. /// @@ -164,7 +168,7 @@ pub enum TaskError { #[error("execution failed: {reason}")] #[non_exhaustive] Fail { - /// Human-readable failure reason. + /// Readable failure reason. reason: String, /// Process-style exit code, when available. /// @@ -183,12 +187,6 @@ pub enum TaskError { } impl TaskError { - /// Creates a retry-eligible timeout error. - #[must_use] - pub const fn timeout(timeout: Duration) -> Self { - TaskError::Timeout { timeout } - } - /// Creates a retryable failure with no source error. pub fn fail(reason: impl Into) -> Self { TaskError::Fail { @@ -237,6 +235,12 @@ impl TaskError { } } + /// Creates a retry-eligible timeout error. + #[must_use] + pub const fn timeout(timeout: Duration) -> Self { + TaskError::Timeout { timeout } + } + /// Sets or clears the process-style exit code on `Fail` or `Fatal`. /// /// Pass an integer to set it or `None` to clear it. This has no effect on `Timeout` or `Canceled`. diff --git a/src/events/bus.rs b/src/events/bus.rs index 167521b..d5fd629 100644 --- a/src/events/bus.rs +++ b/src/events/bus.rs @@ -7,12 +7,17 @@ //! //! ```text //! Supervisor / registry / task runners +//! │ publish(Event) //! ▼ -//! bounded broadcast bus +//! bounded broadcast bus +//! │ runtime receiver //! ▼ -//! subscriber listener -//! ▼ -//! per-subscriber queues +//! subscriber listener +//! │ +//! ├──► alive tracker +//! ├──► subscriber queue 1 +//! ├──► subscriber queue 2 +//! └──► subscriber queue N //! ``` //! //! Each `subscribe()` call creates an independent receiver position. @@ -26,7 +31,8 @@ //! - If a receiver is too slow, it gets `RecvError::Lagged(n)` and skips old events. //! - If there are no active receivers, published events are dropped. //! -//! This bus is only for observability. It does not provide durable, at-least-once, or exactly-once delivery. +//! This bus is only for observability. +//! It does not provide durable, at-least-once, or exactly-once delivery. use std::sync::Arc; use tokio::sync::broadcast; @@ -52,9 +58,6 @@ pub(crate) struct Bus { /// /// This is an internal runtime primitive. /// Public user code normally consumes events through [`Subscribe`](crate::Subscribe), not by subscribing to `Bus` directly. -/// -/// The bus is cloneable and multi-producer. -/// Receivers are independent, but they share one bounded broadcast buffer. impl Bus { /// Creates a new bus. /// diff --git a/src/events/event.rs b/src/events/event.rs index 90bd405..69ac297 100644 --- a/src/events/event.rs +++ b/src/events/event.rs @@ -16,7 +16,6 @@ //! Use it to sort observed events and detect gaps. //! It is not stored across process restarts. //! -//! `seq` is not a causal clock. //! With concurrent publishers, it shows event construction order, not a guaranteed order of runtime effects or subscriber callbacks. //! //! ## Fields @@ -36,8 +35,8 @@ //! `timeout_ms`, `delay_ms`, and `duration_ms` use whole milliseconds. //! Values above `u32::MAX` milliseconds are stored as `u32::MAX`. //! -//! Treat `reason` as human-readable text unless the event points to a constant in [`reasons`](crate::reasons). -//! Use [`EventKind::as_label`] for a stable event label. +//! Treat `reason` a readable text unless the event points to a constant in [`reasons`](crate::reasons). +//! > Use [`EventKind::as_label`] for a stable event label. //! //! ## Example //! @@ -63,7 +62,9 @@ use std::time::{Duration, SystemTime}; use crate::identity::TaskId; -/// Process-local counter for `seq` values. It wraps after `2^64` allocations. +/// Process-local counter for `seq` values. +/// +/// It wraps after `2^64` allocations. static EVENT_SEQ: AtomicU64 = AtomicU64::new(1); /// Describes what happened in the runtime. @@ -83,6 +84,17 @@ pub enum EventKind { /// - `seq`: process-local sequence SubscriberPanicked, + /// An internal runtime component failed. + /// + /// This includes a caught panic or a worker that did not join cleanly. + /// + /// Sets: + /// - `task`: runtime component name + /// - `reason`: diagnostic failure details + /// - `at`: wall-clock timestamp + /// - `seq`: process-local sequence + RuntimeFailure, + /// An event was lost because a subscriber path fell behind or closed. /// /// Sets: @@ -237,6 +249,7 @@ pub enum EventKind { /// Sets: /// - `id`: task run identity /// - `task`: task name + /// - `reason`: [`FORCE_TERMINATED_AFTER_GRACE`](crate::reasons::FORCE_TERMINATED_AFTER_GRACE) when the actor was force-aborted or `None` /// - `at`: wall-clock timestamp /// - `seq`: process-local sequence TaskRemoved, @@ -298,7 +311,7 @@ pub enum EventKind { /// Sets: /// - `task`: slot name /// - `id`: the submission's [`TaskId`] - /// - `reason`: a human-readable admission summary, e.g. `admission=Queue status=admitting` or + /// - `reason`: a readable admission summary, e.g. `admission=Queue status=admitting` or /// `started_from_queue depth=N` (exact text is diagnostic, not a stable contract) ControllerSubmitted, @@ -308,7 +321,7 @@ pub enum EventKind { /// /// Sets: /// - `task`: slot name - /// - `reason`: human-readable transition text; it is not a stable machine contract + /// - `reason`: readable transition text; it is not a stable machine contract ControllerSlotTransition, } @@ -318,6 +331,15 @@ impl EventKind { /// The label is the snake_case form of the variant name. /// Use it as an event name in tracing or as a metrics label value. /// + /// ```text + /// EventKind::TaskStarting + /// │ as_label() + /// ▼ + /// "task_starting" + /// ├── log field: event="task_starting" + /// └── metric label: event="task_starting" + /// ``` + /// /// ```rust /// use taskvisor::EventKind; /// @@ -328,6 +350,7 @@ impl EventKind { pub fn as_label(&self) -> &'static str { match self { EventKind::SubscriberPanicked => "subscriber_panicked", + EventKind::RuntimeFailure => "runtime_failure", EventKind::SubscriberOverflow => "subscriber_overflow", EventKind::ShutdownRequested => "shutdown_requested", EventKind::AllStoppedWithinGrace => "all_stopped_within_grace", @@ -391,13 +414,13 @@ impl BackoffSource { /// - other optional fields are set depending on the [`EventKind`] /// /// Fields are public for reading. Create an event with [`Event::new`] and add optional values with the `with_*` builders. -/// Use `..` when matching the struct because more fields may be added. +/// > Use `..` when matching the struct because more fields may be added. /// /// # Also /// /// - [`EventKind`] - event classification /// - [`Subscribe`](crate::Subscribe) - user-defined event handler trait -/// - `LogWriter` (feature = `logging`) - built-in human-readable event printer +/// - `LogWriter` (feature = `logging`) - built-in readable event printer #[derive(Clone)] #[non_exhaustive] pub struct Event { @@ -459,7 +482,7 @@ impl Event { } } - /// Attaches a human-readable reason. + /// Attaches a readable reason. #[inline] #[must_use] pub fn with_reason(mut self, reason: impl Into>) -> Self { @@ -569,13 +592,24 @@ impl Event { .with_reason(info) } + /// Creates an internal runtime failure event. + #[inline] + #[must_use] + pub fn runtime_failure(component: impl Into>, reason: impl Into>) -> Self { + Event::new(EventKind::RuntimeFailure) + .with_task(component) + .with_reason(reason) + } + /// Returns `true` for internal diagnostic events. #[inline] #[must_use] pub fn is_internal_diagnostic(&self) -> bool { matches!( self.kind, - EventKind::SubscriberOverflow | EventKind::SubscriberPanicked + EventKind::SubscriberOverflow + | EventKind::SubscriberPanicked + | EventKind::RuntimeFailure ) } } @@ -631,6 +665,7 @@ mod tests { fn event_kind_labels_are_stable() { let cases = [ (EventKind::SubscriberPanicked, "subscriber_panicked"), + (EventKind::RuntimeFailure, "runtime_failure"), (EventKind::SubscriberOverflow, "subscriber_overflow"), (EventKind::ShutdownRequested, "shutdown_requested"), (EventKind::AllStoppedWithinGrace, "all_stopped_within_grace"), @@ -714,15 +749,19 @@ mod tests { } #[test] - fn is_internal_diagnostic_covers_both_variants() { - for kind in [EventKind::SubscriberOverflow, EventKind::SubscriberPanicked] { + fn is_internal_diagnostic_covers_all_variants() { + for kind in [ + EventKind::SubscriberOverflow, + EventKind::SubscriberPanicked, + EventKind::RuntimeFailure, + ] { assert!(Event::new(kind).is_internal_diagnostic(), "{kind:?}"); } assert!(!Event::new(EventKind::TaskStarting).is_internal_diagnostic()); } #[test] - fn subscriber_factories_set_kind_task_and_reason() { + fn diagnostic_factories_set_kind_task_and_reason() { let overflow = Event::subscriber_overflow("my-sub", "full"); assert_eq!(overflow.kind, EventKind::SubscriberOverflow); assert_eq!( @@ -740,6 +779,14 @@ mod tests { assert_eq!(panicked.kind, EventKind::SubscriberPanicked); assert_eq!(panicked.task.as_deref(), Some("my-sub")); assert_eq!(panicked.reason.as_deref(), Some("boom")); + + let runtime_failure = Event::runtime_failure("registry", "listener join failed"); + assert_eq!(runtime_failure.kind, EventKind::RuntimeFailure); + assert_eq!(runtime_failure.task.as_deref(), Some("registry")); + assert_eq!( + runtime_failure.reason.as_deref(), + Some("listener join failed") + ); } #[test] diff --git a/src/events/mod.rs b/src/events/mod.rs index 7a3bef1..6bc6a16 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -1,6 +1,7 @@ //! # Runtime events //! -//! Events describe what Taskvisor is doing. Use them for logs, metrics, dashboards, alerts, and tests. +//! Events describe what Taskvisor is doing. +//! Use them for logs, metrics, dashboards, alerts, and tests. //! //! | Type | Role | //! |-------------------|--------------------------------------------| diff --git a/src/identity.rs b/src/identity.rs index 851d06e..ae89aa6 100644 --- a/src/identity.rs +++ b/src/identity.rs @@ -6,16 +6,53 @@ //! Controller `submit*` methods return it after queueing, before slot admission. //! The same ID therefore also identifies a controller submission that is rejected without running. //! +//! ## One identity across the lifecycle +//! +//! Every submission accepted into Taskvisor's command path has one `TaskId`. +//! It is allocated before the first admission decision, then passed unchanged through every stage that the submission reaches: +//! +//! ```text +//! allocate TaskId A +//! │ +//! ├── direct add ───────────────────────────────────────────────┐ +//! │ │ +//! └── optional controller ─► queue[A] ─► slot admission[A] ─────┤ +//! │ │ +//! └──► rejected[A] (no actor) │ +//! ▼ +//! registry admission[A] +//! │ │ +//! rejected[A] ◄──────────┘ └──► registry[A] +//! (no actor) │ +//! ▼ +//! actor[A] ─► attempt 1, 2, ... +//! ▼ +//! cleanup[A] +//! ``` +//! +//! No new `TaskId` is allocated at admission or between retry attempts. +//! Queue management, registry membership, the actor, and completion tracking carry the same `A`. +//! Related lifecycle events expose it so callers can correlate cancellation, logs, and metrics. +//! //! ## TaskId vs Name vs Slot //! -//! | Concept | Owned by | Meaning | -//! |------------------|------------|-------------------------------------------------------------| -//! | [`TaskId`] | Taskvisor | identity of one submission and its run, if admitted | -//! | task name | task | human label used for logs, metrics, and registry uniqueness | -//! | controller slot | controller | admission key for "one at a time" scheduling | +//! | Concept | Owned by | Meaning | +//! |------------------|------------|-------------------------------------------------------| +//! | [`TaskId`] | taskvisor | identity of one submission across its full lifecycle | +//! | task name | task | label used for logs, metrics, and registry uniqueness | +//! | controller slot | controller | admission key for "one at a time" scheduling | +//! +//! A task name is unique only while its registry entry exists. +//! After terminal cleanup removes that entry, a later submission may reuse the same name. +//! Reusing a name does not reuse its identity: +//! +//! ```text +//! first submission: name = "worker", slot = "jobs", TaskId = A +//! terminal cleanup: removes A and releases the name "worker" +//! later submission: name = "worker", slot = "jobs", TaskId = B (B != A) +//! ``` //! -//! A task name may be reused after the old task is removed. -//! A `TaskId` is allocated from a process-local `u64` counter. +//! Each `TaskId` is allocated from a process-local `u64` counter. //! //! The counter is not stored across process restarts, and a `TaskId` is not a UUID. //! If external systems need a persistent identity, store their own ID next to this one. @@ -32,17 +69,15 @@ use std::sync::atomic::{AtomicU64, Ordering}; /// reaching that limit is outside normal operation. static TASK_ID_SEQ: AtomicU64 = AtomicU64::new(1); -/// Opaque process-local identity of one task submission and its run, if admitted. +/// Opaque process-local identity of one task submission. /// -/// A `TaskId` is allocated by Taskvisor and carried on submission and lifecycle [`Event`](crate::Event)s. -/// A controller submission keeps the same id from queueing through admission, execution, and terminal cleanup. +/// Taskvisor allocates it once. With the `controller` feature, this happens before admission so queued work can already be addressed and correlated. +/// If admitted, the same value becomes the registry key and remains unchanged through all attempts and terminal cleanup. /// /// Rejected work keeps its id even though no task body ran. /// -/// Use it to: -/// - cancel or remove a task by identity, -/// - correlate events for the same submission and run, -/// - keep stable log and metrics references within one process run. +/// Pass the returned value to cancellation and removal operations. +/// The same value correlates the submission's lifecycle [`Event`](crate::Event)s, completion, logs, and metrics within the current process. /// /// Do not parse its display output. /// Use [`get`](Self::get) when you need the numeric value. @@ -51,7 +86,7 @@ static TASK_ID_SEQ: AtomicU64 = AtomicU64::new(1); pub struct TaskId(u64); impl TaskId { - /// Allocates the next submission/run identity. + /// Allocates the next submission identity. /// /// Internal only: Taskvisor owns identity allocation across direct runtime registration and controller pre-admission. #[inline] diff --git a/src/policies/backoff.rs b/src/policies/backoff.rs index e1767da..3a32e73 100644 --- a/src/policies/backoff.rs +++ b/src/policies/backoff.rs @@ -47,7 +47,7 @@ //! ``` //! //! Named constructors: [`constant`](BackoffPolicy::constant) and [`exponential`](BackoffPolicy::exponential). -//! For a custom growth factor use [`new`](BackoffPolicy::new). +//! > For a custom growth factor use [`new`](BackoffPolicy::new). use std::time::Duration; @@ -287,7 +287,7 @@ impl BackoffPolicy { /// For [`JitterPolicy::None`], [`JitterPolicy::Full`], and [`JitterPolicy::Equal`], the jittered delay never exceeds the base. /// [`JitterPolicy::RandomizedBand`] uses a wider band and may return a delay larger than the base. /// - /// This method has no memory. One result does not change later results. + /// > This method has no memory. One result does not change later results. /// /// After jitter, Taskvisor applies the user floor. /// For a non-zero base, it also applies a `1ms` safety floor, capped at `max`. diff --git a/src/policies/jitter.rs b/src/policies/jitter.rs index 9311f52..12b507f 100644 --- a/src/policies/jitter.rs +++ b/src/policies/jitter.rs @@ -86,8 +86,7 @@ impl JitterPolicy { } } - /// Chooses a uniform random delay in - /// `[lower, min(upper_seed × 3, max)]` for [`RandomizedBand`](Self::RandomizedBand). + /// Chooses a uniform random delay in `[lower, min(upper_seed × 3, max)]` for [`RandomizedBand`](Self::RandomizedBand). /// /// This method first clamps `lower` to `max`. /// For any other policy, it applies that policy to the clamped `lower`; `upper_seed` is unused. diff --git a/src/policies/mod.rs b/src/policies/mod.rs index 91b9575..7393355 100644 --- a/src/policies/mod.rs +++ b/src/policies/mod.rs @@ -11,16 +11,14 @@ //! ├── Ok(()) ────────────────► RestartPolicy //! │ ├── stop //! │ └── wait for Always.interval, then run again -//! │ //! ├── Fail / Timeout ────────► RestartPolicy //! │ ├── stop //! │ └── BackoffPolicy ──► wait ──► run again -//! │ //! └── Fatal / Canceled ──────► stop //! ``` //! //! A [`TaskSpec`](crate::TaskSpec) can set these values for one task. -//! If it does not, it inherits them from [`TaskDefaults`](crate::TaskDefaults). +//! > If it does not, it inherits them from [`TaskDefaults`](crate::TaskDefaults). //! //! ## Default behavior //! diff --git a/src/policies/restart.rs b/src/policies/restart.rs index b3cfb0e..27c67e6 100644 --- a/src/policies/restart.rs +++ b/src/policies/restart.rs @@ -19,7 +19,8 @@ /// Decides whether a task starts another attempt. /// -/// This policy decides restart eligibility. Retry timing is controlled by [`BackoffPolicy`](crate::BackoffPolicy). +/// This policy decides restart eligibility. +/// Retry timing is controlled by [`BackoffPolicy`](crate::BackoffPolicy). /// Include a wildcard arm when matching because new policies may be added. /// /// # Also diff --git a/src/prelude.rs b/src/prelude.rs index ac4b364..868b847 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -1,13 +1,17 @@ //! # Common imports //! -//! Import the prelude when building ordinary Taskvisor applications: -//! //! ```rust //! use taskvisor::prelude::*; //! ``` //! -//! The prelude includes the main runtime, task, policy, event, subscriber, error, and identity types. -//! It does not change behavior or enable features. +//! The prelude includes: +//! - identity types +//! - main runtime +//! - subscriber +//! - policy +//! - event +//! - error +//! - task /// Core supervisor runtime. pub use crate::core::{ diff --git a/src/reasons.rs b/src/reasons.rs index af451e2..9059d1b 100644 --- a/src/reasons.rs +++ b/src/reasons.rs @@ -1,8 +1,8 @@ //! # Stable `reason` values //! -//! Some events and outcomes contain a `reason` text. -//! Only the values and prefixes in this module are stable for machine matching. -//! Other reason text may change between releases. +//! Some events and outcomes have a `reason` string. +//! Code can check the stable values and prefixes in this module. +//! Other reason strings are not stable and may change between releases. //! //! Exact value: //! @@ -21,20 +21,20 @@ //! //! ## Where each value appears //! -//! | Constant | Used by | Meaning | -//! |------------------------------|--------------------------------------------------------|----------------------------------------| -//! | [`POLICY_EXHAUSTED_SUCCESS`] | `ActorExhausted` | Normal one-shot finish. | -//! | [`TASK_RETURNED_CANCELED`] | `ActorExhausted` | The task stopped itself. | -//! | [`MAX_RETRIES_EXCEEDED`] | `ActorExhausted` (prefix) | Retry limit reached. | -//! | [`ALREADY_EXISTS`] | `TaskAddFailed`, `TaskOutcome::Rejected` | A task with this name already exists. | -//! | [`BATCH_REJECTED`] | `TaskAddFailed` | Another item rejected the whole batch. | -//! | [`QUEUE_FULL`] | `ControllerRejected`, `TaskOutcome::Rejected` (prefix) | The slot queue is full. | -//! | [`REMOVED_FROM_QUEUE`] | `ControllerRejected`, `TaskOutcome::Rejected` | A queued task was removed. | -//! | [`SUPERSEDED_BY_REPLACE`] | `ControllerRejected`, `TaskOutcome::Rejected` | A newer `Replace` task took its place. | -//! | [`CONTROLLER_SHUTTING_DOWN`] | `ControllerRejected`, `TaskOutcome::Rejected` | The controller is shutting down. | +//! | Constant | Used by | Meaning | +//! |--------------------------------------|--------------------------------------------------------|----------------------------------------| +//! | [`POLICY_EXHAUSTED_SUCCESS`] | `ActorExhausted` | Normal one-shot finish. | +//! | [`TASK_RETURNED_CANCELED`] | `ActorExhausted` | The task stopped itself. | +//! | [`MAX_RETRIES_EXCEEDED`] | `ActorExhausted` (prefix) | Retry limit reached. | +//! | [`ALREADY_EXISTS`] | `TaskAddFailed`, `TaskOutcome::Rejected` | A task with this name already exists. | +//! | [`BATCH_REJECTED`] | `TaskAddFailed` | Another item rejected the whole batch. | +//! | [`QUEUE_FULL`] | `ControllerRejected`, `TaskOutcome::Rejected` (prefix) | The slot queue is full. | +//! | [`DROP_IF_RUNNING`] | `ControllerRejected`, `TaskOutcome::Rejected` (prefix) | A busy slot rejected new work. | +//! | [`REMOVED_FROM_QUEUE`] | `ControllerRejected`, `TaskOutcome::Rejected` | A queued task was removed. | +//! | [`SUPERSEDED_BY_REPLACE`] | `ControllerRejected`, `TaskOutcome::Rejected` | A newer `Replace` task took its place. | +//! | [`CONTROLLER_SHUTTING_DOWN`] | `ControllerRejected`, `TaskOutcome::Rejected` | The controller is shutting down. | +//! | [`FORCE_TERMINATED_AFTER_GRACE`] | `TaskRemoved` | The actor was force-aborted. | //! -//! Controller rejection values are used only with the `controller` feature. -//! The constants themselves are always available. /// `ActorExhausted` reason: the task finished successfully under a `Never`/`OnFailure` policy. /// This is not an error. @@ -59,6 +59,15 @@ pub const SUPERSEDED_BY_REPLACE: &str = "superseded_by_replace"; /// Rejection reason: the controller is shutting down. pub const CONTROLLER_SHUTTING_DOWN: &str = "controller_shutting_down"; +/// `ControllerRejected` and `TaskOutcome::Rejected` reason **prefix**: `DropIfRunning` rejected a submission because its slot was busy. +/// +/// The full reason is `dropped: slot busy ()`. +/// Match with `starts_with`, not equality. +pub const DROP_IF_RUNNING: &str = "dropped: slot busy"; + +/// `TaskRemoved` reason: the actor did not stop within its grace window and was force-aborted. +pub const FORCE_TERMINATED_AFTER_GRACE: &str = "force_terminated_after_grace"; + /// `ActorExhausted` reason **prefix**: the task stopped after it used all retry attempts. /// /// The full reason is `max_retries_exceeded(/): `. diff --git a/src/subscribers/embedded/log.rs b/src/subscribers/embedded/log.rs index 3627486..6c34f0a 100644 --- a/src/subscribers/embedded/log.rs +++ b/src/subscribers/embedded/log.rs @@ -35,7 +35,7 @@ use crate::events::{Event, EventKind}; use crate::subscribers::Subscribe; -/// Prints human-readable events to standard output. +/// Prints a readable events to standard output. /// /// Implements [`Subscribe`] and prints `[seq] [event-type] key=value ...` with relevant metadata. /// Output is intended for people and is not a stable machine-readable format. @@ -141,6 +141,13 @@ impl LogWriter { or(e.reason.as_deref(), "unknown") ); } + EventKind::RuntimeFailure => { + println!( + "{head} component={} reason=\"{}\"", + or(e.task.as_deref(), "none"), + or(e.reason.as_deref(), "unknown") + ); + } // Terminals. EventKind::ActorExhausted => { diff --git a/src/subscribers/embedded/tracing.rs b/src/subscribers/embedded/tracing.rs index 97c0053..63ac1fc 100644 --- a/src/subscribers/embedded/tracing.rs +++ b/src/subscribers/embedded/tracing.rs @@ -1,7 +1,6 @@ //! # Bridge to `tracing` //! //! [`TracingBridge`] converts every event it receives into one structured [`tracing`] event. -//! Subscriber delivery is best-effort, so this bridge does not make event delivery reliable. //! //! Each tracing event uses target `taskvisor` and contains: //! - a level based on the event severity (see [`TracingBridge`]), @@ -30,7 +29,6 @@ use crate::subscribers::Subscribe; /// Level mapping: /// - `ERROR`: task failed, actor dead, subscriber panicked. /// - `WARN`: timeout, grace exceeded, subscriber overflow, add failed, controller rejected. -/// Also `actor_exhausted` when the reason starts with `max_retries_exceeded`: the task permanently gave up. /// - `INFO`: lifecycle milestones (stopped, canceled, added, removed, shutdown, submitted, and `actor_exhausted` for every other reason). /// - `DEBUG`: chatty events (starting, backoff, add/remove requests, slot transitions). /// @@ -45,9 +43,10 @@ pub struct TracingBridge; /// Maps an event to a tracing level. fn level_for(e: &Event) -> Level { match e.kind { - EventKind::TaskFailed | EventKind::ActorDead | EventKind::SubscriberPanicked => { - Level::ERROR - } + EventKind::TaskFailed + | EventKind::ActorDead + | EventKind::SubscriberPanicked + | EventKind::RuntimeFailure => Level::ERROR, EventKind::TimeoutHit | EventKind::GraceExceeded @@ -85,7 +84,6 @@ fn level_for(e: &Event) -> Level { impl Subscribe for TracingBridge { fn on_event(&self, e: &Event) { - // `tracing::event!` needs a const level, one macro call per level. macro_rules! emit { ($level:expr) => { tracing::event!( diff --git a/src/subscribers/subscriber.rs b/src/subscribers/subscriber.rs index 477ac09..9909d71 100644 --- a/src/subscribers/subscriber.rs +++ b/src/subscribers/subscriber.rs @@ -94,7 +94,7 @@ pub trait Subscribe: Send + Sync + 'static { /// The return type guarantees that the queue can hold at least one event. /// If the queue is full, Taskvisor drops the new ordinary event for this subscriber and tries to report `SubscriberOverflow`. /// - /// Default: `1024`. + /// > Default: `1024`. fn queue_capacity(&self) -> NonZeroUsize { DEFAULT_QUEUE_CAPACITY } diff --git a/src/subscribers/subscriber_set.rs b/src/subscribers/subscriber_set.rs index 2703f7a..d103286 100644 --- a/src/subscribers/subscriber_set.rs +++ b/src/subscribers/subscriber_set.rs @@ -298,7 +298,7 @@ impl SubscriberSet { } } -/// Extracts a human-readable message from a panic payload. +/// Extracts a readable message from a panic payload. fn extract_panic_info(panic_err: &Box) -> String { let any = &**panic_err; if let Some(msg) = any.downcast_ref::<&'static str>() { @@ -850,7 +850,11 @@ mod tests { #[tokio::test] async fn panic_on_internal_diagnostic_does_not_republish() { - for diagnostic in [EventKind::SubscriberPanicked, EventKind::SubscriberOverflow] { + for diagnostic in [ + EventKind::SubscriberPanicked, + EventKind::SubscriberOverflow, + EventKind::RuntimeFailure, + ] { let bus = Bus::new(64); let mut rx = bus.subscribe(); let set = SubscriberSet::new(vec![PanicSub::new()], bus.clone()); diff --git a/src/tasks/context.rs b/src/tasks/context.rs index 1388f5f..98086f8 100644 --- a/src/tasks/context.rs +++ b/src/tasks/context.rs @@ -16,10 +16,12 @@ use crate::error::TaskError; /// /// ```text /// remove / cancel / shutdown -/// v -/// TaskContext cancelled -/// v -/// task returns TaskError::Canceled +/// ▼ +/// TaskContext becomes cancelled +/// ├── cancelled() / is_cancelled() +/// │ └──► task stops and returns TaskError::Canceled +/// └── run_until_cancelled(future) +/// └──► Err(TaskError::Canceled) /// ``` /// /// Await [`cancelled`](Self::cancelled), check [`is_cancelled`](Self::is_cancelled), or wrap a cancellation-safe future in [`run_until_cancelled`](Self::run_until_cancelled). @@ -95,7 +97,7 @@ impl TaskContext { /// If the context is already cancelled, `fut` is not polled. /// When cancellation wins, `fut` is dropped. /// - /// Use this method only with futures that are safe to cancel by dropping. + /// > Use this method only with futures that are safe to cancel by dropping. /// /// This is a short form of `tokio::select!` for common worker loops: /// diff --git a/src/tasks/spec.rs b/src/tasks/spec.rs index bff5a46..ec502cb 100644 --- a/src/tasks/spec.rs +++ b/src/tasks/spec.rs @@ -23,12 +23,18 @@ fn normalize_timeout(timeout: Option) -> Option { /// A `with_*` method always sets an explicit value and wins over the default. /// /// ```text -/// TaskSpec TaskDefaults -/// restart = OnFailure restart = OnFailure -/// timeout = inherit + timeout = 30s -/// | -/// v -/// resolved at admission: restart = OnFailure, timeout = 30s +/// Resolve each field at registry admission: +/// +/// explicit value +/// TaskSpec.restart = Never ─────────────────────────────────► restart = Never +/// TaskDefaults.restart = OnFailure ─────────────────────────────────► not selected +/// +/// inherited value +/// TaskSpec.timeout = inherit ──► TaskDefaults.timeout = 30s ──► timeout = 30s +/// +/// ResolvedTaskSpec +/// ├── restart = Never +/// └── timeout = 30s /// ``` /// /// | Constructor | Restart setting | Other settings | @@ -53,7 +59,8 @@ fn normalize_timeout(timeout: Option) -> Option { /// .with_max_retries(NonZeroU32::new(5).unwrap()); /// ``` /// -/// `max_retries = 5` allows the first failed attempt plus five retries. +/// > `max_retries = 5` allows the first failed attempt plus five retries. +/// /// A successful attempt resets this count; an `Always` task may still have more than six attempts over its full lifetime. /// /// ## See Also @@ -63,17 +70,27 @@ fn normalize_timeout(timeout: Option) -> Option { #[derive(Clone)] #[must_use] pub struct TaskSpec { + /// Restart policy selected explicitly or inherited from [`TaskDefaults`]. restart: Override, + /// Backoff policy selected explicitly or inherited from [`TaskDefaults`]. backoff: Override, + /// Per-attempt timeout; `Set(None)` explicitly disables an inherited timeout. timeout: Override>, + /// Retry limit; `Set(None)` explicitly selects unlimited retries. max_retries: Override>, + /// Task object reused across all attempts started from this spec. task: TaskRef, } +/// Origin of one `TaskSpec` setting before admission resolves defaults. +/// +/// For `Override>`, `Inherit` and `Set(None)` are intentionally distinct. #[derive(Clone, Copy, Debug)] enum Override { + /// Resolve the field from [`TaskDefaults`] at registry admission. Inherit, + /// Use this explicit value instead of the corresponding default. Set(T), } @@ -134,7 +151,7 @@ impl TaskSpec { /// Creates a spec that inherits every execution setting. /// /// The supervisor resolves restart, backoff, timeout, and retry limit from its [`TaskDefaults`] when it accepts the task. - /// A later `with_*` call sets that one field explicitly. + /// > A later `with_*` call sets that one field explicitly. pub fn from_defaults(task: TaskRef) -> Self { Self { restart: Override::Inherit, @@ -147,12 +164,13 @@ impl TaskSpec { /// Creates a spec with explicit main settings. /// - /// Prefer the named constructors for common cases: - /// [`once`](Self::once), [`restartable`](Self::restartable), [`periodic`](Self::periodic). + /// Prefer the named constructors for common cases: [`once`](Self::once), [`restartable`](Self::restartable), [`periodic`](Self::periodic). /// /// `timeout` accepts a [`Duration`] or `Option`. /// `None` and zero disable the attempt timeout. - /// The retry limit is set to unlimited; change it with [`with_max_retries`](Self::with_max_retries). + /// + /// The retry limit is set to unlimited. + /// > _change it with [`with_max_retries`](Self::with_max_retries)_. pub fn new( task: TaskRef, restart: RestartPolicy, @@ -171,7 +189,7 @@ impl TaskSpec { /// Creates a one-shot task that never restarts. /// /// Backoff, timeout, and retry limit are inherited from [`TaskDefaults`]. - /// Override them with the matching `with_*` methods. + /// > Override them with the matching `with_*` methods. pub fn once(task: TaskRef) -> Self { Self { restart: Override::Set(RestartPolicy::Never), @@ -185,7 +203,7 @@ impl TaskSpec { /// Creates a task that restarts after retryable failures. /// /// Success, fatal failure, and cancellation stop the task. - /// Backoff, timeout, and retry limit are inherited from [`TaskDefaults`]. + /// > Backoff, timeout, and retry limit are inherited from [`TaskDefaults`]. pub fn restartable(task: TaskRef) -> Self { Self { restart: Override::Set(RestartPolicy::OnFailure), @@ -199,14 +217,15 @@ impl TaskSpec { /// Creates a task that runs again after each success. /// /// After success, the supervisor waits `every` before the next attempt. - /// A zero value means no configured interval; a small internal guard still prevents an instant task from creating a hot loop. + /// A zero value means no configured interval; + /// > _a small internal guard still prevents an instant task from creating a hot loop._ /// /// Retryable failures use the backoff policy, not `every`. /// A retry limit can stop the task after repeated failures. /// Fatal failure and cancellation always stop it. /// /// The interval starts after an attempt completes. - /// This is not a wall-clock schedule such as "daily at 03:00". + /// > *This is not a wall-clock schedule such as "daily at 03:00".* /// /// ```rust /// use std::time::Duration; @@ -309,6 +328,7 @@ impl TaskSpec { /// /// Pass a [`NonZeroU32`] to set a limit. /// Pass `None` for unlimited retries, including when [`TaskDefaults`] has a limit. + /// /// A success resets the count. #[doc(alias = "retry limit")] #[doc(alias = "retry budget")] @@ -320,8 +340,9 @@ impl TaskSpec { /// Sets a retry limit from a raw integer. /// /// # Errors + /// /// Returns [`ConfigError::Zero`] when `max_retries` is zero. - /// Use [`with_max_retries`](Self::with_max_retries) with `None` for unlimited retries. + /// > Use [`with_max_retries`](Self::with_max_retries) with `None` for unlimited retries. pub fn try_with_max_retries(self, max_retries: u32) -> Result { let max_retries = NonZeroU32::new(max_retries).ok_or(ConfigError::Zero { field: "max_retries", diff --git a/src/tasks/task.rs b/src/tasks/task.rs index be43e05..3e53e09 100644 --- a/src/tasks/task.rs +++ b/src/tasks/task.rs @@ -12,16 +12,16 @@ pub type BoxTaskFuture = Pin> + Se pub type TaskRef = Arc; /// Async work managed by a [`Supervisor`](crate::Supervisor). -/// -/// Use [`TaskFn`](crate::TaskFn) unless you need a custom task type. +/// > Use [`TaskFn`](crate::TaskFn) unless you need a custom task type. /// /// ## Attempt Contract /// /// The supervisor calls [`spawn`](Task::spawn) once for every attempt. -/// `spawn` **must return a new future on every call**. -/// Never store and reuse a future. +/// > `spawn` **must return a new future on every call**. /// -/// The task object may keep shared state between attempts. +/// The same task object is reused for every attempt. +/// State stored in its fields can therefore survive retries. +/// Each returned future is attempt-local; its local values do not carry over to the next attempt. /// /// ```text /// spawn(ctx) -> attempt 1 -> retry delay -> spawn(ctx) -> attempt 2 @@ -31,11 +31,11 @@ pub type TaskRef = Arc; /// /// Long-running tasks should listen to [`TaskContext::cancelled`] or use [`TaskContext::run_until_cancelled`]. /// Return [`TaskError::Canceled`] when the task stops for that reason. -/// This is a normal stop and is never retried. +/// > **This is a normal stop and is never retried.** /// /// Short tasks may ignore cancellation if they finish quickly. /// During shutdown, a task that does not stop before the grace period is aborted. -/// Dropping its future does not roll back external side effects. +/// Dropping its future does not rollback external side effects. /// /// | Result | Meaning | Can restart? | /// |-------------------------|-------------------|-------------------------------------------------------------------| @@ -46,11 +46,11 @@ pub type TaskRef = Arc; /// | [`TaskError::Fatal`] | Permanent failure | No | /// /// A panic in `spawn` or in the returned future is caught and treated as a retryable failure. -/// Do not use panic for normal error handling. +/// > **Do not use panic for normal error handling.** /// /// ## See Also /// -/// - For the closure-based implementation see [`TaskFn`](crate::TaskFn). +/// - For the closure-based implementation and a shared-state example, see [`TaskFn`](crate::TaskFn). /// - To configure restart, backoff, and timeout see [`TaskSpec`](crate::TaskSpec). pub trait Task: Send + Sync + 'static { /// Returns the stable name used for registration and observability. diff --git a/tests/controller.rs b/tests/controller.rs index b7bbfe7..bd8e331 100644 --- a/tests/controller.rs +++ b/tests/controller.rs @@ -496,10 +496,9 @@ async fn drop_if_running_rejects_busy_submission_without_starting_it() { event.task.as_deref() == Some("s") && event.kind == EventKind::ControllerRejected && event.id == Some(rejected_id) - && event - .reason - .as_deref() - .is_some_and(|reason| reason.contains("dropped: slot busy")) + && event.reason.as_deref().is_some_and(|reason| { + reason.starts_with(taskvisor::reasons::DROP_IF_RUNNING) + }) }) }) .await diff --git a/tests/identity.rs b/tests/identity.rs index aa0bf44..c009bfc 100644 --- a/tests/identity.rs +++ b/tests/identity.rs @@ -392,7 +392,8 @@ async fn individually_removed_stuck_task_is_force_aborted_after_grace() { events.iter().any(|event| { event.id == Some(id) && event.kind == EventKind::TaskRemoved - && event.reason.as_deref() == Some("force_terminated_after_grace") + && event.reason.as_deref() + == Some(taskvisor::reasons::FORCE_TERMINATED_AFTER_GRACE) }) }) .await,