diff --git a/Cargo.toml b/Cargo.toml index 5ac06d6..25d86da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,13 +4,13 @@ version = "0.6.0" edition = "2024" rust-version = "1.90.0" -description = "Task supervisor for Tokio: restarts background tasks on failure with exponential backoff and jitter, graceful shutdown, and lifecycle events" +description = "In-process Tokio task supervisor with retries, reliable outcomes, and per-key queue/replace/reject admission" repository = "https://github.com/soltiHQ/taskvisor" documentation = "https://docs.rs/taskvisor" license = "Apache-2.0" readme = "README.md" -keywords = ["supervisor", "tokio", "retry", "restart", "backoff"] +keywords = ["tokio", "supervisor", "keyed", "admission-control", "retry"] categories = ["asynchronous", "concurrency"] exclude = [ @@ -23,7 +23,7 @@ exclude = [ [features] controller = ["dep:dashmap"] -default = [] +default = ["controller"] logging = [] test-util = [] tracing = ["dep:tracing"] @@ -56,6 +56,10 @@ required-features = ["controller"] name = "admission" required-features = ["controller"] +[[example]] +name = "tenant_sync" +required-features = ["controller"] + [[example]] name = "tracing" required-features = ["tracing"] diff --git a/README.md b/README.md index 8765f9f..eb9a1ae 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,13 @@ [![Minimum Rust 1.90](https://img.shields.io/badge/rust-1.90%2B-orange.svg)](https://rust-lang.org) [![Apache 2.0](https://img.shields.io/badge/license-Apache2.0-blue.svg)](./LICENSE) -> In-process task supervision for Tokio - from long-running workers to one-shot jobs. +> Queue, replace, or reject Tokio work independently per key—with supervised lifecycles and reliable outcomes. -Write ordinary async code. Taskvisor adds restart and backoff, cooperative shutdown, dynamic task management, typed lifecycle events, reliable final outcomes, and optional per-slot admission control. +Taskvisor is an in-process task supervisor for Tokio services. Write ordinary async code. -[Quick start](#quick-start) - [Examples](#examples) - [Production limits](#production-limits) +Taskvisor adds restart and backoff, cooperative shutdown, dynamic task management, typed lifecycle events, and reliable final outcomes. Its controller gives each key one owner and resolves conflicts by policy: queue, replace, or reject. + +| [Quick start](#quick-start) | [Examples](#examples) | [Production limits](#production-limits) | ## The loop you stop writing @@ -41,6 +43,12 @@ supervisor Taskvisor owns the lifecycle machinery. Your task keeps the application logic. +## Check the fit first + +If you have a small fixed set of workers and only need retry plus graceful shutdown, Taskvisor is probably more lifecycle than you need. Start with `JoinSet` or `TaskTracker`, `CancellationToken`, and a retry crate such as `backon`. + +> Taskvisor is aimed at services where work is added dynamically or contends per key, and needs one in-process contract for admission, supervised cancellation, and reliable final outcomes. + ## Quick start ```toml @@ -91,47 +99,84 @@ attempt 3 Every attempt gets a fresh future. A retryable failure follows the configured backoff; a successful `restartable` task stops. `Supervisor::run` returns only after lifecycle cleanup finishes. -For a resident worker that runs until Ctrl+C, see [worker.rs](examples/worker.rs). For a reconnecting queue consumer, see [queue_consumer.rs](examples/queue_consumer.rs). +For a resident worker that runs until Ctrl+C, see [worker.rs](examples/worker.rs). + +For a queue consumer that retries a failed broker connection, see [queue_consumer.rs](examples/queue_consumer.rs). + +## One key, one owner + +Retry alone does not resolve conflicting work for the same resource. The controller gives each key one owner while allowing different keys to run concurrently: + +```text +sync tenant-42/rev-1 is running +sync tenant-42/rev-2 arrives ──► retire rev-1, then run rev-2 +sync tenant-17/rev-1 arrives ──► run independently +``` + +The controller API is enabled by default. Configuring a controller at runtime remains explicit: + +```toml +taskvisor = "0.6" +``` + +```rust,ignore +let request = ControllerSpec::replace(TaskSpec::once(sync_tenant_42_rev_2)) + .with_slot("tenant-42"); + +let (_id, waiter) = handle.submit_and_watch(request).await?; +let outcome = waiter.wait().await?; +``` + +- `DropIfRunning` rejects the conflict without starting it. See [admission.rs](examples/admission.rs); +- `Replace` makes the newest submission the next owner only after the old owner finishes cancellation cleanup. See the runnable [tenant-42 conflict example](examples/tenant_sync.rs); +- `Queue` preserves FIFO order. See [slots.rs](examples/slots.rs). ## Why Taskvisor? -Restart and backoff are the baseline. Taskvisor also provides: +Restart and backoff are the baseline. +Taskvisor also provides: - **Cooperative shutdown with a deadline.** Tasks observe `TaskContext`; tasks that miss the grace period are force-aborted. - **Reliable final outcomes.** `TaskWaiter` reports how watched work ended even when best-effort events are dropped. - **Typed lifecycle events.** Logs, metrics, traces, and live status consume one structured event model. - **Dynamic management.** Add, list, cancel, remove, and watch tasks through `SupervisorHandle`. -- **Admission control.** The optional controller applies `Queue`, `Replace`, or `DropIfRunning` per named slot. +- **Admission control.** The controller applies `Queue`, `Replace`, or `DropIfRunning` per named slot when configured. - **Explicit limits.** Configure per-attempt timeout, retry budget, global concurrency, and bounded queues. -`JoinSet` and `TaskTracker` help own and join spawned futures. Taskvisor owns the restart policy and the task's complete lifecycle contract. +`JoinSet` and `TaskTracker` help own and join spawned futures. Taskvisor owns the restart policy and the managed in-process task lifecycle. -It is a good fit for queue consumers, pollers, sync loops, connection keepers, periodic work, and one-shot in-process jobs that need admission or a reliable outcome. +Taskvisor is strongest when dynamic or keyed work needs conflict handling and a reliable outcome. It also supports queue consumers, pollers, sync loops, connection keepers, periodic work, and one-shot in-process jobs. When the primary requirement is different, use a more specialized tool: -| You need | Better fit | -|-------------------------------------------------|--------------------------------------------------------------------------------------------------| -| Retry one future | [backon](https://crates.io/crates/backon) or [tokio-retry](https://crates.io/crates/tokio-retry) | -| Persist and recover jobs after a process restart | [apalis](https://crates.io/crates/apalis) | -| Actors with addresses and mailboxes | [ractor](https://crates.io/crates/ractor) or [kameo](https://crates.io/crates/kameo) | -| Structured subsystem shutdown without restarts | [tokio-graceful-shutdown](https://crates.io/crates/tokio-graceful-shutdown) | +| You need | Better fit | +|-------------------------------------------------------------|---------------------------------------------------------------------------------------------------| +| A small fixed set of workers with retry and cancellation | `JoinSet`/`TaskTracker` + `CancellationToken` + [backon](https://crates.io/crates/backon) | +| Retry one future | [backon](https://crates.io/crates/backon) or [tokio-retry](https://crates.io/crates/tokio-retry) | +| Persist and recover jobs after a process restart | [apalis](https://crates.io/crates/apalis) with a persistent storage backend | +| Actors with addresses and mailboxes | [ractor](https://crates.io/crates/ractor) or [kameo](https://crates.io/crates/kameo) | +| Structured subsystem shutdown without restarts | [tokio-graceful-shutdown](https://crates.io/crates/tokio-graceful-shutdown) | -## Core model +## Main API model -Four types form the main API: +These type groups form the main API: -| Type | Purpose | -|--------------------|------------------------------------------------------------| -| `TaskFn` or `Task` | The async work. A new future is created for every attempt. | -| `TaskSpec` | Restart policy, backoff, timeout, and retry limit. | -| `Supervisor` | Owns task lifecycle, shutdown, and event delivery. | -| `SupervisorHandle` | Adds, removes, cancels, and watches tasks at runtime. | +| Type | Purpose | +|----------------------------------------|------------------------------------------------------------| +| `TaskFn` or `Task` | The async work. A new future is created for every attempt. | +| `TaskSpec` | Restart policy, backoff, timeout, and retry limit. | +| `Supervisor` and `SupervisorHandle` | Own task lifecycle, shutdown, and dynamic management. | +| `TaskWaiter` and `TaskOutcome` | Deliver the reliable final result of watched work. | +| `ControllerSpec` and `AdmissionPolicy` | Resolve queue, replace, or reject conflicts per slot. | +| `PreparedSubmission` | Expose a submission ID before controller events can start. | +| `ControllerConfig` | Sets controller queue and command limits. |

Taskvisor core lifecycle: the Supervisor runs one task attempt, then either schedules another after failure backoff or an optional interval, or produces a final outcome for watched tasks

+The diagram shows common registered-task outcomes. `TaskOutcome` also distinguishes fatal failure, force-abort, and task-runner panic. Controller rejection produces `TaskOutcome::Rejected` before task registration. + Retries for one task run in sequence. Two attempts for the same `TaskId` never run at the same time. Active task names must be unique; the name can be reused after terminal cleanup, with a new `TaskId`. There are two runtime modes: @@ -141,7 +186,7 @@ There are two runtime modes: | `supervisor.run(specs)` | Tasks are known at startup | Taskvisor waits for completion or an OS signal. | | `supervisor.serve()` | Tasks are added at runtime | Your code calls `handle.shutdown().await`. | -`Supervisor::run(...).await == Ok(())` means the supervisor lifecycle and cleanup completed successfully. It does not mean that every task succeeded. Use a watched outcome when application logic needs that answer. +`Supervisor::run(...).await == Ok(())` means the supervisor lifecycle and cleanup completed successfully. It does not mean that every task succeeded, and `run` does not return per-task outcomes. Register work through `add_and_watch` or `submit_and_watch` when application logic needs the final result. ## Choose task behavior @@ -267,21 +312,15 @@ async fn wait_for_task( } ``` -Events carry a process-local sequence number and, where relevant, task identity, attempt, duration, reason, timeout, delay, and exit code. Stable string labels are available for telemetry. +Events carry a process-local sequence number and, where relevant, task identity, attempt, duration, timeout, delay, and exit code. `TaskFinished` carries `TaskOutcomeKind` for terminal telemetry. Rejected work carries `TaskOutcomeKind::Rejected` plus a `RejectionKind` explaining why it did not start. Treat `reason` as diagnostic text; do not parse it for branching, metrics, or alerts. Stable enum labels are available for telemetry. Each subscriber has its own bounded FIFO queue. Its synchronous callback runs on Tokio's blocking pool. A slow subscriber cannot block publishers, but its queue may fill and lose events. Keep callbacks short and forward async work to another channel. See [subscriber.rs](examples/subscriber.rs), the `TracingBridge` in [tracing.rs](examples/tracing.rs), and the Prometheus counters in [metrics.rs](examples/metrics.rs). -## Admission control (feature: `controller`) +## Admission control per key -Enable the controller on the dependency: - -```toml -taskvisor = { version = "0.6", features = ["controller"] } -``` - -The controller groups submissions into named slots. At most one task can occupy a slot; different slots can run concurrently. +The controller groups submissions into named slots. At most one task can occupy a slot; different slots can run concurrently. The `controller` feature is enabled by default; adding a controller to a supervisor remains explicit through `Supervisor::builder().with_controller(...)`. | Policy | Busy-slot behavior | Typical use | |-----------------|------------------------------------------------------------------------------|------------------------------------------| @@ -309,11 +348,13 @@ async fn submit_to_slot( } ``` -`submit().await?` confirms controller intake; admission happens later. `submit_and_watch` returns a final outcome. Work that never starts resolves to `TaskOutcome::Rejected`. +`submit().await?` confirms controller intake; admission happens later. `submit_and_watch` returns a final outcome. A submission rejected before registry admission resolves to `TaskOutcome::Rejected`; admitted work resolves to the registered task's terminal outcome. Queue depth is bounded per slot. `Replace` changes only the queue head; FIFO items behind it remain queued. `controller_snapshot()` returns a best-effort, non-transactional view of slot status and queue depth. -See [slots.rs](examples/slots.rs) and [admission.rs](examples/admission.rs) for complete programs. +Slots govern admission, not lifecycle addressing. Cancellation and removal operate by `TaskId` or registered task name; there is no slot-wide cancel/remove operation. Stopping the current owner does not automatically purge a queued replacement in the same slot. + +See [tenant_sync.rs](examples/tenant_sync.rs) for the tenant-42 conflict, [slots.rs](examples/slots.rs) for a policy reference, and [admission.rs](examples/admission.rs) for watched admission and rejection. ## Configuration @@ -367,6 +408,7 @@ Taskvisor defines an in-process lifecycle. Keep these boundaries explicit: - Subscriber callbacks may still run on Tokio's blocking pool after their drain deadline. Tokio runtime shutdown may wait for such callbacks. - Periodic tasks use an interval after completion. They do not provide calendar scheduling or missed-run recovery. - The controller coordinates tasks inside one supervisor. +- Controller slots are admission keys, not cancellation keys. There is no atomic "stop the current owner and purge its slot queue" operation. - With `panic = "unwind"`, Taskvisor catches task-future panics. It cannot recover from `panic = "abort"`, process aborts, memory exhaustion, or failures outside the process. For a service deployment, call the joined shutdown path, make resident tasks cancellation-aware, set finite timeouts and retry limits where endless retry is unsafe, monitor lifecycle failures and overflow, and use watched outcomes for decisions that depend on completion. @@ -375,18 +417,24 @@ The crate forbids unsafe Rust with `#![forbid(unsafe_code)]`. ## Feature flags -Taskvisor has no default features. The core depends on `tokio`, `tokio-util`, `thiserror`, and `fastrand`. +The `controller` feature is enabled by default so the keyed admission API is present in the standard install. The controller still has no runtime effect unless configured with `with_controller`. Use `default-features = false` to omit it and its `dashmap` dependency. -| Feature | Adds | -|----------------------|---------------------------------------------------------------| -| `controller` | Slot-based admission control; adds `dashmap`. | -| `tracing` | `TracingBridge` for the `tracing` ecosystem. | -| `logging` | `LogWriter`, a simple event writer for demos and small tools. | -| `tokio-util-interop` | Access to the raw cancellation token in `TaskContext`. | -| `test-util` | Helpers for code that integrates with Taskvisor. | +| Feature | Default | Adds | +|----------------------|---------|---------------------------------------------------------------| +| `controller` | yes | Slot-based admission control; adds `dashmap`. | +| `tracing` | no | `TracingBridge` for the `tracing` ecosystem. | +| `logging` | no | `LogWriter`, a simple event writer for demos and small tools. | +| `tokio-util-interop` | no | Access to the raw cancellation token in `TaskContext`. | +| `test-util` | no | Helpers for code that integrates with Taskvisor. | ```toml -taskvisor = { version = "0.6", features = ["controller", "tracing"] } +taskvisor = { version = "0.6", features = ["tracing"] } +``` + +Core-only install: + +```toml +taskvisor = { version = "0.6", default-features = false } ``` ## Examples @@ -397,21 +445,51 @@ From a cloned repository checkout, run the smallest example with: cargo run --example basic ``` -| Example | What it shows | -|-------------------------------------------------|----------------------------------------------------------| -| [basic.rs](examples/basic.rs) | One task, one run, one exit. | -| [worker.rs](examples/worker.rs) | A long-running worker with graceful cancellation. | -| [periodic.rs](examples/periodic.rs) | Repeated execution after an interval. | -| [multiple.rs](examples/multiple.rs) | Several restart policies in one supervisor. | -| [queue_consumer.rs](examples/queue_consumer.rs) | Reconnect after a consumer failure. | -| [cpu_job.rs](examples/cpu_job.rs) | Supervise CPU-heavy work without blocking Tokio workers. | -| [subscriber.rs](examples/subscriber.rs) | Handle typed lifecycle events. | -| [tracing.rs](examples/tracing.rs) | Forward events to `tracing` (`tracing` feature). | -| [metrics.rs](examples/metrics.rs) | Build Prometheus counters from events. | -| [dynamic.rs](examples/dynamic.rs) | Add, list, cancel, and remove tasks at runtime. | -| [outcomes.rs](examples/outcomes.rs) | Await the final result of a task. | -| [slots.rs](examples/slots.rs) | Compare controller policies (`controller` feature). | -| [admission.rs](examples/admission.rs) | Observe admission and rejection (`controller` feature). | +Choose the shortest path for your use case: + +- New to supervision: `basic` → `worker` → `outcomes`. +- Need per-key coordination: `tenant_sync` → `slots` → `admission`. + +The full catalog follows. + +### Start here + +| Example | What it shows | +|-------------------------------------------|---------------------------------------------------| +| [basic.rs](examples/basic.rs) | One task, one run, one exit. | +| [worker.rs](examples/worker.rs) | A long-running worker with graceful cancellation. | +| [periodic.rs](examples/periodic.rs) | Repeated execution after an interval. | +| [multiple.rs](examples/multiple.rs) | Several restart policies in one supervisor. | + +### Real patterns + +| Example | What it shows | +|--------------------------------------------------|-----------------------------------------------------------| +| [queue_consumer.rs](examples/queue_consumer.rs) | Retry a failed broker connection. | +| [cpu_job.rs](examples/cpu_job.rs) | Supervise CPU-heavy work without blocking Tokio workers. | + +### Observability + +| Example | What it shows | +|-------------------------------------------|--------------------------------------------------| +| [subscriber.rs](examples/subscriber.rs) | Handle typed lifecycle events. | +| [tracing.rs](examples/tracing.rs) | Forward events to `tracing` (`tracing` feature). | +| [metrics.rs](examples/metrics.rs) | Build Prometheus counters from events. | + +### Dynamic work and outcomes + +| Example | What it shows | +|---------------------------------------|--------------------------------------------------| +| [dynamic.rs](examples/dynamic.rs) | Add, list, cancel, and remove tasks at runtime. | +| [outcomes.rs](examples/outcomes.rs) | Await reliable outcomes, including a timeout. | + +### Keyed admission + +| Example | What it shows | +|-------------------------------------------------|-----------------------------------------------------| +| [tenant_sync.rs](examples/tenant_sync.rs) | Keep only the latest sync revision per tenant. | +| [slots.rs](examples/slots.rs) | Compare queue, replace, and reject policies. | +| [admission.rs](examples/admission.rs) | Observe typed admission and rejection outcomes. | ## Benchmarks @@ -419,17 +497,14 @@ The repository includes Criterion suites for lifecycle, throughput, subscriber f ```bash cargo bench -cargo bench --bench controller --features controller ``` ## Contributing -Issues and pull requests are welcome. Read the [contributing guide](https://github.com/soltiHQ/.github/blob/main/CONTRIBUTING.md) before a large change. +Issues and pull requests are welcome. Start with the [source architecture guide](src/ARCHITECTURE.md) to understand the runtime flows, then read the [contributing guide](https://github.com/soltiHQ/.github/blob/main/CONTRIBUTING.md) before a large change. If Taskvisor earns a place in your stack, a GitHub star helps other Rust developers find it. -## -

diff --git a/examples/admission.rs b/examples/admission.rs index 013e71e..f6a6138 100644 --- a/examples/admission.rs +++ b/examples/admission.rs @@ -6,31 +6,26 @@ //! `submit_and_watch` also returns a `TaskWaiter`: //! //! - an admitted task resolves to its final runtime outcome; -//! - a task that never starts resolves to `TaskOutcome::Rejected` with a reason. +//! - a submission rejected before registry admission resolves to `TaskOutcome::Rejected` with a typed `RejectionKind` and diagnostic reason; its task body never starts. //! //! This example shows both paths and reads a live controller snapshot. //! Use the waiter when rejection affects application logic. //! Events are best-effort and are better suited to logs and metrics. //! -//! Run with -//! `cargo run --example admission --features controller`. - -#[cfg(not(feature = "controller"))] -compile_error!( - "This example requires the `controller` feature: cargo run --example admission --features controller" -); +//! Run with `cargo run --example admission`. use std::sync::Arc; use std::time::Duration; -use taskvisor::ControllerSpec; use taskvisor::prelude::*; +use taskvisor::{ControllerConfig, ControllerSpec, RejectionKind}; use tokio::sync::Notify; -/// A job that runs for `dur`, observing cancellation. -fn job(name: &'static str, dur: Duration) -> TaskSpec { +/// A job that runs for `duration`, observing cancellation. +fn job(name: &'static str, duration: Duration) -> TaskSpec { let task: TaskRef = TaskFn::arc(name, move |ctx| async move { - ctx.run_until_cancelled(tokio::time::sleep(dur)).await?; + ctx.run_until_cancelled(tokio::time::sleep(duration)) + .await?; Ok(()) }); TaskSpec::once(task) @@ -52,10 +47,10 @@ fn gated_job(name: &'static str, started: Arc, release: Arc) -> #[tokio::main(flavor = "current_thread")] async fn main() -> Result<(), Box> { - let sup = Supervisor::builder(SupervisorConfig::default()) - .with_controller(taskvisor::ControllerConfig::default()) + let supervisor = Supervisor::builder(SupervisorConfig::default()) + .with_controller(ControllerConfig::default()) .build(); - let handle = sup.serve(); + let handle = supervisor.serve(); println!("Slot 'deploy' admits at most one task at a time.\n"); @@ -101,7 +96,11 @@ async fn main() -> Result<(), Box> { ) .await?; match v2.wait().await? { - TaskOutcome::Rejected { reason, .. } => { + TaskOutcome::Rejected { + kind: RejectionKind::SlotBusy, + reason, + .. + } => { println!(" deploy-v2 -> Rejected ({reason}) - never ran\n"); } other => println!(" deploy-v2 -> {other:?} (unexpected)\n"), @@ -113,6 +112,5 @@ async fn main() -> Result<(), Box> { println!(" deploy-v1 -> {:?}", v1.wait().await?); handle.shutdown().await?; - println!("\nDone."); Ok(()) } diff --git a/examples/basic.rs b/examples/basic.rs index 9a3ce03..bf2cadc 100644 --- a/examples/basic.rs +++ b/examples/basic.rs @@ -21,9 +21,8 @@ async fn main() -> Result<(), Box> { Ok(()) }); - let sup = Supervisor::new(SupervisorConfig::default(), vec![]); - sup.run(vec![TaskSpec::once(task)]).await?; + let supervisor = Supervisor::new(SupervisorConfig::default(), vec![]); + supervisor.run(vec![TaskSpec::once(task)]).await?; - println!("Done."); Ok(()) } diff --git a/examples/cpu_job.rs b/examples/cpu_job.rs index e0a14b7..58162a4 100644 --- a/examples/cpu_job.rs +++ b/examples/cpu_job.rs @@ -25,8 +25,8 @@ fn sum_of_primes(limit: u64) -> u64 { #[tokio::main(flavor = "current_thread")] async fn main() -> Result<(), Box> { - let sup = Supervisor::new(SupervisorConfig::default(), vec![]); - let handle = sup.serve(); + let supervisor = Supervisor::new(SupervisorConfig::default(), vec![]); + let handle = supervisor.serve(); let attempts = Arc::new(AtomicU32::new(0)); let job: TaskRef = TaskFn::arc("prime-sum", { diff --git a/examples/dynamic.rs b/examples/dynamic.rs index 568aed8..7a4a8b1 100644 --- a/examples/dynamic.rs +++ b/examples/dynamic.rs @@ -44,16 +44,17 @@ fn make_worker(name: &'static str) -> TaskSpec { #[tokio::main(flavor = "current_thread")] async fn main() -> Result<(), Box> { - let sup = Supervisor::new(SupervisorConfig::default(), vec![]); + let supervisor = Supervisor::new(SupervisorConfig::default(), vec![]); // serve() starts listeners and returns a handle for dynamic management. - let handle = sup.serve(); + let handle = supervisor.serve(); // Add workers dynamically println!("Adding worker-a and worker-b..."); let id_a = handle.add(make_worker("worker-a")).await?; let id_b = handle.add(make_worker("worker-b")).await?; + // Demo pacing only: add() has already confirmed registration. tokio::time::sleep(Duration::from_secs(1)).await; println!("Active: {:?}", handle.list().await); @@ -71,6 +72,7 @@ async fn main() -> Result<(), Box> { // Add worker-c println!("\nAdding worker-c..."); handle.add(make_worker("worker-c")).await?; + // Demo pacing only: let the terminal show a worker-c tick. tokio::time::sleep(Duration::from_millis(500)).await; // Cancel worker-b @@ -82,12 +84,12 @@ async fn main() -> Result<(), Box> { handle.is_alive("worker-b").await ); + // Demo pacing only: let worker-c keep ticking before the final snapshot. tokio::time::sleep(Duration::from_millis(500)).await; println!("\nActive: {:?}", handle.list().await); // Graceful shutdown (consumes the handle) println!("\nShutting down..."); handle.shutdown().await?; - println!("Done."); Ok(()) } diff --git a/examples/metrics.rs b/examples/metrics.rs index 7e9ceb3..4ea18f9 100644 --- a/examples/metrics.rs +++ b/examples/metrics.rs @@ -1,13 +1,14 @@ //! # Metrics from lifecycle events //! //! This example implements `Subscribe` and maps each event to a Prometheus counter. -//! `EventKind::as_label` provides stable, machine-readable values such as `task_failed` and `backoff_scheduled`. +//! `EventKind::as_label` provides stable, machine-readable values such as `attempt_failed` and `backoff_scheduled`. //! //! A real service would expose the registry on its metrics endpoint. //! This example prints the Prometheus text format when it exits. //! //! The `task` label uses task names. Keep these names bounded and stable. //! Do not put request IDs, user IDs, or other unbounded values in metric labels. +//! Terminal classification comes from `TaskOutcomeKind`; diagnostic `reason` text is deliberately not used as a label. //! //! Run with `cargo run --example metrics`. @@ -25,10 +26,14 @@ struct PromSubscriber { } impl Subscribe for PromSubscriber { - fn on_event(&self, e: &Event) { - let task = e.task.as_deref().unwrap_or("none"); + fn on_event(&self, event: &Event) { + let task = event.task.as_deref().unwrap_or("none"); + let outcome = event + .outcome_kind + .map(TaskOutcomeKind::as_label) + .unwrap_or("none"); self.events - .with_label_values(&[e.kind.as_label(), task]) + .with_label_values(&[event.kind.as_label(), outcome, task]) .inc(); } @@ -46,7 +51,7 @@ async fn main() -> Result<(), Box> { let registry = Registry::new(); let events = IntCounterVec::new( Opts::new("taskvisor_events_total", "Supervisor lifecycle events"), - &["event", "task"], + &["event", "outcome", "task"], )?; registry.register(Box::new(events.clone()))?; @@ -67,11 +72,11 @@ async fn main() -> Result<(), Box> { let spec = TaskSpec::restartable(flaky) .with_backoff(BackoffPolicy::constant(Duration::from_millis(100))); - let subs: Vec> = vec![Arc::new(PromSubscriber { + let subscribers: Vec> = vec![Arc::new(PromSubscriber { events: events.clone(), })]; - let sup = Supervisor::new(SupervisorConfig::default(), subs); - sup.run(vec![spec]).await?; + let supervisor = Supervisor::new(SupervisorConfig::default(), subscribers); + supervisor.run(vec![spec]).await?; // In a real service: serve this string at GET /metrics. let mut buf = Vec::new(); diff --git a/examples/multiple.rs b/examples/multiple.rs index b04904f..8e4e8bb 100644 --- a/examples/multiple.rs +++ b/examples/multiple.rs @@ -6,7 +6,7 @@ //! |-------------|--------------------|---------------------------------------| //! | `one-shot` | never restart | run once | //! | `resilient` | restart on failure | fail twice, then succeed | -//! | `always-on` | always restart | wait 500 ms after each successful run | +//! | `recurring` | periodic | wait 500 ms after each successful run | //! //! The resilient task uses exponential backoff and a retry budget. //! Only failure-driven restarts consume that budget. @@ -52,13 +52,13 @@ async fn main() -> Result<(), Box> { } }); - // Always-on: repeats every 500ms until Ctrl+C + // Recurring: repeats every 500ms until Ctrl+C let cycle = Arc::new(AtomicU32::new(0)); - let always_on: TaskRef = TaskFn::arc("always-on", move |_ctx| { + let recurring: TaskRef = TaskFn::arc("recurring", move |_ctx| { let cycle = Arc::clone(&cycle); async move { let n = cycle.fetch_add(1, Ordering::Relaxed) + 1; - println!("[always-on] cycle #{n}"); + println!("[recurring] cycle #{n}"); tokio::time::sleep(Duration::from_millis(300)).await; Ok(()) } @@ -66,7 +66,7 @@ async fn main() -> Result<(), Box> { let specs = vec![ TaskSpec::once(one_shot), - TaskSpec::periodic(always_on, Duration::from_millis(500)), + TaskSpec::periodic(recurring, Duration::from_millis(500)), TaskSpec::restartable(resilient) .with_backoff( BackoffPolicy::exponential(Duration::from_millis(200)) @@ -75,10 +75,9 @@ async fn main() -> Result<(), Box> { .with_max_retries(NonZeroU32::new(3).unwrap()), ]; - let cfg = SupervisorConfig::default().with_grace(Duration::from_secs(5)); - let sup = Supervisor::new(cfg, vec![]); - sup.run(specs).await?; + let config = SupervisorConfig::default().with_grace(Duration::from_secs(5)); + let supervisor = Supervisor::new(config, vec![]); + supervisor.run(specs).await?; - println!("All tasks finished."); Ok(()) } diff --git a/examples/outcomes.rs b/examples/outcomes.rs index 1e1f175..8f920c6 100644 --- a/examples/outcomes.rs +++ b/examples/outcomes.rs @@ -1,7 +1,7 @@ //! # Outcomes: wait for the final result //! //! `add_and_watch` returns a `TaskWaiter`. -//! The waiter resolves after the task entry has stopped, including all restarts allowed by its policy. +//! The waiter resolves after all allowed attempts end, the managed runner is joined, and registry membership is removed. //! //! Taskvisor has two result paths: //! @@ -10,8 +10,8 @@ //! | lifecycle events | logs, metrics, live progress | bounded and best-effort | //! | `TaskOutcome` | final business decision | dedicated terminal channel | //! -//! This example handles successful, failed, and canceled tasks. -//! Other outcomes cover fatal errors, force-abort, actor panic, and controller rejection. +//! This example handles successful, retry-exhausted, timed-out, and canceled tasks. +//! Other outcomes cover fatal errors, force-abort, task-runner panic, and controller rejection. //! `TaskWaiter::wait` can still return an error if the runtime closes the terminal channel unexpectedly. //! //! Run with `cargo run --example outcomes`. @@ -22,11 +22,12 @@ use std::sync::atomic::{AtomicU32, Ordering}; use std::time::Duration; use taskvisor::prelude::*; +use tokio::sync::Notify; #[tokio::main(flavor = "current_thread")] async fn main() -> Result<(), Box> { - let sup = Supervisor::new(SupervisorConfig::default(), vec![]); - let handle = sup.serve(); + let supervisor = Supervisor::new(SupervisorConfig::default(), vec![]); + let handle = supervisor.serve(); // 1) A one-shot job that succeeds -> Completed. println!("=== Completed ==="); @@ -38,7 +39,7 @@ async fn main() -> Result<(), Box> { println!(" import -> {:?}\n", waiter.wait().await?); // 2) A task that always fails, with a bounded retry budget -> Failed. - // Note the outcome's reason/exit_code are identical to the ActorExhausted event. + // Its reason/exit_code are identical to the typed TaskFinished event. println!("=== Failed (retries exhausted) ==="); let attempts = Arc::new(AtomicU32::new(0)); let flaky: TaskRef = TaskFn::arc("sync", move |_ctx| { @@ -61,19 +62,41 @@ async fn main() -> Result<(), Box> { other => println!(" sync -> {other:?}\n"), } - // 3) A long-running worker we cancel -> Canceled. + // 3) A one-shot task that exceeds its per-attempt deadline -> Failed. + println!("=== Failed (attempt timed out) ==="); + let slow: TaskRef = TaskFn::arc("slow-report", |_ctx| async { + tokio::time::sleep(Duration::from_secs(1)).await; + Ok(()) + }); + let timed = TaskSpec::once(slow).with_timeout(Duration::from_millis(20)); + match handle.add_and_watch(timed).await?.1.wait().await? { + TaskOutcome::Failed { reason, .. } => { + println!(" slow-report -> Failed: {reason}\n"); + } + other => println!(" slow-report -> {other:?}\n"), + } + + // 4) A long-running worker we cancel -> Canceled. println!("=== Canceled ==="); - let worker: TaskRef = TaskFn::arc("worker", |ctx| async move { - ctx.cancelled().await; - Err(TaskError::Canceled) + let started = Arc::new(Notify::new()); + let worker: TaskRef = TaskFn::arc("worker", { + let started = Arc::clone(&started); + move |ctx| { + let started = Arc::clone(&started); + async move { + started.notify_one(); + ctx.cancelled().await; + Err(TaskError::Canceled) + } + } }); let (id, waiter) = handle.add_and_watch(TaskSpec::restartable(worker)).await?; - tokio::time::sleep(Duration::from_millis(100)).await; + // The task body, rather than a timer, confirms that the worker started. + started.notified().await; println!(" cancelling worker..."); handle.cancel(id).await?; println!(" worker -> {:?}\n", waiter.wait().await?); handle.shutdown().await?; - println!("Done."); Ok(()) } diff --git a/examples/periodic.rs b/examples/periodic.rs index b79871b..9d72c24 100644 --- a/examples/periodic.rs +++ b/examples/periodic.rs @@ -27,8 +27,8 @@ async fn main() -> Result<(), Box> { let spec = TaskSpec::periodic(heartbeat, Duration::from_secs(2)); - let sup = Supervisor::new(SupervisorConfig::default(), vec![]); - sup.run(vec![spec]).await?; + let supervisor = Supervisor::new(SupervisorConfig::default(), vec![]); + supervisor.run(vec![spec]).await?; Ok(()) } diff --git a/examples/queue_consumer.rs b/examples/queue_consumer.rs index 644c14e..8b51c19 100644 --- a/examples/queue_consumer.rs +++ b/examples/queue_consumer.rs @@ -1,11 +1,11 @@ -//! # Queue consumer: reconnect after failure +//! # Queue consumer: retry a failed broker connection //! //! This example models a long-lived broker consumer. //! An in-process channel stands in for Kafka, Redis, SQS, or another client. //! //! One task attempt represents one connection session. -//! A connection error returns `TaskError::fail`. -//! The supervisor starts a new session after exponential backoff and jitter. +//! The first connection attempt returns `TaskError::fail`. +//! The supervisor retries it after exponential backoff and jitter. //! A clean return stops the `OnFailure` task. //! //! The receive operation uses `TaskContext::run_until_cancelled`. @@ -67,16 +67,15 @@ async fn main() -> Result<(), Box> { } }); - // Reconnect policy: base delays of 100ms, 200ms, 400ms, ... capped at 5s, with jitter. + // Retry policy: base delays of 100ms, 200ms, 400ms, ... capped at 5s, with jitter. let spec = TaskSpec::restartable(consumer).with_backoff( BackoffPolicy::exponential(Duration::from_millis(100)) .with_max(Duration::from_secs(5)) .with_jitter(JitterPolicy::Equal), ); - let sup = Supervisor::new(SupervisorConfig::default(), vec![]); - sup.run(vec![spec]).await?; + let supervisor = Supervisor::new(SupervisorConfig::default(), vec![]); + supervisor.run(vec![spec]).await?; - println!("Done."); Ok(()) } diff --git a/examples/slots.rs b/examples/slots.rs index 8db6702..a2829a6 100644 --- a/examples/slots.rs +++ b/examples/slots.rs @@ -1,6 +1,6 @@ //! # Slot admission policies //! -//! The optional controller groups tasks by a slot key. +//! The controller groups tasks by a slot key. //! One task can run in a slot at a time. //! Different slot keys are independent. //! @@ -12,22 +12,19 @@ //! //! The slot key defaults to the task name. //! `with_slot` can set an explicit key. +//! Each scenario below uses different task names with one shared slot to make that distinction visible. //! `add` bypasses the controller; `submit` uses its admission rules. //! An `Ok(id)` from `submit` confirms intake, not final admission. -//! Use`submit_and_watch` when application logic needs the final result. +//! Use `submit_and_watch` when application logic needs the final result. //! `Replace` does not clear the full FIFO queue; items behind the head remain. //! -//! Run with `cargo run --example slots --features controller`. - -#[cfg(not(feature = "controller"))] -compile_error!( - "This example requires the `controller` feature: cargo run --example slots --features controller" -); +//! Run with `cargo run --example slots`. use std::sync::Arc; use std::time::Duration; use taskvisor::prelude::*; +use taskvisor::{ControllerConfig, ControllerSpec}; use tokio::sync::Notify; fn job(name: &'static str, duration: Duration) -> TaskSpec { @@ -65,25 +62,27 @@ fn job_with_start( #[tokio::main(flavor = "current_thread")] async fn main() -> Result<(), Box> { - let sup = Supervisor::builder(SupervisorConfig::default()) - .with_controller(taskvisor::ControllerConfig::default()) + let supervisor = Supervisor::builder(SupervisorConfig::default()) + .with_controller(ControllerConfig::default()) .build(); // serve() returns a handle for dynamic task submission. - let handle = sup.serve(); + let handle = supervisor.serve(); // Queue: tasks run sequentially println!("=== Queue Policy ==="); - println!("Submit 3 jobs with the same name — they run one-by-one.\n"); + println!("Submit 3 differently named jobs to one slot — they run one-by-one.\n"); let mut queued = Vec::new(); - for i in 1..=3 { - let spec = job("queued-job", Duration::from_millis(400)); - let (_id, waiter) = handle - .submit_and_watch(taskvisor::ControllerSpec::queue(spec)) - .await?; + for (index, name) in ["queued-job-1", "queued-job-2", "queued-job-3"] + .into_iter() + .enumerate() + { + let spec = job(name, Duration::from_millis(400)); + let request = ControllerSpec::queue(spec).with_slot("queue-demo"); + let (_id, waiter) = handle.submit_and_watch(request).await?; queued.push(waiter); - println!(" submitted #{i}"); + println!(" submitted #{}", index + 1); } for (index, waiter) in queued.into_iter().enumerate() { println!(" queued #{} -> {:?}", index + 1, waiter.wait().await?); @@ -95,19 +94,17 @@ async fn main() -> Result<(), Box> { let long_started = Arc::new(Notify::new()); let long = job_with_start( - "replace-job", + "replace-v1", Duration::from_secs(5), Some(Arc::clone(&long_started)), ); - let (_long_id, long_waiter) = handle - .submit_and_watch(taskvisor::ControllerSpec::replace(long)) - .await?; + let long_request = ControllerSpec::replace(long).with_slot("replace-demo"); + let (_long_id, long_waiter) = handle.submit_and_watch(long_request).await?; long_started.notified().await; - let short = job("replace-job", Duration::from_millis(200)); - let (_short_id, short_waiter) = handle - .submit_and_watch(taskvisor::ControllerSpec::replace(short)) - .await?; + let short = job("replace-v2", Duration::from_millis(200)); + let short_request = ControllerSpec::replace(short).with_slot("replace-demo"); + let (_short_id, short_waiter) = handle.submit_and_watch(short_request).await?; println!(" long -> {:?}", long_waiter.wait().await?); println!(" short -> {:?}", short_waiter.wait().await?); @@ -117,24 +114,21 @@ async fn main() -> Result<(), Box> { let first_started = Arc::new(Notify::new()); let first = job_with_start( - "drop-job", + "drop-v1", Duration::from_millis(600), Some(Arc::clone(&first_started)), ); - let (_first_id, first_waiter) = handle - .submit_and_watch(taskvisor::ControllerSpec::drop_if_running(first)) - .await?; + let first_request = ControllerSpec::drop_if_running(first).with_slot("drop-demo"); + let (_first_id, first_waiter) = handle.submit_and_watch(first_request).await?; first_started.notified().await; - let second = job("drop-job", Duration::from_millis(100)); - let (_second_id, second_waiter) = handle - .submit_and_watch(taskvisor::ControllerSpec::drop_if_running(second)) - .await?; + let second = job("drop-v2", Duration::from_millis(100)); + let second_request = ControllerSpec::drop_if_running(second).with_slot("drop-demo"); + let (_second_id, second_waiter) = handle.submit_and_watch(second_request).await?; println!(" second -> {:?}", second_waiter.wait().await?); println!(" first -> {:?}", first_waiter.wait().await?); - // Shutdown (consumes the handle) - println!("\nDone."); + // Joined shutdown consumes the handle. handle.shutdown().await?; Ok(()) diff --git a/examples/subscriber.rs b/examples/subscriber.rs index 5f3486e..cdb695a 100644 --- a/examples/subscriber.rs +++ b/examples/subscriber.rs @@ -26,18 +26,18 @@ use taskvisor::prelude::*; /// A simple metrics subscriber that counts lifecycle events. struct Metrics { starts: AtomicU64, - stops: AtomicU64, + successes: AtomicU64, failures: AtomicU64, - retries: AtomicU64, + backoffs: AtomicU64, } impl Metrics { fn new() -> Self { Self { starts: AtomicU64::new(0), - stops: AtomicU64::new(0), + successes: AtomicU64::new(0), failures: AtomicU64::new(0), - retries: AtomicU64::new(0), + backoffs: AtomicU64::new(0), } } @@ -45,26 +45,26 @@ impl Metrics { println!(); println!("--- Metrics ---"); println!(" starts: {}", self.starts.load(Ordering::Relaxed)); - println!(" stops: {}", self.stops.load(Ordering::Relaxed)); + println!(" successes: {}", self.successes.load(Ordering::Relaxed)); println!(" failures: {}", self.failures.load(Ordering::Relaxed)); - println!(" retries: {}", self.retries.load(Ordering::Relaxed)); + println!(" backoffs: {}", self.backoffs.load(Ordering::Relaxed)); } } impl Subscribe for Metrics { - fn on_event(&self, ev: &Event) { - match ev.kind { - EventKind::TaskStarting => { + fn on_event(&self, event: &Event) { + match event.kind { + EventKind::AttemptStarting => { self.starts.fetch_add(1, Ordering::Relaxed); } - EventKind::TaskStopped => { - self.stops.fetch_add(1, Ordering::Relaxed); + EventKind::AttemptSucceeded => { + self.successes.fetch_add(1, Ordering::Relaxed); } - EventKind::TaskFailed => { + EventKind::AttemptFailed => { self.failures.fetch_add(1, Ordering::Relaxed); } EventKind::BackoffScheduled => { - self.retries.fetch_add(1, Ordering::Relaxed); + self.backoffs.fetch_add(1, Ordering::Relaxed); } _ => {} } @@ -104,9 +104,9 @@ async fn main() -> Result<(), Box> { // restartable() uses exponential backoff from 200ms to 30s with equal jitter. let spec = TaskSpec::restartable(flaky); - let subs: Vec> = vec![Arc::clone(&metrics) as _]; - let sup = Supervisor::new(SupervisorConfig::default(), subs); - sup.run(vec![spec]).await?; + let subscribers: Vec> = vec![Arc::clone(&metrics) as _]; + let supervisor = Supervisor::new(SupervisorConfig::default(), subscribers); + supervisor.run(vec![spec]).await?; metrics.report(); Ok(()) diff --git a/examples/tenant_sync.rs b/examples/tenant_sync.rs new file mode 100644 index 0000000..9415828 --- /dev/null +++ b/examples/tenant_sync.rs @@ -0,0 +1,132 @@ +//! # Latest-wins synchronization per tenant +//! +//! A tenant is the natural admission key: +//! - syncs for the same tenant never overlap; +//! - only the newest waiting revision survives replacement; +//! - another tenant can sync independently. +//! +//! Reliable `TaskWaiter` outcomes, not lifecycle events, drive the decisions in this example. +//! +//! Run with `cargo run --example tenant_sync`. + +use std::sync::Arc; + +use taskvisor::prelude::*; +use taskvisor::{ControllerConfig, ControllerSpec, RejectionKind}; +use tokio::sync::Notify; + +#[derive(Debug)] +struct SyncGate { + started: Notify, + cancel_observed: Notify, + finish: Notify, + cancel_cleanup: Notify, + hold_cancel_cleanup: bool, +} + +impl SyncGate { + fn new(hold_cancel_cleanup: bool) -> Arc { + Arc::new(Self { + started: Notify::new(), + cancel_observed: Notify::new(), + finish: Notify::new(), + cancel_cleanup: Notify::new(), + hold_cancel_cleanup, + }) + } +} + +fn tenant_sync(tenant: &'static str, revision: u64, gate: Arc) -> TaskSpec { + let name = format!("sync/{tenant}/rev-{revision}"); + let task: TaskRef = TaskFn::arc(name, move |ctx| { + let gate = Arc::clone(&gate); + async move { + println!(" {tenant} revision {revision}: started"); + gate.started.notify_one(); + + tokio::select! { + biased; + _ = ctx.cancelled() => { + println!(" {tenant} revision {revision}: replacement requested"); + gate.cancel_observed.notify_one(); + if gate.hold_cancel_cleanup { + gate.cancel_cleanup.notified().await; + } + println!(" {tenant} revision {revision}: cleanup finished"); + Err(TaskError::Canceled) + } + _ = gate.finish.notified() => { + println!(" {tenant} revision {revision}: applied"); + Ok(()) + } + } + } + }); + TaskSpec::once(task) +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Box> { + let supervisor = Supervisor::builder(SupervisorConfig::default()) + .with_controller(ControllerConfig::default()) + .build(); + let handle = supervisor.serve(); + + let tenant_42_old = SyncGate::new(true); + let tenant_42_latest = SyncGate::new(false); + let tenant_17 = SyncGate::new(false); + + let (_, old_waiter) = handle + .submit_and_watch( + ControllerSpec::replace(tenant_sync("tenant-42", 1, Arc::clone(&tenant_42_old))) + .with_slot("tenant-42"), + ) + .await?; + tenant_42_old.started.notified().await; + + let (_, other_waiter) = handle + .submit_and_watch( + ControllerSpec::replace(tenant_sync("tenant-17", 1, Arc::clone(&tenant_17))) + .with_slot("tenant-17"), + ) + .await?; + tenant_17.started.notified().await; + println!("different tenant slots are running together\n"); + + let (_, pending_waiter) = handle + .submit_and_watch( + ControllerSpec::replace(tenant_sync("tenant-42", 2, SyncGate::new(false))) + .with_slot("tenant-42"), + ) + .await?; + tenant_42_old.cancel_observed.notified().await; + println!("tenant-42 revision 2 waits for revision 1 cleanup\n"); + + let (_, latest_waiter) = handle + .submit_and_watch( + ControllerSpec::replace(tenant_sync("tenant-42", 3, Arc::clone(&tenant_42_latest))) + .with_slot("tenant-42"), + ) + .await?; + match pending_waiter.wait().await? { + TaskOutcome::Rejected { + kind: RejectionKind::SupersededByReplace, + .. + } => println!("tenant-42 revision 2 -> superseded before start\n"), + other => panic!("unexpected tenant-42 revision 2 outcome: {other:?}"), + } + + tenant_42_old.cancel_cleanup.notify_one(); + println!("tenant-42 revision 1 -> {:?}", old_waiter.wait().await?); + tenant_42_latest.started.notified().await; + + tenant_42_latest.finish.notify_one(); + tenant_17.finish.notify_one(); + let (latest_outcome, other_outcome) = + tokio::try_join!(latest_waiter.wait(), other_waiter.wait())?; + println!("tenant-42 revision 3 -> {latest_outcome:?}"); + println!("tenant-17 revision 1 -> {other_outcome:?}"); + + handle.shutdown().await?; + Ok(()) +} diff --git a/examples/tracing.rs b/examples/tracing.rs index 63dea25..6a32a2f 100644 --- a/examples/tracing.rs +++ b/examples/tracing.rs @@ -8,11 +8,6 @@ //! //! Run with `cargo run --example tracing --features tracing`. -#[cfg(not(feature = "tracing"))] -compile_error!( - "This example requires the `tracing` feature: cargo run --example tracing --features tracing" -); - use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; use std::time::Duration; @@ -50,9 +45,9 @@ async fn main() -> Result<(), Box> { .with_backoff(BackoffPolicy::constant(Duration::from_millis(100))); // One line: supervisor events flow into your log pipeline. - let subs: Vec> = vec![Arc::new(TracingBridge)]; - let sup = Supervisor::new(SupervisorConfig::default(), subs); - sup.run(vec![spec]).await?; + let subscribers: Vec> = vec![Arc::new(TracingBridge)]; + let supervisor = Supervisor::new(SupervisorConfig::default(), subscribers); + supervisor.run(vec![spec]).await?; Ok(()) } diff --git a/examples/worker.rs b/examples/worker.rs index 5af6a13..14c6a34 100644 --- a/examples/worker.rs +++ b/examples/worker.rs @@ -40,8 +40,8 @@ async fn main() -> Result<(), Box> { let spec = TaskSpec::restartable(worker); - let sup = Supervisor::new(SupervisorConfig::default(), vec![]); - sup.run(vec![spec]).await?; + let supervisor = Supervisor::new(SupervisorConfig::default(), vec![]); + supervisor.run(vec![spec]).await?; Ok(()) } diff --git a/src/ARCHITECTURE.md b/src/ARCHITECTURE.md new file mode 100644 index 0000000..8a9e65f --- /dev/null +++ b/src/ARCHITECTURE.md @@ -0,0 +1,366 @@ +# Taskvisor source guide + +This document is a reading map for contributors. + +It shows which module owns each decision and how data moves through the runtime. The Rust source and its module-level documentation remain the source of truth. + +## Recommended reading order + +Read the code in this order if you are new to the repository: + +| Step | Files | Question answered | +|------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------| +| 1 | [`lib.rs`](lib.rs), [`prelude.rs`](prelude.rs) | What is public | +| 2 | [`tasks/`](tasks), [`policies/`](policies), [`core/task_defaults.rs`](core/task_defaults.rs) | What describes a task and its retry rules | +| 3 | [`core/builder.rs`](core/builder.rs), [`core/supervisor.rs`](core/supervisor.rs), [`core/handle.rs`](core/handle.rs), [`core/owner.rs`](core/owner.rs) | How is the runtime built, owned, and exposed | +| 4 | [`core/runtime.rs`](core/runtime.rs), [`core/runtime/management.rs`](core/runtime/management.rs), [`core/runtime/lifecycle.rs`](core/runtime/lifecycle.rs) | How do public calls enter the runtime | +| 5 | [`core/registry.rs`](core/registry.rs), [`core/registry/`](core/registry) | Which task state is authoritative, and how is it cleaned up | +| 6 | [`core/actor.rs`](core/actor.rs), [`core/runner.rs`](core/runner.rs) | How does one task run, retry, time out, and stop | +| 7 | [`core/outcome.rs`](core/outcome.rs), [`events/`](events), [`subscribers/`](subscribers) | Which results are reliable, and which signals are observability | +| 8 | [`controller/mod.rs`](controller/mod.rs), [`controller/prepared.rs`](controller/prepared.rs), [`controller/slot.rs`](controller/slot.rs), [`controller/core/`](controller/core) | How does per-slot queue/replace/reject admission work | +| 9 | [`core/runtime/shutdown_workflow.rs`](core/runtime/shutdown_workflow.rs), [`core/shutdown.rs`](core/shutdown.rs), [`controller/core/shutdown.rs`](controller/core/shutdown.rs) | How is one shared shutdown coordinated | + +After the module documentation, read the integration tests by behavior: [`tests/watch.rs`](../tests/watch.rs), [`tests/identity.rs`](../tests/identity.rs), [`tests/controller.rs`](../tests/controller.rs), and [`tests/shutdown.rs`](../tests/shutdown.rs). + +## Runtime map + +`SupervisorBuilder` wires the runtime but does not start its background tasks. + +`Supervisor::run` and `Supervisor::serve` call the idempotent runtime start path. + +The same component may appear in more than one path below. Repetition is only for layout; each label refers to the same runtime component. + +### Construction and task execution + +```mermaid +%%{init: {"flowchart": {"curve": "linear"}}}%% +flowchart TB + App["Application"] + Builder["SupervisorBuilder"] + Supervisor["Supervisor / SupervisorHandle"] + Core["SupervisorCore"] + Registry["Registry: authoritative task membership"] + Actor["TaskActor: one registered task"] + Runner["run_once: one attempt"] + Task["Task + TaskSpec + policies"] + Shutdown["ShutdownCoordinator"] + Waiter["TaskWaiter / TaskOutcome"] + + App --> Supervisor + Builder -->|returns| Supervisor + Supervisor --> Core + Builder -->|constructs| Core + Core -->|bounded command channel| Registry + Registry -->|spawns and joins| Actor + Actor --> Runner + Runner --> Task + Core --> Shutdown + Registry -->|direct one-shot| Waiter +``` + +### Controller admission + +```mermaid +%%{init: {"flowchart": {"curve": "linear"}}}%% +flowchart TB + Builder["SupervisorBuilder"] + Supervisor["Supervisor / SupervisorHandle"] + Prepared["PreparedSubmission: reserved TaskId + ControllerSpec"] + Controller["Controller: per-slot admission"] + Core["SupervisorCore"] + Waiter["TaskWaiter / TaskOutcome"] + + Builder -->|constructs when configured| Controller + Supervisor -->|prepare_submission| Prepared + Supervisor -->|submit shortcuts| Controller + Prepared -->|single-use submit| Controller + Controller -->|accepted work| Core + Controller -->|direct rejection outcome| Waiter +``` + +### Best-effort observability + +```mermaid +%%{init: {"flowchart": {"curve": "linear"}}}%% +flowchart TB + Registry["Registry: authoritative task membership"] + Actor["TaskActor: one registered task"] + Controller["Controller: per-slot admission"] + Shutdown["ShutdownCoordinator"] + Bus["Bus: best-effort broadcast"] + Observability["Event relay: alive view + subscriber queues"] + + Registry -. lifecycle events .-> Bus + Actor -. attempt events .-> Bus + Controller -. admission events .-> Bus + Shutdown -. shutdown events .-> Bus + Bus -. best-effort broadcast .-> Observability +``` + +The controller is compiled by the default `controller` feature, but it is a runtime opt-in: it exists only when a builder receives a `ControllerConfig`. Direct `add*` methods bypass slot admission; `submit*` methods use it. + +`PreparedSubmission` is only a command-side hand-off. It allocates the controller submission's `TaskId` and holds its `ControllerSpec`, but it does not publish or enqueue anything. Consuming it sends the same ordered controller command as the ordinary `submit*` shortcuts. This lets an integrating application install `application ID -> TaskId` correlation before events for that `TaskId` can begin. + +## Direct task lifecycle + +The registry, not the event stream, owns task identity and cleanup. A watched add uses direct replies in both directions: one reply for admission and one final outcome after the actor join and membership removal. + +```mermaid +sequenceDiagram + participant Caller + participant Handle as SupervisorHandle + participant Core as SupervisorCore + participant Queue as Registry command channel + participant Registry + participant TaskActor + participant Waiter as TaskWaiter + + Caller->>Handle: add_and_watch(TaskSpec) + Handle->>Core: add watched task + Core->>Queue: Add command + reply + outcome sender + Queue->>Registry: listener receives command + Registry->>Registry: resolve defaults and index ID + name + Registry->>TaskActor: spawn behind start gate + Registry-->>Core: direct Add decision + Registry->>TaskActor: open start gate + Core-->>Caller: TaskId + TaskWaiter + + loop Sequential attempts + TaskActor->>TaskActor: permit, run_once, policy decision + end + + TaskActor-->>Registry: reliable completion ID + Registry->>TaskActor: claim and join actor + Registry->>Registry: remove ID + name indexes + Registry-->>Waiter: direct TaskOutcome + Waiter-->>Caller: final outcome +``` + +For a static `run(tasks)` batch, the registry indexes every accepted entry, attempts all `TaskAdded` publications, and attempts the direct batch reply before one shared start gate releases any task body. + +## One actor and its attempts + +`run_once` owns one attempt: task invocation, panic capture, the attempt timeout, and the attempt terminal event. + +`TaskActor` owns the surrounding loop: the concurrency permit, restart policy, backoff, retry budget, and cancellation between attempts. + +The retry loop and actor exits are shown separately below. Repeated phase names refer to the same actor phases. + +### Retry loop + +```mermaid +%%{init: {"flowchart": {"curve": "linear"}}}%% +flowchart TB + Start(( )) + Permit("Wait for concurrency permit") + Attempt("Run one attempt") + SuccessDelay("Success interval or restart floor") + FailureDelay("Failure backoff") + + Start --> Permit + Permit -->|permit acquired| Attempt + Attempt -->|success, Always restarts| SuccessDelay + Attempt -->|retryable, retry allowed| FailureDelay + SuccessDelay -->|delay complete| Permit + FailureDelay -->|delay complete| Permit +``` + +### Exit paths + +```mermaid +%%{init: {"flowchart": {"curve": "linear"}}}%% +flowchart TB + Permit("Wait for concurrency permit") + Attempt("Run one attempt") + SuccessDelay("Success interval or restart floor") + FailureDelay("Failure backoff") + Completed("ActorExitReason::Completed") + Exhausted("ActorExitReason::Exhausted") + Fatal("ActorExitReason::Fatal") + Canceled("ActorExitReason::Canceled") + End(( )) + + Permit -->|runtime canceled| Canceled + Attempt -->|success, policy stops| Completed + Attempt -->|retry not allowed| Exhausted + Attempt -->|fatal error| Fatal + Attempt -->|cooperative cancellation| Canceled + SuccessDelay -->|runtime canceled| Canceled + FailureDelay -->|runtime canceled| Canceled + + Completed --> End + Exhausted --> End + Fatal --> End + Canceled --> End +``` + +Important boundaries: + +- Attempt numbers start at `1`. +- `max_retries` counts retries after the first failed attempt, not total attempts. +- A success resets the failure retry counter. +- A concurrency permit is held only while `run_once` is active. +- Task panics caught by `run_once` become retryable `TaskError::Fail` values. +- The actor returns an internal `ActorExitReason`; the registry maps it to `TaskOutcome` after joining the actor and removing its ID and name indexes. +- Force-abort and an outer Tokio join failure are cleanup results, not normal actor exits. + +## Events and reliable outcomes are separate paths + +Events support diagnostics, metrics, and best-effort liveness views. They do not drive cleanup, watched outcomes, or controller slot ownership. + +```mermaid +%%{init: {"flowchart": {"curve": "linear"}}}%% +flowchart LR + Runtime["Runtime components"] + Bus["Broadcast Bus"] + Relay["Event relay"] + Alive["AliveTracker"] + Queues["Per-subscriber bounded queues"] + Subscribers["Subscriber callbacks"] + + Cleanup["Registry terminal cleanup"] + Reject["Controller rejection"] + Outcome["Outcome one-shot"] + Waiter["TaskWaiter"] + + Runtime -. best-effort events .-> Bus + Bus -. best-effort broadcast .-> Relay + Relay -. event-derived update .-> Alive + Relay -. bounded enqueue .-> Queues + Queues -. blocking-pool delivery .-> Subscribers + + Cleanup -->|direct final result| Outcome + Reject -->|direct rejected result| Outcome + Outcome --> Waiter +``` + +Use the following source according to the question being asked: + +| Question | Source | +|----------|--------| +| Is a task still registered or being removed? | `SupervisorHandle::list`, backed by the registry | +| What final result did this watched task produce? | `TaskWaiter`, backed by a direct one-shot | +| What happened for logging or metrics? | Events and subscribers | +| Which tasks look alive from observed lifecycle events? | `alive_snapshot` / `is_alive`; these views may lag | +| What is the current controller view? | `controller_snapshot`; it is a rolling diagnostic snapshot, not a transaction | + +"Reliable" here means that a watched result does not depend on the lossy event path. It does not add persistence across process termination. + +## Controller admission + +The controller is a serialized admission layer before the registry. One loop owns slot transitions and processes ordered commands, direct registry Add decisions, terminal `RemovalCompletion` signals, and the reliable runtime shutdown token. + +```mermaid +%%{init: {"flowchart": {"curve": "linear"}}}%% +flowchart LR + Commands["Ordered submit and identity commands"] + AddDecision["Direct registry Add decision"] + Completion["Terminal RemovalCompletion"] + Shutdown["Runtime shutdown token"] + Loop["Single controller loop"] + Slots["Per-slot state + queue"] + Registry["SupervisorCore / Registry"] + Rejected["TaskOutcome::Rejected"] + Events["Best-effort controller events"] + + Commands --> Loop + AddDecision --> Loop + Completion --> Loop + Shutdown --> Loop + Loop --> Slots + Loop -->|admit or remove owner| Registry + Loop -->|resolve watched rejection| Rejected + Loop -. observability only .-> Events +``` + +The internal slot phases are: + +```mermaid +%%{init: {"flowchart": {"curve": "linear"}}}%% +flowchart LR + Start(( )) + Idle("Idle") + Admitting("Admitting") + Running("Running") + CancelPending("CancelPendingAdmission") + Terminating("Terminating") + + Start --> Idle + Idle -->|submit or advance queued head| Admitting + Admitting -->|registry accepts Add| Running + Admitting -->|registry rejects Add| Idle + Admitting -->|Replace arrives before Add decision| CancelPending + CancelPending -->|registry accepts Add, then removal starts| Terminating + CancelPending -->|registry rejects Add| Idle + Running -->|Replace requests removal| Terminating + Running -->|terminal registry completion| Idle + Terminating -->|terminal registry completion| Idle +``` + +Policy behavior around those phases: + +- `Queue` appends work in FIFO order until `ControllerConfig::max_slot_queue` is reached. +- `Replace` replaces the queued head and rejects the displaced head as `SupersededByReplace`. Existing FIFO work behind the head remains queued. +- `DropIfRunning` rejects new work while the slot has an owner. +- A successful removal request does not free the slot. Only terminal `RemovalCompletion`, after the registry has joined the actor and removed both identity indexes, frees it. +- A task name and a controller slot are different keys. The registry still enforces global task-name uniqueness. + +## Shared shutdown + +Explicit shutdown, a received OS signal, and natural completion join one cancellation-safe shutdown operation. The first trigger installs the operation; all callers wait for its cached result. + +```mermaid +%%{init: {"flowchart": {"curve": "linear"}}}%% +flowchart LR + Explicit["Explicit request"] + Signal["Received OS signal"] + Natural["Registry becomes empty"] + Coordinator["ShutdownCoordinator: first trigger wins"] + Close["Close management admission,
fence committed registry commands"] + Drain["Cancel and join tasks
within grace"] + Verdict{"All task cleanup
finished?"} + Tail["Cleanup tail: join controller,
cancel runtime token,
join registry listener and event relay,
close subscriber workers"] + Result["Cache and return
one shared result"] + + Explicit --> Coordinator + Signal --> Coordinator + Natural --> Coordinator + Coordinator --> Close + Close --> Drain + Drain --> Verdict + Verdict -->|AllStoppedWithinGrace| Tail + Verdict -->|GraceExceeded + stuck task names| Tail + Tail --> Result +``` + +Subscriber shutdown has its own timeout and happens after the task grace phase. Every common cleanup phase is attempted even if an earlier phase reports an internal failure. If OS signal setup itself fails, shutdown still closes admission and runs the common cleanup tail, but it does not run the normal task-drain branch. Dropping the last runtime owner is only a synchronous fallback: it closes admission and cancels tokens, but cannot await or report graceful cleanup. + +## Where to make a change + +| Change | Start here | Verify here | +|-------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------| +| Public task contract or task configuration | [`tasks/`](tasks), [`core/task_defaults.rs`](core/task_defaults.rs) | [`tests/defaults.rs`](../tests/defaults.rs), rustdoc examples | +| Attempt timeout, panic, or terminal event | [`core/runner.rs`](core/runner.rs) | unit tests in that module, [`tests/timeout.rs`](../tests/timeout.rs), [`tests/failure.rs`](../tests/failure.rs) | +| Restart, retry, backoff, or cancellation between attempts | [`core/actor.rs`](core/actor.rs), [`policies/`](policies) | actor unit tests, [`tests/failure.rs`](../tests/failure.rs), [`tests/lifecycle.rs`](../tests/lifecycle.rs) | +| Task identity, duplicate names, add/remove/cancel semantics | [`core/registry/`](core/registry), [`core/runtime/management.rs`](core/runtime/management.rs) | [`tests/identity.rs`](../tests/identity.rs), [`tests/watch.rs`](../tests/watch.rs), [`tests/concurrency.rs`](../tests/concurrency.rs) | +| Final watched outcomes | [`core/outcome.rs`](core/outcome.rs), [`core/registry/removal.rs`](core/registry/removal.rs) | [`tests/watch.rs`](../tests/watch.rs) | +| Event fields or delivery | [`events/`](events), [`core/runtime/event_relay.rs`](core/runtime/event_relay.rs), [`subscribers/`](subscribers) | [`tests/lifecycle.rs`](../tests/lifecycle.rs), subscriber unit tests | +| Per-slot queue/replace/reject behavior | [`controller/slot.rs`](controller/slot.rs), [`controller/core/admission.rs`](controller/core/admission.rs), [`controller/core/queue.rs`](controller/core/queue.rs) | [`tests/controller.rs`](../tests/controller.rs), controller unit tests | +| Shutdown order or grace behavior | [`core/runtime/shutdown_workflow.rs`](core/runtime/shutdown_workflow.rs), [`core/registry/removal.rs`](core/registry/removal.rs), [`controller/core/shutdown.rs`](controller/core/shutdown.rs) | [`tests/shutdown.rs`](../tests/shutdown.rs), [`tests/ownership.rs`](../tests/ownership.rs) | +| User-facing story | [`../README.md`](../README.md), [`../examples/`](../examples), [`lib.rs`](lib.rs) | `cargo test --all-features`, `cargo test --doc --all-features` | + +## Invariants to preserve + +Before changing a coordination path, check these constraints in the owning module and its tests: + +1. Registry membership is authoritative; events never add or remove membership. +2. Task ID and name indexes change under the same registry state lock. +3. A name stays reserved until terminal join cleanup removes it. +4. Exactly one cleanup owner claims and joins each actor handle. +5. Accepted task bodies start only after indexing and the admission publication/reply attempts. +6. Watched outcomes resolve outside the best-effort event path, after the actor join and registry membership removal. +7. The serialized controller loop owns slot transitions; queued work starts only after the current Add is rejected or the current owner reaches terminal removal completion. +8. Management admission closes and committed commands are fenced before shutdown drains tasks. +9. Concurrent shutdown callers join the same operation and receive its cached result. + +When a change crosses one of these boundaries, update both the module-level documentation and the relevant diagram in this guide. diff --git a/src/controller/config.rs b/src/controller/config.rs index e3ef5c4..f8ef4ee 100644 --- a/src/controller/config.rs +++ b/src/controller/config.rs @@ -44,9 +44,8 @@ pub struct ControllerConfig { /// When that cap is reached, the controller stops draining new commands; later commands remain in the bounded channel. /// /// When the command channel is full: - /// - `submit()` waits for capacity, - /// - `submit_and_watch()` waits for capacity, - /// - `try_submit()` and `try_submit_and_watch()` return [`ControllerError::Full`](crate::ControllerError::Full), + /// - async `submit()` and `submit_and_watch()` methods wait for capacity, + /// - `try_submit()` and `try_submit_and_watch()` methods return [`ControllerError::Full`](crate::ControllerError::Full), /// - `remove()`, `cancel()`, and `cancel_with_timeout()` wait for capacity, /// - `try_remove()`, `try_cancel()`, and `try_cancel_with_timeout()` return [`RuntimeError::CommandQueueFull`](crate::RuntimeError::CommandQueueFull). /// diff --git a/src/controller/core/handle.rs b/src/controller/core/handle.rs index cf9ea33..b016f54 100644 --- a/src/controller/core/handle.rs +++ b/src/controller/core/handle.rs @@ -26,8 +26,17 @@ impl ControllerHandle { /// `Ok(id)` means the channel accepted the command. /// /// The controller may not have applied the admission policy yet, and runtime admission happens later. + #[cfg(test)] pub async fn submit(&self, spec: ControllerSpec) -> Result { let id = TaskId::next(); + self.submit_prepared(id, spec).await + } + + pub(crate) async fn submit_prepared( + &self, + id: TaskId, + spec: ControllerSpec, + ) -> Result { self.tx .send(ControllerCommand::Submit(Submission { id, @@ -43,8 +52,17 @@ impl ControllerHandle { /// /// `ControllerError::Full` means the controller command channel is full. /// It does not mean the target slot queue is full. + #[cfg(test)] pub fn try_submit(&self, spec: ControllerSpec) -> Result { let id = TaskId::next(); + self.try_submit_prepared(id, spec) + } + + pub(crate) fn try_submit_prepared( + &self, + id: TaskId, + spec: ControllerSpec, + ) -> Result { self.tx .try_send(ControllerCommand::Submit(Submission { id, @@ -66,11 +84,20 @@ impl ControllerHandle { /// /// `Ok((id, rx))` means the channel accepted the command. /// The controller may not have applied the slot policy yet. + #[cfg(test)] pub async fn submit_and_watch( &self, spec: ControllerSpec, ) -> Result<(TaskId, oneshot::Receiver), ControllerError> { let id = TaskId::next(); + self.submit_prepared_and_watch(id, spec).await + } + + pub(crate) async fn submit_prepared_and_watch( + &self, + id: TaskId, + spec: ControllerSpec, + ) -> Result<(TaskId, oneshot::Receiver), ControllerError> { let (tx, rx) = oneshot::channel(); self.tx .send(ControllerCommand::Submit(Submission { @@ -88,11 +115,20 @@ impl ControllerHandle { /// The returned receiver has the same completion semantics as [`submit_and_watch`](Self::submit_and_watch). /// /// `ControllerError::Full` means the controller command channel is full, not that the target slot rejected the submission. + #[cfg(test)] pub fn try_submit_and_watch( &self, spec: ControllerSpec, ) -> Result<(TaskId, oneshot::Receiver), ControllerError> { let id = TaskId::next(); + self.try_submit_prepared_and_watch(id, spec) + } + + pub(crate) fn try_submit_prepared_and_watch( + &self, + id: TaskId, + spec: ControllerSpec, + ) -> Result<(TaskId, oneshot::Receiver), ControllerError> { let (tx, rx) = oneshot::channel(); self.tx .try_send(ControllerCommand::Submit(Submission { diff --git a/src/controller/core/identity.rs b/src/controller/core/identity.rs index 24b471c..4016e27 100644 --- a/src/controller/core/identity.rs +++ b/src/controller/core/identity.rs @@ -68,7 +68,7 @@ impl Controller { /// Removes one queued, not-yet-admitted submission by identity. /// /// Returns `true` only when this call claimed the queued submission. - /// A claimed watched submission resolves as `Rejected("removed_from_queue")` because its task body never ran. + /// A claimed watched submission resolves as `Rejected { kind: RemovedFromQueue, .. }` because its task body never ran. async fn remove_queued_submission( &self, id: TaskId, diff --git a/src/controller/core/queue.rs b/src/controller/core/queue.rs index ee1d1f0..02930e5 100644 --- a/src/controller/core/queue.rs +++ b/src/controller/core/queue.rs @@ -69,7 +69,7 @@ impl Controller { /// Implements latest-wins replacement for the queue head only. /// - /// If the queue has a head, that head is rejected as `superseded_by_replace` and replaced by the new submission. + /// If the queue has a head, that head is rejected with [`RejectionKind::SupersededByReplace`] and replaced by the new submission. /// FIFO items behind it stay in place. /// /// If the queue is empty, the new submission becomes the head. diff --git a/src/controller/core/tests.rs b/src/controller/core/tests.rs index 6c810d1..2ed6217 100644 --- a/src/controller/core/tests.rs +++ b/src/controller/core/tests.rs @@ -1260,7 +1260,7 @@ async fn registry_reply_marks_slot_running_without_task_added() { .await .expect("controller intake must accept the submission"); for _ in 0..16 { - controller_bus.publish(Event::new(EventKind::TaskStarting).with_task("noise")); + controller_bus.publish(Event::new(EventKind::AttemptStarting).with_task("noise")); } let reached_running = poll_until(Duration::from_secs(2), || async { diff --git a/src/controller/error.rs b/src/controller/error.rs index 4c7c28c..a044c32 100644 --- a/src/controller/error.rs +++ b/src/controller/error.rs @@ -16,8 +16,8 @@ pub enum ControllerError { /// The ordered controller command queue is full. /// - /// Returned only by `try_submit` and `try_submit_and_watch`. - /// Use async `submit` or `submit_and_watch` to wait for command capacity. + /// Returned only by fail-fast `try_submit*` methods, including those on [`PreparedSubmission`](crate::PreparedSubmission). + /// Use the corresponding async submit method to wait for command capacity. #[error("submission queue full")] Full, diff --git a/src/controller/mod.rs b/src/controller/mod.rs index 164b44d..b1cc7cb 100644 --- a/src/controller/mod.rs +++ b/src/controller/mod.rs @@ -74,7 +74,7 @@ //! `Queue` and `DropIfRunning` also leave the current owner's status unchanged. //! //! After a registered owner, the next queued task starts only after terminal registry cleanup. -//! At that point, the old task actor has been joined and its task name is free. +//! At that point, the old managed runner has been joined and its task name is free. //! If registration is rejected, no registered owner exists; the controller can try the next queued item as soon as it receives that direct decision. //! Events are only for observability; they do not drive slot state. //! @@ -84,9 +84,16 @@ //! - Configure with [`SupervisorBuilder::with_controller`](crate::SupervisorBuilder::with_controller). //! - Submit with [`SupervisorHandle::submit`](crate::SupervisorHandle::submit) or [`SupervisorHandle::submit_and_watch`](crate::SupervisorHandle::submit_and_watch). //! Their `try_*` forms return immediately instead of waiting when the command channel is full. +//! - When application correlation must exist before lifecycle events can start, +//! call [`SupervisorHandle::prepare_submission`](crate::SupervisorHandle::prepare_submission), +//! store its [`TaskId`](crate::TaskId), then consume the returned [`PreparedSubmission`]. //! - Remove or cancel by the [`TaskId`](crate::TaskId) returned from submission. //! - Read current slot state with [`SupervisorHandle::controller_snapshot`](crate::SupervisorHandle::controller_snapshot). //! +//! Slots control admission only. There is no slot-wide cancel/remove operation. +//! Canceling a registered owner by ID or name does not automatically purge a +//! queued replacement for the same slot. +//! //! ## Example //! //! ```rust,no_run @@ -118,6 +125,8 @@ //! The returned `TaskId` is allocated before runtime admission. //! If admission succeeds, the runtime uses the same ID. //! Before admission, it still identifies queued work for cancellation, outcomes, and event correlation. +//! A [`PreparedSubmission`] exposes that ID before controller intake. Preparing +//! alone does not enqueue work or publish an event. //! //! [`ControllerConfig::queue_capacity`] bounds the controller command queue and separately caps registry-backed remove/cancel operations. //! A new `Queue` submission is rejected when the slot's pending depth is already [`ControllerConfig::max_slot_queue`] or greater. @@ -166,6 +175,9 @@ pub(crate) use core::Controller; mod error; pub use error::ControllerError; +mod prepared; +pub use prepared::PreparedSubmission; + mod spec; pub use spec::ControllerSpec; diff --git a/src/controller/prepared.rs b/src/controller/prepared.rs new file mode 100644 index 0000000..2ff9bd0 --- /dev/null +++ b/src/controller/prepared.rs @@ -0,0 +1,148 @@ +//! A controller submission whose identity is visible before intake. + +use super::{ControllerError, ControllerSpec, core::ControllerHandle}; +use crate::{TaskId, TaskWaiter}; + +/// A single-use controller submission with a preallocated [`TaskId`]. +/// +/// Create this value with [`SupervisorHandle::prepare_submission`](crate::SupervisorHandle::prepare_submission). +/// Preparation allocates the identity but does not enqueue work or publish an event. +/// This lets an application install its own correlation mapping before the controller can emit an event for the submission. +/// +/// Call [`submit`](Self::submit), [`try_submit`](Self::try_submit), [`submit_and_watch`](Self::submit_and_watch), or +/// [`try_submit_and_watch`](Self::try_submit_and_watch) to consume the value and commit it to the same controller path used by [`SupervisorHandle`](crate::SupervisorHandle). +/// Dropping it without submitting starts no work and publishes no event. +/// +/// This type is intentionally not [`Clone`]. One prepared value can commit at most one controller submission. +/// +/// ## Example +/// +/// ```rust,no_run +/// use taskvisor::prelude::*; +/// +/// # #[tokio::main] +/// # async fn main() -> Result<(), Box> { +/// let supervisor = Supervisor::builder(SupervisorConfig::default()) +/// .with_controller(ControllerConfig::default()) +/// .build(); +/// let handle = supervisor.serve(); +/// +/// let task = TaskFn::arc("sync-tenant-42", |_ctx| async { Ok(()) }); +/// let request = ControllerSpec::replace(TaskSpec::once(task)).with_slot("tenant-42"); +/// let prepared = handle.prepare_submission(request)?; +/// let id = prepared.id(); +/// +/// // Store application_id -> id here. No event for `id` can exist yet. +/// let (submitted_id, waiter) = prepared.submit_and_watch().await?; +/// assert_eq!(submitted_id, id); +/// assert!(waiter.wait().await?.is_success()); +/// +/// handle.shutdown().await?; +/// # Ok(()) +/// # } +/// ``` +#[must_use = "a prepared submission starts no work until a submit method consumes it"] +pub struct PreparedSubmission { + controller: ControllerHandle, + id: TaskId, + spec: ControllerSpec, +} + +impl PreparedSubmission { + pub(crate) fn new(controller: ControllerHandle, spec: ControllerSpec) -> Self { + Self { + controller, + id: TaskId::next(), + spec, + } + } + + /// Returns the identity reserved for this submission. + /// + /// Preparation itself emits no event. + /// + /// After a submit method is called, this identity is used unchanged through controller admission, registry execution, events, cancellation, and the final outcome. + #[must_use] + pub fn id(&self) -> TaskId { + self.id + } + + /// Returns the controller specification that will be submitted. + #[must_use = "use the prepared controller specification"] + pub fn spec(&self) -> &ControllerSpec { + &self.spec + } + + /// Waits for controller-command capacity and commits this submission. + /// + /// `Ok(id)` confirms only controller intake. Slot admission and registry registration happen later. + /// + /// # Errors + /// + /// Returns [`ControllerError::Closed`] if the controller command channel is closed. + pub async fn submit(self) -> Result { + let Self { + controller, + id, + spec, + } = self; + controller.submit_prepared(id, spec).await + } + + /// Commits this submission only if controller-command capacity is available now. + /// + /// # Errors + /// + /// - [`ControllerError::Full`] when the controller command queue has no capacity. + /// - [`ControllerError::Closed`] when the controller command channel is closed. + pub fn try_submit(self) -> Result { + let Self { + controller, + id, + spec, + } = self; + controller.try_submit_prepared(id, spec) + } + + /// Waits for controller-command capacity, commits this submission, and returns its final-outcome waiter. + /// + /// The waiter has the same reliable completion semantics as [`SupervisorHandle::submit_and_watch`](crate::SupervisorHandle::submit_and_watch). + /// + /// # Errors + /// + /// Returns [`ControllerError::Closed`] if the controller command channel is closed. + pub async fn submit_and_watch(self) -> Result<(TaskId, TaskWaiter), ControllerError> { + let Self { + controller, + id, + spec, + } = self; + let (submitted_id, rx) = controller.submit_prepared_and_watch(id, spec).await?; + Ok((submitted_id, TaskWaiter::new(submitted_id, rx))) + } + + /// Commits this watched submission only if controller-command capacity is available now. + /// + /// # Errors + /// + /// - [`ControllerError::Full`] when the controller command queue has no capacity. + /// - [`ControllerError::Closed`] when the controller command channel is closed. + pub fn try_submit_and_watch(self) -> Result<(TaskId, TaskWaiter), ControllerError> { + let Self { + controller, + id, + spec, + } = self; + let (submitted_id, rx) = controller.try_submit_prepared_and_watch(id, spec)?; + Ok((submitted_id, TaskWaiter::new(submitted_id, rx))) + } +} + +impl std::fmt::Debug for PreparedSubmission { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PreparedSubmission") + .field("id", &self.id) + .field("spec", &self.spec) + .finish_non_exhaustive() + } +} diff --git a/src/controller/spec.rs b/src/controller/spec.rs index 576881a..e70e163 100644 --- a/src/controller/spec.rs +++ b/src/controller/spec.rs @@ -50,6 +50,7 @@ use crate::TaskSpec; /// /// - [`AdmissionPolicy`] - how concurrent submissions to the same slot are handled /// - [`TaskSpec`] - task restart, backoff, timeout, and retry settings +/// - [`SupervisorHandle::prepare_submission`](crate::SupervisorHandle::prepare_submission) - reserve the submission identity before events can start /// - [`SupervisorHandle::submit`](crate::SupervisorHandle::submit) - submit without waiting for the final outcome /// - [`SupervisorHandle::submit_and_watch`](crate::SupervisorHandle::submit_and_watch) - submit and get a [`TaskWaiter`](crate::TaskWaiter) for the final [`TaskOutcome`](crate::TaskOutcome) #[derive(Clone)] diff --git a/src/core/actor.rs b/src/core/actor.rs index 70dfd65..a4387a8 100644 --- a/src/core/actor.rs +++ b/src/core/actor.rs @@ -6,7 +6,7 @@ //! ## Flow //! //! ```text -//! wait for permit -> TaskStarting -> run attempt -> release permit -> apply policy +//! wait for permit -> AttemptStarting -> run attempt -> release permit -> apply policy //! ``` //! //! | Attempt result | Actor decision | @@ -20,10 +20,11 @@ //! //! ## Events and Final State //! -//! `TaskStopped`, `TaskCanceled`, and `TaskFailed` describe one attempt. -//! `BackoffScheduled`, `ActorExhausted`, and `ActorDead` describe the actor's decision. +//! `AttemptSucceeded`, `AttemptCanceled`, and `AttemptFailed` describe one attempt. +//! `BackoffScheduled` describes the decision to start another attempt. +//! After this actor returns, the registry publishes one typed `TaskFinished` event from the resulting [`TaskOutcome`](crate::TaskOutcome). //! -//! A `TaskFailed` event is not a final outcome; another attempt may follow it. +//! > An `AttemptFailed` event is not a final outcome; another attempt may follow it. //! //! ## Rules //! @@ -51,7 +52,6 @@ use crate::{ events::{Bus, Event, EventKind}, identity::TaskId, policies::{BackoffPolicy, RestartPolicy}, - reasons, tasks::Task, }; @@ -90,7 +90,7 @@ pub(crate) enum ActorExitReason { /// - the error is not retryable /// - the retry budget (`max_retries`) is used up Exhausted { - /// Final failure message. Same text as the `ActorExhausted` event reason. + /// Diagnostic final failure message. reason: Arc, /// Numeric exit code from a process-like task, if any. exit_code: Option, @@ -101,14 +101,14 @@ pub(crate) enum ActorExitReason { /// Actor stopped because of runtime shutdown, explicit removal, or `TaskError::Canceled`. /// /// This maps to [`TaskOutcome::Canceled`](crate::TaskOutcome). - /// Depending on where cancellation happened, there may be no actor-level terminal event. + /// Depending on where cancellation happened, there may be no attempt-level terminal event. Canceled, /// Actor stopped because the task returned a fatal error. /// /// Fatal errors are not retried. Fatal { - /// Fatal error message. Same text as the `ActorDead` event reason. + /// Diagnostic fatal error message. reason: Arc, /// Numeric exit code from a process-like task, if any. exit_code: Option, @@ -193,16 +193,7 @@ impl TaskActor { tokio::select! { res = &mut fut => match res { Ok(p) => Some(p), - Err(_closed) => { - self.bus.publish( - Event::new(EventKind::ActorExhausted) - .with_task(task_name.clone()) - .with_id(id) - .with_attempt(attempt) - .with_reason("semaphore_closed"), - ); - return ActorExitReason::Canceled; - } + Err(_closed) => return ActorExitReason::Canceled, }, _ = runtime_token.cancelled() => { return ActorExitReason::Canceled; @@ -219,7 +210,7 @@ impl TaskActor { attempt = attempt.saturating_add(1); self.bus.publish( - Event::new(EventKind::TaskStarting) + Event::new(EventKind::AttemptStarting) .with_task(task_name.clone()) .with_id(id) .with_attempt(attempt), @@ -276,13 +267,6 @@ impl TaskActor { if runtime_token.is_cancelled() { return ActorExitReason::Canceled; } - self.bus.publish( - Event::new(EventKind::ActorExhausted) - .with_task(task_name.clone()) - .with_id(id) - .with_attempt(attempt) - .with_reason(reasons::POLICY_EXHAUSTED_SUCCESS), - ); return ActorExitReason::Completed; } } @@ -292,15 +276,6 @@ impl TaskActor { let exit_code = e.exit_code(); let source: Option = e.into_source().map(Arc::from); - let mut ev = Event::new(EventKind::ActorDead) - .with_task(task_name.clone()) - .with_id(id) - .with_attempt(attempt) - .with_reason(Arc::clone(&reason)); - if let Some(code) = exit_code { - ev = ev.with_exit_code(code); - } - self.bus.publish(ev); return ActorExitReason::Fatal { reason, exit_code, @@ -308,16 +283,6 @@ impl TaskActor { }; } Err(TaskError::Canceled) => { - if runtime_token.is_cancelled() { - return ActorExitReason::Canceled; - } - self.bus.publish( - Event::new(EventKind::ActorExhausted) - .with_task(task_name.clone()) - .with_id(id) - .with_attempt(attempt) - .with_reason(reasons::TASK_RETURNED_CANCELED), - ); return ActorExitReason::Canceled; } Err(e) => { @@ -336,11 +301,8 @@ impl TaskActor { self.params.max_retries.filter(|_| retries_exhausted) { Arc::from(format!( - "{}({}/{}): {}", - reasons::MAX_RETRIES_EXCEEDED, - backoff_attempt, - limit.get(), - e + "retry limit reached after {backoff_attempt} of {} retries: {e}", + limit.get() )) } else { Arc::from(e.to_string()) @@ -348,15 +310,6 @@ impl TaskActor { let exit_code = e.exit_code(); let source: Option = e.into_source().map(Arc::from); - let mut ev = Event::new(EventKind::ActorExhausted) - .with_task(task_name.clone()) - .with_id(id) - .with_attempt(attempt) - .with_reason(Arc::clone(&reason)); - if let Some(code) = exit_code { - ev = ev.with_exit_code(code); - } - self.bus.publish(ev); return ActorExitReason::Exhausted { reason, exit_code, @@ -530,10 +483,7 @@ mod tests { let reason = a.run(CancellationToken::new()).await; match reason { ActorExitReason::Exhausted { reason, .. } => { - assert!( - reason.contains("max_retries_exceeded"), - "reason must mention exhausted budget: {reason}" - ); + assert!(reason.contains("retry limit reached")); } other => panic!("expected Exhausted, got {other:?}"), } diff --git a/src/core/alive.rs b/src/core/alive.rs index a0b597b..a5c1864 100644 --- a/src/core/alive.rs +++ b/src/core/alive.rs @@ -24,13 +24,13 @@ //! //! - Lifecycle events create and update entries. Reconciliation can remove them. //! - While an entry exists, events with `seq <= last_seq` for the same key are ignored as stale. -//! - `TaskStarting` marks the current attempt as alive. +//! - `AttemptStarting` marks the current attempt as alive. //! - Stop, failure, cancellation, and actor-end events mark it as not alive. //! - `TaskRemoved` deletes the cache entry. //! - `reconcile` removes identities no longer present in the registry. //! - Queries can lag or miss state because event delivery is best-effort. //! -//! Alive means that the latest applied lifecycle event for a task run is `TaskStarting`. +//! Alive means that the latest applied lifecycle event for a task run is `AttemptStarting`. //! A registered task can be marked not alive while it waits for a permit, retry, successful restart, or terminal cleanup. use std::{ @@ -93,18 +93,18 @@ impl AliveTracker { /// Events are applied only when `ev.seq > last_seq` for the same cache key ([`TaskId`] when present, otherwise the task name): /// /// ```text - /// update(TaskStopped, seq=100) -> alive=false, last_seq=100 - /// update(TaskStarting, seq=99) -> ignored as stale + /// update(AttemptSucceeded, seq=100) -> alive=false, last_seq=100 + /// update(AttemptStarting, seq=99) -> ignored as stale /// ``` pub async fn update(&self, ev: &Event) -> bool { let relevant = matches!( ev.kind, - EventKind::TaskStarting - | EventKind::TaskStopped - | EventKind::TaskCanceled - | EventKind::TaskFailed - | EventKind::ActorExhausted - | EventKind::ActorDead + EventKind::AttemptStarting + | EventKind::AttemptSucceeded + | EventKind::AttemptCanceled + | EventKind::AttemptFailed + | EventKind::AttemptTimedOut + | EventKind::TaskFinished | EventKind::TaskRemoved ); if !relevant { @@ -144,12 +144,12 @@ impl AliveTracker { } let next_alive = match ev.kind { - EventKind::TaskStarting => true, - EventKind::TaskStopped - | EventKind::TaskCanceled - | EventKind::TaskFailed - | EventKind::ActorExhausted - | EventKind::ActorDead => false, + EventKind::AttemptStarting => true, + EventKind::AttemptSucceeded + | EventKind::AttemptCanceled + | EventKind::AttemptFailed + | EventKind::AttemptTimedOut + | EventKind::TaskFinished => false, _ => entry.alive, }; @@ -216,7 +216,7 @@ mod tests { .update(&evi(EventKind::BackoffScheduled, "slot-name", 1, id)) .await; tracker - .update(&evi(EventKind::TaskStarting, "real-name", 2, id)) + .update(&evi(EventKind::AttemptStarting, "real-name", 2, id)) .await; assert!( @@ -236,10 +236,10 @@ mod tests { let new = crate::identity::TaskId::next(); tracker - .update(&evi(EventKind::TaskStarting, "x", 1, old)) + .update(&evi(EventKind::AttemptStarting, "x", 1, old)) .await; tracker - .update(&evi(EventKind::TaskStarting, "x", 3, new)) + .update(&evi(EventKind::AttemptStarting, "x", 3, new)) .await; tracker .update(&evi(EventKind::TaskRemoved, "x", 4, old)) @@ -264,10 +264,12 @@ mod tests { async fn stale_and_equal_sequence_events_are_rejected() { for (incoming_seq, case) in [(99, "stale"), (100, "equal")] { let tracker = AliveTracker::new(); - tracker.update(&ev(EventKind::TaskStopped, "t1", 100)).await; + tracker + .update(&ev(EventKind::AttemptSucceeded, "t1", 100)) + .await; let changed = tracker - .update(&ev(EventKind::TaskStarting, "t1", incoming_seq)) + .update(&ev(EventKind::AttemptStarting, "t1", incoming_seq)) .await; assert!(!changed, "{case} event must not change state"); assert!( @@ -281,8 +283,10 @@ mod tests { async fn task_removed_deletes_entry() { let tracker = AliveTracker::new(); assert!( - tracker.update(&ev(EventKind::TaskStarting, "t1", 1)).await, - "first TaskStarting should change alive from false to true" + tracker + .update(&ev(EventKind::AttemptStarting, "t1", 1)) + .await, + "first AttemptStarting should change alive from false to true" ); assert!(tracker.is_alive("t1").await); @@ -290,14 +294,18 @@ mod tests { assert!(changed, "TaskRemoved should report change"); assert!(!tracker.is_alive("t1").await); - tracker.update(&ev(EventKind::TaskStarting, "t1", 3)).await; + tracker + .update(&ev(EventKind::AttemptStarting, "t1", 3)) + .await; assert!(tracker.is_alive("t1").await, "fresh entry after removal"); } #[tokio::test] async fn stale_task_removed_ignored() { let tracker = AliveTracker::new(); - tracker.update(&ev(EventKind::TaskStarting, "t1", 10)).await; + tracker + .update(&ev(EventKind::AttemptStarting, "t1", 10)) + .await; let changed = tracker.update(&ev(EventKind::TaskRemoved, "t1", 5)).await; assert!(!changed); @@ -319,16 +327,16 @@ mod tests { async fn snapshot_returns_alive_sorted() { let tracker = AliveTracker::new(); tracker - .update(&ev(EventKind::TaskStarting, "charlie", 1)) + .update(&ev(EventKind::AttemptStarting, "charlie", 1)) .await; tracker - .update(&ev(EventKind::TaskStarting, "alpha", 2)) + .update(&ev(EventKind::AttemptStarting, "alpha", 2)) .await; tracker - .update(&ev(EventKind::TaskStarting, "bravo", 3)) + .update(&ev(EventKind::AttemptStarting, "bravo", 3)) .await; tracker - .update(&ev(EventKind::TaskStopped, "bravo", 4)) + .update(&ev(EventKind::AttemptSucceeded, "bravo", 4)) .await; let alive = tracker.snapshot().await; @@ -339,7 +347,7 @@ mod tests { #[tokio::test] async fn event_without_task_name_ignored() { let tracker = AliveTracker::new(); - let mut e = Event::new(EventKind::TaskStarting); + let mut e = Event::new(EventKind::AttemptStarting); e.seq = 1; let changed = tracker.update(&e).await; assert!(!changed); @@ -349,7 +357,9 @@ mod tests { #[tokio::test] async fn non_lifecycle_event_is_ignored_and_keeps_alive() { let tracker = AliveTracker::new(); - tracker.update(&ev(EventKind::TaskStarting, "t1", 1)).await; + tracker + .update(&ev(EventKind::AttemptStarting, "t1", 1)) + .await; let changed = tracker .update(&ev(EventKind::BackoffScheduled, "t1", 2)) @@ -357,22 +367,26 @@ mod tests { assert!(!changed, "non-lifecycle event should be ignored"); assert!(tracker.is_alive("t1").await); - let changed = tracker.update(&ev(EventKind::TaskStopped, "t1", 2)).await; - assert!(changed, "TaskStopped with seq=2 should still apply"); + let changed = tracker + .update(&ev(EventKind::AttemptSucceeded, "t1", 2)) + .await; + assert!(changed, "AttemptSucceeded with seq=2 should still apply"); assert!(!tracker.is_alive("t1").await); } #[tokio::test] async fn all_death_events_set_alive_false() { for kind in [ - EventKind::TaskStopped, - EventKind::TaskCanceled, - EventKind::TaskFailed, - EventKind::ActorExhausted, - EventKind::ActorDead, + EventKind::AttemptSucceeded, + EventKind::AttemptCanceled, + EventKind::AttemptFailed, + EventKind::AttemptTimedOut, + EventKind::TaskFinished, ] { let tracker = AliveTracker::new(); - tracker.update(&ev(EventKind::TaskStarting, "t", 1)).await; + tracker + .update(&ev(EventKind::AttemptStarting, "t", 1)) + .await; let changed = tracker.update(&ev(kind, "t", 2)).await; assert!(changed, "{kind:?} should set alive=false"); assert!( @@ -389,10 +403,10 @@ mod tests { let orphan = crate::identity::TaskId::next(); tracker - .update(&evi(EventKind::TaskStarting, "kept", 1, kept)) + .update(&evi(EventKind::AttemptStarting, "kept", 1, kept)) .await; tracker - .update(&evi(EventKind::TaskStarting, "orphan", 2, orphan)) + .update(&evi(EventKind::AttemptStarting, "orphan", 2, orphan)) .await; assert!(tracker.is_alive("orphan").await); diff --git a/src/core/handle.rs b/src/core/handle.rs index 7e5b753..18933db 100644 --- a/src/core/handle.rs +++ b/src/core/handle.rs @@ -21,6 +21,7 @@ //! |--------------------------------------------|-------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------| //! | `add`, `try_add` | Registry admission and runner creation | First attempt started | //! | `add_and_watch`, `try_add_and_watch` | Registry admission; the returned waiter later confirms terminal cleanup | Task success at method return | +//! | `prepare_submission` | A controller `TaskId` is reserved and visible to the caller | Queue intake; preparation publishes no event | //! | `submit`, `try_submit` | Controller command-queue acceptance | Slot admission, registry admission, or task start | //! | `submit_and_watch`, `try_submit_and_watch` | Controller command-queue acceptance; the returned waiter later confirms rejection or terminal cleanup | Slot or registry admission at method return | //! | `remove`, `try_remove` | The stop claim was decided; queued controller work is removed before return | Terminal cleanup of registered work | @@ -149,7 +150,7 @@ impl SupervisorHandle { /// Adds a task and returns a waiter for its final outcome. /// /// Registration is the same as [`add`](Self::add). - /// The [`TaskWaiter`] resolves after all retries end, the registry joins the task actor, and registry membership is removed. + /// The [`TaskWaiter`] resolves after all retries end, the registry joins the managed runner, and registry membership is removed. /// /// > It uses a direct completion channel, not best-effort lifecycle events. /// @@ -268,7 +269,7 @@ impl SupervisorHandle { /// Returns the authoritative registry view as `(id, name)` pairs. /// /// The list comes from the registry and is sorted by [`TaskId`]. - /// It includes every registry entry, whether its actor is running, waiting for a permit, backoff, or restart interval, already finished but not cleaned up, or being removed. + /// It includes every registry entry, whether its managed task is running, waiting for a permit, in backoff or a restart interval, already finished but not cleaned up, or being removed. /// /// See [`alive_snapshot`](Self::alive_snapshot) for the best-effort list of task names currently marked alive. pub async fn list(&self) -> Vec<(TaskId, Arc)> { @@ -308,7 +309,7 @@ impl SupervisorHandle { /// Cancels work by identity and waits for terminal cleanup. /// - /// For registered work, this returns after the actor is joined and its name and identity are released. + /// For registered work, this returns after the managed runner is joined and its name and identity are released. /// For queued controller work, removal is already complete when this method returns. /// /// `Ok(true)` is returned only to the call that claimed the stop. @@ -482,6 +483,34 @@ impl SupervisorHandle { self.core().shutdown().await } + /// Prepares a controller submission and exposes its identity before intake. + /// + /// This allocates the [`TaskId`] but does not enqueue work or publish an event. + /// Install any application-level correlation for [`PreparedSubmission::id`](crate::PreparedSubmission::id), then consume the prepared value with one of its submit methods. + /// + /// Use the ordinary [`submit`](Self::submit) and [`submit_and_watch`](Self::submit_and_watch) shortcuts when correlation does not need to exist before lifecycle events can start. + /// + /// Requires the `controller` feature. + /// + /// # Errors + /// + /// Returns [`ControllerError::NotConfigured`](crate::ControllerError::NotConfigured) + /// when this supervisor was built without a controller. + #[cfg(feature = "controller")] + #[cfg_attr(docsrs, doc(cfg(feature = "controller")))] + pub fn prepare_submission( + &self, + spec: crate::controller::ControllerSpec, + ) -> Result { + match &self.controller { + Some(controller) => Ok(crate::controller::PreparedSubmission::new( + controller.handle(), + spec, + )), + None => Err(crate::controller::ControllerError::NotConfigured), + } + } + /// Sends a task to the controller and returns its reserved [`TaskId`]. /// /// `Ok(id)` confirms only that the controller queue accepted the submission. @@ -502,10 +531,7 @@ impl SupervisorHandle { &self, spec: crate::controller::ControllerSpec, ) -> Result { - match &self.controller { - Some(ctrl) => ctrl.handle().submit(spec).await, - None => Err(crate::controller::ControllerError::NotConfigured), - } + self.prepare_submission(spec)?.submit().await } /// Submits only if the controller queue has capacity now. @@ -525,10 +551,7 @@ impl SupervisorHandle { &self, spec: crate::controller::ControllerSpec, ) -> Result { - match &self.controller { - Some(ctrl) => ctrl.handle().try_submit(spec), - None => Err(crate::controller::ControllerError::NotConfigured), - } + self.prepare_submission(spec)?.try_submit() } /// Sends a task to the controller and returns a final-outcome waiter. @@ -536,7 +559,7 @@ impl SupervisorHandle { /// The waiter uses the direct completion channel, not the best-effort event bus. /// It normally resolves to [`TaskOutcome::Rejected`](crate::TaskOutcome::Rejected) when controller or registry admission rejects the submission. /// If the completion sender closes first, [`TaskWaiter::wait`] returns [`RuntimeError::ShuttingDown`]. - /// If admitted, the waiter resolves after the task finishes and its actor is joined. + /// If admitted, the waiter resolves after the task finishes and its managed runner is joined. /// /// Requires the `controller` feature. /// @@ -550,13 +573,7 @@ impl SupervisorHandle { &self, spec: crate::controller::ControllerSpec, ) -> Result<(TaskId, TaskWaiter), crate::controller::ControllerError> { - match &self.controller { - Some(ctrl) => { - let (id, rx) = ctrl.handle().submit_and_watch(spec).await?; - Ok((id, TaskWaiter::new(id, rx))) - } - None => Err(crate::controller::ControllerError::NotConfigured), - } + self.prepare_submission(spec)?.submit_and_watch().await } /// Submits watched work only if the controller queue has capacity now. @@ -577,13 +594,7 @@ impl SupervisorHandle { &self, spec: crate::controller::ControllerSpec, ) -> Result<(TaskId, TaskWaiter), crate::controller::ControllerError> { - match &self.controller { - Some(ctrl) => { - let (id, rx) = ctrl.handle().try_submit_and_watch(spec)?; - Ok((id, TaskWaiter::new(id, rx))) - } - None => Err(crate::controller::ControllerError::NotConfigured), - } + self.prepare_submission(spec)?.try_submit_and_watch() } /// Returns a best-effort rolling snapshot of controller slots. diff --git a/src/core/mod.rs b/src/core/mod.rs index ec951bb..21fe8c6 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -49,7 +49,7 @@ //! - Event sequence numbers help sort observations, but do not prove causal order between concurrent tasks. mod outcome; -pub use outcome::{TaskOutcome, TaskWaiter}; +pub use outcome::{TaskOutcome, TaskOutcomeKind, TaskWaiter}; mod runtime; pub(crate) use runtime::SupervisorCore; diff --git a/src/core/outcome.rs b/src/core/outcome.rs index 26ae6d9..6b487ee 100644 --- a/src/core/outcome.rs +++ b/src/core/outcome.rs @@ -5,13 +5,13 @@ //! //! A [`TaskWaiter`] answers "how did this task end?" for one [`TaskId`]. //! It receives one final [`TaskOutcome`] through a direct one-shot channel, outside the event bus. -//! For an admitted task, the registry sends the outcome after joining the actor and removing its membership. +//! For an admitted task, the registry sends the outcome after joining its managed runner and removing registry membership. //! Event-bus lag does not affect this path. If the outcome cannot be delivered, [`TaskWaiter::wait`] returns an error instead of guessing the result. //! //! ## Successful Direct-Add Flow //! //! ```text -//! Caller Runtime Task actor +//! Caller Runtime Managed runner //! │ │ │ //! ├── add_and_watch(spec) ───────►│ │ //! │ ├── register and spawn ─────────►│ @@ -19,7 +19,7 @@ //! │ │ │ attempts / retries //! │ await waiter.wait() │ │ //! │ │◄─────── terminal signal ───────┤ -//! │ │ join actor │ +//! │ │ join runner │ //! │ │ remove TaskId and name │ //! │◄──── TaskOutcome (oneshot) ───┤ │ //! ``` @@ -30,7 +30,7 @@ //! ## Guarantees //! //! - One waiter follows one [`TaskId`]. -//! - For admitted work, it resolves after all retries end and the registry joins the task actor. +//! - For admitted work, it resolves after all retries end and the registry joins the managed runner. //! - Dropping a waiter is safe and does not cancel the task. //! - If the runtime drops the sender before it creates an outcome, [`TaskWaiter::wait`] returns an error instead of inventing a result. //! @@ -46,9 +46,51 @@ use crate::error::{RuntimeError, SharedError}; use crate::events::RejectionKind; use crate::identity::TaskId; +/// Machine-readable category of a final [`TaskOutcome`]. +/// +/// This lightweight enum mirrors [`TaskOutcome`] without carrying diagnostic +/// text or source errors. It is used by lifecycle [`Event`](crate::Event) +/// values so metrics, dashboards, and alerts never need to parse `reason`. +/// +/// Match with a wildcard arm because new outcome categories may be added. +#[non_exhaustive] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum TaskOutcomeKind { + /// Final attempt succeeded and policy stopped the task. + Completed, + /// A non-fatal failure reached a policy or retry-limit stop condition. + Failed, + /// The task reported a permanent failure. + Fatal, + /// Cancellation was requested or reported cooperatively. + Canceled, + /// The runtime aborted the task before cooperative stop completed. + ForceAborted, + /// The internal task runner panicked. + Panicked, + /// Admission rejected the work before its task body ran. + Rejected, +} + +impl TaskOutcomeKind { + /// Returns the stable machine-readable label used by events, logs, and metrics. + #[must_use] + pub const fn as_label(self) -> &'static str { + match self { + Self::Completed => "outcome_completed", + Self::Failed => "outcome_failed", + Self::Fatal => "outcome_fatal", + Self::Canceled => "outcome_canceled", + Self::ForceAborted => "outcome_force_aborted", + Self::Panicked => "outcome_panicked", + Self::Rejected => "outcome_rejected", + } + } +} + /// Final result of one watched task or controller submission. /// -/// For admitted work, this value is sent after the retry loop ends, the actor is joined, and registry membership is removed. +/// For admitted work, this value is sent after the retry loop ends, the managed runner is joined, and registry membership is removed. /// A controller can instead return [`Rejected`](Self::Rejected) before the task starts. /// /// This enum is non-exhaustive. @@ -68,7 +110,7 @@ use crate::identity::TaskId; /// | [`Fatal`](Self::Fatal) | Task reported a permanent failure | /// | [`Canceled`](Self::Canceled) | Cooperative cancellation | /// | [`ForceAborted`](Self::ForceAborted) | Runtime aborted before cooperative stop | -/// | [`Panicked`](Self::Panicked) | Internal actor panicked | +/// | [`Panicked`](Self::Panicked) | Internal task runner panicked | /// | [`Rejected`](Self::Rejected) | Task body never ran | /// /// ## See Also @@ -95,7 +137,10 @@ pub enum TaskOutcome { /// - the retry budget is used up. #[non_exhaustive] Failed { - /// Final failure message. Same text as the `ActorExhausted` event reason. + /// Diagnostic final failure message. + /// + /// This text is not a machine-readable category and may change. + /// Use [`TaskOutcome::kind`] for branching, metrics, and alerts. reason: Arc, /// Numeric exit code from a process-like task, if any. exit_code: Option, @@ -108,7 +153,10 @@ pub enum TaskOutcome { /// Fatal errors are not retried. #[non_exhaustive] Fatal { - /// Fatal error message. Same text as the `ActorDead` event reason. + /// Diagnostic fatal error message. + /// + /// This text is not a machine-readable category and may change. + /// Use [`TaskOutcome::kind`] for branching, metrics, and alerts. reason: Arc, /// Numeric exit code from a process-like task, if any. exit_code: Option, @@ -121,13 +169,13 @@ pub enum TaskOutcome { /// This can come from shutdown, explicit removal, or the task returning [`TaskError::Canceled`](crate::TaskError::Canceled). Canceled, - /// The runtime aborted the actor before cooperative stop completed. + /// The runtime aborted the managed task runner before cooperative stop completed. /// /// This normally happens after the configured grace period. /// Last-owner fallback and signal-setup failure cleanup cannot wait for that period. ForceAborted, - /// The internal actor panicked. + /// The internal task runner panicked. /// /// This guards against a runtime bug. /// Panics inside the user task are caught earlier and become retryable failures instead. @@ -146,12 +194,28 @@ pub enum TaskOutcome { Rejected { /// Stable category for machine-readable handling. kind: RejectionKind, - /// Readable rejection details. + /// Readable diagnostic rejection details. + /// + /// Use `kind` instead of parsing this text. reason: Arc, }, } impl TaskOutcome { + /// Returns the machine-readable category of this outcome. + #[must_use] + pub const fn kind(&self) -> TaskOutcomeKind { + match self { + TaskOutcome::Completed => TaskOutcomeKind::Completed, + TaskOutcome::Failed { .. } => TaskOutcomeKind::Failed, + TaskOutcome::Fatal { .. } => TaskOutcomeKind::Fatal, + TaskOutcome::Canceled => TaskOutcomeKind::Canceled, + TaskOutcome::ForceAborted => TaskOutcomeKind::ForceAborted, + TaskOutcome::Panicked => TaskOutcomeKind::Panicked, + TaskOutcome::Rejected { .. } => TaskOutcomeKind::Rejected, + } + } + /// Returns `true` only for [`Completed`](Self::Completed). #[must_use] pub fn is_success(&self) -> bool { @@ -205,7 +269,7 @@ impl TaskOutcome { /// /// let outcome = TaskOutcome::rejected_for_tests( /// taskvisor::RejectionKind::QueueFull, - /// "queue_full", + /// "slot queue reached capacity", /// ); /// assert_eq!(outcome.as_label(), "outcome_rejected"); /// ``` @@ -242,15 +306,7 @@ impl TaskOutcome { /// Useful for logs, metrics, and telemetry. #[must_use] pub fn as_label(&self) -> &'static str { - match self { - TaskOutcome::Completed => "outcome_completed", - TaskOutcome::Failed { .. } => "outcome_failed", - TaskOutcome::Fatal { .. } => "outcome_fatal", - TaskOutcome::Canceled => "outcome_canceled", - TaskOutcome::ForceAborted => "outcome_force_aborted", - TaskOutcome::Panicked => "outcome_panicked", - TaskOutcome::Rejected { .. } => "outcome_rejected", - } + self.kind().as_label() } } @@ -261,7 +317,7 @@ impl TaskOutcome { /// - [`SupervisorHandle::try_add_and_watch`](crate::SupervisorHandle::try_add_and_watch) #[cfg_attr( feature = "controller", - doc = "- [`SupervisorHandle::submit_and_watch`](crate::SupervisorHandle::submit_and_watch)\n- [`SupervisorHandle::try_submit_and_watch`](crate::SupervisorHandle::try_submit_and_watch)" + doc = "- [`SupervisorHandle::submit_and_watch`](crate::SupervisorHandle::submit_and_watch)\n- [`SupervisorHandle::try_submit_and_watch`](crate::SupervisorHandle::try_submit_and_watch)\n- [`PreparedSubmission::submit_and_watch`](crate::PreparedSubmission::submit_and_watch)\n- [`PreparedSubmission::try_submit_and_watch`](crate::PreparedSubmission::try_submit_and_watch)" )] /// /// [`wait`](Self::wait) consumes the waiter. @@ -342,11 +398,14 @@ mod tests { TaskOutcome::Fatal { reason, exit_code: None, .. } if reason.as_ref() == "bad config" )); - let rejected = TaskOutcome::rejected_for_tests(RejectionKind::QueueFull, "queue_full"); + let rejected = TaskOutcome::rejected_for_tests( + RejectionKind::QueueFull, + "slot queue reached capacity", + ); assert!(matches!( &rejected, TaskOutcome::Rejected { kind: RejectionKind::QueueFull, reason, .. } - if reason.as_ref() == "queue_full" + if reason.as_ref() == "slot queue reached capacity" )); assert!(rejected.source().is_none()); } @@ -354,13 +413,19 @@ mod tests { #[test] fn labels_and_success_flags_are_stable_for_every_variant() { let cases = [ - (TaskOutcome::Completed, "outcome_completed", true), + ( + TaskOutcome::Completed, + TaskOutcomeKind::Completed, + "outcome_completed", + true, + ), ( TaskOutcome::Failed { reason: Arc::from("x"), exit_code: None, source: None, }, + TaskOutcomeKind::Failed, "outcome_failed", false, ), @@ -370,17 +435,34 @@ mod tests { exit_code: Some(1), source: None, }, + TaskOutcomeKind::Fatal, "outcome_fatal", false, ), - (TaskOutcome::Canceled, "outcome_canceled", false), - (TaskOutcome::ForceAborted, "outcome_force_aborted", false), - (TaskOutcome::Panicked, "outcome_panicked", false), + ( + TaskOutcome::Canceled, + TaskOutcomeKind::Canceled, + "outcome_canceled", + false, + ), + ( + TaskOutcome::ForceAborted, + TaskOutcomeKind::ForceAborted, + "outcome_force_aborted", + false, + ), + ( + TaskOutcome::Panicked, + TaskOutcomeKind::Panicked, + "outcome_panicked", + false, + ), ( TaskOutcome::Rejected { kind: RejectionKind::AdmissionFailed, reason: Arc::from("x"), }, + TaskOutcomeKind::Rejected, "outcome_rejected", false, ), @@ -388,11 +470,15 @@ mod tests { let labels: std::collections::HashSet<_> = cases .iter() - .map(|(outcome, expected_label, expected_success)| { - assert_eq!(outcome.as_label(), *expected_label); - assert_eq!(outcome.is_success(), *expected_success, "{expected_label}"); - outcome.as_label() - }) + .map( + |(outcome, expected_kind, expected_label, expected_success)| { + assert_eq!(outcome.kind(), *expected_kind); + assert_eq!(outcome.as_label(), *expected_label); + assert_eq!(expected_kind.as_label(), *expected_label); + assert_eq!(outcome.is_success(), *expected_success, "{expected_label}"); + outcome.as_label() + }, + ) .collect(); assert_eq!(labels.len(), cases.len(), "labels must remain distinct"); } diff --git a/src/core/registry/removal.rs b/src/core/registry/removal.rs index 76fe8d8..94eaa95 100644 --- a/src/core/registry/removal.rs +++ b/src/core/registry/removal.rs @@ -17,7 +17,6 @@ use crate::{ core::{actor::ActorExitReason, outcome::TaskOutcome}, events::{Bus, Event, EventKind}, identity::TaskId, - reasons, }; /// Terminal result passed from the single join owner to registry cleanup. @@ -418,15 +417,7 @@ impl Registry { Self::report_join(bus, id, &entry.label, res, outcome); } JoinCompletion::ForceAborted => { - if let Some(done) = outcome { - let _ = done.send(TaskOutcome::ForceAborted); - } - bus.publish( - Event::new(EventKind::TaskRemoved) - .with_task(Arc::clone(&entry.label)) - .with_id(id) - .with_reason(reasons::FORCE_TERMINATED_AFTER_GRACE), - ); + Self::report_outcome(bus, id, &entry.label, TaskOutcome::ForceAborted, outcome); } } pending_joins.dec(id); @@ -444,7 +435,7 @@ impl Registry { /// Reports the result of a joined actor. /// - /// Sends the watched [`TaskOutcome`] if present, publishes `ActorDead` for an actor panic, and always publishes `TaskRemoved` for this joined actor. + /// Maps the join result once, then publishes and delivers the same final outcome. fn report_join( bus: &Bus, id: TaskId, @@ -452,18 +443,48 @@ impl Registry { res: Result, done: Option, ) { - if let Err(e) = &res - && e.is_panic() - { - bus.publish( - Event::new(EventKind::ActorDead) - .with_task(name) - .with_id(id) - .with_reason("actor_panic"), - ); + let outcome = Self::outcome_of(res); + Self::report_outcome(bus, id, name, outcome, done); + } + + /// Publishes one typed terminal event, delivers the watched outcome, then publishes registry removal. + /// + /// The event and waiter are classified from the same [`TaskOutcome`] value. + /// `reason` remains diagnostic; callers branch on [`TaskOutcomeKind`](crate::TaskOutcomeKind). + fn report_outcome( + bus: &Bus, + id: TaskId, + name: &str, + outcome: TaskOutcome, + done: Option, + ) { + let mut finished = Event::new(EventKind::TaskFinished) + .with_task(name) + .with_id(id) + .with_outcome_kind(outcome.kind()); + match &outcome { + TaskOutcome::Failed { + reason, exit_code, .. + } + | TaskOutcome::Fatal { + reason, exit_code, .. + } => { + finished = finished.with_reason(Arc::clone(reason)); + if let Some(code) = exit_code { + finished = finished.with_exit_code(*code); + } + } + TaskOutcome::ForceAborted => { + finished = finished.with_reason("task did not stop within grace; force-aborted"); + } + TaskOutcome::Panicked => { + finished = finished.with_reason("internal task runner panicked"); + } + TaskOutcome::Completed | TaskOutcome::Canceled | TaskOutcome::Rejected { .. } => {} } + bus.publish(finished); if let Some(done) = done { - let _ = done.send(Self::outcome_of(res)); + let _ = done.send(outcome); } bus.publish( Event::new(EventKind::TaskRemoved) diff --git a/src/core/registry/tests.rs b/src/core/registry/tests.rs index 5ff5989..4646a37 100644 --- a/src/core/registry/tests.rs +++ b/src/core/registry/tests.rs @@ -292,7 +292,7 @@ async fn add_reply_commits_state_without_event_confirmation() { assert_eq!(registry.list().await, vec![(id, Arc::from("reply-add"))]); for _ in 0..4 { - bus.publish(Event::new(EventKind::TaskStarting).with_task("noise")); + bus.publish(Event::new(EventKind::AttemptStarting).with_task("noise")); } assert!( matches!(stale_events.try_recv(), Err(TryRecvError::Lagged(_))), @@ -363,7 +363,7 @@ async fn single_add_publishes_added_before_starting() { let entry = order.entry(id).or_default(); match event.kind { EventKind::TaskAdded => entry.0 = Some(position), - EventKind::TaskStarting => { + EventKind::AttemptStarting => { entry.1.get_or_insert(position); } _ => {} @@ -373,10 +373,10 @@ async fn single_add_publishes_added_before_starting() { assert_eq!(order.len(), TASKS, "every registration must be observed"); for (id, (added, starting)) in order { let added = added.unwrap_or_else(|| panic!("{id} is missing TaskAdded")); - let starting = starting.unwrap_or_else(|| panic!("{id} is missing TaskStarting")); + let starting = starting.unwrap_or_else(|| panic!("{id} is missing AttemptStarting")); assert!( added < starting, - "{id} delivered TaskStarting before TaskAdded: added={added}, starting={starting}" + "{id} delivered AttemptStarting before TaskAdded: added={added}, starting={starting}" ); } @@ -449,7 +449,7 @@ async fn dropped_batch_reply_still_starts_after_all_added_events() { .collect(); let starting: Vec<_> = observed .iter() - .filter(|event| event.kind == EventKind::TaskStarting) + .filter(|event| event.kind == EventKind::AttemptStarting) .collect(); assert_eq!(added.len(), 2); assert_eq!(starting.len(), 2); @@ -799,8 +799,10 @@ async fn concurrent_cancel_commands_share_one_terminal_completion() { assert!(decisions.iter().all(|decision| !decision.is_complete())); assert!(registry.contains(id).await); assert!( - std::iter::from_fn(|| events.try_recv().ok()) - .all(|event| event.id != Some(id) || event.kind != EventKind::TaskRemoved), + std::iter::from_fn(|| events.try_recv().ok()).all(|event| { + event.id != Some(id) + || !matches!(event.kind, EventKind::TaskFinished | EventKind::TaskRemoved) + }), "terminal cleanup cannot happen before the task is released" ); @@ -813,13 +815,19 @@ async fn concurrent_cancel_commands_share_one_terminal_completion() { tokio::time::timeout(Duration::from_secs(2), registry.wait_until_empty()) .await .expect("terminal cleanup must remove the task"); - let removed = std::iter::from_fn(|| events.try_recv().ok()) - .filter(|event| event.id == Some(id) && event.kind == EventKind::TaskRemoved) - .count(); + let terminal: Vec<_> = std::iter::from_fn(|| events.try_recv().ok()) + .filter(|event| { + event.id == Some(id) + && matches!(event.kind, EventKind::TaskFinished | EventKind::TaskRemoved) + }) + .collect(); + assert_eq!(terminal.len(), 2); + assert_eq!(terminal[0].kind, EventKind::TaskFinished); assert_eq!( - removed, 1, - "shared cancellation must publish one terminal event" + terminal[0].outcome_kind, + Some(crate::TaskOutcomeKind::Canceled) ); + assert_eq!(terminal[1].kind, EventKind::TaskRemoved); stop_registry(®istry, &token).await; } @@ -1192,9 +1200,10 @@ async fn forged_terminal_event_does_not_remove_running_actor() { ); bus.publish( - Event::new(EventKind::ActorExhausted) + Event::new(EventKind::TaskFinished) .with_task("ignore-terminal-event") - .with_id(id), + .with_id(id) + .with_outcome_kind(crate::TaskOutcomeKind::Completed), ); let barrier_id = TaskId::next(); @@ -1260,17 +1269,20 @@ async fn outer_actor_panic_is_reaped_by_completion_channel() { .is_empty() ); - let mut actor_dead = 0; + let mut task_finished = 0; let mut task_removed = 0; while let Ok(event) = events.try_recv() { - if event.id == Some(id) && event.kind == EventKind::ActorDead { - actor_dead += 1; + if event.id == Some(id) + && event.kind == EventKind::TaskFinished + && event.outcome_kind == Some(crate::TaskOutcomeKind::Panicked) + { + task_finished += 1; } if event.id == Some(id) && event.kind == EventKind::TaskRemoved { task_removed += 1; } } - assert_eq!(actor_dead, 1); + assert_eq!(task_finished, 1); assert_eq!(task_removed, 1); stop_registry(®istry, &token).await; @@ -1319,13 +1331,23 @@ async fn remove_path_owns_cleanup_when_completion_signal_arrives() { receive_reply(send_remove(&tx, TaskId::next()), "completion barrier reply",).await, Ok(false) )); - let removed_count = std::iter::from_fn(|| events.try_recv().ok()) - .filter(|event| event.id == Some(id) && event.kind == EventKind::TaskRemoved) - .count(); + let terminal: Vec<_> = std::iter::from_fn(|| events.try_recv().ok()) + .filter(|event| { + event.id == Some(id) + && matches!(event.kind, EventKind::TaskFinished | EventKind::TaskRemoved) + }) + .collect(); assert_eq!( - removed_count, 1, - "stale completion signal must not duplicate terminal cleanup" + terminal.len(), + 2, + "stale completion must not duplicate events" ); + assert_eq!(terminal[0].kind, EventKind::TaskFinished); + assert_eq!( + terminal[0].outcome_kind, + Some(crate::TaskOutcomeKind::Canceled) + ); + assert_eq!(terminal[1].kind, EventKind::TaskRemoved); stop_registry(®istry, &token).await; } @@ -1390,8 +1412,10 @@ async fn completion_claim_before_remove_emits_one_terminal_event() { ); assert!(!joined_cancel.is_complete()); assert!( - std::iter::from_fn(|| events.try_recv().ok()) - .all(|event| event.id != Some(id) || event.kind != EventKind::TaskRemoved), + std::iter::from_fn(|| events.try_recv().ok()).all(|event| { + event.id != Some(id) + || !matches!(event.kind, EventKind::TaskFinished | EventKind::TaskRemoved) + }), "remove must not report termination while cleanup is still joining" ); @@ -1416,13 +1440,19 @@ async fn completion_claim_before_remove_emits_one_terminal_event() { .await, Ok(false) )); - let removed_count = std::iter::from_fn(|| events.try_recv().ok()) - .filter(|event| event.id == Some(id) && event.kind == EventKind::TaskRemoved) - .count(); + let terminal: Vec<_> = std::iter::from_fn(|| events.try_recv().ok()) + .filter(|event| { + event.id == Some(id) + && matches!(event.kind, EventKind::TaskFinished | EventKind::TaskRemoved) + }) + .collect(); + assert_eq!(terminal.len(), 2, "completion-first must publish one pair"); + assert_eq!(terminal[0].kind, EventKind::TaskFinished); assert_eq!( - removed_count, 1, - "completion-first race must publish one terminal event" + terminal[0].outcome_kind, + Some(crate::TaskOutcomeKind::Completed) ); + assert_eq!(terminal[1].kind, EventKind::TaskRemoved); stop_registry(®istry, &token).await; } diff --git a/src/core/runner.rs b/src/core/runner.rs index 97085cb..18866b6 100644 --- a/src/core/runner.rs +++ b/src/core/runner.rs @@ -7,18 +7,20 @@ //! //! | Attempt result | Events | Returned result | //! |-----------------------------------------------------|---------------------------------|----------------------| -//! | `Ok(())` | `TaskStopped` | `Ok(())` | -//! | `TaskError::Canceled` | `TaskCanceled` | Same error | -//! | Task-returned `Fail`, `Fatal`, or `Timeout` | `TaskFailed` | Same error | -//! | Panic while calling `spawn()` or polling its future | `TaskFailed` | `TaskError::Fail` | -//! | Configured attempt timer expires | `TimeoutHit`, then `TaskFailed` | `TaskError::Timeout` | +//! | `Ok(())` | `AttemptSucceeded` | `Ok(())` | +//! | `TaskError::Canceled` | `AttemptCanceled` | Same error | +//! | Task-returned `Fail`, `Fatal`, or `Timeout` | `AttemptFailed` | Same error | +//! | Panic while calling `spawn()` or polling its future | `AttemptFailed` | `TaskError::Fail` | +//! | Configured attempt timer expires | `AttemptTimedOut` | `TaskError::Timeout` | //! //! ## Rules //! -//! - Each completed call publishes one final attempt event: `TaskStopped`,`TaskCanceled`, or `TaskFailed`. Force-aborting the actor can drop an in-flight call before that event. +//! - Each completed call publishes one final attempt event: `AttemptSucceeded`, +//! `AttemptCanceled`, `AttemptFailed`, or `AttemptTimedOut`. Force-aborting +//! the managed runner can drop an in-flight call before that event. //! - Each attempt gets a child cancellation token. Parent cancellation reaches it, but child cancellation does not affect the parent. //! - Panics while calling `spawn()` or polling its future become retryable [`TaskError::Fail`] values. -//! - `TimeoutHit` is published only when the configured attempt timer expires. +//! - `AttemptTimedOut` is published only when the configured attempt timer expires. //! - `TaskError::Canceled` is a cooperative stop, not a failure. use std::future::Future; @@ -79,14 +81,16 @@ fn panic_to_error(payload: &(dyn std::any::Any + Send)) -> TaskError { /// A positive timeout limits this attempt only. /// When the configured timer expires, the attempt future is dropped and is no longer polled. /// The child token is then cancelled so work that cloned it can observe cancellation. -/// `TimeoutHit` is published before the final `TaskFailed`. +/// A configured timeout publishes `AttemptTimedOut` as the attempt's single +/// terminal event. A task that explicitly returns `TaskError::Timeout` instead +/// follows the ordinary `AttemptFailed` path. /// `None` and zero mean no timeout. /// /// ### Cancellation /// /// Parent cancellation reaches the attempt context. /// A cooperative task should observe [`TaskContext::cancelled`](crate::TaskContext::cancelled) and return [`TaskError::Canceled`]. -/// This publishes `TaskCanceled`, not `TaskFailed`. +/// This publishes `AttemptCanceled`, not `AttemptFailed`. /// /// ### Panic Handling /// @@ -118,7 +122,7 @@ pub async fn run_once( Err(_elapsed) => { child.cancel(); publish_timeout(bus, id, task.name(), dur, attempt, started.elapsed()); - Err(TaskError::timeout(dur)) + return Err(TaskError::timeout(dur)); } } } else { @@ -141,10 +145,10 @@ pub async fn run_once( } } -/// Publishes `TaskStopped` for a successful attempt. +/// Publishes `AttemptSucceeded` for a successful attempt. fn publish_stopped(bus: &Bus, id: TaskId, name: &str, attempt: u32, duration: Duration) { bus.publish( - Event::new(EventKind::TaskStopped) + Event::new(EventKind::AttemptSucceeded) .with_task(name) .with_id(id) .with_attempt(attempt) @@ -152,10 +156,10 @@ fn publish_stopped(bus: &Bus, id: TaskId, name: &str, attempt: u32, duration: Du ); } -/// Publishes `TaskCanceled` for a cooperative cancellation attempt. +/// Publishes `AttemptCanceled` for a cooperative cancellation attempt. fn publish_canceled(bus: &Bus, id: TaskId, name: &str, attempt: u32, duration: Duration) { bus.publish( - Event::new(EventKind::TaskCanceled) + Event::new(EventKind::AttemptCanceled) .with_task(name) .with_id(id) .with_attempt(attempt) @@ -163,7 +167,7 @@ fn publish_canceled(bus: &Bus, id: TaskId, name: &str, attempt: u32, duration: D ); } -/// Publishes `TaskFailed` with error details and attempt duration. +/// Publishes `AttemptFailed` with error details and attempt duration. fn publish_failed( bus: &Bus, id: TaskId, @@ -172,7 +176,7 @@ fn publish_failed( err: &TaskError, duration: Duration, ) { - let mut ev = Event::new(EventKind::TaskFailed) + let mut ev = Event::new(EventKind::AttemptFailed) .with_task(name) .with_id(id) .with_attempt(attempt) @@ -184,7 +188,7 @@ fn publish_failed( bus.publish(ev); } -/// Publishes `TimeoutHit` before the final timeout `TaskFailed` event. +/// Publishes `AttemptTimedOut` as the configured timeout's terminal attempt event. fn publish_timeout( bus: &Bus, id: TaskId, @@ -194,7 +198,7 @@ fn publish_timeout( duration: Duration, ) { bus.publish( - Event::new(EventKind::TimeoutHit) + Event::new(EventKind::AttemptTimedOut) .with_task(name) .with_id(id) .with_timeout(dur) @@ -239,7 +243,7 @@ mod tests { } #[tokio::test(start_paused = true)] - async fn timeout_returns_timeout_and_publishes_timeout_hit() { + async fn timeout_returns_timeout_and_publishes_attempt_timed_out() { let bus = Bus::new(16); let mut rx = bus.subscribe(); let parent = CancellationToken::new(); @@ -260,8 +264,8 @@ mod tests { } assert!( std::iter::from_fn(|| rx.try_recv().ok()) - .any(|event| event.kind == EventKind::TimeoutHit), - "a timeout result must be accompanied by TimeoutHit" + .any(|event| event.kind == EventKind::AttemptTimedOut), + "a timeout result must be accompanied by AttemptTimedOut" ); } @@ -289,16 +293,16 @@ mod tests { .expect("task succeeds"); let stopped = std::iter::from_fn(|| rx.try_recv().ok()) - .find(|event| event.kind == EventKind::TaskStopped) - .expect("a successful attempt must publish TaskStopped"); + .find(|event| event.kind == EventKind::AttemptSucceeded) + .expect("a successful attempt must publish AttemptSucceeded"); assert_eq!( stopped.attempt, Some(3), - "TaskStopped must carry the attempt number" + "AttemptSucceeded must carry the attempt number" ); let measured = stopped .duration_ms - .expect("TaskStopped must carry the attempt duration"); + .expect("AttemptSucceeded must carry the attempt duration"); assert!( measured >= 20, "attempt duration must reflect the ~30ms of work, got {measured}ms" diff --git a/src/core/runtime/tests.rs b/src/core/runtime/tests.rs index 55c2ee9..c83d502 100644 --- a/src/core/runtime/tests.rs +++ b/src/core/runtime/tests.rs @@ -227,7 +227,7 @@ async fn subscriber_listener_reports_bus_lag_as_overflow() { for i in 0..500 { core.bus - .publish(Event::new(EventKind::TaskStarting).with_task(format!("f{i}"))); + .publish(Event::new(EventKind::AttemptStarting).with_task(format!("f{i}"))); } let saw_lag = timeout( @@ -261,7 +261,7 @@ async fn drain_pending_delivers_retained_tail_after_a_lag_gap() { set.start(); for i in 0..5 { - bus.publish(Event::new(EventKind::TaskStarting).with_task(format!("t{i}"))); + bus.publish(Event::new(EventKind::AttemptStarting).with_task(format!("t{i}"))); } SupervisorCore::drain_pending(&mut rx, &alive, &set).await; @@ -271,7 +271,7 @@ async fn drain_pending_delivers_retained_tail_after_a_lag_gap() { assert!( delivered .iter() - .any(|e| e.kind == EventKind::TaskStarting && e.task.as_deref() == Some("t4")), + .any(|e| e.kind == EventKind::AttemptStarting && e.task.as_deref() == Some("t4")), "newest retained event must reach subscribers despite a lag gap" ); } @@ -394,7 +394,7 @@ async fn shutdown_panic_still_runs_cleanup_before_caching_result() { let delivered_before_probe = seen.lock().unwrap().len(); core.subs.emit_arc(Arc::new( - Event::new(EventKind::TaskStarting).with_task("closed-probe"), + Event::new(EventKind::AttemptStarting).with_task("closed-probe"), )); assert_eq!( seen.lock().unwrap().len(), @@ -1285,7 +1285,7 @@ async fn static_run_batch_uses_one_queue_slot_with_lagged_observer() { let mut stale_events = core.bus.subscribe(); for index in 0..4 { core.bus - .publish(Event::new(EventKind::TaskStarting).with_task(format!("noise-{index}"))); + .publish(Event::new(EventKind::AttemptStarting).with_task(format!("noise-{index}"))); } assert!(matches!( stale_events.try_recv(), @@ -1547,7 +1547,7 @@ async fn cancel_uses_registry_completion_when_event_bus_lags() { for _ in 0..16 { core.bus - .publish(Event::new(EventKind::TaskStarting).with_task("noise")); + .publish(Event::new(EventKind::AttemptStarting).with_task("noise")); } assert!( matches!(stale_events.try_recv(), Err(TryRecvError::Lagged(_))), diff --git a/src/events/bus.rs b/src/events/bus.rs index d5fd629..d46b7ce 100644 --- a/src/events/bus.rs +++ b/src/events/bus.rs @@ -113,16 +113,16 @@ mod tests { let mut a = bus.subscribe(); let mut b = bus.subscribe(); - bus.publish(Event::new(EventKind::TaskStarting)); + bus.publish(Event::new(EventKind::AttemptStarting)); - assert_eq!(a.recv().await.unwrap().kind, EventKind::TaskStarting); - assert_eq!(b.recv().await.unwrap().kind, EventKind::TaskStarting); + assert_eq!(a.recv().await.unwrap().kind, EventKind::AttemptStarting); + assert_eq!(b.recv().await.unwrap().kind, EventKind::AttemptStarting); } #[tokio::test] async fn publish_without_subscribers_is_dropped() { let bus = Bus::new(16); - bus.publish(Event::new(EventKind::TaskStarting)); + bus.publish(Event::new(EventKind::AttemptStarting)); let mut rx = bus.subscribe(); assert!(matches!(rx.try_recv(), Err(TryRecvError::Empty))); @@ -134,7 +134,7 @@ mod tests { let mut rx = bus.subscribe(); for _ in 0..4 { - bus.publish(Event::new(EventKind::TaskStarting)); + bus.publish(Event::new(EventKind::AttemptStarting)); } let err = rx @@ -145,6 +145,6 @@ mod tests { matches!(err, RecvError::Lagged(_)), "expected Lagged, got {err:?}" ); - assert_eq!(rx.recv().await.unwrap().kind, EventKind::TaskStarting); + assert_eq!(rx.recv().await.unwrap().kind, EventKind::AttemptStarting); } } diff --git a/src/events/event.rs b/src/events/event.rs index 1d25006..1d856ad 100644 --- a/src/events/event.rs +++ b/src/events/event.rs @@ -10,6 +10,7 @@ //! | [`Event`] | Event payload and metadata | //! | [`BackoffSource`] | Why a `BackoffScheduled` event was emitted | //! | [`RejectionKind`] | Machine-readable submission rejection | +//! | [`TaskOutcomeKind`]| Machine-readable final task outcome | //! //! ## Sequence numbers //! @@ -32,13 +33,14 @@ //! - `id`: the stable [`TaskId`] for one submission and run. //! - `attempt`: task attempt number, starting from 1. //! - `task`: usually a task name. Subscriber diagnostics use it for the subscriber name, and controller events use it for the slot name. +//! - `outcome_kind`: machine-readable final outcome for `TaskFinished` and rejected work. //! - `rejection_kind`: machine-readable category for a rejected add or controller submission. //! //! `timeout_ms`, `delay_ms`, and `duration_ms` use whole milliseconds. //! Values above `u32::MAX` milliseconds are stored as `u32::MAX`. //! -//! Treat `reason` as readable text unless the event points to a constant in [`reasons`](crate::reasons). -//! Use [`RejectionKind`] instead of parsing rejection text. +//! Treat `reason` as readable diagnostic text, not schema. +//! Use [`TaskOutcomeKind`] and [`RejectionKind`] for machine decisions. //! > Use [`EventKind::as_label`] for a stable event label. //! //! ## Example @@ -47,13 +49,13 @@ //! use std::time::Duration; //! use taskvisor::{Event, EventKind}; //! -//! let ev = Event::new(EventKind::TaskFailed) +//! let ev = Event::new(EventKind::AttemptFailed) //! .with_task("demo-task") //! .with_reason("boom") //! .with_attempt(3) //! .with_duration(Duration::from_millis(42)); //! -//! assert_eq!(ev.kind, EventKind::TaskFailed); +//! assert_eq!(ev.kind, EventKind::AttemptFailed); //! assert_eq!(ev.task.as_deref(), Some("demo-task")); //! assert_eq!(ev.reason.as_deref(), Some("boom")); //! assert_eq!(ev.duration_ms, Some(42)); @@ -63,7 +65,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; use std::time::{Duration, SystemTime}; -use crate::identity::TaskId; +use crate::{TaskOutcomeKind, identity::TaskId}; /// Process-local counter for `seq` values. /// @@ -138,7 +140,7 @@ pub enum EventKind { /// - `attempt`: attempt number (1-based for this task run) /// - `at`: wall-clock timestamp /// - `seq`: process-local sequence - TaskStarting, + AttemptStarting, /// A task attempt returned `Ok(())`. /// @@ -150,7 +152,7 @@ pub enum EventKind { /// - `task`: task name /// - `attempt`: attempt number /// - `duration_ms`: attempt duration - TaskStopped, + AttemptSucceeded, /// Task attempt returned [`TaskError::Canceled`](crate::TaskError::Canceled). /// @@ -159,11 +161,13 @@ pub enum EventKind { /// - `task`: task name /// - `attempt`: attempt number /// - `duration_ms`: attempt duration - TaskCanceled, + AttemptCanceled, /// A task attempt returned a failure. /// - /// This includes retryable failures, timeouts, and fatal errors. + /// This includes retryable failures, fatal errors, task-returned timeouts, + /// and panics caught while running user code. A configured per-attempt + /// deadline instead emits [`AttemptTimedOut`](Self::AttemptTimedOut). /// A later event shows whether Taskvisor retries or reaches a terminal state. /// /// Sets: @@ -173,19 +177,17 @@ pub enum EventKind { /// - `duration_ms`: attempt duration /// - `reason`: error message /// - `exit_code`: process-like exit code, when available - TaskFailed, + AttemptFailed, /// Task exceeded its configured timeout for this attempt. /// - /// A timeout is followed by a `TaskFailed` event carrying `TaskError::Timeout`. - /// /// Sets: /// - `id`: task run identity /// - `task`: task name /// - `attempt`: attempt number /// - `timeout_ms`: configured timeout /// - `duration_ms`: elapsed attempt duration - TimeoutHit, + AttemptTimedOut, /// The next attempt was scheduled after success or failure. /// @@ -229,8 +231,9 @@ pub enum EventKind { /// Sets: /// - `id`: task run identity of the rejected add request /// - `task`: task name + /// - `outcome_kind`: [`TaskOutcomeKind::Rejected`] /// - `rejection_kind`: [`RejectionKind::AlreadyExists`] or [`RejectionKind::BatchRejected`] - /// - `reason`: e.g. "already_exists" or "batch_rejected" + /// - `reason`: diagnostic rejection details /// - `at`: wall-clock timestamp /// - `seq`: process-local sequence TaskAddFailed, @@ -248,48 +251,30 @@ pub enum EventKind { /// - `seq`: process-local sequence TaskRemoveRequested, - /// Task was removed from the supervisor (after join/cleanup). + /// Task was removed from the supervisor after terminal cleanup. /// /// Sets: /// - `id`: task run identity /// - `task`: task name - /// - `reason`: [`FORCE_TERMINATED_AFTER_GRACE`](crate::reasons::FORCE_TERMINATED_AFTER_GRACE) when the actor was force-aborted or `None` /// - `at`: wall-clock timestamp /// - `seq`: process-local sequence TaskRemoved, - /// A task reached a non-fatal terminal state and will not restart. + /// A registered task reached its final outcome and will not start another attempt. /// - /// Emitted when: - /// - `RestartPolicy::Never` stops after success or a retryable failure - /// - `RestartPolicy::OnFailure` stops after success - /// - the retry limit is reached after retryable failures - /// - the task returns `TaskError::Canceled` without a runtime cancellation + /// This is emitted once after the task runner is joined and before + /// [`TaskRemoved`](Self::TaskRemoved). It also covers force-abort and an + /// internal runner panic, even when no attempt-level terminal event exists. /// /// Sets: /// - `id`: task run identity /// - `task`: task name - /// - `attempt`: last attempt number - /// - `reason`: optional message - /// - `exit_code`: numeric exit code (process-like runtimes); `None` otherwise - /// - `at`: wall-clock timestamp - /// - `seq`: process-local sequence - ActorExhausted, - - /// A task reached a fatal terminal state and will not restart. - /// - /// Emitted when: - /// - Task returned `TaskError::Fatal` - /// - /// Sets: - /// - `id`: task run identity - /// - `task`: task name - /// - `attempt`: last attempt number - /// - `reason`: fatal error message - /// - `exit_code`: numeric exit code when the fatal error has one; `None` for logical errors + /// - `outcome_kind`: stable machine-readable final category + /// - `reason`: optional diagnostic detail; never parse it as schema + /// - `exit_code`: process-like exit code, when available /// - `at`: wall-clock timestamp /// - `seq`: process-local sequence - ActorDead, + TaskFinished, #[cfg(feature = "controller")] #[cfg_attr(docsrs, doc(cfg(feature = "controller")))] @@ -298,6 +283,7 @@ pub enum EventKind { /// Sets: /// - `task`: slot name, when known /// - `id`: the rejected submission's [`TaskId`] + /// - `outcome_kind`: [`TaskOutcomeKind::Rejected`] /// - `rejection_kind`: stable machine-readable rejection category /// - `reason`: readable rejection details ControllerRejected, @@ -333,18 +319,18 @@ impl EventKind { /// Use it as an event name in tracing or as a metrics label value. /// /// ```text - /// EventKind::TaskStarting + /// EventKind::AttemptStarting /// │ as_label() /// ▼ - /// "task_starting" - /// ├── log field: event="task_starting" - /// └── metric label: event="task_starting" + /// "attempt_starting" + /// ├── log field: event="attempt_starting" + /// └── metric label: event="attempt_starting" /// ``` /// /// ```rust /// use taskvisor::EventKind; /// - /// assert_eq!(EventKind::TaskStarting.as_label(), "task_starting"); + /// assert_eq!(EventKind::AttemptStarting.as_label(), "attempt_starting"); /// assert_eq!(EventKind::BackoffScheduled.as_label(), "backoff_scheduled"); /// ``` #[must_use] @@ -356,19 +342,18 @@ impl EventKind { EventKind::ShutdownRequested => "shutdown_requested", EventKind::AllStoppedWithinGrace => "all_stopped_within_grace", EventKind::GraceExceeded => "grace_exceeded", - EventKind::TaskStarting => "task_starting", - EventKind::TaskStopped => "task_stopped", - EventKind::TaskCanceled => "task_canceled", - EventKind::TaskFailed => "task_failed", - EventKind::TimeoutHit => "timeout_hit", + EventKind::AttemptStarting => "attempt_starting", + EventKind::AttemptSucceeded => "attempt_succeeded", + EventKind::AttemptCanceled => "attempt_canceled", + EventKind::AttemptFailed => "attempt_failed", + EventKind::AttemptTimedOut => "attempt_timed_out", EventKind::BackoffScheduled => "backoff_scheduled", EventKind::TaskAddRequested => "task_add_requested", EventKind::TaskAdded => "task_added", EventKind::TaskAddFailed => "task_add_failed", EventKind::TaskRemoveRequested => "task_remove_requested", EventKind::TaskRemoved => "task_removed", - EventKind::ActorExhausted => "actor_exhausted", - EventKind::ActorDead => "actor_dead", + EventKind::TaskFinished => "task_finished", #[cfg(feature = "controller")] EventKind::ControllerRejected => "controller_rejected", #[cfg(feature = "controller")] @@ -489,13 +474,24 @@ pub struct Event { pub delay_ms: Option, /// Elapsed duration of the attempt in milliseconds. pub duration_ms: Option, - /// Only values documented in [`reasons`](crate::reasons) are stable. + /// Human-readable diagnostic detail. + /// + /// This text is not schema and may change. Use typed fields such as + /// [`outcome_kind`](Self::outcome_kind) and + /// [`rejection_kind`](Self::rejection_kind) for machine decisions. pub reason: Option>, + /// Machine-readable final category for `TaskFinished` and rejected work. + /// + /// Use [`TaskOutcomeKind`] for branching and [`TaskOutcomeKind::as_label`] + /// for telemetry labels. + pub outcome_kind: Option, /// Machine-readable category for `TaskAddFailed` and `ControllerRejected`. /// /// Readable details remain available in [`reason`](Self::reason). pub rejection_kind: Option, - /// Attempt count (starting from 1). + /// 1-based number of the attempt described by an attempt-level or backoff event. + /// + /// This is not the total number of attempts and is not set on `TaskFinished`. pub attempt: Option, /// This is normally a task name. Subscriber diagnostics use it for a subscriber name, and controller events use it for a slot name. pub task: Option>, @@ -530,6 +526,7 @@ impl Event { duration_ms: None, attempt: None, reason: None, + outcome_kind: None, rejection_kind: None, task: None, id: None, @@ -545,11 +542,23 @@ impl Event { self } + /// Attaches a machine-readable final outcome category. + #[inline] + #[must_use] + pub fn with_outcome_kind(mut self, kind: TaskOutcomeKind) -> Self { + self.outcome_kind = Some(kind); + self + } + /// Attaches a machine-readable submission rejection category. + /// + /// This also sets [`outcome_kind`](Self::outcome_kind) to + /// [`TaskOutcomeKind::Rejected`]. #[inline] #[must_use] pub fn with_rejection_kind(mut self, kind: RejectionKind) -> Self { self.rejection_kind = Some(kind); + self.outcome_kind = Some(TaskOutcomeKind::Rejected); self } @@ -694,6 +703,9 @@ impl std::fmt::Debug for Event { if let Some(ref reason) = self.reason { d.field("reason", reason); } + if let Some(outcome_kind) = self.outcome_kind { + d.field("outcome_kind", &outcome_kind); + } if let Some(rejection_kind) = self.rejection_kind { d.field("rejection_kind", &rejection_kind); } @@ -722,8 +734,8 @@ mod tests { #[test] fn seq_increases_monotonically() { - let a = Event::new(EventKind::TaskStarting); - let b = Event::new(EventKind::TaskStopped); + let a = Event::new(EventKind::AttemptStarting); + let b = Event::new(EventKind::AttemptSucceeded); assert!(b.seq > a.seq, "seq must grow: {} vs {}", a.seq, b.seq); } @@ -736,19 +748,18 @@ mod tests { (EventKind::ShutdownRequested, "shutdown_requested"), (EventKind::AllStoppedWithinGrace, "all_stopped_within_grace"), (EventKind::GraceExceeded, "grace_exceeded"), - (EventKind::TaskStarting, "task_starting"), - (EventKind::TaskStopped, "task_stopped"), - (EventKind::TaskCanceled, "task_canceled"), - (EventKind::TaskFailed, "task_failed"), - (EventKind::TimeoutHit, "timeout_hit"), + (EventKind::AttemptStarting, "attempt_starting"), + (EventKind::AttemptSucceeded, "attempt_succeeded"), + (EventKind::AttemptCanceled, "attempt_canceled"), + (EventKind::AttemptFailed, "attempt_failed"), + (EventKind::AttemptTimedOut, "attempt_timed_out"), (EventKind::BackoffScheduled, "backoff_scheduled"), (EventKind::TaskAddRequested, "task_add_requested"), (EventKind::TaskAdded, "task_added"), (EventKind::TaskAddFailed, "task_add_failed"), (EventKind::TaskRemoveRequested, "task_remove_requested"), (EventKind::TaskRemoved, "task_removed"), - (EventKind::ActorExhausted, "actor_exhausted"), - (EventKind::ActorDead, "actor_dead"), + (EventKind::TaskFinished, "task_finished"), #[cfg(feature = "controller")] (EventKind::ControllerRejected, "controller_rejected"), #[cfg(feature = "controller")] @@ -767,13 +778,14 @@ mod tests { #[test] fn new_event_leaves_all_optionals_empty() { - let ev = Event::new(EventKind::TaskStarting); + let ev = Event::new(EventKind::AttemptStarting); assert_eq!(ev.timeout_ms, None); assert_eq!(ev.delay_ms, None); assert_eq!(ev.duration_ms, None); assert_eq!(ev.attempt, None); assert_eq!(ev.exit_code, None); assert_eq!(ev.reason, None); + assert_eq!(ev.outcome_kind, None); assert_eq!(ev.rejection_kind, None); assert_eq!(ev.task, None); assert_eq!(ev.id, None); @@ -799,6 +811,11 @@ mod tests { for (kind, expected) in cases { assert_eq!(kind.as_label(), expected, "{kind:?}"); } + + let rejected = + Event::new(EventKind::TaskAddFailed).with_rejection_kind(RejectionKind::AlreadyExists); + assert_eq!(rejected.rejection_kind, Some(RejectionKind::AlreadyExists)); + assert_eq!(rejected.outcome_kind, Some(TaskOutcomeKind::Rejected)); } #[test] @@ -809,9 +826,12 @@ mod tests { type ReadMs = fn(&Event) -> Option; let cases: [(&str, EventKind, Builder, ReadMs); 3] = [ - ("timeout", EventKind::TimeoutHit, Event::with_timeout, |e| { - e.timeout_ms - }), + ( + "timeout", + EventKind::AttemptTimedOut, + Event::with_timeout, + |e| e.timeout_ms, + ), ( "delay", EventKind::BackoffScheduled, @@ -820,7 +840,7 @@ mod tests { ), ( "duration", - EventKind::TaskStopped, + EventKind::AttemptSucceeded, Event::with_duration, |e| e.duration_ms, ), @@ -845,7 +865,7 @@ mod tests { ] { assert!(Event::new(kind).is_internal_diagnostic(), "{kind:?}"); } - assert!(!Event::new(EventKind::TaskStarting).is_internal_diagnostic()); + assert!(!Event::new(EventKind::AttemptStarting).is_internal_diagnostic()); } #[test] @@ -879,7 +899,10 @@ mod tests { #[test] fn with_exit_code_keeps_sign() { - for (kind, code) in [(EventKind::TaskFailed, 42), (EventKind::ActorDead, -1)] { + for (kind, code) in [ + (EventKind::AttemptFailed, 42), + (EventKind::TaskFinished, -1), + ] { assert_eq!(Event::new(kind).with_exit_code(code).exit_code, Some(code)); } } @@ -913,13 +936,19 @@ mod tests { #[test] fn debug_renders_exit_code_only_when_set() { - let ev = Event::new(EventKind::ActorExhausted).with_exit_code(137); + let ev = Event::new(EventKind::TaskFinished) + .with_outcome_kind(TaskOutcomeKind::ForceAborted) + .with_exit_code(137); assert!( format!("{ev:?}").contains("exit_code: 137"), "Debug must surface exit_code when present" ); + assert!( + format!("{ev:?}").contains("outcome_kind: ForceAborted"), + "Debug must surface outcome_kind when present" + ); - let none = Event::new(EventKind::TaskStopped); + let none = Event::new(EventKind::AttemptSucceeded); assert!( !format!("{none:?}").contains("exit_code"), "Debug must omit exit_code when absent" diff --git a/src/events/mod.rs b/src/events/mod.rs index c1e527f..71b68cc 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -3,12 +3,13 @@ //! Events describe what Taskvisor is doing. //! Use them for logs, metrics, dashboards, alerts, and tests. //! -//! | Type | Role | -//! |-------------------|--------------------------------------------| -//! | [`EventKind`] | Event classification | -//! | [`Event`] | Event payload and metadata | -//! | [`BackoffSource`] | Why a `BackoffScheduled` event was emitted | -//! | [`RejectionKind`] | Machine-readable submission rejection | +//! | Type | Role | +//! |---------------------------------------------|--------------------------------------------| +//! | [`EventKind`] | Event classification | +//! | [`Event`] | Event payload and metadata | +//! | [`BackoffSource`] | Why a `BackoffScheduled` event was emitted | +//! | [`RejectionKind`] | Machine-readable submission rejection | +//! | [`TaskOutcomeKind`](crate::TaskOutcomeKind) | Machine-readable final outcome | //! //! ## Events and final outcomes are different //! @@ -30,29 +31,29 @@ //! //! ```text //! Add: -//! TaskAddRequested ──► TaskAdded ──► TaskStarting +//! TaskAddRequested ──► TaskAdded ──► AttemptStarting //! └──► TaskAddFailed //! //! Attempt: -//! TaskStarting ──► TaskStopped -//! ├──► TaskCanceled -//! ├──► TaskFailed -//! └──► TimeoutHit ──► TaskFailed +//! AttemptStarting ──► AttemptSucceeded +//! ├──► AttemptCanceled ──► TaskFinished(Canceled) +//! ├──► AttemptFailed +//! └──► AttemptTimedOut //! //! Successful attempt: -//! TaskStopped ──► ActorExhausted -//! ├──► BackoffScheduled(Success) ──► TaskStarting -//! └──► TaskStarting (Always without an interval) +//! AttemptSucceeded ──► TaskFinished(Completed) +//! ├──► BackoffScheduled(Success) ──► AttemptStarting +//! └──► AttemptStarting (Always without an interval) //! -//! Retryable failure: -//! TaskFailed ──► BackoffScheduled(Failure) ──► TaskStarting -//! └──► ActorExhausted +//! Retryable failure or configured timeout: +//! AttemptFailed | AttemptTimedOut ──► BackoffScheduled(Failure) ──► AttemptStarting +//! └──► TaskFinished(Failed) //! //! Fatal failure: -//! TaskFailed ──► ActorDead +//! AttemptFailed ──► TaskFinished(Fatal) //! //! Registry cleanup: -//! [optional TaskRemoveRequested] ──► TaskRemoved +//! [optional TaskRemoveRequested] ──► TaskFinished ──► TaskRemoved //! //! Queued controller removal (feature `controller`): //! TaskRemoveRequested ──► ControllerRejected(RemovedFromQueue) @@ -66,13 +67,16 @@ //! //! ## Read the Stream Safely //! -//! - [`TaskFailed`](EventKind::TaskFailed) describes one attempt. -//! The task may retry. [`ActorExhausted`](EventKind::ActorExhausted) and [`ActorDead`](EventKind::ActorDead) mean that no more attempts will start. +//! - [`AttemptFailed`](EventKind::AttemptFailed) describes one attempt. +//! The task may retry. [`TaskFinished`](EventKind::TaskFinished) means that a registered task has reached its final [`TaskOutcomeKind`](crate::TaskOutcomeKind). +//! Cancellation while waiting for a permit or backoff can produce `TaskFinished(Canceled)` without an `AttemptCanceled` event. //! - [`Event::seq`] is process-local construction order. //! It can help sort events and detect gaps, but it is not a causal clock. //! - Use [`EventKind::as_label`] for a stable telemetry label. -//! Treat free-form [`Event::reason`] text as diagnostic unless it is documented in [`reasons`](crate::reasons). +//! Treat free-form [`Event::reason`] text as diagnostic, never as schema. +//! Use [`TaskOutcomeKind`](crate::TaskOutcomeKind) and [`RejectionKind`] for machine decisions. //! - Use [`RejectionKind`] for machine-readable handling of `TaskAddFailed` and `ControllerRejected`. +//! Rejected work never enters the registry, so it has no `TaskFinished` or `TaskRemoved` event. //! //! ## Subscribers //! diff --git a/src/identity.rs b/src/identity.rs index ae89aa6..a9a7b7f 100644 --- a/src/identity.rs +++ b/src/identity.rs @@ -4,6 +4,8 @@ //! //! Direct `add*` methods return it after the registry accepts a task. //! Controller `submit*` methods return it after queueing, before slot admission. +//! Controller `prepare_submission` exposes it earlier, before the submission can +//! publish any event. //! The same ID therefore also identifies a controller submission that is rejected without running. //! //! ## One identity across the lifecycle @@ -18,20 +20,20 @@ //! │ │ //! └── optional controller ─► queue[A] ─► slot admission[A] ─────┤ //! │ │ -//! └──► rejected[A] (no actor) │ +//! └──► rejected[A] (not registered)│ //! ▼ //! registry admission[A] //! │ │ //! rejected[A] ◄──────────┘ └──► registry[A] -//! (no actor) │ +//! (not registered) │ //! ▼ -//! actor[A] ─► attempt 1, 2, ... +//! task runner[A] ─► attempt 1, 2, ... //! ▼ //! cleanup[A] //! ``` //! //! No new `TaskId` is allocated at admission or between retry attempts. -//! Queue management, registry membership, the actor, and completion tracking carry the same `A`. +//! Queue management, registry membership, the managed runner, and completion tracking carry the same `A`. //! Related lifecycle events expose it so callers can correlate cancellation, logs, and metrics. //! //! ## TaskId vs Name vs Slot @@ -72,6 +74,7 @@ static TASK_ID_SEQ: AtomicU64 = AtomicU64::new(1); /// Opaque process-local identity of one task submission. /// /// Taskvisor allocates it once. With the `controller` feature, this happens before admission so queued work can already be addressed and correlated. +/// A prepared controller submission exposes the allocated value before controller intake and before any event for that value can be published. /// If admitted, the same value becomes the registry key and remains unchanged through all attempts and terminal cleanup. /// /// Rejected work keeps its id even though no task body ran. diff --git a/src/lib.rs b/src/lib.rs index 3ecf151..7d6bda8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,9 +1,9 @@ //! # taskvisor //! -//! Taskvisor supervises long-running Tokio tasks. -//! It can restart failed work, slow down retries, time out each attempt, and stop tasks during shutdown. +//! Taskvisor is an in-process Tokio task supervisor with retries, reliable outcomes, and keyed queue/replace/reject admission. +//! It can time out attempts, stop tasks during shutdown, and dynamically manage work. //! -//! Use it for workers, consumers, connection loops, and other background work that should have a clear lifecycle. +//! Use it for dynamic or keyed background work that needs conflict handling and a clear lifecycle. //! //! ## Start Here //! @@ -75,7 +75,7 @@ //! it returns a runtime error if the completion channel closes first. //! //! Create one with [`SupervisorHandle::add_and_watch`] or its fail-fast `try_*` form. -//! With the `controller` feature, use `submit_and_watch` or `try_submit_and_watch`. +//! With a configured controller, use `submit_and_watch` or `try_submit_and_watch`. //! //! ## Quick Start //! @@ -98,47 +98,81 @@ //! //! ## Main Types //! -//! | Need | Types | -//! |------------------|--------------------------------------------------------| -//! | Define work | [`Task`], [`TaskFn`], [`TaskContext`], [`TaskSpec`] | -//! | Run work | [`Supervisor`], [`SupervisorHandle`] | -//! | Set defaults | [`SupervisorConfig`], [`TaskDefaults`] | -//! | Control retries | [`RestartPolicy`], [`BackoffPolicy`], [`JitterPolicy`] | -//! | Observe progress | [`Event`], [`EventKind`], [`Subscribe`] | -//! | Wait for the end | [`TaskWaiter`], [`TaskOutcome`] | -//! | Handle errors | [`Error`], [`TaskError`], [`RuntimeError`] | +//! | Need | Types | +//! |------------------|----------------------------------------------------------------| +//! | Define work | [`Task`], [`TaskFn`], [`TaskContext`], [`TaskSpec`] | +//! | Run work | [`Supervisor`], [`SupervisorHandle`] | +//! | Set defaults | [`SupervisorConfig`], [`TaskDefaults`] | +//! | Control retries | [`RestartPolicy`], [`BackoffPolicy`], [`JitterPolicy`] | +//! | Observe progress | [`Event`], [`EventKind`], [`Subscribe`] | +//! | Wait for the end | [`TaskWaiter`], [`TaskOutcome`], [`TaskOutcomeKind`] | +//! | Admit keyed work | [`ControllerSpec`], [`PreparedSubmission`], [`AdmissionPolicy`], [`ControllerConfig`] | +//! | Handle errors | [`Error`], [`TaskError`], [`RuntimeError`] | //! //! Main types are re-exported at the crate root. //! The module pages explain each area in more detail: //! [`tasks`], [`policies`], [`events`], [`subscribers`], [`core`], and [`identity`]. //! -//! ## Optional Features +//! ## Keyed Admission +//! +//! The default `controller` feature provides per-slot admission. Configure it with +//! [`SupervisorBuilder::with_controller`], then submit a [`ControllerSpec`] that +//! queues, replaces, or rejects work when the slot already has an owner. +//! Different slots are independent, subject to the supervisor's global limits. +//! +//! ## Feature Flags //! //! - `tracing`: forwards lifecycle events to `tracing`. //! - `logging`: simple event logging for examples and development. //! - `tokio-util-interop`: exposes the underlying Tokio cancellation token. -//! - `controller`: slot-based admission with queue, replace, and reject rules. +//! - `controller` (default): slot-based admission with queue, replace, and reject rules. //! - `test-util`: constructors for task contexts, identities, and outcomes in tests. //! //! ## Examples //! -//! Repository examples go from simple to advanced ([browse them on GitHub](https://github.com/soltiHQ/taskvisor/tree/main/examples)): -//! -//! | Example | What it shows | -//! |---------------------------------------------------------------------------------------------|-------------------------------------------------------------------| -//! | [basic](https://github.com/soltiHQ/taskvisor/blob/main/examples/basic.rs) | Run one task and exit — the minimal wiring | -//! | [worker](https://github.com/soltiHQ/taskvisor/blob/main/examples/worker.rs) | A long-running worker that stops cleanly on Ctrl+C | -//! | [periodic](https://github.com/soltiHQ/taskvisor/blob/main/examples/periodic.rs) | Repeat a job after each successful cycle | -//! | [multiple](https://github.com/soltiHQ/taskvisor/blob/main/examples/multiple.rs) | Several tasks with different restart rules under one supervisor | -//! | [queue_consumer](https://github.com/soltiHQ/taskvisor/blob/main/examples/queue_consumer.rs) | A message consumer that reconnects after failures | -//! | [cpu_job](https://github.com/soltiHQ/taskvisor/blob/main/examples/cpu_job.rs) | Run CPU-heavy work on rayon, supervised, without blocking Tokio | -//! | [subscriber](https://github.com/soltiHQ/taskvisor/blob/main/examples/subscriber.rs) | React to lifecycle events with your own handler | -//! | [tracing](https://github.com/soltiHQ/taskvisor/blob/main/examples/tracing.rs) | Send supervisor events into your logs (feature `tracing`) | -//! | [metrics](https://github.com/soltiHQ/taskvisor/blob/main/examples/metrics.rs) | Count lifecycle events as Prometheus metrics | -//! | [dynamic](https://github.com/soltiHQ/taskvisor/blob/main/examples/dynamic.rs) | Add, cancel, and remove tasks while the app is running | -//! | [outcomes](https://github.com/soltiHQ/taskvisor/blob/main/examples/outcomes.rs) | Wait for a task's final result: done, failed, or canceled | -//! | [slots](https://github.com/soltiHQ/taskvisor/blob/main/examples/slots.rs) | Limit concurrency per slot: queue, replace, or drop the newcomer | -//! | [admission](https://github.com/soltiHQ/taskvisor/blob/main/examples/admission.rs) | Find out if your submission ran or was rejected | +//! Choose a short path, or [browse all examples on GitHub](https://github.com/soltiHQ/taskvisor/tree/main/examples): +//! +//! - New to supervision: `basic` → `worker` → `outcomes`. +//! - Need per-key coordination: `tenant_sync` → `slots` → `admission`. +//! +//! ### Start here +//! +//! | Example | What it shows | +//! |-------------------------------------------------------------------------------------------|-----------------------------------------------------------------| +//! | [basic](https://github.com/soltiHQ/taskvisor/blob/main/examples/basic.rs) | Run one task and exit — the minimal wiring | +//! | [worker](https://github.com/soltiHQ/taskvisor/blob/main/examples/worker.rs) | A long-running worker that stops cleanly on Ctrl+C | +//! | [periodic](https://github.com/soltiHQ/taskvisor/blob/main/examples/periodic.rs) | Repeat a job after each successful cycle | +//! | [multiple](https://github.com/soltiHQ/taskvisor/blob/main/examples/multiple.rs) | Several restart rules under one supervisor | +//! +//! ### Real patterns +//! +//! | Example | What it shows | +//! |---------------------------------------------------------------------------------------------|-----------------------------------------------------------------| +//! | [queue_consumer](https://github.com/soltiHQ/taskvisor/blob/main/examples/queue_consumer.rs) | Retry a failed broker connection | +//! | [cpu_job](https://github.com/soltiHQ/taskvisor/blob/main/examples/cpu_job.rs) | Run CPU-heavy work on rayon without blocking Tokio | +//! +//! ### Observability +//! +//! | Example | What it shows | +//! |-----------------------------------------------------------------------------------------|-----------------------------------------------------------| +//! | [subscriber](https://github.com/soltiHQ/taskvisor/blob/main/examples/subscriber.rs) | React to lifecycle events with your own handler | +//! | [tracing](https://github.com/soltiHQ/taskvisor/blob/main/examples/tracing.rs) | Send events into `tracing` (feature `tracing`) | +//! | [metrics](https://github.com/soltiHQ/taskvisor/blob/main/examples/metrics.rs) | Build Prometheus counters from lifecycle events | +//! +//! ### Dynamic work and outcomes +//! +//! | Example | What it shows | +//! |-----------------------------------------------------------------------------------------|----------------------------------------------------------| +//! | [dynamic](https://github.com/soltiHQ/taskvisor/blob/main/examples/dynamic.rs) | Add, list, cancel, and remove tasks at runtime | +//! | [outcomes](https://github.com/soltiHQ/taskvisor/blob/main/examples/outcomes.rs) | Wait for reliable outcomes, including a timeout | +//! +//! ### Keyed admission +//! +//! | Example | What it shows | +//! |-----------------------------------------------------------------------------------------------|-------------------------------------------------------| +//! | [tenant_sync](https://github.com/soltiHQ/taskvisor/blob/main/examples/tenant_sync.rs) | Keep only the latest sync revision per tenant | +//! | [slots](https://github.com/soltiHQ/taskvisor/blob/main/examples/slots.rs) | Compare queue, replace, and reject policies | +//! | [admission](https://github.com/soltiHQ/taskvisor/blob/main/examples/admission.rs) | Observe typed admission and rejection outcomes | #![forbid(unsafe_code)] #![warn(missing_docs)] @@ -152,7 +186,7 @@ struct ReadmeDoctests; pub mod core; pub use core::{ ConfigError, Supervisor, SupervisorBuilder, SupervisorConfig, SupervisorHandle, TaskDefaults, - TaskOutcome, TaskWaiter, + TaskOutcome, TaskOutcomeKind, TaskWaiter, }; pub mod tasks; @@ -175,7 +209,7 @@ pub use identity::TaskId; pub mod prelude; -pub mod reasons; +pub(crate) mod reasons; #[cfg(feature = "controller")] #[cfg_attr(docsrs, doc(cfg(feature = "controller")))] @@ -184,7 +218,7 @@ pub mod controller; #[cfg_attr(docsrs, doc(cfg(feature = "controller")))] pub use controller::{ AdmissionPolicy, ControllerConfig, ControllerError, ControllerSnapshot, ControllerSpec, - SlotStatusKind, SlotView, + PreparedSubmission, SlotStatusKind, SlotView, }; #[cfg(feature = "logging")] diff --git a/src/prelude.rs b/src/prelude.rs index 573be34..93ac41f 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -16,7 +16,7 @@ /// Core supervisor runtime. pub use crate::core::{ ConfigError, Supervisor, SupervisorBuilder, SupervisorConfig, SupervisorHandle, TaskDefaults, - TaskOutcome, TaskWaiter, + TaskOutcome, TaskOutcomeKind, TaskWaiter, }; /// Task abstractions and task specs. @@ -44,7 +44,7 @@ pub use crate::identity::TaskId; #[cfg_attr(docsrs, doc(cfg(feature = "controller")))] pub use crate::controller::{ AdmissionPolicy, ControllerConfig, ControllerError, ControllerSnapshot, ControllerSpec, - SlotStatusKind, SlotView, + PreparedSubmission, SlotStatusKind, SlotView, }; /// Built-in logging subscriber. diff --git a/src/reasons.rs b/src/reasons.rs index 9059d1b..0501486 100644 --- a/src/reasons.rs +++ b/src/reasons.rs @@ -1,81 +1,30 @@ -//! # Stable `reason` values +//! Internal fragments used to compose readable diagnostics. //! -//! Some events and outcomes have a `reason` string. -//! Code can check the stable values and prefixes in this module. -//! Other reason strings are not stable and may change between releases. -//! -//! Exact value: -//! -//! ```rust -//! use taskvisor::reasons::ALREADY_EXISTS; -//! assert_eq!("already_exists", ALREADY_EXISTS); -//! ``` -//! -//! Prefix with details: -//! -//! ```rust -//! use taskvisor::reasons::MAX_RETRIES_EXCEEDED; -//! let reason = "max_retries_exceeded(3/3): connection lost"; -//! assert!(reason.starts_with(MAX_RETRIES_EXCEEDED)); -//! ``` -//! -//! ## Where each value appears -//! -//! | Constant | Used by | Meaning | -//! |--------------------------------------|--------------------------------------------------------|----------------------------------------| -//! | [`POLICY_EXHAUSTED_SUCCESS`] | `ActorExhausted` | Normal one-shot finish. | -//! | [`TASK_RETURNED_CANCELED`] | `ActorExhausted` | The task stopped itself. | -//! | [`MAX_RETRIES_EXCEEDED`] | `ActorExhausted` (prefix) | Retry limit reached. | -//! | [`ALREADY_EXISTS`] | `TaskAddFailed`, `TaskOutcome::Rejected` | A task with this name already exists. | -//! | [`BATCH_REJECTED`] | `TaskAddFailed` | Another item rejected the whole batch. | -//! | [`QUEUE_FULL`] | `ControllerRejected`, `TaskOutcome::Rejected` (prefix) | The slot queue is full. | -//! | [`DROP_IF_RUNNING`] | `ControllerRejected`, `TaskOutcome::Rejected` (prefix) | A busy slot rejected new work. | -//! | [`REMOVED_FROM_QUEUE`] | `ControllerRejected`, `TaskOutcome::Rejected` | A queued task was removed. | -//! | [`SUPERSEDED_BY_REPLACE`] | `ControllerRejected`, `TaskOutcome::Rejected` | A newer `Replace` task took its place. | -//! | [`CONTROLLER_SHUTTING_DOWN`] | `ControllerRejected`, `TaskOutcome::Rejected` | The controller is shutting down. | -//! | [`FORCE_TERMINATED_AFTER_GRACE`] | `TaskRemoved` | The actor was force-aborted. | -//! - -/// `ActorExhausted` reason: the task finished successfully under a `Never`/`OnFailure` policy. -/// This is not an error. -pub const POLICY_EXHAUSTED_SUCCESS: &str = "policy_exhausted_success"; - -/// `ActorExhausted` reason: the task body returned [`TaskError::Canceled`](crate::TaskError::Canceled), but the runtime did not cancel it. -/// This is not an error. -pub const TASK_RETURNED_CANCELED: &str = "task_returned_canceled"; +//! These strings are deliberately not public API and carry no compatibility guarantee. +//! Runtime consumers must branch on typed fields such as `TaskOutcomeKind` and `RejectionKind`, never parse `reason`. /// Registration/rejection reason: another registered task already uses this name. -pub const ALREADY_EXISTS: &str = "already_exists"; +pub(crate) const ALREADY_EXISTS: &str = "a registered task already uses this name"; /// Registration reason: another item caused an all-or-nothing static batch to be rejected. -pub const BATCH_REJECTED: &str = "batch_rejected"; +pub(crate) const BATCH_REJECTED: &str = "another item rejected the all-or-nothing batch"; /// Rejection reason: a queued task was removed before it started. -pub const REMOVED_FROM_QUEUE: &str = "removed_from_queue"; +#[cfg(feature = "controller")] +pub(crate) const REMOVED_FROM_QUEUE: &str = "removed from controller queue before start"; /// Rejection reason: a newer `Replace` task took this task's place. -pub const SUPERSEDED_BY_REPLACE: &str = "superseded_by_replace"; +#[cfg(feature = "controller")] +pub(crate) const SUPERSEDED_BY_REPLACE: &str = "superseded by a newer replacement"; /// Rejection reason: the controller is shutting down. -pub const CONTROLLER_SHUTTING_DOWN: &str = "controller_shutting_down"; - -/// `ControllerRejected` and `TaskOutcome::Rejected` reason **prefix**: `DropIfRunning` rejected a submission because its slot was busy. -/// -/// The full reason is `dropped: slot busy ()`. -/// Match with `starts_with`, not equality. -pub const DROP_IF_RUNNING: &str = "dropped: slot busy"; - -/// `TaskRemoved` reason: the actor did not stop within its grace window and was force-aborted. -pub const FORCE_TERMINATED_AFTER_GRACE: &str = "force_terminated_after_grace"; +#[cfg(feature = "controller")] +pub(crate) const CONTROLLER_SHUTTING_DOWN: &str = "controller is shutting down"; -/// `ActorExhausted` reason **prefix**: the task stopped after it used all retry attempts. -/// -/// The full reason is `max_retries_exceeded(/): `. -/// Match with `starts_with`, not equality. -pub const MAX_RETRIES_EXCEEDED: &str = "max_retries_exceeded"; +/// Diagnostic text used when `DropIfRunning` rejects a busy slot. +#[cfg(feature = "controller")] +pub(crate) const DROP_IF_RUNNING: &str = "slot is busy; DropIfRunning rejected the submission"; -/// Rejection reason **prefix**: a controller slot queue is full. -/// -/// The full reason is `queue_full: /`. -/// Match with `starts_with`, not equality. -pub const QUEUE_FULL: &str = "queue_full"; +/// Diagnostic text used when a controller slot queue is full. +#[cfg(feature = "controller")] +pub(crate) const QUEUE_FULL: &str = "slot queue is full"; diff --git a/src/subscribers/embedded/log.rs b/src/subscribers/embedded/log.rs index 4d8e418..a7bc62b 100644 --- a/src/subscribers/embedded/log.rs +++ b/src/subscribers/embedded/log.rs @@ -9,12 +9,12 @@ //! //! ## Example output //! ```text -//! [001] [task_starting] task=worker attempt=1 -//! [002] [task_failed] task=worker reason="connection refused" attempt=1 +//! [001] [attempt_starting] task=worker attempt=1 +//! [002] [attempt_failed] task=worker reason="connection refused" attempt=1 //! [003] [backoff_scheduled] task=worker source=failure delay=2s after_attempt=1 reason="connection refused" -//! [004] [timeout_hit] task=worker timeout=5s -//! [005] [task_stopped] task=worker -//! [006] [actor_exhausted] task=worker reason=policy +//! [004] [attempt_timed_out] task=worker timeout=5s +//! [005] [attempt_succeeded] task=worker +//! [006] [task_finished] task=worker outcome=outcome_completed //! [007] [task_add_requested] task=new-worker //! [008] [task_added] task=new-worker //! [009] [task_remove_requested] task=old-worker @@ -83,8 +83,8 @@ impl LogWriter { } // Task lifecycle and management: task name only. - EventKind::TaskStopped - | EventKind::TaskCanceled + EventKind::AttemptSucceeded + | EventKind::AttemptCanceled | EventKind::TaskAddRequested | EventKind::TaskAdded | EventKind::TaskRemoveRequested @@ -92,14 +92,14 @@ impl LogWriter { println!("{head} task={}", or(e.task.as_deref(), "none")); } - EventKind::TaskStarting => { + EventKind::AttemptStarting => { println!( "{head} task={} attempt={}", or(e.task.as_deref(), "none"), e.attempt.unwrap_or(0) ); } - EventKind::TaskFailed => { + EventKind::AttemptFailed => { println!( "{head} task={} reason=\"{}\" attempt={}", or(e.task.as_deref(), "none"), @@ -114,7 +114,7 @@ impl LogWriter { or(e.reason.as_deref(), "unknown") ); } - EventKind::TimeoutHit => { + EventKind::AttemptTimedOut => { println!( "{head} task={} timeout={}", or(e.task.as_deref(), "none"), @@ -149,20 +149,18 @@ impl LogWriter { ); } - // Terminals. - EventKind::ActorExhausted => { - println!( - "{head} task={} reason=\"{}\"", - or(e.task.as_deref(), "none"), - or(e.reason.as_deref(), "policy") - ); - } - EventKind::ActorDead => { - println!( - "{head} task={} reason=\"{}\"", - or(e.task.as_deref(), "none"), - or(e.reason.as_deref(), "fatal") - ); + // Registered-task terminal outcome. + EventKind::TaskFinished => { + let task = or(e.task.as_deref(), "none"); + let outcome = e + .outcome_kind + .map(|kind| kind.as_label()) + .unwrap_or("unknown"); + if let Some(reason) = e.reason.as_deref() { + println!("{head} task={task} outcome={outcome} reason=\"{reason}\""); + } else { + println!("{head} task={task} outcome={outcome}"); + } } // Controller: the `task` field carries the slot name. @@ -207,9 +205,9 @@ mod tests { #[test] fn event_head_keeps_the_full_sequence_number() { - let mut event = Event::new(EventKind::TaskStarting); + let mut event = Event::new(EventKind::AttemptStarting); event.seq = 12_345; - assert_eq!(event_head(&event), "[12345] [task_starting]"); + assert_eq!(event_head(&event), "[12345] [attempt_starting]"); } } diff --git a/src/subscribers/embedded/tracing.rs b/src/subscribers/embedded/tracing.rs index 440f99b..7d063bc 100644 --- a/src/subscribers/embedded/tracing.rs +++ b/src/subscribers/embedded/tracing.rs @@ -5,7 +5,7 @@ //! Each tracing event uses target `taskvisor` and contains: //! - a level based on the event severity (see [`TracingBridge`]), //! - structured fields: `event` (the stable label), `seq`, and the optional payload fields that are set -//! (`task`, `id`, `attempt`, `reason`, `delay_ms`, `timeout_ms`, `duration_ms`, `exit_code`, `backoff_source`). +//! (`task`, `id`, `attempt`, `reason`, `outcome_kind`, `rejection_kind`, `delay_ms`, `timeout_ms`, `duration_ms`, `exit_code`, `backoff_source`). //! //! Unset optional fields are not recorded. //! @@ -20,17 +20,17 @@ use tracing::Level; +use crate::TaskOutcomeKind; use crate::events::{Event, EventKind}; -use crate::reasons::MAX_RETRIES_EXCEEDED; use crate::subscribers::Subscribe; /// Sends runtime events to [`tracing`] as structured events. /// /// Level mapping: -/// - `ERROR`: task failed, actor dead, subscriber panicked. -/// - `WARN`: timeout, grace exceeded, subscriber overflow, add failed, controller rejected. -/// - `INFO`: lifecycle milestones (stopped, canceled, added, removed, shutdown, submitted, and `actor_exhausted` for every other reason). -/// - `DEBUG`: chatty events (starting, backoff, add/remove requests, slot transitions). +/// - `ERROR`: failed attempts, fatal/panicked terminal outcomes, subscriber panics, runtime failures. +/// - `WARN`: timeouts, non-fatal failed/force-aborted outcomes, grace exceeded, overflow, and rejection. +/// - `INFO`: successful/canceled attempts and task outcomes, registration, removal, and shutdown milestones. +/// - `DEBUG`: attempt starts, backoff, management requests, and slot transitions. /// /// ## Also /// @@ -43,32 +43,32 @@ pub struct TracingBridge; /// Maps an event to a tracing level. fn level_for(e: &Event) -> Level { match e.kind { - EventKind::TaskFailed - | EventKind::ActorDead - | EventKind::SubscriberPanicked - | EventKind::RuntimeFailure => Level::ERROR, + EventKind::AttemptFailed | EventKind::SubscriberPanicked | EventKind::RuntimeFailure => { + Level::ERROR + } - EventKind::TimeoutHit + EventKind::AttemptTimedOut | EventKind::GraceExceeded | EventKind::SubscriberOverflow | EventKind::TaskAddFailed => Level::WARN, - EventKind::ActorExhausted => { - let gave_up = e - .reason - .as_deref() - .is_some_and(|r| r.starts_with(MAX_RETRIES_EXCEEDED)); - if gave_up { Level::WARN } else { Level::INFO } - } + EventKind::TaskFinished => match e.outcome_kind { + Some(TaskOutcomeKind::Fatal | TaskOutcomeKind::Panicked) => Level::ERROR, + Some( + TaskOutcomeKind::Failed | TaskOutcomeKind::ForceAborted | TaskOutcomeKind::Rejected, + ) + | None => Level::WARN, + Some(TaskOutcomeKind::Completed | TaskOutcomeKind::Canceled) => Level::INFO, + }, - EventKind::TaskStopped - | EventKind::TaskCanceled + EventKind::AttemptSucceeded + | EventKind::AttemptCanceled | EventKind::TaskAdded | EventKind::TaskRemoved | EventKind::ShutdownRequested | EventKind::AllStoppedWithinGrace => Level::INFO, - EventKind::TaskStarting + EventKind::AttemptStarting | EventKind::BackoffScheduled | EventKind::TaskAddRequested | EventKind::TaskRemoveRequested => Level::DEBUG, @@ -101,6 +101,7 @@ impl Subscribe for TracingBridge { exit_code = e.exit_code.map(i64::from), backoff_source = e.backoff_source.map(|s| s.as_label()), rejection_kind = e.rejection_kind.map(|kind| kind.as_label()), + outcome_kind = e.outcome_kind.map(TaskOutcomeKind::as_label), ) }; } @@ -183,16 +184,19 @@ mod tests { } #[test] - fn task_failed_maps_to_error_with_structured_fields() { - let e = Event::new(EventKind::TaskFailed) + fn attempt_failed_maps_to_error_with_structured_fields() { + let e = Event::new(EventKind::AttemptFailed) .with_task("worker") .with_reason("boom") .with_attempt(2); let (level, fields) = capture_one(&e); - assert_eq!(level, Level::ERROR, "TaskFailed must map to ERROR"); - assert_eq!(fields.get("event").map(String::as_str), Some("task_failed")); + assert_eq!(level, Level::ERROR, "AttemptFailed must map to ERROR"); + assert_eq!( + fields.get("event").map(String::as_str), + Some("attempt_failed") + ); assert_eq!(fields.get("task").map(String::as_str), Some("worker")); assert_eq!(fields.get("reason").map(String::as_str), Some("boom")); assert_eq!(fields.get("attempt").map(String::as_str), Some("2")); @@ -201,10 +205,9 @@ mod tests { #[test] fn levels_match_event_severity() { let cases = [ - (EventKind::TaskStopped, Level::INFO), - (EventKind::TaskStarting, Level::DEBUG), - (EventKind::TimeoutHit, Level::WARN), - (EventKind::ActorDead, Level::ERROR), + (EventKind::AttemptSucceeded, Level::INFO), + (EventKind::AttemptStarting, Level::DEBUG), + (EventKind::AttemptTimedOut, Level::WARN), (EventKind::GraceExceeded, Level::WARN), (EventKind::BackoffScheduled, Level::DEBUG), ]; @@ -215,19 +218,28 @@ mod tests { } #[test] - fn actor_exhausted_level_depends_on_reason() { - for (reason, expected) in [ - (Some("max_retries_exceeded(3/3): boom"), Level::WARN), - (Some("policy_exhausted_success"), Level::INFO), - (Some("task_returned_canceled"), Level::INFO), - (None, Level::INFO), + fn task_finished_level_and_field_depend_on_outcome_kind() { + for (outcome_kind, expected) in [ + (TaskOutcomeKind::Completed, Level::INFO), + (TaskOutcomeKind::Canceled, Level::INFO), + (TaskOutcomeKind::Failed, Level::WARN), + (TaskOutcomeKind::ForceAborted, Level::WARN), + (TaskOutcomeKind::Rejected, Level::WARN), + (TaskOutcomeKind::Fatal, Level::ERROR), + (TaskOutcomeKind::Panicked, Level::ERROR), ] { - let mut e = Event::new(EventKind::ActorExhausted).with_task("worker"); - if let Some(reason) = reason { - e = e.with_reason(reason); - } + let e = Event::new(EventKind::TaskFinished) + .with_task("worker") + .with_outcome_kind(outcome_kind) + .with_reason("free-form diagnostic text"); let (level, _) = capture_one(&e); - assert_eq!(level, expected, "wrong level for reason {reason:?}"); + assert_eq!(level, expected, "wrong level for {outcome_kind:?}"); + + let (_, fields) = capture_one(&e); + assert_eq!( + fields.get("outcome_kind").map(String::as_str), + Some(outcome_kind.as_label()) + ); } } @@ -250,6 +262,8 @@ mod tests { "duration_ms", "exit_code", "backoff_source", + "rejection_kind", + "outcome_kind", ] { assert!( !fields.contains_key(absent), diff --git a/src/subscribers/subscriber.rs b/src/subscribers/subscriber.rs index 9909d71..f2d50f0 100644 --- a/src/subscribers/subscriber.rs +++ b/src/subscribers/subscriber.rs @@ -38,7 +38,7 @@ //! //! impl Subscribe for Metrics { //! fn on_event(&self, ev: &Event) { -//! if matches!(ev.kind, EventKind::TaskFailed) { +//! if matches!(ev.kind, EventKind::AttemptFailed) { //! // update counters, push to a channel, etc. //! } //! } diff --git a/src/subscribers/subscriber_set.rs b/src/subscribers/subscriber_set.rs index d103286..9e2d442 100644 --- a/src/subscribers/subscriber_set.rs +++ b/src/subscribers/subscriber_set.rs @@ -323,7 +323,7 @@ mod tests { use tokio::sync::broadcast; fn ev(task: &str) -> Arc { - Arc::new(Event::new(EventKind::TaskStarting).with_task(task)) + Arc::new(Event::new(EventKind::AttemptStarting).with_task(task)) } fn kind_ev(kind: EventKind) -> Arc { diff --git a/tests/controller.rs b/tests/controller.rs index bd8e331..e8e4675 100644 --- a/tests/controller.rs +++ b/tests/controller.rs @@ -3,6 +3,7 @@ mod common; use std::num::NonZeroUsize; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -36,9 +37,9 @@ async fn submit_running(handle: &SupervisorHandle, spec: ControllerSpec) -> Task id } -async fn expect_rejected(waiter: TaskWaiter) -> Arc { +async fn expect_rejected(waiter: TaskWaiter) -> RejectionKind { match waiter.wait().await.expect("waiter errored") { - TaskOutcome::Rejected { reason, .. } => reason, + TaskOutcome::Rejected { kind, .. } => kind, other => panic!("expected Rejected, got {other:?}"), } } @@ -75,6 +76,100 @@ fn controller_spec_components_are_configured_through_accessors() { // Watched submission contracts. +#[tokio::test(flavor = "current_thread")] +async fn prepared_submission_exposes_identity_before_events_and_preserves_it() { + let (handle, collector) = served_controller(ControllerConfig::default()); + + with_timeout(10, async { + let request = ControllerSpec::queue(TaskSpec::once(make_ok_once("prepared-watched"))) + .with_slot("prepared-slot"); + let prepared = handle + .prepare_submission(request) + .expect("controller is configured"); + let reserved_id = prepared.id(); + + assert_eq!(prepared.spec().slot_name(), "prepared-slot"); + assert!( + collector.by_id(reserved_id).is_empty(), + "preparation must not publish an event" + ); + + let (submitted_id, waiter) = prepared + .submit_and_watch() + .await + .expect("prepared submission must enter the controller queue"); + assert_eq!(submitted_id, reserved_id); + assert_eq!(waiter.id(), reserved_id); + assert!(matches!(waiter.wait().await, Ok(TaskOutcome::Completed))); + + assert!( + collector + .wait_until(Duration::from_secs(2), |events| { + events.iter().any(|event| { + event.id == Some(reserved_id) && event.kind == EventKind::TaskRemoved + }) + }) + .await, + "the prepared identity must be used through terminal cleanup" + ); + assert!( + collector.by_id(reserved_id).iter().any(|event| { + event.kind == EventKind::AttemptStarting && event.attempt == Some(1) + }) + ); + + handle.shutdown().await.expect("shutdown ok"); + }) + .await; +} + +#[tokio::test(flavor = "current_thread")] +async fn dropping_prepared_submission_starts_no_work_and_publishes_no_event() { + let (handle, collector) = served_controller(ControllerConfig::default()); + let starts = Arc::new(AtomicUsize::new(0)); + let task_starts = Arc::clone(&starts); + let task = TaskFn::arc("prepared-dropped", move |_ctx| { + let starts = Arc::clone(&task_starts); + async move { + starts.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + }); + + with_timeout(10, async { + let prepared = handle + .prepare_submission( + ControllerSpec::queue(TaskSpec::once(task)).with_slot("prepared-dropped"), + ) + .expect("controller is configured"); + let dropped_id = prepared.id(); + drop(prepared); + + let (barrier_id, barrier) = handle + .submit_and_watch(ControllerSpec::queue(TaskSpec::once(make_ok_once( + "prepared-drop-barrier", + )))) + .await + .expect("barrier submission"); + assert!(matches!(barrier.wait().await, Ok(TaskOutcome::Completed))); + assert!( + collector + .wait_until(Duration::from_secs(2), |events| { + events.iter().any(|event| { + event.id == Some(barrier_id) && event.kind == EventKind::TaskRemoved + }) + }) + .await + ); + + assert_eq!(starts.load(Ordering::SeqCst), 0); + assert!(collector.by_id(dropped_id).is_empty()); + + handle.shutdown().await.expect("shutdown ok"); + }) + .await; +} + #[tokio::test(flavor = "current_thread")] async fn watched_submit_variants_resolve_completed_for_admitted_tasks() { let (handle, _collector) = served_controller(ControllerConfig::default()); @@ -94,11 +189,16 @@ async fn watched_submit_variants_resolve_completed_for_admitted_tasks() { "an admitted task that succeeds must resolve Completed, got {outcome:?}" ); - let (id, waiter) = handle - .try_submit_and_watch(ControllerSpec::queue(TaskSpec::once(make_ok_once( + let prepared = handle + .prepare_submission(ControllerSpec::queue(TaskSpec::once(make_ok_once( "try-watched-ok", )))) + .expect("controller is configured"); + let reserved_id = prepared.id(); + let (id, waiter) = prepared + .try_submit_and_watch() .expect("the controller queue has capacity"); + assert_eq!(id, reserved_id); assert_eq!(waiter.id(), id); assert!(matches!(waiter.wait().await, Ok(TaskOutcome::Completed))); @@ -109,7 +209,7 @@ async fn watched_submit_variants_resolve_completed_for_admitted_tasks() { #[tokio::test(flavor = "current_thread")] async fn submit_and_watch_resolves_rejected_on_drop_if_running() { - let (handle, _collector) = served_controller(ControllerConfig::default()); + let (handle, collector) = served_controller(ControllerConfig::default()); with_timeout(10, async { submit_running( @@ -118,7 +218,7 @@ async fn submit_and_watch_resolves_rejected_on_drop_if_running() { ) .await; - let (_id, waiter) = handle + let (id, waiter) = handle .submit_and_watch( ControllerSpec::drop_if_running(TaskSpec::restartable(make_coop("dropped-w"))) .with_slot("s"), @@ -126,11 +226,38 @@ async fn submit_and_watch_resolves_rejected_on_drop_if_running() { .await .expect("submit_and_watch accepted into channel"); - let reason = expect_rejected(waiter).await; + let outcome = waiter.wait().await.expect("waiter errored"); + assert!(matches!( + outcome, + TaskOutcome::Rejected { + kind: RejectionKind::SlotBusy, + .. + } + )); assert!( - reason.contains("dropped"), - "rejection reason must explain why: {reason}" + collector + .wait_until(Duration::from_secs(2), |events| { + events.iter().any(|event| { + event.id == Some(id) && event.kind == EventKind::ControllerRejected + }) + }) + .await ); + let by_id = collector.by_id(id); + assert!(by_id.iter().any(|event| { + event.kind == EventKind::ControllerRejected + && event.outcome_kind == Some(TaskOutcomeKind::Rejected) + && event.rejection_kind == Some(RejectionKind::SlotBusy) + })); + assert!(by_id.iter().all(|event| { + !matches!( + event.kind, + EventKind::TaskAdded + | EventKind::AttemptStarting + | EventKind::TaskFinished + | EventKind::TaskRemoved + ) + })); handle.shutdown().await.expect("shutdown ok"); }) @@ -167,7 +294,10 @@ async fn cancel_immediately_removes_a_watched_queued_submission() { "a queued submission can be claimed only once" ); - assert_eq!(&*expect_rejected(waiter).await, "removed_from_queue"); + assert_eq!( + expect_rejected(waiter).await, + RejectionKind::RemovedFromQueue + ); handle.shutdown().await.expect("shutdown ok"); }) @@ -237,7 +367,8 @@ async fn remove_of_queued_submission_purges_it_before_start() { events.iter().any(|event| { event.kind == EventKind::ControllerRejected && event.id == Some(victim_id) - && event.reason.as_deref() == Some("removed_from_queue") + && event.rejection_kind == Some(RejectionKind::RemovedFromQueue) + && event.outcome_kind == Some(TaskOutcomeKind::Rejected) }) }) .await, @@ -265,7 +396,7 @@ async fn remove_of_queued_submission_purges_it_before_start() { collector .by_label("queued-victim") .iter() - .all(|e| e.kind != EventKind::TaskStarting), + .all(|e| e.kind != EventKind::AttemptStarting), "a removed queued submission must never start" ); @@ -303,7 +434,7 @@ async fn shutdown_does_not_start_queued_tasks() { || collector .by_label("queued") .iter() - .all(|e| e.kind != EventKind::TaskStarting), + .all(|e| e.kind != EventKind::AttemptStarting), "queued task must not start during shutdown" ); } @@ -329,6 +460,14 @@ async fn submit_without_controller_is_consistent_across_construction_paths() { for (constructor, supervisor, spec) in cases { let handle = supervisor.serve(); + assert!( + matches!( + handle.prepare_submission(spec.clone()), + Err(ControllerError::NotConfigured) + ), + "prepare_submission must reject a supervisor created through {constructor}" + ); + assert_eq!( handle.submit(spec.clone()).await, Err(ControllerError::NotConfigured), @@ -401,7 +540,7 @@ async fn idle_submit_admits_emits_submitted_then_running_transition() { collector .by_label("runner-7") .iter() - .any(|e| { e.kind == EventKind::TaskStarting && e.id == Some(id) }), + .any(|e| { e.kind == EventKind::AttemptStarting && e.id == Some(id) }), "the lifecycle must run under the id minted at submit()" ); @@ -496,9 +635,8 @@ async fn drop_if_running_rejects_busy_submission_without_starting_it() { event.task.as_deref() == Some("s") && event.kind == EventKind::ControllerRejected && event.id == Some(rejected_id) - && event.reason.as_deref().is_some_and(|reason| { - reason.starts_with(taskvisor::reasons::DROP_IF_RUNNING) - }) + && event.outcome_kind == Some(TaskOutcomeKind::Rejected) + && event.rejection_kind == Some(RejectionKind::SlotBusy) }) }) .await @@ -663,10 +801,8 @@ async fn queue_full_rejects_with_controller_rejected_event() { events.iter().any(|event| { event.task.as_deref() == Some("s") && event.kind == EventKind::ControllerRejected - && event - .reason - .as_deref() - .is_some_and(|reason| reason.contains("queue_full")) + && event.rejection_kind == Some(RejectionKind::QueueFull) + && event.outcome_kind == Some(TaskOutcomeKind::Rejected) }) }) .await diff --git a/tests/failure.rs b/tests/failure.rs index 9164d31..f73acf4 100644 --- a/tests/failure.rs +++ b/tests/failure.rs @@ -2,6 +2,7 @@ mod common; +use std::num::NonZeroUsize; use std::sync::Arc; use std::time::Duration; @@ -17,27 +18,24 @@ async fn run_to_completion(spec: TaskSpec) -> Arc { } #[tokio::test(flavor = "current_thread")] -async fn task_failure_exit_code_propagates_to_terminal_events() { +async fn task_failure_exit_code_propagates_to_attempt_and_task_events() { for (name, expected_code) in [("fail-code", Some(7)), ("logical", None)] { let collector = run_to_completion(TaskSpec::once(make_fail(name, expected_code))).await; - let exhausted = collector - .wait_for(EventKind::ActorExhausted, Duration::from_secs(2)) + let finished = collector + .wait_for(EventKind::TaskFinished, Duration::from_secs(2)) .await - .unwrap_or_else(|| panic!("{name}: ActorExhausted was not observed")); + .unwrap_or_else(|| panic!("{name}: TaskFinished was not observed")); let failed = collector - .find(EventKind::TaskFailed) - .unwrap_or_else(|| panic!("{name}: TaskFailed was not observed")); + .find(EventKind::AttemptFailed) + .unwrap_or_else(|| panic!("{name}: AttemptFailed was not observed")); - assert_eq!(failed.exit_code, expected_code, "{name}: TaskFailed"); - assert_eq!(exhausted.exit_code, expected_code, "{name}: exhausted"); + assert_eq!(failed.exit_code, expected_code, "{name}: AttemptFailed"); + assert_eq!(finished.exit_code, expected_code, "{name}: TaskFinished"); + assert_eq!(finished.outcome_kind, Some(TaskOutcomeKind::Failed)); assert_eq!(failed.attempt, Some(1), "{name}: first attempt"); assert!( - !exhausted - .reason - .as_deref() - .unwrap_or("") - .contains("max_retries_exceeded"), - "{name}: RestartPolicy::Never is not retry-budget exhaustion" + finished.reason.is_some(), + "{name}: diagnostic detail is retained" ); } } @@ -57,8 +55,8 @@ async fn panicking_task_is_reaped_and_run_returns() { "panicked task must be reaped (TaskRemoved published)" ); assert!( - collector.any_reason_contains(EventKind::TaskFailed, "panic"), - "panic must surface as TaskFailed with a panic reason" + collector.any_reason_contains(EventKind::AttemptFailed, "panic"), + "panic must surface as AttemptFailed with a panic reason" ); } @@ -91,11 +89,12 @@ async fn panicking_task_restarts_per_policy_then_succeeds() { .wait_until(Duration::from_secs(2), |events| { events.iter().any(|event| { event.task.as_deref() == Some("flaky-panic") - && event.kind == EventKind::ActorExhausted + && event.kind == EventKind::TaskFinished + && event.outcome_kind == Some(TaskOutcomeKind::Completed) }) }) .await, - "actor must finish normally after panics are retried" + "task must finish normally after panics are retried" ); } @@ -110,14 +109,16 @@ async fn task_returning_canceled_without_cancellation_is_reaped() { let collector = run_to_completion(spec).await; - assert!( - collector.any_reason_contains(EventKind::ActorExhausted, "task_returned_canceled"), - "spurious Canceled must surface as ActorExhausted with an explicit reason" + let finished = collector.find(EventKind::TaskFinished).unwrap(); + assert_eq!(finished.outcome_kind, Some(TaskOutcomeKind::Canceled)); + assert_eq!( + finished.reason, None, + "classification must not require reason text" ); } #[tokio::test(flavor = "current_thread")] -async fn cooperative_cancellation_returning_ok_yields_task_stopped() { +async fn cooperative_cancellation_returning_ok_yields_succeeded_attempt_and_canceled_task() { let (handle, collector) = served_with_collector(SupervisorConfig::default().with_grace(Duration::from_secs(5))); @@ -140,9 +141,11 @@ async fn cooperative_cancellation_returning_ok_yields_task_stopped() { ); let by_id = collector.by_id(id); - assert!(by_id.iter().any(|e| e.kind == EventKind::TaskStopped)); - assert!(by_id.iter().all(|e| e.kind != EventKind::TaskFailed)); - assert!(by_id.iter().all(|e| e.kind != EventKind::ActorDead)); + assert!(by_id.iter().any(|e| e.kind == EventKind::AttemptSucceeded)); + assert!(by_id.iter().all(|e| e.kind != EventKind::AttemptFailed)); + assert!(by_id.iter().any(|e| { + e.kind == EventKind::TaskFinished && e.outcome_kind == Some(TaskOutcomeKind::Canceled) + })); assert!(by_id.iter().any(|e| e.kind == EventKind::TaskRemoved)); let _ = handle.shutdown().await; @@ -151,7 +154,7 @@ async fn cooperative_cancellation_returning_ok_yields_task_stopped() { } #[tokio::test(flavor = "current_thread")] -async fn cancellation_returning_canceled_error_yields_task_canceled() { +async fn cancellation_returning_canceled_error_yields_canceled_attempt_and_task() { let (handle, collector) = served_with_collector(SupervisorConfig::default().with_grace(Duration::from_secs(5))); @@ -180,19 +183,84 @@ async fn cancellation_returning_canceled_error_yields_task_canceled() { let by_id = collector.by_id(id); assert!( - by_id.iter().any(|e| e.kind == EventKind::TaskCanceled), - "graceful cancellation must surface as TaskCanceled" + by_id.iter().any(|e| e.kind == EventKind::AttemptCanceled), + "graceful cancellation must surface as AttemptCanceled" ); assert!( - by_id.iter().all(|e| e.kind != EventKind::TaskStopped), - "TaskStopped is reserved for successful attempts" + by_id.iter().all(|e| e.kind != EventKind::AttemptSucceeded), + "AttemptSucceeded is reserved for successful attempts" ); - assert!(by_id.iter().all(|e| e.kind != EventKind::TaskFailed)); - assert!(by_id.iter().all(|e| e.kind != EventKind::ActorExhausted)); - assert!(by_id.iter().all(|e| e.kind != EventKind::ActorDead)); + assert!(by_id.iter().all(|e| e.kind != EventKind::AttemptFailed)); + assert!(by_id.iter().any(|e| { + e.kind == EventKind::TaskFinished && e.outcome_kind == Some(TaskOutcomeKind::Canceled) + })); assert!(by_id.iter().any(|e| e.kind == EventKind::TaskRemoved)); let _ = handle.shutdown().await; }) .await; } + +#[tokio::test(flavor = "current_thread")] +async fn cancellation_while_waiting_for_a_permit_finishes_without_an_attempt() { + let (handle, collector) = served_with_collector( + SupervisorConfig::default().with_max_concurrent(NonZeroUsize::new(1)), + ); + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let permit_owner: TaskRef = TaskFn::arc("permit-owner", { + let started = Arc::clone(&started); + let release = Arc::clone(&release); + move |_ctx: TaskContext| { + let started = Arc::clone(&started); + let release = Arc::clone(&release); + async move { + started.notify_one(); + release.notified().await; + Ok(()) + } + } + }); + + handle.add(TaskSpec::once(permit_owner)).await.unwrap(); + started.notified().await; + + let (id, waiter) = handle + .add_and_watch(TaskSpec::once(make_ok_once("permit-waiter"))) + .await + .unwrap(); + assert!(handle.cancel(id).await.unwrap()); + assert!(matches!( + waiter.wait().await.unwrap(), + TaskOutcome::Canceled + )); + assert!( + collector + .wait_until(Duration::from_secs(2), |events| { + events + .iter() + .any(|event| event.id == Some(id) && event.kind == EventKind::TaskRemoved) + }) + .await + ); + + let by_id = collector.by_id(id); + assert!( + by_id + .iter() + .all(|event| event.kind != EventKind::AttemptStarting) + ); + assert_eq!( + by_id + .iter() + .filter(|event| { + event.kind == EventKind::TaskFinished + && event.outcome_kind == Some(TaskOutcomeKind::Canceled) + }) + .count(), + 1 + ); + + release.notify_one(); + handle.shutdown().await.unwrap(); +} diff --git a/tests/identity.rs b/tests/identity.rs index c009bfc..893b22e 100644 --- a/tests/identity.rs +++ b/tests/identity.rs @@ -42,7 +42,7 @@ async fn add_confirms_registration_returns_id_and_starts_task() { poll_until(Duration::from_secs(2), || async { handle.is_alive("worker").await && collector.by_id(id).iter().any(|e| { - e.kind == EventKind::TaskStarting && e.task.as_deref() == Some("worker") + e.kind == EventKind::AttemptStarting && e.task.as_deref() == Some("worker") }) }) .await @@ -389,16 +389,24 @@ async fn individually_removed_stuck_task_is_force_aborted_after_grace() { assert!( collector .wait_until(Duration::from_secs(3), |events| { - events.iter().any(|event| { - event.id == Some(id) - && event.kind == EventKind::TaskRemoved - && event.reason.as_deref() - == Some(taskvisor::reasons::FORCE_TERMINATED_AFTER_GRACE) - }) + events + .iter() + .any(|event| event.id == Some(id) && event.kind == EventKind::TaskRemoved) }) .await, "stuck task must be force-aborted after grace, not leaked" ); + assert_eq!( + collector + .by_id(id) + .iter() + .filter(|event| { + event.kind == EventKind::TaskFinished + && event.outcome_kind == Some(TaskOutcomeKind::ForceAborted) + }) + .count(), + 1 + ); let _ = handle.shutdown().await; }) .await; @@ -512,7 +520,7 @@ async fn events_carry_correct_id_across_full_lifecycle() { ); let by_id = collector.by_id(id); - assert!(by_id.iter().any(|e| e.kind == EventKind::TaskStarting)); + assert!(by_id.iter().any(|e| e.kind == EventKind::AttemptStarting)); assert!(by_id.iter().any(|e| e.kind == EventKind::TaskRemoved)); for e in collector.by_label("life") { if let Some(eid) = e.id { diff --git a/tests/lifecycle.rs b/tests/lifecycle.rs index 4d37949..ff2e3e6 100644 --- a/tests/lifecycle.rs +++ b/tests/lifecycle.rs @@ -43,54 +43,52 @@ fn supervisor_builder_is_nameable_from_public_api() { } #[tokio::test(flavor = "current_thread")] -async fn never_oneshot_success_emits_starting_stopped_exhausted_once() { +async fn never_oneshot_success_emits_attempt_and_typed_task_finish_once() { let collector = run_static(vec![TaskSpec::once(make_ok_once("oneshot"))]).await; - assert_eq!(collector.count(EventKind::TaskStarting), 1); - assert_eq!(collector.count(EventKind::TaskStopped), 1); - assert_eq!(collector.count(EventKind::ActorExhausted), 1); + assert_eq!(collector.count(EventKind::AttemptStarting), 1); + assert_eq!(collector.count(EventKind::AttemptSucceeded), 1); + assert_eq!(collector.count(EventKind::TaskFinished), 1); assert_eq!(collector.count(EventKind::TaskRemoved), 1); - assert_eq!(collector.count(EventKind::TaskFailed), 0); + assert_eq!(collector.count(EventKind::AttemptFailed), 0); assert_eq!(collector.count(EventKind::BackoffScheduled), 0); - assert_eq!(collector.count(EventKind::ActorDead), 0); - - let exhausted = collector.find(EventKind::ActorExhausted).unwrap(); - assert_eq!( - exhausted.reason.as_deref(), - Some("policy_exhausted_success") - ); - let stopped = collector.find(EventKind::TaskStopped).unwrap(); + let finished = collector.find(EventKind::TaskFinished).unwrap(); + assert_eq!(finished.outcome_kind, Some(TaskOutcomeKind::Completed)); + assert_eq!(finished.reason, None); + let stopped = collector.find(EventKind::AttemptSucceeded).unwrap(); assert!( stopped.duration_ms.is_some(), - "terminal TaskStopped must carry attempt duration" + "terminal AttemptSucceeded must carry attempt duration" + ); + assert!( + finished.seq > stopped.seq, + "TaskFinished must follow the attempt" + ); + let removed = collector.find(EventKind::TaskRemoved).unwrap(); + assert!( + removed.seq > finished.seq, + "TaskRemoved must follow TaskFinished" ); - assert!(exhausted.seq > stopped.seq, "exhausted must follow stopped"); } #[tokio::test(flavor = "current_thread")] -async fn never_oneshot_failure_emits_taskfailed_then_exhausted_no_backoff() { +async fn never_oneshot_failure_emits_failed_attempt_then_failed_task() { let task = TaskFn::arc("fail-once", |_ctx: TaskContext| async move { Err(TaskError::fail("boom".to_string())) }); let collector = run_static(vec![TaskSpec::once(task)]).await; - assert_eq!(collector.count(EventKind::TaskStarting), 1); - assert_eq!(collector.count(EventKind::TaskFailed), 1); - assert_eq!(collector.count(EventKind::ActorExhausted), 1); + assert_eq!(collector.count(EventKind::AttemptStarting), 1); + assert_eq!(collector.count(EventKind::AttemptFailed), 1); + assert_eq!(collector.count(EventKind::TaskFinished), 1); assert_eq!(collector.count(EventKind::BackoffScheduled), 0); - assert_eq!(collector.count(EventKind::ActorDead), 0); - assert_eq!(collector.count(EventKind::TaskStopped), 0); + assert_eq!(collector.count(EventKind::AttemptSucceeded), 0); - let failed = collector.find(EventKind::TaskFailed).unwrap(); + let failed = collector.find(EventKind::AttemptFailed).unwrap(); assert!(failed.reason.as_deref().unwrap().contains("boom")); - let exhausted = collector.find(EventKind::ActorExhausted).unwrap(); - assert!( - !exhausted - .reason - .as_deref() - .unwrap_or("") - .contains("max_retries_exceeded") - ); + let finished = collector.find(EventKind::TaskFinished).unwrap(); + assert_eq!(finished.outcome_kind, Some(TaskOutcomeKind::Failed)); + assert!(finished.reason.as_deref().unwrap().contains("boom")); } #[tokio::test(flavor = "current_thread")] @@ -111,48 +109,47 @@ async fn on_failure_flaky_retries_then_succeeds_failure_source_backoff() { let spec = TaskSpec::restartable(task).with_backoff(fast_backoff()); let collector = run_static(vec![spec]).await; - assert_eq!(collector.count(EventKind::TaskStarting), 3); - assert_eq!(collector.count(EventKind::TaskFailed), 2); + assert_eq!(collector.count(EventKind::AttemptStarting), 3); + assert_eq!(collector.count(EventKind::AttemptFailed), 2); assert_eq!(collector.count(EventKind::BackoffScheduled), 2); - assert_eq!(collector.count(EventKind::TaskStopped), 1); - assert_eq!(collector.count(EventKind::ActorExhausted), 1); - assert_eq!(collector.count(EventKind::ActorDead), 0); + assert_eq!(collector.count(EventKind::AttemptSucceeded), 1); + assert_eq!(collector.count(EventKind::TaskFinished), 1); for b in collector.find_all(EventKind::BackoffScheduled) { assert_eq!(b.backoff_source, Some(BackoffSource::Failure)); assert_eq!(b.delay_ms, Some(1)); assert!(b.reason.as_deref().unwrap().contains("transient-err")); } - let exhausted = collector.find(EventKind::ActorExhausted).unwrap(); - assert_eq!( - exhausted.reason.as_deref(), - Some("policy_exhausted_success") - ); - assert!( - !collector.any_reason_contains(EventKind::ActorExhausted, "max_retries_exceeded"), - "unlimited retries must end on success, not retry-budget exhaustion" - ); + let finished = collector.find(EventKind::TaskFinished).unwrap(); + assert_eq!(finished.outcome_kind, Some(TaskOutcomeKind::Completed)); + assert_eq!(finished.reason, None); } #[tokio::test(flavor = "current_thread")] -async fn on_failure_fatal_emits_actordead_with_exit_code_no_retry() { +async fn on_failure_fatal_emits_fatal_task_finish_with_exit_code_no_retry() { let spec = TaskSpec::restartable(make_fatal("fatal-task", Some(7))); let collector = run_static(vec![spec]).await; - assert_eq!(collector.count(EventKind::TaskStarting), 1); - assert_eq!(collector.count(EventKind::TaskFailed), 1); - assert_eq!(collector.count(EventKind::ActorDead), 1); + assert_eq!(collector.count(EventKind::AttemptStarting), 1); + assert_eq!(collector.count(EventKind::AttemptFailed), 1); + assert_eq!(collector.count(EventKind::TaskFinished), 1); assert_eq!(collector.count(EventKind::BackoffScheduled), 0); - assert_eq!(collector.count(EventKind::ActorExhausted), 0); - assert_eq!(collector.count(EventKind::TaskStopped), 0); + assert_eq!(collector.count(EventKind::AttemptSucceeded), 0); assert_eq!( - collector.find(EventKind::TaskFailed).unwrap().exit_code, + collector.find(EventKind::AttemptFailed).unwrap().exit_code, Some(7) ); - let dead = collector.find(EventKind::ActorDead).unwrap(); - assert_eq!(dead.exit_code, Some(7)); - assert!(dead.reason.as_deref().unwrap().contains("unrecoverable")); + let finished = collector.find(EventKind::TaskFinished).unwrap(); + assert_eq!(finished.outcome_kind, Some(TaskOutcomeKind::Fatal)); + assert_eq!(finished.exit_code, Some(7)); + assert!( + finished + .reason + .as_deref() + .unwrap() + .contains("unrecoverable") + ); } #[tokio::test(flavor = "current_thread")] @@ -161,8 +158,15 @@ async fn fatal_no_restart_under_always_interval_none() { .with_restart(RestartPolicy::Always { interval: None }); let collector = run_static(vec![spec]).await; - assert_eq!(collector.count(EventKind::TaskStarting), 1); - assert_eq!(collector.count(EventKind::ActorDead), 1); + assert_eq!(collector.count(EventKind::AttemptStarting), 1); + assert_eq!(collector.count(EventKind::TaskFinished), 1); + assert_eq!( + collector + .find(EventKind::TaskFinished) + .unwrap() + .outcome_kind, + Some(TaskOutcomeKind::Fatal) + ); assert_eq!(collector.count(EventKind::BackoffScheduled), 0); } @@ -177,12 +181,12 @@ async fn max_retries_allows_initial_attempt_plus_configured_retries() { let expected_attempts = usize::try_from(retries + 1).unwrap(); assert_eq!( - collector.count(EventKind::TaskStarting), + collector.count(EventKind::AttemptStarting), expected_attempts, "retry limit {retries}" ); assert_eq!( - collector.count(EventKind::TaskFailed), + collector.count(EventKind::AttemptFailed), expected_attempts, "retry limit {retries}" ); @@ -191,23 +195,21 @@ async fn max_retries_allows_initial_attempt_plus_configured_retries() { usize::try_from(retries).unwrap(), "retry limit {retries}" ); - assert_eq!(collector.count(EventKind::ActorExhausted), 1); - assert_eq!(collector.count(EventKind::ActorDead), 0); + assert_eq!(collector.count(EventKind::TaskFinished), 1); assert!( collector - .find_all(EventKind::TaskFailed) + .find_all(EventKind::AttemptFailed) .iter() .all(|event| event.exit_code == Some(42)), "retry limit {retries}: every failed attempt keeps the exit code" ); - let exhausted = collector.find(EventKind::ActorExhausted).unwrap(); - assert_eq!(exhausted.exit_code, Some(42)); - let reason = exhausted.reason.as_deref().unwrap(); - assert!(reason.contains("max_retries_exceeded"), "got: {reason}"); + let finished = collector.find(EventKind::TaskFinished).unwrap(); + assert_eq!(finished.outcome_kind, Some(TaskOutcomeKind::Failed)); + assert_eq!(finished.exit_code, Some(42)); assert!( - reason.contains(&format!("({retries}/{retries})")), - "got: {reason}" + finished.reason.as_deref().unwrap().contains("boom"), + "diagnostic detail should retain the final task error" ); } } @@ -236,7 +238,7 @@ async fn always_interval_none_restarts_repeatedly_no_backoff_scheduled() { counter.load(Ordering::SeqCst) >= 5 && events .iter() - .filter(|event| event.kind == EventKind::TaskStarting) + .filter(|event| event.kind == EventKind::AttemptStarting) .count() >= 5 }) @@ -244,7 +246,7 @@ async fn always_interval_none_restarts_repeatedly_no_backoff_scheduled() { "immediate-restart loop and its observable start events should reach 5 runs" ); assert_eq!(collector.count(EventKind::BackoffScheduled), 0); - assert!(collector.count(EventKind::TaskStarting) >= 5); + assert!(collector.count(EventKind::AttemptStarting) >= 5); let _ = handle.shutdown().await; }) .await; @@ -290,7 +292,7 @@ async fn always_interval_some_emits_success_source_backoff_between_runs() { assert_eq!(b.delay_ms, Some(5)); assert_eq!(b.reason, None); } - assert_eq!(collector.count(EventKind::TaskFailed), 0); + assert_eq!(collector.count(EventKind::AttemptFailed), 0); let _ = handle.shutdown().await; }) .await; @@ -333,17 +335,14 @@ async fn success_driven_restart_does_not_consume_failure_retry_budget() { .await, "task and its first failure events should settle before assertions" ); - assert_eq!(collector.count(EventKind::TaskFailed), 1); + assert_eq!(collector.count(EventKind::AttemptFailed), 1); let failure_backoffs = collector .find_all(EventKind::BackoffScheduled) .into_iter() .filter(|b| b.backoff_source == Some(BackoffSource::Failure)) .count(); assert_eq!(failure_backoffs, 1); - assert!( - !collector.any_reason_contains(EventKind::ActorExhausted, "max_retries_exceeded"), - "budget reset means it must never exhaust on max-retries" - ); + assert_eq!(collector.count(EventKind::TaskFinished), 0); let _ = handle.shutdown().await; }) .await; @@ -358,20 +357,23 @@ async fn static_run_multiple_oneshots_all_complete_run_returns_ok() { ]; let collector = run_static(specs).await; - assert_eq!(collector.count(EventKind::TaskStarting), 3); - assert_eq!(collector.count(EventKind::TaskStopped), 3); - assert_eq!(collector.count(EventKind::ActorExhausted), 3); + assert_eq!(collector.count(EventKind::AttemptStarting), 3); + assert_eq!(collector.count(EventKind::AttemptSucceeded), 3); + assert_eq!(collector.count(EventKind::TaskFinished), 3); assert_eq!(collector.count(EventKind::TaskRemoved), 3); for label in ["a", "b", "c"] { let evs = collector.by_label(label); assert!( - evs.iter().any(|e| e.kind == EventKind::TaskStarting), - "missing TaskStarting for {label}" + evs.iter().any(|e| e.kind == EventKind::AttemptStarting), + "missing AttemptStarting for {label}" ); assert!( - evs.iter().any(|e| e.kind == EventKind::ActorExhausted), - "missing ActorExhausted for {label}" + evs.iter().any(|e| { + e.kind == EventKind::TaskFinished + && e.outcome_kind == Some(TaskOutcomeKind::Completed) + }), + "missing completed TaskFinished for {label}" ); } } @@ -425,7 +427,10 @@ async fn duplicate_static_batch_starts_no_task_body() { for label in ["unique", "duplicate"] { assert!(collector.by_label(label).iter().all(|event| { - !matches!(event.kind, EventKind::TaskAdded | EventKind::TaskStarting) + !matches!( + event.kind, + EventKind::TaskAdded | EventKind::AttemptStarting + ) })); } @@ -435,18 +440,15 @@ async fn duplicate_static_batch_starts_no_task_body() { .find(|event| event.kind == EventKind::TaskAddFailed) .expect("unique item must receive its batch rejection event"); assert_eq!( - unique_failure.reason.as_deref(), - Some(taskvisor::reasons::BATCH_REJECTED) + unique_failure.rejection_kind, + Some(RejectionKind::BatchRejected) ); - let duplicate_reasons: Vec<_> = collector + assert_eq!(unique_failure.outcome_kind, Some(TaskOutcomeKind::Rejected)); + let duplicate_kinds: Vec<_> = collector .by_label("duplicate") .into_iter() .filter(|event| event.kind == EventKind::TaskAddFailed) - .filter_map(|event| event.reason) + .filter_map(|event| event.rejection_kind) .collect(); - assert!( - duplicate_reasons - .iter() - .any(|reason| reason.as_ref() == taskvisor::reasons::ALREADY_EXISTS) - ); + assert!(duplicate_kinds.contains(&RejectionKind::AlreadyExists)); } diff --git a/tests/ownership.rs b/tests/ownership.rs index 968b0b4..5338332 100644 --- a/tests/ownership.rs +++ b/tests/ownership.rs @@ -243,7 +243,9 @@ async fn last_owner_drop_rejects_queued_controller_work() { assert!(matches!( with_timeout(2, waiter.wait()).await, - Ok(TaskOutcome::Rejected { reason, .. }) - if reason.as_ref() == taskvisor::reasons::CONTROLLER_SHUTTING_DOWN + Ok(TaskOutcome::Rejected { + kind: RejectionKind::ControllerShuttingDown, + .. + }) )); } diff --git a/tests/timeout.rs b/tests/timeout.rs index efb5af1..7113191 100644 --- a/tests/timeout.rs +++ b/tests/timeout.rs @@ -17,14 +17,14 @@ async fn run_to_exhaustion(spec: TaskSpec) -> Arc { .await .expect("run() should return Ok"); collector - .wait_for(EventKind::ActorExhausted, Duration::from_secs(2)) + .wait_for(EventKind::TaskFinished, Duration::from_secs(2)) .await - .expect("ActorExhausted was not observed"); + .expect("TaskFinished was not observed"); collector } #[tokio::test(flavor = "current_thread", start_paused = true)] -async fn per_attempt_timeout_emits_timeout_hit_before_task_failed_then_retries() { +async fn configured_timeout_emits_one_terminal_attempt_event_then_retries() { let task = TaskFn::arc("slow", |_ctx: TaskContext| async move { tokio::time::sleep(Duration::from_secs(3600)).await; Ok(()) @@ -35,33 +35,24 @@ async fn per_attempt_timeout_emits_timeout_hit_before_task_failed_then_retries() .with_max_retries(NonZeroU32::new(1).unwrap()); let collector = run_to_exhaustion(spec).await; - assert_eq!(collector.count(EventKind::TimeoutHit), 2); - assert_eq!(collector.count(EventKind::TaskFailed), 2); + assert_eq!(collector.count(EventKind::AttemptTimedOut), 2); + assert_eq!(collector.count(EventKind::AttemptFailed), 0); assert_eq!(collector.count(EventKind::BackoffScheduled), 1); - assert_eq!(collector.count(EventKind::ActorExhausted), 1); + assert_eq!(collector.count(EventKind::TaskFinished), 1); for attempt in 1..=2u32 { let hit = collector - .find_all(EventKind::TimeoutHit) + .find_all(EventKind::AttemptTimedOut) .into_iter() .find(|e| e.attempt == Some(attempt)) .unwrap(); - let failed = collector - .find_all(EventKind::TaskFailed) - .into_iter() - .find(|e| e.attempt == Some(attempt)) - .unwrap(); - assert!(hit.seq < failed.seq, "TimeoutHit must precede TaskFailed"); - assert!(failed.reason.as_deref().unwrap().contains("timed out")); - assert_eq!(failed.exit_code, None); + assert_eq!(hit.timeout_ms, Some(50)); } - let reason = collector - .find(EventKind::ActorExhausted) - .unwrap() - .reason - .unwrap(); - assert!(reason.contains("max_retries_exceeded") && reason.contains("(1/1)")); + let finished = collector.find(EventKind::TaskFinished).unwrap(); + assert_eq!(finished.outcome_kind, Some(TaskOutcomeKind::Failed)); + let reason = finished.reason.unwrap(); + assert!(reason.contains("timed out")); } #[tokio::test(flavor = "current_thread", start_paused = true)] @@ -83,13 +74,12 @@ async fn timeout_then_success_unlimited_retries_exhausts_on_success() { .with_backoff(fast_backoff()); let collector = run_to_exhaustion(spec).await; - assert_eq!(collector.count(EventKind::TimeoutHit), 1); - assert_eq!(collector.count(EventKind::TaskFailed), 1); + assert_eq!(collector.count(EventKind::AttemptTimedOut), 1); + assert_eq!(collector.count(EventKind::AttemptFailed), 0); assert_eq!(collector.count(EventKind::BackoffScheduled), 1); - assert_eq!(collector.count(EventKind::TaskStarting), 2); - assert_eq!(collector.count(EventKind::TaskStopped), 1); - assert_eq!(collector.count(EventKind::ActorExhausted), 1); - assert_eq!(collector.count(EventKind::ActorDead), 0); + assert_eq!(collector.count(EventKind::AttemptStarting), 2); + assert_eq!(collector.count(EventKind::AttemptSucceeded), 1); + assert_eq!(collector.count(EventKind::TaskFinished), 1); assert_eq!( collector @@ -98,14 +88,9 @@ async fn timeout_then_success_unlimited_retries_exhausts_on_success() { .backoff_source, Some(BackoffSource::Failure) ); - assert_eq!( - collector - .find(EventKind::ActorExhausted) - .unwrap() - .reason - .as_deref(), - Some("policy_exhausted_success") - ); + let finished = collector.find(EventKind::TaskFinished).unwrap(); + assert_eq!(finished.outcome_kind, Some(TaskOutcomeKind::Completed)); + assert_eq!(finished.reason, None); } #[tokio::test(flavor = "current_thread", start_paused = true)] @@ -117,15 +102,28 @@ async fn zero_timeout_means_no_timeout_task_runs_to_completion() { let spec = TaskSpec::once(task).with_timeout(Duration::ZERO); let collector = run_to_exhaustion(spec).await; - assert_eq!(collector.count(EventKind::TimeoutHit), 0); - assert_eq!(collector.count(EventKind::TaskStopped), 1); - assert_eq!(collector.count(EventKind::TaskFailed), 0); + assert_eq!(collector.count(EventKind::AttemptTimedOut), 0); + assert_eq!(collector.count(EventKind::AttemptSucceeded), 1); + assert_eq!(collector.count(EventKind::AttemptFailed), 0); + let finished = collector.find(EventKind::TaskFinished).unwrap(); + assert_eq!(finished.outcome_kind, Some(TaskOutcomeKind::Completed)); + assert_eq!(finished.reason, None); +} + +#[tokio::test(flavor = "current_thread")] +async fn task_returned_timeout_is_an_attempt_failure_not_a_configured_deadline() { + let task = TaskFn::arc("reported-timeout", |_ctx: TaskContext| async move { + Err(TaskError::timeout(Duration::from_secs(7))) + }); + let collector = run_to_exhaustion(TaskSpec::once(task)).await; + + assert_eq!(collector.count(EventKind::AttemptTimedOut), 0); + assert_eq!(collector.count(EventKind::AttemptFailed), 1); assert_eq!( collector - .find(EventKind::ActorExhausted) + .find(EventKind::TaskFinished) .unwrap() - .reason - .as_deref(), - Some("policy_exhausted_success") + .outcome_kind, + Some(TaskOutcomeKind::Failed) ); } diff --git a/tests/watch.rs b/tests/watch.rs index 2d39e69..eeb1614 100644 --- a/tests/watch.rs +++ b/tests/watch.rs @@ -33,26 +33,27 @@ async fn outcome_reason_is_byte_identical_to_the_event_reason() { .wait_until(Duration::from_secs(2), |events| { events .iter() - .any(|event| event.id == Some(id) && event.kind == EventKind::ActorExhausted) + .any(|event| event.id == Some(id) && event.kind == EventKind::TaskFinished) }) .await ); let event = collector .by_id(id) .into_iter() - .find(|e| e.kind == EventKind::ActorExhausted) - .expect("ActorExhausted event for the run"); + .find(|e| e.kind == EventKind::TaskFinished) + .expect("TaskFinished event for the run"); + assert_eq!(event.outcome_kind, Some(TaskOutcomeKind::Failed)); match outcome { TaskOutcome::Failed { reason, exit_code, .. } => { - assert!(reason.contains("max_retries_exceeded")); + assert!(reason.contains("boom")); assert_eq!(exit_code, Some(9)); assert_eq!( &*reason, event.reason.as_deref().expect("event carries a reason"), - "TaskOutcome reason must be byte-identical to the ActorExhausted reason" + "TaskOutcome reason must be byte-identical to the TaskFinished reason" ); assert_eq!(exit_code, event.exit_code, "exit_code must match too"); }