diff --git a/CHANGELOG.md b/CHANGELOG.md index 908a7f8..408bc02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,41 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +## [0.3.0] - 2026-08-09 + +Adds jitter. Purely additive apart from one collision noted under Upgrading. + +### Added +- `Backoff::jittered()` and `Backoff::jittered_with_seed(seed)`. Wraps any strategy, including one + you wrote, so each delay becomes a uniform random value in `0 ..= delay` ("full jitter"). Opt-in: + the default schedule stays deterministic. The seeded form is for tests. +- `DecorrelatedBackoff`, from a validated `DecorrelatedBackoffConfig`. Draws each delay from + `base ..= prev * 3`, capped at `max_delay`, and never below `base`. Seedable with `with_seed`. +- `Clock` for `&C` and `Arc`, on both the async and blocking traits, so a mock clock can be + passed as `.clock(&mock)` and still be read afterwards. +- `ExponentialBackoffConfig { factor: 1, .. }` documented as the way to get a constant delay; there + is no separate `ConstantBackoff` type. + +### Changed +- New required dependency: `fastrand` (no transitive dependencies, not feature-gated). + +### Upgrading + +0.2.0 code compiles unchanged, with one exception. If you wrote `impl Clock for &YourClock` +yourself, it now collides with the impl this release adds and the build fails with +`E0119: conflicting implementations`. Delete yours; this release makes it redundant. +`cargo-semver-checks` does not flag added impls, so it would not have warned you. +[ADR005](https://github.com/azeemshaik025/mettle/blob/main/docs/adr/ADR005.md) explains why the +impls are there and why we shipped them anyway. + +`Jittered` and `DecorrelatedBackoff` are deliberately not `Clone`, because a copy carries the RNG +state and replays the same delays. Keep the config and build a fresh strategy from it. + +Why the jitter side is shaped the way it is, including why there is no *mode* to choose: +[ADR004](https://github.com/azeemshaik025/mettle/blob/main/docs/adr/ADR004.md). + ## [0.2.0] - 2026-07-26 Supersedes the yanked 0.1.1: that release removed public items in a patch, which was a breaking diff --git a/Cargo.lock b/Cargo.lock index 13bef64..81a7c27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,10 +2,17 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "mettle" -version = "0.2.0" +version = "0.3.0" dependencies = [ + "fastrand", "pin-project-lite", "tokio", "tracing", diff --git a/Cargo.toml b/Cargo.toml index 5248d36..5e74194 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "mettle" -version = "0.2.0" +version = "0.3.0" edition = "2024" rust-version = "1.85" -description = "A resilience toolkit for Rust: async and blocking retries with configurable backoff." +description = "A resilience toolkit for Rust: async and blocking retries with configurable backoff and jitter." readme = "README.md" license = "MIT OR Apache-2.0" keywords = ["retry", "backoff", "resilience", "async", "tokio"] @@ -20,6 +20,7 @@ rustdoc-args = ["--cfg", "docsrs"] tokio = { version = "1", features = ["time"], optional = true } pin-project-lite = { version = "0.2", optional = true } tracing = { version = "0.1", default-features = false, features = ["std"] } +fastrand = "2" [features] default = ["async", "blocking"] diff --git a/README.md b/README.md index abff4b1..77fb583 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,27 @@ let body = retry(|| async { fetch(&url).await }) No async runtime? The blocking twin is identical but ends in `.call()` instead of `.await`. +Retrying on a fixed schedule means every client that failed together retries together, so a service +that is coming back up gets a synchronized wave. Jitter spreads them out: + +```rust +use mettle::{Backoff, DecorrelatedBackoff, DecorrelatedBackoffConfig, ExponentialBackoff}; + +// Randomize any strategy's delays into 0 ..= delay ("full jitter")... +let backoff = ExponentialBackoff::default().jittered(); + +// ...or use decorrelated jitter, where each delay is drawn from the previous one. +let backoff = DecorrelatedBackoff::new(DecorrelatedBackoffConfig::default())?; +``` + +Which one: `.jittered()` works on any strategy, including one you wrote, and spreads delays as +widely as possible. `DecorrelatedBackoff` is its own strategy and never draws below its `base`, so +reach for it when you want a floor under every wait. The trade is that a floor also means never +retrying sooner than `base`, so a dependency that frees up early isn't picked up until then. + +Both seed from entropy by default and take a fixed seed (`with_seed`) when you want a test to +replay the same delays. + ## Tools Each tool comes with a runnable example. Start there: diff --git a/docs/adr/ADR004.md b/docs/adr/ADR004.md new file mode 100644 index 0000000..36dd0bc --- /dev/null +++ b/docs/adr/ADR004.md @@ -0,0 +1,110 @@ +# ADR004: Jitter, a fourth dependency, and why randomized strategies aren't `Clone` + +**Status:** Accepted + +## Context + +Retrying with a fixed backoff means every client that failed at the same moment retries at the +same moment. The dependency that just fell over gets a synchronized wave of traffic the instant +it comes back, which is the thundering herd. Jitter is the fix, and it's the first thing in +mettle that needs randomness, so it forces three decisions we hadn't had to make yet: what to +depend on, how many knobs to expose, and what `Clone` means once a type carries RNG state. + +Shipping it also exposed two gaps in how testable the crate actually was. Those are ADR005. + +## Decisions + +**1. `fastrand` is a required dependency, not a feature.** +Jitter needs a random number generator. `fastrand` gives us three things (`Rng::new()`, +`Rng::with_seed(u64)`, and `rng.u64(range)`) and nothing else comes with it. + +ADR003 moved Tokio behind a feature so sync-only users didn't pay for a runtime they never +touched, so the obvious move was to do the same here: `fastrand` optional, behind a `jitter` +feature. We measured instead of guessing. `fastrand` is 1198 lines with zero transitive +dependencies, and rebuilding it costs about the same as rebuilding the whole `tracing` tree +(`tracing` plus `tracing-core` plus `once_cell`), which ADR003 already accepts unconditionally. +Gating it would have put `#[cfg]` on roughly a dozen public items including a trait method, +doubled the feature matrix CI has to cover, and given anyone on a minimal build a missing-type +error instead of a working `Jitter`. That's a lot of friction to save less than what a dependency +we already took costs. + +The weight class is what matters, not the count. Tokio is a runtime; `fastrand` is a PRNG. We'd +make the same call again for something this size and revisit it for anything larger. + +We also considered inlining a small PRNG to keep the dependency count at two. We didn't, because +seeding from entropy portably is the part that's actually easy to get wrong, and getting it wrong +means a fleet seeded identically, which is the exact failure jitter exists to prevent. + +**1b. The RNG is a concrete `fastrand::Rng`, not a trait.** +`Clock` is injected through a trait, so it's fair to ask why randomness isn't. The two look +symmetrical and aren't. + +You cannot seed the system clock. The only way a test controls time is to replace the source +outright, which is why `Clock` has to be a trait. Randomness has a cheaper answer: seeding *is* the +determinism mechanism, and `with_seed` already gives complete reproducibility without replacing +anything. A trait would buy the ability to swap the *algorithm*, which is a different capability and +one nobody has asked for. Jitter does not need to be unpredictable to an adversary. + +The coupling is also small enough that the abstraction isn't needed to keep our options open. We use +exactly three things (`Rng::new`, `Rng::with_seed`, `rng.u64(range)`), `fastrand` appears in no +public signature, and swapping it would be a change inside one file. Against that, a trait means a +second type parameter on `Jittered` and `DecorrelatedBackoff` forever. + +What we do owe users is honesty about what a seed guarantees: the same delays within a build, not +across `fastrand` versions. Every seeded entry point says so, and this crate's own tests assert +properties rather than golden values for that reason. + +**2. One kind of jitter, with no mode to choose.** +`Jittered` applies full jitter: every delay becomes a uniform random value in `0 ..= delay`. +There is no `Jitter` enum and no argument to `jittered()`. + +We shipped `Full` and `Equal` first, mirroring the AWS post that names them. Then we read the post's +own conclusion, which is that equal jitter "does slightly more work than Full Jitter, and takes much +longer", and simulated both against mettle's actual implementations under a contended resource. +Equal lost to full on both total calls and completion time in every configuration we tried, at +several client counts and base delays. AWS's own SDK agrees: full jitter is the default for its +STANDARD and ADAPTIVE retry modes, and equal survives only on the LEGACY throttling path. + +An option nobody should pick is not an option, it's a trap, and the name "equal" reads as the safe +middle choice, which is exactly backwards. So it's gone, and with one mode left the enum was pure +ceremony. + +The cost is the extension point. `Jitter` was `#[non_exhaustive]`, so adding a fourth mode later +would have been free; now it would need a separate `jittered_with(mode)` method. We accept that: +the literature defines three modes, we ship the one that measures best, and decorrelated cannot be +a mode at all (see below). + +**2b. Decorrelated jitter is its own `Backoff`, not a mode.** +A mode is a per-delay function: hand it a delay, get a randomized delay. Decorrelated doesn't fit +that shape. Its next delay is drawn from a range set by the delay it actually drew last time +(`next = min(cap, rand(base, prev * 3))`), so the randomness is inside the recurrence and there's +no deterministic sequence underneath to wrap. It ships as `DecorrelatedBackoff`. + +It is also the answer for anyone who wants a floor, which is what `Equal` was reaching for and +getting wrong: decorrelated never draws below `base`. The trade is real and documented, since a +floor also means never retrying sooner than `base`. + +**3. Randomized strategies are not `Clone`.** +`ExponentialBackoff` is `Clone` and cloning it is the right thing to do: you get the same +deterministic sequence, which is what you asked for. Cloning `Jittered` or `DecorrelatedBackoff` +copies the RNG state, so every copy replays identical delays. A user who builds one and clones it +per request puts every concurrent request on that host into lockstep, and nothing warns them. +That's the herd again, one host at a time. + +We considered a manual `Clone` that reseeds, and rejected it: a `Clone` that doesn't clone breaks +the trait's contract and would silently break seeded tests. So the randomized types just aren't +`Clone`. Keep the config, which is `Clone`, and build a fresh strategy from it. + +This direction is also the reversible one. Adding `Clone` later is a minor version; taking it away +is a breaking change. + +## Consequences + +Jitter is available to everyone with no feature flag to discover, at the cost of one small +dependency. The `Backoff` trait now has strategies that behave differently under `Clone`, which +the trait docs have to explain rather than leaving to be discovered. Anyone reusing a randomized +policy across calls has to hold the config rather than the strategy, which is one extra line and +the only shape that's actually correct. + +Every randomized path is seedable, so none of this costs us deterministic tests. Making that +seeding pleasant to reach is ADR005. diff --git a/docs/adr/ADR005.md b/docs/adr/ADR005.md new file mode 100644 index 0000000..6243b91 --- /dev/null +++ b/docs/adr/ADR005.md @@ -0,0 +1,71 @@ +# ADR005: Making the tested path as usable as the production path + +**Status:** Accepted + +## Context + +mettle's headline claim is that you can test a resilience policy exactly, with no sleeping and no +real clock. Shipping jitter put that claim under load for the first time, because a randomized +policy is only testable if you can pin the randomness *and* still read back what happened. + +Two things turned out to be wrong, and neither was a bug in the sense a test would catch. Both were +found the same way: by writing the API from outside the crate, in a separate crate with a path +dependency, doing what someone who had only read the README would do. That took about ten minutes +and found two problems that had been there since 0.1.0. + +## Decisions + +**1. The readable path and the deterministic path are the same shape.** +`jittered()` seeds from entropy, which is what production wants. `jittered_with_seed(seed)` fixes +the seed, which is what a test wants. + +The seeded constructor already existed as `Jittered::with_seed(inner, seed)`, so the combinator +looks redundant. It isn't. Without it, turning on jitter meant a production line reading +`ExponentialBackoff::default().jittered()` and a test line reading +`Jittered::with_seed(ExponentialBackoff::default(), 42)`. Those are different constructions, so a +test no longer visibly exercised the thing it was testing. For a crate that sells deterministic +testing, the deterministic path being the awkward one is the wrong way round. + +Seeding in production is a mistake, since a fleet seeded identically retries in lockstep, so the +method's docs say so outright rather than leaving it implied. + +**2. `Clock` and `Now` are implemented for `&C` and `Arc`.** +Both traits are taken by value. `retry(..).clock(c)` owns its clock, and a `CircuitBreaker` owns its +time source for its whole life. Without a reference impl a test can hand over its mock and then +never read it back, which defeats the entire point of injecting time. Passing `&mock` was a compile +error; the only way through was to give the mock interior sharing so it could be cloned. + +`&C` covers the case where the retry doesn't outlive the clock. `Arc` covers the case where it +does, which is the usual shape for a breaker and for anything spawned. + +**3. Before a release, use the crate from outside it.** +This is the decision that generalises, so it is written down as one. + +We missed the reference-impl gap for two releases because our own tests worked around it on day +one. Every mock in this repo wraps its state in `Arc>` and gets passed by `.clone()`, even +in single-threaded blocking tests where nothing is shared. Written once, copied everywhere, and from +the inside it looked like "how you write a mock clock here" rather than like a tax. Nobody notices a +papercut they have already bandaged. + +A test suite that lives inside the crate inherits its own solutions. It can be thorough and +mutation-checked and still tell you nothing about whether the front door opens. So the release +checklist includes building a throwaway crate against a path dependency and writing the code a +first-time reader would write. + +## Consequences + +A mock clock is now an ordinary struct with a `Cell` or a `RefCell`, not something that needs +interior sharing before it can be passed at all. `examples/breaker.rs` lost its `Rc>` for +exactly this reason, which is the clearest evidence the friction was real: our own example was +working around it. + +Decision 2 is the one part of 0.3.0 that can break an existing build. Someone who hit the same +papercut and wrote `impl Clock for &TheirClock` themselves now collides with ours, and the build +fails with `E0119: conflicting implementations`. We shipped it anyway: the affected population is +small, the failure is a compile error rather than silent misbehaviour, and the fix is deleting a +line this release makes redundant. `cargo-semver-checks` does not flag added impls, so nothing in CI +would have caught it and the CHANGELOG says it out loud instead. + +Decision 3 costs a few minutes per release and has no automation behind it. The cheap way to make it +structural later is a `tests/` directory, since integration tests compile against the crate's public +face and would have caught this without anyone thinking to look. diff --git a/docs/adr/README.md b/docs/adr/README.md index 610d556..9811e6b 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -9,3 +9,5 @@ changes, edit its file and say what changed and why. | [001](ADR001.md) | Sans-IO core, hand-written async, one crate | Accepted | | [002](ADR002.md) | Public API, open traits, one validated `Backoff` | Accepted | | [003](ADR003.md) | Dependencies and Cargo features | Accepted | +| [004](ADR004.md) | Jitter, a fourth dependency, and `Clone` | Accepted | +| [005](ADR005.md) | Making the tested path as usable as the production path | Accepted | diff --git a/examples/retry.rs b/examples/retry.rs index c99e7e0..21a6adc 100644 --- a/examples/retry.rs +++ b/examples/retry.rs @@ -11,9 +11,15 @@ //! - `.max_elapsed(d)` gives up once the next wait would push total time past `d` //! - `.clock(clock)` supplies the time source (default: Tokio) //! +//! The default schedule is deterministic, which means a fleet of clients that failed together +//! retries together. `.jittered()` spreads them out; see step 4. +//! //! Run with: `cargo run --example retry` -use mettle::{ExponentialBackoff, ExponentialBackoffConfig, retry}; +use mettle::{ + Backoff, DecorrelatedBackoff, DecorrelatedBackoffConfig, ExponentialBackoff, + ExponentialBackoffConfig, retry, +}; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; use std::time::Duration; @@ -24,7 +30,7 @@ enum FetchError { } #[tokio::main] -async fn main() { +async fn main() -> Result<(), mettle::BackoffConfigError> { // 1) Simplest form: pass the operation, take every default. let result = retry(flaky_fetch).await; println!("1. defaults: {result:?}"); // Ok("user data"), succeeds on attempt 3 @@ -42,6 +48,27 @@ async fn main() { .when(|e| matches!(e, FetchError::Timeout)) .await; println!("3. full: {result:?}"); // Ok("user data"), succeeds on attempt 3 + + // 4) Jitter, so a fleet doesn't retry in lockstep. `.jittered()` randomizes each delay into + // `0 ..= delay`; `DecorrelatedBackoff` instead draws each delay from the previous one and + // never goes below `base`. Both are opt-in: the default above stays deterministic. + let result = retry(flaky_fetch) + .backoff(fast_backoff().jittered()) + .when(|e| matches!(e, FetchError::Timeout)) + .await; + println!("4. jittered: {result:?}"); // same outcome, unpredictable delays + + let result = retry(flaky_fetch) + .backoff(DecorrelatedBackoff::new(DecorrelatedBackoffConfig { + base: Duration::from_millis(20), + max_retries: 5, + max_delay: Duration::from_secs(1), + })?) + .when(|e| matches!(e, FetchError::Timeout)) + .await; + println!("5. decorrel.: {result:?}"); // every delay at least 20ms + + Ok(()) } /// A flaky call: fails with `Timeout` twice, then succeeds. It resets after each success, so it diff --git a/src/backoff.rs b/src/backoff.rs index 81018dd..d0f970e 100644 --- a/src/backoff.rs +++ b/src/backoff.rs @@ -1,31 +1,89 @@ //! Backoff strategies: whether to retry, and how long to wait before each attempt. //! //! A [`Backoff`] is a stateful sequence of delays. `retry` takes it by value and drives that -//! owned instance; reuse one policy across calls by passing a fresh (or cloned) value. +//! owned instance; reuse one policy across calls by passing a fresh value. use std::num::NonZeroU32; use std::time::Duration; -// Defaults for `ExponentialBackoffConfig::default()`. +// Defaults shared by both config `Default` impls; `DEFAULT_FACTOR` is exponential-only. const DEFAULT_FACTOR: u32 = 2; const DEFAULT_BASE: Duration = Duration::from_millis(100); const DEFAULT_MAX_RETRIES: u32 = 3; const DEFAULT_MAX_DELAY: Duration = Duration::from_secs(30); +/// The multiplier in decorrelated jitter's `rand(base, prev * 3)`. Part of the published +/// algorithm, not a tuning knob. +const DECORRELATED_MULTIPLIER: u32 = 3; + /// A stateful sequence of retry delays. Pure: no I/O, no sleeping, no clock reads. /// /// A backoff is consumed as it runs, since each [`next_delay`](Backoff::next_delay) advances it, /// so `retry` takes it **by value** and drives that owned instance. A freshly constructed value -/// must represent an un-started sequence; to reuse one policy across calls, pass a fresh (or -/// cloned) value each time. +/// must represent an un-started sequence, so reuse one policy across calls by passing a fresh +/// value each time. A deterministic strategy such as [`ExponentialBackoff`] can be cloned +/// instead; the randomized ones are deliberately not [`Clone`], since a copy would replay the +/// same delays. /// /// Open on purpose: add a custom strategy by implementing it. pub trait Backoff { /// Delay before the next retry, or `None` to give up (e.g. retries exhausted). fn next_delay(&mut self) -> Option; + + /// Wrap this strategy so every delay becomes a uniform random value in `0 ..= delay`, seeding + /// the RNG from entropy. + /// + /// This is "full jitter" from AWS's *Exponential Backoff and Jitter*. Composes with any + /// strategy, including one you wrote, which is the point: reach for it to stop a fleet of + /// clients retrying in lockstep. + /// + /// ``` + /// use mettle::{Backoff, ExponentialBackoff}; + /// + /// let mut backoff = ExponentialBackoff::default().jittered(); + /// let delay = backoff.next_delay(); // somewhere in 0 ..= 100ms + /// # let _ = delay; + /// ``` + /// + /// Because the floor is zero, a retry can fire almost immediately. That is the mechanism, not + /// a flaw: it is what lets a freed-up dependency be picked up at once. If you need a floor + /// under every wait, use [`DecorrelatedBackoff`] instead, which never goes below its `base`. + fn jittered(self) -> Jittered + where + Self: Sized, + { + Jittered::new(self) + } + + /// As [`jittered`](Backoff::jittered), but with a fixed `seed`, so the delays repeat exactly. + /// + /// For tests. Turning on jitter otherwise costs you the ability to assert what your retry + /// waited, which is the one thing this crate is built to let you do: + /// + /// ``` + /// use mettle::{Backoff, ExponentialBackoff}; + /// + /// let delays = |seed| { + /// let mut b = ExponentialBackoff::default().jittered_with_seed(seed); + /// std::iter::from_fn(move || b.next_delay()).collect::>() + /// }; + /// assert_eq!(delays(42), delays(42)); + /// ``` + /// + /// Don't reach for this in production. A fleet that all seeds the same way retries in + /// lockstep, which is the thundering herd jitter exists to prevent. + /// + /// A seed replays the same delays within a build. The exact values are not part of this + /// crate's API contract, so assert on properties rather than on specific numbers. + fn jittered_with_seed(self, seed: u64) -> Jittered + where + Self: Sized, + { + Jittered::with_seed(self, seed) + } } -/// Why an [`ExponentialBackoff`] configuration was rejected. +/// Why a backoff configuration was rejected. Not every strategy can produce every variant. #[derive(Debug, Clone, PartialEq, Eq)] #[non_exhaustive] pub enum BackoffConfigError { @@ -33,7 +91,7 @@ pub enum BackoffConfigError { ZeroBase, /// `factor` was zero, so delays would collapse to zero (a busy-loop). ZeroFactor, - /// `max_delay` was smaller than `base`, which would cap the first delay below `base`. + /// `max_delay` was smaller than `base`, which would cap delays below `base`. MaxDelayBelowBase, } @@ -57,7 +115,7 @@ impl std::error::Error for BackoffConfigError {} /// ``` #[derive(Debug, Clone)] pub struct ExponentialBackoffConfig { - /// Growth multiplier applied each retry (must be >= 1). + /// Growth multiplier applied each retry (must be >= 1). Set it to `1` for a constant delay. pub factor: u32, /// Delay before the first retry (must be non-zero). pub base: Duration, @@ -89,7 +147,8 @@ impl Default for ExponentialBackoffConfig { /// `retry` consumes it by value, so pass a fresh (or cloned) one to run the same policy again. /// /// Delays are deterministic: no jitter is applied, so a given config always yields the same -/// sequence. Jitter is planned; until then, wrap this in a custom [`Backoff`] if you need it. +/// sequence. Add randomness by wrapping it with [`Backoff::jittered`], for example +/// `ExponentialBackoff::default().jittered()`. #[derive(Debug, Clone)] pub struct ExponentialBackoff { factor: NonZeroU32, @@ -154,6 +213,206 @@ impl Backoff for ExponentialBackoff { } } +/// A uniform random `Duration` in `lo ..= hi`, or `lo` if `hi <= lo`. Computed in `u64` +/// nanoseconds; real delays sit well below that bound, and larger inputs saturate to it. +fn rand_duration(rng: &mut fastrand::Rng, lo: Duration, hi: Duration) -> Duration { + let lo = lo.as_nanos().min(u64::MAX as u128) as u64; + let hi = hi.as_nanos().min(u64::MAX as u128) as u64; + if hi <= lo { + return Duration::from_nanos(lo); + } + Duration::from_nanos(lo + rng.u64(0..=(hi - lo))) +} + +/// Any [`Backoff`] wrapped so each delay becomes a uniform random value in `0 ..= delay`. +/// +/// This is "full jitter": the widest spread, so the density of clients attempting at any instant +/// is as low as it can be. There is no mode to choose. AWS's own measurements had the alternative +/// ("equal jitter", a floor at `delay/2`) doing more work *and* finishing later, so shipping it as +/// an option would only invite people to pick the worse one. +/// +/// Want a floor under every wait? That's [`DecorrelatedBackoff`], which never draws below its +/// `base`. Note the trade: a floor means never retrying sooner than `base`, so a dependency that +/// frees up early isn't picked up until then. +/// +/// The inner strategy stays deterministic; only this layer is random. Its RNG is seedable with +/// [`with_seed`](Jittered::with_seed) so jittered retries stay reproducible in tests. Usually +/// built with [`Backoff::jittered`] rather than named directly. +/// +/// Not [`Clone`] on purpose: a copy would carry the RNG state and replay the same delays, which +/// is the lockstep jitter exists to prevent. Wrap a fresh inner strategy instead. +#[derive(Debug)] +pub struct Jittered { + inner: B, + rng: fastrand::Rng, +} + +impl Jittered { + /// Wrap `inner`, seeding the RNG from entropy. + pub fn new(inner: B) -> Self { + Self { + inner, + rng: fastrand::Rng::new(), + } + } + + /// Wrap `inner` with a fixed `seed`, for reproducible tests. + /// + /// A seed replays the same delays within a build. The exact values are not part of this + /// crate's API contract, so assert on properties rather than on specific numbers. + pub fn with_seed(inner: B, seed: u64) -> Self { + Self { + inner, + rng: fastrand::Rng::with_seed(seed), + } + } +} + +impl Backoff for Jittered { + fn next_delay(&mut self) -> Option { + let delay = self.inner.next_delay()?; + Some(rand_duration(&mut self.rng, Duration::ZERO, delay)) + } +} + +/// Parameters for [`DecorrelatedBackoff::new`]. Fill only what differs from [`Default`]: +/// +/// ``` +/// # use mettle::DecorrelatedBackoffConfig; +/// let _ = DecorrelatedBackoffConfig { max_retries: 8, ..Default::default() }; +/// ``` +#[derive(Debug, Clone)] +pub struct DecorrelatedBackoffConfig { + /// Lower bound on every delay, and where the first draw starts from (must be non-zero). + pub base: Duration, + /// Number of retries; `0` means one attempt, no retries + /// (total attempts = `max_retries + 1`). + pub max_retries: u32, + /// Upper bound on any single delay (must be >= `base`). + pub max_delay: Duration, +} + +impl Default for DecorrelatedBackoffConfig { + /// Sensible defaults: 100 ms base, 3 retries, 30 s cap. + fn default() -> Self { + Self { + base: DEFAULT_BASE, + max_retries: DEFAULT_MAX_RETRIES, + max_delay: DEFAULT_MAX_DELAY, + } + } +} + +/// Decorrelated jitter: each delay is drawn uniformly from `base ..= prev * 3`, capped at +/// `max_delay`, for at most `max_retries` retries. The formula is the one from AWS's +/// "Exponential Backoff and Jitter". +/// +/// ``` +/// use mettle::{Backoff, DecorrelatedBackoff, DecorrelatedBackoffConfig}; +/// +/// let mut backoff = DecorrelatedBackoff::new(DecorrelatedBackoffConfig::default())?; +/// let delay = backoff.next_delay(); // somewhere in 100ms ..= 300ms +/// # let _ = delay; +/// # Ok::<_, mettle::BackoffConfigError>(()) +/// ``` +/// +/// This is a strategy of its own rather than something [`Jittered`] could produce, because the +/// randomness lives in the recurrence: each range is set by the delay that was actually drawn last +/// time, so there is no deterministic sequence underneath to wrap. +/// +/// One thing differs from [`ExponentialBackoff`]: the first delay is already random, somewhere in +/// `base ..= base * 3`, rather than exactly `base`. Against a jittered exponential, the difference +/// is the floor. Every delay here is at least `base`, where [`Backoff::jittered`] can return anything +/// down to zero. Don't stack the two by calling [`jittered`](Backoff::jittered) on this: the +/// randomness is already in the recurrence, and wrapping it throws the `base` floor away. +/// +/// Build it with [`new`](DecorrelatedBackoff::new), or [`with_seed`](DecorrelatedBackoff::with_seed) +/// to make the delays reproducible in tests. Not [`Clone`] on purpose: a copy would carry the RNG +/// state and replay the same delays, which is the lockstep this strategy exists to prevent. Keep +/// the [`DecorrelatedBackoffConfig`] around and build a fresh one per call instead. +#[derive(Debug)] +pub struct DecorrelatedBackoff { + base: Duration, + max_delay: Duration, + prev: Duration, // the last delay handed out; starts at `base` + retries_left: u32, + rng: fastrand::Rng, +} + +impl DecorrelatedBackoff { + /// Validate a [`DecorrelatedBackoffConfig`] into a ready-to-run backoff, seeding the RNG from + /// entropy. + /// + /// # Errors + /// Returns [`BackoffConfigError`] if the config would produce a degenerate + /// (e.g. zero-delay) sequence. + pub fn new(config: DecorrelatedBackoffConfig) -> Result { + Self::build(config, fastrand::Rng::new()) + } + + /// As [`new`](DecorrelatedBackoff::new), but with a fixed `seed`, for reproducible tests. A + /// seed replays the same delays within a build; the exact values are not part of this crate's + /// API contract. + /// + /// # Errors + /// Returns [`BackoffConfigError`] if the config would produce a degenerate + /// (e.g. zero-delay) sequence. + pub fn with_seed( + config: DecorrelatedBackoffConfig, + seed: u64, + ) -> Result { + Self::build(config, fastrand::Rng::with_seed(seed)) + } + + fn build( + config: DecorrelatedBackoffConfig, + rng: fastrand::Rng, + ) -> Result { + let DecorrelatedBackoffConfig { + base, + max_retries, + max_delay, + } = config; + if base.is_zero() { + // Zero is absorbing here: prev = 0 makes every later range [0, 0] too. + return Err(BackoffConfigError::ZeroBase); + } + if max_delay < base { + return Err(BackoffConfigError::MaxDelayBelowBase); + } + Ok(Self { + base, + max_delay, + prev: base, + retries_left: max_retries, + rng, + }) + } +} + +impl Backoff for DecorrelatedBackoff { + fn next_delay(&mut self) -> Option { + if self.retries_left == 0 { + return None; // retries exhausted — give up + } + self.retries_left -= 1; + + // Draw from [base, prev * 3], then cap, which is the formula as published. + // `saturating_mul` so an enormous `prev` pins the top of the range at `Duration::MAX` + // instead of panicking. + let hi = self.prev.saturating_mul(DECORRELATED_MULTIPLIER); + let delay = rand_duration(&mut self.rng, self.base, hi) + .min(self.max_delay) + // `.min` alone can't drop below `base`, since `max_delay >= base` is validated. The + // `.max` is for a `base` past the u64-nanosecond range `rand_duration` works in, + // where the draw itself saturates low. Absurd as an input, but it keeps the + // "every delay is at least `base`" guarantee total rather than almost-total. + .max(self.base); + self.prev = delay; // the capped value seeds the next draw + Some(delay) + } +} + #[cfg(test)] mod tests { use super::*; @@ -183,6 +442,13 @@ mod tests { assert_eq!(b.next_delay(), None); // 5 retries used up } + #[test] + fn factor_of_one_holds_the_delay_constant() { + // Documented on `ExponentialBackoffConfig::factor`, and the reason there's no separate + // `ConstantBackoff` type. + assert_eq!(drain(exp(7, 1, 100, 4)), vec![secs(7); 4]); + } + #[test] fn delay_is_capped_at_max() { let mut b = exp(10, 10, 30, 4); @@ -206,8 +472,10 @@ mod tests { max_delay: Duration::MAX, }) .unwrap(); - let _ = b.next_delay(); // saturating_mul must not overflow-panic - let _ = b.next_delay(); + // `saturating_mul` must not overflow-panic, and must saturate rather than wrap to zero: + // a zero delay here would be the busy-loop the config validation exists to prevent. + assert!(b.next_delay().unwrap() > Duration::ZERO); + assert!(b.next_delay().unwrap() > Duration::ZERO); } #[test] @@ -236,4 +504,274 @@ mod tests { Err(BackoffConfigError::MaxDelayBelowBase) )); } + + // --- jitter --- + + fn exp6() -> ExponentialBackoff { + exp(1, 2, 100, 6) // underlying: 1, 2, 4, 8, 16, 32 (seconds) + } + + fn drain(mut b: impl Backoff) -> Vec { + std::iter::from_fn(move || b.next_delay()).collect() + } + + #[test] + fn full_jitter_stays_within_bounds() { + // Full jitter: every delay lies in [0, the underlying delay], and the sequence still ends + // exactly when the inner strategy is exhausted. + let plain = drain(exp6()); + let mut j = Jittered::with_seed(exp6(), 42); + for p in &plain { + let d = j.next_delay().unwrap(); + assert!( + d <= *p, + "full jitter exceeded the base delay: {d:?} > {p:?}" + ); + } + assert_eq!(j.next_delay(), None); + } + + #[test] + fn seed_makes_jitter_reproducible() { + // Same seed yields an identical sequence (so jittered retries stay testable); different + // seeds generally differ, which guards against a constant or broken RNG. + let seq = |seed| drain(Jittered::with_seed(exp6(), seed)); + assert_eq!(seq(7), seq(7)); + assert_ne!(seq(1), seq(2)); + } + + #[test] + fn jitter_actually_moves_the_delay() { + // `d <= p` alone is satisfied by the identity function, so pin that jitter really + // randomizes: across the sequence it must land both below and above the halfway mark. + let jittered = drain(Jittered::with_seed(exp6(), 5)); + let plain = drain(exp6()); + assert!( + jittered.iter().zip(&plain).any(|(d, p)| *d < *p / 2), + "jitter never dropped below half the plain delay" + ); + assert!( + jittered.iter().zip(&plain).any(|(d, p)| *d > *p / 2), + "jitter never rose above half the plain delay" + ); + } + + #[test] + fn jittered_combinator_wraps_the_inner_strategy() { + // `Backoff::jittered` is the documented entry point, so drive it rather than only the + // `Jittered::` constructors. It must bound each delay by the inner strategy's own value + // and end exactly when the inner one does. + let mut b = exp6().jittered(); + let plain = drain(exp6()); + for p in &plain { + let d = b.next_delay().unwrap(); + assert!( + d <= *p, + "combinator exceeded the inner delay: {d:?} for {p:?}" + ); + } + assert_eq!(b.next_delay(), None); + } + + #[test] + fn jittered_with_seed_matches_the_named_constructor() { + // The combinator is the ergonomic path; it must not be a second-class one. Seeding + // through it has to give exactly what naming the type gives. + assert_eq!( + drain(exp6().jittered_with_seed(42)), + drain(Jittered::with_seed(exp6(), 42)) + ); + // And it must actually honour the seed rather than quietly reading entropy. + assert_eq!( + drain(exp6().jittered_with_seed(7)), + drain(exp6().jittered_with_seed(7)) + ); + assert_ne!( + drain(exp6().jittered_with_seed(1)), + drain(exp6().jittered_with_seed(2)) + ); + } + + #[test] + fn unseeded_jitter_differs_between_instances() { + // The entropy-seeded path is what stops a fleet retrying in lockstep, and it's the reason + // these types aren't `Clone`. A regression to a fixed seed would pass every other test + // here. Two independent RNGs colliding across six delays is a 2^-64 event. + assert_ne!(drain(exp6().jittered()), drain(exp6().jittered())); + } + + #[test] + fn jitter_handles_zero_and_extreme_delays() { + // A custom strategy can hand back zero or enormous delays; jitter must not panic. + struct Fixed(std::vec::IntoIter); + impl Backoff for Fixed { + fn next_delay(&mut self) -> Option { + self.0.next() + } + } + let inner = Fixed(vec![Duration::ZERO, Duration::MAX, secs(1)].into_iter()); + let mut j = Jittered::with_seed(inner, 1); + assert_eq!(j.next_delay(), Some(Duration::ZERO)); // rand(0..=0) + let _ = j.next_delay().unwrap(); // Duration::MAX saturates, no panic + assert!(j.next_delay().unwrap() <= secs(1)); + assert_eq!(j.next_delay(), None); + } + + // --- decorrelated jitter --- + + fn dec(base: u64, max_delay: u64, max_retries: u32, seed: u64) -> DecorrelatedBackoff { + DecorrelatedBackoff::with_seed( + DecorrelatedBackoffConfig { + base: secs(base), + max_retries, + max_delay: secs(max_delay), + }, + seed, + ) + .unwrap() + } + + #[test] + fn decorrelated_draws_each_delay_from_the_previous_one() { + // The recurrence, checked step by step across seeds: every delay sits in + // [base, min(cap, prev * 3)]. The `base` floor is what stops it collapsing toward zero, + // and it's the first non-zero `lo` anything passes to `rand_duration`. + let (base, cap) = (secs(1), secs(30)); + let mut ever_above_double = false; + for seed in 0..16 { + let mut prev = base; + let mut b = dec(1, 30, 40, seed); + while let Some(d) = b.next_delay() { + let hi = (prev * 3).min(cap); + assert!( + d >= base && d <= hi, + "{d:?} outside [{base:?}, {hi:?}] (seed {seed})" + ); + ever_above_double |= d > prev * 2 && d < cap; + prev = d; + } + } + // That bound is one-sided, so a smaller multiplier would satisfy it too. Only a range that + // really runs to `prev * 3` can land a delay past `prev * 2`. + assert!(ever_above_double, "no delay ever exceeded `prev * 2`"); + } + + #[test] + fn decorrelated_feeds_the_capped_delay_into_the_next_draw() { + // `prev` must be the delay handed out, not the raw draw. Feeding the uncapped draw back in + // lets `prev` grow without bound, so the sequence pins to the cap and stops coming down. + // Every per-delay bound still holds under that mutation, so what separates the two is how + // often the sequence recovers below the cap. + let cap = secs(2); + let (mut below, mut total) = (0u32, 0u32); + for seed in 0..32 { + for d in drain(dec(1, 2, 32, seed)) { + below += u32::from(d < cap); + total += 1; + } + } + // Feeding back the capped value holds this near 20%. Feeding back the raw draw collapses + // it to roughly 3%, since `prev` runs away after a handful of retries. + assert!( + below * 10 >= total, + "sequence stopped recovering below the cap: {below}/{total}" + ); + } + + #[test] + fn unseeded_decorrelated_differs_between_instances() { + // Same reasoning as the jitter twin: the entropy-seeded path is the whole reason this type + // isn't `Clone`, and a regression to a fixed seed would pass every other test here. + let cfg = || DecorrelatedBackoffConfig { + base: secs(1), + max_retries: 8, + max_delay: secs(60), + }; + assert_ne!( + drain(DecorrelatedBackoff::new(cfg()).unwrap()), + drain(DecorrelatedBackoff::new(cfg()).unwrap()) + ); + } + + #[test] + fn decorrelated_first_delay_is_already_random() { + // Unlike `ExponentialBackoff`, whose first delay is exactly `base`, `prev` starts at + // `base` so the very first delay is drawn from [base, base * 3]. + let firsts: Vec<_> = (0..16) + .map(|seed| dec(1, 1000, 1, seed).next_delay().unwrap()) + .collect(); + assert!(firsts.iter().all(|d| *d >= secs(1) && *d <= secs(3))); + assert!( + firsts.iter().any(|d| *d != secs(1)), + "first delay never moved off `base`" + ); + } + + #[test] + fn decorrelated_gives_up_after_max_retries() { + assert_eq!(drain(dec(1, 100, 5, 3)).len(), 5); + assert_eq!(dec(1, 100, 0, 3).next_delay(), None); // no retries — exactly one attempt + } + + #[test] + fn decorrelated_seed_is_reproducible() { + // Same seed, same sequence, so a jittered retry stays testable; different seeds differ, + // which guards against a constant or broken RNG. + let seq = |seed| drain(dec(1, 60, 8, seed)); + assert_eq!(seq(7), seq(7)); + assert_ne!(seq(1), seq(2)); + } + + #[test] + fn decorrelated_reaches_the_cap() { + // The cap applies after the draw, so `max_delay` is a value the sequence actually hits + // rather than an asymptote it approaches. + let hit = (0..32).any(|seed| drain(dec(1, 4, 24, seed)).contains(&secs(4))); + assert!(hit, "cap was never reached"); + } + + #[test] + fn decorrelated_base_equal_to_cap_is_constant() { + // Degenerate but legal: every draw is capped straight back to `base`, so the RNG can't + // move it. + assert_eq!(drain(dec(5, 5, 4, 12345)), vec![secs(5); 4]); + } + + #[test] + fn decorrelated_huge_base_does_not_panic() { + let mut b = DecorrelatedBackoff::with_seed( + DecorrelatedBackoffConfig { + base: Duration::from_secs(u64::MAX / 2), + max_retries: 3, + max_delay: Duration::MAX, + }, + 1, + ) + .unwrap(); + // `prev * 3` must saturate rather than overflow-panic. This `base` is also past the + // u64-nanosecond range `rand_duration` works in, where the draw saturates low, so it + // pins the `.max(base)` that keeps the "every delay is at least `base`" guarantee total. + let base = Duration::from_secs(u64::MAX / 2); + assert!(b.next_delay().unwrap() >= base); + assert!(b.next_delay().unwrap() >= base); + } + + #[test] + fn decorrelated_rejects_degenerate_configs() { + assert!(matches!( + DecorrelatedBackoff::new(DecorrelatedBackoffConfig { + base: secs(0), + ..Default::default() + }), + Err(BackoffConfigError::ZeroBase) + )); + assert!(matches!( + DecorrelatedBackoff::new(DecorrelatedBackoffConfig { + base: secs(10), + max_delay: secs(5), + ..Default::default() + }), + Err(BackoffConfigError::MaxDelayBelowBase) + )); + } } diff --git a/src/blocking/clock.rs b/src/blocking/clock.rs index 18e07ae..16de7e4 100644 --- a/src/blocking/clock.rs +++ b/src/blocking/clock.rs @@ -17,6 +17,28 @@ pub trait Clock { fn sleep(&self, dur: Duration); } +// Same reasoning as the async twin: a test wants to keep its mock so it can advance time and read +// back what was slept, and without these `.clock(&mock)` doesn't compile. +impl Clock for &C { + fn now(&self) -> Instant { + (**self).now() + } + + fn sleep(&self, dur: Duration) { + (**self).sleep(dur); + } +} + +impl Clock for std::sync::Arc { + fn now(&self) -> Instant { + (**self).now() + } + + fn sleep(&self, dur: Duration) { + (**self).sleep(dur); + } +} + /// A [`Clock`] backed by [`std::thread::sleep`] and [`std::time::Instant`]. #[derive(Debug, Clone, Copy, Default)] pub struct StdClock; diff --git a/src/clock.rs b/src/clock.rs index 6ac9487..24f21ff 100644 --- a/src/clock.rs +++ b/src/clock.rs @@ -18,6 +18,35 @@ pub trait Clock { fn sleep(&self, dur: Duration) -> Self::Sleep; } +// A mock clock is normally something the test wants to keep hold of, so it can advance time and +// read back what was slept. Without these, `.clock(&mock)` fails to compile and the mock has to +// wrap its own state in `Rc`/`Arc` just to be usable, which is friction on the exact path this +// crate exists to make easy. Use `&C` when the retry doesn't outlive the clock, `Arc` when it +// does. +impl Clock for &C { + type Sleep = C::Sleep; + + fn now(&self) -> Instant { + (**self).now() + } + + fn sleep(&self, dur: Duration) -> Self::Sleep { + (**self).sleep(dur) + } +} + +impl Clock for std::sync::Arc { + type Sleep = C::Sleep; + + fn now(&self) -> Instant { + (**self).now() + } + + fn sleep(&self, dur: Duration) -> Self::Sleep { + (**self).sleep(dur) + } +} + /// A [`Clock`] backed by Tokio's timer, so `now` and `sleep` read the same clock /// (and both honor `tokio::time::pause`). #[derive(Debug, Clone, Copy, Default)] diff --git a/src/lib.rs b/src/lib.rs index e5c0c42..50ac609 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,7 @@ //! don't hand-roll retry-and-backoff logic in every project. //! //! Available now: `retry()` (async) and `blocking::retry()` (sync), both with configurable -//! backoff. Timeout and circuit breaking are planned. +//! backoff and optional jitter. Timeout and circuit breaking are planned. //! //! # Quickstart //! @@ -27,6 +27,29 @@ //! the builder methods, then `.await`. No async runtime? The blocking twin is identical but ends //! in `.call()` instead of `.await`. //! +//! # Jitter +//! +//! A fixed schedule means every client that failed together retries together, so a service coming +//! back up gets a synchronized wave. Two ways to spread that out, both opt-in: +//! +//! ``` +//! use mettle::{Backoff, DecorrelatedBackoff, DecorrelatedBackoffConfig, ExponentialBackoff}; +//! +//! // Randomize any strategy's delays into `0 ..= delay` ("full jitter"). +//! let spread = ExponentialBackoff::default().jittered(); +//! +//! // Or draw each delay from the previous one, never below `base`. +//! let floored = DecorrelatedBackoff::new(DecorrelatedBackoffConfig::default())?; +//! # let _ = (spread, floored); +//! # Ok::<_, mettle::BackoffConfigError>(()) +//! ``` +//! +//! [`jittered`](Backoff::jittered) wraps any strategy, including one you wrote. +//! [`DecorrelatedBackoff`] is its own strategy and keeps a floor under every wait, at the cost of +//! never retrying sooner than `base`. Both seed from entropy; use +//! [`jittered_with_seed`](Backoff::jittered_with_seed) or +//! [`DecorrelatedBackoff::with_seed`] when a test needs the delays to repeat. +//! //! # Observability //! //! Every retry emits a [`tracing`](https://docs.rs/tracing) event on target `mettle::retry` at @@ -58,7 +81,10 @@ mod shared; #[cfg(test)] mod test_support; -pub use backoff::{Backoff, BackoffConfigError, ExponentialBackoff, ExponentialBackoffConfig}; +pub use backoff::{ + Backoff, BackoffConfigError, DecorrelatedBackoff, DecorrelatedBackoffConfig, + ExponentialBackoff, ExponentialBackoffConfig, Jittered, +}; #[cfg(feature = "async")] pub use clock::Clock; #[cfg(feature = "async")] diff --git a/src/retry.rs b/src/retry.rs index 677ec62..c19d458 100644 --- a/src/retry.rs +++ b/src/retry.rs @@ -685,4 +685,55 @@ mod tests { assert_eq!(out, Ok(42)); assert_eq!(events.get(), 2); // one event per retry } + + // --- randomized strategies driven through the real driver --- + + #[tokio::test] + async fn drives_a_decorrelated_backoff() { + // The backoff tests cover the delay sequence in isolation; this covers the wiring, that a + // strategy holding its own RNG survives being moved into the future and polled. Seeded, so + // the delays are fixed even though the strategy is randomized. + use crate::backoff::{DecorrelatedBackoff, DecorrelatedBackoffConfig}; + + let clock = MockClock::new(); + let out: Result = retry(|| async { Err("boom") }) + .backoff( + DecorrelatedBackoff::with_seed( + DecorrelatedBackoffConfig { + base: secs(1), + max_retries: 4, + max_delay: secs(20), + }, + 7, + ) + .unwrap(), + ) + .clock(clock.clone()) + .await; + + assert_eq!(out, Err("boom")); + let slept = clock.slept(); + assert_eq!(slept.len(), 4); // max_retries sleeps, then give up + assert!( + slept.iter().all(|d| *d >= secs(1) && *d <= secs(20)), + "delays escaped [base, max_delay]: {slept:?}" + ); + } + + #[tokio::test] + async fn drives_a_jittered_backoff() { + let clock = MockClock::new(); + let out: Result = retry(|| async { Err("boom") }) + .backoff(crate::backoff::Jittered::with_seed(backoff(3), 42)) + .clock(clock.clone()) + .await; + + assert_eq!(out, Err("boom")); + // Underlying exponential is 1s, 2s, 4s; full jitter can only shrink each one. + let slept = clock.slept(); + assert_eq!(slept.len(), 3); + for (d, cap) in slept.iter().zip([secs(1), secs(2), secs(4)]) { + assert!(*d <= cap, "jittered delay {d:?} exceeded {cap:?}"); + } + } }