diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34c7631..174a93e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -231,6 +231,38 @@ jobs: sudo timeout --kill-after=15s 300s env "PATH=$HOME/.cargo/bin:$PATH" "HOME=$HOME" \ "CARGO_HOME=${CARGO_HOME:-$HOME/.cargo}" "RUSTUP_HOME=${RUSTUP_HOME:-$HOME/.rustup}" \ ./target/debug/lab test crates/blackwall-lab/scenarios/flowspec-auto-bird.kdl lab-junit-flowspec-auto.xml + - name: "Run the lab gate (armed: protected skip + rate re-announce)" + # M1 arming interlocks C1/C4 against real BIRD2: a target inside a + # `protect`ed prefix never reaches BIRD's RIB while a normal target + # in the same prefix does, and a FlowSpec rule re-asserted with a + # changed rate re-announces the new action. + timeout-minutes: 6 + run: | + trap 'rc=$?; exec 1>&3 2>&4; cat lab-gate.log; exit $rc' EXIT + exec 3>&1 4>&2 >lab-gate.log 2>&1 + sudo timeout --kill-after=15s 300s env "PATH=$HOME/.cargo/bin:$PATH" "HOME=$HOME" \ + "CARGO_HOME=${CARGO_HOME:-$HOME/.cargo}" "RUSTUP_HOME=${RUSTUP_HOME:-$HOME/.rustup}" \ + ./target/debug/lab test crates/blackwall-lab/scenarios/armed-protect-reannounce-bird.kdl lab-junit-armed-protect-reannounce.xml + - name: "Run the lab gate (armed: no-reset on un-negotiated SAFI)" + # M1 arming interlock C3 against real BIRD2: a peer that never + # negotiates the FlowSpec SAFI is skipped, not NOTIFICATION-reset. + timeout-minutes: 6 + run: | + trap 'rc=$?; exec 1>&3 2>&4; cat lab-gate.log; exit $rc' EXIT + exec 3>&1 4>&2 >lab-gate.log 2>&1 + sudo timeout --kill-after=15s 300s env "PATH=$HOME/.cargo/bin:$PATH" "HOME=$HOME" \ + "CARGO_HOME=${CARGO_HOME:-$HOME/.cargo}" "RUSTUP_HOME=${RUSTUP_HOME:-$HOME/.rustup}" \ + ./target/debug/lab test crates/blackwall-lab/scenarios/armed-noneg-flowspec-bird.kdl lab-junit-armed-noneg-flowspec.xml + - name: "Run the lab gate (armed: SIGUSR1 disarm)" + # M1 arming interlock C5 against real BIRD2: SIGUSR1 withdraws every + # active mitigation from BIRD's RIB while the daemon keeps running. + timeout-minutes: 6 + run: | + trap 'rc=$?; exec 1>&3 2>&4; cat lab-gate.log; exit $rc' EXIT + exec 3>&1 4>&2 >lab-gate.log 2>&1 + sudo timeout --kill-after=15s 300s env "PATH=$HOME/.cargo/bin:$PATH" "HOME=$HOME" \ + "CARGO_HOME=${CARGO_HOME:-$HOME/.cargo}" "RUSTUP_HOME=${RUSTUP_HOME:-$HOME/.rustup}" \ + ./target/debug/lab test crates/blackwall-lab/scenarios/armed-disarm-bird.kdl lab-junit-armed-disarm.xml - name: Run the lab gate (xdp AF_XDP redirect) # B3.1: end-to-end AF_XDP redirect on a veth pair (attach xdp_filter, bind # an AF_XDP socket into XSKS, inject a UDP frame, assert it lands on the @@ -302,3 +334,6 @@ jobs: lab-junit-bird-gen.xml lab-junit-flowspec.xml lab-junit-flowspec-auto.xml + lab-junit-armed-protect-reannounce.xml + lab-junit-armed-noneg-flowspec.xml + lab-junit-armed-disarm.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index fbfb0de..c5588e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ All notable changes to this project are documented here, following - nftables rendering bound the `prerouting` filter chain to the managed interface (`type filter hook prerouting … device`), which the kernel rejects — only ingress/egress chains may bind a device — so `blackwall-nft::apply` failed on any real ruleset. The chain is now unbound and classification is scoped per-rule with an `iifname` match, the correct pattern for a prerouting filter chain. Because an unbound chain runs for every interface, the closed posture (`default_state` closed) is now enforced by an explicit interface-scoped terminal `drop` rule instead of a chain-wide drop policy, so it no longer black-holes loopback or other-interface host traffic. Caught by the new deception↔scanner lab gate, the first end-to-end run of `apply` against a real `nft`. ### Added +- M1 arming interlock (AS214806 milestone M1) — the safety guards that make removing the `shadow` directive and letting the mitigation plane act for real *safe*. Six control-plane interlocks: (**C1**) **anycast self-protection** — a new repeatable `protect ` directive; a target inside a protected prefix is skipped *before* the eligibility check across RTBH, FlowSpec, and the XDP auto-sink, so a flood of your own anycast VIP can never blackhole your own service (a manual `add` of a protected target now returns `Rejected`, not a silent forever-pending `Deferred`). It also applies under `shadow`, so it stops false "would-blackhole-own-VIP" records in the observation window. (**C2**) **confirm-before-active** — the mitigation managers now roll back the in-memory active entry when the BGP announce / eBPF-map write fails, so the control plane never believes it mitigated something the router/kernel didn't take (previously a failed announce left a phantom "active" entry that deduped future detections into an invisible protection gap); surfaced as `blackwall_{rtbh,flowspec,xdp}_apply_failures_total`. (**C3**) **capability-gated OPEN** — FlowSpec/IPv6 announces *and* withdraws are gated on the peer's negotiated AFI/SAFIs, so a peer that never negotiated SAFI 133 no longer NOTIFICATION-resets the session in a loop (skipped + `blackwall_bgp_unnegotiated_announce_skipped_total{safi}`). (**C4**) a re-asserted FlowSpec rule whose action changed (e.g. tightening a rate-limit to a full drop mid-attack) now re-announces instead of silently no-op'ing. (**C6**) a cross-plane **rate cap** (`max-new-per-min` on the `rtbh` directive) bounds the transient blast radius of a detection storm or bug — new mitigations over the ceiling are rejected + counted (`blackwall_mitigations_ratecapped_total{plane}`), live-only. (**C5**) an **in-daemon disarm** on `SIGUSR1` — withdraws every announced route then keeps detecting + recording but applies nothing (record-only), so an operator can instantly stop mitigating without losing the detector; a `blackwall_armed` gauge reads 1 live / 0 shadow / 0 disarmed. All guards are inert under `shadow` (except C1's recording) and every new config field defaults to today's behavior. Proven end-to-end by three armed-mode lab gates against real BIRD 2.17.1 (protected-skip, rate re-announce, no-reset-on-un-negotiated-peer, SIGUSR1 withdraw-all). Arming remains `remove shadow + restart`; RPKI cross-check, the XDP data-plane armed-only bugs, per-plane arming, runtime re-arm, and a control-API disarm endpoint are deferred follow-ons. - BIRD iBGP-snippet generator (deployment #3): `blackwalld bird-config --config ` generates BIRD's side of the blackwall↔BIRD iBGP session from blackwall's own config, so prefix lists and session params aren't hand-maintained in both `blackwall.conf` and `bird.conf`. A pure `blackwall_bgp::render_bird_ibgp` emits an `include` file: `OWN_V4/V6` prefix defines plus one MP-BGP `protocol bgp blackwall` session (ipv4/ipv6/flow4/flow6 channels; unicast import filters `net ~ [prefix+]`, flow filters `net.dst ~ [prefix+]`; MD5 → an `include "blackwall-secret.conf";` reference so the secret never lands in the generated file; GTSM → `ttl security on`). A new `rtbh local-addr=` sets blackwall's BGP source, emitted as BIRD's `neighbor` and bound by the speaker as its source so the two sides match by construction. Validated against real **BIRD 2.17.1** by a new `bird-gen` lab gate (the generated include establishes the session and imports both a `/32` blackhole and a FlowSpec rule) plus a `bird -p` parse-check. BIRD stays the fan-out point — blackwall injects once, BIRD re-advertises to every upstream via its existing per-peer filters. Non-breaking. - Network-wide shadow mode (deployment #9): a global, opt-in `shadow` config directive that makes the mitigation plane log + record + meter every RTBH/FlowSpec/XDP mitigation the daemon *would* apply, **without executing it** — the interlock for running a detection-only deployment live on the security boundary. Shadow decorates the execution boundary only (a `ShadowBgpExecutor` that holds no BGP handle, so it's structurally incapable of announcing; no-op journals keep the mirror empty; the real iBGP session isn't spawned; the XDP map-apply is gated), while detection/selection/controller logic runs identically. Intended actions surface via INFO logs, a `blackwall_shadow_would_mitigate_total{plane,action}` counter, and `audit_log` rows (queryable through `/v1/audit`), with a startup `WARN: SHADOW MODE` banner. Non-breaking (absent `shadow` = live behavior). - Anycast telemetry ingest (deployment #1/#2, sub-project D): the sFlow agent address (previously parsed then discarded) is now threaded into every `FlowObservation`, so the central `flow` daemon reads feeds from many anycast POPs as one logical view. A `pop agent= sampling=` config directive maps agents to POP names + expected rates; detections are tagged with the contributing POPs and a top-N attacker source-block (/24 v4, /48 v6) rollup; per-agent liveness and a sampling-sanity clamp (bounded to known agents, closing a spoofed-agent memory-DoS) guard the feed, exposed as `blackwall_flow_{pop_last_seen_seconds,agent_sampling_mismatch_total,unknown_agent_observations_total}`. `blackwalld sensor render-hsflowd` generates each POP's hsflowd.conf from the POP-map. Non-breaking (no `pop` block = per-victim detection as before, everything tagged `unknown`). diff --git a/bin/blackwalld/src/main.rs b/bin/blackwalld/src/main.rs index 5c4cc7a..57a0c04 100644 --- a/bin/blackwalld/src/main.rs +++ b/bin/blackwalld/src/main.rs @@ -661,6 +661,7 @@ fn rtbh_config_from( max_blackholes: rtbh.max_blackholes, hold_down: rtbh.hold_down, max_ttl: rtbh.max_ttl, + protected_prefixes: policy.protected_prefixes.clone(), } } @@ -677,6 +678,7 @@ fn flowspec_config_from( max_rules: fs.max_rules, hold_down: fs.hold_down, max_ttl: fs.max_ttl, + protected_prefixes: policy.protected_prefixes.clone(), } } @@ -760,6 +762,46 @@ async fn wait_for_shutdown() { } } +/// Listen for SIGUSR1 and fan a one-shot disarm command out to every +/// RTBH/FlowSpec/XDP manager task via `disarm_tx` (C5: in-daemon kill +/// switch). +/// +/// Each manager task withdraws its own active mitigations and switches to +/// record-only on receipt (see [`rtbh_manager_task`]/[`flowspec_manager_task`]/ +/// [`xdp_manager_task`]'s `disarm_rx` arm); this task only relays the signal +/// and flips the shared `blackwall_armed` gauge to `0`. One-way: a second +/// SIGUSR1 re-broadcasts, but every manager's own `disarm` is idempotent (a +/// no-op once already disarmed), and there is no re-arm signal — only a +/// restart clears it. Detached for the process's lifetime (mirrors +/// [`bgp_supervisor`]); if the signal handler fails to install (rare — e.g. +/// exhausted signalfd resources), this logs once and returns, leaving +/// SIGTERM/SIGINT shutdown (handled separately by [`wait_for_shutdown`]) +/// unaffected. +async fn disarm_signal_task( + disarm_tx: tokio::sync::broadcast::Sender<()>, + armed: std::sync::Arc, +) { + use tokio::signal::unix::{signal, SignalKind}; + let Ok(mut usr1) = signal(SignalKind::user_defined1()) else { + tracing::warn!( + "failed to install SIGUSR1 handler; in-daemon disarm (C5) is unavailable this run" + ); + return; + }; + loop { + if usr1.recv().await.is_none() { + return; + } + tracing::warn!( + "WARN: DISARMED — SIGUSR1 received: withdrawing all mitigations, now recording only (one-way; restart to re-arm)" + ); + armed.store(0, std::sync::atomic::Ordering::Relaxed); + // No receivers (e.g. no rtbh/flowspec/xdp block configured) is not + // an error — there is simply nothing to disarm. + let _ = disarm_tx.send(()); + } +} + /// Observe the BGP session and log loudly when it leaves `Established` — a down /// session means auto-mitigations are not reaching the peer (issue #79). Purely /// observational; the session task drives reconnect itself. Exits when the @@ -795,11 +837,22 @@ async fn rtbh_manager_task( mut manager: blackwall_rtbh::RtbhManager, mut rx: mpsc::Receiver, request_store: std::sync::Arc, + protected_metrics: Arc, + apply_failure_metrics: Arc, + ratecapped_metrics: Arc, + mut disarm_rx: tokio::sync::broadcast::Receiver<()>, ) where B: blackwall_rtbh::manager::BgpExecutor + Send + 'static, J: blackwall_rtbh::manager::BlackholeJournal + Send + 'static, { let mut ticker = tokio::time::interval(std::time::Duration::from_secs(1)); + // Once `disarm_tx` (held by `disarm_signal_task`) is gone — e.g. SIGUSR1 + // registration failed at startup — `disarm_rx.recv()` would return + // `Err(Closed)` on every poll forever; without this guard that turns + // into a busy loop (the branch is always immediately ready). Once + // observed, the `if` precondition below permanently disables polling + // this branch, so the task falls back to `rx`/`ticker` only. + let mut disarm_open = true; loop { tokio::select! { maybe_ev = rx.recv() => { @@ -811,11 +864,39 @@ async fn rtbh_manager_task( } } } + disarmed = disarm_rx.recv(), if disarm_open => { + // C5: withdraw every active blackhole and switch to + // record-only. A `Lagged` delivery still means "disarm was + // requested" (the payload is `()`, nothing to miss), so it + // is treated the same as `Ok`. + match disarmed { + Ok(()) | Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => { + tracing::warn!("RTBH: DISARMED — mitigations withdrawn, now recording only"); + manager.disarm(mono_now()).await; + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => { + disarm_open = false; + } + } + } _ = ticker.tick() => { // Mandatory: without this, a `Cleared` arriving before hold-down // elapses is deferred and never completed. manager.tick(mono_now(), wall_now()).await; + protected_metrics.rtbh.store( + manager.protected_skipped(), + std::sync::atomic::Ordering::Relaxed, + ); + apply_failure_metrics.store( + manager.apply_failures(), + std::sync::atomic::Ordering::Relaxed, + ); + ratecapped_metrics.rtbh.store( + manager.ratecapped(), + std::sync::atomic::Ordering::Relaxed, + ); + match request_store.pending_requests().await { Ok(reqs) => { for req in reqs { @@ -869,7 +950,9 @@ async fn apply_request( } }, "remove" => { - manager.apply_remove(req.target, wall_now()).await; + manager + .apply_remove(req.target, mono_now(), wall_now()) + .await; // Cancel any other still-pending add for the same target: the // operator's remove is the newer intent, so a pending add must // not later announce this target once capacity frees. @@ -909,11 +992,17 @@ async fn flowspec_manager_task( mut manager: blackwall_rtbh::FlowSpecManager, mut rx: mpsc::Receiver, request_store: std::sync::Arc, + protected_metrics: Arc, + apply_failure_metrics: Arc, + ratecapped_metrics: Arc, + mut disarm_rx: tokio::sync::broadcast::Receiver<()>, ) where B: blackwall_rtbh::manager::BgpExecutor + Send + 'static, J: blackwall_rtbh::flowspec_manager::FlowSpecJournal + Send + 'static, { let mut ticker = tokio::time::interval(std::time::Duration::from_secs(1)); + // See the matching guard in `rtbh_manager_task` for why this exists. + let mut disarm_open = true; loop { tokio::select! { maybe_ev = rx.recv() => { @@ -934,10 +1023,36 @@ async fn flowspec_manager_task( } } } + disarmed = disarm_rx.recv(), if disarm_open => { + // C5: see the matching arm in `rtbh_manager_task` for why a + // `Lagged` delivery still counts as "disarm was requested". + match disarmed { + Ok(()) | Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => { + tracing::warn!("FlowSpec: DISARMED — mitigations withdrawn, now recording only"); + manager.disarm(mono_now()).await; + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => { + disarm_open = false; + } + } + } _ = ticker.tick() => { // Mandatory: completes deferred clears / TTL expiry. manager.tick(mono_now(), wall_now()).await; + protected_metrics.flowspec.store( + manager.protected_skipped(), + std::sync::atomic::Ordering::Relaxed, + ); + apply_failure_metrics.store( + manager.apply_failures(), + std::sync::atomic::Ordering::Relaxed, + ); + ratecapped_metrics.flowspec.store( + manager.ratecapped(), + std::sync::atomic::Ordering::Relaxed, + ); + match request_store.pending_flowspec_requests().await { Ok(reqs) => { for req in reqs { @@ -997,7 +1112,7 @@ async fn apply_flowspec_request( } }, "remove" => { - manager.apply_remove(rule, wall_now()).await; + manager.apply_remove(rule, mono_now(), wall_now()).await; // The operator's remove is the newer intent: cancel any earlier // still-pending add for the same flow key. if let Err(err) = request_store @@ -1095,10 +1210,15 @@ async fn xdp_manager_task( mut rx: mpsc::Receiver, request_store: std::sync::Arc, auto_enabled: bool, + protected_metrics: Arc, + apply_failure_metrics: Arc, + mut disarm_rx: tokio::sync::broadcast::Receiver<()>, ) where J: blackwall_xdp::XdpJournal + 'static, { let mut ticker = tokio::time::interval(std::time::Duration::from_secs(1)); + // See the matching guard in `rtbh_manager_task` for why this exists. + let mut disarm_open = true; loop { tokio::select! { maybe_ev = rx.recv() => { @@ -1114,10 +1234,32 @@ async fn xdp_manager_task( } } } + disarmed = disarm_rx.recv(), if disarm_open => { + // C5: see the matching arm in `rtbh_manager_task` for why a + // `Lagged` delivery still counts as "disarm was requested". + match disarmed { + Ok(()) | Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => { + tracing::warn!("XDP: DISARMED — mitigations withdrawn, now recording only"); + manager.disarm(mono_now()).await; + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => { + disarm_open = false; + } + } + } _ = ticker.tick() => { // Drains any journal mirror-writes queued by a transient DB blip. manager.tick().await; + protected_metrics.xdp.store( + manager.protected_skipped(), + std::sync::atomic::Ordering::Relaxed, + ); + apply_failure_metrics.store( + manager.apply_failures(), + std::sync::atomic::Ordering::Relaxed, + ); + match request_store.xdp_pending_requests().await { Ok(reqs) => { for req in reqs { @@ -1410,6 +1552,52 @@ async fn run() -> Result<(), Box> { // the metrics endpoint. Built unconditionally — harmless all-zero // counters when shadow mode is off. let shadow_metrics = std::sync::Arc::new(shadow::ShadowMetrics::default()); + // Shared anycast self-protection (C1) skip counters: each manager + // task below copies its controller's `protected_skipped()` value + // in here on every tick, in BOTH shadow and live sessions (unlike + // `shadow_metrics`, this guard is not shadow-specific). Built + // unconditionally — harmless all-zero counters when RTBH/FlowSpec/ + // XDP aren't configured. + let protected_skipped_metrics = + std::sync::Arc::new(shadow::ProtectedSkippedMetrics::default()); + // RTBH announces that failed at the BGP executor and were rolled + // back (C2): copied from `RtbhManager::apply_failures` on every + // tick, mirroring `protected_skipped_metrics` above. Built + // unconditionally — harmless all-zero counter when no `rtbh` + // block is configured. + let rtbh_apply_failure_metrics = + std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + // FlowSpec announces that failed at the BGP executor and were + // rolled back (C2): copied from `FlowSpecManager::apply_failures` + // on every tick, mirroring `rtbh_apply_failure_metrics` above. + // Built unconditionally — harmless all-zero counter when no + // `flowspec` block is configured. + let flowspec_apply_failure_metrics = + std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + // XDP executor (eBPF-map) applies that failed and were rolled + // back (C2): copied from `XdpManager::apply_failures` on every + // tick, mirroring `rtbh_apply_failure_metrics` above. Built + // unconditionally — harmless all-zero counter when no `xdp` + // block is configured. + let xdp_apply_failure_metrics = + std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + // C6 cross-plane new-mitigation rate cap skip counters: copied + // from `RtbhManager`/`FlowSpecManager::ratecapped` on every tick, + // mirroring `protected_skipped_metrics` above. Built + // unconditionally — harmless all-zero counters when no `rtbh` + // block is configured or `max-new-per-min` is unset. + let ratecapped_metrics = std::sync::Arc::new(shadow::RatecappedMetrics::default()); + // In-daemon disarm kill switch (C5): `blackwall_armed` starts at + // 1 (live) or 0 (shadow) and is flipped to 0 exactly once, on a + // SIGUSR1 disarm — there is no path back to 1 short of a + // restart. `disarm_tx` is the broadcast sender the SIGUSR1 + // listener (spawned below, once every manager task below has + // subscribed) uses to fan a single disarm command out to every + // RTBH/FlowSpec/XDP manager task; each subscribes via + // `disarm_tx.subscribe()` when it is spawned. + let blackwall_armed = + std::sync::Arc::new(std::sync::atomic::AtomicU8::new(u8::from(!policy.shadow))); + let (disarm_tx, _disarm_rx) = tokio::sync::broadcast::channel::<()>(8); let sink: std::sync::Arc = match policy.rtbh.clone() { @@ -1423,6 +1611,27 @@ async fn run() -> Result<(), Box> { let channel_cap = rtbh.max_blackholes.max(1024); let (tx, rx) = mpsc::channel::(channel_cap); + // C6 cross-plane rate cap on new mitigations: ONE limiter + // built from the rtbh block's `max-new-per-min` knob, + // shared between RTBH and FlowSpec below (FlowSpec reuses + // this block). Only wired on the live path (`!policy.shadow`) + // — under shadow nothing is really announced, so + // rate-capping would only corrupt the would-mitigate + // signal; `None` here (either shadow, or the knob absent) + // means neither manager ever gets `.with_rate_limiter` + // called, so both stay unlimited (non-breaking default). + let rate_limiter: Option< + std::sync::Arc>, + > = if policy.shadow { + None + } else { + rtbh.max_new_per_min.map(|n| { + std::sync::Arc::new(std::sync::Mutex::new( + blackwall_rtbh::ArmingRateLimiter::new(n), + )) + }) + }; + // The live BGP handle, threaded to the FlowSpec // construction below so its live branch reuses this same // iBGP session. `None` in shadow mode (no real session is @@ -1440,7 +1649,15 @@ async fn run() -> Result<(), Box> { // No rehydrate: the shadow mirror is intentionally // empty — nothing was ever really announced, so there // is nothing to replay. - tokio::spawn(rtbh_manager_task(manager, rx, store.clone())); + tokio::spawn(rtbh_manager_task( + manager, + rx, + store.clone(), + protected_skipped_metrics.clone(), + rtbh_apply_failure_metrics.clone(), + ratecapped_metrics.clone(), + disarm_tx.subscribe(), + )); None } else { let peer = blackwall_bgp::PeerConfig { @@ -1465,6 +1682,9 @@ async fn run() -> Result<(), Box> { let journal: blackwall_state::Store = (*store).clone(); let mut manager = blackwall_rtbh::RtbhManager::new(controller, bgp.clone(), journal); + if let Some(limiter) = &rate_limiter { + manager = manager.with_rate_limiter(limiter.clone()); + } // Rehydrate the controller from the announced mirror // before this session starts accepting new @@ -1483,7 +1703,15 @@ async fn run() -> Result<(), Box> { .collect(); manager.rehydrate(rehydrate_rows, mono_now()).await; - tokio::spawn(rtbh_manager_task(manager, rx, store.clone())); + tokio::spawn(rtbh_manager_task( + manager, + rx, + store.clone(), + protected_skipped_metrics.clone(), + rtbh_apply_failure_metrics.clone(), + ratecapped_metrics.clone(), + disarm_tx.subscribe(), + )); Some(bgp) }; @@ -1530,6 +1758,10 @@ async fn run() -> Result<(), Box> { fs_manager, fs_rx, store.clone(), + protected_skipped_metrics.clone(), + flowspec_apply_failure_metrics.clone(), + ratecapped_metrics.clone(), + disarm_tx.subscribe(), )); } Some(bgp) => { @@ -1539,6 +1771,14 @@ async fn run() -> Result<(), Box> { bgp, fs_journal, ); + // FlowSpec reuses the rtbh block's + // `max-new-per-min` knob: the SAME shared + // limiter as the RTBH manager above, so + // ONE cap governs the combined + // cross-plane announce rate (C6). + if let Some(limiter) = &rate_limiter { + fs_manager = fs_manager.with_rate_limiter(limiter.clone()); + } // Rehydrate FlowSpec rules from the announced mirror. let fs_mirror = store.list_active_flowspec().await?; @@ -1570,6 +1810,10 @@ async fn run() -> Result<(), Box> { fs_manager, fs_rx, store.clone(), + protected_skipped_metrics.clone(), + flowspec_apply_failure_metrics.clone(), + ratecapped_metrics.clone(), + disarm_tx.subscribe(), )); } } @@ -1724,6 +1968,7 @@ async fn run() -> Result<(), Box> { policy.prefixes.clone(), XDP_MAX_ENTRIES, default_pps, + policy.protected_prefixes.clone(), ); let (xdp_tx, xdp_rx) = mpsc::channel::(4096); @@ -1757,6 +2002,9 @@ async fn run() -> Result<(), Box> { xdp_rx, store.clone(), auto_enabled, + protected_skipped_metrics.clone(), + xdp_apply_failure_metrics.clone(), + disarm_tx.subscribe(), )) } else { let executor = shadow::XdpExec::Live(dataplane.clone()); @@ -1781,6 +2029,9 @@ async fn run() -> Result<(), Box> { xdp_rx, store.clone(), auto_enabled, + protected_skipped_metrics.clone(), + xdp_apply_failure_metrics.clone(), + disarm_tx.subscribe(), )) }; xdp_shutdown = Some((handle, dataplane)); @@ -1795,6 +2046,11 @@ async fn run() -> Result<(), Box> { sink }; + // C5: every RTBH/FlowSpec/XDP manager task above has now + // subscribed to `disarm_tx`, so it is safe to start listening + // for the operator's SIGUSR1 disarm signal. + tokio::spawn(disarm_signal_task(disarm_tx, blackwall_armed.clone())); + // Optional Prometheus metrics endpoint. if let Some(metrics_listen) = policy.metrics_listen { let sources = metrics::MetricsSources { @@ -1807,6 +2063,12 @@ async fn run() -> Result<(), Box> { afxdp_udp_responses: afxdp_udp_metric.clone(), agent_stats: Some(agent_snapshot.clone()), shadow: Some(shadow_metrics.clone()), + protected_skipped: Some(protected_skipped_metrics.clone()), + ratecapped: Some(ratecapped_metrics.clone()), + rtbh_apply_failures: Some(rtbh_apply_failure_metrics.clone()), + flowspec_apply_failures: Some(flowspec_apply_failure_metrics.clone()), + xdp_apply_failures: Some(xdp_apply_failure_metrics.clone()), + armed: Some(blackwall_armed.clone()), }; tokio::spawn(metrics::metrics_server(metrics_listen, sources)); } @@ -2060,6 +2322,12 @@ async fn run() -> Result<(), Box> { afxdp_udp_responses: None, agent_stats: None, shadow: None, + protected_skipped: None, + ratecapped: None, + rtbh_apply_failures: None, + flowspec_apply_failures: None, + xdp_apply_failures: None, + armed: None, }; tokio::spawn(metrics::metrics_server(metrics_listen, sources)); } diff --git a/bin/blackwalld/src/metrics.rs b/bin/blackwalld/src/metrics.rs index 1a6c34e..66ebebb 100644 --- a/bin/blackwalld/src/metrics.rs +++ b/bin/blackwalld/src/metrics.rs @@ -36,6 +36,37 @@ pub(crate) struct MetricsSources { /// Shadow-mode "would mitigate" counters (RTBH/FlowSpec/XDP); `None` /// outside the flow daemon (no RTBH/FlowSpec/XDP managers to shadow). pub shadow: Option>, + /// Per-plane anycast self-protection skip counters (C1); `None` outside + /// the flow daemon (no RTBH/FlowSpec/XDP managers to guard). Unlike + /// `shadow`, populated in both shadow AND live sessions. + pub protected_skipped: Option>, + /// RTBH announces that failed at the BGP executor and were rolled back + /// (C2, `RtbhManager::apply_failures`); `None` when no `rtbh` block is + /// configured. Copied from the manager once per tick, mirroring how + /// `protected_skipped` reaches this endpoint. + pub rtbh_apply_failures: Option>, + /// FlowSpec announces that failed at the BGP executor and were rolled + /// back (C2, `FlowSpecManager::apply_failures`); `None` when no + /// `flowspec` block is configured. Copied from the manager once per + /// tick, mirroring `rtbh_apply_failures`. + pub flowspec_apply_failures: Option>, + /// XDP executor (eBPF-map) applies that failed and were rolled back (C2, + /// `blackwall_xdp::manager::XdpManager::apply_failures`); `None` when no + /// `xdp` block is configured. Copied from the manager once per tick, + /// mirroring `rtbh_apply_failures`. + pub xdp_apply_failures: Option>, + /// Per-plane cross-plane new-mitigation rate cap (C6) skip counters + /// (`RtbhManager`/`FlowSpecManager::ratecapped`); `None` outside the flow + /// daemon (no managers to cap). Populated (all-zero) even when no `rtbh` + /// block is configured or `max-new-per-min` is unset (unlimited), mirroring + /// `protected_skipped`. + pub ratecapped: Option>, + /// Whether mitigations are actually being applied: `1` live, `0` under + /// `shadow` or after an in-daemon disarm (C5, SIGUSR1) — flipped to `0` + /// exactly once, at process start (shadow) or on disarm; there is no + /// path back to `1` short of a restart. `None` outside the flow daemon + /// (no RTBH/FlowSpec/XDP managers to arm). + pub armed: Option>, } /// Correctly-rounded `u64 -> f64` without an `as` cast: `u32 -> f64` is exact @@ -116,6 +147,39 @@ async fn gather(sources: &MetricsSources) -> Vec { }); } + if let Some(rtbh_apply_failures) = &sources.rtbh_apply_failures { + m.push(Metric { + name: "blackwall_rtbh_apply_failures_total", + help: "RTBH announces that failed at the BGP executor and were rolled back (C2)", + kind: MetricKind::Counter, + value: u64_to_f64(rtbh_apply_failures.load(std::sync::atomic::Ordering::Relaxed)), + }); + } + if let Some(flowspec_apply_failures) = &sources.flowspec_apply_failures { + m.push(Metric { + name: "blackwall_flowspec_apply_failures_total", + help: "FlowSpec announces that failed at the BGP executor and were rolled back (C2)", + kind: MetricKind::Counter, + value: u64_to_f64(flowspec_apply_failures.load(std::sync::atomic::Ordering::Relaxed)), + }); + } + if let Some(xdp_apply_failures) = &sources.xdp_apply_failures { + m.push(Metric { + name: "blackwall_xdp_apply_failures_total", + help: "XDP executor (eBPF-map) applies that failed and were rolled back (C2)", + kind: MetricKind::Counter, + value: u64_to_f64(xdp_apply_failures.load(std::sync::atomic::Ordering::Relaxed)), + }); + } + if let Some(armed) = &sources.armed { + m.push(Metric { + name: "blackwall_armed", + help: "Whether mitigations are actually applied: 1 live, 0 shadow or disarmed (C5)", + kind: MetricKind::Gauge, + value: f64::from(armed.load(std::sync::atomic::Ordering::Relaxed)), + }); + } + let s = &sources.store; match s.list_active_blackholes().await { Ok(v) => m.push(Metric { @@ -428,6 +492,93 @@ fn shadow_block(sources: &MetricsSources) -> Option { Some(out) } +/// Render `blackwall_mitigations_protected_skipped_total{plane}` from the +/// shared per-plane counters, or `None` when they aren't wired up (outside +/// the flow daemon). Labels are a fixed, known-at-compile-time set, but still +/// hand-written since [`Metric`] only carries unlabelled series — mirrors +/// [`shadow_block`]. +fn protected_skipped_block(sources: &MetricsSources) -> Option { + use std::sync::atomic::Ordering; + + let counters = sources.protected_skipped.as_ref()?; + let mut out = String::new(); + let _ = writeln!( + out, + "# HELP blackwall_mitigations_protected_skipped_total Targets skipped because they fell inside a configured protected prefix (own VIP), by plane" + ); + let _ = writeln!( + out, + "# TYPE blackwall_mitigations_protected_skipped_total counter" + ); + for (plane, counter) in [ + ("rtbh", &counters.rtbh), + ("flowspec", &counters.flowspec), + ("xdp", &counters.xdp), + ] { + let _ = writeln!( + out, + "blackwall_mitigations_protected_skipped_total{{plane=\"{plane}\"}} {}", + counter.load(Ordering::Relaxed) + ); + } + Some(out) +} + +/// Render `blackwall_mitigations_ratecapped_total{plane}` (C6 cross-plane +/// new-mitigation rate cap) from the shared per-plane counters, or `None` +/// when they aren't wired up (outside the flow daemon) — mirrors +/// [`protected_skipped_block`]. +fn ratecapped_block(sources: &MetricsSources) -> Option { + use std::sync::atomic::Ordering; + + let counters = sources.ratecapped.as_ref()?; + let mut out = String::new(); + let _ = writeln!( + out, + "# HELP blackwall_mitigations_ratecapped_total New-mitigation announces skipped because the shared cross-plane rate cap (C6, max-new-per-min) was at capacity, by plane" + ); + let _ = writeln!(out, "# TYPE blackwall_mitigations_ratecapped_total counter"); + for (plane, counter) in [("rtbh", &counters.rtbh), ("flowspec", &counters.flowspec)] { + let _ = writeln!( + out, + "blackwall_mitigations_ratecapped_total{{plane=\"{plane}\"}} {}", + counter.load(Ordering::Relaxed) + ); + } + Some(out) +} + +/// Render `blackwall_bgp_unnegotiated_announce_skipped_total{safi}` (C3, C3 +/// follow-up) from the live `BgpHandle`, or `None` when no `rtbh`/`flowspec` +/// block is configured (no BGP session to report — mirrors `sources.bgp` +/// used by [`gather`]'s session-state/reconnect metrics). Labels are a fixed, +/// known-at-compile-time set, but still hand-written since [`Metric`] only +/// carries unlabelled series — mirrors [`shadow_block`]/[`protected_skipped_block`]. +fn unnegotiated_announce_skipped_block(sources: &MetricsSources) -> Option { + let bgp = sources.bgp.as_ref()?; + let counts = bgp.unnegotiated_skip_counts(); + let mut out = String::new(); + let _ = writeln!( + out, + "# HELP blackwall_bgp_unnegotiated_announce_skipped_total FlowSpec/IPv6 announces or withdraws skipped because the peer never negotiated that SAFI in its OPEN (C3, C3 follow-up)" + ); + let _ = writeln!( + out, + "# TYPE blackwall_bgp_unnegotiated_announce_skipped_total counter" + ); + for (safi, count) in [ + ("flowspec_v4", counts.flowspec_v4), + ("flowspec_v6", counts.flowspec_v6), + ("ipv6_unicast", counts.ipv6_unicast), + ] { + let _ = writeln!( + out, + "blackwall_bgp_unnegotiated_announce_skipped_total{{safi=\"{safi}\"}} {count}" + ); + } + Some(out) +} + /// Serve `/metrics` forever. Each connection is handled on its own task so a /// slow client cannot block scrapes; a bind failure disables the endpoint (and /// is logged) without taking down the daemon. @@ -481,6 +632,24 @@ async fn handle_conn(mut sock: tokio::net::TcpStream, sources: &MetricsSources) } body.push_str(&shadow); } + if let Some(protected) = protected_skipped_block(sources) { + if !body.is_empty() { + body.push('\n'); + } + body.push_str(&protected); + } + if let Some(ratecapped) = ratecapped_block(sources) { + if !body.is_empty() { + body.push('\n'); + } + body.push_str(&ratecapped); + } + if let Some(unnegotiated) = unnegotiated_announce_skipped_block(sources) { + if !body.is_empty() { + body.push('\n'); + } + body.push_str(&unnegotiated); + } format!( "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len() diff --git a/bin/blackwalld/src/shadow.rs b/bin/blackwalld/src/shadow.rs index 1c64db2..530046a 100644 --- a/bin/blackwalld/src/shadow.rs +++ b/bin/blackwalld/src/shadow.rs @@ -25,6 +25,42 @@ pub struct ShadowMetrics { pub xdp_rate_limit: AtomicU64, } +/// Per-plane counters backing the `blackwall_mitigations_protected_skipped_total` +/// metric (C1 anycast self-protection: RTBH/FlowSpec/XDP each skip a target +/// that falls inside a configured `protected_prefixes` entry, own VIP, +/// before ever reaching eligibility). Unlike [`ShadowMetrics`], this fires in +/// BOTH shadow and live sessions — the guard runs inside each pure +/// controller itself, so it applies regardless of mode. Built unconditionally +/// (harmless all-zero counters when RTBH/FlowSpec/XDP aren't configured); +/// each manager task copies its controller's `protected_skipped()` counter in +/// here on every tick, mirroring how `CollectorMetrics::set_min_sample_suppressed` +/// is kept in sync from the flow detector. +#[derive(Default)] +pub struct ProtectedSkippedMetrics { + /// RTBH targets skipped because they were in a protected prefix. + pub rtbh: AtomicU64, + /// FlowSpec targets skipped because they were in a protected prefix. + pub flowspec: AtomicU64, + /// XDP detections skipped because the victim was in a protected prefix. + pub xdp: AtomicU64, +} + +/// Per-plane counters backing the `blackwall_mitigations_ratecapped_total` +/// metric (C6 cross-plane new-mitigation rate cap: `RtbhManager`/ +/// `FlowSpecManager` each skip a NEW announce once the shared +/// `blackwall_rtbh::ArmingRateLimiter` is at capacity). Mirrors +/// [`ProtectedSkippedMetrics`]: built unconditionally (harmless all-zero +/// counters when no `rtbh` block is configured, or when its +/// `max-new-per-min` knob is absent/unlimited); each manager task copies its +/// `ratecapped()` counter in here on every tick. +#[derive(Default)] +pub struct RatecappedMetrics { + /// RTBH announces skipped by the shared rate cap. + pub rtbh: AtomicU64, + /// FlowSpec announces skipped by the shared rate cap. + pub flowspec: AtomicU64, +} + /// Records shadow actions to the audit log + metrics + INFO log. /// /// Wired in place of a real `BgpExecutor`/journal pair (via diff --git a/crates/blackwall-bgp/src/lib.rs b/crates/blackwall-bgp/src/lib.rs index a6ba4bd..e7bc074 100644 --- a/crates/blackwall-bgp/src/lib.rs +++ b/crates/blackwall-bgp/src/lib.rs @@ -19,5 +19,6 @@ pub use render::{render_bird_ibgp, BirdGenError}; pub use route::{Origin, Route}; pub use session_net::{ spawn, BgpHandle, BgpSendError, PeerConfig, PeerConfigError, SessionCommand, SessionState, + UnnegotiatedSkipCounts, }; pub use update::{build_announce, build_withdraw}; diff --git a/crates/blackwall-bgp/src/session_net.rs b/crates/blackwall-bgp/src/session_net.rs index 83d0506..a8912d6 100644 --- a/crates/blackwall-bgp/src/session_net.rs +++ b/crates/blackwall-bgp/src/session_net.rs @@ -100,6 +100,139 @@ pub enum PeerConfigError { #[error("BGP session task is not running; command dropped")] pub struct BgpSendError; +// ── C3: capability-gated announces/withdraws ──────────────────────────────── + +/// Which negotiated AFI/SAFI an announce or withdraw needs, per RFC 4760 +/// MP-BGP capabilities (RFC 8955/8956 §133 for FlowSpec). +/// +/// [`announce_allowed`] gates a pending announce *or withdraw* against the +/// peer's [`OpenMsg`] before it is written to the wire — sending an UPDATE +/// (MP_REACH_NLRI or MP_UNREACH_NLRI alike) for a SAFI the peer never +/// advertised in its own OPEN causes some implementations (and our own +/// bird-config default) to NOTIFICATION-reset the session, dropping every +/// other active mitigation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AnnounceSafi { + /// AFI 1 / SAFI 1 — unconditionally allowed; see [`announce_allowed`]. + Ipv4Unicast, + /// AFI 2 / SAFI 1. + Ipv6Unicast, + /// AFI 1 / SAFI 133. + FlowSpecV4, + /// AFI 2 / SAFI 133. + FlowSpecV6, +} + +impl AnnounceSafi { + /// The SAFI an announce for `prefix` needs (IPv4 vs IPv6 unicast). + fn for_route(prefix: &IpNet) -> Self { + match prefix { + IpNet::V4(_) => AnnounceSafi::Ipv4Unicast, + IpNet::V6(_) => AnnounceSafi::Ipv6Unicast, + } + } + + /// The SAFI a FlowSpec announce for `dst` needs (v4 vs v6 FlowSpec). + fn for_flowspec(dst: &IpNet) -> Self { + match dst { + IpNet::V4(_) => AnnounceSafi::FlowSpecV4, + IpNet::V6(_) => AnnounceSafi::FlowSpecV6, + } + } + + /// The `safi` label used for `tracing::warn!` and the + /// `blackwall_bgp_unnegotiated_announce_skipped_total` metric. + fn label(self) -> &'static str { + match self { + AnnounceSafi::Ipv4Unicast => "ipv4_unicast", + AnnounceSafi::Ipv6Unicast => "ipv6_unicast", + AnnounceSafi::FlowSpecV4 => "flowspec_v4", + AnnounceSafi::FlowSpecV6 => "flowspec_v6", + } + } +} + +/// Pure gate: does `peer`'s negotiated capability set (decoded from its OPEN) +/// allow an announce or withdraw needing `safi`? +/// +/// IPv4 unicast is unconditionally allowed — every peer we support negotiates +/// it, and it is not the SAFI the C3 bug concerns. IPv6 unicast and both +/// FlowSpec AFIs gate on the corresponding field decoded from the peer's +/// [`OpenMsg`] at handshake (`decode_open`), so an under-negotiating peer is +/// skipped instead of NOTIFICATION-reset. +fn announce_allowed(peer: &OpenMsg, safi: AnnounceSafi) -> bool { + match safi { + AnnounceSafi::Ipv4Unicast => true, + AnnounceSafi::Ipv6Unicast => peer.ipv6_unicast, + AnnounceSafi::FlowSpecV4 => peer.flowspec_v4, + AnnounceSafi::FlowSpecV6 => peer.flowspec_v6, + } +} + +/// Read-only session context for [`established_loop`]: static peer config +/// plus the C3 capability-gating inputs (the peer's negotiated SAFIs and the +/// shared skip counters). Grouped into a single parameter so +/// `established_loop` stays under clippy's `too_many_arguments` limit. +struct EstablishedCtx<'a> { + cfg: &'a PeerConfig, + peer_open: &'a OpenMsg, + unnegotiated_skipped: &'a UnnegotiatedSkipCounters, +} + +/// A snapshot of unnegotiated-SAFI announce/withdraw-skip counts (C3), for +/// the `/metrics` endpoint (`blackwall_bgp_unnegotiated_announce_skipped_total{safi}`). +/// The metric name predates the withdraw-side gate (C3 follow-up) but its +/// semantics — "an UPDATE for this SAFI was skipped because the peer never +/// negotiated it" — cover both directions, so it is reused rather than +/// split. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct UnnegotiatedSkipCounts { + /// FlowSpec v4 (AFI 1 / SAFI 133) announces/withdraws skipped because the + /// peer never negotiated that SAFI. + pub flowspec_v4: u64, + /// FlowSpec v6 (AFI 2 / SAFI 133) announces/withdraws skipped. + pub flowspec_v6: u64, + /// IPv6 unicast (AFI 2 / SAFI 1) announces/withdraws skipped. + pub ipv6_unicast: u64, +} + +/// Live counters backing [`UnnegotiatedSkipCounts`]; shared between the +/// session task (writer) and [`BgpHandle`] (reader, for `/metrics`). +/// +/// `pub(crate)` (rather than private) because [`run`] — itself `pub` for +/// [`spawn`] to call across the module boundary — takes an `Arc` of this type +/// as a parameter, and Rust requires a `pub` item's signature to only +/// reference types at least as visible as the item itself. +#[derive(Debug, Default)] +pub(crate) struct UnnegotiatedSkipCounters { + flowspec_v4: AtomicU64, + flowspec_v6: AtomicU64, + ipv6_unicast: AtomicU64, +} + +impl UnnegotiatedSkipCounters { + /// Record one skipped announce or withdraw for `safi`. `Ipv4Unicast` is + /// never gated (see [`announce_allowed`]) and is intentionally not + /// counted. + fn record(&self, safi: AnnounceSafi) { + let counter = match safi { + AnnounceSafi::FlowSpecV4 => &self.flowspec_v4, + AnnounceSafi::FlowSpecV6 => &self.flowspec_v6, + AnnounceSafi::Ipv6Unicast => &self.ipv6_unicast, + AnnounceSafi::Ipv4Unicast => return, + }; + counter.fetch_add(1, Ordering::Relaxed); + } + + fn snapshot(&self) -> UnnegotiatedSkipCounts { + UnnegotiatedSkipCounts { + flowspec_v4: self.flowspec_v4.load(Ordering::Relaxed), + flowspec_v6: self.flowspec_v6.load(Ordering::Relaxed), + ipv6_unicast: self.ipv6_unicast.load(Ordering::Relaxed), + } + } +} + impl PeerConfig { /// Validate the configuration for an iBGP-injection session. /// @@ -161,6 +294,7 @@ pub struct BgpHandle { tx: mpsc::Sender, state: watch::Receiver, reconnects: Arc, + unnegotiated_skipped: Arc, } impl BgpHandle { @@ -183,6 +317,15 @@ impl BgpHandle { self.reconnects.load(Ordering::Relaxed) } + /// FlowSpec/IPv6 announces or withdraws skipped because the peer never + /// negotiated the required SAFI in its OPEN (C3, C3 follow-up) — see + /// [`announce_allowed`]. For the `/metrics` endpoint's + /// `blackwall_bgp_unnegotiated_announce_skipped_total{safi}`. + #[must_use] + pub fn unnegotiated_skip_counts(&self) -> UnnegotiatedSkipCounts { + self.unnegotiated_skipped.snapshot() + } + /// Announce a route to the BGP peer. /// /// The route is stored in the session's active set and re-announced on @@ -270,12 +413,14 @@ pub fn spawn(cfg: PeerConfig) -> Result<(BgpHandle, tokio::task::JoinHandle<()>) let (tx, rx) = mpsc::channel(256); let (state_tx, state_rx) = watch::channel(SessionState::Idle); let reconnects = Arc::new(AtomicU64::new(0)); + let unnegotiated_skipped = Arc::new(UnnegotiatedSkipCounters::default()); let handle = BgpHandle { tx, state: state_rx, reconnects: Arc::clone(&reconnects), + unnegotiated_skipped: Arc::clone(&unnegotiated_skipped), }; - let join = tokio::spawn(run(cfg, rx, state_tx, reconnects)); + let join = tokio::spawn(run(cfg, rx, state_tx, reconnects, unnegotiated_skipped)); Ok((handle, join)) } @@ -291,6 +436,7 @@ pub async fn run( mut commands: mpsc::Receiver, state: watch::Sender, reconnects: Arc, + unnegotiated_skipped: Arc, ) { let mut active: HashMap = HashMap::new(); let mut active_flowspec: HashMap, FlowSpecRule> = HashMap::new(); @@ -307,6 +453,7 @@ pub async fn run( &mut active_flowspec, &state, &mut consecutive_failures, + &unnegotiated_skipped, ) .await; match outcome { @@ -390,6 +537,7 @@ async fn session_once( active_flowspec: &mut HashMap, FlowSpecRule>, state: &watch::Sender, consecutive_failures: &mut u32, + unnegotiated_skipped: &UnnegotiatedSkipCounters, ) -> SessionOutcome { // ── 1. TCP connect ────────────────────────────────────────────────────── let mut stream = match connect_peer( @@ -573,33 +721,68 @@ async fn session_once( let _ = state.send(SessionState::Established); // ── 5. Re-announce the full active set ────────────────────────────────── + // C3: gate on the peer's negotiated SAFIs — `active`/`active_flowspec` may + // hold routes/rules queued while disconnected (or negotiated with a + // *different* prior session), so this reconnect path is just as capable + // of sending a session-resetting UPDATE as the live command arms below. + let mut skipped_routes = 0usize; for route in active.values() { + let safi = AnnounceSafi::for_route(&route.prefix); + if !announce_allowed(&peer_open, safi) { + unnegotiated_skipped.record(safi); + skipped_routes += 1; + warn!( + peer = %cfg.peer_addr, prefix = %route.prefix, safi = safi.label(), + "skipping re-announce: peer did not negotiate this SAFI" + ); + continue; + } let pkt = build_announce(route); if let Err(e) = stream.write_all(&pkt).await { return SessionOutcome::Reconnect(format!("re-announce write failed: {e}")); } } - if !active.is_empty() { - debug!(peer = %cfg.peer_addr, count = active.len(), "re-announced active routes"); + if active.len() > skipped_routes { + debug!( + peer = %cfg.peer_addr, + count = active.len() - skipped_routes, + "re-announced active routes" + ); } + let mut skipped_flowspec = 0usize; for rule in active_flowspec.values() { + let safi = AnnounceSafi::for_flowspec(&rule.dst); + if !announce_allowed(&peer_open, safi) { + unnegotiated_skipped.record(safi); + skipped_flowspec += 1; + warn!( + peer = %cfg.peer_addr, dst = %rule.dst, safi = safi.label(), + "skipping FlowSpec re-announce: peer did not negotiate this SAFI" + ); + continue; + } let pkt = build_flowspec_announce(rule); if let Err(e) = stream.write_all(&pkt).await { return SessionOutcome::Reconnect(format!("FlowSpec re-announce write failed: {e}")); } } - if !active_flowspec.is_empty() { + if active_flowspec.len() > skipped_flowspec { debug!( peer = %cfg.peer_addr, - count = active_flowspec.len(), + count = active_flowspec.len() - skipped_flowspec, "re-announced active FlowSpec rules" ); } // ── 6. Established event loop ─────────────────────────────────────────── - established_loop( + let ctx = EstablishedCtx { cfg, + peer_open: &peer_open, + unnegotiated_skipped, + }; + established_loop( + &ctx, commands, active, active_flowspec, @@ -701,7 +884,7 @@ async fn await_peer_confirmation( /// commands. Returns when the session needs to reconnect or the command /// channel closes. async fn established_loop( - cfg: &PeerConfig, + ctx: &EstablishedCtx<'_>, commands: &mut mpsc::Receiver, active: &mut HashMap, active_flowspec: &mut HashMap, FlowSpecRule>, @@ -709,6 +892,9 @@ async fn established_loop( buf: &mut Vec, hold_secs: u16, ) -> SessionOutcome { + let cfg = ctx.cfg; + let peer_open = ctx.peer_open; + let unnegotiated_skipped = ctx.unnegotiated_skipped; // Fix 3: when hold_secs == 0 (RFC 4271: no keepalive/hold timers), park the // keepalive arm on `pending()` so we never send unsolicited KEEPALIVEs. // When hold_secs > 0, keepalive every hold/3 (min 1 s). RFC 4271 §6.7. @@ -849,46 +1035,97 @@ async fn established_loop( } Some(SessionCommand::Announce(route)) => { let prefix = route.prefix; - let pkt = build_announce(&route); - active.insert(prefix, route); - if let Err(e) = stream.write_all(&pkt).await { - return SessionOutcome::Reconnect( - format!("announce write failed: {e}") + // C3: gate IPv6-unicast announces on the peer's negotiated + // SAFI (IPv4 unicast is unconditionally allowed). `active` + // still tracks the route so a later reconnect (to a peer + // that *does* negotiate it) re-announces it. + let safi = AnnounceSafi::for_route(&prefix); + if !announce_allowed(peer_open, safi) { + unnegotiated_skipped.record(safi); + active.insert(prefix, route); + warn!( + peer = %cfg.peer_addr, %prefix, safi = safi.label(), + "skipping announce: peer did not negotiate this SAFI" ); + } else { + let pkt = build_announce(&route); + active.insert(prefix, route); + if let Err(e) = stream.write_all(&pkt).await { + return SessionOutcome::Reconnect( + format!("announce write failed: {e}") + ); + } + debug!(peer = %cfg.peer_addr, %prefix, "announced"); } - debug!(peer = %cfg.peer_addr, %prefix, "announced"); } Some(SessionCommand::Withdraw(prefix)) => { + // C3 follow-up: gate withdraws the same way announces are + // gated — MP_UNREACH_NLRI for a SAFI the peer never + // negotiated is just as session-reset-worthy as MP_REACH. active.remove(&prefix); - let pkt = build_withdraw(&prefix); - if let Err(e) = stream.write_all(&pkt).await { - return SessionOutcome::Reconnect( - format!("withdraw write failed: {e}") + let safi = AnnounceSafi::for_route(&prefix); + if !announce_allowed(peer_open, safi) { + unnegotiated_skipped.record(safi); + warn!( + peer = %cfg.peer_addr, %prefix, safi = safi.label(), + "skipping withdraw: peer did not negotiate this SAFI" ); + } else { + let pkt = build_withdraw(&prefix); + if let Err(e) = stream.write_all(&pkt).await { + return SessionOutcome::Reconnect( + format!("withdraw write failed: {e}") + ); + } + debug!(peer = %cfg.peer_addr, %prefix, "withdrawn"); } - debug!(peer = %cfg.peer_addr, %prefix, "withdrawn"); } Some(SessionCommand::AnnounceFlowSpec(rule)) => { let key = crate::flowspec::encode_flowspec_nlri(&rule); - let pkt = build_flowspec_announce(&rule); - active_flowspec.insert(key, rule); - if let Err(e) = stream.write_all(&pkt).await { - return SessionOutcome::Reconnect( - format!("FlowSpec announce write failed: {e}") + // C3: gate on the peer's negotiated FlowSpec SAFI (133) for + // this rule's address family. `active_flowspec` still tracks + // the rule so a later reconnect can re-announce it. + let safi = AnnounceSafi::for_flowspec(&rule.dst); + if !announce_allowed(peer_open, safi) { + unnegotiated_skipped.record(safi); + let dst = rule.dst; + active_flowspec.insert(key, rule); + warn!( + peer = %cfg.peer_addr, %dst, safi = safi.label(), + "skipping FlowSpec announce: peer did not negotiate this SAFI" ); + } else { + let pkt = build_flowspec_announce(&rule); + active_flowspec.insert(key, rule); + if let Err(e) = stream.write_all(&pkt).await { + return SessionOutcome::Reconnect( + format!("FlowSpec announce write failed: {e}") + ); + } + debug!(peer = %cfg.peer_addr, "FlowSpec rule announced"); } - debug!(peer = %cfg.peer_addr, "FlowSpec rule announced"); } Some(SessionCommand::WithdrawFlowSpec(rule)) => { + // C3 follow-up: same gate as AnnounceFlowSpec — a + // withdraw is still an MP_UNREACH_NLRI for SAFI 133. let key = crate::flowspec::encode_flowspec_nlri(&rule); active_flowspec.remove(&key); - let pkt = build_flowspec_withdraw(&rule); - if let Err(e) = stream.write_all(&pkt).await { - return SessionOutcome::Reconnect( - format!("FlowSpec withdraw write failed: {e}") + let safi = AnnounceSafi::for_flowspec(&rule.dst); + if !announce_allowed(peer_open, safi) { + unnegotiated_skipped.record(safi); + warn!( + peer = %cfg.peer_addr, dst = %rule.dst, safi = safi.label(), + "skipping FlowSpec withdraw: peer did not negotiate this SAFI" ); + } else { + let pkt = build_flowspec_withdraw(&rule); + if let Err(e) = stream.write_all(&pkt).await { + return SessionOutcome::Reconnect( + format!("FlowSpec withdraw write failed: {e}") + ); + } + debug!(peer = %cfg.peer_addr, "FlowSpec rule withdrawn"); } - debug!(peer = %cfg.peer_addr, "FlowSpec rule withdrawn"); } } } @@ -1236,4 +1473,260 @@ mod tests { std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST) ); } + + // ── C3: capability-gated announces ────────────────────────────────────── + + /// A minimal `OpenMsg` with every AFI/SAFI disabled, for tests to + /// selectively enable one field at a time. + fn peer_stub() -> OpenMsg { + OpenMsg { + asn: 65001, + hold_time: 90, + router_id: 0x0A00_0001, + ipv4_unicast: true, + ipv6_unicast: false, + flowspec_v4: false, + flowspec_v6: false, + } + } + + #[test] + fn ipv4_unicast_is_always_allowed() { + // Even a peer that negotiated nothing else must still get IPv4 unicast. + assert!(announce_allowed(&peer_stub(), AnnounceSafi::Ipv4Unicast)); + } + + #[test] + fn flowspec_announce_skipped_when_peer_did_not_negotiate_safi_133() { + let peer = peer_stub(); + assert!(!announce_allowed(&peer, AnnounceSafi::FlowSpecV4)); + let peer2 = OpenMsg { + flowspec_v4: true, + ..peer + }; + assert!(announce_allowed(&peer2, AnnounceSafi::FlowSpecV4)); + } + + #[test] + fn flowspec_v6_gates_independently_of_flowspec_v4() { + let mut peer = peer_stub(); + peer.flowspec_v4 = true; + // v4 negotiated, v6 not — a v6 FlowSpec announce must still be gated. + assert!(!announce_allowed(&peer, AnnounceSafi::FlowSpecV6)); + peer.flowspec_v6 = true; + assert!(announce_allowed(&peer, AnnounceSafi::FlowSpecV6)); + } + + #[test] + fn ipv6_unicast_gates_on_peer_capability() { + let peer = peer_stub(); + assert!(!announce_allowed(&peer, AnnounceSafi::Ipv6Unicast)); + let peer2 = OpenMsg { + ipv6_unicast: true, + ..peer + }; + assert!(announce_allowed(&peer2, AnnounceSafi::Ipv6Unicast)); + } + + #[test] + fn announce_safi_for_route_dispatches_on_address_family() { + let v4: IpNet = "10.0.0.1/32".parse().unwrap(); + let v6: IpNet = "2001:db8::1/128".parse().unwrap(); + assert_eq!(AnnounceSafi::for_route(&v4), AnnounceSafi::Ipv4Unicast); + assert_eq!(AnnounceSafi::for_route(&v6), AnnounceSafi::Ipv6Unicast); + } + + #[test] + fn announce_safi_for_flowspec_dispatches_on_address_family() { + let v4: IpNet = "10.0.0.0/24".parse().unwrap(); + let v6: IpNet = "2001:db8::/32".parse().unwrap(); + assert_eq!(AnnounceSafi::for_flowspec(&v4), AnnounceSafi::FlowSpecV4); + assert_eq!(AnnounceSafi::for_flowspec(&v6), AnnounceSafi::FlowSpecV6); + } + + #[test] + fn unnegotiated_skip_counters_record_and_snapshot_per_safi() { + let counters = UnnegotiatedSkipCounters::default(); + counters.record(AnnounceSafi::FlowSpecV4); + counters.record(AnnounceSafi::FlowSpecV4); + counters.record(AnnounceSafi::FlowSpecV6); + counters.record(AnnounceSafi::Ipv6Unicast); + // Ipv4Unicast is never gated in practice; recording it is a no-op. + counters.record(AnnounceSafi::Ipv4Unicast); + + let snap = counters.snapshot(); + assert_eq!( + snap, + UnnegotiatedSkipCounts { + flowspec_v4: 2, + flowspec_v6: 1, + ipv6_unicast: 1, + } + ); + } + + // ── C3 follow-up: capability-gated withdraws ──────────────────────────── + + use crate::{FlowAction, Origin}; + + /// Drives [`established_loop`] over a real loopback TCP pair with exactly + /// one queued command, then returns the bytes written to the peer side + /// plus the final unnegotiated-skip snapshot. + /// + /// `hold_secs = 0` parks the keepalive/hold `select!` arms on `pending()`, + /// and dropping the command sender right after queuing ends the loop via + /// `CommandsExhausted` on the next iteration — so this needs no sleeps or + /// timeouts to bound `established_loop` itself. + async fn run_one_command( + peer_open: &OpenMsg, + active: &mut HashMap, + active_flowspec: &mut HashMap, FlowSpecRule>, + command: SessionCommand, + ) -> (Vec, UnnegotiatedSkipCounts) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind loopback listener"); + let addr = listener.local_addr().expect("listener local_addr"); + let (mut server, mut client) = tokio::join!( + async { listener.accept().await.expect("accept").0 }, + async { TcpStream::connect(addr).await.expect("connect") } + ); + + let (tx, mut rx) = mpsc::channel(4); + tx.send(command).await.expect("queue command"); + drop(tx); + + let peer_cfg = cfg(65001, 65001, 0); + let unnegotiated_skipped = UnnegotiatedSkipCounters::default(); + let ctx = EstablishedCtx { + cfg: &peer_cfg, + peer_open, + unnegotiated_skipped: &unnegotiated_skipped, + }; + let mut buf = Vec::new(); + let outcome = established_loop( + &ctx, + &mut rx, + active, + active_flowspec, + &mut client, + &mut buf, + 0, + ) + .await; + match outcome { + SessionOutcome::CommandsExhausted => {} + SessionOutcome::Reconnect(reason) => panic!("unexpected reconnect: {reason}"), + } + + // Bound the read: a skip writes nothing, so the server side never + // sees data or EOF — a short timeout distinguishes "nothing was ever + // sent" from a hang, without racing the (already-awaited) write. + let mut received = Vec::new(); + let mut tmp = [0u8; 256]; + while let Ok(Ok(n)) = + tokio::time::timeout(Duration::from_millis(50), server.read(&mut tmp)).await + { + if n == 0 { + break; + } + received.extend_from_slice(&tmp[..n]); + } + (received, unnegotiated_skipped.snapshot()) + } + + fn flowspec_rule() -> FlowSpecRule { + FlowSpecRule { + dst: "203.0.113.0/24".parse().unwrap(), + protocol: Some(17), + dst_port: Some(53), + action: FlowAction::TrafficRate(0.0), + } + } + + #[tokio::test] + async fn flowspec_withdraw_skipped_when_peer_did_not_negotiate_safi_133() { + let peer_open = peer_stub(); // flowspec_v4 = false + let mut active = HashMap::new(); + let mut active_flowspec = HashMap::new(); + let rule = flowspec_rule(); + active_flowspec.insert(crate::flowspec::encode_flowspec_nlri(&rule), rule.clone()); + + let (received, counts) = run_one_command( + &peer_open, + &mut active, + &mut active_flowspec, + SessionCommand::WithdrawFlowSpec(rule), + ) + .await; + + assert!( + received.is_empty(), + "no MP_UNREACH bytes should reach a peer that never negotiated FlowSpec v4" + ); + assert_eq!(counts.flowspec_v4, 1); + assert!( + active_flowspec.is_empty(), + "withdrawn rule still drops from the active set" + ); + } + + #[tokio::test] + async fn ipv6_withdraw_skipped_when_peer_did_not_negotiate_ipv6_unicast() { + let peer_open = peer_stub(); // ipv6_unicast = false + let mut active = HashMap::new(); + let mut active_flowspec = HashMap::new(); + let prefix: IpNet = "2001:db8::1/128".parse().unwrap(); + active.insert( + prefix, + Route { + prefix, + next_hop: "2001:db8::ffff".parse().unwrap(), + origin: Origin::Igp, + communities: vec![], + large_communities: vec![], + }, + ); + + let (received, counts) = run_one_command( + &peer_open, + &mut active, + &mut active_flowspec, + SessionCommand::Withdraw(prefix), + ) + .await; + + assert!( + received.is_empty(), + "no MP_UNREACH bytes should reach a peer that never negotiated IPv6 unicast" + ); + assert_eq!(counts.ipv6_unicast, 1); + assert!( + active.is_empty(), + "withdrawn prefix still drops from the active set" + ); + } + + #[tokio::test] + async fn flowspec_withdraw_written_when_negotiated() { + // Non-regression: a peer that *did* negotiate FlowSpec v4 must still + // get the withdraw on the wire, unaffected by the new gate. + let mut peer_open = peer_stub(); + peer_open.flowspec_v4 = true; + let mut active = HashMap::new(); + let mut active_flowspec = HashMap::new(); + let rule = flowspec_rule(); + active_flowspec.insert(crate::flowspec::encode_flowspec_nlri(&rule), rule.clone()); + + let (received, counts) = run_one_command( + &peer_open, + &mut active, + &mut active_flowspec, + SessionCommand::WithdrawFlowSpec(rule.clone()), + ) + .await; + + assert_eq!(received, build_flowspec_withdraw(&rule)); + assert_eq!(counts.flowspec_v4, 0); + } } diff --git a/crates/blackwall-config/src/parser.rs b/crates/blackwall-config/src/parser.rs index 3bd1967..c17beb2 100644 --- a/crates/blackwall-config/src/parser.rs +++ b/crates/blackwall-config/src/parser.rs @@ -29,6 +29,7 @@ pub fn parse(lines: &[Line]) -> Result { let mut stateless_tcp_ports: Vec = Vec::new(); let mut pops: Vec = Vec::new(); let mut shadow = false; + let mut protected_prefixes: Vec = Vec::new(); let mut i = 0; while i < lines.len() { @@ -43,6 +44,10 @@ pub fn parse(lines: &[Line]) -> Result { expect_len(line, 2, " ")?; prefixes.push(parse_cidr(line, &line.words[1])?); } + "protect" => { + expect_len(line, 2, "protect ")?; + protected_prefixes.push(parse_cidr(line, &line.words[1])?); + } "default" => { expect_len(line, 2, "default deception|drop")?; default_state = match line.words[1].as_str() { @@ -222,6 +227,7 @@ pub fn parse(lines: &[Line]) -> Result { | "md5" | "gtsm-hops" | "local-addr" + | "max-new-per-min" ) { return Err(ConfigError::BadValue { line: line.number, @@ -321,6 +327,16 @@ pub fn parse(lines: &[Line]) -> Result { } None => None, }; + // C6: cross-plane rate cap on new mitigations. Absent => + // `None` => unlimited (today's behavior, non-breaking). + // `0` is accepted (rejects every new announce) rather than + // treated as "unset" — an operator may deliberately want to + // freeze new arming while still allowing existing + // blackholes/rules and withdraws to proceed. + let max_new_per_min: Option = match kv.get("max-new-per-min") { + Some(v) => Some(v.parse().map_err(|_| bad("rtbh max-new-per-min", v))?), + None => None, + }; rtbh = Some(RtbhPolicy { local_asn, peer_asn, @@ -340,6 +356,7 @@ pub fn parse(lines: &[Line]) -> Result { .get("local-addr") .map(|v| v.parse().map_err(|_| bad("local-addr", v))) .transpose()?, + max_new_per_min, }); } "flowspec" => { @@ -833,6 +850,7 @@ pub fn parse(lines: &[Line]) -> Result { xdp, stateless_tcp_ports, shadow, + protected_prefixes, }) } @@ -1633,6 +1651,34 @@ tenant t { assert_eq!(p.rtbh.unwrap().local_addr, None); } + #[test] + fn rtbh_max_new_per_min_defaults_none_unlimited() { + let p = parse_text( + "interface wan eth0\nipv4 203.0.113.0/24\nrtbh peer=10.0.0.2:179 local-as=65000 peer-as=65000 router-id=10.0.0.1 next-hop-v4=192.0.2.1 max=8 hold-down=60s\n", + ).unwrap(); + assert_eq!( + p.rtbh.unwrap().max_new_per_min, + None, + "absent max-new-per-min must be unlimited (non-breaking)" + ); + } + + #[test] + fn rtbh_parses_max_new_per_min() { + let p = parse_text( + "interface wan eth0\nipv4 203.0.113.0/24\nrtbh peer=10.0.0.2:179 local-as=65000 peer-as=65000 router-id=10.0.0.1 next-hop-v4=192.0.2.1 max=8 hold-down=60s max-new-per-min=30\n", + ).unwrap(); + assert_eq!(p.rtbh.unwrap().max_new_per_min, Some(30)); + } + + #[test] + fn rtbh_rejects_bad_max_new_per_min() { + let err = parse_text( + "interface wan eth0\nipv4 203.0.113.0/24\nrtbh peer=10.0.0.2:179 local-as=65000 peer-as=65000 router-id=10.0.0.1 next-hop-v4=192.0.2.1 max=8 hold-down=60s max-new-per-min=notanumber\n", + ); + assert!(err.is_err()); + } + #[test] fn rtbh_rejects_bad_local_addr() { let err = parse_text( @@ -2261,4 +2307,23 @@ flowspec concentration=0.8 max-flows=4 rate=0 max-rules=256 hold-down=60s bogus= assert!(e.to_string().contains("line 7")); assert!(e.to_string().contains("xyz")); } + + #[test] + fn parses_protect_prefixes() { + let p = parse_text( + "interface wan eth0\nipv4 203.0.113.0/24\nprotect 203.0.113.53/32\nprotect 2001:db8::53/128\n", + ).unwrap(); + assert_eq!( + p.protected_prefixes, + vec![ + "203.0.113.53/32".parse::().unwrap(), + "2001:db8::53/128".parse::().unwrap(), + ] + ); + } + #[test] + fn protect_defaults_empty() { + let p = parse_text("interface wan eth0\nipv4 203.0.113.0/24\n").unwrap(); + assert!(p.protected_prefixes.is_empty()); + } } diff --git a/crates/blackwall-core/src/policy.rs b/crates/blackwall-core/src/policy.rs index caa9e16..8d6246c 100644 --- a/crates/blackwall-core/src/policy.rs +++ b/crates/blackwall-core/src/policy.rs @@ -77,4 +77,8 @@ pub struct Policy { /// Shadow mode: log + record + meter mitigations (RTBH/FlowSpec/XDP) /// without applying them. `false` (the default) is live. pub shadow: bool, + /// Prefixes Blackwall must never mitigate against (own anycast VIPs and + /// similar always-safe destinations), set via the repeatable `protect` + /// directive. Empty (the default) protects nothing extra. + pub protected_prefixes: Vec, } diff --git a/crates/blackwall-core/src/resolve.rs b/crates/blackwall-core/src/resolve.rs index 2ab78ab..88956d1 100644 --- a/crates/blackwall-core/src/resolve.rs +++ b/crates/blackwall-core/src/resolve.rs @@ -139,6 +139,7 @@ mod tests { flowtable: None, xdp: None, stateless_tcp_ports: Vec::new(), + protected_prefixes: Vec::new(), shadow: false, } } @@ -305,6 +306,7 @@ mod tests { flowtable: None, xdp: None, stateless_tcp_ports: Vec::new(), + protected_prefixes: Vec::new(), shadow: false, }; let resolved = policy.resolve().expect("empty policy resolves"); diff --git a/crates/blackwall-core/src/rtbh.rs b/crates/blackwall-core/src/rtbh.rs index 1e80004..33ec6f5 100644 --- a/crates/blackwall-core/src/rtbh.rs +++ b/crates/blackwall-core/src/rtbh.rs @@ -40,6 +40,14 @@ pub struct RtbhPolicy { /// source and emitted as BIRD's `neighbor`. `None` = OS-chosen source (no /// generated BIRD session possible). Its family should match `peer_addr`. pub local_addr: Option, + /// Cross-plane cap (C6) on how many NEW mitigations (BGP announces) may + /// be armed per rolling 60s window, shared between RTBH and FlowSpec + /// (FlowSpec reuses this block) — a safety ceiling on the *arrival rate* + /// of mitigations, distinct from `max_blackholes`/`FlowSpecPolicy::max_rules` + /// which only bound the steady-state active-set size. `None` (the + /// default; absent `max-new-per-min` key) is unlimited — today's + /// behavior. + pub max_new_per_min: Option, } #[cfg(test)] @@ -61,6 +69,7 @@ mod tests { md5: Some(crate::Md5Secret::new("pw".into())), gtsm_hops: Some(1), local_addr: Some("10.222.255.2".parse().unwrap()), + max_new_per_min: Some(60), }; let json = serde_json::to_string(&p).unwrap(); let back: RtbhPolicy = serde_json::from_str(&json).unwrap(); diff --git a/crates/blackwall-deception/tests/interop.rs b/crates/blackwall-deception/tests/interop.rs index d344ea0..831d04e 100644 --- a/crates/blackwall-deception/tests/interop.rs +++ b/crates/blackwall-deception/tests/interop.rs @@ -123,6 +123,7 @@ async fn serves_deception_banner() { flowtable: None, xdp: None, stateless_tcp_ports: Vec::new(), + protected_prefixes: Vec::new(), shadow: false, }; @@ -190,6 +191,7 @@ async fn serves_deception_under_load() { flowtable: None, xdp: None, stateless_tcp_ports: Vec::new(), + protected_prefixes: Vec::new(), shadow: false, }; @@ -279,6 +281,7 @@ fn serves_stateless_syn_cookie() { // The stateless-tier port under test (Component 2c wiring): deception // TCP on 8080 is routed to the engine's NFQUEUE instead of tproxy. stateless_tcp_ports: vec![8080], + protected_prefixes: Vec::new(), shadow: false, }; @@ -364,6 +367,7 @@ fn serves_stateless_syn_cookie_v6() { flowtable: None, xdp: None, stateless_tcp_ports: vec![8080], + protected_prefixes: Vec::new(), shadow: false, }; diff --git a/crates/blackwall-discovery/src/reconcile.rs b/crates/blackwall-discovery/src/reconcile.rs index 394eede..cce7d04 100644 --- a/crates/blackwall-discovery/src/reconcile.rs +++ b/crates/blackwall-discovery/src/reconcile.rs @@ -131,6 +131,7 @@ mod tests { flowtable: None, xdp: None, stateless_tcp_ports: Vec::new(), + protected_prefixes: Vec::new(), shadow: false, } } diff --git a/crates/blackwall-lab/scenarios/armed-disarm-bird.kdl b/crates/blackwall-lab/scenarios/armed-disarm-bird.kdl new file mode 100644 index 0000000..f867934 --- /dev/null +++ b/crates/blackwall-lab/scenarios/armed-disarm-bird.kdl @@ -0,0 +1,48 @@ +// Armed-mode gate (M1), C5 (in-daemon SIGUSR1 disarm): with an RTBH +// blackhole AND a FlowSpec rule both live in real BIRD's RIB, sending +// SIGUSR1 to the running speaker process must withdraw both — proving +// `RtbhManager::disarm`/`FlowSpecManager::disarm` (Task 9) end to end — while +// the process itself keeps running (a crash on signal receipt would be its +// own, worse failure: it would take every other in-flight mitigation with +// it). The driver installs its own SIGUSR1 handler and calls the same +// `disarm` methods `blackwalld`'s `disarm_signal_task` +// (bin/blackwalld/src/main.rs) calls, standing in for the full daemon +// without needing Postgres. +topology "armed-disarm-bird" { + node "peer" { + daemon "bird" local-as="214806" neighbor-node="speaker" neighbor-as="214806" import="all" passive="yes" flowspec="yes" + } + node "speaker" { + run "speaker" \ + cmd="target/debug/lab-tests/blackwall-rtbh-armed_disarm_interop --ignored --nocapture" \ + env="BW_BGP_PEER={peer.addr}:179" + } + link "peer" "speaker" subnet="10.0.0.0/30" +} + +scenario "sigusr1-withdraws-all-and-keeps-running" { + step wait node="peer" until="bgp-established" timeout="20s" + + // Both mitigations must be live in BIRD's RIB before disarming. + step assert node="peer" cmd="birdc show route 203.0.113.20/32 all" \ + contains="(65535,666)" timeout="15s" + step assert node="peer" cmd="birdc show route table flow4tab" \ + contains="dst 203.0.113.21/32; proto 17; dport 53" timeout="15s" + + // C5: signal the running driver process (it installed its own SIGUSR1 + // handler and stands in for blackwalld here — see the topology comment). + step exec node="speaker" cmd="pkill -USR1 -f blackwall-rtbh-armed_disarm_interop" + + // The process must still be alive after the signal — a crash here would + // be a worse failure than a missed withdraw (it takes every other + // in-flight mitigation down with it). + step assert node="speaker" cmd="pgrep -f blackwall-rtbh-armed_disarm_interop" \ + exit="0" timeout="5s" + + // Both mitigations must now be withdrawn from BIRD's RIB. + step assert node="peer" \ + cmd="birdc show route table flow4tab | grep -c 203.0.113.21/32" \ + equals="0" timeout="15s" + step assert node="peer" cmd="birdc show route 203.0.113.20/32" \ + contains="Network not found" timeout="15s" +} diff --git a/crates/blackwall-lab/scenarios/armed-noneg-flowspec-bird.kdl b/crates/blackwall-lab/scenarios/armed-noneg-flowspec-bird.kdl new file mode 100644 index 0000000..04f1440 --- /dev/null +++ b/crates/blackwall-lab/scenarios/armed-noneg-flowspec-bird.kdl @@ -0,0 +1,39 @@ +// Armed-mode gate (M1), C3 (capability-gated announce, no-reset): a peer +// BIRD2 configured WITHOUT the flow4/flow6 channels never negotiates SAFI +// 133 in its OPEN, so blackwall's `announce_allowed` gate (blackwall-bgp +// commit b2bae52) must skip the FlowSpec announce instead of writing an +// UPDATE for an unnegotiated SAFI — which some peers (and BIRD, confirmed +// live) NOTIFICATION-reset the session over, dropping every other active +// mitigation. This reuses the same `flowspec_interop` driver as +// flowspec-bird.kdl (it always attempts both a covering unicast route and a +// FlowSpec announce); the only difference is this peer's `daemon "bird"` has +// no `flowspec="yes"`, so it never advertises the SAFI to negotiate. +topology "armed-noneg-flowspec-bird" { + node "peer" { + // Deliberately no `flowspec="yes"`: this peer never negotiates SAFI + // 133, so the driver's FlowSpec announce below must be skipped, not + // sent — proving the C3 gate rather than the codec/session path + // flowspec-bird.kdl already covers. + daemon "bird" local-as="214806" neighbor-node="speaker" neighbor-as="214806" import="all" passive="yes" + } + node "speaker" { + run "speaker" \ + cmd="target/debug/lab-tests/blackwall-bgp-flowspec_interop --ignored --nocapture" \ + env="BW_BGP_PEER={peer.addr}:179" + } + link "peer" "speaker" subnet="10.0.0.0/30" +} + +scenario "flowspec-skipped-session-survives" { + step wait node="peer" until="bgp-established" timeout="20s" + + // The covering unicast route is ordinary IPv4 unicast (always + // negotiated) and is unaffected by C3 — it must still land normally. + step assert node="peer" cmd="birdc show route 203.0.113.0/24" \ + contains="203.0.113.0/24" timeout="15s" + + // C3: no NOTIFICATION-reset from the (skipped) FlowSpec announce + // attempt — the session must still be Established. + step assert node="peer" cmd="birdc show protocols peer_speaker" \ + contains="Established" timeout="15s" +} diff --git a/crates/blackwall-lab/scenarios/armed-protect-reannounce-bird.kdl b/crates/blackwall-lab/scenarios/armed-protect-reannounce-bird.kdl new file mode 100644 index 0000000..b61bad3 --- /dev/null +++ b/crates/blackwall-lab/scenarios/armed-protect-reannounce-bird.kdl @@ -0,0 +1,52 @@ +// Armed-mode gate (M1): a native BGP speaker running ARMED (no `shadow`) +// against real BIRD2 proves two of the arming interlocks end to end: +// - C1 (anycast self-protection): a target inside a `protect`ed prefix is +// rejected by the manager before any BGP announce is attempted — it must +// never reach BIRD's RIB — while a normal eligible target in the SAME +// prefix DOES appear (the positive control that rules out "the whole +// session is broken"). +// - C4 (rate re-announce): a FlowSpec rule re-asserted with a changed rate +// re-announces the new action rather than silently keeping the stale +// one. BIRD 2.17 renders the RFC 8955 §7.1 traffic-rate extended +// community as `(generic, 0x80060000, )`; the +// 1,000,000.0 bytes/sec f32 used here is `0x49742400` (confirmed live +// against BIRD 2.17.1 — `0.0` renders as `0x0`). +// +// Same RFC 8955 §6 "safe update" validation gotcha as flowspec-bird.kdl: +// resolved by the driver announcing a covering 203.0.113.0/24 unicast route +// first, so BIRD's validator finds a same-peer covering route for all three +// FlowSpec targets (.7, .8, .9) below. +topology "armed-protect-reannounce-bird" { + node "peer" { + daemon "bird" local-as="214806" neighbor-node="speaker" neighbor-as="214806" import="all" passive="yes" flowspec="yes" + } + node "speaker" { + run "speaker" \ + cmd="target/debug/lab-tests/blackwall-rtbh-armed_protect_reannounce_interop --ignored --nocapture" \ + env="BW_BGP_PEER={peer.addr}:179" + } + link "peer" "speaker" subnet="10.0.0.0/30" +} + +scenario "protected-skip-and-rate-reannounce" { + step wait node="peer" until="bgp-established" timeout="20s" + + // Positive control: the normal eligible target (.7) IS announced. + step assert node="peer" cmd="birdc show route table flow4tab" \ + contains="dst 203.0.113.7/32; proto 17; dport 53" timeout="15s" + + // C1: the protected target (.9) never reaches BIRD's RIB — apply_add + // returned Rejected before any BGP announce was attempted. `grep -c` + // over flow4tab counting zero matches is the "no route" assertion the + // matcher's contains/equals/exit vocabulary can express. + step assert node="peer" \ + cmd="birdc show route table flow4tab | grep -c 203.0.113.9/32" \ + equals="0" timeout="15s" + + // C4: the re-asserted rule (.8) shows its NEW rate's extended community + // in BIRD's RIB, proving the re-announce actually happened. + step assert node="peer" cmd="birdc show route table flow4tab all" \ + contains="dst 203.0.113.8/32; proto 17; dport 53" timeout="15s" + step assert node="peer" cmd="birdc show route table flow4tab all" \ + contains="0x49742400" timeout="15s" +} diff --git a/crates/blackwall-nft/src/render.rs b/crates/blackwall-nft/src/render.rs index 1ee81d7..17ad0e6 100644 --- a/crates/blackwall-nft/src/render.rs +++ b/crates/blackwall-nft/src/render.rs @@ -722,6 +722,7 @@ mod tests { flowtable: None, xdp: None, stateless_tcp_ports: Vec::new(), + protected_prefixes: Vec::new(), shadow: false, } } @@ -753,6 +754,7 @@ mod tests { flowtable: None, xdp: None, stateless_tcp_ports: Vec::new(), + protected_prefixes: Vec::new(), shadow: false, } } @@ -1260,6 +1262,7 @@ mod tests { flowtable: None, xdp: None, stateless_tcp_ports: Vec::new(), + protected_prefixes: Vec::new(), shadow: false, }; let ruleset = render(&policy).expect("render empty"); diff --git a/crates/blackwall-nft/tests/apply_netns.rs b/crates/blackwall-nft/tests/apply_netns.rs index 9d758c8..46206a1 100644 --- a/crates/blackwall-nft/tests/apply_netns.rs +++ b/crates/blackwall-nft/tests/apply_netns.rs @@ -35,6 +35,7 @@ fn sample() -> Policy { flowtable: None, xdp: None, stateless_tcp_ports: Vec::new(), + protected_prefixes: Vec::new(), shadow: false, } } @@ -93,6 +94,7 @@ fn stale_set_elements_removed_on_second_apply() { flowtable: None, xdp: None, stateless_tcp_ports: Vec::new(), + protected_prefixes: Vec::new(), shadow: false, }; blackwall_nft::apply(&policy_empty).expect("second apply"); diff --git a/crates/blackwall-rtbh/Cargo.toml b/crates/blackwall-rtbh/Cargo.toml index 60a3f43..df7e556 100644 --- a/crates/blackwall-rtbh/Cargo.toml +++ b/crates/blackwall-rtbh/Cargo.toml @@ -15,7 +15,10 @@ async-trait = { workspace = true } thiserror = { workspace = true } [dev-dependencies] -tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "time"] } +# `signal` is needed only by the `armed_disarm_interop` gate driver (C5: it +# installs its own SIGUSR1 handler, mirroring blackwalld's disarm_signal_task) +# — not by any unit test, so it stays scoped to dev-dependencies. +tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "time", "signal"] } [lints] workspace = true diff --git a/crates/blackwall-rtbh/src/controller.rs b/crates/blackwall-rtbh/src/controller.rs index 9b04c07..f182019 100644 --- a/crates/blackwall-rtbh/src/controller.rs +++ b/crates/blackwall-rtbh/src/controller.rs @@ -46,6 +46,10 @@ pub struct RtbhConfig { /// Maximum lifetime of an auto blackhole (hygiene backstop against a dropped /// or missed `Cleared`); `None` disables the TTL. pub max_ttl: Option, + /// Prefixes that must never be blackholed (own anycast VIPs and similar + /// always-safe destinations), from `Policy.protected_prefixes`. Empty + /// (the default) protects nothing extra. + pub protected_prefixes: Vec, } /// A decision the [`RtbhController`] emits for the sink to execute. @@ -69,6 +73,9 @@ pub enum RtbhAction { pub struct RtbhController { config: RtbhConfig, active: HashMap, + /// Count of targets skipped by the protected-prefix guard (see + /// [`Self::protected_skipped`]). + protected_skipped: u64, } impl RtbhController { @@ -78,6 +85,7 @@ impl RtbhController { Self { config, active: HashMap::new(), + protected_skipped: 0, } } @@ -212,6 +220,22 @@ impl RtbhController { .any(|p| p.contains(&target)) } + /// Whether `target` falls inside a configured protected prefix (own + /// anycast VIP or similar always-safe destination that must never be + /// mitigated). + /// + /// Pure accessor over [`RtbhConfig::protected_prefixes`]; mirrors + /// [`Self::is_eligible`] so a caller (e.g. the manager) can classify a + /// rejected `manual_add` without duplicating the controller's + /// self-protection logic. + #[must_use] + pub fn is_protected(&self, target: IpAddr) -> bool { + self.config + .protected_prefixes + .iter() + .any(|p| p.contains(&target)) + } + /// Whether a next-hop is configured for `target`'s address family. /// /// Pure accessor over [`RtbhConfig::next_hop_v4`] / `next_hop_v6`; lets a @@ -225,6 +249,16 @@ impl RtbhController { } } + /// Number of targets skipped because they fell inside a configured + /// [`RtbhConfig::protected_prefixes`] entry (own anycast VIP or similar + /// always-safe destination) — the anycast self-protection guard in + /// [`Self::insert_blackhole`]. Surfaced for `/metrics` + /// (`blackwall_mitigations_protected_skipped_total{plane="rtbh"}`). + #[must_use] + pub fn protected_skipped(&self) -> u64 { + self.protected_skipped + } + fn request_clear(&mut self, target: IpAddr, now: u64) -> Vec { let hold_ms = u64::try_from(self.config.hold_down.as_millis()).unwrap_or(u64::MAX); match self.active.get_mut(&target) { @@ -252,6 +286,19 @@ impl RtbhController { announced_at: u64, origin: BlackholeOrigin, ) -> Vec { + // Anycast self-protection (C1): a protected prefix (own VIP) must + // never be blackholed, even when it also falls inside an eligible + // prefix — checked BEFORE eligibility, and decisive. + if self + .config + .protected_prefixes + .iter() + .any(|p| p.contains(&target)) + { + tracing::warn!(%target, "RTBH: target in a protected prefix; skipping (never mitigate own service)"); + self.protected_skipped = self.protected_skipped.saturating_add(1); + return Vec::new(); + } if !self .config .eligible_prefixes @@ -289,6 +336,25 @@ impl RtbhController { vec![RtbhAction::Announce(route)] } + /// Undo a just-inserted active entry after its BGP announce failed (C2: + /// commit-after-confirm). + /// + /// Removes `target` from the active set and emits nothing — the router + /// never took the route, so there is nothing to withdraw. Must only be + /// called immediately after an [`RtbhAction::Announce`] was returned for + /// `target` (by [`Self::on_event`], [`Self::manual_add`], or + /// [`Self::resume`]): [`Self::insert_blackhole`] emits `Announce` only + /// when it performs a brand-new insert — a re-assertion, an + /// Auto-to-Manual upgrade, an at-cap target, or an ineligible/protected + /// target all return an empty vector instead and never reach this call. + /// So at the point of a failed announce, `active[target]` is guaranteed + /// to still be exactly the entry this call is undoing — never a + /// pre-existing `Manual` blackhole or one touched by anything else in + /// between. + pub fn rollback(&mut self, target: IpAddr) { + self.active.remove(&target); + } + fn host_route(&self, target: IpAddr) -> Option { let next_hop = match target { IpAddr::V4(_) => self.config.next_hop_v4.map(IpAddr::V4), @@ -330,6 +396,7 @@ mod tests { max_blackholes: 2, hold_down: Duration::from_secs(10), max_ttl: None, + protected_prefixes: Vec::new(), } } @@ -628,6 +695,31 @@ mod tests { assert!(c.manual_add(ip("203.0.113.7"), 0).is_empty(), "at cap"); } + #[test] + fn protected_target_is_skipped_even_when_eligible() { + // 203.0.113.53 is inside the eligible /24 but is carved out as a + // protected VIP: it must never be blackholed. + let mut c = RtbhController::new(RtbhConfig { + protected_prefixes: vec![net("203.0.113.53/32")], + ..cfg() + }); + let actions = c.on_event(&DetectionEvent::Opened(det("203.0.113.53")), 1_000); + assert!(actions.is_empty(), "protected VIP must not be blackholed"); + assert!(c.active_blackholes().is_empty()); + assert_eq!(c.protected_skipped(), 1); + } + + #[test] + fn unprotected_eligible_target_still_mitigates() { + let mut c = RtbhController::new(RtbhConfig { + protected_prefixes: vec![net("203.0.113.53/32")], + ..cfg() + }); + let actions = c.on_event(&DetectionEvent::Opened(det("203.0.113.7")), 1_000); + assert!(matches!(actions.as_slice(), [RtbhAction::Announce(_)])); + assert_eq!(c.protected_skipped(), 0); + } + #[test] fn is_eligible_checks_configured_prefixes() { let c = RtbhController::new(cfg()); diff --git a/crates/blackwall-rtbh/src/flowspec_controller.rs b/crates/blackwall-rtbh/src/flowspec_controller.rs index 7043c82..853d17d 100644 --- a/crates/blackwall-rtbh/src/flowspec_controller.rs +++ b/crates/blackwall-rtbh/src/flowspec_controller.rs @@ -24,6 +24,18 @@ struct ActiveEntry { last_activity: u64, origin: BlackholeOrigin, clear_requested_at: Option, + /// The rule this entry held immediately before an in-place update whose + /// `Announce` has not yet been confirmed (C4 rollback support). `None` + /// for a brand-new entry (nothing to revert to — [`FlowSpecController::rollback`] + /// removes it outright on failure). `Some(prior)` for a re-assert or + /// manual-upgrade that changed the action in place: the router still + /// holds `prior` since the new announce was never confirmed, so a + /// failed announce must restore it rather than delete the entry or + /// leave the unconfirmed rule stuck in place. Left stale (harmlessly) + /// after a successful announce — it is only ever read by `rollback`, + /// which is called solely, and immediately, after the very `Announce` + /// that last set this field. + pending_prior: Option, } /// FlowSpec policy configuration. @@ -43,6 +55,10 @@ pub struct FlowSpecConfig { /// Maximum lifetime of an auto rule (hygiene backstop against a dropped /// or missed clear); `None` disables the TTL. pub max_ttl: Option, + /// Prefixes that must never have FlowSpec rules installed against them + /// (own anycast VIPs and similar always-safe destinations), from + /// `Policy.protected_prefixes`. Empty (the default) protects nothing extra. + pub protected_prefixes: Vec, } /// A decision the [`FlowSpecController`] emits for the sink to execute. @@ -68,6 +84,9 @@ pub enum FlowSpecAction { pub struct FlowSpecController { config: FlowSpecConfig, active: HashMap, + /// Count of targets skipped by the protected-prefix guard (see + /// [`Self::protected_skipped`]). + protected_skipped: u64, } impl FlowSpecController { @@ -77,6 +96,7 @@ impl FlowSpecController { Self { config, active: HashMap::new(), + protected_skipped: 0, } } @@ -91,10 +111,14 @@ impl FlowSpecController { /// /// # Returns /// - /// An `Announce` action per newly-installed rule. A rule already active is - /// re-asserted (its pending clear, if any, is cancelled and its TTL anchor - /// refreshed) without emitting a new action. Ineligible targets or a full - /// capacity cap are ignored. + /// An `Announce` action per newly-installed rule. A rule already active + /// (same destination/protocol/port) is re-asserted: its pending clear, if + /// any, is cancelled and its TTL anchor refreshed. If the incoming rule's + /// action differs from the stored one (e.g. tightening a rate-limit to a + /// full drop mid-attack), the stored rule is updated and a fresh + /// `Announce` is emitted (C4: `FlowKey` excludes the action, so identity + /// alone can't detect this); an unchanged re-assert emits nothing. + /// Ineligible targets or a full capacity cap are ignored. pub fn install( &mut self, target: IpAddr, @@ -163,7 +187,9 @@ impl FlowSpecController { /// /// If a rule with the same `(dst, protocol, dst_port)` is already active as /// `Auto`, this upgrades it to `Manual` (and cancels any pending deferred - /// clear) instead of re-announcing. + /// clear). If the incoming rule's action also differs from the stored one + /// (C4: `FlowKey` excludes the action), the stored rule is updated and a + /// fresh `Announce` is emitted instead of the usual silent upgrade. /// /// # Arguments /// @@ -173,14 +199,24 @@ impl FlowSpecController { /// /// # Returns /// - /// An `Announce` action if newly installed, empty vector if upgraded, - /// ineligible, or at cap. + /// An `Announce` action if newly installed or upgraded with a changed + /// action; empty vector if upgraded with no action change, ineligible, or + /// at cap. pub fn manual_add(&mut self, rule: FlowSpecRule, now: u64) -> Vec { let key = key_of(&rule); if let Some(e) = self.active.get_mut(&key) { // Already active: upgrade to Manual + cancel any pending clear. e.origin = BlackholeOrigin::Manual; e.clear_requested_at = None; + // Same C4 blind spot as `insert_rule`: an operator re-asserting + // with a changed action (e.g. tightening the rate while also + // taking manual ownership) must re-announce, not silently keep + // the stale rate under the new Manual origin. + if e.rule.action != rule.action { + e.pending_prior = Some(e.rule.clone()); + e.rule = rule.clone(); + return vec![FlowSpecAction::Announce(rule)]; + } return Vec::new(); } let target = rule.dst.addr(); @@ -285,6 +321,32 @@ impl FlowSpecController { .any(|p| p.contains(&target)) } + /// Whether `target`'s host route falls inside a configured protected + /// prefix (own anycast VIP or similar always-safe destination that must + /// never be mitigated). + /// + /// Pure accessor over [`FlowSpecConfig::protected_prefixes`]; mirrors + /// [`Self::is_eligible`] so a caller (e.g. the manager) can classify a + /// rejected `manual_add` without duplicating the controller's + /// self-protection logic. + #[must_use] + pub fn is_protected(&self, target: IpAddr) -> bool { + self.config + .protected_prefixes + .iter() + .any(|p| p.contains(&target)) + } + + /// Number of targets skipped because they fell inside a configured + /// [`FlowSpecConfig::protected_prefixes`] entry (own anycast VIP or + /// similar always-safe destination) — the anycast self-protection guard + /// in [`Self::insert_rule`]. Surfaced for `/metrics` + /// (`blackwall_mitigations_protected_skipped_total{plane="flowspec"}`). + #[must_use] + pub fn protected_skipped(&self) -> u64 { + self.protected_skipped + } + fn insert_rule( &mut self, target: IpAddr, @@ -292,6 +354,20 @@ impl FlowSpecController { announced_at: u64, origin: BlackholeOrigin, ) -> Vec { + // Anycast self-protection (C1): a protected prefix (own VIP) must + // never have a FlowSpec rule installed against it, even when it also + // falls inside an eligible prefix — checked BEFORE eligibility, and + // decisive. + if self + .config + .protected_prefixes + .iter() + .any(|p| p.contains(&target)) + { + tracing::warn!(%target, "FlowSpec: target in a protected prefix; skipping (never mitigate own service)"); + self.protected_skipped = self.protected_skipped.saturating_add(1); + return Vec::new(); + } if !self.is_eligible(target) { tracing::warn!(%target, "FlowSpec: target outside eligible prefixes; ignoring"); return Vec::new(); @@ -303,6 +379,18 @@ impl FlowSpecController { // TTL anchor so `tick` does not withdraw a flow under attack again. e.clear_requested_at = None; e.last_activity = announced_at; + // `FlowKey` excludes the traffic-rate action (C4), so a re-assert + // with a DIFFERENT action (e.g. tightening a rate-limit to a full + // drop mid-attack) must update the stored rule and re-announce — + // otherwise the stale rate lingers forever. BGP FlowSpec + // semantics: re-announcing the same NLRI with a new action + // updates the rule in place, so a single fresh `Announce` + // suffices (no explicit withdraw needed). + if e.rule.action != rule.action { + e.pending_prior = Some(e.rule.clone()); + e.rule = rule.clone(); + return vec![FlowSpecAction::Announce(rule)]; + } return Vec::new(); } if self.active.len() >= self.config.max_rules { @@ -317,10 +405,51 @@ impl FlowSpecController { last_activity: announced_at, origin, clear_requested_at: None, + pending_prior: None, }, ); vec![FlowSpecAction::Announce(rule)] } + + /// Undo an unconfirmed active-entry mutation after its BGP announce + /// failed (C2: commit-after-confirm; extended by C4 to updates-in-place). + /// + /// Emits nothing — the router never took the failed announce, so there + /// is nothing to withdraw. Must only be called immediately after a + /// [`FlowSpecAction::Announce`] was returned for `key` (by + /// [`Self::install`], [`Self::manual_add`], or [`Self::resume`]): + /// [`Self::insert_rule`] emits `Announce` only for a brand-new insert or + /// a re-assertion whose action changed (likewise + /// [`Self::manual_add`]'s upgrade branch) — an unchanged re-assertion, an + /// at-cap key, or an ineligible/protected target all return an empty + /// vector instead and never reach this call. So at the point of a failed + /// announce, `active[key]` is guaranteed to still be exactly the entry + /// this call is undoing — never a pre-existing rule or one touched by + /// anything else in between. + /// + /// Two cases, distinguished by [`ActiveEntry::pending_prior`]: + /// * Brand-new insert (`pending_prior` is `None`): the entry is removed + /// outright — it never existed as far as the router is concerned. + /// * Update-in-place (`pending_prior` is `Some(prior)`, set by + /// [`Self::insert_rule`]/[`Self::manual_add`] just before returning the + /// `Announce` this call is undoing): the entry's rule is restored to + /// `prior` rather than deleted — the router still holds `prior` since + /// the new rule was never confirmed, and the target is still under + /// attack. (The entry's `origin`, if flipped `Auto` -> `Manual` by a + /// manual-add upgrade, is left as `Manual`: the operator's intent to + /// own the rule stands independent of whether the accompanying rate + /// change landed.) + /// + /// Mirrors [`crate::controller::RtbhController::rollback`]. + pub fn rollback(&mut self, key: FlowKey) { + if let Some(e) = self.active.get_mut(&key) { + if let Some(prior) = e.pending_prior.take() { + e.rule = prior; + return; + } + } + self.active.remove(&key); + } } /// Derive the `FlowKey` a rule is stored/looked-up under. @@ -344,6 +473,7 @@ mod tests { max_rules: 3, hold_down: Duration::from_secs(10), max_ttl: None, + protected_prefixes: Vec::new(), } } @@ -611,4 +741,125 @@ mod tests { assert!(c.is_eligible(ip("203.0.113.7"))); assert!(!c.is_eligible(ip("198.51.100.7"))); } + + #[test] + fn protected_target_is_skipped_even_when_eligible() { + // 203.0.113.53 is inside the eligible /24 but is carved out as a + // protected VIP: it must never get a FlowSpec rule installed. + let mut c = FlowSpecController::new(FlowSpecConfig { + protected_prefixes: vec![net("203.0.113.53/32")], + ..cfg() + }); + let actions = c.install(ip("203.0.113.53"), &[(17, 53, 1000.0)], 1_000); + assert!(actions.is_empty(), "protected VIP must not get a rule"); + assert!(c.active_rules().is_empty()); + assert_eq!(c.protected_skipped(), 1); + } + + #[test] + fn unprotected_eligible_target_still_mitigates() { + let mut c = FlowSpecController::new(FlowSpecConfig { + protected_prefixes: vec![net("203.0.113.53/32")], + ..cfg() + }); + let actions = c.install(ip("203.0.113.7"), &[(17, 53, 1000.0)], 1_000); + assert_eq!(actions.len(), 1); + assert!(matches!(actions.as_slice(), [FlowSpecAction::Announce(_)])); + assert_eq!(c.protected_skipped(), 0); + } + + #[test] + fn tightening_flowspec_rate_mid_attack_re_announces() { + let mut c = FlowSpecController::new(cfg()); + // install rate-limit + let a1 = c.install(ip("203.0.113.7"), &[(17, 53, 1000.0)], 1_000); + assert!(matches!(a1.as_slice(), [FlowSpecAction::Announce(_)])); + // same (dst,proto,port) but tighten to full drop (rate 0.0): + let a2 = c.install(ip("203.0.113.7"), &[(17, 53, 0.0)], 2_000); + let [FlowSpecAction::Announce(r)] = a2.as_slice() else { + panic!("changed action must re-announce, not no-op: {a2:?}"); + }; + assert_eq!(r.action, FlowAction::TrafficRate(0.0)); + // identical re-assert stays a no-op: + let a3 = c.install(ip("203.0.113.7"), &[(17, 53, 0.0)], 3_000); + assert!(a3.is_empty()); + } + + #[test] + fn manual_add_upgrade_with_changed_action_re_announces() { + // A Manual re-assert that also tightens the rate must re-announce, + // not silently keep the stale rate under the new Manual origin. + let mut c = FlowSpecController::new(cfg()); + assert_eq!( + c.install(ip("203.0.113.7"), &[(17, 53, 1000.0)], 0).len(), + 1 + ); // Auto + let actions = c.manual_add(rule("203.0.113.7/32", 17, 53, 0.0), 1_000); + let [FlowSpecAction::Announce(r)] = actions.as_slice() else { + panic!("changed action on manual upgrade must re-announce: {actions:?}"); + }; + assert_eq!(r.action, FlowAction::TrafficRate(0.0)); + let snap = c.active_rules(); + assert_eq!(snap.len(), 1); + assert_eq!( + snap[0].2, + BlackholeOrigin::Manual, + "still upgraded to Manual" + ); + } + + #[test] + fn rollback_after_changed_action_restores_prior_rule_not_delete() { + // A re-assert that changes the action optimistically updates the + // stored rule before the caller confirms the BGP announce (mirrors + // the fresh-insert C2 pattern). If that announce then fails, the + // router still holds the OLD rule (it never saw the new one), so + // rollback must restore the prior rule rather than deleting the + // entry outright — deleting would forget an attack mitigation that + // is, in fact, still live on the router. + let mut c = FlowSpecController::new(cfg()); + c.install(ip("203.0.113.7"), &[(17, 53, 1000.0)], 1_000); + let a2 = c.install(ip("203.0.113.7"), &[(17, 53, 0.0)], 2_000); + let [FlowSpecAction::Announce(r)] = a2.as_slice() else { + panic!("expected Announce: {a2:?}") + }; + let key = key_of(r); + + c.rollback(key); // simulates the tightened announce failing + + // Past hold-down, the surviving entry must withdraw the PRIOR rule + // (rate 1000.0), proving it was restored rather than left as the + // failed 0.0 rule or dropped entirely. + let withdraw = c.clear_target(ip("203.0.113.7"), 20_000); + let [FlowSpecAction::Withdraw(w)] = withdraw.as_slice() else { + panic!("entry must survive rollback, not vanish: {withdraw:?}") + }; + assert_eq!(w.action, FlowAction::TrafficRate(1000.0)); + } + + #[test] + fn manual_add_rollback_after_changed_action_restores_prior_rule() { + let mut c = FlowSpecController::new(cfg()); + c.install(ip("203.0.113.7"), &[(17, 53, 1000.0)], 0); // Auto + let actions = c.manual_add(rule("203.0.113.7/32", 17, 53, 0.0), 1_000); + let [FlowSpecAction::Announce(r)] = actions.as_slice() else { + panic!("expected Announce: {actions:?}") + }; + let key = key_of(r); + + c.rollback(key); // simulates the tightened manual announce failing + + // manual_remove looks up by (dst, protocol, dst_port) only, so any + // rate in the passed-in rule is irrelevant to the lookup; the + // returned Withdraw carries the actual stored rule. + let withdrawn = c.manual_remove(rule("203.0.113.7/32", 17, 53, 0.0)); + let [FlowSpecAction::Withdraw(w)] = withdrawn.as_slice() else { + panic!("entry must survive rollback, not vanish: {withdrawn:?}") + }; + assert_eq!( + w.action, + FlowAction::TrafficRate(1000.0), + "rollback must restore the prior rule, not the failed one" + ); + } } diff --git a/crates/blackwall-rtbh/src/flowspec_manager.rs b/crates/blackwall-rtbh/src/flowspec_manager.rs index 7494569..6988e7a 100644 --- a/crates/blackwall-rtbh/src/flowspec_manager.rs +++ b/crates/blackwall-rtbh/src/flowspec_manager.rs @@ -15,10 +15,12 @@ use crate::controller::BlackholeOrigin; use crate::flowspec_controller::{key_of, FlowKey, FlowSpecAction, FlowSpecController}; use crate::manager::{ApplyOutcome, BgpExecutor, JournalError}; +use crate::rate_limit::ArmingRateLimiter; use async_trait::async_trait; use blackwall_bgp::FlowSpecRule; use blackwall_flow::FlowRule; use std::net::IpAddr; +use std::sync::{Arc, Mutex}; /// Mirrors FlowSpec rule state into persistent storage. /// @@ -67,18 +69,48 @@ impl MirrorOp { } } +/// Outcome of [`FlowSpecManager::execute_and_journal_announce`]. +/// +/// Mirrors [`crate::manager`]'s private `AnnounceOutcome`. The auto path +/// (`apply_open`/`tick`, via [`FlowSpecManager::execute_and_journal`]) +/// ignores this — auto re-detection naturally compensates for a skip on its +/// next tick. [`FlowSpecManager::apply_add`] (the manual path) consumes it +/// to report a truthful [`ApplyOutcome`] rather than always claiming +/// [`ApplyOutcome::Applied`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AnnounceOutcome { + /// The announce reached BGP; a journal-mirror failure afterward is still + /// `Applied` (self-healed via [`FlowSpecManager::retry_pending_mirror`]) + /// — the live rule is active either way. + Applied, + /// Skipped: the shared cross-plane [`ArmingRateLimiter`] (C6) was at + /// capacity. The controller entry was rolled back. + RateCapped, + /// Skipped: the manager is [`FlowSpecManager::disarm`]ed (C5), + /// record-only. The controller entry was rolled back. + Disarmed, + /// Attempted and failed at the [`BgpExecutor`] (C2). The controller + /// entry was rolled back. + Failed, +} + /// Single-owner FlowSpec manager. /// /// Owns the pure [`FlowSpecController`] plus the I/O boundary: it executes the /// controller's decisions on a [`BgpExecutor`] and mirrors auto/manual rule -/// state via a [`FlowSpecJournal`]. A BGP failure is logged and the action is -/// not journaled — but note this is a known limitation, not a retry -/// mechanism: on a failed first announce the controller entry is kept in -/// memory while the rule itself is never re-announced automatically. A -/// journal failure after a successful BGP operation is logged, never causes a -/// live rule to be withdrawn, and is queued as a `MirrorOp` for a bounded -/// self-heal retry on the next [`FlowSpecManager::tick`] — the BGP outcome is -/// never re-issued, only the mirror write. +/// state via a [`FlowSpecJournal`]. A BGP announce failure is logged, the +/// action is not journaled, and the controller's freshly-inserted active +/// entry is rolled back via [`FlowSpecController::rollback`] (C2: +/// commit-after-confirm) — the control plane never believes an unconfirmed +/// announce succeeded, so a future detection for the same rule is not deduped +/// against a phantom entry. There is no retry queue for this: while the +/// underlying attack persists, the detector naturally re-emits the detection +/// on its next tick and the manager re-attempts through the same path. This +/// differs from a *journal* failure after a successful BGP operation, which +/// is logged, never causes a live rule to be withdrawn, and is queued as a +/// `MirrorOp` for a bounded self-heal retry on the next +/// [`FlowSpecManager::tick`] — the BGP outcome is never re-issued, only the +/// mirror write. pub struct FlowSpecManager { controller: FlowSpecController, bgp: B, @@ -87,6 +119,32 @@ pub struct FlowSpecManager { /// succeeded; retried (never re-issued to BGP) by /// `FlowSpecManager::retry_pending_mirror` on the next tick. pending_mirror: Vec, + /// Count of announces that failed at the BGP executor, each rolled back + /// (see [`Self::apply_failures`]). + apply_failures: u64, + /// Cross-plane cap (C6) on the arrival rate of NEW mitigations, shared + /// with the sibling `RtbhManager` via the same `Arc>` so ONE + /// limiter governs the combined RTBH+FlowSpec announce rate. `None` (the + /// default from [`Self::new`]) is unlimited — `main.rs` only attaches + /// `Some` via [`Self::with_rate_limiter`] on the live path (never under + /// shadow, where nothing is really announced). + rate_limiter: Option>>, + /// Count of announces skipped because [`Self::rate_limiter`] was at + /// capacity (C6) — a SKIP (never attempted), distinct from + /// [`Self::apply_failures`] (attempted and failed at BGP). See + /// [`Self::ratecapped`]. + ratecapped: u64, + /// One-way in-daemon disarm kill switch (C5), flipped by [`Self::disarm`]. + /// While set, [`Self::execute_and_journal_announce`] skips every new + /// `Announce` (never reaches [`Self::bgp`]) while detection + selection + /// keep running unchanged. There is no re-arm entry point; a fresh + /// process (restart) is the only way back to armed. Mirrors + /// [`crate::manager::RtbhManager::disarmed`]. + disarmed: bool, + /// Count of announces skipped because [`Self::disarmed`] was set (C5) — + /// a SKIP (never attempted), distinct from both [`Self::apply_failures`] + /// and [`Self::ratecapped`]. See [`Self::disarmed_skips`]. + disarmed_skips: u64, } impl FlowSpecManager { @@ -97,9 +155,27 @@ impl FlowSpecManager { bgp, journal, pending_mirror: Vec::new(), + apply_failures: 0, + rate_limiter: None, + ratecapped: 0, + disarmed: false, + disarmed_skips: 0, } } + /// Attach a shared cross-plane rate cap (C6) on new mitigations. + /// + /// Non-breaking: absent (the default from [`Self::new`]) is unlimited. + /// `main.rs` wires `Some` only on the live path (`!policy.shadow`) — the + /// shadow-mode construction never calls this, so shadow sessions are + /// always unlimited (rate-capping a mitigation that is never really + /// announced would only corrupt the would-mitigate signal). + #[must_use] + pub fn with_rate_limiter(mut self, limiter: Arc>) -> Self { + self.rate_limiter = Some(limiter); + self + } + /// Install the flow-scoped rules selected for `target` (from a /// [`blackwall_flow::FlowMitigationEvent::Open`]) and execute + journal /// the resulting announces. @@ -123,7 +199,7 @@ impl FlowSpecManager { .collect(); let actions = self.controller.install(target, &tuples, mono_now); for action in actions { - self.execute_and_journal(action, wall_now).await; + self.execute_and_journal(action, mono_now, wall_now).await; } } @@ -134,7 +210,7 @@ impl FlowSpecManager { pub async fn apply_clear(&mut self, target: IpAddr, mono_now: u64, wall_now: u64) { let actions = self.controller.clear_target(target, mono_now); for action in actions { - self.execute_and_journal(action, wall_now).await; + self.execute_and_journal(action, mono_now, wall_now).await; } } @@ -163,7 +239,7 @@ impl FlowSpecManager { self.retry_pending_mirror().await; let actions = self.controller.tick(mono_now); for action in actions { - self.execute_and_journal(action, wall_now).await; + self.execute_and_journal(action, mono_now, wall_now).await; } } @@ -172,8 +248,9 @@ impl FlowSpecManager { /// Returns [`ApplyOutcome::Applied`] if newly installed or upgraded from /// `Auto` to `Manual` (re-journaled as `Manual` in the latter case), /// [`ApplyOutcome::Deferred`] if the manager is at capacity, or - /// [`ApplyOutcome::Rejected`] if the target is ineligible. Unlike RTBH, - /// FlowSpec carries no next-hop, so there is no next-hop rejection case. + /// [`ApplyOutcome::Rejected`] if the target is protected or ineligible. + /// Unlike RTBH, FlowSpec carries no next-hop, so there is no next-hop + /// rejection case. pub async fn apply_add( &mut self, rule: FlowSpecRule, @@ -184,9 +261,22 @@ impl FlowSpecManager { let target = rule.dst.addr(); let actions = self.controller.manual_add(rule.clone(), mono_now); if let Some(FlowSpecAction::Announce(r)) = actions.into_iter().next() { - self.execute_and_journal_announce(r, BlackholeOrigin::Manual, wall_now) + let outcome = self + .execute_and_journal_announce(r, BlackholeOrigin::Manual, mono_now, wall_now) .await; - return ApplyOutcome::Applied; + return match outcome { + AnnounceOutcome::Applied => ApplyOutcome::Applied, + // The window will have room again; the request row stays + // `pending` and is retried next tick. + AnnounceOutcome::RateCapped => ApplyOutcome::Deferred, + // One-way: retrying is pointless until re-armed via restart. + AnnounceOutcome::Disarmed => ApplyOutcome::Rejected(format!( + "{target} was not announced: manager is disarmed (C5)" + )), + // No auto re-detection exists for a manual request, so a + // failed BGP announce must be retried, not marked applied. + AnnounceOutcome::Failed => ApplyOutcome::Deferred, + }; } // Empty result: either already active (upgrade), at cap, or rejected. if self.is_active(key) { @@ -205,6 +295,15 @@ impl FlowSpecManager { } return ApplyOutcome::Applied; } + // Checked before eligibility: a protected target is typically ALSO + // eligible (that's the point — protected VIPs live inside eligible + // prefixes), so it must be rejected outright here rather than falling + // through to Deferred, which would retry forever and never resolve. + if self.controller.is_protected(target) { + return ApplyOutcome::Rejected(format!( + "{target} is inside a protected prefix and is never mitigated" + )); + } if !self.controller.is_eligible(target) { return ApplyOutcome::Rejected(format!("{target} is outside eligible prefixes")); } @@ -212,10 +311,16 @@ impl FlowSpecManager { } /// Manually withdraw a rule (bypasses hold-down). - pub async fn apply_remove(&mut self, rule: FlowSpecRule, wall_now: u64) { + /// + /// `mono_now` is accepted for symmetry with the other entry points that + /// funnel through [`Self::execute_and_journal`] (it is unused here: a + /// manual removal only ever produces a `Withdraw`, never an `Announce`, + /// so the shared rate limiter — which only gates `Announce` — is never + /// consulted on this path). + pub async fn apply_remove(&mut self, rule: FlowSpecRule, mono_now: u64, wall_now: u64) { let actions = self.controller.manual_remove(rule); for action in actions { - self.execute_and_journal(action, wall_now).await; + self.execute_and_journal(action, mono_now, wall_now).await; } } @@ -256,6 +361,90 @@ impl FlowSpecManager { self.controller.active_rules() } + /// Number of targets skipped by the controller's protected-prefix guard + /// (own anycast VIPs never mitigated). Surfaced for `/metrics`; see + /// [`crate::manager::RtbhManager::protected_skipped`] for the analogous + /// RTBH accessor. + #[must_use] + pub fn protected_skipped(&self) -> u64 { + self.controller.protected_skipped() + } + + /// Count of announces that failed at the [`BgpExecutor`] (C2). Each + /// failure rolls back the controller's freshly-inserted active entry + /// (see [`FlowSpecController::rollback`]) so the control plane never + /// believes an unconfirmed announce is active. Surfaced for `/metrics` + /// as `blackwall_flowspec_apply_failures_total`, mirroring + /// [`crate::manager::RtbhManager::apply_failures`]. + #[must_use] + pub fn apply_failures(&self) -> u64 { + self.apply_failures + } + + /// Count of announces skipped because the shared cross-plane + /// [`ArmingRateLimiter`] (C6) was at capacity. Each skip rolls back the + /// controller's freshly-inserted active entry (never left as a phantom + /// active rule) and is distinct from [`Self::apply_failures`] — a + /// rate-cap skip was never attempted at all. Surfaced for `/metrics` as + /// `blackwall_mitigations_ratecapped_total{plane="flowspec"}`. + #[must_use] + pub fn ratecapped(&self) -> u64 { + self.ratecapped + } + + /// In-daemon disarm kill switch (C5): withdraw every currently-active + /// rule and switch to record-only for the rest of this process's life. + /// + /// Mirrors [`crate::manager::RtbhManager::disarm`]: each active rule is + /// withdrawn on BGP best-effort (a withdraw `Err` is logged and the + /// sweep continues), no journal write happens (disarm is runtime-only — + /// a restart re-arms and [`Self::rehydrate`]s the same active set), and + /// once disarmed every subsequent `Announce` is skipped in + /// [`Self::execute_and_journal_announce`] and counted in + /// [`Self::disarmed_skips`]. One-way and idempotent. + /// + /// `mono_now` is accepted for symmetry with the other entry points that + /// funnel through the execute path; it is unused here. + pub async fn disarm(&mut self, _mono_now: u64) { + if self.disarmed { + return; + } + self.disarmed = true; + let keys: Vec = self + .controller + .active_rules() + .into_iter() + .map(|(key, ..)| key) + .collect(); + for key in keys { + // The action on this synthetic rule is never read: `manual_remove` + // looks the entry up by `(dst, protocol, dst_port)` only (see + // `key_of`) and withdraws the *stored* rule, action included. + let placeholder = FlowSpecRule { + dst: key.0, + protocol: Some(key.1), + dst_port: Some(key.2), + action: blackwall_bgp::FlowAction::TrafficRate(0.0), + }; + for action in self.controller.manual_remove(placeholder) { + if let FlowSpecAction::Withdraw(rule) = action { + if let Err(e) = self.bgp.withdraw_flowspec(rule).await { + tracing::warn!(?key, error = %e, "FlowSpec: disarm withdraw failed; continuing best-effort"); + } + } + } + } + tracing::warn!("FlowSpec: DISARMED — mitigations withdrawn, now recording only"); + } + + /// Count of new-mitigation announces skipped because the manager was + /// [`Self::disarm`]ed (C5) — a SKIP (never attempted), distinct from + /// both [`Self::apply_failures`] and [`Self::ratecapped`]. + #[must_use] + pub fn disarmed_skips(&self) -> u64 { + self.disarmed_skips + } + fn is_active(&self, key: FlowKey) -> bool { self.controller .active_rules() @@ -276,10 +465,10 @@ impl FlowSpecManager { } /// Execute one controller action on BGP and mirror it into the journal. - async fn execute_and_journal(&mut self, action: FlowSpecAction, wall_now: u64) { + async fn execute_and_journal(&mut self, action: FlowSpecAction, mono_now: u64, wall_now: u64) { match action { FlowSpecAction::Announce(rule) => { - self.execute_and_journal_announce(rule, BlackholeOrigin::Auto, wall_now) + self.execute_and_journal_announce(rule, BlackholeOrigin::Auto, mono_now, wall_now) .await; } FlowSpecAction::Withdraw(rule) => { @@ -299,16 +488,49 @@ impl FlowSpecManager { } } + /// Execute one NEW-mitigation `Announce` on BGP and mirror it into the + /// journal. + /// + /// First consults the shared [`ArmingRateLimiter`] (C6), if attached: a + /// rejection rolls back the controller's freshly-inserted active entry + /// (same "commit-after-confirm" discipline as a BGP failure, see the + /// module docs) and counts against [`Self::ratecapped`] — never + /// [`Self::apply_failures`], since the announce was never attempted, not + /// attempted-and-failed. Only reached for `Announce` actions. async fn execute_and_journal_announce( &mut self, rule: FlowSpecRule, origin: BlackholeOrigin, + mono_now: u64, wall_now: u64, - ) { + ) -> AnnounceOutcome { let key = key_of(&rule); + if self.disarmed { + tracing::warn!( + ?key, + "FlowSpec: disarmed (C5); skipping announce, recording only" + ); + self.controller.rollback(key); + self.disarmed_skips = self.disarmed_skips.saturating_add(1); + return AnnounceOutcome::Disarmed; + } + if let Some(limiter) = &self.rate_limiter { + let allowed = limiter + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .try_acquire(mono_now); + if !allowed { + tracing::warn!(?key, "FlowSpec: cross-plane new-mitigation rate cap exceeded (C6); skipping announce, not activating"); + self.controller.rollback(key); + self.ratecapped = self.ratecapped.saturating_add(1); + return AnnounceOutcome::RateCapped; + } + } if let Err(e) = self.bgp.announce_flowspec(rule.clone()).await { - tracing::warn!(?key, error = %e, "FlowSpec: BGP announce failed; not journaling"); - return; + tracing::warn!(?key, error = %e, "FlowSpec: BGP announce failed; rolling back active entry, not journaling"); + self.controller.rollback(key); + self.apply_failures = self.apply_failures.saturating_add(1); + return AnnounceOutcome::Failed; } if let Err(e) = self .journal @@ -322,6 +544,7 @@ impl FlowSpecManager { at_ms: wall_now, }); } + AnnounceOutcome::Applied } /// Drain-retry queued mirror writes left over from a transient journal @@ -476,6 +699,7 @@ mod tests { max_rules: 2, hold_down: Duration::from_secs(10), max_ttl: None, + protected_prefixes: Vec::new(), } } fn ip(s: &str) -> IpAddr { @@ -648,6 +872,118 @@ mod tests { ); } + #[tokio::test] + async fn failed_announce_does_not_leave_a_phantom_active_entry() { + // BGP fails: the router never took the rule, so the control plane + // must NOT believe it did (C2) — the freshly-inserted active entry + // must be rolled back, not left as a phantom "active" rule. + let mut m = mgr(true, false); // BGP fails + m.apply_open( + ip("203.0.113.7"), + &[flow_rule("203.0.113.7", 17, 53, 0.0)], + 1_000, + 1_000, + ) + .await; + let key = key_of(&rule("203.0.113.7/32", 17, 53, 0.0)); + assert!( + !m.is_active(key), + "a failed announce must not leave a phantom active entry" + ); + assert_eq!(m.apply_failures(), 1); + + // A subsequent identical detection re-attempts (not deduped against + // a phantom active entry) — no retry queue, just the natural + // re-detection on the next tick. + m.apply_open( + ip("203.0.113.7"), + &[flow_rule("203.0.113.7", 17, 53, 0.0)], + 2_000, + 2_000, + ) + .await; + assert_eq!(m.apply_failures(), 2); + } + + #[tokio::test] + async fn successful_announce_activates_and_journals_no_apply_failures() { + let mut m = mgr(false, false); + m.apply_open( + ip("203.0.113.7"), + &[flow_rule("203.0.113.7", 17, 53, 0.0)], + 1_000, + 1_000, + ) + .await; + let key = key_of(&rule("203.0.113.7/32", 17, 53, 0.0)); + assert!(m.is_active(key)); + assert_eq!(m.apply_failures(), 0); + } + + #[tokio::test] + async fn rate_capped_announce_is_skipped_not_activated_and_not_an_apply_failure() { + // C6: mirrors `manager::tests::rate_capped_announce_is_skipped_...` + // for the FlowSpec side of the SAME shared limiter type. + let mut m = + mgr(false, false).with_rate_limiter(Arc::new(Mutex::new(ArmingRateLimiter::new(1)))); + m.apply_open( + ip("203.0.113.7"), + &[flow_rule("203.0.113.7", 17, 53, 0.0)], + 1_000, + 1_000, + ) + .await; + let key1 = key_of(&rule("203.0.113.7/32", 17, 53, 0.0)); + assert!(m.is_active(key1), "first announce is admitted"); + + m.apply_open( + ip("203.0.113.8"), + &[flow_rule("203.0.113.8", 17, 53, 0.0)], + 1_500, + 1_500, + ) + .await; + let key2 = key_of(&rule("203.0.113.8/32", 17, 53, 0.0)); + assert!( + !m.is_active(key2), + "rate-capped announce must not leave a phantom active entry" + ); + assert_eq!( + m.bgp().announced.lock().unwrap().len(), + 1, + "the rate-capped announce must never reach BGP" + ); + assert_eq!(m.ratecapped(), 1); + assert_eq!( + m.apply_failures(), + 0, + "a rate-cap skip is not an apply_failure (never attempted)" + ); + } + + #[tokio::test] + async fn no_rate_limiter_attached_is_unlimited() { + // Non-breaking: a manager with no limiter attached (the default from + // `new`) behaves exactly as before this feature existed. + let mut m = mgr(false, false); + m.apply_open( + ip("203.0.113.7"), + &[ + flow_rule("203.0.113.7", 17, 53, 0.0), + flow_rule("203.0.113.7", 6, 80, 0.0), + ], + 0, + 0, + ) + .await; + assert_eq!( + m.bgp().announced.lock().unwrap().len(), + 2, + "still bounded by max_rules=2 in cfg(), not by any rate cap" + ); + assert_eq!(m.ratecapped(), 0); + } + #[tokio::test] async fn tick_drains_pending_mirror_once_journal_recovers() { let mut m = mgr_transient_journal_failures(1); @@ -699,7 +1035,7 @@ mod tests { m.journal().announced.lock().unwrap()[0].1, BlackholeOrigin::Manual ); - m.apply_remove(r, 1000).await; + m.apply_remove(r, 1000, 1000).await; assert!(m.active().is_empty()); assert_eq!(m.journal().withdrawn.lock().unwrap().len(), 1); } @@ -744,6 +1080,41 @@ mod tests { ); } + #[tokio::test] + async fn apply_add_protected_target_is_rejected_not_deferred() { + // Target sits inside BOTH an eligible prefix and a protected prefix — + // exactly the overlap the protected-prefix guard exists for (an + // anycast VIP inside a customer-eligible block). A manual add must be + // classified as Rejected, not Deferred: a Deferred outcome leaves the + // request row 'pending' forever, retried every tick, indistinguishable + // from a transient capacity wait that will never resolve (C1 follow-up). + let mut m = FlowSpecManager::new( + FlowSpecController::new(FlowSpecConfig { + protected_prefixes: vec!["203.0.113.53/32".parse().unwrap()], + ..cfg() + }), + FakeBgp::default(), + FakeJournal::default(), + ); + let outcome = m + .apply_add(rule("203.0.113.53/32", 17, 53, 0.0), 0, 0) + .await; + match &outcome { + ApplyOutcome::Rejected(reason) => { + assert!( + reason.contains("protected"), + "reason should mention 'protected': {reason}" + ); + } + other => panic!("protected target must be Rejected, not {other:?}"), + } + assert!(m.active().is_empty()); + assert!( + m.bgp().announced.lock().unwrap().is_empty(), + "no Announce may be executed for a protected target" + ); + } + #[tokio::test] async fn rehydrate_reannounces() { let mut m = mgr(false, false); @@ -789,7 +1160,7 @@ mod tests { 1000, ) .await; - m.apply_remove(rule("203.0.113.7/32", 17, 53, 0.0), 2000) + m.apply_remove(rule("203.0.113.7/32", 17, 53, 0.0), 2000, 2000) .await; m.apply_open( ip("203.0.113.7"), @@ -805,6 +1176,108 @@ mod tests { ); } + #[tokio::test] + async fn disarm_withdraws_all_and_switches_to_record_only() { + let mut m = mgr(false, false); + m.apply_open( + ip("203.0.113.7"), + &[flow_rule("203.0.113.7", 17, 53, 0.0)], + 1_000, + 1_000, + ) + .await; + let key = key_of(&rule("203.0.113.7/32", 17, 53, 0.0)); + assert!(m.is_active(key)); + + m.disarm(2_000).await; + + assert_eq!( + m.bgp().withdrawn.lock().unwrap().len(), + 1, + "disarm must withdraw every active rule" + ); + assert!(!m.is_active(key), "disarm must clear the active set"); + + // A subsequent detection is recorded, not executed. + m.apply_open( + ip("203.0.113.8"), + &[flow_rule("203.0.113.8", 17, 53, 0.0)], + 3_000, + 3_000, + ) + .await; + let key2 = key_of(&rule("203.0.113.8/32", 17, 53, 0.0)); + assert!(!m.is_active(key2)); + assert_eq!(m.bgp().announced.lock().unwrap().len(), 1); + assert_eq!(m.apply_failures(), 0); + assert_eq!(m.ratecapped(), 0); + assert_eq!(m.disarmed_skips(), 1); + } + + #[tokio::test] + async fn apply_add_while_disarmed_is_rejected_not_applied() { + // C5 + final-review fix: a manual add while disarmed must be + // classified Rejected (retrying is pointless — there is no re-arm + // entry point), never Applied — an "applied" operator-request row + // is never retried, which would silently lose operator intent. + let mut m = mgr(false, false); + m.disarm(0).await; + + let outcome = m + .apply_add(rule("203.0.113.7/32", 17, 53, 0.0), 1_000, 1_000) + .await; + match &outcome { + ApplyOutcome::Rejected(reason) => { + assert!( + reason.contains("disarmed"), + "reason should mention 'disarmed': {reason}" + ); + } + other => panic!("disarmed manual add must be Rejected, not {other:?}"), + } + let key = key_of(&rule("203.0.113.7/32", 17, 53, 0.0)); + assert!( + !m.is_active(key), + "a disarmed manual add must not leave a phantom active entry" + ); + assert!(m.bgp().announced.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn apply_add_while_rate_capped_is_deferred_not_applied() { + // C6 + final-review fix: a manual add rejected by the shared rate + // limiter must be classified Deferred (the request row stays + // `pending` and is retried next tick, once the window has room), + // never Applied. + let mut m = + mgr(false, false).with_rate_limiter(Arc::new(Mutex::new(ArmingRateLimiter::new(1)))); + // Exhaust the limiter's one slot for this window via an auto path. + m.apply_open( + ip("203.0.113.7"), + &[flow_rule("203.0.113.7", 17, 53, 0.0)], + 1_000, + 1_000, + ) + .await; + let key1 = key_of(&rule("203.0.113.7/32", 17, 53, 0.0)); + assert!(m.is_active(key1)); + + let outcome = m + .apply_add(rule("203.0.113.8/32", 17, 53, 0.0), 1_500, 1_500) + .await; + assert_eq!( + outcome, + ApplyOutcome::Deferred, + "a rate-capped manual add must be Deferred, not Applied" + ); + let key2 = key_of(&rule("203.0.113.8/32", 17, 53, 0.0)); + assert!( + !m.is_active(key2), + "a rate-capped manual add must not leave a phantom active entry" + ); + assert_eq!(m.ratecapped(), 1); + } + #[tokio::test] async fn queued_announce_then_withdraw_for_same_key_coalesces_to_withdraw() { let mut m = mgr_transient_journal_failures(2); @@ -817,7 +1290,7 @@ mod tests { .await; assert_eq!(m.pending_mirror_len(), 1); - m.apply_remove(rule("203.0.113.7/32", 17, 53, 0.0), 2000) + m.apply_remove(rule("203.0.113.7/32", 17, 53, 0.0), 2000, 2000) .await; assert_eq!( m.pending_mirror_len(), diff --git a/crates/blackwall-rtbh/src/lib.rs b/crates/blackwall-rtbh/src/lib.rs index b30a3d6..0510eee 100644 --- a/crates/blackwall-rtbh/src/lib.rs +++ b/crates/blackwall-rtbh/src/lib.rs @@ -7,6 +7,7 @@ pub mod controller; pub mod flowspec_controller; pub mod flowspec_manager; pub mod manager; +pub mod rate_limit; pub mod shadow; pub use controller::{BlackholeOrigin, RtbhAction, RtbhConfig, RtbhController}; @@ -15,6 +16,7 @@ pub use flowspec_manager::{FlowSpecJournal, FlowSpecManager}; pub use manager::{ ApplyOutcome, BgpError, BgpExecutor, BlackholeJournal, JournalError, RtbhManager, }; +pub use rate_limit::ArmingRateLimiter; pub use shadow::{NoOpJournal, ShadowAction, ShadowBgpExecutor, ShadowRecorder}; /// Executes BGP commands against a live session via [`blackwall_bgp::BgpHandle`]. diff --git a/crates/blackwall-rtbh/src/manager.rs b/crates/blackwall-rtbh/src/manager.rs index ef810d0..f35deb5 100644 --- a/crates/blackwall-rtbh/src/manager.rs +++ b/crates/blackwall-rtbh/src/manager.rs @@ -6,11 +6,13 @@ //! traits so `blackwall-rtbh` stays free of any DB dependency. use crate::controller::{BlackholeOrigin, RtbhAction, RtbhController}; +use crate::rate_limit::ArmingRateLimiter; use async_trait::async_trait; use blackwall_bgp::Route; use blackwall_flow::DetectionEvent; use ipnet::IpNet; use std::net::IpAddr; +use std::sync::{Arc, Mutex}; /// Executes BGP announce/withdraw commands. /// @@ -77,14 +79,18 @@ pub enum ApplyOutcome { /// /// Owns the pure [`RtbhController`] plus the I/O boundary: it executes the /// controller's decisions on a [`BgpExecutor`] and mirrors auto/manual state -/// via a [`BlackholeJournal`]. A BGP failure is logged and the action is not -/// journaled — but note this is a known limitation, not a retry mechanism: -/// on a failed first announce the controller entry is kept in memory while -/// the route itself is never re-announced automatically. A journal failure -/// after a successful BGP operation is logged, never causes a live -/// blackhole to be withdrawn, and is queued as a `MirrorOp` for a bounded -/// self-heal retry on the next [`RtbhManager::tick`] — the BGP outcome is -/// never re-issued, only the mirror write. +/// via a [`BlackholeJournal`]. A BGP announce failure is logged, the action +/// is not journaled, and the controller's freshly-inserted active entry is +/// rolled back via [`RtbhController::rollback`] (C2: commit-after-confirm) — +/// the control plane never believes an unconfirmed announce succeeded, so a +/// future detection for the same target is not deduped against a phantom +/// entry. There is no retry queue for this: while the underlying attack +/// persists, the detector naturally re-emits the detection on its next tick +/// and the manager re-attempts through the same path. This differs from a +/// *journal* failure after a successful BGP operation, which is logged, +/// never causes a live blackhole to be withdrawn, and is queued as a +/// `MirrorOp` for a bounded self-heal retry on the next [`RtbhManager::tick`] +/// — the BGP outcome is never re-issued, only the mirror write. pub struct RtbhManager { controller: RtbhController, bgp: B, @@ -93,6 +99,33 @@ pub struct RtbhManager { /// succeeded; retried (never re-issued to BGP) by /// `RtbhManager::retry_pending_mirror` on the next tick. pending_mirror: Vec, + /// Count of announces that failed at the BGP executor, each rolled back + /// (see [`Self::apply_failures`]). + apply_failures: u64, + /// Cross-plane cap (C6) on the arrival rate of NEW mitigations, shared + /// with the sibling `FlowSpecManager` via the same `Arc>` so ONE + /// limiter governs the combined RTBH+FlowSpec announce rate. `None` (the + /// default from [`Self::new`]) is unlimited — [`main`](../../bin/blackwalld) + /// only attaches `Some` via [`Self::with_rate_limiter`] on the live path + /// (never under shadow, where nothing is really announced). + rate_limiter: Option>>, + /// Count of announces skipped because [`Self::rate_limiter`] was at + /// capacity (C6) — a SKIP (never attempted), distinct from + /// [`Self::apply_failures`] (attempted and failed at BGP). See + /// [`Self::ratecapped`]. + ratecapped: u64, + /// One-way in-daemon disarm kill switch (C5), flipped by [`Self::disarm`]. + /// While set, [`Self::execute_and_journal_announce`] skips every new + /// `Announce` (never reaches [`Self::bgp`]) while detection + selection + /// keep running unchanged — the manager keeps recording, it just stops + /// applying. There is no re-arm entry point; a fresh process (restart) + /// is the only way back to armed. + disarmed: bool, + /// Count of announces skipped because [`Self::disarmed`] was set (C5) — + /// a SKIP (never attempted), distinct from both [`Self::apply_failures`] + /// (attempted and failed) and [`Self::ratecapped`] (skipped for a + /// different reason). See [`Self::disarmed_skips`]. + disarmed_skips: u64, } /// A journal mirror write that failed and is queued for a self-heal retry. @@ -120,6 +153,30 @@ impl MirrorOp { } } +/// Outcome of [`RtbhManager::execute_and_journal_announce`]. +/// +/// The auto path (`apply_event`/`tick`, via [`RtbhManager::execute_and_journal`]) +/// ignores this — auto re-detection naturally compensates for a skip on its +/// next tick. [`RtbhManager::apply_add`] (the manual path) consumes it to +/// report a truthful [`ApplyOutcome`] rather than always claiming +/// [`ApplyOutcome::Applied`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AnnounceOutcome { + /// The announce reached BGP; a journal-mirror failure afterward is still + /// `Applied` (self-healed via [`RtbhManager::retry_pending_mirror`]) — + /// the live blackhole is active either way. + Applied, + /// Skipped: the shared cross-plane [`ArmingRateLimiter`] (C6) was at + /// capacity. The controller entry was rolled back. + RateCapped, + /// Skipped: the manager is [`RtbhManager::disarm`]ed (C5), record-only. + /// The controller entry was rolled back. + Disarmed, + /// Attempted and failed at the [`BgpExecutor`] (C2). The controller + /// entry was rolled back. + Failed, +} + impl RtbhManager { /// Wrap a controller with a BGP executor and a journal. pub fn new(controller: RtbhController, bgp: B, journal: J) -> Self { @@ -128,9 +185,27 @@ impl RtbhManager { bgp, journal, pending_mirror: Vec::new(), + apply_failures: 0, + rate_limiter: None, + ratecapped: 0, + disarmed: false, + disarmed_skips: 0, } } + /// Attach a shared cross-plane rate cap (C6) on new mitigations. + /// + /// Non-breaking: absent (the default from [`Self::new`]) is unlimited. + /// `main.rs` wires `Some` only on the live path (`!policy.shadow`) — the + /// shadow-mode construction never calls this, so shadow sessions are + /// always unlimited (rate-capping a mitigation that is never really + /// announced would only corrupt the would-mitigate signal). + #[must_use] + pub fn with_rate_limiter(mut self, limiter: Arc>) -> Self { + self.rate_limiter = Some(limiter); + self + } + /// Feed one detection event through the controller and execute + journal /// the resulting actions. /// @@ -143,7 +218,7 @@ impl RtbhManager { pub async fn apply_event(&mut self, event: &DetectionEvent, mono_now: u64, wall_now: u64) { let actions = self.controller.on_event(event, mono_now); for action in actions { - self.execute_and_journal(action, wall_now).await; + self.execute_and_journal(action, mono_now, wall_now).await; } } @@ -158,7 +233,7 @@ impl RtbhManager { self.retry_pending_mirror().await; let actions = self.controller.tick(mono_now); for action in actions { - self.execute_and_journal(action, wall_now).await; + self.execute_and_journal(action, mono_now, wall_now).await; } } @@ -167,8 +242,8 @@ impl RtbhManager { /// Returns [`ApplyOutcome::Applied`] if newly installed or upgraded from /// `Auto` to `Manual` (re-journaled as `Manual` in the latter case), /// [`ApplyOutcome::Deferred`] if the manager is at capacity, or - /// [`ApplyOutcome::Rejected`] if the target is ineligible or has no - /// next-hop for its address family. + /// [`ApplyOutcome::Rejected`] if the target is protected, ineligible, or + /// has no next-hop for its address family. pub async fn apply_add( &mut self, target: IpAddr, @@ -177,9 +252,28 @@ impl RtbhManager { ) -> ApplyOutcome { let actions = self.controller.manual_add(target, mono_now); if let Some(RtbhAction::Announce(route)) = actions.into_iter().next() { - self.execute_and_journal_announce(target, route, BlackholeOrigin::Manual, wall_now) + let outcome = self + .execute_and_journal_announce( + target, + route, + BlackholeOrigin::Manual, + mono_now, + wall_now, + ) .await; - return ApplyOutcome::Applied; + return match outcome { + AnnounceOutcome::Applied => ApplyOutcome::Applied, + // The window will have room again; the request row stays + // `pending` and is retried next tick. + AnnounceOutcome::RateCapped => ApplyOutcome::Deferred, + // One-way: retrying is pointless until re-armed via restart. + AnnounceOutcome::Disarmed => ApplyOutcome::Rejected(format!( + "{target} was not announced: manager is disarmed (C5)" + )), + // No auto re-detection exists for a manual request, so a + // failed BGP announce must be retried, not marked applied. + AnnounceOutcome::Failed => ApplyOutcome::Deferred, + }; } // Empty result: either already active (upgrade), at cap, or rejected. if self.is_active(target) { @@ -199,6 +293,15 @@ impl RtbhManager { } return ApplyOutcome::Applied; } + // Checked before eligibility: a protected target is typically ALSO + // eligible (that's the point — protected VIPs live inside eligible + // prefixes), so it must be rejected outright here rather than falling + // through to Deferred, which would retry forever and never resolve. + if self.controller.is_protected(target) { + return ApplyOutcome::Rejected(format!( + "{target} is inside a protected prefix and is never mitigated" + )); + } if !self.controller.is_eligible(target) { return ApplyOutcome::Rejected(format!("{target} is outside eligible prefixes")); } @@ -209,10 +312,16 @@ impl RtbhManager { } /// Manually withdraw a target (bypasses hold-down). - pub async fn apply_remove(&mut self, target: IpAddr, wall_now: u64) { + /// + /// `mono_now` is accepted for symmetry with the other entry points that + /// funnel through [`Self::execute_and_journal`] (it is unused here: a + /// manual removal only ever produces a `Withdraw`, never an `Announce`, + /// so the shared rate limiter — which only gates `Announce` — is never + /// consulted on this path). + pub async fn apply_remove(&mut self, target: IpAddr, mono_now: u64, wall_now: u64) { let actions = self.controller.manual_remove(target); for action in actions { - self.execute_and_journal(action, wall_now).await; + self.execute_and_journal(action, mono_now, wall_now).await; } } @@ -251,6 +360,92 @@ impl RtbhManager { self.controller.active_blackholes() } + /// Number of targets skipped by the controller's protected-prefix guard + /// (own anycast VIPs never mitigated). Surfaced for `/metrics`; the + /// owning task periodically copies this into a shared counter read by + /// the metrics endpoint, mirroring how `min_sample_suppressed` reaches + /// `/metrics` from the flow detector. + #[must_use] + pub fn protected_skipped(&self) -> u64 { + self.controller.protected_skipped() + } + + /// Count of announces that failed at the [`BgpExecutor`] (C2). Each + /// failure rolls back the controller's freshly-inserted active entry + /// (see [`RtbhController::rollback`]) so the control plane never + /// believes an unconfirmed announce is active. Surfaced for `/metrics` + /// as `blackwall_rtbh_apply_failures_total`, mirroring how + /// [`Self::protected_skipped`] reaches the endpoint. + #[must_use] + pub fn apply_failures(&self) -> u64 { + self.apply_failures + } + + /// Count of announces skipped because the shared cross-plane + /// [`ArmingRateLimiter`] (C6) was at capacity. Each skip rolls back the + /// controller's freshly-inserted active entry (never left as a phantom + /// active mitigation) and is distinct from [`Self::apply_failures`] — a + /// rate-cap skip was never attempted at all. Surfaced for `/metrics` as + /// `blackwall_mitigations_ratecapped_total{plane="rtbh"}`. + #[must_use] + pub fn ratecapped(&self) -> u64 { + self.ratecapped + } + + /// In-daemon disarm kill switch (C5): withdraw every currently-active + /// blackhole and switch to record-only for the rest of this process's + /// life. + /// + /// Each active target is withdrawn on BGP best-effort — a withdraw + /// `Err` is logged and the sweep continues with the next target (never + /// aborts), mirroring how [`Self::execute_and_journal`] already treats a + /// withdraw failure elsewhere: log and move on. No journal write happens + /// here, unlike a normal withdraw: disarm is a *runtime-only* state, not + /// a persisted decision — a restart re-arms and [`Self::rehydrate`]s the + /// very same active set from the journal, exactly as if disarm had never + /// happened. Once disarmed, every subsequent `Announce` reaching + /// [`Self::execute_and_journal_announce`] is skipped (never sent to + /// [`Self::bgp`]) and counted in [`Self::disarmed_skips`] — detection + /// and selection keep running unchanged (visibility retained), only the + /// apply step is gated. One-way: calling this again is a no-op (idempotent + /// under a repeated SIGUSR1), and there is no re-arm entry point. + /// + /// `mono_now` is accepted for symmetry with the other entry points that + /// funnel through the execute path; it is unused here (a disarm withdraw + /// bypasses hold-down via [`RtbhController::manual_remove`] and needs no + /// time arithmetic). + pub async fn disarm(&mut self, _mono_now: u64) { + if self.disarmed { + return; + } + self.disarmed = true; + let targets: Vec = self + .controller + .active_blackholes() + .into_iter() + .map(|(target, ..)| target) + .collect(); + for target in targets { + for action in self.controller.manual_remove(target) { + if let RtbhAction::Withdraw(prefix) = action { + if let Err(e) = self.bgp.withdraw(prefix).await { + tracing::warn!(%target, error = %e, "RTBH: disarm withdraw failed; continuing best-effort"); + } + } + } + } + tracing::warn!("RTBH: DISARMED — mitigations withdrawn, now recording only"); + } + + /// Count of new-mitigation announces skipped because the manager was + /// [`Self::disarm`]ed (C5) — a SKIP (never attempted), distinct from + /// both [`Self::apply_failures`] (attempted and failed at BGP) and + /// [`Self::ratecapped`] (skipped for a different reason). + #[must_use] + pub fn disarmed_skips(&self) -> u64 { + self.disarmed_skips + } + fn is_active(&self, target: IpAddr) -> bool { self.controller .active_blackholes() @@ -271,13 +466,14 @@ impl RtbhManager { } /// Execute one controller action on BGP and mirror it into the journal. - async fn execute_and_journal(&mut self, action: RtbhAction, wall_now: u64) { + async fn execute_and_journal(&mut self, action: RtbhAction, mono_now: u64, wall_now: u64) { match action { RtbhAction::Announce(route) => { self.execute_and_journal_announce( ip_of(&route.prefix), route, BlackholeOrigin::Auto, + mono_now, wall_now, ) .await; @@ -299,16 +495,48 @@ impl RtbhManager { } } + /// Execute one NEW-mitigation `Announce` on BGP and mirror it into the + /// journal. + /// + /// First consults the shared [`ArmingRateLimiter`] (C6), if attached: a + /// rejection rolls back the controller's freshly-inserted active entry + /// (same "commit-after-confirm" discipline as a BGP failure, see the + /// module docs) and counts against [`Self::ratecapped`] — never + /// [`Self::apply_failures`], since the announce was never attempted, not + /// attempted-and-failed. Only reached for `Announce` actions (never a + /// `Withdraw` or a controller re-assertion/refresh, which don't produce + /// `Announce` at all). async fn execute_and_journal_announce( &mut self, target: IpAddr, route: Route, origin: BlackholeOrigin, + mono_now: u64, wall_now: u64, - ) { + ) -> AnnounceOutcome { + if self.disarmed { + tracing::warn!(%target, "RTBH: disarmed (C5); skipping announce, recording only"); + self.controller.rollback(target); + self.disarmed_skips = self.disarmed_skips.saturating_add(1); + return AnnounceOutcome::Disarmed; + } + if let Some(limiter) = &self.rate_limiter { + let allowed = limiter + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .try_acquire(mono_now); + if !allowed { + tracing::warn!(%target, "RTBH: cross-plane new-mitigation rate cap exceeded (C6); skipping announce, not activating"); + self.controller.rollback(target); + self.ratecapped = self.ratecapped.saturating_add(1); + return AnnounceOutcome::RateCapped; + } + } if let Err(e) = self.bgp.announce(route).await { - tracing::warn!(%target, error = %e, "RTBH: BGP announce failed; not journaling"); - return; + tracing::warn!(%target, error = %e, "RTBH: BGP announce failed; rolling back active entry, not journaling"); + self.controller.rollback(target); + self.apply_failures = self.apply_failures.saturating_add(1); + return AnnounceOutcome::Failed; } if let Err(e) = self.journal.record_announce(target, origin, wall_now).await { tracing::error!(%target, error = %e, "RTBH: journal write failed after announce; keeping active"); @@ -318,6 +546,7 @@ impl RtbhManager { at_ms: wall_now, }); } + AnnounceOutcome::Applied } /// Drain-retry queued mirror writes left over from a transient journal @@ -388,6 +617,11 @@ mod tests { announced: Mutex>, withdrawn: Mutex>, fail: bool, + /// Independent withdraw-only failure toggle, for exercising disarm's + /// best-effort tolerance of a withdraw `Err` without also blocking + /// the announce that must precede it (unlike `fail`, which fails + /// both). + fail_withdraw: bool, } #[async_trait] impl BgpExecutor for FakeBgp { @@ -399,7 +633,7 @@ mod tests { Ok(()) } async fn withdraw(&self, prefix: IpNet) -> Result<(), BgpError> { - if self.fail { + if self.fail || self.fail_withdraw { return Err(BgpError); } self.withdrawn.lock().unwrap().push(prefix); @@ -481,6 +715,7 @@ mod tests { max_blackholes: 2, hold_down: Duration::from_secs(10), max_ttl: None, + protected_prefixes: Vec::new(), } } fn det(ip: &str) -> Detection { @@ -606,6 +841,92 @@ mod tests { ); } + #[tokio::test] + async fn failed_announce_does_not_leave_a_phantom_active_entry() { + // BGP fails: the router never took the route, so the control plane + // must NOT believe it did (C2) — the freshly-inserted active entry + // must be rolled back, not left as a phantom "active" mitigation. + let mut m = mgr(true, false); // BGP fails + m.apply_event(&DetectionEvent::Opened(det("203.0.113.7")), 1_000, 1_000) + .await; + assert!( + !m.is_active(ip("203.0.113.7")), + "a failed announce must not leave a phantom active entry" + ); + assert_eq!(m.apply_failures(), 1); + + // A subsequent identical detection re-attempts (not deduped against + // a phantom active entry) — no retry queue, just the natural + // re-detection on the next tick. + m.apply_event(&DetectionEvent::Opened(det("203.0.113.7")), 2_000, 2_000) + .await; + assert_eq!(m.apply_failures(), 2); + } + + #[tokio::test] + async fn successful_announce_activates_and_journals_no_apply_failures() { + let mut m = mgr(false, false); + m.apply_event(&DetectionEvent::Opened(det("203.0.113.7")), 1_000, 1_000) + .await; + assert!(m.is_active(ip("203.0.113.7"))); + assert_eq!(m.apply_failures(), 0); + } + + #[tokio::test] + async fn rate_capped_announce_is_skipped_not_activated_and_not_an_apply_failure() { + // C6: a shared limiter admitting only 1 announce per minute. The + // second Opened in the same window must be SKIPPED (never reach + // BGP), rolled back so it is not left as a phantom active entry, and + // counted as `ratecapped` — NOT `apply_failures` (it was never + // attempted, unlike a BGP failure). + let mut m = + mgr(false, false).with_rate_limiter(Arc::new(Mutex::new(ArmingRateLimiter::new(1)))); + m.apply_event(&DetectionEvent::Opened(det("203.0.113.7")), 1_000, 1_000) + .await; + assert!(m.is_active(ip("203.0.113.7")), "first announce is admitted"); + + m.apply_event(&DetectionEvent::Opened(det("203.0.113.8")), 1_500, 1_500) + .await; + assert!( + !m.is_active(ip("203.0.113.8")), + "rate-capped announce must not leave a phantom active entry" + ); + assert!( + m.bgp().announced.lock().unwrap().len() == 1, + "the rate-capped announce must never reach BGP" + ); + assert_eq!(m.ratecapped(), 1); + assert_eq!( + m.apply_failures(), + 0, + "a rate-cap skip is not an apply_failure (never attempted)" + ); + + // Once the window rolls, the target is admitted normally. + m.apply_event(&DetectionEvent::Opened(det("203.0.113.8")), 61_500, 61_500) + .await; + assert!(m.is_active(ip("203.0.113.8"))); + assert_eq!(m.ratecapped(), 1); + } + + #[tokio::test] + async fn no_rate_limiter_attached_is_unlimited() { + // Non-breaking: a manager with no limiter attached (the default from + // `new`) behaves exactly as before this feature existed. + let mut m = mgr(false, false); + for i in 0..10u8 { + let target = format!("203.0.113.{}", i + 1); + m.apply_add(target.parse().unwrap(), u64::from(i), u64::from(i)) + .await; + } + assert_eq!( + m.bgp().announced.lock().unwrap().len(), + 2, + "still bounded by max_blackholes=2 in cfg(), not by any rate cap" + ); + assert_eq!(m.ratecapped(), 0); + } + #[tokio::test] async fn apply_add_rejects_ineligible_and_defers_at_cap() { let mut m = mgr(false, false); @@ -627,6 +948,39 @@ mod tests { ); } + #[tokio::test] + async fn apply_add_protected_target_is_rejected_not_deferred() { + // Target sits inside BOTH an eligible prefix and a protected prefix — + // exactly the overlap the protected-prefix guard exists for (an + // anycast VIP inside a customer-eligible block). A manual add must be + // classified as Rejected, not Deferred: a Deferred outcome leaves the + // request row 'pending' forever, retried every tick, indistinguishable + // from a transient capacity wait that will never resolve (C1 follow-up). + let mut m = RtbhManager::new( + RtbhController::new(RtbhConfig { + protected_prefixes: vec!["203.0.113.53/32".parse().unwrap()], + ..cfg() + }), + FakeBgp::default(), + FakeJournal::default(), + ); + let outcome = m.apply_add(ip("203.0.113.53"), 0, 0).await; + match &outcome { + ApplyOutcome::Rejected(reason) => { + assert!( + reason.contains("protected"), + "reason should mention 'protected': {reason}" + ); + } + other => panic!("protected target must be Rejected, not {other:?}"), + } + assert!(m.active().is_empty()); + assert!( + m.bgp().announced.lock().unwrap().is_empty(), + "no Announce may be executed for a protected target" + ); + } + #[tokio::test] async fn apply_add_upgrade_rejournals_as_manual() { let mut m = mgr(false, false); @@ -646,7 +1000,7 @@ mod tests { async fn apply_remove_withdraws_and_journals() { let mut m = mgr(false, false); m.apply_add(ip("203.0.113.7"), 0, 0).await; - m.apply_remove(ip("203.0.113.7"), 1000).await; + m.apply_remove(ip("203.0.113.7"), 1000, 1000).await; assert!(m.active().is_empty()); assert_eq!(m.journal().withdrawn.lock().unwrap().len(), 1); } @@ -759,7 +1113,7 @@ mod tests { .await; assert_eq!(m.pending_mirror_len(), 1); - m.apply_remove(ip("203.0.113.7"), 2000).await; + m.apply_remove(ip("203.0.113.7"), 2000, 2000).await; assert_eq!( m.pending_mirror_len(), 1, @@ -782,7 +1136,7 @@ mod tests { let mut m = mgr(false, true); // BGP ok, journal always fails m.apply_event(&DetectionEvent::Opened(det("203.0.113.7")), 0, 1000) .await; - m.apply_remove(ip("203.0.113.7"), 2000).await; + m.apply_remove(ip("203.0.113.7"), 2000, 2000).await; m.apply_event(&DetectionEvent::Opened(det("203.0.113.7")), 3000, 3000) .await; assert_eq!( @@ -792,6 +1146,143 @@ mod tests { ); } + #[tokio::test] + async fn disarm_withdraws_all_and_switches_to_record_only() { + // C5: disarm must withdraw every active blackhole on BGP (best + // effort), clear the active set, and thereafter skip every new + // Announce (record-only) — detection/selection keep running (the + // manager still accepts events), only the apply step is gated. + let mut m = mgr(false, false); + m.apply_event(&DetectionEvent::Opened(det("203.0.113.7")), 1_000, 1_000) + .await; + assert!(m.is_active(ip("203.0.113.7"))); + + m.disarm(2_000).await; + + assert!( + m.bgp() + .withdrawn + .lock() + .unwrap() + .contains(&"203.0.113.7/32".parse::().unwrap()), + "disarm must withdraw every active target" + ); + assert!( + !m.is_active(ip("203.0.113.7")), + "disarm must clear the active set" + ); + + // A subsequent detection is recorded (the controller still runs), + // but must NOT be announced — record-only. + m.apply_event(&DetectionEvent::Opened(det("203.0.113.8")), 3_000, 3_000) + .await; + assert_eq!( + m.bgp().announced.lock().unwrap().len(), + 1, + "no new announce may execute once disarmed" + ); + assert!( + !m.is_active(ip("203.0.113.8")), + "a disarmed skip must not leave a phantom active entry" + ); + assert_eq!( + m.apply_failures(), + 0, + "a disarmed skip is not an apply_failure (never attempted)" + ); + assert_eq!(m.ratecapped(), 0, "a disarmed skip is not a rate-cap skip"); + assert_eq!(m.disarmed_skips(), 1); + } + + #[tokio::test] + async fn disarm_tolerates_a_withdraw_error() { + // Best-effort: a withdraw Err during disarm must not abort the + // sweep (a second active target is still withdrawn) or stop the + // manager from switching to record-only. + let mut m = RtbhManager::new( + RtbhController::new(cfg()), + FakeBgp { + fail_withdraw: true, + ..Default::default() + }, + FakeJournal::default(), + ); + m.apply_event(&DetectionEvent::Opened(det("203.0.113.7")), 0, 0) + .await; + m.apply_event(&DetectionEvent::Opened(det("203.0.113.8")), 0, 0) + .await; + assert_eq!(m.active().len(), 2); + + m.disarm(1_000).await; + + assert!( + m.bgp().withdrawn.lock().unwrap().is_empty(), + "every withdraw errored, so none was recorded by the fake" + ); + assert!( + m.active().is_empty(), + "disarm clears the active set even when every withdraw errors (best-effort)" + ); + + // Record-only holds even though disarm itself never got a + // confirmed withdraw. + m.apply_event(&DetectionEvent::Opened(det("203.0.113.9")), 2_000, 2_000) + .await; + assert!(!m.is_active(ip("203.0.113.9"))); + } + + #[tokio::test] + async fn apply_add_while_disarmed_is_rejected_not_applied() { + // C5 + final-review fix: a manual add while disarmed must be + // classified Rejected (retrying is pointless — there is no re-arm + // entry point), never Applied — an "applied" operator-request row + // is never retried, which would silently lose operator intent. + let mut m = mgr(false, false); + m.disarm(0).await; + + let outcome = m.apply_add(ip("203.0.113.7"), 1_000, 1_000).await; + match &outcome { + ApplyOutcome::Rejected(reason) => { + assert!( + reason.contains("disarmed"), + "reason should mention 'disarmed': {reason}" + ); + } + other => panic!("disarmed manual add must be Rejected, not {other:?}"), + } + assert!( + !m.is_active(ip("203.0.113.7")), + "a disarmed manual add must not leave a phantom active entry" + ); + assert!(m.bgp().announced.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn apply_add_while_rate_capped_is_deferred_not_applied() { + // C6 + final-review fix: a manual add rejected by the shared rate + // limiter must be classified Deferred (the request row stays + // `pending` and is retried next tick, once the window has room), + // never Applied. + let mut m = + mgr(false, false).with_rate_limiter(Arc::new(Mutex::new(ArmingRateLimiter::new(1)))); + // Exhaust the limiter's one slot for this window via an auto path. + m.apply_event(&DetectionEvent::Opened(det("203.0.113.7")), 1_000, 1_000) + .await; + assert!(m.is_active(ip("203.0.113.7"))); + + let outcome = m.apply_add(ip("203.0.113.8"), 1_500, 1_500).await; + assert_eq!( + outcome, + ApplyOutcome::Deferred, + "a rate-capped manual add must be Deferred, not Applied" + ); + assert!( + !m.is_active(ip("203.0.113.8")), + "a rate-capped manual add must not leave a phantom active entry" + ); + assert_eq!(m.ratecapped(), 1); + } + #[tokio::test] async fn manual_upgrade_journal_failure_self_heals_as_manual() { // An Auto entry is active but its mirror write failed; the operator diff --git a/crates/blackwall-rtbh/src/rate_limit.rs b/crates/blackwall-rtbh/src/rate_limit.rs new file mode 100644 index 0000000..e5e5dfb --- /dev/null +++ b/crates/blackwall-rtbh/src/rate_limit.rs @@ -0,0 +1,114 @@ +//! Cross-plane rate cap on new mitigations (C6). +//! +//! `max_blackholes`/`max_rules` bound the *steady-state* size of the active +//! set, but neither bounds how fast new entries can arrive: a detection +//! storm (or a bug feeding the detector garbage) can walk either cap up to +//! its ceiling within seconds, mass-blackholing legitimate destinations +//! before an operator can react. [`ArmingRateLimiter`] is a safety ceiling on +//! the *arrival rate* of NEW mitigations (BGP `Announce`s) — orthogonal to, +//! and layered underneath, the existing per-plane count caps. + +use std::collections::VecDeque; + +/// Width of the sliding window, in milliseconds. +const WINDOW_MS: u64 = 60_000; + +/// A sliding 60-second-window rate limiter over new-mitigation timestamps. +/// +/// Pure and deterministic: callers pass `now_ms` explicitly rather than the +/// limiter reading the clock itself, so it is unit-testable without real +/// time and reusable across both the RTBH and FlowSpec managers (a single +/// shared instance, behind an `Arc>`, governs the combined +/// cross-plane announce rate — see `blackwall_rtbh::manager::RtbhManager`'s +/// and `blackwall_rtbh::flowspec_manager::FlowSpecManager`'s +/// `with_rate_limiter`). +#[derive(Debug)] +pub struct ArmingRateLimiter { + max_per_min: u32, + /// Timestamps of admitted announces still inside the trailing window, + /// oldest first. + window: VecDeque, +} + +impl ArmingRateLimiter { + /// A limiter admitting at most `max_per_min` new announces in any + /// trailing 60_000 ms window. + #[must_use] + pub fn new(max_per_min: u32) -> Self { + Self { + max_per_min, + window: VecDeque::new(), + } + } + + /// Attempt to admit one new-mitigation announce at `now_ms`. + /// + /// Drops timestamps that have aged out of the trailing 60_000 ms window, + /// then admits (and records) the attempt iff fewer than `max_per_min` + /// remain; otherwise rejects without recording, so a rejected attempt + /// never itself counts toward the window. + pub fn try_acquire(&mut self, now_ms: u64) -> bool { + let window_start = now_ms.saturating_sub(WINDOW_MS); + while let Some(&oldest) = self.window.front() { + if oldest < window_start { + self.window.pop_front(); + } else { + break; + } + } + let in_window = u32::try_from(self.window.len()).unwrap_or(u32::MAX); + if in_window < self.max_per_min { + self.window.push_back(now_ms); + true + } else { + false + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rate_cap_rejects_excess_in_window_and_refills() { + let mut lim = ArmingRateLimiter::new(2); // 2 per minute + assert!(lim.try_acquire(1_000)); + assert!(lim.try_acquire(1_500)); + assert!(!lim.try_acquire(2_000), "3rd in-window is rejected"); + // 61s later the window has rolled: + assert!(lim.try_acquire(63_000)); + } + + #[test] + fn zero_max_rejects_everything() { + let mut lim = ArmingRateLimiter::new(0); + assert!(!lim.try_acquire(1_000)); + assert!(!lim.try_acquire(100_000)); + } + + #[test] + fn rejected_attempt_is_not_recorded() { + // A rejected attempt must not itself occupy a window slot — otherwise + // a burst of rejects could wedge the limiter shut even after room + // frees up within the same window. + let mut lim = ArmingRateLimiter::new(1); + assert!(lim.try_acquire(0)); + assert!(!lim.try_acquire(100)); + assert!(!lim.try_acquire(200)); + // Still only the first (admitted) timestamp counts; once it ages out + // of the window (60_000 ms is still exactly in-window; 60_001 is + // not), capacity returns. + assert!(!lim.try_acquire(60_000)); + assert!(lim.try_acquire(60_001)); + } + + #[test] + fn independent_windows_do_not_interfere() { + let mut lim = ArmingRateLimiter::new(1); + assert!(lim.try_acquire(0)); + assert!(!lim.try_acquire(30_000), "still inside the first window"); + assert!(lim.try_acquire(60_001), "window has fully rolled"); + assert!(!lim.try_acquire(90_000), "back inside the new window"); + } +} diff --git a/crates/blackwall-rtbh/tests/armed_disarm_interop.rs b/crates/blackwall-rtbh/tests/armed_disarm_interop.rs new file mode 100644 index 0000000..4a2f9eb --- /dev/null +++ b/crates/blackwall-rtbh/tests/armed_disarm_interop.rs @@ -0,0 +1,160 @@ +//! In-daemon disarm gate (C5): after RTBH + FlowSpec routes are announced to +//! real BIRD, `SIGUSR1` must withdraw everything from BIRD's RIB while the +//! process keeps running — proving `RtbhManager::disarm` / +//! `FlowSpecManager::disarm` (Task 9) end to end against real BIRD. Mirrors +//! `blackwalld`'s own `disarm_signal_task` (bin/blackwalld/src/main.rs) +//! narrowed to just the two managers under test — no DB/Postgres dependency, +//! same `Noop*Journal` pattern as the sibling interop drivers. Ignored in +//! CI; run by the lab's `armed-disarm-bird` scenario. +//! BW_BGP_PEER=10.0.0.1:179 cargo test -p blackwall-rtbh --test armed_disarm_interop -- --ignored --nocapture + +use async_trait::async_trait; +use blackwall_bgp::{spawn, FlowAction, FlowSpecRule, Origin, PeerConfig, Route}; +use blackwall_rtbh::{ + ApplyOutcome, BlackholeJournal, BlackholeOrigin, FlowSpecConfig, FlowSpecController, + FlowSpecJournal, FlowSpecManager, JournalError, RtbhConfig, RtbhController, RtbhManager, +}; +use std::net::IpAddr; +use std::time::Duration; +use tokio::signal::unix::{signal, SignalKind}; + +/// A no-op RTBH journal: this test only exercises the BGP path against real +/// BIRD, not persistence (covered elsewhere with fakes / real Postgres). +struct NoopBlackholeJournal; + +#[async_trait] +impl BlackholeJournal for NoopBlackholeJournal { + async fn record_announce( + &self, + _target: IpAddr, + _origin: BlackholeOrigin, + _at_ms: u64, + ) -> Result<(), JournalError> { + Ok(()) + } + async fn record_withdraw(&self, _target: IpAddr, _at_ms: u64) -> Result<(), JournalError> { + Ok(()) + } +} + +/// A no-op FlowSpec journal, for the same reason. +struct NoopFlowSpecJournal; + +#[async_trait] +impl FlowSpecJournal for NoopFlowSpecJournal { + async fn record_announce( + &self, + _rule: FlowSpecRule, + _origin: BlackholeOrigin, + _at_ms: u64, + ) -> Result<(), JournalError> { + Ok(()) + } + async fn record_withdraw(&self, _rule: FlowSpecRule, _at_ms: u64) -> Result<(), JournalError> { + Ok(()) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "needs a live BGP peer (BIRD); run in the netns lab"] +async fn sigusr1_withdraws_all_and_keeps_running() { + let peer: std::net::SocketAddr = std::env::var("BW_BGP_PEER") + .expect("set BW_BGP_PEER=ip:179") + .parse() + .unwrap(); + let (handle, _join) = spawn(PeerConfig { + local_asn: 214_806, + peer_asn: 214_806, + peer_addr: peer, + router_id: "10.222.255.99".parse().unwrap(), + hold_time: 90, + md5: None, + gtsm_hops: None, + local_addr: std::env::var("BW_BGP_LOCAL_ADDR") + .ok() + .map(|s| s.parse().expect("BW_BGP_LOCAL_ADDR must be an IP address")), + }) + .expect("valid iBGP config"); + tokio::time::sleep(Duration::from_secs(3)).await; // let the session establish + + // Covering unicast route (RFC 8955 §6 "safe update" validation): without + // this, BIRD rejects the FlowSpec rule below as unvalidated. + handle + .announce(Route { + prefix: "203.0.113.0/24".parse().unwrap(), + next_hop: "10.0.0.1".parse().unwrap(), + origin: Origin::Igp, + communities: vec![], + large_communities: vec![], + }) + .await + .expect("announce covering route"); + + // Install the SIGUSR1 handler BEFORE announcing anything: a signal that + // arrives between "route is live" and "handler installed" would be lost + // under the default disposition, wedging the scenario's `step exec`. + let mut usr1 = signal(SignalKind::user_defined1()).expect("install SIGUSR1 handler"); + + let mut rtbh_mgr = RtbhManager::new( + RtbhController::new(RtbhConfig { + eligible_prefixes: vec!["203.0.113.0/24".parse().unwrap()], + blackhole_communities: vec![(65535, 666)], + next_hop_v4: Some("10.222.255.99".parse().unwrap()), + next_hop_v6: None, + max_blackholes: 64, + hold_down: Duration::from_secs(0), + max_ttl: None, + protected_prefixes: Vec::new(), + }), + handle.clone(), + NoopBlackholeJournal, + ); + let mut flowspec_mgr = FlowSpecManager::new( + FlowSpecController::new(FlowSpecConfig { + eligible_prefixes: vec!["203.0.113.0/24".parse().unwrap()], + max_rules: 64, + hold_down: Duration::from_secs(0), + max_ttl: None, + protected_prefixes: Vec::new(), + }), + handle, + NoopFlowSpecJournal, + ); + + assert_eq!( + rtbh_mgr + .apply_add("203.0.113.20".parse().unwrap(), 0, 0) + .await, + ApplyOutcome::Applied + ); + assert_eq!( + flowspec_mgr + .apply_add( + FlowSpecRule { + dst: "203.0.113.21/32".parse().unwrap(), + protocol: Some(17), + dst_port: Some(53), + action: FlowAction::TrafficRate(0.0), + }, + 0, + 0, + ) + .await, + ApplyOutcome::Applied + ); + + tokio::time::sleep(Duration::from_secs(3)).await; // let BIRD import both before the scenario disarms + + usr1.recv().await.expect("SIGUSR1 channel closed"); + + // Mirrors `disarm_signal_task`: fan the disarm out to every manager. + rtbh_mgr.disarm(1000).await; + flowspec_mgr.disarm(1000).await; + + // Prove the process is alive well past the signal — the scenario sends + // SIGUSR1, then polls both for continued liveness (`pgrep`) and for the + // RIB entries disappearing, all inside this window — before exiting + // normally (never crashing on receipt, unlike an unhandled default + // SIGUSR1 disposition would). + tokio::time::sleep(Duration::from_secs(15)).await; +} diff --git a/crates/blackwall-rtbh/tests/armed_protect_reannounce_interop.rs b/crates/blackwall-rtbh/tests/armed_protect_reannounce_interop.rs new file mode 100644 index 0000000..8b41dd4 --- /dev/null +++ b/crates/blackwall-rtbh/tests/armed_protect_reannounce_interop.rs @@ -0,0 +1,122 @@ +//! Armed-mode gate: protected-prefix skip (C1) + FlowSpec rate re-announce +//! on action change (C4) against a real BGP peer (BIRD2). Ignored in CI; run +//! by the lab's `armed-protect-reannounce-bird` scenario. +//! BW_BGP_PEER=10.0.0.1:179 cargo test -p blackwall-rtbh --test armed_protect_reannounce_interop -- --ignored --nocapture +//! +//! Same RFC 8955 §6 "safe update" validation gotcha as `flowspec_interop.rs` +//! (in `blackwall-bgp/tests`): BIRD only accepts a FlowSpec route whose +//! destination is covered by a unicast route from the same origin AS, so we +//! announce a covering `203.0.113.0/24` route before any FlowSpec rule — +//! all three targets below (.7, .8, .9) fall inside it. + +use async_trait::async_trait; +use blackwall_bgp::{spawn, FlowAction, FlowSpecRule, Origin, PeerConfig, Route}; +use blackwall_rtbh::{ + ApplyOutcome, BlackholeOrigin, FlowSpecConfig, FlowSpecController, FlowSpecJournal, + FlowSpecManager, JournalError, +}; +use std::time::Duration; + +/// A no-op journal: this test only exercises the BGP path against real BIRD, +/// not persistence (covered elsewhere with fakes / real Postgres). +struct NoopFlowSpecJournal; + +#[async_trait] +impl FlowSpecJournal for NoopFlowSpecJournal { + async fn record_announce( + &self, + _rule: FlowSpecRule, + _origin: BlackholeOrigin, + _at_ms: u64, + ) -> Result<(), JournalError> { + Ok(()) + } + async fn record_withdraw(&self, _rule: FlowSpecRule, _at_ms: u64) -> Result<(), JournalError> { + Ok(()) + } +} + +fn rule(dst: &str, rate: f32) -> FlowSpecRule { + FlowSpecRule { + dst: dst.parse().unwrap(), + protocol: Some(17), + dst_port: Some(53), + action: FlowAction::TrafficRate(rate), + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "needs a live BGP peer (BIRD); run in the netns lab"] +async fn protected_skip_and_rate_reannounce_on_real_bird() { + let peer: std::net::SocketAddr = std::env::var("BW_BGP_PEER") + .expect("set BW_BGP_PEER=ip:179") + .parse() + .unwrap(); + let (handle, _join) = spawn(PeerConfig { + local_asn: 214_806, + peer_asn: 214_806, + peer_addr: peer, + router_id: "10.222.255.99".parse().unwrap(), + hold_time: 90, + md5: None, + gtsm_hops: None, + local_addr: std::env::var("BW_BGP_LOCAL_ADDR") + .ok() + .map(|s| s.parse().expect("BW_BGP_LOCAL_ADDR must be an IP address")), + }) + .expect("valid iBGP config"); + tokio::time::sleep(Duration::from_secs(3)).await; // let the session establish + + // Covering unicast route (RFC 8955 §6 "safe update" validation): without + // this, BIRD rejects every FlowSpec rule below as unvalidated. + handle + .announce(Route { + prefix: "203.0.113.0/24".parse().unwrap(), + next_hop: "10.0.0.1".parse().unwrap(), + origin: Origin::Igp, + communities: vec![], + large_communities: vec![], + }) + .await + .expect("announce covering route"); + + let mut mgr = FlowSpecManager::new( + FlowSpecController::new(FlowSpecConfig { + eligible_prefixes: vec!["203.0.113.0/24".parse().unwrap()], + max_rules: 64, + hold_down: Duration::from_secs(0), + max_ttl: None, + // .9 is the "own anycast VIP" stand-in (C1): inside the eligible + // prefix, but never mitigated. + protected_prefixes: vec!["203.0.113.9/32".parse().unwrap()], + }), + handle, + NoopFlowSpecJournal, + ); + + // C1: a target inside a `protect`ed prefix is rejected outright — the + // manager never reaches the BGP executor for it, so BIRD's RIB must + // never see it (asserted by the lab scenario). + let protected_outcome = mgr.apply_add(rule("203.0.113.9/32", 0.0), 0, 0).await; + assert!( + matches!(protected_outcome, ApplyOutcome::Rejected(_)), + "protected target must be rejected, got {protected_outcome:?}" + ); + + // Control: a normal eligible target IS announced — proves the absence + // above is the protected-skip guard at work, not e.g. a broken session. + let normal_outcome = mgr.apply_add(rule("203.0.113.7/32", 0.0), 0, 0).await; + assert_eq!(normal_outcome, ApplyOutcome::Applied); + + // C4: re-asserting an already-active rule with a changed rate (0 -> + // 1,000,000 bytes/sec) must re-announce the new action, not silently + // keep serving the stale one. + let initial = mgr.apply_add(rule("203.0.113.8/32", 0.0), 1000, 1000).await; + assert_eq!(initial, ApplyOutcome::Applied); + let reannounced = mgr + .apply_add(rule("203.0.113.8/32", 1_000_000.0), 2000, 2000) + .await; + assert_eq!(reannounced, ApplyOutcome::Applied); + + tokio::time::sleep(Duration::from_secs(5)).await; // let BIRD import + the scenario assert +} diff --git a/crates/blackwall-rtbh/tests/flowspec_auto_interop.rs b/crates/blackwall-rtbh/tests/flowspec_auto_interop.rs index 8ea87ba..47b36e9 100644 --- a/crates/blackwall-rtbh/tests/flowspec_auto_interop.rs +++ b/crates/blackwall-rtbh/tests/flowspec_auto_interop.rs @@ -160,6 +160,7 @@ async fn selection_routes_to_flowspec_and_rtbh_on_real_bird() { max_rules: 64, hold_down: Duration::from_secs(0), max_ttl: None, + protected_prefixes: Vec::new(), }), handle.clone(), NoopFlowSpecJournal, @@ -173,6 +174,7 @@ async fn selection_routes_to_flowspec_and_rtbh_on_real_bird() { max_blackholes: 64, hold_down: Duration::from_secs(0), max_ttl: None, + protected_prefixes: Vec::new(), }), handle, NoopBlackholeJournal, diff --git a/crates/blackwall-rtbh/tests/interop.rs b/crates/blackwall-rtbh/tests/interop.rs index f3008bb..c8cdf21 100644 --- a/crates/blackwall-rtbh/tests/interop.rs +++ b/crates/blackwall-rtbh/tests/interop.rs @@ -68,6 +68,7 @@ async fn blackholes_a_detected_target() { max_blackholes: 64, hold_down: Duration::from_secs(0), max_ttl: None, + protected_prefixes: Vec::new(), }); let mut manager = RtbhManager::new(controller, handle, NoopJournal); diff --git a/crates/blackwall-state/src/lib.rs b/crates/blackwall-state/src/lib.rs index 53ee546..66a68f5 100644 --- a/crates/blackwall-state/src/lib.rs +++ b/crates/blackwall-state/src/lib.rs @@ -1891,6 +1891,7 @@ mod tests { flowtable: None, xdp: None, stateless_tcp_ports: Vec::new(), + protected_prefixes: Vec::new(), shadow: false, } } @@ -1930,6 +1931,7 @@ mod tests { flowtable: None, xdp: None, stateless_tcp_ports: Vec::new(), + protected_prefixes: Vec::new(), shadow: false, } } diff --git a/crates/blackwall-xdp/src/control.rs b/crates/blackwall-xdp/src/control.rs index d46f0e8..20bc492 100644 --- a/crates/blackwall-xdp/src/control.rs +++ b/crates/blackwall-xdp/src/control.rs @@ -90,6 +90,13 @@ pub struct XdpController { prefixes: Vec, max_entries: usize, default_rate_pps: u64, + /// Prefixes whose victim traffic must never trigger auto-mitigation (own + /// anycast VIPs and similar always-safe destinations), from + /// `Policy.protected_prefixes`. Empty (the default) protects nothing + /// extra. Unrelated to `XdpDataplane::set_protected_prefixes`, which + /// guards the SYN-cookie fast path's protected *ports* — a different + /// subsystem. + protected_prefixes: Vec, /// All currently rate-limited sources, with the origin and effective /// rate/burst that installed them. rate_limited: HashMap, @@ -98,6 +105,9 @@ pub struct XdpController { /// Victim target -> the (auto) sources currently rate-limited on its /// behalf, so a `Cleared` for that target knows which sources to release. by_target: HashMap>, + /// Count of detections skipped by the protected-prefix guard (see + /// [`Self::protected_skipped`]). + protected_skipped: u64, } impl XdpController { @@ -111,15 +121,25 @@ impl XdpController { /// and block entries (mirrors the fixed-size eBPF maps). /// * `default_rate_pps` - The packets/second cap (and burst size) applied /// to sources rate-limited automatically from a detection. + /// * `protected_prefixes` - Own anycast VIPs (and similar) that must + /// never trigger auto-mitigation even when the victim also falls + /// inside `prefixes`; from `Policy.protected_prefixes`. #[must_use] - pub fn new(prefixes: Vec, max_entries: usize, default_rate_pps: u64) -> Self { + pub fn new( + prefixes: Vec, + max_entries: usize, + default_rate_pps: u64, + protected_prefixes: Vec, + ) -> Self { Self { prefixes, max_entries, default_rate_pps, + protected_prefixes, rate_limited: HashMap::new(), blocked_nets: HashMap::new(), by_target: HashMap::new(), + protected_skipped: 0, } } @@ -264,6 +284,68 @@ impl XdpController { self.total_active() >= self.max_entries } + /// Whether `net` is currently blocked (manually or automatically). + /// + /// Lets a caller (e.g. the manager) check freshness *before* calling + /// [`Self::manual_block`], to decide whether a subsequent executor + /// failure should [`Self::rollback`] a brand-new insert or leave an + /// already-active entry untouched. + #[must_use] + pub fn is_blocked(&self, net: IpNet) -> bool { + self.blocked_nets.contains_key(&net) + } + + /// Whether `src` is currently rate-limited (manually or automatically). + /// + /// See [`Self::is_blocked`] for why this is exposed. + #[must_use] + pub fn is_rate_limited(&self, src: IpAddr) -> bool { + self.rate_limited.contains_key(&src) + } + + /// Undo a just-inserted active entry after its executor apply failed + /// (C2: commit-after-confirm). + /// + /// Removes the entry `action` describes from the active set and emits + /// nothing — the map write never took, so there is nothing to unwind on + /// the data-plane side. Only meaningful for [`XdpAction::RateLimit`] and + /// [`XdpAction::Block`] (the two "insert" variants); called on an + /// [`XdpAction::Unblock`] or [`XdpAction::ClearRate`] it is a no-op, + /// since those represent a removal that already happened in the + /// controller's bookkeeping regardless of the executor outcome. + /// + /// Callers must only invoke this for an `action` known to be a brand-new + /// insert (see [`Self::is_blocked`]/[`Self::is_rate_limited`]) — calling + /// it after a re-assertion or a param upgrade of an already-active entry + /// would incorrectly drop state that predates this call. Mirrors + /// `blackwall_rtbh::controller::RtbhController::rollback`. + pub fn rollback(&mut self, action: &XdpAction) { + match *action { + XdpAction::RateLimit { src, victim, .. } => { + self.rate_limited.remove(&src); + if let Some(victim) = victim { + if let Some(sources) = self.by_target.get_mut(&victim) { + sources.remove(&src); + } + } + } + XdpAction::Block { net } => { + self.blocked_nets.remove(&net); + } + XdpAction::Unblock { .. } | XdpAction::ClearRate { .. } => {} + } + } + + /// Number of detections skipped because the victim fell inside a + /// configured protected prefix (own anycast VIP or similar always-safe + /// destination) — the anycast self-protection guard in + /// [`Self::handle_detection`]. Surfaced for `/metrics` + /// (`blackwall_mitigations_protected_skipped_total{plane="xdp"}`). + #[must_use] + pub fn protected_skipped(&self) -> u64 { + self.protected_skipped + } + /// Snapshot the active set (for reconcile mirroring and restart rehydration). #[must_use] pub fn active_entries(&self) -> Vec<(XdpAction, XdpOrigin)> { @@ -332,6 +414,22 @@ impl XdpController { } fn handle_detection(&mut self, d: &Detection) -> Vec { + // Anycast self-protection (C1): a protected victim (own VIP) must + // never trigger auto-mitigation, even when it also falls inside an + // own prefix — checked BEFORE the own-prefix eligibility check, and + // decisive. + if self + .protected_prefixes + .iter() + .any(|p| p.contains(&d.target)) + { + tracing::warn!( + target = %d.target, + "XDP: victim in a protected prefix; skipping (never mitigate own service)" + ); + self.protected_skipped = self.protected_skipped.saturating_add(1); + return Vec::new(); + } if !self.prefixes.iter().any(|p| p.contains(&d.target)) { return Vec::new(); } @@ -431,7 +529,7 @@ mod tests { #[test] fn opened_on_own_victim_rate_limits_each_source() { - let mut c = XdpController::new(own(), 100, 1000); + let mut c = XdpController::new(own(), 100, 1000, Vec::new()); let acts = c.on_detection(&DetectionEvent::Opened(det( "203.0.113.7", vec!["198.51.100.9", "198.51.100.10"], @@ -444,7 +542,7 @@ mod tests { #[test] fn detection_on_foreign_victim_is_ignored() { - let mut c = XdpController::new(own(), 100, 1000); + let mut c = XdpController::new(own(), 100, 1000, Vec::new()); let acts = c.on_detection(&DetectionEvent::Opened(det( "8.8.8.8", vec!["198.51.100.9"], @@ -452,15 +550,41 @@ mod tests { assert!(acts.is_empty()); } + #[test] + fn protected_victim_is_skipped_even_when_own() { + // 203.0.113.53 is inside own address space but is carved out as a + // protected VIP: a detection against it must never rate-limit + // sources on its behalf. + let mut c = XdpController::new(own(), 100, 1000, vec!["203.0.113.53/32".parse().unwrap()]); + let acts = c.on_detection(&DetectionEvent::Opened(det( + "203.0.113.53", + vec!["198.51.100.9"], + ))); + assert!(acts.is_empty(), "protected VIP must not trigger mitigation"); + assert!(c.active_entries().is_empty()); + assert_eq!(c.protected_skipped(), 1); + } + + #[test] + fn unprotected_own_victim_still_mitigates() { + let mut c = XdpController::new(own(), 100, 1000, vec!["203.0.113.53/32".parse().unwrap()]); + let acts = c.on_detection(&DetectionEvent::Opened(det( + "203.0.113.7", + vec!["198.51.100.9"], + ))); + assert_eq!(acts.len(), 1); + assert_eq!(c.protected_skipped(), 0); + } + #[test] fn manual_block_of_own_prefix_is_rejected() { - let mut c = XdpController::new(own(), 100, 1000); + let mut c = XdpController::new(own(), 100, 1000, Vec::new()); assert!(c.manual_block("203.0.113.5/32".parse().unwrap()).is_err()); } #[test] fn cap_defers_beyond_max_entries() { - let mut c = XdpController::new(own(), 1, 1000); + let mut c = XdpController::new(own(), 1, 1000, Vec::new()); let acts = c.on_detection(&DetectionEvent::Opened(det( "203.0.113.7", vec!["198.51.100.9", "198.51.100.10"], @@ -470,7 +594,7 @@ mod tests { #[test] fn cleared_emits_clear_rate_for_each_recorded_source() { - let mut c = XdpController::new(own(), 100, 1000); + let mut c = XdpController::new(own(), 100, 1000, Vec::new()); c.on_detection(&DetectionEvent::Opened(det( "203.0.113.7", vec!["198.51.100.9", "198.51.100.10"], @@ -488,7 +612,7 @@ mod tests { #[test] fn manual_rate_limit_survives_auto_clear() { - let mut c = XdpController::new(own(), 100, 1000); + let mut c = XdpController::new(own(), 100, 1000, Vec::new()); c.on_detection(&DetectionEvent::Opened(det( "203.0.113.7", vec!["198.51.100.9"], @@ -505,14 +629,14 @@ mod tests { #[test] fn manual_block_at_capacity_is_rejected() { - let mut c = XdpController::new(own(), 1, 1000); + let mut c = XdpController::new(own(), 1, 1000, Vec::new()); c.manual_block("198.51.100.0/24".parse().unwrap()).unwrap(); assert!(c.manual_block("198.51.101.0/24".parse().unwrap()).is_err()); } #[test] fn manual_unblock_is_idempotent() { - let mut c = XdpController::new(own(), 100, 1000); + let mut c = XdpController::new(own(), 100, 1000, Vec::new()); let net = "198.51.100.0/24".parse().unwrap(); c.manual_block(net).unwrap(); assert!(c.manual_unblock(net).is_ok()); @@ -523,7 +647,7 @@ mod tests { #[test] fn shared_source_kept_until_last_victim_clears() { // Source X floods two of our own victims, A and B, simultaneously. - let mut c = XdpController::new(own(), 100, 1000); + let mut c = XdpController::new(own(), 100, 1000, Vec::new()); c.on_detection(&DetectionEvent::Opened(det( "203.0.113.7", // A vec!["198.51.100.9"], @@ -562,13 +686,13 @@ mod tests { fn manual_block_of_supernet_covering_own_space_is_rejected() { // Own space is 203.0.113.0/24; a supernet block of 203.0.0.0/16 would // swallow it — the self-block guard must catch this direction too. - let mut c = XdpController::new(own(), 100, 1000); + let mut c = XdpController::new(own(), 100, 1000, Vec::new()); assert!(c.manual_block("203.0.0.0/16".parse().unwrap()).is_err()); } #[test] fn manual_clear_rate_removes_source() { - let mut c = XdpController::new(own(), 100, 1000); + let mut c = XdpController::new(own(), 100, 1000, Vec::new()); let addr: IpAddr = "198.51.100.9".parse().unwrap(); c.manual_rate_limit(addr, 500, 500).unwrap(); assert_eq!(c.active_entries().len(), 1); @@ -583,7 +707,7 @@ mod tests { #[test] fn manual_clear_rate_is_idempotent_for_unknown_source() { - let mut c = XdpController::new(own(), 100, 1000); + let mut c = XdpController::new(own(), 100, 1000, Vec::new()); let addr: IpAddr = "198.51.100.9".parse().unwrap(); let act = c.manual_clear_rate(addr).unwrap(); assert!(matches!(act, XdpAction::ClearRate { src } if src == addr)); @@ -595,7 +719,7 @@ mod tests { // Rate-limit a source via a detection (populates by_target), then // manually clear it. A later `Cleared` for that target must not // re-emit a ClearRate for the already-cleared source. - let mut c = XdpController::new(own(), 100, 1000); + let mut c = XdpController::new(own(), 100, 1000, Vec::new()); c.on_detection(&DetectionEvent::Opened(det( "203.0.113.7", vec!["198.51.100.9"], @@ -616,7 +740,7 @@ mod tests { #[test] fn manual_rate_limit_preserved_in_active_entries() { - let mut c = XdpController::new(own(), 100, 1000); + let mut c = XdpController::new(own(), 100, 1000, Vec::new()); let addr: IpAddr = "198.51.100.9".parse().unwrap(); c.manual_rate_limit(addr, 500, 500).unwrap(); @@ -644,7 +768,7 @@ mod tests { // forever. With the fix, `mark_resumed` repopulates `by_target` from // the resumed action's `victim`, so `Cleared` emits `ClearRate` for // the source, exactly as it would have in the pre-restart session. - let mut c = XdpController::new(own(), 100, 1000); + let mut c = XdpController::new(own(), 100, 1000, Vec::new()); let victim: IpAddr = "203.0.113.7".parse().unwrap(); let src: IpAddr = "198.51.100.9".parse().unwrap(); @@ -680,7 +804,7 @@ mod tests { // A resumed manual rate-limit (no victim) must not create a // `by_target` entry: nothing should auto-clear it, since it was // never tied to a detection in the first place. - let mut c = XdpController::new(own(), 100, 1000); + let mut c = XdpController::new(own(), 100, 1000, Vec::new()); let victim: IpAddr = "203.0.113.7".parse().unwrap(); let src: IpAddr = "198.51.100.9".parse().unwrap(); diff --git a/crates/blackwall-xdp/src/manager.rs b/crates/blackwall-xdp/src/manager.rs index 76265e8..0e584f5 100644 --- a/crates/blackwall-xdp/src/manager.rs +++ b/crates/blackwall-xdp/src/manager.rs @@ -108,18 +108,47 @@ enum MirrorKey { Net(IpNet), } +/// Outcome of [`XdpManager::execute_and_journal`]. +/// +/// The auto path (`on_detection`, which always passes `fresh = true`) +/// ignores this — auto re-detection naturally compensates for a skip on its +/// next tick. [`XdpManager::apply_add`] and [`XdpManager::apply_rate_limit`] +/// (the manual insert paths) consume it to report a truthful +/// [`ApplyOutcome`] rather than always claiming [`ApplyOutcome::Applied`]. +/// There is no rate-capped variant here: unlike `RtbhManager`/ +/// `FlowSpecManager`, `XdpManager` has no [`crate`]-level cross-plane rate +/// limiter attached. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ExecOutcome { + /// The action reached the executor; a journal-mirror failure afterward + /// is still `Applied` (self-healed via + /// [`XdpManager::retry_pending_mirror`]) — the live entry is active + /// either way. + Applied, + /// Skipped: the manager is [`XdpManager::disarm`]ed (C5), record-only. + /// A fresh insert's controller entry was rolled back. + Disarmed, + /// Attempted and failed at the [`XdpExecutor`] (C2). A fresh insert's + /// controller entry was rolled back. + Failed, +} + /// Single-owner XDP manager. /// /// Owns the pure [`XdpController`] plus the I/O boundary: it executes the /// controller's decisions on an [`XdpExecutor`] and mirrors state via an -/// [`XdpJournal`]. An executor failure is logged and the action is not -/// journaled — the controller entry is kept in memory while the map write -/// itself is never retried automatically (a known limitation, mirroring -/// `RtbhManager`'s BGP-failure behavior). A journal failure after a -/// successful executor operation is logged, never causes a live entry to be -/// removed, and is queued as a `MirrorOp` for a bounded self-heal retry on -/// the next [`XdpManager::tick`] — the executor outcome is never re-issued, -/// only the mirror write. +/// [`XdpJournal`]. An executor failure is logged, the action is not +/// journaled, and — for a brand-new insert (a fresh `RateLimit`/`Block`, +/// never a re-assertion or param upgrade of an already-active entry) — the +/// controller's freshly-added active entry is rolled back via +/// [`XdpController::rollback`] (C2: commit-after-confirm), mirroring +/// `RtbhManager`'s BGP-failure rollback. This is not a retry mechanism: the +/// map write itself is never retried automatically, but a future detection +/// for the same source is no longer deduped against a phantom active entry. +/// A journal failure after a successful executor operation is logged, never +/// causes a live entry to be removed, and is queued as a `MirrorOp` for a +/// bounded self-heal retry on the next [`XdpManager::tick`] — the executor +/// outcome is never re-issued, only the mirror write. pub struct XdpManager { controller: XdpController, executor: E, @@ -128,6 +157,21 @@ pub struct XdpManager { /// succeeded; retried (never re-issued to the executor) by /// [`XdpManager::retry_pending_mirror`] on the next tick. pending_mirror: Vec, + /// Count of executor applies that failed, each counted here (see + /// [`Self::apply_failures`]); a fresh insert among them is also rolled + /// back (see [`XdpController::rollback`]). + apply_failures: u64, + /// One-way in-daemon disarm kill switch (C5), flipped by [`Self::disarm`]. + /// While set, [`Self::execute_and_journal`] skips every new install + /// (`Block`/`RateLimit`, never reaching [`Self::executor`]) while + /// detection keeps running unchanged. There is no re-arm entry point; a + /// fresh process (restart) is the only way back to armed. Mirrors + /// `blackwall_rtbh::manager::RtbhManager::disarmed`. + disarmed: bool, + /// Count of installs skipped because [`Self::disarmed`] was set (C5) — + /// a SKIP (never attempted), distinct from [`Self::apply_failures`] + /// (attempted and failed at the executor). See [`Self::disarmed_skips`]. + disarmed_skips: u64, } impl XdpManager { @@ -138,6 +182,9 @@ impl XdpManager { executor, journal, pending_mirror: Vec::new(), + apply_failures: 0, + disarmed: false, + disarmed_skips: 0, } } @@ -146,7 +193,12 @@ impl XdpManager { pub async fn on_detection(&mut self, ev: &blackwall_flow::DetectionEvent, wall_now: u64) { let actions = self.controller.on_detection(ev); for action in actions { - self.execute_and_journal(action, XdpOrigin::Auto, wall_now) + // `XdpController::on_detection` only ever emits a `RateLimit` + // for a source it just freshly inserted (an already-active + // source is deduplicated before an action is produced — see + // `XdpController::handle_detection`), so every action reaching + // here is always a fresh insert. + self.execute_and_journal(action, XdpOrigin::Auto, wall_now, true) .await; } } @@ -157,11 +209,22 @@ impl XdpManager { /// [`ApplyOutcome::Rejected`] if `net` overlaps an own prefix, or /// [`ApplyOutcome::Deferred`] if the manager is at capacity. pub async fn apply_add(&mut self, net: IpNet, wall_now: u64) -> ApplyOutcome { + let fresh = !self.controller.is_blocked(net); match self.controller.manual_block(net) { Ok(action) => { - self.execute_and_journal(action, XdpOrigin::Manual, wall_now) - .await; - ApplyOutcome::Applied + match self + .execute_and_journal(action, XdpOrigin::Manual, wall_now, fresh) + .await + { + ExecOutcome::Applied => ApplyOutcome::Applied, + // One-way: retrying is pointless until re-armed via restart. + ExecOutcome::Disarmed => ApplyOutcome::Rejected(format!( + "{net} was not applied: manager is disarmed (C5)" + )), + // No auto re-detection exists for a manual request, so a + // failed executor apply must be retried, not marked applied. + ExecOutcome::Failed => ApplyOutcome::Deferred, + } } Err(e) if self.controller.overlaps_own_prefix(net) => ApplyOutcome::Rejected(e), Err(_) => ApplyOutcome::Deferred, @@ -173,7 +236,10 @@ impl XdpManager { pub async fn apply_remove(&mut self, net: IpNet, wall_now: u64) -> ApplyOutcome { match self.controller.manual_unblock(net) { Ok(action) => { - self.execute_and_journal(action, XdpOrigin::Manual, wall_now) + // `Unblock` is a removal, not an insert — nothing for a + // failed apply to roll back (`XdpController::rollback` is a + // no-op for this variant regardless). + self.execute_and_journal(action, XdpOrigin::Manual, wall_now, false) .await; ApplyOutcome::Applied } @@ -186,7 +252,9 @@ impl XdpManager { pub async fn apply_clear_rate(&mut self, src: IpAddr, wall_now: u64) -> ApplyOutcome { match self.controller.manual_clear_rate(src) { Ok(action) => { - self.execute_and_journal(action, XdpOrigin::Manual, wall_now) + // `ClearRate` is a removal, not an insert — see the + // `apply_remove` comment above. + self.execute_and_journal(action, XdpOrigin::Manual, wall_now, false) .await; ApplyOutcome::Applied } @@ -205,11 +273,22 @@ impl XdpManager { burst: u64, wall_now: u64, ) -> ApplyOutcome { + let fresh = !self.controller.is_rate_limited(src); match self.controller.manual_rate_limit(src, pps, burst) { Ok(action) => { - self.execute_and_journal(action, XdpOrigin::Manual, wall_now) - .await; - ApplyOutcome::Applied + match self + .execute_and_journal(action, XdpOrigin::Manual, wall_now, fresh) + .await + { + ExecOutcome::Applied => ApplyOutcome::Applied, + // One-way: retrying is pointless until re-armed via restart. + ExecOutcome::Disarmed => ApplyOutcome::Rejected(format!( + "{src} was not rate-limited: manager is disarmed (C5)" + )), + // No auto re-detection exists for a manual request, so a + // failed executor apply must be retried, not marked applied. + ExecOutcome::Failed => ApplyOutcome::Deferred, + } } Err(_) => ApplyOutcome::Deferred, } @@ -248,6 +327,77 @@ impl XdpManager { self.controller.active_entries() } + /// Number of detections skipped by the controller's protected-prefix + /// guard (own anycast VIPs never mitigated). Surfaced for `/metrics`; + /// see `blackwall_rtbh::manager::RtbhManager::protected_skipped` for the + /// analogous RTBH accessor. + #[must_use] + pub fn protected_skipped(&self) -> u64 { + self.controller.protected_skipped() + } + + /// Count of executor (eBPF-map) applies that failed (C2). A failure for + /// a brand-new insert also rolls back the controller's freshly-added + /// active entry (see [`XdpController::rollback`]) so the control plane + /// never believes an unconfirmed map write is active. Surfaced for + /// `/metrics` as `blackwall_xdp_apply_failures_total`, mirroring + /// `blackwall_rtbh::manager::RtbhManager::apply_failures`. + #[must_use] + pub fn apply_failures(&self) -> u64 { + self.apply_failures + } + + /// In-daemon disarm kill switch (C5): withdraw every currently-active + /// block/rate-limit and switch to record-only for the rest of this + /// process's life. + /// + /// Mirrors `blackwall_rtbh::manager::RtbhManager::disarm`: each active + /// entry is undone on the executor best-effort (an apply `Err` is logged + /// and the sweep continues), no journal write happens (disarm is + /// runtime-only — a restart re-arms and [`Self::reapply_active`]s the + /// same active set), and once disarmed every subsequent new + /// `Block`/`RateLimit` install is skipped in [`Self::execute_and_journal`] + /// and counted in [`Self::disarmed_skips`] — an `Unblock`/`ClearRate` + /// (a removal, not an install) is never gated. One-way and idempotent. + /// + /// `mono_now` is accepted for symmetry with the RTBH/FlowSpec managers' + /// `disarm` entry points; it is unused here. + pub async fn disarm(&mut self, _mono_now: u64) { + if self.disarmed { + return; + } + self.disarmed = true; + let actives: Vec = self + .controller + .active_entries() + .into_iter() + .map(|(action, _origin)| action) + .collect(); + for action in actives { + let inverse = match action { + XdpAction::Block { net } => self.controller.manual_unblock(net), + XdpAction::RateLimit { src, .. } => self.controller.manual_clear_rate(src), + // `active_entries` never yields a removal variant. + XdpAction::Unblock { .. } | XdpAction::ClearRate { .. } => continue, + }; + if let Ok(inv) = inverse { + if let Err(e) = self.executor.apply(inv).await { + tracing::warn!(error = %e, ?action, "XDP: disarm apply (withdraw) failed; continuing best-effort"); + } + } + } + tracing::warn!("XDP: DISARMED — mitigations withdrawn, now recording only"); + } + + /// Count of new installs (`Block`/`RateLimit`) skipped because the + /// manager was [`Self::disarm`]ed (C5) — a SKIP (never attempted), + /// distinct from [`Self::apply_failures`] (attempted and failed at the + /// executor). + #[must_use] + pub fn disarmed_skips(&self) -> u64 { + self.disarmed_skips + } + /// Queue a failed mirror write for self-heal, coalescing by identity /// (source or network). /// @@ -263,10 +413,51 @@ impl XdpManager { } /// Execute one controller action on the executor and mirror it into the journal. - async fn execute_and_journal(&mut self, action: XdpAction, origin: XdpOrigin, wall_now: u64) { + /// + /// `fresh` marks whether `action` is a brand-new insert (a first-time + /// `RateLimit`/`Block`) rather than a re-assertion or param upgrade of an + /// already-active entry, or a removal (`Unblock`/`ClearRate`). On an + /// executor failure, a fresh insert's freshly-added active entry is + /// rolled back (C2: commit-after-confirm) so the control plane never + /// believes an unconfirmed map write is active; a non-fresh action has no + /// just-inserted entry to undo, so nothing is rolled back. Either way the + /// failure is counted in `apply_failures`. + async fn execute_and_journal( + &mut self, + action: XdpAction, + origin: XdpOrigin, + wall_now: u64, + fresh: bool, + ) -> ExecOutcome { + if self.disarmed + && matches!( + action, + XdpAction::Block { .. } | XdpAction::RateLimit { .. } + ) + { + tracing::warn!( + ?action, + "XDP: disarmed (C5); skipping install, recording only" + ); + if fresh { + self.controller.rollback(&action); + } + self.disarmed_skips = self.disarmed_skips.saturating_add(1); + return ExecOutcome::Disarmed; + } if let Err(e) = self.executor.apply(action).await { - tracing::warn!(error = %e, ?action, "XDP: executor apply failed; not journaling"); - return; + self.apply_failures = self.apply_failures.saturating_add(1); + if fresh { + tracing::warn!( + error = %e, + ?action, + "XDP: executor apply failed; rolling back active entry, not journaling" + ); + self.controller.rollback(&action); + } else { + tracing::warn!(error = %e, ?action, "XDP: executor apply failed; not journaling"); + } + return ExecOutcome::Failed; } if let Err(e) = self.journal.record(&action, origin, wall_now).await { tracing::error!(error = %e, ?action, "XDP: journal write failed after apply; keeping active"); @@ -276,6 +467,7 @@ impl XdpManager { at_ms: wall_now, }); } + ExecOutcome::Applied } /// Drain-retry queued mirror writes left over from a transient journal failure. @@ -324,11 +516,21 @@ mod tests { struct FakeExecutor { applied: Mutex>, fail: bool, + /// If `Some(n)`, the n-th call (1-indexed) onward fails; earlier + /// calls succeed. Used to simulate a successful fresh insert + /// followed by a failing upgrade apply (Fix 2 regression test). + fail_from_call: Option, + call_count: Mutex, } #[async_trait] impl XdpExecutor for FakeExecutor { async fn apply(&self, action: XdpAction) -> Result<(), XdpExecError> { - if self.fail { + let call_no = { + let mut count = self.call_count.lock().unwrap(); + *count += 1; + *count + }; + if self.fail || self.fail_from_call.is_some_and(|from| call_no >= from) { return Err(XdpExecError); } self.applied.lock().unwrap().push(action); @@ -397,7 +599,7 @@ mod tests { fn mgr(fail_exec: bool, fail_journal: bool) -> XdpManager { XdpManager::new( - XdpController::new(own(), 100, 1000), + XdpController::new(own(), 100, 1000, Vec::new()), FakeExecutor { fail: fail_exec, ..Default::default() @@ -411,7 +613,7 @@ mod tests { fn mgr_transient_journal_failures(n: usize) -> XdpManager { XdpManager::new( - XdpController::new(own(), 100, 1000), + XdpController::new(own(), 100, 1000, Vec::new()), FakeExecutor::default(), FakeJournal { fail_calls_remaining: Mutex::new(n), @@ -480,6 +682,46 @@ mod tests { ); } + #[tokio::test] + async fn executor_failure_does_not_leave_a_phantom_active_entry() { + // The executor (map write) fails: the kernel never installed the + // rate limit, so the control plane must NOT believe it did (C2) — + // the freshly-inserted active entry must be rolled back, not left as + // a phantom "active" mitigation that dedupes future detections. + let mut m = mgr(true, false); // executor fails + m.on_detection( + &DetectionEvent::Opened(det("203.0.113.7", vec!["198.51.100.9"])), + 1000, + ) + .await; + assert!( + m.active().is_empty(), + "a failed executor apply must not leave a phantom active entry" + ); + assert_eq!(m.apply_failures(), 1); + + // A subsequent identical detection re-attempts (not deduped against + // a phantom active entry). + m.on_detection( + &DetectionEvent::Opened(det("203.0.113.7", vec!["198.51.100.9"])), + 2000, + ) + .await; + assert_eq!(m.apply_failures(), 2); + } + + #[tokio::test] + async fn successful_apply_activates_with_no_apply_failures() { + let mut m = mgr(false, false); + m.on_detection( + &DetectionEvent::Opened(det("203.0.113.7", vec!["198.51.100.9"])), + 1000, + ) + .await; + assert_eq!(m.active().len(), 1); + assert_eq!(m.apply_failures(), 0); + } + #[tokio::test] async fn reapply_active_reissues_executor_calls_but_not_journal() { let mut m = mgr(false, false); @@ -522,7 +764,7 @@ mod tests { #[tokio::test] async fn apply_add_defers_at_capacity() { let mut m = XdpManager::new( - XdpController::new(own(), 1, 1000), + XdpController::new(own(), 1, 1000, Vec::new()), FakeExecutor::default(), FakeJournal::default(), ); @@ -604,6 +846,146 @@ mod tests { ); } + #[tokio::test] + async fn disarm_withdraws_all_and_switches_to_record_only() { + let mut m = mgr(false, false); + m.on_detection( + &DetectionEvent::Opened(det("203.0.113.7", vec!["198.51.100.9"])), + 1000, + ) + .await; + assert_eq!(m.active().len(), 1); + + m.disarm(2_000).await; + + assert!(m.active().is_empty(), "disarm must clear the active set"); + // The withdraw (ClearRate) reached the executor. + assert!(m + .executor() + .applied + .lock() + .unwrap() + .iter() + .any(|a| matches!(a, XdpAction::ClearRate { .. }))); + + // A subsequent detection is recorded, not executed. + let applied_before = m.executor().applied.lock().unwrap().len(); + m.on_detection( + &DetectionEvent::Opened(det("203.0.113.7", vec!["198.51.100.10"])), + 3000, + ) + .await; + assert!(m.active().is_empty()); + assert_eq!(m.executor().applied.lock().unwrap().len(), applied_before); + assert_eq!(m.apply_failures(), 0); + assert_eq!(m.disarmed_skips(), 1); + } + + #[tokio::test] + async fn apply_add_while_disarmed_is_rejected_not_applied() { + // C5 + final-review fix: a manual add while disarmed must be + // classified Rejected (retrying is pointless — there is no re-arm + // entry point), never Applied — an "applied" operator-request row + // is never retried, which would silently lose operator intent. + let mut m = mgr(false, false); + m.disarm(0).await; + + let outcome = m.apply_add("198.51.100.0/24".parse().unwrap(), 1_000).await; + match &outcome { + ApplyOutcome::Rejected(reason) => { + assert!( + reason.contains("disarmed"), + "reason should mention 'disarmed': {reason}" + ); + } + other => panic!("disarmed manual add must be Rejected, not {other:?}"), + } + assert!( + m.active().is_empty(), + "a disarmed manual add must not leave a phantom active entry" + ); + } + + #[tokio::test] + async fn apply_rate_limit_while_disarmed_is_rejected_not_applied() { + // Same as above, for the apply_rate_limit manual path. + let mut m = mgr(false, false); + m.disarm(0).await; + + let addr: IpAddr = "198.51.100.9".parse().unwrap(); + let outcome = m.apply_rate_limit(addr, 500, 500, 1_000).await; + match &outcome { + ApplyOutcome::Rejected(reason) => { + assert!( + reason.contains("disarmed"), + "reason should mention 'disarmed': {reason}" + ); + } + other => panic!("disarmed manual rate-limit must be Rejected, not {other:?}"), + } + assert!(m.active().is_empty()); + } + + #[tokio::test] + async fn apply_add_when_executor_fails_is_deferred_not_applied() { + // final-review fix: a manual add whose executor apply fails has no + // auto re-detection to compensate — it must be Deferred (retried + // next tick), not marked Applied. + let mut m = mgr(true, false); // executor always fails + let outcome = m.apply_add("198.51.100.0/24".parse().unwrap(), 0).await; + assert_eq!( + outcome, + ApplyOutcome::Deferred, + "a failed executor apply on a manual add must be Deferred, not Applied" + ); + assert!(m.active().is_empty()); + assert_eq!(m.apply_failures(), 1); + } + + #[tokio::test] + async fn upgrade_apply_failure_does_not_evict_existing_active_entry() { + // Fix 2 (final-review regression test): an already-active + // (successfully-applied) rate-limit entry, then an UPGRADE apply + // (same source, new pps/burst) whose executor FAILS must NOT evict + // the pre-existing active entry — only a FRESH insert's rollback + // undoes anything (`fresh` is computed BEFORE the controller call + // and rollback only fires `if fresh`). + let mut m = XdpManager::new( + XdpController::new(own(), 100, 1000, Vec::new()), + FakeExecutor { + // 1st call (the fresh insert) succeeds; 2nd call (the + // upgrade) and onward fail. + fail_from_call: Some(2), + ..Default::default() + }, + FakeJournal::default(), + ); + let addr: IpAddr = "198.51.100.9".parse().unwrap(); + + let first = m.apply_rate_limit(addr, 500, 500, 0).await; + assert_eq!(first, ApplyOutcome::Applied, "the fresh insert succeeds"); + assert_eq!(m.active().len(), 1); + + let second = m.apply_rate_limit(addr, 999, 999, 1_000).await; + assert_eq!( + second, + ApplyOutcome::Deferred, + "a failed upgrade apply must be Deferred, not Applied" + ); + assert_eq!( + m.active().len(), + 1, + "the pre-existing entry must still be active, not evicted by a failed upgrade" + ); + assert!( + m.active().iter().any( + |(action, _)| matches!(action, XdpAction::RateLimit { src, .. } if *src == addr) + ), + "the surviving entry must still be for the same source" + ); + assert_eq!(m.apply_failures(), 1); + } + #[tokio::test] async fn noop_journal_records_nothing_and_succeeds() { // The shadow-mode journal must accept every record call without error diff --git a/scripts/build-lab-tests.sh b/scripts/build-lab-tests.sh index 719b5d4..1628234 100755 --- a/scripts/build-lab-tests.sh +++ b/scripts/build-lab-tests.sh @@ -33,6 +33,14 @@ pairs=( if grep -rqs flowspec_auto_interop crates/blackwall-rtbh/tests/ 2>/dev/null; then pairs+=("blackwall-rtbh flowspec_auto_interop") fi +# M1 armed-mode gate drivers (protected skip/rate re-announce C1+C4, and +# SIGUSR1 disarm C5) ship in blackwall-rtbh too; same before/after guard. +if grep -rqs armed_protect_reannounce_interop crates/blackwall-rtbh/tests/ 2>/dev/null; then + pairs+=("blackwall-rtbh armed_protect_reannounce_interop") +fi +if grep -rqs armed_disarm_interop crates/blackwall-rtbh/tests/ 2>/dev/null; then + pairs+=("blackwall-rtbh armed_disarm_interop") +fi mkdir -p target/debug/lab-tests for pair in "${pairs[@]}"; do