diff --git a/crates/maxplayer-core/src/delivery_orchestrator.rs b/crates/maxplayer-core/src/delivery_orchestrator.rs index f46c4118a..972b36723 100644 --- a/crates/maxplayer-core/src/delivery_orchestrator.rs +++ b/crates/maxplayer-core/src/delivery_orchestrator.rs @@ -583,7 +583,10 @@ fn drive_acp_agent( &inputs.prompt, workdir, &identity, - AgentRunTimeout::JobDeadline(timeout), + AgentRunTimeout::JobDeadline { + remaining: timeout, + deadline_unix: inputs.deadline_unix, + }, Some(env.clone()), // Servers only: the HOST mounted this container, so there is nothing to mount here. JobAttachments { mcp_servers: inputs.mcp_servers.clone(), extra_mounts: Vec::new() }, diff --git a/crates/maxplayer-core/src/held_tool.rs b/crates/maxplayer-core/src/held_tool.rs index fdd1ed798..6c841e162 100644 --- a/crates/maxplayer-core/src/held_tool.rs +++ b/crates/maxplayer-core/src/held_tool.rs @@ -1701,7 +1701,7 @@ mod live_tests { let McpServer::Stdio(entry) = &attachments.mcp_servers[drive] else { panic!("stdio entries") }; let mut command = vec![entry.command.clone()]; command.extend(entry.args.iter().cloned()); - let prepared = prepare_launch(&command, &policy, &workdir, &identity, Duration::from_secs(120)) + let prepared = prepare_launch(&command, &policy, &workdir, &identity, Duration::from_secs(120), None) .await .expect("prepare the launch"); let mut servers = prepared.mcp_servers.clone(); @@ -1970,7 +1970,16 @@ mod live_tests { prompt, &workdir, &identity, - crate::seller_exec::AgentRunTimeout::JobDeadline(Duration::from_secs(420)), + crate::seller_exec::AgentRunTimeout::JobDeadline { + remaining: Duration::from_secs(420), + // The absolute second this window ends at, taken HERE at construction — the + // one instant at which `now + remaining` IS the deadline rather than a guess. + deadline_unix: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("a clock") + .as_secs() + + 420, + }, None, attachments, ) diff --git a/crates/maxplayer-core/src/lib.rs b/crates/maxplayer-core/src/lib.rs index e6bfa48c0..8693a9290 100644 --- a/crates/maxplayer-core/src/lib.rs +++ b/crates/maxplayer-core/src/lib.rs @@ -106,6 +106,23 @@ mod sandbox_dns_live; /// against its base to tell a regression from a preexisting condition. HARNESS ONLY. #[cfg(all(test, feature = "acp", feature = "wallet"))] mod sandbox_egress_live; +/// Offline validation of a *saved* live containment matrix: the record a live run writes down, and +/// the checks that say whether it actually covers every required case. +/// +/// The live gates in `tests/sandbox_netns_live.rs` are `#[ignore]`d, so an ordinary `cargo test` +/// reports them as ignored and proves nothing about containment. This module is what the offline +/// acceptance entrypoint runs instead: it fails on a missing, duplicated, unknown or unscored case, +/// and on a record that does not name the commit, artifact and host it came from. Ungated for the +/// same reason as the policy modules below — a gate that can be compiled out is not a gate. +pub mod sandbox_evidence; +/// The same policy, on the interface the packets actually leave by. +/// +/// `sandbox_net`'s rules live on the host kernel's `OUTPUT` chain, which a gVisor payload never +/// traverses: it runs its own netstack and hands finished packets to the namespace's veth. This +/// module translates the very same rendered policy into `tc`/`flower` filters on that veth, so the +/// containment stops depending on which runtime the job was launched under. Ungated for the same +/// reason as the renderer it derives from. +pub mod sandbox_iface; /// Host-side network containment for a docker job (#797): which destinations a job may reach, and /// the `iptables` rules that enforce it on the two chains container traffic actually splits across. /// diff --git a/crates/maxplayer-core/src/sandbox_dns_live.rs b/crates/maxplayer-core/src/sandbox_dns_live.rs index 8235f10c4..37ea992f7 100644 --- a/crates/maxplayer-core/src/sandbox_dns_live.rs +++ b/crates/maxplayer-core/src/sandbox_dns_live.rs @@ -414,6 +414,8 @@ async fn contained_delivery( workdir.path(), &identity(), Duration::from_secs(900), + // A live DNS probe, not a job — see the egress probe: no job deadline exists to carry. + None, ) .await .expect("containment must be established"); @@ -819,6 +821,8 @@ async fn gate_a_scenario(tag: &str) -> GateA { workdir.path(), &identity(), Duration::from_secs(300), + // A live DNS probe, not a job: no job deadline exists to carry. + None, ) .await .expect("containment must be established"); @@ -922,6 +926,8 @@ async fn diagnose_v6_resolver_reachability_inside_containment() { workdir.path(), &identity(), Duration::from_secs(300), + // A live DNS probe, not a job: no job deadline exists to carry. + None, ) .await .expect("containment must be established"); @@ -990,6 +996,8 @@ async fn diagnose_v6_resolver_outside_the_denied_ranges() { workdir.path(), &identity(), Duration::from_secs(300), + // A live DNS probe, not a job: no job deadline exists to carry. + None, ) .await .expect("containment must be established"); @@ -1037,6 +1045,8 @@ async fn a_truncated_udp_answer_falls_back_to_tcp_53_inside_containment() { workdir.path(), &identity(), Duration::from_secs(300), + // A live DNS probe, not a job: no job deadline exists to carry. + None, ) .await .expect("containment must be established"); @@ -1094,6 +1104,8 @@ async fn host_stub_discovery_hands_the_job_a_canonical_upstream_not_the_stub() { workdir.path(), &identity(), Duration::from_secs(300), + // A live DNS probe, not a job: no job deadline exists to carry. + None, ) .await .expect("containment must be established from discovered resolvers"); @@ -1139,6 +1151,8 @@ async fn no_usable_resolver_refuses_before_a_holder_or_a_payload_exists() { workdir.path(), &identity(), Duration::from_secs(300), + // A live DNS probe, not a job: no job deadline exists to carry. + None, ) .await .err() @@ -1184,6 +1198,7 @@ async fn a_failed_installer_destroys_the_holder_and_leaves_nothing_to_launch_int None, true, vec![RESOLVER_V4.to_owned()], + 2_000_000_000, ) .await; let error = result.err().expect("an installer that cannot apply must fail the launch"); diff --git a/crates/maxplayer-core/src/sandbox_egress_live.rs b/crates/maxplayer-core/src/sandbox_egress_live.rs index 836f2159e..a2ae88fc0 100644 --- a/crates/maxplayer-core/src/sandbox_egress_live.rs +++ b/crates/maxplayer-core/src/sandbox_egress_live.rs @@ -267,6 +267,9 @@ async fn contained_run(tag: &str, runtime: &str) -> (String, String, String) { workdir.path(), &identity(), Duration::from_secs(300), + // A live egress probe, not a job: there is no job deadline to carry, so the stamp falls + // back to this probe's own lifetime rather than inventing an absolute one. + None, ) .await .expect("containment must establish"); diff --git a/crates/maxplayer-core/src/sandbox_evidence.rs b/crates/maxplayer-core/src/sandbox_evidence.rs new file mode 100644 index 000000000..758cd141b --- /dev/null +++ b/crates/maxplayer-core/src/sandbox_evidence.rs @@ -0,0 +1,903 @@ +//! Offline validation of a **saved** live containment matrix. +//! +//! The live gates in `tests/sandbox_netns_live.rs` need a docker daemon, a gVisor runtime and a +//! built sidecar image, so they are `#[ignore]`d and an ordinary `cargo test` run reports them as +//! *ignored*. That is the right call for replay — and it leaves a hole that a review found: the +//! named offline acceptance command was `cargo test -p maxplayer-core --features acp,wallet`, which +//! validates **nothing** about the live matrix. "0 passed, 11 ignored" and "the matrix is complete" +//! produce the same green. +//! +//! This module closes that hole. A live run writes down what it measured; this validates that +//! record offline, deterministically, with no daemon: +//! +//! * every required case is present, exactly once, with the outcome the matrix requires; +//! * a case with an empty or unrecognised outcome is **unscored**, and unscored fails; +//! * an id nobody requires is refused, because that is what a renamed or truncated record looks +//! like; +//! * the record names the source commit, the artifact that produced it, and the host it ran on, so +//! the matrix is attributable to something rather than floating free. +//! +//! It deliberately does **not** re-run anything. Replay stays separate, and a validator that shells +//! out to docker would be the live gate again under another name. +//! +//! ## The saved format +//! +//! Line oriented, because it has to be writable by hand from a log and diffable in review. `#` +//! starts a comment; blank lines are ignored. Header lines are `key=value`. Case lines start with +//! `case` and carry `key=value` fields: +//! +//! ```text +//! source_head=0721dcec44131cbd0298e035d48b1aa935567088 +//! artifact_sha256=a852e2e4d57fa0ed4873a1c36a2a567b2336901dd05fb1ce9776d69fa9919714 +//! artifact_path=target-linux/debug/deps/sandbox_netns_live-82001bf1221bb710 +//! host=lima:gvisor-repro linux-6.8.0-134-generic aarch64 runsc-release-20260817.0 +//! case id=integrated.denied.v4 outcome=refused log=raw/live-integrated.txt +//! ``` + +/// What a payload did, as recorded. The vocabulary is closed on purpose: a free-text outcome is an +/// unscored outcome, and "it failed" is exactly the ambiguity the live oracle exists to remove. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Outcome { + /// The payload ran and its connection succeeded. + Connected, + /// The payload ran and its connection was refused or timed out. + Refused, + /// The payload's process never reached its first statement. Never containment evidence. + NeverStarted, + /// Preparation refused to launch anything at all (fail-closed). + LaunchRefused, +} + +impl Outcome { + /// Parse the recorded word. `None` for anything else, which the validator reports as unscored + /// rather than guessing a direction. + pub fn parse(word: &str) -> Option { + match word { + "connected" => Some(Self::Connected), + "refused" => Some(Self::Refused), + "never-started" => Some(Self::NeverStarted), + "launch-refused" => Some(Self::LaunchRefused), + _ => None, + } + } + + /// The word a record must carry for this outcome. + pub fn word(self) -> &'static str { + match self { + Self::Connected => "connected", + Self::Refused => "refused", + Self::NeverStarted => "never-started", + Self::LaunchRefused => "launch-refused", + } + } +} + +/// One case the matrix must contain, and the outcome that case is only evidence at. +/// +/// The required outcome is part of the requirement because the direction is the evidence. A leg +/// recorded as `refused` when it must be `connected` is a positive control that did not hold, and a +/// matrix that accepts either has no positive controls. +#[derive(Debug, Clone, Copy)] +pub struct RequiredCase { + pub id: &'static str, + pub outcome: Outcome, + /// What this case establishes, quoted in the failure so a missing entry says why it matters. + pub establishes: &'static str, +} + +/// The complete matrix. A live run that does not produce every one of these has not produced the +/// matrix, and this list is the only place that is written down. +/// +/// Entries whose live legs are not yet authored are **deliberately present**: the requirement comes +/// from the review, not from what happens to exist, and a gate that only demands what is already +/// built cannot report an incomplete matrix. Each one fails as missing until a live run supplies it. +pub const REQUIRED_CASES: &[RequiredCase] = &[ + // ── The separate OUTPUT-only baseline arm ──────────────────────────────────────────────── + RequiredCase { + id: "baseline.output-only.runsc.leak", + outcome: Outcome::Connected, + establishes: "the leak reproduces: with only the OUTPUT chain installed and verified, a \ + gVisor payload reaches a denied destination", + }, + RequiredCase { + id: "baseline.output-only.runc.contained", + outcome: Outcome::Refused, + establishes: "the discriminator: the same OUTPUT policy does contain a runc payload, so the \ + leak above is a runtime property and not a policy that never applied", + }, + RequiredCase { + id: "baseline.veth.runsc.contained", + outcome: Outcome::Refused, + establishes: "the same rendered policy on the veth stops the gVisor payload", + }, + RequiredCase { + id: "baseline.veth.runsc.allowed", + outcome: Outcome::Connected, + establishes: "the veth filters are not a blanket deny", + }, + // ── The integrated production launch path ──────────────────────────────────────────────── + RequiredCase { + id: "integrated.denied.v4", + outcome: Outcome::Refused, + establishes: "a job prepared and launched by production prepare_launch/launch is contained \ + over IPv4 TCP", + }, + RequiredCase { + id: "integrated.allowed.v4", + outcome: Outcome::Connected, + establishes: "the integrated path's positive control over IPv4 TCP", + }, + RequiredCase { + id: "integrated.denied.v6", + outcome: Outcome::Refused, + establishes: "IPv6 TCP containment measured by connection, not by readback — an unfiltered \ + address family is the cheapest bypass there is", + }, + RequiredCase { + id: "integrated.allowed.v6", + outcome: Outcome::Connected, + establishes: "the IPv6 positive control — and, since v6 reaches nothing at all when \ + neighbour discovery is starved, the proof that the denied v6 leg above \ + measured a destination rule rather than a dead stack", + }, + RequiredCase { + id: "integrated.denied.v6-link-local", + outcome: Outcome::Refused, + establishes: "the counter-control for the neighbour-discovery exception: permitting the \ + two ICMPv6 control messages must not permit ordinary traffic to fe80::/10, \ + which is exactly what an over-broad ND allowance opens", + }, + RequiredCase { + id: "integrated.allowed.neighbour-port", + outcome: Outcome::Connected, + establishes: "an allowed destination stays allowed on a second port, ruling out a filter \ + that matched one port number", + }, + RequiredCase { + id: "integrated.denied.neighbour-port", + outcome: Outcome::Refused, + establishes: "a denied destination stays denied on a neighbouring port, so the pinhole is a \ + pinhole and not an open host", + }, + RequiredCase { + id: "integrated.exception.proxy-pinhole", + outcome: Outcome::Connected, + establishes: "the permitted proxy exception really passes through every installed layer", + }, + RequiredCase { + id: "integrated.runc.denied", + outcome: Outcome::Refused, + establishes: "runc compatibility: the added veth filters do not break containment for the \ + runtime that was already contained", + }, + RequiredCase { + id: "integrated.runc.allowed", + outcome: Outcome::Connected, + establishes: "runc compatibility: allowed traffic still flows under the added filters", + }, + RequiredCase { + id: "integrated.never-started.oracle", + outcome: Outcome::NeverStarted, + establishes: "the oracle's red-prove: a payload that could not start is scored NeverStarted \ + and never counted as a denial", + }, + RequiredCase { + id: "integrated.fail-closed.preparation", + outcome: Outcome::LaunchRefused, + establishes: "containment that cannot be installed refuses the launch and starts no payload", + }, + // ── Lifecycle isolation ────────────────────────────────────────────────────────────────── + RequiredCase { + id: "integrated.sibling.contained-after-cleanup", + outcome: Outcome::Refused, + establishes: "a sibling job keeps its containment across another job's teardown", + }, + RequiredCase { + id: "integrated.sibling.allowed-after-cleanup", + outcome: Outcome::Connected, + establishes: "a sibling job keeps working across another job's teardown", + }, + RequiredCase { + id: "host.unaffected.during-cleanup", + outcome: Outcome::Connected, + establishes: "the VM host's own egress is unaffected before, during and after cleanup — no \ + host-global mutation", + }, +]; + +/// A validated record of one case. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SavedCase { + pub id: String, + pub outcome: Outcome, + pub log: String, + /// The `#[test]` function that asserts this case, as `cargo test` prints it. + /// + /// Present so the map from test function to case id stops being the author's word. Without it + /// the record names a log and asserts an outcome, and nothing connects the two: the log could + /// be any green run, the id could be attached to whichever test the author believed owned it, + /// and both readings pass a validator that only checks the log text is nonempty. + /// [`corroborate`] resolves this name in that log. + pub test: String, +} + +/// A validated saved matrix: what produced it, and every case it scored. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SavedMatrix { + /// The commit the artifact was built from, 40 hex. + pub source_head: String, + /// SHA-256 of the executed test binary, 64 hex. + pub artifact_sha256: String, + pub artifact_path: String, + pub host: String, + pub cases: Vec, +} + +/// Every header a record must carry. Identity is not decoration: a matrix that does not say which +/// source, which binary and which host produced it cannot be checked against anything later. +const REQUIRED_HEADERS: &[&str] = &["source_head", "artifact_sha256", "artifact_path", "host"]; + +/// Validate a saved matrix. `Err` carries **every** problem found, not the first: a record with +/// four missing cases should take one round trip to fix, not four. +pub fn validate(text: &str) -> Result> { + let mut problems = Vec::new(); + let mut headers: Vec<(String, String)> = Vec::new(); + let mut cases: Vec = Vec::new(); + let mut seen_ids: Vec = Vec::new(); + + for (number, raw) in text.lines().enumerate() { + let line = raw.split('#').next().unwrap_or("").trim(); + if line.is_empty() { + continue; + } + let at = number + 1; + if let Some(rest) = line.strip_prefix("case ") { + match parse_case(rest, at) { + Ok(case) => { + if seen_ids.contains(&case.id) { + problems.push(format!( + "line {at}: case {:?} is recorded twice — one live measurement per \ + case, or the second silently overwrites the first", + case.id + )); + } else { + seen_ids.push(case.id.clone()); + cases.push(case); + } + } + Err(problem) => problems.push(problem), + } + } else if line.starts_with("case") { + problems.push(format!("line {at}: {line:?} is neither a header nor a `case ` record")); + } else { + match line.split_once('=') { + Some((key, value)) => { + headers.push((key.trim().to_owned(), value.trim().to_owned())) + } + None => problems.push(format!( + "line {at}: {line:?} is not `key=value` and not a `case ` record" + )), + } + } + } + + // Headers: present, unique, non-empty, and the two digests actually digest-shaped. + let mut resolved: Vec<(&str, String)> = Vec::new(); + for name in REQUIRED_HEADERS { + let found: Vec<&(String, String)> = + headers.iter().filter(|(key, _)| key == name).collect(); + match found.as_slice() { + [] => problems.push(format!( + "the record has no {name} — a matrix that does not say what produced it is not \ + attributable to anything" + )), + [(_, value)] if value.is_empty() => { + problems.push(format!("{name} is empty, which is the same as absent")) + } + [(_, value)] => resolved.push((name, value.clone())), + _ => problems.push(format!( + "{name} is given {} times; one record describes one run", + found.len() + )), + } + } + let header = |name: &str| -> String { + resolved + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.clone()) + .unwrap_or_default() + }; + let source_head = header("source_head"); + if !source_head.is_empty() && !is_hex(&source_head, 40) { + problems.push(format!( + "source_head {source_head:?} is not a 40-character hex commit id — an abbreviated head \ + cannot be compared to a published one" + )); + } + let artifact_sha256 = header("artifact_sha256"); + if !artifact_sha256.is_empty() && !is_hex(&artifact_sha256, 64) { + problems.push(format!("artifact_sha256 {artifact_sha256:?} is not a 64-character hex digest")); + } + + // Coverage, in the order the matrix declares, so the failure reads as a checklist. + for required in REQUIRED_CASES { + match cases.iter().find(|case| case.id == required.id) { + None => problems.push(format!( + "missing case {}: {} — a matrix without it is incomplete, not passing", + required.id, required.establishes + )), + Some(case) if case.outcome != required.outcome => problems.push(format!( + "case {} is recorded as {} but is only evidence at {}: {}", + required.id, + case.outcome.word(), + required.outcome.word(), + required.establishes + )), + Some(_) => {} + } + } + for case in &cases { + if !REQUIRED_CASES.iter().any(|required| required.id == case.id) { + problems.push(format!( + "case {:?} is not one the matrix requires — a renamed or truncated id records a \ + measurement against nothing", + case.id + )); + } + } + + if problems.is_empty() { + Ok(SavedMatrix { + source_head, + artifact_sha256, + artifact_path: header("artifact_path"), + host: header("host"), + cases, + }) + } else { + Err(problems) + } +} + +/// `id=… outcome=… log=… test=…`, all four required and none of them empty. +fn parse_case(rest: &str, at: usize) -> Result { + let mut id = None; + let mut outcome_word = None; + let mut log = None; + let mut test = None; + for field in rest.split_whitespace() { + let (key, value) = field.split_once('=').ok_or_else(|| { + format!("line {at}: {field:?} in a case record is not `key=value`") + })?; + // Assigned ONCE. A repeated field used to overwrite the earlier one, so + // `outcome=refused outcome=connected` scored as Connected: the strictest reading of a line + // lost to the last word on it, and a record could carry its own contradiction and still + // pass. There is no honest reading of a case that states two outcomes -- refusing is the + // only answer that cannot be gamed by ordering. + let slot = match key { + "id" => &mut id, + "outcome" => &mut outcome_word, + "log" => &mut log, + "test" => &mut test, + other => { + return Err(format!("line {at}: a case record has no {other:?} field")); + } + }; + if let Some(first) = slot.as_deref() { + return Err(format!( + "line {at}: a case record states {key} twice, {first:?} then {value:?} -- a record \ + that contradicts itself is not evidence, and the second value does not silently \ + win" + )); + } + *slot = Some(value.to_owned()); + } + let id = id.filter(|value| !value.is_empty()).ok_or_else(|| { + format!("line {at}: a case record with no id scores nothing") + })?; + let word = outcome_word.unwrap_or_default(); + if word.is_empty() { + return Err(format!( + "line {at}: case {id} has an empty outcome — an unscored case is a case that was not \ + measured, and it fails rather than passing quietly" + )); + } + let outcome = Outcome::parse(&word).ok_or_else(|| { + format!( + "line {at}: case {id} records outcome {word:?}, which is not one of connected, \ + refused, never-started, launch-refused — an outcome nobody can read is unscored" + ) + })?; + let log = log.filter(|value| !value.is_empty()).ok_or_else(|| { + format!( + "line {at}: case {id} names no log — an outcome with nothing behind it is an assertion, \ + not evidence" + ) + })?; + let test = test.filter(|value| !value.is_empty()).ok_or_else(|| { + format!( + "line {at}: case {id} names no test — without the function that asserts it, the tie \ + between this id and that log is the author's word, which is what the record exists to \ + replace" + ) + })?; + Ok(SavedCase { id, outcome, log, test }) +} + +/// Read each case's named log and require it to show that case's test PASSING. +/// +/// Separate from [`validate`] because this one touches the filesystem: `validate` is a pure reading +/// of the record's text and stays usable on a record whose logs are elsewhere. Everything here is +/// the check the review asked for and the text pass cannot make — that the log behind a case is +/// that case's log, and that it is green. +/// +/// `base` is the directory the record's relative log paths resolve against, i.e. the record's own +/// directory. Absolute paths are refused: a record that reaches outside its own run is not +/// self-contained evidence and could name a log from another machine. +/// +/// What this establishes: the named log exists, contains the named test, and that test is recorded +/// `ok` in it. What it does NOT establish: that the test's assertions are the right ones for the +/// case id. Only reading the test body settles that, and this function makes no claim about it. +pub fn corroborate(base: &std::path::Path, matrix: &SavedMatrix) -> Result<(), Vec> { + let mut problems = Vec::new(); + let mut sources: std::collections::HashMap = std::collections::HashMap::new(); + + for case in &matrix.cases { + let relative = std::path::Path::new(&case.log); + if relative.is_absolute() || case.log.contains("..") { + problems.push(format!( + "case {}: log {:?} is not a path inside the record's own directory — evidence that \ + reaches outside the run is not attributable to it", + case.id, case.log + )); + continue; + } + let path = base.join(relative); + // The check above is LEXICAL, and lexical confinement is not confinement: `raw/green.log` + // contains no `..` and is not absolute, and can still be a symlink to another run's log, or + // to anywhere on the host. Resolving both sides and requiring the result to stay under the + // record's own directory is what actually binds the evidence to this run. + // + // Resolution also fixes the file: what is read below is the path that was just checked, so a + // link cannot be swapped for one that passes and then read as one that would not. + let resolved = match (std::fs::canonicalize(base), std::fs::canonicalize(&path)) { + (Ok(root), Ok(target)) => { + if !target.starts_with(&root) { + problems.push(format!( + "case {}: log {:?} resolves to {}, outside the record's own directory {} \ + — evidence that reaches outside the run is not attributable to it, and a \ + relative-looking path that resolves away is exactly how that happens", + case.id, + case.log, + target.display(), + root.display() + )); + continue; + } + target + } + // An unresolvable path is the ABSENT-log case, not the escaping one, and it keeps the + // wording the absent case already had: a cited log that is not there is a missing gate. + _ => { + problems.push(format!( + "case {}: named log {} could not be read (path does not resolve) — a cited log \ + that is not there is a missing gate, not an absent one", + case.id, + path.display() + )); + continue; + } + }; + let text = match sources.get(&case.log) { + Some(text) => text.clone(), + None => match std::fs::read_to_string(&resolved) { + Ok(text) => { + sources.insert(case.log.clone(), text.clone()); + text + } + Err(error) => { + problems.push(format!( + "case {}: named log {} could not be read ({error}) — a cited log that is not \ + there is a missing gate, not an absent one", + case.id, + path.display() + )); + continue; + } + }, + }; + + // `cargo test` prints `test :: ... ok`, and on failure `... FAILED` plus a + // `failures:` block naming it again. Requiring the `ok` line is what makes a red run + // unciteable; matching the bare name would find it in that failure block too. + let passed = text + .lines() + .filter_map(|line| line.strip_prefix("test ")) + .filter(|line| { + line.split_whitespace() + .next() + .is_some_and(|name| name == case.test || name.ends_with(&format!("::{}", case.test))) + }) + .any(|line| line.ends_with(" ok")); + + if !passed { + let named = text.contains(&case.test); + problems.push(format!( + "case {}: log {} does not record test {} as passing ({}) — the record's outcome {} \ + rests on a run this log does not show", + case.id, + path.display(), + case.test, + if named { + "the test is named there, but not with an `ok` result" + } else { + "the test is not named in that log at all" + }, + case.outcome.word() + )); + } + } + + if problems.is_empty() { Ok(()) } else { Err(problems) } +} + +fn is_hex(value: &str, len: usize) -> bool { + value.len() == len && value.chars().all(|c| c.is_ascii_hexdigit()) +} + +/// The environment variable naming the saved matrix. Set by the offline acceptance entrypoint; when +/// it is unset the validator's own behaviour is still gated by the tests below. +pub const EVIDENCE_PATH_VAR: &str = "MAXPLAYER_LIVE_EVIDENCE"; + +#[cfg(test)] +mod tests { + use super::*; + + /// A complete record, built from the requirement itself so the positive control cannot drift + /// out of date as the matrix grows. + fn complete() -> String { + let mut text = String::from( + "# saved live containment matrix\n\ + source_head=0721dcec44131cbd0298e035d48b1aa935567088\n\ + artifact_sha256=a852e2e4d57fa0ed4873a1c36a2a567b2336901dd05fb1ce9776d69fa9919714\n\ + artifact_path=target-linux/debug/deps/sandbox_netns_live-82001bf1221bb710\n\ + host=lima:gvisor-repro linux-6.8.0-134-generic aarch64 runsc-release-20260817.0\n", + ); + for case in REQUIRED_CASES { + text.push_str(&format!( + "case id={} outcome={} log=raw/{}.txt test=a_test_named_for_{}\n", + case.id, + case.outcome.word(), + case.id, + case.id.replace(['.', '-'], "_") + )); + } + text + } + + /// The positive control. Without it every refusal below could come from a validator that + /// refuses everything. + #[test] + fn a_complete_record_validates_and_carries_its_identities() { + let matrix = validate(&complete()).expect("a complete record must validate"); + assert_eq!(matrix.source_head, "0721dcec44131cbd0298e035d48b1aa935567088"); + assert_eq!(matrix.cases.len(), REQUIRED_CASES.len()); + assert!(matrix.host.contains("runsc"), "{:?}", matrix.host); + } + + /// The gate the review asked for: a missing case fails, and says which and why. + #[test] + fn a_missing_case_fails_and_names_what_it_would_have_shown() { + let dropped = REQUIRED_CASES[0].id; + let text: String = complete() + .lines() + .filter(|line| !line.contains(&format!("id={dropped} "))) + .map(|line| format!("{line}\n")) + .collect(); + let problems = validate(&text).expect_err("an incomplete matrix must not validate"); + assert!( + problems.iter().any(|p| p.contains(dropped) && p.contains("missing case")), + "{problems:?}" + ); + // Every required case is reachable this way, not just the first: a checklist that only + // checks its head is not a checklist. + for case in REQUIRED_CASES { + let text: String = complete() + .lines() + .filter(|line| !line.contains(&format!("id={} ", case.id))) + .map(|line| format!("{line}\n")) + .collect(); + let problems = validate(&text).expect_err("still incomplete"); + assert!(problems.iter().any(|p| p.contains(case.id)), "{}: {problems:?}", case.id); + } + } + + /// An empty or unreadable outcome is unscored, and unscored fails. This is the difference + /// between "11 ignored" and "the matrix is complete". + #[test] + fn an_unscored_case_fails_rather_than_passing_quietly() { + for (record, expect) in [ + ("case id=integrated.denied.v4 outcome= log=raw/x.txt test=t", "empty outcome"), + ( + "case id=integrated.denied.v4 outcome=failed log=raw/x.txt test=t", + "not one of connected", + ), + ("case id=integrated.denied.v4 outcome=refused log= test=t", "names no log"), + ("case id= outcome=refused log=raw/x.txt test=t", "no id scores nothing"), + ( + "case id=integrated.denied.v4 outcome=refused log=raw/x.txt", + "names no test", + ), + ( + "case id=integrated.denied.v4 outcome=refused log=raw/x.txt test=", + "names no test", + ), + ] { + let text = complete() + .lines() + .filter(|line| !line.contains("id=integrated.denied.v4 ")) + .map(|line| format!("{line}\n")) + .collect::() + + record + + "\n"; + let problems = validate(&text).expect_err("an unscored case must fail"); + assert!( + problems.iter().any(|problem| problem.contains(expect)), + "expected {expect:?} in {problems:?}" + ); + } + } + + /// A case recorded in the wrong direction fails. A positive control recorded as a refusal is a + /// control that did not hold, and accepting it would let a blanket-deny ruleset pass. + #[test] + fn a_case_recorded_in_the_wrong_direction_fails() { + let text = complete().replace( + "case id=integrated.allowed.v4 outcome=connected", + "case id=integrated.allowed.v4 outcome=refused", + ); + let problems = validate(&text).expect_err("a failed positive control must fail the matrix"); + assert!( + problems.iter().any(|p| p.contains("integrated.allowed.v4") + && p.contains("only evidence at connected")), + "{problems:?}" + ); + } + + /// A case that states a field twice fails, and the second value does not win. Overwriting made + /// the parser read only the last word: `outcome=refused outcome=connected` scored as Connected, + /// so a record could carry a refusal AND the pass that contradicts it, and the pass is what got + /// counted. Checked on the outcome, where it decides the grade, and on the log, where it + /// decides which file anyone reading the record would go and open. + #[test] + fn a_case_that_states_a_field_twice_fails() { + for (record, expect) in [ + ( + "case id=integrated.denied.v4 outcome=refused outcome=connected log=raw/x.txt \ + test=t", + "states outcome twice", + ), + ( + "case id=integrated.denied.v4 outcome=refused log=raw/x.txt log=raw/other.txt \ + test=t", + "states log twice", + ), + ] { + let text = complete() + .lines() + .filter(|line| !line.contains("id=integrated.denied.v4 ")) + .map(|line| format!("{line}\n")) + .collect::() + + record + + "\n"; + let problems = validate(&text).expect_err("a self-contradicting case must fail"); + assert!( + problems.iter().any(|problem| problem.contains(expect)), + "expected {expect:?} in {problems:?}" + ); + } + } + + /// A duplicate and an unknown id both fail: the first hides a second measurement, the second is + /// what a renamed or truncated record looks like. + #[test] + fn duplicate_and_unknown_case_ids_fail() { + let duplicated = + complete() + "case id=integrated.denied.v4 outcome=connected log=raw/again.txt test=t\n"; + let problems = validate(&duplicated).expect_err("a duplicate must fail"); + assert!(problems.iter().any(|p| p.contains("recorded twice")), "{problems:?}"); + + let unknown = + complete() + "case id=integrated.denied.v5 outcome=refused log=raw/x.txt test=t\n"; + let problems = validate(&unknown).expect_err("an unknown id must fail"); + assert!( + problems.iter().any(|p| p.contains("not one the matrix requires")), + "{problems:?}" + ); + } + + /// Identity headers are required and shaped. An unattributable matrix is not evidence about any + /// particular source or binary. + #[test] + fn a_record_without_source_and_artifact_identity_fails() { + for (name, broken) in [ + ("source_head", complete().replace("source_head=0721dcec44131cbd0298e035d48b1aa935567088\n", "")), + ("artifact_sha256", complete().replace("artifact_sha256=a852e2e4d57fa0ed4873a1c36a2a567b2336901dd05fb1ce9776d69fa9919714\n", "")), + ("artifact_path", complete().replace("artifact_path=target-linux/debug/deps/sandbox_netns_live-82001bf1221bb710\n", "")), + ("host", complete().replace("host=lima:gvisor-repro linux-6.8.0-134-generic aarch64 runsc-release-20260817.0\n", "")), + ] { + let problems = validate(&broken).expect_err("a record missing {name} must fail"); + assert!(problems.iter().any(|p| p.contains(name)), "{name}: {problems:?}"); + } + // Shape, not just presence: an abbreviated head cannot be compared to a published one. + let short = complete().replace("source_head=0721dcec44131cbd0298e035d48b1aa935567088", "source_head=0721dce"); + let problems = validate(&short).expect_err("an abbreviated head must fail"); + assert!(problems.iter().any(|p| p.contains("40-character hex")), "{problems:?}"); + let empty = complete().replace("host=lima:gvisor-repro linux-6.8.0-134-generic aarch64 runsc-release-20260817.0", "host="); + let problems = validate(&empty).expect_err("an empty header must fail"); + assert!(problems.iter().any(|p| p.contains("host is empty")), "{problems:?}"); + } + + /// An empty file fails — the state this repository is in right now. No live matrix has been + /// produced for this candidate, because the live runtime is held, and the gate says so instead + /// of reporting a green. + #[test] + fn an_empty_record_fails_with_the_whole_checklist() { + let problems = validate("").expect_err("nothing measured is not a pass"); + assert!( + problems.len() >= REQUIRED_HEADERS.len() + REQUIRED_CASES.len(), + "an empty record must report every missing header and case, got {}", + problems.len() + ); + } + + /// The acceptance entrypoint's own leg: when a saved matrix is named, validate that file. This + /// is the test `scripts/sandbox-acceptance.sh` runs with the variable set, and it fails when the + /// file is absent, unreadable or incomplete. + #[test] + fn the_named_saved_matrix_validates() { + let Ok(path) = std::env::var(EVIDENCE_PATH_VAR) else { + // Unset: the validator's behaviour is gated by the tests above, and there is nothing + // here to check. The acceptance entrypoint is what sets it. + return; + }; + let text = std::fs::read_to_string(&path).unwrap_or_else(|error| { + panic!( + "{EVIDENCE_PATH_VAR}={path} could not be read: {error}\nA named saved matrix that \ + is not there is a missing gate, not an absent one." + ) + }); + let matrix = match validate(&text) { + Ok(matrix) => matrix, + Err(problems) => panic!( + "the saved live matrix at {path} is not complete:\n {}", + problems.join("\n ") + ), + }; + + // The second leg: every case's named log must actually show that case's test passing. + // Without it the record's logs were checked only for being nonempty text, so a complete + // matrix could cite a green run that never contained the test the row claims. + let base = std::path::Path::new(&path).parent().unwrap_or(std::path::Path::new(".")); + if let Err(problems) = corroborate(base, &matrix) { + panic!( + "the saved live matrix at {path} names logs that do not corroborate it:\n {}", + problems.join("\n ") + ); + } + } + + /// A record whose logs are written next to it, for the corroboration tests below. + fn matrix_with_logs(log_body: &str) -> (std::path::PathBuf, SavedMatrix) { + let dir = std::env::temp_dir().join(format!( + "mx-corroborate-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(dir.join("raw")).expect("temp dir"); + let matrix = validate(&complete()).expect("the fixture record is complete"); + for case in &matrix.cases { + let body = log_body.replace("{test}", &case.test); + std::fs::write(dir.join(&case.log), body).expect("write log"); + } + (dir, matrix) + } + + /// The positive control: logs that record each case's test as passing corroborate the record. + #[test] + fn logs_that_show_each_case_passing_corroborate_the_record() { + let (dir, matrix) = matrix_with_logs( + "running 1 test\ntest sandbox_netns_live::{test} ... ok\n\ntest result: ok. 1 passed\n", + ); + assert_eq!(corroborate(&dir, &matrix), Ok(())); + let _ = std::fs::remove_dir_all(&dir); + } + + /// The check the review asked for: a log that is nonempty, green, and about something else does + /// NOT corroborate the case. This is the exact hole — the old validator accepted any nonempty + /// log text, so a real green run of unrelated tests satisfied every row. + #[test] + fn a_green_log_that_never_names_the_test_does_not_corroborate_it() { + let (dir, matrix) = matrix_with_logs( + "running 1 test\ntest some::other_test ... ok\n\ntest result: ok. 1 passed\n", + ); + let problems = corroborate(&dir, &matrix) + .expect_err("a log that never names the test cannot stand behind it"); + assert_eq!(problems.len(), matrix.cases.len(), "every case must be reported, not the first"); + assert!(problems[0].contains("not named in that log at all"), "{}", problems[0]); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A log that names the test but records it FAILED does not corroborate it either. Matching the + /// bare name would have found it in cargo's `failures:` block and passed a red run. + #[test] + fn a_log_recording_the_test_as_failed_does_not_corroborate_it() { + let (dir, matrix) = matrix_with_logs( + "running 1 test\ntest sandbox_netns_live::{test} ... FAILED\n\nfailures:\n \ + sandbox_netns_live::{test}\n\ntest result: FAILED. 0 passed; 1 failed\n", + ); + let problems = corroborate(&dir, &matrix).expect_err("a failed test is not evidence"); + assert!(problems[0].contains("not with an `ok` result"), "{}", problems[0]); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A cited log that is not there fails; and a log path reaching outside the record's directory + /// is refused rather than followed. + #[test] + fn an_absent_or_escaping_log_is_refused() { + let (dir, matrix) = matrix_with_logs("test x ... ok\n"); + std::fs::remove_file(dir.join(&matrix.cases[0].log)).expect("remove one log"); + let problems = corroborate(&dir, &matrix).expect_err("a missing log is a missing gate"); + assert!(problems[0].contains("could not be read"), "{}", problems[0]); + + let mut escaping = matrix.clone(); + escaping.cases[0].log = "../elsewhere/green.log".to_owned(); + let problems = + corroborate(&dir, &escaping).expect_err("a log outside the run is not its evidence"); + assert!(problems[0].contains("inside the record's own directory"), "{}", problems[0]); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A log that LOOKS local and resolves elsewhere is refused. + /// + /// The R3 verdict named this exactly: the confinement check was lexical, not symlink + /// confinement. `raw/green.log` is neither absolute nor contains `..`, so it passes every + /// textual test — and can still be a link to another run's green log, or to anywhere on the + /// host. Lexical checks are the ones an attacker-shaped mistake walks straight around, and here + /// the "attacker" is just a copied directory or a convenience symlink someone left behind. + /// + /// The positive control matters as much as the refusal: a REGULAR file at the same path must + /// still corroborate, or this check would be indistinguishable from one that refuses everything. + #[test] + fn a_log_that_is_a_symlink_out_of_the_record_directory_is_refused() { + let (dir, matrix) = matrix_with_logs( + "running 1 test\ntest sandbox_netns_live::{test} ... ok\n\ntest result: ok. 1 passed\n", + ); + assert_eq!(corroborate(&dir, &matrix), Ok(()), "positive control: a real local log passes"); + + // Somewhere else entirely, holding a log that would corroborate if it were followed. + let outside = std::env::temp_dir().join(format!("mx-outside-{}", std::process::id())); + std::fs::create_dir_all(&outside).expect("outside dir"); + let elsewhere = outside.join("green.log"); + let named = &matrix.cases[0].log; + let local = dir.join(named); + std::fs::copy(&local, &elsewhere).expect("a green log outside the record directory"); + + // Replace the local log with a link to it. The recorded path does not change at all. + std::fs::remove_file(&local).expect("remove the real log"); + std::os::unix::fs::symlink(&elsewhere, &local).expect("symlink"); + + let problems = corroborate(&dir, &matrix) + .expect_err("a log resolving outside the record directory is not its evidence"); + assert!( + problems[0].contains("outside the record's own directory"), + "the refusal must name the escape, not some other complaint: {}", + problems[0] + ); + + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_dir_all(&outside); + } +} diff --git a/crates/maxplayer-core/src/sandbox_iface.rs b/crates/maxplayer-core/src/sandbox_iface.rs new file mode 100644 index 000000000..e95378671 --- /dev/null +++ b/crates/maxplayer-core/src/sandbox_iface.rs @@ -0,0 +1,2123 @@ +//! Job-local egress filtering on the holder namespace's own veth. +//! +//! # The hole this closes +//! +//! [`crate::sandbox_net`] renders an iptables plan and [`crate::sandbox_netns`] installs it into the +//! job's network namespace. That plan contains a `runc` job completely. It does **not** contain a +//! `runsc` (gVisor) one: gVisor runs its own userspace netstack and hands finished packets to the +//! sandbox's network endpoint, so a payload's egress never traverses the host kernel's `OUTPUT` +//! chain. The rules are installed, the readback proves they are installed, and the denied +//! destinations stay reachable anyway — measured on this repo's own fixtures, both address families, +//! over TCP. +//! +//! The packets do cross one thing the host kernel owns: the **veth** the namespace is built on. +//! Every packet leaving that namespace, from any runtime, is transmitted on that interface. So the +//! filter goes there, as a `clsact` egress qdisc with `flower` classifiers, installed by the same +//! trusted sidecar that installs the iptables plan and before any payload exists. +//! +//! # Why this is not a replacement for the iptables plan +//! +//! Both are installed. The iptables plan keeps doing what it already did — in particular it is what +//! LOGs a job probing the LAN, which `tc` has no equivalent of — and the veth filter closes the +//! runtime-dependent gap underneath it. Removing either one would be a widening, and neither is +//! sufficient alone. +//! +//! # Where the policy comes from +//! +//! Nowhere here. Every prefix, every exception and their order are **derived from +//! [`NetPolicy::rules`]** — the same rendered policy the iptables plan is built from, unit-tested in +//! `sandbox_net`. This module translates that policy into `tc` argv; it does not hold a second copy +//! of it. A denied range added there appears here without anyone remembering to, and the parity +//! tests below fail if a translation is ever dropped. +//! +//! # What is deliberately different from the iptables rendering +//! +//! **The drops carry no protocol match.** The iptables drops do not either, but this is the property +//! the whole exercise turns on: the demonstrated leak was TCP, and a TCP-only filter would look +//! green against the very fixture that found the hole while leaving UDP and everything else open. +//! [`drops_are_protocol_independent`] is the test that keeps it that way. +//! +//! **Loopback is not filtered and does not need to be.** Docker's embedded resolver answers at +//! `127.0.0.11` inside the namespace, and loopback traffic is never transmitted on the veth, so an +//! egress filter on that interface cannot reach it. The `sandbox_net` invariant that loopback is +//! never denied survives here by construction rather than by a rule. + +use crate::sandbox_net::{Family, NetPolicy}; + +/// The qdisc the filters attach to. +/// +/// `clsact` rather than the older `ingress` qdisc because it is the one that offers an **egress** +/// hook, which is the direction a payload's traffic leaves in. It is also classless and holds no +/// queueing behaviour of its own: attaching it changes no scheduling, adds no shaping, and cannot +/// reorder or delay a packet it does not drop. +pub const EGRESS_QDISC: &str = "clsact"; + +/// The `tc` hook filters are attached to. +pub const EGRESS_HOOK: &str = "egress"; + +/// The first `tc` priority this module uses; each rendered filter takes the next one. +/// +/// `tc` evaluates filters in ascending `pref` and takes the **first match**, which is the same +/// first-match semantics `iptables` gives an `OUTPUT` chain of terminating targets. So the order is +/// not re-derived here: the filters are emitted in exactly [`NetPolicy::rules`]' order and numbered +/// consecutively, and every ordering property that file establishes and tests — the metadata drop +/// ahead of everything that could pass it, the proxy pinhole ahead of the range drop that covers the +/// gateway — is carried across rather than reinvented. +/// +/// Starting at 100 rather than 1 leaves room below for a future filter that must precede all of +/// these, and makes a hand-added filter visible as an out-of-band number in a readback. +pub const PREF_BASE: u16 = 100; + +/// One rendered `tc` filter, kept as data so the plan can be printed, compared and tested rather +/// than only executed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IfaceFilter { + pub family: Family, + pub pref: u16, + /// The destination prefix, exactly as the policy spells it. + /// + /// `None` only where the policy rule itself names no `-d`: the neighbour-advertisement + /// exception, which is narrowed by ICMPv6 type and hop limit instead. Spelling that as + /// `::/0` would be the same match written as a wildcard, and a wildcard destination is the + /// one thing this field must never quietly acquire. + pub dst: Option, + /// `Some("tcp")` for an exception that names a protocol; **always `None` for a drop**. + pub ip_proto: Option, + /// The destination port match in `tc` spelling (`49200-49299`), if the source rule had one. + pub dst_port: Option, + /// The ICMPv6 message type, for the two neighbour-discovery exceptions. `tc` takes this as + /// `type` and prints it as `icmp_type`. + pub icmp_type: Option, + /// The hop-limit match (`255`), which is what confines an ND exception to this link. + pub ip_ttl: Option, + /// `pass` or `drop`. + pub action: &'static str, + /// Why this filter exists, carried from the policy rule it was derived from. + pub why: &'static str, +} + +impl IfaceFilter { + /// The `tc` argv that installs this filter on `dev`, without the leading binary name. + pub fn add_argv(&self, dev: &str) -> Vec { + let mut argv: Vec = ["filter", "add", "dev", dev, EGRESS_HOOK] + .into_iter() + .map(String::from) + .collect(); + // `pref` before `protocol` before `flower`: the exact argument order the prototype gate + // installed and read back on a live kernel. `tc` accepts other orders, but this is the one + // with a measurement behind it. + argv.push("pref".into()); + argv.push(self.pref.to_string()); + argv.push("protocol".into()); + argv.push(tc_protocol(self.family).into()); + argv.push("flower".into()); + if let Some(proto) = &self.ip_proto { + argv.push("ip_proto".into()); + argv.push(proto.clone()); + } + // `type` on the way in, `icmp_type` on the way back — measured on a live kernel, not + // assumed symmetric. + if let Some(icmp_type) = &self.icmp_type { + argv.push("type".into()); + argv.push(icmp_type.clone()); + } + if let Some(dst) = &self.dst { + argv.push("dst_ip".into()); + argv.push(dst.clone()); + } + if let Some(ttl) = &self.ip_ttl { + argv.push("ip_ttl".into()); + argv.push(ttl.clone()); + } + if let Some(port) = &self.dst_port { + argv.push("dst_port".into()); + argv.push(port.clone()); + } + argv.push("action".into()); + argv.push(self.action.to_owned()); + argv + } + + /// How this filter names its destination in a refusal message. A filter matching on type and + /// hop limit has none, and saying so is more useful than printing an empty string. + fn dst_label(&self) -> &str { + self.dst.as_deref().unwrap_or("no destination (ICMPv6 type match)") + } +} + +/// The complete interface plan for one job: the qdisc, then the filters in the order they must be +/// installed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IfacePlan { + pub dev: String, + pub filters: Vec, +} + +impl IfacePlan { + /// Derive the plan for `dev` from `policy`. + /// + /// `Err` rather than a silent omission whenever a policy rule cannot be translated: an + /// untranslatable deny is a hole in this layer, and the only safe response is to refuse to launch + /// the payload. "Rendered fewer filters than the policy has drops" must never be representable. + pub fn derive(dev: &str, policy: &NetPolicy) -> Result { + let mut filters = Vec::new(); + let mut rendered = 0u16; + + for rule in policy.rules() { + let target = rule.target(); + let action = match target { + Some("ACCEPT") => "pass", + Some("DROP") => "drop", + // LOG rules have no `tc` equivalent and are not containment: they observe. They stay + // in the iptables plan, which still runs. Skipping them here is not a widening — + // there is nothing to widen, a LOG rule denies nothing. + Some("LOG") => continue, + other => { + return Err(format!( + "policy rule {:?} jumps to {other:?}, which this layer cannot translate — \ + refusing to render a partial interface filter", + rule.args + )) + } + }; + + let icmp_type = arg_after(&rule.args, "--icmpv6-type").map(str::to_owned); + let ip_ttl = arg_after(&rule.args, "--hl-eq").map(str::to_owned); + let dst = rule.destination(); + + // A rule with no `-d` is translatable only when something else narrows it as tightly. + // That is exactly the neighbour-advertisement exception: ICMPv6 type 136 at hop limit + // 255, which flower expresses natively (`ip_proto icmpv6 type 136 ip_ttl 255`) and + // which was measured accepted and read back on a live kernel. Anything else with no + // destination is still a refusal — a destination-less ACCEPT is an egress hole and a + // destination-less DROP has no flower prefix. + if dst.is_none() && !(icmp_type.is_some() && ip_ttl.is_some()) { + return Err(format!( + "policy rule {:?} names no -d destination and no ICMPv6 type/hop-limit \ + narrowing, so it has no flower equivalent", + rule.args + )); + } + + rendered += 1; + let pref = PREF_BASE + rendered; + + filters.push(IfaceFilter { + family: rule.family, + pref, + dst: dst.map(str::to_owned), + // Both carried for a pass and both dropped for a drop, for the same reason + // `ip_proto` is: a deny narrowed to one ICMPv6 type denies only that type. + icmp_type: match action { + "pass" => icmp_type, + _ => None, + }, + ip_ttl: match action { + "pass" => ip_ttl, + _ => None, + }, + // Protocol is carried for an exception — a pinhole must stay as narrow as the + // iptables one, and widening it here would be a widening of containment. It is + // dropped for a deny, because the leak this closes is protocol-independent and a + // TCP-only deny is the exact shape of the bug. + ip_proto: match action { + "pass" => arg_after(&rule.args, "-p").map(str::to_owned), + _ => None, + }, + dst_port: match action { + "pass" => arg_after(&rule.args, "--dport").map(to_tc_port_range), + _ => None, + }, + action, + why: rule.why, + }); + } + + if filters.is_empty() { + return Err( + "the policy rendered no interface filters at all — refusing an unfiltered veth" + .to_owned(), + ); + } + // Checked on the render as well as on the readback. A shadowed pinhole is not a typo, it is + // the measured failure `sandbox_net` documents — the ACCEPT appended below the range drop + // that covers the gateway, leaving every job without its model while every shape test stays + // green — and it must not be renderable, let alone installable. + no_shadowed_exception(&filters)?; + + Ok(Self { dev: dev.to_owned(), filters }) + } + + /// Every `tc` argv this plan runs, in order: the qdisc first, then the filters. + pub fn install_plan(&self) -> Vec> { + let mut plan = vec![vec![ + "qdisc".to_owned(), + "add".to_owned(), + "dev".to_owned(), + self.dev.clone(), + EGRESS_QDISC.to_owned(), + ]]; + plan.extend(self.filters.iter().map(|filter| filter.add_argv(&self.dev))); + plan + } + + /// How many filters this plan installs for one address family. + pub fn filter_count(&self, family: Family) -> usize { + self.filters.iter().filter(|filter| filter.family == family).count() + } +} + +/// The plan as the sidecar reads it: one `tc ` line per step, plus the count, so the caller +/// can cross-check the sidecar's echoed total against what was rendered. +/// +/// The same cross-check `sandbox_netns::plan_stdin` exists for, for the same reason: a truncated +/// stdin applies perfectly and exits 0, and only the count reveals it. +pub fn plan_stdin(plan: &IfacePlan) -> (String, usize) { + let steps = plan.install_plan(); + let mut out = String::new(); + for step in &steps { + out.push_str("tc"); + for arg in step { + out.push(' '); + out.push_str(arg); + } + out.push('\n'); + } + (out, steps.len()) +} + +// --------------------------------------------------------------------------------------------- +// Which interface, and the proof it is the right one +// --------------------------------------------------------------------------------------------- + +/// One link as `ip -details -oneline link show` reports it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Link { + pub index: u32, + pub name: String, + /// The peer's ifindex from a `name@ifN` suffix. A veth has one; nothing else does. + pub peer_index: Option, + /// The link kind `-details` prints (`veth`, `bridge`, …); `None` for a plain device. + pub kind: Option, + pub loopback: bool, + pub up: bool, +} + +/// `docker run` argv that enumerates the links inside the holder's namespace. +/// +/// **Unprivileged.** Listing links needs no capability, and this container must not be able to +/// change one: the whole question it answers is "what is here", and a reader that could also move an +/// interface would be a worse answer to it. `--entrypoint ip` replaces the applier, so the image's +/// privileged entrypoint is not reachable from here even if the arguments were wrong. +pub fn link_probe_argv(holder_name: &str, image: &str) -> Vec { + [ + "docker", + "run", + "--rm", + "--network", + &crate::sandbox_netns::NetnsHolder::network_mode_for(holder_name), + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--entrypoint", + "ip", + image, + "-details", + "-oneline", + "link", + "show", + ] + .into_iter() + .map(String::from) + .collect() +} + +/// Parse `ip -details -oneline link show` output. +/// +/// **A nonempty line that does not parse is a refusal, not a skip.** Silently dropping unparsable +/// records let a namespace holding `lo`, a veth and one malformed third record present itself as the +/// two-link shape [`select_egress_link`] accepts — the discarded record is exactly the link that +/// would have forced a refusal. +pub fn parse_links(stdout: &str) -> Result, String> { + let mut links = Vec::new(); + for (number, line) in stdout.lines().enumerate() { + let at = number + 1; + if line.trim().is_empty() { + continue; + } + // `: [@peer]: mtu … \ link/ … [kind] …` + let mut head = line.splitn(3, ": "); + let index: u32 = head + .next() + .and_then(|field| field.trim().parse().ok()) + .ok_or(format!("line {at}: link record names no ifindex: {line:?}"))?; + let name_field = head + .next() + .ok_or(format!("line {at}: link record names no interface: {line:?}"))? + .trim(); + if name_field.is_empty() { + return Err(format!("line {at}: link record has an empty interface name: {line:?}")); + } + let rest = head.next().unwrap_or_default(); + let (name, peer_index) = match name_field.split_once('@') { + Some((name, peer)) => ( + name.to_owned(), + peer.strip_prefix("if").and_then(|digits| digits.parse().ok()), + ), + None => (name_field.to_owned(), None), + }; + let tokens: Vec<&str> = rest.split_whitespace().collect(); + if !tokens.iter().any(|token| token.starts_with("link/")) { + return Err(format!( + "line {at}: link record for {name:?} names no link type: {line:?} — an unreadable \ + record is refused, because the link it hides is the one that would force a refusal" + )); + } + links.push(Link { + index, + name, + peer_index, + kind: LINK_KINDS + .iter() + .find(|kind| tokens.contains(kind)) + .map(|kind| (*kind).to_owned()), + loopback: tokens.iter().any(|token| *token == "link/loopback"), + up: rest + .split_once('>') + .map(|(flags, _)| flags.contains(",UP") || flags.contains("`, so it is in the holder's namespace by +/// construction. These checks are what catches the case where that construction did **not** hold — +/// a mis-resolved holder name, a runtime that ignored the network mode, a future caller that passes +/// the wrong container. The failure being designed against is filtering the **host's** interface, +/// which would be a host-global mutation this design forbids outright, so every ambiguity refuses. +pub fn select_egress_link(links: &[Link]) -> Result { + if links.is_empty() { + return Err("the namespace reported no links at all — the probe did not see a namespace" + .to_owned()); + } + // A namespace holding one job has exactly two links: loopback and one veth. The host's has many, + // and `docker0` or any bridge among them is the loudest possible "this is not a job namespace". + if let Some(bridge) = links.iter().find(|link| link.kind.as_deref() == Some("bridge")) { + return Err(format!( + "the namespace contains a bridge ({}) — this is a host or shared namespace, not a job's, \ + and nothing here may filter it", + bridge.name + )); + } + if !links.iter().any(|link| link.loopback) { + return Err("the namespace has no loopback link, so it is not a namespace this build made" + .to_owned()); + } + + let candidates: Vec<&Link> = links.iter().filter(|link| !link.loopback).collect(); + let [candidate] = candidates.as_slice() else { + return Err(format!( + "expected exactly one non-loopback link in the job's namespace, found {}: {:?} — \ + filtering one of several would leave the others open", + candidates.len(), + candidates.iter().map(|link| &link.name).collect::>() + )); + }; + if candidate.kind.as_deref() != Some("veth") { + return Err(format!( + "{} is a {:?}, not a veth — a physical or host-owned interface is never filtered by a job", + candidate.name, candidate.kind + )); + } + match candidate.peer_index { + None => { + return Err(format!( + "{} names no peer index, so it is not one end of a veth pair this namespace owns", + candidate.name + )) + } + Some(peer) if peer == candidate.index => { + return Err(format!( + "{} claims itself as its own veth peer (ifindex {peer})", + candidate.name + )) + } + Some(_) => {} + } + if !candidate.up { + return Err(format!( + "{} is down; filtering a down interface proves nothing about the one the job will use", + candidate.name + )); + } + Ok((*candidate).clone()) +} + +/// The `tc` spelling of an address family's ethertype selector. +/// +/// A free function rather than a method on [`Family`] so this whole layer adds nothing to +/// `sandbox_net`: that file is being rewritten by two other lanes at the same time, and a filter +/// that lives entirely in its own module is a filter that cannot lose a three-way merge. +pub fn tc_protocol(family: Family) -> &'static str { + match family { + Family::V4 => "ip", + Family::V6 => "ipv6", + } +} + +/// What `flower` prints back for the ethertype it matched. `tc` takes `protocol ip` on the command +/// line and lists `eth_type ipv4`, so readback compares this spelling rather than [`tc_protocol`]. +pub fn tc_eth_type(family: Family) -> &'static str { + match family { + Family::V4 => "ipv4", + Family::V6 => "ipv6", + } +} + +/// iptables spells a port range `49200:49299`; `tc` flower spells it `49200-49299`. +/// +/// The singleton is the case that cannot be translated mechanically. A one-port proxy range is +/// supported configuration — `PortRange::new` accepts equal endpoints and `PortRange::parse` +/// documents a bare `"49200"` — and iptables renders it `49221:49221`. Replacing the colon would +/// produce `dst_port 49221-49221`, which `tc` flower refuses outright: +/// +/// ```text +/// Illegal "dst_port" - max value should be greater than min value +/// ``` +/// +/// The filter then never installs, establishment refuses the launch, and a configuration the rest +/// of the stack accepts cannot run at all. `tc` spells that same match as the bare port, so an +/// equal-endpoint range collapses to one port here. A range with distinct endpoints keeps both: +/// collapsing those too would narrow the pinhole to its first port and deny the rest. +fn to_tc_port_range(dport: &str) -> String { + match dport.split_once(':') { + Some((start, end)) if start == end => start.to_owned(), + _ => dport.replace(':', "-"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sandbox_net::PortRange; + + const DEV: &str = "eth0"; + + fn policy() -> NetPolicy { + NetPolicy { + gateway: "172.17.0.1".to_owned(), + proxy_ports: Some(PortRange::new(49200, 49299).expect("valid range")), + log_connections: true, + // No resolver in the base fixture, so the expected filter sets below stay the ones + // these tests were written against. The resolver pinholes need no separate translation + // here: this plan is derived from `policy.rules()`, so every ACCEPT the renderer emits + // — proxy, resolver or neighbour discovery — reaches the veth by the same path. + dns_resolvers: Vec::new(), + } + } + + fn plan() -> IfacePlan { + IfacePlan::derive(DEV, &policy()).expect("the shipped policy must render") + } + + fn plan_for(start: u16, end: u16) -> IfacePlan { + let policy = NetPolicy { + proxy_ports: Some(PortRange::new(start, end).expect("a valid range")), + ..policy() + }; + IfacePlan::derive(DEV, &policy).expect("a supported policy must render") + } + + fn rendered_ports(plan: &IfacePlan) -> Vec { + plan.filters.iter().filter_map(|f| f.dst_port.clone()).collect() + } + + /// A supported SINGLE-PORT proxy range must render a filter `tc` will actually accept. + /// + /// `PortRange::new` accepts equal endpoints and `PortRange::parse` documents a bare `"49200"`, + /// so a one-port proxy range is supported configuration, not abuse. iptables spells it + /// `49221:49221`; a mechanical colon-to-hyphen translation spells it `dst_port 49221-49221`, + /// and `tc` flower REJECTS that outright — `Illegal "dst_port" - max value should be greater + /// than min value`. The filter never installs, establishment refuses the launch, and a + /// supported configuration cannot run at all. This is a fail-closed availability defect, not a + /// packet escape, and it is still a defect. + #[test] + fn a_single_port_proxy_range_renders_a_filter_tc_accepts() { + let plan = plan_for(49221, 49221); + let ports = rendered_ports(&plan); + assert!(!ports.is_empty(), "the pinhole must still carry a port match: {plan:#?}"); + for port in &ports { + assert!( + !port.contains('-'), + "tc rejects an equal-endpoint range; a single port must render bare: dst_port {port}" + ); + assert_eq!(port, "49221", "the one supported port is the one that must be matched"); + } + } + + /// The negative half of the same repair: collapsing EQUAL endpoints must not collapse a real + /// range. Without this, "fix the singleton" could be satisfied by emitting a bare start port + /// for every range, which would silently narrow the pinhole to one port and deny the rest. + #[test] + fn a_multi_port_proxy_range_still_renders_as_a_range() { + let ports = rendered_ports(&plan_for(49200, 49299)); + assert!( + ports.iter().any(|port| port == "49200-49299"), + "a real range must keep both endpoints: {ports:#?}" + ); + for port in &ports { + assert!( + !port.chars().all(|c| c.is_ascii_digit()), + "a multi-port range must not collapse to a single port: dst_port {port}" + ); + } + } + + /// And the invariant behind both halves: `tc` rejects every `N-N`, so the renderer must never + /// emit one for ANY supported range, at either end of the port space. + #[test] + fn no_supported_range_ever_renders_an_equal_endpoint_tc_range() { + for (start, end) in + [(49221u16, 49221u16), (1, 1), (65535, 65535), (49200, 49299), (1, 65535)] + { + for port in rendered_ports(&plan_for(start, end)) { + if let Some((low, high)) = port.split_once('-') { + assert_ne!( + low, high, + "tc rejects dst_port {port} from range {start}-{end}: max must exceed min" + ); + } + } + } + } + + /// Render a plan the way `tc filter show` prints it. + /// + /// **This is a shape, not a measurement, and it cannot prove the parser reads real `tc` output** + /// — for the positive case it is circular by construction. It exists so the *mutations* below + /// have something to mutate. The anchors against reality are + /// [`the_parser_reads_the_shape_tc_actually_prints`], which parses a literal capture, and the + /// live gate in `tests/sandbox_iface_live.rs`, which asks a real kernel. + fn as_tc_output(plan: &IfacePlan) -> String { + let mut out = String::new(); + for filter in &plan.filters { + let protocol = tc_protocol(filter.family); + out.push_str(&format!( + "filter protocol {protocol} pref {} flower chain 0 \n", + filter.pref + )); + out.push_str(&format!( + "filter protocol {protocol} pref {} flower chain 0 handle 0x1 \n", + filter.pref + )); + out.push_str(&format!(" eth_type {}\n", tc_eth_type(filter.family))); + if let Some(proto) = &filter.ip_proto { + out.push_str(&format!(" ip_proto {proto}\n")); + } + if let Some(ttl) = &filter.ip_ttl { + out.push_str(&format!(" ip_ttl {ttl}\n")); + } + if let Some(dst) = &filter.dst { + out.push_str(&format!(" dst_ip {dst}\n")); + } + if let Some(icmp_type) = &filter.icmp_type { + out.push_str(&format!(" icmp_type {icmp_type}\n")); + } + if let Some(port) = &filter.dst_port { + out.push_str(&format!(" dst_port {port}\n")); + } + out.push_str(" skip_hw\n"); + out.push_str(" not_in_hw\n"); + out.push_str(&format!("\taction order 1: gact action {}\n", filter.action)); + out.push_str("\t random type none pass val 0\n"); + out.push_str("\t index 1 ref 1 bind 1\n"); + } + out + } + + /// THE point of this module. The leak it closes was found over TCP, so a TCP-only translation + /// would pass every fixture that found it and contain nothing else. + #[test] + fn drops_are_protocol_independent() { + let plan = plan(); + let offenders: Vec<&IfaceFilter> = plan + .filters + .iter() + .filter(|filter| filter.action == "drop" && filter.ip_proto.is_some()) + .collect(); + assert!( + offenders.is_empty(), + "a drop filter names a protocol, so every other protocol reaches the denied prefix: \ + {offenders:?}" + ); + } + + /// Parity, in the direction that matters: nothing the iptables policy denies may be missing here. + /// A translation that quietly rendered seven of eight denied ranges would otherwise be invisible. + #[test] + fn every_policy_deny_is_translated_for_its_own_family() { + let policy = policy(); + let plan = plan(); + for rule in policy.rules().iter().filter(|rule| rule.target() == Some("DROP")) { + let destination = rule.destination().expect("a deny names a destination"); + assert!( + plan.filters.iter().any(|filter| { + filter.family == rule.family + && filter.dst.as_deref() == Some(destination) + && filter.action == "drop" + }), + "the policy denies {destination} on {:?} and the interface plan does not: {:#?}", + rule.family, + plan.filters + ); + } + } + + /// And parity in the other direction, which is where a widening would hide: every exception here + /// must be one the policy already makes, as narrow as the policy makes it. + #[test] + fn every_exception_is_the_policys_own_and_no_wider() { + let policy = policy(); + let plan = plan(); + let accepts: Vec<_> = + policy.rules().into_iter().filter(|rule| rule.target() == Some("ACCEPT")).collect(); + let passes: Vec<_> = plan.filters.iter().filter(|f| f.action == "pass").collect(); + assert_eq!(accepts.len(), passes.len(), "{passes:#?}"); + for (rule, filter) in accepts.iter().zip(passes.iter()) { + assert_eq!(filter.dst.as_deref(), rule.destination()); + assert_eq!( + filter.ip_proto.as_deref(), + arg_after(&rule.args, "-p"), + "the pinhole must stay bound to the protocol the policy bound it to" + ); + // Each exception keeps every narrowing its policy rule had. The port range belongs to + // the v4 pinhole; the ICMPv6 type and the hop limit belong to the two ND exceptions, + // and dropping either of those here would widen this hook past the iptables one. + assert_eq!(filter.dst_port.as_deref(), arg_after(&rule.args, "--dport").map(|_| "49200-49299")); + assert_eq!(filter.icmp_type.as_deref(), arg_after(&rule.args, "--icmpv6-type")); + assert_eq!(filter.ip_ttl.as_deref(), arg_after(&rule.args, "--hl-eq")); + } + assert!( + passes.iter().any(|filter| filter.dst_port.is_some()), + "the v4 pinhole's port range must be among the exceptions checked above" + ); + assert_eq!( + passes.iter().filter(|filter| filter.icmp_type.is_some()).count(), + 2, + "both ND exceptions must carry their ICMPv6 type" + ); + } + + /// A policy with no pinhole is a valid policy; it must not silently gain one, and must not render + /// an empty plan either. + #[test] + fn a_policy_without_a_pinhole_renders_only_denies() { + let mut unconfigured = policy(); + unconfigured.proxy_ports = None; + let plan = IfacePlan::derive(DEV, &unconfigured).expect("renders"); + assert!( + plan.filters + .iter() + .filter(|filter| filter.family == Family::V4) + .all(|filter| filter.action == "drop"), + "a policy with no pinhole must open nothing on v4: {:#?}", + plan.filters + ); + // v6 still passes neighbour discovery, and only that: without it the namespace reaches no + // v6 address at all, which is not containment but a dead stack that cannot tell a denied + // destination from an allowed one. Each pass carries its ICMPv6 type and its hop limit. + let v6_passes: Vec<&IfaceFilter> = plan + .filters + .iter() + .filter(|filter| filter.family == Family::V6 && filter.action == "pass") + .collect(); + assert_eq!(v6_passes.len(), 2, "{v6_passes:#?}"); + for filter in &v6_passes { + assert_eq!(filter.ip_proto.as_deref(), Some("icmpv6"), "{filter:#?}"); + assert_eq!(filter.ip_ttl.as_deref(), Some("255"), "{filter:#?}"); + assert!(filter.icmp_type.is_some(), "{filter:#?}"); + assert!(filter.dst_port.is_none(), "an ND exception opens no port: {filter:#?}"); + } + assert!(plan.filter_count(Family::V4) > 0 && plan.filter_count(Family::V6) > 0); + } + + #[test] + fn the_plan_installs_the_clsact_qdisc_before_any_filter() { + let plan = plan(); + let steps = plan.install_plan(); + assert_eq!(steps[0], vec!["qdisc", "add", "dev", DEV, EGRESS_QDISC]); + assert!(steps[1..].iter().all(|step| step[0] == "filter"), "{steps:#?}"); + + let (stdin, count) = plan_stdin(&plan); + assert_eq!(count, steps.len()); + assert_eq!(stdin.lines().count(), count, "one line per step, or the count lies"); + assert!(stdin.lines().all(|line| line.starts_with("tc ")), "{stdin}"); + assert!( + !stdin.lines().any(|line| line.contains('\t')), + "a tab in a plan line would split into an argument the applier never rendered" + ); + } + + /// Both families, on the interface as well as in the chain. An unfiltered family is the cheapest + /// bypass there is. + #[test] + fn both_address_families_are_filtered() { + let plan = plan(); + assert!(plan.filter_count(Family::V4) > 0); + assert!(plan.filter_count(Family::V6) > 0); + } + + #[test] + fn prefix_containment_knows_what_covers_the_gateway() { + assert_eq!(prefix_contains("172.16.0.0/12", "172.17.0.1"), Some(true)); + assert_eq!(prefix_contains("10.0.0.0/8", "172.17.0.1"), Some(false)); + assert_eq!(prefix_contains("169.254.0.0/16", "169.254.169.254/32"), Some(true)); + assert_eq!(prefix_contains("fc00::/7", "fd00::1"), Some(true)); + assert_eq!(prefix_contains("fe80::/10", "fd00::1"), Some(false)); + assert_eq!(prefix_contains("10.0.0.0/8", "fd00::1"), Some(false)); + assert_eq!(prefix_contains("not-a-prefix", "10.0.0.1"), None); + } + + /// The measured iptables failure, reproduced in `tc` terms: the pinhole below the range drop that + /// covers the gateway. Present, correct, and never reached. + #[test] + fn a_shadowed_pinhole_is_refused_rather_than_installed() { + let shadowed = vec![ + IfaceFilter { + family: Family::V4, + pref: 101, + dst: Some("172.16.0.0/12".into()), + ip_proto: None, + dst_port: None, + icmp_type: None, + ip_ttl: None, + action: "drop", + why: "range deny", + }, + IfaceFilter { + family: Family::V4, + pref: 102, + dst: Some("172.17.0.1".into()), + ip_proto: Some("tcp".into()), + dst_port: Some("49200-49299".into()), + icmp_type: None, + ip_ttl: None, + action: "pass", + why: "the proxy pinhole", + }, + ]; + let refused = no_shadowed_exception(&shadowed).expect_err("must refuse"); + assert!(refused.contains("inert"), "{refused}"); + // The same two, the right way round, must be accepted — otherwise this test passes for a + // checker that refuses everything. + let mut ordered = shadowed; + ordered.swap(0, 1); + ordered[0].pref = 101; + ordered[1].pref = 102; + no_shadowed_exception(&ordered).expect("the pinhole above its covering deny is correct"); + } + + /// The parser **and the verifier**, against a literal `tc filter show dev eth0 egress` block + /// rather than against something this file generated. + /// + /// The verifier leg is the one that matters: every other `verify_readback` test feeds it + /// [`as_tc_output`], which this file writes, so a parser and a renderer that agreed on a shape + /// the kernel never prints would pass all of them together. Here the expectation is hand-written + /// from the two filters the capture describes, and the input is the kernel's own text — + /// statistics lines, `installed`/`used` suffixes, trailing spaces and all. + #[test] + fn the_parser_and_the_verifier_both_read_the_shape_tc_actually_prints() { + const CAPTURE: &str = "\ +filter protocol ip pref 102 flower chain 0 +filter protocol ip pref 102 flower chain 0 handle 0x1 + eth_type ipv4 + ip_proto tcp + dst_ip 172.17.0.1 + dst_port 49200-49299 + skip_hw + not_in_hw + action order 1: gact action pass + random type none pass val 0 + index 1 ref 1 bind 1 installed 2 sec used 2 sec + Action statistics: + Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) +filter protocol ipv6 pref 111 flower chain 0 +filter protocol ipv6 pref 111 flower chain 0 handle 0x1 + eth_type ipv6 + dst_ip fc00::/7 + skip_hw + not_in_hw + action order 1: gact action drop + random type none pass val 0 + index 2 ref 1 bind 1 installed 2 sec used 0 sec + Action statistics: + Sent 168 bytes 4 pkt (dropped 4, overlimits 0 requeues 0) +"; + let parsed = parse_filters(CAPTURE).expect("real tc output must parse"); + assert_eq!(parsed.len(), 2, "the handle-less header lines are not filters: {parsed:#?}"); + assert_eq!(parsed[0].protocol, "ip"); + assert_eq!(parsed[0].pref, 102); + assert_eq!(parsed[0].chain, ACTIVE_CHAIN); + assert_eq!(parsed[0].handle, "0x1"); + assert_eq!(parsed[0].key("dst_ip"), Some("172.17.0.1")); + assert_eq!(parsed[0].key("ip_proto"), Some("tcp")); + assert_eq!(parsed[0].key("dst_port"), Some("49200-49299")); + assert_eq!(parsed[0].key("eth_type"), Some("ipv4")); + assert_eq!(parsed[0].actions, vec!["pass".to_owned()]); + assert_eq!(parsed[1].protocol, "ipv6"); + assert_eq!(parsed[1].key("dst_ip"), Some("fc00::/7")); + assert_eq!(parsed[1].key("ip_proto"), None); + assert_eq!(parsed[1].actions, vec!["drop".to_owned()]); + // Complete accounting: every key the capture prints is retained, none invented. + assert_eq!( + parsed[1].keys.iter().map(|(key, _)| key.as_str()).collect::>(), + vec!["eth_type", "dst_ip"] + ); + + // The non-circular positive control. This plan is written out by hand from what the capture + // above *means* — a proxy pinhole and a unique-local deny — and never rendered by this file. + // + // The capture is one filter short of a verifiable plan: the box it came from had no IPv4 + // drop, and `verify_readback` refuses a plan with an unfiltered family. So the third block + // is derived from the capture's own IPv6 drop block — every byte of layout, indentation, + // statistics and trailing whitespace is the kernel's, and only the family tokens and the + // prefix are changed. That keeps this leg a test of real `tc` text rather than of + // [`as_tc_output`]. + let v6_drop_block: String = CAPTURE + .lines() + .skip_while(|line| !line.starts_with("filter protocol ipv6")) + .map(|line| format!("{line}\n")) + .collect(); + assert!(v6_drop_block.contains("gact action drop"), "{v6_drop_block}"); + let v4_drop_block = v6_drop_block + .replace("protocol ipv6", "protocol ip") + .replace("pref 111", "pref 101") + .replace("eth_type ipv6", "eth_type ipv4") + .replace("dst_ip fc00::/7", "dst_ip 10.0.0.0/8"); + let captured_text = format!("{CAPTURE}{v4_drop_block}"); + let captured = IfacePlan { + dev: DEV.to_owned(), + filters: vec![ + IfaceFilter { + family: Family::V4, + pref: 102, + dst: Some("172.17.0.1".to_owned()), + ip_proto: Some("tcp".to_owned()), + dst_port: Some("49200-49299".to_owned()), + icmp_type: None, + ip_ttl: None, + action: "pass", + why: "the proxy pinhole, as the kernel printed it", + }, + IfaceFilter { + family: Family::V6, + pref: 111, + dst: Some("fc00::/7".to_owned()), + ip_proto: None, + dst_port: None, + icmp_type: None, + ip_ttl: None, + action: "drop", + why: "the unique-local deny, as the kernel printed it", + }, + IfaceFilter { + family: Family::V4, + pref: 101, + dst: Some("10.0.0.0/8".to_owned()), + ip_proto: None, + dst_port: None, + icmp_type: None, + ip_ttl: None, + action: "drop", + why: "the private-range deny, in the kernel's own layout", + }, + ], + }; + captured + .verify_readback(&captured_text) + .expect("the verifier must accept real tc output for the plan it describes"); + + // ...and it is not accepting it blindly: one word of that same capture changed, and the + // same verifier refuses. Without this leg the acceptance above could come from a verifier + // that accepts anything shaped like tc output. + let parked = + captured_text.replace("ipv6 pref 111 flower chain 0", "ipv6 pref 111 flower chain 7"); + let refused = captured + .verify_readback(&parked) + .expect_err("a real capture with one filter parked in chain 7 must be refused"); + assert!(refused.contains("chain"), "{refused}"); + + // F1, strict token accounting: the lines this parser *skips* are the ones worth hiding in. + // Statistics lines and action bookkeeping carry nothing that is compared, so each is read to + // the end and refused on anything unrecognised rather than skipped wholesale. + let hidden: &[(&str, &str, &str)] = &[ + ( + "a match key smuggled onto a statistics line", + "Sent 168 bytes 4 pkt (dropped 4, overlimits 0 requeues 0) ", + "Sent 168 bytes 4 pkt (dropped 4, overlimits 0 requeues 0) dst_ip 0.0.0.0/0", + ), + // R2, F1: the blacklist that preceded this grammar asked only whether a statistics + // line contained a RECOGNISED predicate, so a narrowing key it had never heard of rode + // through untouched and left the verification unchanged. `src_ip` is that key: it is in + // neither token list, and it narrows the rule to one source. + ( + "a source narrowing appended to a statistics line", + "Sent 168 bytes 4 pkt (dropped 4, overlimits 0 requeues 0) ", + "Sent 168 bytes 4 pkt (dropped 4, overlimits 0 requeues 0) src_ip 192.0.2.123", + ), + ( + "the same narrowing in its IPv6 spelling", + "Sent 168 bytes 4 pkt (dropped 4, overlimits 0 requeues 0) ", + "Sent 168 bytes 4 pkt (dropped 4, overlimits 0 requeues 0) src_ip 2001:db8::123", + ), + ( + "arbitrary text no list will ever contain", + "Sent 168 bytes 4 pkt (dropped 4, overlimits 0 requeues 0) ", + "Sent 168 bytes 4 pkt (dropped 4, overlimits 0 requeues 0) frobnicate 7", + ), + ( + "a counter replaced by something that is not a count", + "Sent 168 bytes 4 pkt (dropped 4, overlimits 0 requeues 0) ", + "Sent 168 bytes 4 pkt (dropped dst_ip, overlimits 0 requeues 0) ", + ), + ( + "an unread token on the action bookkeeping line", + " index 2 ref 1 bind 1 installed 2 sec used 0 sec", + " index 2 ref 1 bind 1 installed 2 sec used 0 sec goto chain 9", + ), + ( + "a probability restored under a `random type none` prefix", + " random type none pass val 0\n\t index 2", + " random type none pass val 7\n\t index 2", + ), + ( + "a non-numeric hardware count, whose value this parser steps over", + " skip_hw\n\tnot_in_hw\n\taction order 1: gact action drop", + " skip_hw in_hw_count dst_ip\n\tnot_in_hw\n\taction order 1: gact action drop", + ), + ]; + for (what, from, to) in hidden { + let mutated = captured_text.replace(from, to); + assert_ne!(&mutated, &captured_text, "{what}: the mutation did not apply"); + let refused = captured + .verify_readback(&mutated) + .expect_err(&format!("{what} must not verify")); + assert!(!refused.is_empty(), "{what}: {refused}"); + } + } + + /// Rewrite only the lines that belong to a filter of `protocol`, leaving the other family's + /// lines exactly as they were. + /// + /// Every mutation below has to be applied per family and refused in both: a check that fires for + /// IPv4 and not for IPv6 is not a check, it is an IPv6-shaped hole with a passing test over it. + /// `edit` returns `Some(replacement)` for a line it rewrites (the replacement may be several + /// lines) and `None` to leave it alone. + fn in_family(text: &str, protocol: &str, edit: impl Fn(&str) -> Option) -> String { + let mut out: Vec = Vec::new(); + let mut current = String::new(); + for line in text.lines() { + if let Some(rest) = line.trim_start().strip_prefix("filter protocol ") { + current = rest.split_whitespace().next().unwrap_or_default().to_owned(); + } + match edit(line).filter(|_| current == protocol) { + Some(replacement) => out.push(replacement), + None => out.push(line.to_owned()), + } + } + let mut text = out.join("\n"); + text.push('\n'); + text + } + + /// F1: the substitutions that survive a *lossy* readback untouched. Every one of these is a + /// valid rule `tc` would accept and list, agreeing on protocol, pref, destination, ip_proto, + /// port and terminal action — every field the old projection compared — while being a + /// different filter. Both families, because a check applied to one is a bypass in the other. + #[test] + fn a_rule_that_differs_only_in_what_the_old_parser_discarded_is_refused() { + let plan = plan(); + let faithful = as_tc_output(&plan); + + // 1. A NARROWER DENY: the same destination drop, restricted to one source. Traffic from + // every other source misses it, and nothing the old parser read has changed. + for (family, src) in + [("v4", " src_ip 192.0.2.123\n"), ("v6", " src_ip 2001:db8::123\n")] + { + let anchor = if family == "v4" { " dst_ip 10.0.0.0/8\n" } else { " dst_ip fc00::/7\n" }; + assert!(faithful.contains(anchor), "fixture anchor {anchor:?} missing"); + let narrowed = faithful.replace(anchor, &format!("{anchor}{src}")); + let refused = plan + .verify_readback(&narrowed) + .expect_err("a source-restricted deny must not verify as the full deny"); + assert!(refused.contains("src_ip"), "{family}: {refused}"); + } + + // 2. AN INACTIVE CHAIN: installed, listed, never consulted on the egress path. + for family in ["ip", "ipv6"] { + let parked = faithful.replace( + &format!("filter protocol {family} pref"), + &format!("filter protocol {family} PREF_MARKER"), + ); + let parked = parked.replace("PREF_MARKER", "pref"); + let parked = parked + .lines() + .map(|line| { + if line.trim_start().starts_with(&format!("filter protocol {family} ")) { + line.replace("chain 0", "chain 7") + } else { + line.to_owned() + } + }) + .collect::>() + .join("\n"); + let refused = plan + .verify_readback(&parked) + .expect_err("a filter in an unreferenced chain must not verify"); + assert!(refused.contains("chain"), "{family}: {refused}"); + } + + // 3. A SECOND ACTION after the terminal one — the old parser kept only the last verb it saw. + for family in ["ip", "ipv6"] { + let two_actions = in_family(&faithful, family, |line| { + line.trim_start() + .starts_with("action order 1: gact action") + .then(|| format!("{line}\n\taction order 2: gact action pass")) + }); + let refused = plan + .verify_readback(&two_actions) + .expect_err("two actions on one filter must not verify"); + assert!(refused.contains("action"), "{family}: {refused}"); + } + + // 4. A DUPLICATE key, where the second silently overwrote the first. + for family in ["ip", "ipv6"] { + let duplicated = in_family(&faithful, family, |line| { + line.trim_start().starts_with("dst_ip ").then(|| format!("{line}\n{line}")) + }); + let refused = plan + .verify_readback(&duplicated) + .expect_err("a duplicated match key must not verify"); + assert!(refused.contains("twice"), "{family}: {refused}"); + } + + // 5. A DIFFERENT CLASSIFIER matching by different rules under the same header fields. + for family in ["ip", "ipv6"] { + let u32_classifier = + in_family(&faithful, family, |line| Some(line.replace("flower", "u32"))); + let refused = plan + .verify_readback(&u32_classifier) + .expect_err("a non-flower classifier must not verify"); + assert!(refused.contains("classifier"), "{family}: {refused}"); + } + + // 6. TRUNCATION: the listing stops after a header, before the rule it describes. + for family in ["ip", "ipv6"] { + let cut = format!( + "{}\nfilter protocol {family} pref 140 flower chain 0\n", + faithful.trim_end() + ); + let refused = + plan.verify_readback(&cut).expect_err("a truncated listing must not verify"); + assert!( + refused.contains("truncated") || refused.contains("filters"), + "{family}: {refused}" + ); + } + + // 7. An unknown predicate that is not a known bypass — refused because it is unread, not + // because this module happens to know what it does. + for family in ["ip", "ipv6"] { + let unknown = in_family(&faithful, family, |line| { + line.trim_start() + .starts_with("dst_ip ") + .then(|| format!("{line}\n tcp_flags 0x2")) + }); + let refused = + plan.verify_readback(&unknown).expect_err("an unknown predicate must not verify"); + assert!(refused.contains("unknown match key"), "{family}: {refused}"); + } + + // A verifier that refused everything would pass all seven. The real shape still verifies. + plan.verify_readback(&faithful).expect("the faithful readback must still verify"); + } + + /// `skip_sw` means the software path never evaluates the rule: listed, and inert. + #[test] + fn a_filter_the_software_path_never_evaluates_is_refused() { + let plan = plan(); + let inert = as_tc_output(&plan).replace(" skip_hw\n", " skip_sw\n"); + let refused = plan.verify_readback(&inert).expect_err("skip_sw must not verify"); + assert!(refused.contains("skip_sw"), "{refused}"); + } + + #[test] + fn a_faithful_readback_verifies() { + let plan = plan(); + plan.verify_readback(&as_tc_output(&plan)).expect("the plan must verify against itself"); + } + + /// Each of these is a specific way containment can be broken while everything else stays intact, + /// and each must be named by the refusal. A verifier that returns `Ok` for all of them passes + /// [`a_faithful_readback_verifies`] just as well. + #[test] + fn a_broken_readback_is_refused_and_says_what_broke() { + let plan = plan(); + let faithful = as_tc_output(&plan); + + let empty = plan.verify_readback("").expect_err("an unfiltered veth must not verify"); + assert!(empty.contains("expected"), "{empty}"); + + // One filter never landed — a partial application, which exits 0 for every rule that did. + let truncated: String = faithful + .lines() + .take_while(|line| !line.contains("pref 111")) + .collect::>() + .join("\n"); + let short = plan.verify_readback(&truncated).expect_err("a short list must not verify"); + assert!(short.contains("egress filters"), "{short}"); + + // The pinhole widened to every port, still exactly one pass filter. + let widened = faithful.replace("dst_port 49200-49299", "dst_port 1-65535"); + let widened = plan.verify_readback(&widened).expect_err("a widened pinhole must not verify"); + assert!(widened.contains("dst_port"), "{widened}"); + + // A deny that became TCP-only: the exact bug this module exists to close, installed under + // the exact rule name that is supposed to close it. + let tcp_only = faithful.replace(" dst_ip 10.0.0.0/8", " ip_proto tcp\n dst_ip 10.0.0.0/8"); + let tcp_only = + plan.verify_readback(&tcp_only).expect_err("a TCP-only deny must not verify"); + assert!(tcp_only.contains("ip_proto"), "{tcp_only}"); + + // A deny quietly turned into a pass. + let flipped = faithful.replacen("gact action drop", "gact action pass", 1); + let flipped = plan.verify_readback(&flipped).expect_err("a flipped action must not verify"); + assert!(flipped.contains("action"), "{flipped}"); + + // An extra filter nobody rendered. + let injected = format!( + "{faithful}filter protocol ip pref 99 flower chain 0 handle 0x9 \n dst_ip 0.0.0.0/0\n\ + \taction order 1: gact action pass\n" + ); + let injected = plan.verify_readback(&injected).expect_err("an extra filter must not verify"); + assert!(injected.contains("egress filters"), "{injected}"); + } + + /// A v4-only namespace reads as complete and routes straight out over v6. + #[test] + fn a_readback_missing_a_whole_family_is_refused() { + let policy = policy(); + let plan = plan(); + let v4_only: Vec = + plan.filters.iter().filter(|f| f.family == Family::V4).cloned().collect(); + let v4_plan = IfacePlan { dev: DEV.to_owned(), filters: v4_only }; + let refused = plan + .verify_readback(&as_tc_output(&v4_plan)) + .expect_err("a v4-only namespace must not verify against a two-family plan"); + assert!(refused.contains("egress filters"), "{refused}"); + + // …and even a plan that only ever asked for v4 must be refused, because the policy denies v6 + // and this layer is not allowed to be narrower than the policy. + let v4_refused = v4_plan + .verify_readback(&as_tc_output(&v4_plan)) + .expect_err("a plan with no v6 filters must not verify"); + assert!(v4_refused.contains("ipv6"), "{v4_refused}"); + assert!(policy.rule_count(Family::V6) > 0); + } + + // -- interface identity ------------------------------------------------------------------- + + const HOLDER_LINKS: &str = "\ +1: lo: mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000\\ link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 promiscuity 0 minmtu 0 maxmtu 0 numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535 +107: eth0@if108: mtu 1500 qdisc noqueue state UP mode DEFAULT group default \\ link/ether 02:42:ac:11:00:02 brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 0 minmtu 68 maxmtu 65535 veth numtxqueues 4 numrxqueues 4 gso_max_size 65536 gso_max_segs 65535 +"; + + /// The host's own namespace, which this must never filter: a physical interface and a bridge. + const HOST_LINKS: &str = "\ +1: lo: mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000\\ link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 +2: eth0: mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000\\ link/ether 5a:94:ef:12:00:01 brd ff:ff:ff:ff:ff:ff +3: docker0: mtu 1500 qdisc noqueue state UP mode DEFAULT group default \\ link/ether 02:42:1b:aa:bb:cc brd ff:ff:ff:ff:ff:ff promiscuity 0 bridge forward_delay 1500 +108: veth9a1b@if107: mtu 1500 qdisc noqueue master docker0 state UP mode DEFAULT group default \\ link/ether 9a:1b:2c:3d:4e:5f brd ff:ff:ff:ff:ff:ff link-netnsid 1 promiscuity 1 veth +"; + + /// Fixture links that are expected to be readable. A fixture that stopped parsing would + /// otherwise turn every identity test below into a vacuous refusal. + fn links_of(text: &str) -> Vec { + parse_links(text).expect("fixture link records must parse") + } + + #[test] + fn the_job_veth_is_selected_inside_the_holders_namespace() { + let links = links_of(HOLDER_LINKS); + assert_eq!(links.len(), 2, "{links:#?}"); + let chosen = select_egress_link(&links).expect("the holder's veth must be selectable"); + assert_eq!(chosen.name, "eth0"); + assert_eq!(chosen.index, 107); + assert_eq!(chosen.peer_index, Some(108)); + assert_eq!(chosen.kind.as_deref(), Some("veth")); + } + + /// The refusal that matters most: a host namespace is never filtered, and the error says so + /// rather than picking whichever interface sorted first. + #[test] + fn the_hosts_own_namespace_is_refused() { + let refused = select_egress_link(&links_of(HOST_LINKS)).expect_err("must refuse"); + assert!(refused.contains("bridge") || refused.contains("host"), "{refused}"); + } + + /// F1: a link record that does not parse is refused, not skipped. Skipping it let a namespace + /// holding `lo`, a veth and one unreadable third link present the exact two-link shape + /// [`select_egress_link`] accepts — and the discarded record is the one that would have forced + /// the refusal. + #[test] + fn an_unreadable_link_record_is_refused_rather_than_skipped() { + let with_garbage = format!("{HOLDER_LINKS}109: eth1@if110: mtu 1500 \n"); + let refused = parse_links(&with_garbage) + .expect_err("a record with no link type must not be silently dropped"); + assert!(refused.contains("link type"), "{refused}"); + + assert!(parse_links("not a link record at all\n").is_err()); + assert!(parse_links("7: : \\ link/ether 02:42 veth \n").is_err(), "empty name"); + // The honest capture still parses, so this is not a parser that refuses everything. + assert_eq!(links_of(HOLDER_LINKS).len(), 2); + } + + #[test] + fn every_ambiguous_or_wrong_shaped_namespace_is_refused() { + assert!(select_egress_link(&[]).is_err(), "no links at all"); + assert!( + select_egress_link(&links_of( + "107: eth0@if108: mtu 1500 \\ link/ether 02:42 veth \n" + )) + .is_err(), + "a namespace with no loopback is not one this build made" + ); + + let two_veths = format!( + "{HOLDER_LINKS}109: eth1@if110: mtu 1500 \\ \ + link/ether 02:42:ac:11:00:03 veth \n" + ); + let ambiguous = select_egress_link(&links_of(&two_veths)).expect_err("must refuse"); + assert!(ambiguous.contains("exactly one"), "{ambiguous}"); + + let not_a_veth = HOLDER_LINKS.replace(" veth numtxqueues", " numtxqueues"); + let refused = select_egress_link(&links_of(¬_a_veth)).expect_err("must refuse"); + assert!(refused.contains("veth"), "{refused}"); + + let down = HOLDER_LINKS.replace("", ""); + let refused = select_egress_link(&links_of(&down)).expect_err("must refuse"); + assert!(refused.contains("down"), "{refused}"); + } + + #[test] + fn the_probe_and_the_applier_are_different_containers_with_different_powers() { + let probe = link_probe_argv("maxplayer-netns-j1", "img"); + assert!(probe.contains(&"--cap-drop".to_owned())); + assert!(!probe.contains(&"NET_ADMIN".to_owned()), "a reader needs no capability: {probe:?}"); + assert!(probe.contains(&"container:maxplayer-netns-j1".to_owned()), "{probe:?}"); + + let applier = iface_sidecar_argv("maxplayer-netns-j1", "img"); + assert!(applier.contains(&"NET_ADMIN".to_owned())); + assert!(applier.contains(&"--interactive".to_owned()), "the plan arrives on stdin"); + assert!(applier.contains(&"/usr/local/bin/apply-iface".to_owned()), "{applier:?}"); + + // The readback is a third container running a different verb, so it cannot install anything. + let readback = filter_readback_argv("maxplayer-netns-j1", "img", DEV); + assert!(readback.windows(2).any(|pair| pair == ["--entrypoint", "tc"]), "{readback:?}"); + assert!(readback.contains(&"show".to_owned()) && !readback.contains(&"add".to_owned())); + } +} + +/// Does `prefix` (CIDR, or a bare address meaning a host route) contain `address`? +/// +/// `None` when either side does not parse or the families differ — an unanswerable question, which +/// every caller here treats as "cannot prove it is safe" rather than "safe". +pub fn prefix_contains(prefix: &str, address: &str) -> Option { + use std::net::IpAddr; + + let (network, bits) = match prefix.split_once('/') { + Some((network, len)) => (network.parse::().ok()?, len.parse::().ok()?), + None => { + let network = prefix.parse::().ok()?; + let bits = if network.is_ipv4() { 32 } else { 128 }; + (network, bits) + } + }; + let address = address.split('/').next()?.parse::().ok()?; + + match (network, address) { + (IpAddr::V4(network), IpAddr::V4(address)) => { + if bits > 32 { + return None; + } + let mask = if bits == 0 { 0 } else { u32::MAX << (32 - bits) }; + Some(u32::from(network) & mask == u32::from(address) & mask) + } + (IpAddr::V6(network), IpAddr::V6(address)) => { + if bits > 128 { + return None; + } + let mask = if bits == 0 { 0 } else { u128::MAX << (128 - bits) }; + Some(u128::from(network) & mask == u128::from(address) & mask) + } + // Different families never contain one another, and `tc` keeps them on separate ethertypes + // anyway. + _ => Some(false), + } +} + +/// Refuse a filter list in which a `pass` is shadowed by an earlier `drop` covering its destination. +/// +/// The failure it names is inertness, not incorrectness: the exception is present, spelled right, +/// and never reached. `sandbox_net` measured this exact shape on the iptables side — the pinhole +/// appended after the range drops "leaves it inert" — and a first-match classifier reproduces it +/// faithfully unless someone checks. +fn no_shadowed_exception(filters: &[IfaceFilter]) -> Result<(), String> { + for pass in filters.iter().filter(|filter| filter.action == "pass") { + for drop in filters + .iter() + .filter(|filter| filter.action == "drop" && filter.family == pass.family) + .filter(|filter| filter.pref < pass.pref) + { + // A pass with no destination matches every address, so *any* earlier drop in its + // family shadows part of it. That is the conservative reading, and it is the right one + // here: the ND exceptions must sit above the range drops, and an ND exception the + // drops cover is the measured failure this whole check exists for — a namespace whose + // neighbour discovery is dropped cannot reach any v6 address at all. + let covered = match (&drop.dst, &pass.dst) { + (Some(drop_dst), Some(pass_dst)) => prefix_contains(drop_dst, pass_dst) != Some(false), + _ => true, + }; + if covered { + return Err(format!( + "the exception for {} at pref {} sits below the drop for {} at pref {}, which \ + covers it — tc takes the first match, so that exception is inert ({})", + pass.dst_label(), + pass.pref, + drop.dst_label(), + drop.pref, + pass.why + )); + } + } + } + Ok(()) +} + +/// The value following `flag` in an argv, if present. +fn arg_after<'a>(args: &'a [String], flag: &str) -> Option<&'a str> { + args.iter() + .position(|arg| arg == flag) + .and_then(|at| args.get(at + 1)) + .map(String::as_str) +} + +// --------------------------------------------------------------------------------------------- +// Asking the kernel what it actually holds +// --------------------------------------------------------------------------------------------- + +/// `docker run` argv for the sidecar that applies an interface plan. +/// +/// The same shape as `sandbox_netns::sidecar_argv` and for the same reasons: `NET_ADMIN` and nothing +/// else, in a container that joins the namespace, applies a plan it did not choose, and exits before +/// the payload exists. `--entrypoint` names the interface applier rather than the iptables one, so a +/// plan of one kind cannot be fed to the applier for the other. +pub fn iface_sidecar_argv(holder_name: &str, image: &str) -> Vec { + [ + "docker", + "run", + "--rm", + "--interactive", + "--network", + &crate::sandbox_netns::NetnsHolder::network_mode_for(holder_name), + "--cap-drop", + "ALL", + "--cap-add", + "NET_ADMIN", + "--security-opt", + "no-new-privileges", + "--entrypoint", + "/usr/local/bin/apply-iface", + image, + ] + .into_iter() + .map(String::from) + .collect() +} + +/// `docker run` argv that reads the installed filters back out of the namespace. +/// +/// A **separate container** running a **different verb** (`filter show`, not `filter add`), for the +/// reason `sandbox_netns::readback_argv` states: the question is what the kernel holds, not whether +/// the installer believes it succeeded. Both families come back in one read, because the order +/// between them is part of what is verified and two reads could not see it. +pub fn filter_readback_argv(holder_name: &str, image: &str, dev: &str) -> Vec { + [ + "docker", + "run", + "--rm", + "--network", + &crate::sandbox_netns::NetnsHolder::network_mode_for(holder_name), + "--cap-drop", + "ALL", + "--cap-add", + "NET_ADMIN", + "--security-opt", + "no-new-privileges", + "--entrypoint", + "tc", + image, + "filter", + "show", + "dev", + dev, + EGRESS_HOOK, + ] + .into_iter() + .map(String::from) + .collect() +} + +/// The only chain reached from the `clsact` egress hook by default. A filter parked in any other +/// chain is installed, listed, and never consulted — it looks exactly like containment and is none. +pub const ACTIVE_CHAIN: u32 = 0; + +/// The classifier this module installs, and the only one readback will bless. +pub const CLASSIFIER: &str = "flower"; + +/// The match keys a rendered filter may carry. **This list is the security boundary**: any other +/// predicate — `src_ip` above all — narrows what the rule matches while leaving every field this +/// module compares untouched, so an unknown key is refused rather than ignored. +const KNOWN_KEYS: &[&str] = + &["eth_type", "dst_ip", "ip_proto", "dst_port", "icmp_type", "ip_ttl"]; + +/// Valueless tokens `tc` prints that say nothing about what the rule matches. `skip_sw` is +/// deliberately absent: it means the software path never evaluates the rule, so a filter carrying it +/// is listed and inert exactly like one in an unreferenced chain. +const KNOWN_FLAGS: &[&str] = &["not_in_hw", "in_hw", "skip_hw"]; + +/// The `gact` verbs this module renders, and the only ones a readback may name. `tc` reprints the +/// verb inside the `random type none val 0` detail line, so the same list gates both places. +const GACT_VERBS: &[&str] = &["pass", "drop"]; + +/// One filter as `tc filter show` prints it, with **every token accounted for**. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct ReadbackFilter { + /// `ip` or `ipv6`. + pub protocol: String, + pub pref: u16, + /// The chain the filter sits in. Only [`ACTIVE_CHAIN`] is on the egress path. + pub chain: u32, + pub handle: String, + /// Match keys in printed order, every one of them — not a chosen projection. + pub keys: Vec<(String, String)>, + /// The `gact` verbs, in printed order. More than one is a refusal, not a last-one-wins. + pub actions: Vec, +} + +impl ReadbackFilter { + /// The value of one match key, if the filter carries it. + pub fn key(&self, name: &str) -> Option<&str> { + self.keys.iter().find(|(key, _)| key == name).map(|(_, value)| value.as_str()) + } + + fn describe(&self) -> String { + let keys: Vec = + self.keys.iter().map(|(key, value)| format!("{key} {value}")).collect(); + format!( + "protocol {} pref {} chain {} handle {} [{}] actions {:?}", + self.protocol, + self.pref, + self.chain, + self.handle, + keys.join(", "), + self.actions + ) + } +} + +/// Parse `tc filter show dev egress` output strictly, in kernel order. +/// +/// **Every token is consumed or the parse fails.** The previous version of this function recorded a +/// chosen projection — protocol, pref, destination, ip_proto, port, last action — and silently +/// dropped the rest, which let a rule that was narrower (`src_ip` added), parked off the egress path +/// (`chain 7`), or carrying a second action compare equal to the rule that was meant to be there. +/// A comparison cannot recover information the parser discarded, so nothing is discarded. +/// +/// `tc` prints a bare `filter protocol … pref … flower chain 0` header line per priority **and** a +/// second line carrying `handle`, followed by the match keys. Only the handle-bearing block is a +/// filter; counting the header too would double every total and make a namespace holding half the +/// plan look complete. The header must be followed by its handle line: a header alone is truncation. +pub fn parse_filters(stdout: &str) -> Result, String> { + let mut filters: Vec = Vec::new(); + let mut pending_header: Option<(String, u16, u32)> = None; + + for (number, line) in stdout.lines().enumerate() { + let at = number + 1; + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + let fields: Vec<&str> = trimmed.split_whitespace().collect(); + + if fields[0] == "filter" { + let header = parse_filter_header(&fields, at)?; + match header.handle.clone() { + None => { + if pending_header.is_some() { + return Err(format!( + "line {at}: a second bare filter header arrived before the first one's \ + handle line — the listing is truncated or interleaved" + )); + } + pending_header = Some((header.protocol, header.pref, header.chain)); + } + Some(handle) => { + if let Some((protocol, pref, chain)) = pending_header.take() { + if protocol != header.protocol + || pref != header.pref + || chain != header.chain + { + return Err(format!( + "line {at}: the handle line (protocol {} pref {} chain {}) does not \ + match the header it follows (protocol {protocol} pref {pref} chain \ + {chain})", + header.protocol, header.pref, header.chain + )); + } + } + filters.push(ReadbackFilter { + protocol: header.protocol, + pref: header.pref, + chain: header.chain, + handle, + keys: Vec::new(), + actions: Vec::new(), + }); + } + } + continue; + } + + let Some(current) = filters.last_mut() else { + return Err(format!( + "line {at}: {trimmed:?} appears before any filter header — this is not the output \ + of `tc filter show` on an egress hook" + )); + }; + + if fields[0] == "action" { + parse_action_line(current, &fields, at)?; + continue; + } + if is_counter_line(&fields) { + check_counter_line(current, &fields, at)?; + continue; + } + if is_action_detail(&fields) { + parse_action_detail(current, &fields, at)?; + continue; + } + parse_key_line(current, &fields, at)?; + } + + if let Some((protocol, pref, _)) = pending_header { + return Err(format!( + "the listing ends on a bare header (protocol {protocol} pref {pref}) with no handle line \ + — truncated output is not a verified namespace" + )); + } + for filter in &filters { + if filter.actions.is_empty() { + return Err(format!( + "filter {} carries no action — a classifier that matches and does nothing is not \ + containment, and a listing that ends mid-filter is truncation", + filter.describe() + )); + } + } + Ok(filters) +} + +struct FilterHeader { + protocol: String, + pref: u16, + chain: u32, + handle: Option, +} + +/// `filter protocol ip pref 100 flower chain 0 [handle 0x1]`, and nothing else. +fn parse_filter_header(fields: &[&str], at: usize) -> Result { + let want = |index: usize, keyword: &str| -> Result<(), String> { + match fields.get(index) { + Some(field) if *field == keyword => Ok(()), + other => Err(format!( + "line {at}: expected {keyword:?} at position {index} of a filter header, found \ + {other:?}" + )), + } + }; + want(1, "protocol")?; + want(3, "pref")?; + want(6, "chain")?; + + let protocol = (*fields.get(2).ok_or(format!("line {at}: filter header names no protocol"))?) + .to_owned(); + let pref: u16 = fields + .get(4) + .ok_or(format!("line {at}: filter header names no pref"))? + .parse() + .map_err(|_| format!("line {at}: {:?} is not a pref", fields[4]))?; + let kind = *fields.get(5).ok_or(format!("line {at}: filter header names no classifier"))?; + if kind != CLASSIFIER { + return Err(format!( + "line {at}: classifier is {kind:?}, expected {CLASSIFIER:?} — a different classifier \ + matches by different rules, whatever the listing looks like" + )); + } + let chain: u32 = fields + .get(7) + .ok_or(format!("line {at}: filter header names no chain"))? + .parse() + .map_err(|_| format!("line {at}: {:?} is not a chain number", fields[7]))?; + + let handle = match fields.len() { + 8 => None, + 10 if fields[8] == "handle" => Some(fields[9].to_owned()), + _ => { + return Err(format!( + "line {at}: unexpected trailing tokens in a filter header: {:?}", + &fields[8.min(fields.len())..] + )) + } + }; + Ok(FilterHeader { protocol, pref, chain, handle }) +} + +/// `action order 1: gact action drop` +fn parse_action_line( + filter: &mut ReadbackFilter, + fields: &[&str], + at: usize, +) -> Result<(), String> { + if fields.get(1) != Some(&"order") { + return Err(format!("line {at}: malformed action line {fields:?}")); + } + let order: usize = fields + .get(2) + .and_then(|field| field.trim_end_matches(':').parse().ok()) + .ok_or(format!("line {at}: action line names no order"))?; + if order != filter.actions.len() + 1 { + return Err(format!( + "line {at}: action order {order} arrived after {} action(s) — the listing is out of \ + order or a line is missing", + filter.actions.len() + )); + } + let kind = *fields.get(3).ok_or(format!("line {at}: action line names no kind"))?; + if kind != "gact" { + return Err(format!( + "line {at}: action kind is {kind:?}, expected \"gact\" — this module renders nothing \ + else, and an unrecognised action can do anything at all" + )); + } + if fields.get(4) != Some(&"action") { + return Err(format!("line {at}: malformed gact action line {fields:?}")); + } + let verb = *fields.get(5).ok_or(format!("line {at}: gact names no verb"))?; + if fields.len() > 6 { + return Err(format!("line {at}: unexpected trailing tokens after the verb: {:?}", &fields[6..])); + } + filter.actions.push(verb.to_owned()); + Ok(()) +} + +fn is_action_detail(fields: &[&str]) -> bool { + matches!(fields[0], "random" | "index") +} + +/// The counter block `tc` prints under an action in the captured real output. Packet and byte +/// counts carry no match semantics, so they are consumed rather than refused — but only these +/// spellings, so an unrecognised line is still a refusal. +fn is_counter_line(fields: &[&str]) -> bool { + matches!(fields[0], "Sent" | "backlog") + || (fields[0] == "Action" && fields.get(1) == Some(&"statistics:")) +} + +/// Tokens that may only ever appear where this parser reads semantics: on a header, an action line +/// or a match-key line. Finding one inside a statistics line means the line is not the statistics +/// line it looks like. +const SEMANTIC_TOKENS: &[&str] = + &["filter", "flower", "action", "gact", "chain", "handle", "pref", "protocol"]; + +/// A statistics line carries counts, and counts carry nothing this module compares. It is consumed +/// rather than parsed field by field — the numbers differ on every run — but it is first checked to +/// contain no token that could carry match or action semantics, so "consumed" cannot become a place +/// to hide a predicate. +fn check_counter_line( + filter: &ReadbackFilter, + fields: &[&str], + at: usize, +) -> Result<(), String> { + // Read to the end against the shape `tc` actually prints, rather than scanned for known-bad + // tokens. The blacklist this replaces asked only whether a statistics line contained a + // recognised predicate, so anything it had never heard of rode through untouched: appending + // `src_ip 192.0.2.123` to a faithful `Sent` line left the parsed rule and the verification + // unchanged, and `src_ip` is a narrowing predicate. A list of what is forbidden cannot refuse + // what nobody thought to forbid; a grammar refuses everything it does not positively read. + let unread = |what: String| { + Err(format!( + "line {at}: filter {} carries {what} in a statistics line ({fields:?}) — statistics are \ + skipped, so anything unread inside one is skipped with it, predicate or not", + filter.describe() + )) + }; + // Counters print as bare integers except where iproute2 glues punctuation on: `(dropped 4,` + // and the closing `0)`. The units on a backlog (`0b`, `0p`) are handled at their position. + let counter = |token: &str| { + let trimmed = token.trim_start_matches('(').trim_end_matches([',', ')']); + !trimmed.is_empty() && trimmed.bytes().all(|byte| byte.is_ascii_digit()) + }; + + match fields[0] { + // `Sent 168 bytes 4 pkt (dropped 4, overlimits 0 requeues 0)` + "Sent" => { + let expected: [(usize, &str); 6] = [ + (2, "bytes"), + (4, "pkt"), + (5, "(dropped"), + (7, "overlimits"), + (9, "requeues"), + (10, ""), + ]; + if fields.len() != 11 { + return unread(format!("{} tokens where a Sent line has 11", fields.len())); + } + for (index, word) in expected { + if !word.is_empty() && fields[index] != word { + return unread(format!("{:?} where a Sent line has {word:?}", fields[index])); + } + } + for index in [1, 3, 6, 8, 10] { + if !counter(fields[index]) { + return unread(format!("{:?} where a Sent line has a count", fields[index])); + } + } + } + // `backlog 0b 0p requeues 0` + "backlog" => { + if fields.len() != 5 { + return unread(format!("{} tokens where a backlog line has 5", fields.len())); + } + if !fields[1].ends_with('b') || !counter(fields[1].trim_end_matches('b')) { + return unread(format!("{:?} where a backlog line has a byte count", fields[1])); + } + if !fields[2].ends_with('p') || !counter(fields[2].trim_end_matches('p')) { + return unread(format!("{:?} where a backlog line has a packet count", fields[2])); + } + if fields[3] != "requeues" || !counter(fields[4]) { + return unread(format!("{:?} where a backlog line has requeues", &fields[3..])); + } + } + // `Action statistics:` introduces the counters below it and carries nothing else. + _ => { + if fields.len() != 2 { + return unread(format!("{:?} after an Action statistics header", &fields[2..])); + } + } + } + Ok(()) +} + +/// The lines `tc` prints under an action, token by token. +/// +/// `random type none` is the only randomness accepted: any other spelling is a rule that drops a +/// *fraction* of what it claims to drop. The `index` line is bookkeeping — install order, reference +/// counts, age — but it is still read to the end, because "the rest of this line is bookkeeping" is +/// exactly the assumption an added token would hide behind. +fn parse_action_detail( + filter: &ReadbackFilter, + fields: &[&str], + at: usize, +) -> Result<(), String> { + let unexpected = |what: &str| { + Err(format!( + "line {at}: filter {} carries an action detail this parser does not read ({what} in \ + {fields:?}) — an unread token is an unchecked token", + filter.describe() + )) + }; + + if fields[0] == "random" { + if fields.get(1..3) != Some(&["type", "none"][..]) { + return Err(format!( + "line {at}: filter {} carries a randomised action ({fields:?}) — a probabilistic \ + drop passes traffic it claims to stop", + filter.describe() + )); + } + // `random type none pass val 0` is what iproute2 prints for a non-random gact; nothing + // else is accepted after `none`. + return match &fields[3..] { + [] => Ok(()), + [verb, "val", value] if GACT_VERBS.contains(verb) && *value == "0" => Ok(()), + rest => unexpected(&format!("{rest:?}")), + }; + } + + // `index 1 ref 1 bind 1 installed 2 sec used 2 sec [firstused 2 sec]` + let mut index = 1; + if !fields.get(index).is_some_and(|value| value.parse::().is_ok()) { + return unexpected("a non-numeric action index"); + } + index += 1; + while index < fields.len() { + let token = fields[index]; + let value = fields.get(index + 1); + let numeric = value.is_some_and(|value| value.parse::().is_ok()); + match token { + "ref" | "bind" if numeric => index += 2, + "installed" | "used" | "firstused" | "expires" if numeric => { + index += 2; + if fields.get(index) == Some(&"sec") { + index += 1; + } + } + _ => return unexpected(&format!("{token:?}")), + } + } + Ok(()) +} + +/// Match keys and hardware flags. Unknown predicates and duplicates are refusals. +fn parse_key_line( + filter: &mut ReadbackFilter, + fields: &[&str], + at: usize, +) -> Result<(), String> { + let mut index = 0; + while index < fields.len() { + let token = fields[index]; + if KNOWN_FLAGS.contains(&token) { + index += 1; + continue; + } + if token == "in_hw_count" { + let count = fields.get(index + 1).ok_or(format!( + "line {at}: in_hw_count has no value — truncated output is not a verified namespace" + ))?; + if count.parse::().is_err() { + return Err(format!( + "line {at}: in_hw_count is {count:?}, not a count — this parser skips the \ + value, so anything may be hiding in it" + )); + } + index += 2; + continue; + } + if !KNOWN_KEYS.contains(&token) { + return Err(format!( + "line {at}: unknown match key or token {token:?} on filter {} — an unrecognised \ + predicate narrows what the rule matches while every compared field stays equal, \ + so it is refused rather than ignored", + filter.describe() + )); + } + let value = *fields.get(index + 1).ok_or(format!( + "line {at}: match key {token:?} has no value — truncated output is not a verified \ + namespace" + ))?; + if filter.keys.iter().any(|(key, _)| key == token) { + return Err(format!( + "line {at}: match key {token:?} appears twice on one filter — a duplicate silently \ + overwrote the first value in the old parser" + )); + } + filter.keys.push((token.to_owned(), value.to_owned())); + index += 2; + } + Ok(()) +} + +/// `tc` prints a single address without its prefix length; the policy spells one with it. +fn normalise_prefix(address: &str) -> &str { + address + .strip_suffix("/32") + .or_else(|| address.strip_suffix("/128")) + .unwrap_or(address) +} + +impl IfacePlan { + /// Verify a live namespace against this plan, from that namespace's own `tc filter show` output. + /// + /// `Ok(())` means the filters are in force; an `Err` names what is wrong and is a reason to + /// refuse the job. What is checked, and why each one is here rather than assumed: + /// + /// * **every rendered filter is present, in order, with its own match keys** — a filter list + /// that merely has the right length can be the right length and the wrong policy. + /// * **no extra filters** — an unrendered `pass` at a low priority is an egress hole, and it is + /// the one edit that leaves every other property intact. + /// * **every exception is at a lower `pref` than every drop** — checked against the kernel's + /// answer, not against the render, because the render is what is under suspicion. + /// * **no drop carries an `ip_proto` match** — the leak this closes is protocol-independent, and + /// a TCP-only drop passes the very fixture that found it. + /// * **both families are filtered** — an unfiltered address family is the cheapest bypass there + /// is. + /// * **the classifier, the chain and the exact set of match keys** — checked by the parser and + /// again here. A rule that is `flower` on the active chain with exactly the rendered keys is + /// the rule that was asked for; a rule that merely agrees on the fields an older parser chose + /// to read can be narrower (`src_ip`), parked off the egress path (`chain 7`), or carry a + /// second action, and every one of those reads as a pass. + pub fn verify_readback(&self, stdout: &str) -> Result<(), String> { + let live = parse_filters(stdout)?; + if live.len() != self.filters.len() { + return Err(format!( + "the namespace holds {} egress filters, expected {} — {:?}", + live.len(), + self.filters.len(), + live + )); + } + + for (at, (want, got)) in self.filters.iter().zip(live.iter()).enumerate() { + let protocol = tc_protocol(want.family); + if got.protocol != protocol || got.pref != want.pref { + return Err(format!( + "filter {at} is protocol {} pref {}, expected {protocol} pref {} ({})", + got.protocol, got.pref, want.pref, want.why + )); + } + if got.chain != ACTIVE_CHAIN { + return Err(format!( + "filter {at} (pref {}, {}) sits in chain {}, not the active chain \ + {ACTIVE_CHAIN} — a filter in an unreferenced chain is listed, is never \ + consulted on egress, and looks exactly like containment", + want.pref, + want.dst_label(), + got.chain + )); + } + if got.actions.len() != 1 { + return Err(format!( + "filter {at} (pref {}, {}) carries {} actions {:?}, expected exactly one — a \ + second action runs after the first and can undo it", + want.pref, + want.dst_label(), + got.actions.len(), + got.actions + )); + } + if got.actions[0] != want.action { + return Err(format!( + "filter {at} (pref {}, {}) has action {:?}, expected {}", + want.pref, + want.dst_label(), + got.actions[0], + want.action + )); + } + + // The whole key set, compared as a set. Anything the render did not ask for is a + // different rule, whatever the fields an older parser happened to read. + let mut expected: Vec<(&str, String)> = + vec![("eth_type", tc_eth_type(want.family).to_owned())]; + if let Some(dst) = want.dst.as_deref() { + expected.push(("dst_ip", normalise_prefix(dst).to_owned())); + } + if let Some(proto) = want.ip_proto.as_deref() { + expected.push(("ip_proto", proto.to_owned())); + } + if let Some(port) = want.dst_port.as_deref() { + expected.push(("dst_port", port.to_owned())); + } + if let Some(icmp_type) = want.icmp_type.as_deref() { + expected.push(("icmp_type", icmp_type.to_owned())); + } + if let Some(ttl) = want.ip_ttl.as_deref() { + expected.push(("ip_ttl", ttl.to_owned())); + } + let mut seen: Vec<(&str, String)> = got + .keys + .iter() + .map(|(key, value)| { + let value = if key == "dst_ip" { + normalise_prefix(value).to_owned() + } else { + value.clone() + }; + (key.as_str(), value) + }) + .collect(); + expected.sort(); + seen.sort(); + if seen != expected { + return Err(format!( + "filter {at} (pref {}, {}) matches on {:?}, expected exactly {:?} — {}", + want.pref, + want.dst_label(), + seen, + expected, + want.why + )); + } + } + + // Order, read off the kernel's own list rather than off the render above — the render is + // what is under suspicion, so re-deriving the answer from it would prove nothing. + let live_filters: Vec = live + .iter() + .map(|filter| IfaceFilter { + family: if filter.protocol == tc_protocol(Family::V6) { + Family::V6 + } else { + Family::V4 + }, + pref: filter.pref, + dst: filter.key("dst_ip").map(str::to_owned), + ip_proto: filter.key("ip_proto").map(str::to_owned), + dst_port: filter.key("dst_port").map(str::to_owned), + icmp_type: filter.key("icmp_type").map(str::to_owned), + ip_ttl: filter.key("ip_ttl").map(str::to_owned), + action: match filter.actions.first().map(String::as_str) { + Some("pass") => "pass", + _ => "drop", + }, + why: "read back from the live namespace", + }) + .collect(); + no_shadowed_exception(&live_filters)?; + if live.iter().any(|filter| { + filter.actions.first().map(String::as_str) == Some("drop") + && filter.key("ip_proto").is_some() + }) { + return Err( + "a live drop filter carries an ip_proto match — the containment this closes is \ + protocol-independent and a TCP-only drop is the bug, not the fix" + .to_owned(), + ); + } + for family in [Family::V4, Family::V6] { + let protocol = tc_protocol(family); + if !live.iter().any(|filter| { + filter.protocol == protocol + && filter.actions.first().map(String::as_str) == Some("drop") + }) { + return Err(format!( + "the namespace holds no {protocol} drop filter — an unfiltered address family is \ + the cheapest bypass there is" + )); + } + } + Ok(()) + } +} diff --git a/crates/maxplayer-core/src/sandbox_net.rs b/crates/maxplayer-core/src/sandbox_net.rs index 42f023b27..221bb1b70 100644 --- a/crates/maxplayer-core/src/sandbox_net.rs +++ b/crates/maxplayer-core/src/sandbox_net.rs @@ -112,6 +112,31 @@ pub const DENIED_DESTINATIONS: &[&str] = &[ /// carries no destination a job legitimately needs. pub const DENIED_DESTINATIONS_V6: &[&str] = &["fc00::/7", "fe80::/10", "ff00::/8"]; +/// The solicited-node multicast range, and the *only* multicast destination this policy lets out. +/// +/// Every IPv6 address has exactly one solicited-node address derived from its low 24 bits, and +/// Neighbour Solicitation is sent there rather than to the peer (RFC 4861 §7.2.2) — the peer's +/// link-layer address is precisely what is not yet known. `ff02::1` (all-nodes) and `ff02::2` +/// (all-routers) are outside this /104 and stay dropped by [`DENIED_DESTINATIONS_V6`]. +pub const ND_SOLICITED_NODE_MULTICAST: &str = "ff02::1:ff00:0/104"; + +/// ICMPv6 Neighbour Solicitation. +pub const ICMPV6_NEIGHBOUR_SOLICITATION: &str = "135"; + +/// ICMPv6 Neighbour Advertisement — the reply half. Sent to whoever solicited, which on this link is +/// a link-local address, so without an exception `fe80::/10` drops every answer this namespace owes. +pub const ICMPV6_NEIGHBOUR_ADVERTISEMENT: &str = "136"; + +/// The hop limit RFC 4861 §11 requires on a received ND message, and the reason these two exceptions +/// cannot be relayed in from off-link: a router decrements, so 255 can only have been set by a node +/// on this very link. It is the standard ND admission check, applied here to what leaves. +pub const ND_HOP_LIMIT: &str = "255"; + +/// How the two neighbour-discovery exceptions are named in a readback failure. Roles rather than +/// rule text, so a reader is told which reach is missing instead of being handed an argv to diff. +pub const ND_SOLICITATION_ROLE: &str = "the neighbour-solicitation exception"; +pub const ND_ADVERTISEMENT_ROLE: &str = "the neighbour-advertisement exception"; + /// Log lines are rate-limited so a job cannot fill the seller's disk by hammering a denied address. const LOG_RATE: &str = "6/min"; const LOG_BURST: &str = "12"; @@ -351,6 +376,13 @@ pub struct ReadbackRule { pub chain: String, /// Every predicate, in printed order. pub predicates: Vec, + /// Why this line could not be accounted for token by token, when it could not. + /// + /// A malformed line is kept rather than dropped, on purpose. Dropping it would shrink the rule + /// count and hide an unexpected rule from the very check that exists to notice one; keeping it + /// leaves it countable and visible, while [`ReadbackRule::value`] refuses to interpret it so it + /// can never satisfy a rule the policy requires. + pub malformed: Option, } /// An exception rule reduced to the three things that decide what it lets through, and only after @@ -404,8 +436,18 @@ impl ReadbackRule { let chain = fields.next()?.to_owned(); let mut predicates: Vec = Vec::new(); let mut negated = false; + // Every token must land somewhere. The two cases below used to be discarded in + // silence, which let a line that is NOT the one this policy renders read back as one + // that is: the retained predicates were identical, so the rule compared equal to the + // canonical form and was accepted. + let mut malformed: Option = None; for field in fields { if field == "!" { + if negated { + malformed.get_or_insert_with(|| { + format!("`{line}` repeats `!` with no flag between them") + }); + } // iptables prints the inversion as its own token, before the flag. negated = true; continue; @@ -420,9 +462,19 @@ impl ReadbackRule { } else if let Some(current) = predicates.last_mut() { // A flag can take more than one value: `--tcp-flags FIN,SYN,RST,ACK SYN`. current.values.push(field.to_owned()); + } else { + // A bare value with no flag to attach to, e.g. `-A OUTPUT garbage -p ...`. + malformed.get_or_insert_with(|| { + format!("`{line}` carries `{field}` before any flag") + }); } } - Some(Self { chain, predicates }) + if negated { + // A trailing `!` inverts the flag that never came. The pending inversion used to + // be dropped when the loop ended, so the line parsed as its un-inverted twin. + malformed.get_or_insert_with(|| format!("`{line}` ends with a dangling `!`")); + } + Some(Self { chain, predicates, malformed }) }) .collect() } @@ -434,6 +486,11 @@ impl ReadbackRule { /// of those is a different rule from the one this policy renders, and answering with the value /// anyway is how an inverted match passes for a positive one. pub fn value(&self, key: &str) -> Option<&str> { + // A line with an unaccounted token is not interpreted at all. Answering for its retained + // predicates would be answering about a rule this parser demonstrably did not read whole. + if self.malformed.is_some() { + return None; + } let mut matching = self.predicates.iter().filter(|predicate| predicate.key == key); let first = matching.next()?; if matching.next().is_some() || first.negated || first.values.len() != 1 { @@ -546,6 +603,120 @@ impl ReadbackRule { dport: normalize_dport(&dport), }) } + + /// The neighbour-discovery exception this rule is, named by role, if its shape is one this + /// policy renders. + /// + /// ND is judged here rather than through [`ReadbackRule::as_exception`] because it is a + /// different shape, not a variant of the same one: no transport, no port, and a destination on + /// the solicitation only. Projecting it through the udp/tcp reader would reject every correct + /// ND rule and, worse, invite loosening that reader until it accepted them. + /// + /// Narrowed on three axes at once, because each alone is a hole: a type-135 ACCEPT without the + /// destination reaches every multicast group, one without the hop-limit match carries traffic + /// relayed from off-link, and one that named no type at all would permit every ICMPv6 message + /// including echo — the one a payload can actually author. Extra and inverted predicates are + /// refused for exactly the reasons `as_exception` refuses them. + pub fn as_nd_exception(&self) -> Result<&'static str, String> { + if self.chain != OUTPUT_CHAIN { + return Err(format!( + "an ICMPv6 ACCEPT in chain {} rather than {OUTPUT_CHAIN} — it does not filter this \ + job's egress", + self.chain + )); + } + if let Some(negated) = self.predicates.iter().find(|predicate| predicate.negated) { + return Err(format!( + "an ICMPv6 ACCEPT whose {} match is INVERTED — it permits the complement of the \ + neighbour-discovery rule this policy renders", + negated.key + )); + } + if self.target() != Some("ACCEPT") { + return Err("a rule read as a neighbour-discovery exception does not jump to ACCEPT" + .to_owned()); + } + + let icmpv6_type = self.value("--icmpv6-type").ok_or_else(|| { + "an ICMPv6 ACCEPT that names no single message type — untyped, it permits every ICMPv6 \ + message, including the echo a payload can author" + .to_owned() + })?; + match self.value("--hl-eq") { + Some(ND_HOP_LIMIT) => {} + other => { + return Err(format!( + "an ICMPv6 ACCEPT at hop limit {other:?} rather than {ND_HOP_LIMIT} — without \ + that match the exception is not confined to this link, since only an on-link \ + node can present a hop limit a router has not decremented" + )); + } + } + + let role = match icmpv6_type { + ICMPV6_NEIGHBOUR_SOLICITATION => match self.value("-d") { + Some(ND_SOLICITED_NODE_MULTICAST) => ND_SOLICITATION_ROLE, + other => { + return Err(format!( + "a neighbour solicitation aimed at {other:?} rather than \ + {ND_SOLICITED_NODE_MULTICAST} — a widened solicitation is still exactly \ + one ACCEPT, and it reaches every multicast group" + )); + } + }, + ICMPV6_NEIGHBOUR_ADVERTISEMENT => { + if let Some(destination) = self.value("-d") { + return Err(format!( + "a neighbour advertisement narrowed to {destination:?} — the answer is owed \ + to whichever on-link node solicited, so a fixed destination makes it inert" + )); + } + ND_ADVERTISEMENT_ROLE + } + other => { + return Err(format!( + "an ICMPv6 ACCEPT for message type {other:?} — this policy permits only \ + neighbour solicitation ({ICMPV6_NEIGHBOUR_SOLICITATION}) and advertisement \ + ({ICMPV6_NEIGHBOUR_ADVERTISEMENT}), and never echo" + )); + } + }; + + for predicate in &self.predicates { + let permitted = match predicate.key.as_str() { + "-p" | "--icmpv6-type" | "--hl-eq" | "-j" => true, + "-d" => role == ND_SOLICITATION_ROLE, + // The two match modules iptables inserts for its own predicates, and nothing else. + "-m" => predicate.values == ["icmp6"] || predicate.values == ["hl"], + _ => false, + }; + if !permitted { + return Err(format!( + "an ICMPv6 ACCEPT carrying `{} {}`, a predicate this policy never renders on a \ + neighbour-discovery exception", + predicate.key, + predicate.values.join(" ") + )); + } + // `-m` legitimately appears twice: `-m icmp6` for the type match and `-m hl` for the + // hop-limit one. Everything else is printed once per rule. + let allowed_repeats = usize::from(predicate.key == "-m"); + let seen = self + .predicates + .iter() + .filter(|other| other.key == predicate.key) + .count(); + if seen > 1 + allowed_repeats { + return Err(format!( + "an ICMPv6 ACCEPT carrying {seen} `{}` predicates — this is not the plan this \ + policy sent", + predicate.key + )); + } + } + + Ok(role) + } } /// The containment policy for one job's namespace. @@ -690,8 +861,41 @@ impl NetPolicy { )); } - // IPv6. No pinhole and no logging split — the proxy is v4, and a job has no legitimate v6 - // destination inside these ranges. + // IPv6 neighbour discovery, and *only* neighbour discovery. These two ACCEPTs must precede + // the drops below, for the same reason the v4 pinhole does: `ff00::/8` otherwise shadows + // the solicitation and `fe80::/10` the advertisement. + // + // Measured in a live namespace, not reasoned about: with the drops alone, a job cannot + // reach *any* v6 address — allowed or denied — because Neighbour Solicitation goes to a + // solicited-node MULTICAST address, `ff00::/8` drops it, and the neighbour entry ends in + // state FAILED. That is denial by a dead v6 stack rather than by the destination policy, + // and it makes the v6 half of this policy untestable: every address fails identically + // whether or not it is denied. + // + // These exceptions are carried by the kernel's own ND, never by a job's payload: emitting + // ICMPv6 type 135/136 needs a raw socket, a job's namespace runs `--cap-drop ALL`, and a + // ping socket can only send echo (type 128), which stays dropped. The narrowing is + // destination, type and hop limit together — see [`ND_SOLICITED_NODE_MULTICAST`] and + // [`ND_HOP_LIMIT`]. + rules.push(Rule::new( + Family::V6, + vec![ + "-p", "icmpv6", "--icmpv6-type", ICMPV6_NEIGHBOUR_SOLICITATION, "-d", + ND_SOLICITED_NODE_MULTICAST, "-m", "hl", "--hl-eq", ND_HOP_LIMIT, "-j", "ACCEPT", + ], + "neighbour solicitation, or the namespace cannot resolve any v6 peer at all", + )); + rules.push(Rule::new( + Family::V6, + vec![ + "-p", "icmpv6", "--icmpv6-type", ICMPV6_NEIGHBOUR_ADVERTISEMENT, "-m", "hl", + "--hl-eq", ND_HOP_LIMIT, "-j", "ACCEPT", + ], + "the answering half of neighbour discovery, owed to a link-local solicitor", + )); + + // IPv6 destination denial. No pinhole and no logging split — the proxy is v4, and a job has + // no legitimate v6 destination inside these ranges. for denied in DENIED_DESTINATIONS_V6 { rules.push(Rule::new( Family::V6, @@ -759,6 +963,16 @@ impl NetPolicy { /// to be looked at. pub fn verify_readback(&self, family: Family, stdout: &str) -> Result<(), String> { let found = ReadbackRule::parse_all(stdout); + // Before anything is counted or matched: a line this parser could not account for token by + // token is not evidence about the namespace, in either direction. Refusing here keeps the + // failure legible instead of surfacing later as a missing DROP. + if let Some(bad) = found.iter().find_map(|rule| rule.malformed.as_deref()) { + return Err(format!( + "{} printed a rule this parser cannot account for token by token: {bad} — an \ + unreadable readback is not an acceptable one", + family.binary() + )); + } let expected = self.rule_count(family); if found.len() != expected { return Err(format!( @@ -854,6 +1068,7 @@ impl NetPolicy { let mut unrendered: Vec = Vec::new(); let mut matched: Vec<(String, usize)> = Vec::new(); + let mut nd_seen: Vec<&'static str> = Vec::new(); for (at, rule) in found.iter().enumerate() { // Judged by the jump, so a rule whose `-j` is repeated or inverted still arrives here // rather than being skipped as "not an ACCEPT". @@ -864,6 +1079,28 @@ impl NetPolicy { if !jumps_to_accept { continue; } + // Neighbour discovery is judged on its own terms, and only in v6. An ICMPv6 ACCEPT in + // the v4 chain is not a thing this policy renders at all. + if matches!(rule.value("-p"), Some("icmpv6" | "ipv6-icmp")) { + if family == Family::V4 { + unrendered.push(format!("an ICMPv6 ACCEPT in the v4 chain (at index {at})")); + continue; + } + match rule.as_nd_exception() { + Ok(role) if nd_seen.contains(&role) => { + unrendered.push(format!("a second `{role}` (at index {at})")); + } + Ok(role) => { + nd_seen.push(role); + // Into `matched`, so the position checks below apply to ND too: appended + // under the range DROPs it is inert, and the namespace is back to a v6 + // stack that resolves no neighbour at all. + matched.push((role.to_owned(), at)); + } + Err(why) => unrendered.push(format!("{why} (at index {at})")), + } + continue; + } match rule.as_exception(family) { Ok(exception) => { match expected @@ -881,11 +1118,22 @@ impl NetPolicy { } } - let missing: Vec = expected + let mut missing: Vec = expected .iter() .filter(|(_, _, taken)| !*taken) .map(|(role, exception, _)| format!("{role} (`{exception}`)")) .collect(); + if family == Family::V6 { + // Both halves are required, and their absence is reported as missing reach rather than + // as a count: without them the namespace resolves no v6 neighbour at all, every v6 + // destination fails alike, and the denials this policy exists to prove become + // unattributable. + for role in [ND_SOLICITATION_ROLE, ND_ADVERTISEMENT_ROLE] { + if !nd_seen.contains(&role) { + missing.push(role.to_owned()); + } + } + } if !unrendered.is_empty() || !missing.is_empty() { // Both halves in one error deliberately: "an ACCEPT nobody rendered" and "an exception // that is gone" are usually the same edit seen from two sides, and reporting only one @@ -1041,20 +1289,35 @@ mod tests { "no configured range means the gateway is never singled out for access: {:?}", rule.args ); - assert_ne!( - rule.args.last().map(String::as_str), - Some("ACCEPT"), - "an unconfigured range must close the namespace, not accept anything: {:?}", - rule.args - ); + // Scoped to v4, because v6 carries two ACCEPTs that are not a pinhole and do not + // depend on one: neighbour discovery is installed whether or not a proxy exists, and + // without it the namespace cannot reach any v6 address to be contained from. Their + // narrowness is proved in `a_widened_neighbour_discovery_exception_is_refused`. + if rule.family == Family::V4 { + assert_ne!( + rule.args.last().map(String::as_str), + Some("ACCEPT"), + "an unconfigured range must close the namespace, not accept anything: {:?}", + rule.args + ); + } + } + // The v6 exceptions are exactly the two ND rules in both configurations — an unconfigured + // policy must not acquire a third. + for policy in [&configured, &unconfigured] { + let v6_accepts = policy + .rules() + .into_iter() + .filter(|rule| rule.family == Family::V6 && rule.target() == Some("ACCEPT")) + .count(); + assert_eq!(v6_accepts, 2, "only neighbour discovery is permitted over v6"); } // Positive control: the same assertions MUST fail on a configured policy, or they are // asserting nothing and would pass against a renderer that never emits a pinhole at all. assert!( - configured - .rules() - .iter() - .any(|rule| rule.args.last().map(String::as_str) == Some("ACCEPT")), + configured.rules().iter().any(|rule| { + rule.family == Family::V4 && rule.args.last().map(String::as_str) == Some("ACCEPT") + }), "the configured case must open the pinhole this test proves the unconfigured case does \ not" ); @@ -1175,12 +1438,19 @@ mod tests { plan.iter().any(|(bin, _)| *bin == "ip6tables"), "no v6 rules in the plan — the family would be left unfiltered" ); - for (binary, argv) in &plan { - let v6_arg = argv.iter().any(|arg| arg.contains("::")); - if v6_arg { - assert_eq!(*binary, "ip6tables", "v6 rule handed to iptables: {argv:?}"); - } else { - assert_eq!(*binary, "iptables", "v4 rule handed to ip6tables: {argv:?}"); + // The family is the rule's own, not a guess from its text. A v6 rule need not mention a v6 + // address at all — the neighbour-advertisement exception matches on ICMPv6 type and hop + // limit only — so a `contains("::")` heuristic would hand it to `iptables` and call that + // correct. + for (rule, (binary, argv)) in policy().rules().iter().zip(&plan) { + assert_eq!( + *binary, + rule.family.binary(), + "{:?} rule handed to {binary}: {argv:?}", + rule.family + ); + if argv.iter().any(|arg| arg.contains("::")) { + assert_eq!(*binary, "ip6tables", "v6 address handed to iptables: {argv:?}"); } } } @@ -1240,9 +1510,17 @@ mod tests { -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix \"sbx-net-deny:\" -A OUTPUT -d 240.0.0.0/4 -j DROP"; - /// The v6 readback, measured in the same run. Textually identical to what was sent — these rules - /// carry no match module and no bare address, so there is nothing for iptables to rewrite. + /// The v6 readback, measured in a live namespace on 2026-09-14 (`lima:gvisor-repro`), pasted as + /// printed. + /// + /// The three DROPs come back textually identical — no match module, no bare address, nothing for + /// iptables to rewrite. The two ND exceptions do not: iptables **moves `-d` ahead of `-p`** and + /// makes the match module explicit (`-p icmpv6` ⇒ `-p ipv6-icmp -m icmp6`). Sent as + /// `-p icmpv6 --icmpv6-type 135 -d ff02::1:ff00:0/104 …`, printed as the first line below. One + /// more reason [`NetPolicy::verify_readback`] checks properties rather than strings. const MEASURED_V6: &str = "\ +-A OUTPUT -d ff02::1:ff00:0/104 -p ipv6-icmp -m icmp6 --icmpv6-type 135 -m hl --hl-eq 255 -j ACCEPT +-A OUTPUT -p ipv6-icmp -m icmp6 --icmpv6-type 136 -m hl --hl-eq 255 -j ACCEPT -A OUTPUT -d fc00::/7 -j DROP -A OUTPUT -d fe80::/10 -j DROP -A OUTPUT -d ff00::/8 -j DROP"; @@ -1365,11 +1643,107 @@ mod tests { assert!(above.contains("above the metadata DROP"), "{above}"); // A v6 range missing. - let v6_short = "-A OUTPUT -d fc00::/7 -j DROP\n-A OUTPUT -d fe80::/10 -j DROP\n-A OUTPUT -d fc00::/7 -j DROP"; + // Five rules, so the count check passes and the missing `ff00::/8` is what fires: the ND + // pair, two of the three drops, and one repeated. + let v6_short = &MEASURED_V6.replace("-A OUTPUT -d ff00::/8 -j DROP", "-A OUTPUT -d fc00::/7 -j DROP"); let v6_missing = policy.verify_readback(Family::V6, v6_short).expect_err("v6"); assert!(v6_missing.contains("ff00::/8"), "{v6_missing}"); } + /// The neighbour-discovery exceptions are the only v6 traffic this policy permits, and each of + /// the three narrowings is load-bearing on its own. Every mutation below keeps the rule count + /// and every denied DROP intact — so each one is refused by the ND check itself rather than by + /// the count or the denial check firing first, which is what makes this a test of the + /// narrowing. + #[test] + fn a_widened_neighbour_discovery_exception_is_refused() { + let policy = measured_policy(); + assert_eq!(policy.verify_readback(Family::V6, MEASURED_V6), Ok(())); + + // The solicitation reaching every multicast group instead of solicited-node only. Still + // exactly one ACCEPT, still type 135, still hop limit 255. + let all_groups = MEASURED_V6.replace("-d ff02::1:ff00:0/104 ", ""); + let refused = policy + .verify_readback(Family::V6, &all_groups) + .expect_err("a solicitation with no destination narrowing"); + assert!(refused.contains(ND_SOLICITED_NODE_MULTICAST), "{refused}"); + + // `ff02::1` (all-nodes) is the reachable target that narrowing exists to exclude, and it + // is not inside the solicited-node range. + let all_nodes = MEASURED_V6.replace("ff02::1:ff00:0/104", "ff02::1/128"); + let refused = policy + .verify_readback(Family::V6, &all_nodes) + .expect_err("a solicitation aimed at all-nodes"); + assert!(refused.contains(ND_SOLICITED_NODE_MULTICAST), "{refused}"); + + // Without the hop-limit match the exception is no longer confined to this link: a router + // decrements, so only an on-link node can present 255. + for stripped in [ + MEASURED_V6.replace( + "--icmpv6-type 135 -m hl --hl-eq 255", + "--icmpv6-type 135", + ), + MEASURED_V6.replace( + "--icmpv6-type 136 -m hl --hl-eq 255", + "--icmpv6-type 136", + ), + ] { + let refused = policy + .verify_readback(Family::V6, &stripped) + .expect_err("an ND exception with no hop-limit match"); + assert!(refused.contains(ND_HOP_LIMIT), "{refused}"); + } + + // An echo request (type 128) is what an unprivileged payload can actually emit, so an + // exception carrying it is the hole a job could use. Type is the only thing changed. + let echo = MEASURED_V6.replace("--icmpv6-type 136", "--icmpv6-type 128"); + let refused = policy + .verify_readback(Family::V6, &echo) + .expect_err("an echo-request exception"); + assert!(refused.contains("advertisement"), "{refused}"); + } + + /// Order is load-bearing in v6 exactly as it is in v4, and getting it wrong is silent: the + /// drops shadow the exceptions, neighbour discovery dies, and every v6 destination becomes + /// unreachable — which reads as containment while proving nothing about it. + #[test] + fn a_neighbour_discovery_accept_below_the_drops_is_refused() { + let policy = measured_policy(); + let mut lines: Vec<&str> = MEASURED_V6.lines().collect(); + let advertisement = lines.remove(1); + lines.push(advertisement); + let shadowed = lines.join("\n"); + + let refused = policy + .verify_readback(Family::V6, &shadowed) + .expect_err("an ND ACCEPT appended below the range drops"); + // The ND exceptions are checked for position by the SAME ordering check that guards the + // pinhole, so this refusal is worded like its siblings above: it names the rule, its index, + // and the drop that precedes it. Asserted on that wording rather than on a single adjective + // so the control cannot be satisfied by an unrelated refusal, and it must still name the + // advertisement specifically — a message about some other rule would not prove this one is + // guarded. + assert!(refused.contains("below the first range DROP"), "{refused}"); + assert!(refused.contains("neighbour-advertisement"), "{refused}"); + assert!(refused.contains("inert"), "{refused}"); + + // And the rules() order this guards is the order actually installed. + let all = policy.rules(); + let v6: Vec<&Rule> = all.iter().filter(|rule| rule.family == Family::V6).collect(); + let first_drop = v6 + .iter() + .position(|rule| rule.target() == Some("DROP")) + .expect("a v6 drop"); + let last_accept = v6 + .iter() + .rposition(|rule| rule.target() == Some("ACCEPT")) + .expect("a v6 accept"); + assert!( + last_accept < first_drop, + "the ND exceptions must be installed above the range drops" + ); + } + /// A seat with no contained credential renders no pinhole, so any ACCEPT in its namespace is one /// nobody asked for. #[test] @@ -1596,6 +1970,142 @@ mod tests { ); } + /// A canonical ND ACCEPT with one unflagged token wedged in after the chain name. + /// + /// The retained predicates are IDENTICAL to the canonical rule's, which is exactly why this + /// slipped through: the stray token was dropped in silence, so every comparison this module + /// makes on predicates alone answered the same for both lines. + #[test] + fn an_unflagged_token_before_any_flag_is_not_read_back_as_the_canonical_rule() { + const CANONICAL: &str = + "-A OUTPUT -p ipv6-icmp -m icmp6 --icmpv6-type 136 -m hl --hl-eq 255 -j ACCEPT"; + let smuggled_text = CANONICAL.replace("-A OUTPUT ", "-A OUTPUT garbage "); + + let canonical_rules = ReadbackRule::parse_all(CANONICAL); + let smuggled_rules = ReadbackRule::parse_all(&smuggled_text); + let canonical = &canonical_rules[0]; + let smuggled = &smuggled_rules[0]; + + assert_eq!( + canonical.predicates, smuggled.predicates, + "the counterexample rests on the retained predicates being identical" + ); + assert!(canonical.malformed.is_none(), "the canonical rule must still parse"); + let reason = + smuggled.malformed.as_deref().expect("an unflagged token must be accounted for"); + assert!(reason.contains("garbage"), "the reason must name the token: {reason}"); + assert_eq!( + smuggled.target(), + None, + "a line this parser did not read whole must not answer for its target" + ); + } + + /// A trailing `!` inverts the flag that never came. The pending inversion used to be discarded + /// when the loop ended, so the line read back as its un-inverted twin. + #[test] + fn a_trailing_inversion_is_not_discarded_when_the_line_ends() { + const CANONICAL: &str = "-A OUTPUT -d 2001:db8::53/128 -p udp -m udp --dport 53 -j ACCEPT"; + let dangling_text = format!("{CANONICAL} !"); + + let canonical_rules = ReadbackRule::parse_all(CANONICAL); + let dangling_rules = ReadbackRule::parse_all(&dangling_text); + let canonical = &canonical_rules[0]; + let dangling = &dangling_rules[0]; + + assert_eq!( + canonical.predicates, dangling.predicates, + "the counterexample rests on the retained predicates being identical" + ); + assert!(canonical.malformed.is_none(), "the canonical rule must still parse"); + let reason = dangling.malformed.as_deref().expect("a dangling `!` must be accounted for"); + assert!(reason.contains("dangling"), "{reason}"); + assert_eq!( + dangling.value("-d"), + None, + "a line this parser did not read whole must not answer for its destination" + ); + } + + /// The whole readback is refused, not just the one line: a namespace that prints something this + /// parser cannot account for is not evidence about that namespace in either direction. + /// + /// The malformed line is deliberately kept in the parsed list rather than dropped — dropping it + /// would shrink the rule count and hide an unexpected rule from the check that exists to notice + /// one — so the refusal has to come from the accounting, not from a count mismatch. + #[test] + fn a_line_that_cannot_be_accounted_for_refuses_the_whole_readback() { + let policy = policy_with_resolvers(&["10.0.0.2"]); + let good = readback_with_resolver(); + assert_eq!(policy.verify_readback(Family::V4, &good), Ok(()), "positive control"); + + let smuggled = good.replace( + "-A OUTPUT -d 10.0.0.2/32 -p udp", + "-A OUTPUT garbage -d 10.0.0.2/32 -p udp", + ); + assert_eq!( + ReadbackRule::parse_all(&smuggled).len(), + ReadbackRule::parse_all(&good).len(), + "the malformed line must stay countable, or it escapes the unexpected-rule check" + ); + let refusal = policy + .verify_readback(Family::V4, &smuggled) + .expect_err("a line with an unaccounted token must refuse the readback"); + assert!(refusal.contains("account for"), "{refusal}"); + } + + /// The ND readback failure the verdict names, reproduced at the decision site on the canonical + /// ND ACCEPT rules themselves. + /// + /// The siblings above establish that the PARSER marks these shapes. That is not the claim that + /// matters. The claim that matters is the one the verdict made about behaviour: a canonical ND + /// ACCEPT carrying a leading unflagged token after `OUTPUT`, or a dangling final `!`, "retains + /// the same accepted predicates" — i.e. verification still passes, because the tokens the parser + /// could not consume changed nothing it went on to check. + /// + /// So this runs [`NetPolicy::verify_readback`] over the REAL measured v6 readback, mutated only + /// in those two ways, and requires refusal. Both mutations leave the predicate set the checker + /// reads untouched — that is precisely why tolerating them was a hole rather than a cosmetic + /// defect. + #[test] + fn a_canonical_nd_accept_carrying_an_unconsumed_token_refuses_the_readback() { + let policy = measured_policy(); + assert_eq!( + policy.verify_readback(Family::V6, MEASURED_V6), + Ok(()), + "positive control: the unmutated measured readback must verify, or this test would \ + pass for the wrong reason" + ); + + // Shape 1: an unflagged token between the chain and the first predicate. + let smuggled = MEASURED_V6.replace( + "-A OUTPUT -p ipv6-icmp -m icmp6 --icmpv6-type 136", + "-A OUTPUT garbage -p ipv6-icmp -m icmp6 --icmpv6-type 136", + ); + assert_ne!(smuggled, MEASURED_V6, "the mutation must have applied"); + assert_eq!( + ReadbackRule::parse_all(&smuggled).len(), + ReadbackRule::parse_all(MEASURED_V6).len(), + "the malformed line must stay countable, or it escapes the unexpected-rule check \ + instead of being refused by the accounting" + ); + let refusal = policy + .verify_readback(Family::V6, &smuggled) + .expect_err("an ND ACCEPT with an unconsumed leading token must refuse the readback"); + assert!(refusal.contains("account for"), "{refusal}"); + + // Shape 2: a dangling inversion with nothing after it to invert. + let dangling = MEASURED_V6.replace( + "--icmpv6-type 136 -m hl --hl-eq 255 -j ACCEPT", + "--icmpv6-type 136 -m hl --hl-eq 255 -j ACCEPT !", + ); + assert_ne!(dangling, MEASURED_V6, "the mutation must have applied"); + let refusal = policy + .verify_readback(Family::V6, &dangling) + .expect_err("an ND ACCEPT ending in a dangling `!` must refuse the readback"); + assert!(refusal.contains("account for"), "{refusal}"); + } + /// The pinhole count is per family, because the two chains are installed by different binaries /// and verified separately. A total would make a v6-only seat look like it owed v4 rules. #[test] diff --git a/crates/maxplayer-core/src/sandbox_netns.rs b/crates/maxplayer-core/src/sandbox_netns.rs index d971d88fa..1a541f123 100644 --- a/crates/maxplayer-core/src/sandbox_netns.rs +++ b/crates/maxplayer-core/src/sandbox_netns.rs @@ -64,20 +64,786 @@ pub const HOLDER_LABEL: &str = "ai.maxplayer.netns-holder"; /// this whole module chooses whenever it has to choose. pub const HOLDER_SEAT_LABEL: &str = "ai.maxplayer.netns-holder-seat"; +/// The docker label carrying the absolute unix second after which this seat may remove the +/// container **without consulting anything in this process**. +/// +/// **Why the expiry is written into the container instead of remembered.** Everything this module +/// used to rely on to finish a cleanup — a retained owner, a supervisor, a watch on the daemon's +/// event stream — lives in the process that created the container, and so dies with it. A stamp on +/// the container itself is the one record that survives a `SIGKILL`, a crash mid-create, and a +/// container the daemon only materialises after this process is gone. The sweep that reads it needs +/// no memory of the job at all: it asks docker what exists, and the answer carries its own verdict. +/// +/// **Why `deadline + grace` and not a fixed cap.** A seat's jobs do not share a lifetime — the +/// deadline is `--job-timeout-secs`, else the offer's own deadline, else the default, chosen per job +/// by [`crate::seller::job_deadline_unix`]. A single global age would either strangle a long job +/// that was legitimately awarded a long deadline, or leave a short one lying around for hours. This +/// label carries the job's OWN effective deadline plus [`CLEANUP_GRACE_SECS`], so each container is +/// judged against the lifetime its own job was actually granted. +/// +/// A container carrying no expiry label, or one that does not parse, is **never** swept on this +/// path: an unreadable stamp is not an expired one, and the boot reaper remains the backstop for +/// anything older than this build. +pub const HOLDER_CLEANUP_AFTER_LABEL: &str = "ai.maxplayer.netns-cleanup-after"; + +/// The docker label naming what the container was for: the namespace holder, or one of the +/// short-lived helpers that join its namespace. +/// +/// Carried so the sweep can report what it removed in terms an operator can act on, and so a future +/// role can be excluded without having to guess from a container name. +pub const HOLDER_ROLE_LABEL: &str = "ai.maxplayer.netns-role"; + +/// The holder that owns the job's network namespace for the whole run. +pub const ROLE_HOLDER: &str = "holder"; + +/// A short-lived helper that joins the holder's namespace (plan applier, readback probe). +pub const ROLE_HELPER: &str = "helper"; + +/// The docker label carrying the job id on a **helper**, as [`HOLDER_LABEL`] carries it on a holder. +/// +/// **A separate label, and the separation is load-bearing.** The boot reaper selects on the presence +/// of [`HOLDER_LABEL`] ([`list_holders_argv`]), and it removes what it selects by container id. Were +/// a helper to carry that same label, the reaper would count a joiner as a namespace holder — and +/// the one protection it has against removing a live one, the `container:` attachment check, is +/// about the holder's id and says nothing about a helper's. Helpers are therefore attributable to a +/// job for reporting, and invisible to the reaper's selection, which is the property that keeps the +/// two cleanup paths from overlapping. +pub const HELPER_JOB_LABEL: &str = "ai.maxplayer.netns-job"; + +/// The `--label` arguments every job-owned helper container carries: owning seat, this job's own +/// cleanup stamp, its role, and the job it belongs to. +/// +/// The seat and the stamp are the two the sweep judges by, and they are the same two the holder +/// carries, produced here from the same values so the holder and its joiners expire **together**. +/// A helper that outlived its holder by carrying no stamp is exactly the leftover this work exists +/// to remove; a helper stamped differently from its holder would be a second, quieter deadline. +/// +/// Every value here is derived by the seller from what it is itself enforcing. None of it is +/// reachable from a job's payload, which is why a request cannot ask for a helper that never +/// expires or one that belongs to another seat. +#[must_use] +pub fn helper_label_args(job_id: &str, seat: &str, cleanup_after: u64) -> Vec { + [ + "--label".to_owned(), + format!("{HOLDER_SEAT_LABEL}={seat}"), + "--label".to_owned(), + format!("{HOLDER_CLEANUP_AFTER_LABEL}={cleanup_after}"), + "--label".to_owned(), + format!("{HOLDER_ROLE_LABEL}={ROLE_HELPER}"), + "--label".to_owned(), + format!("{HELPER_JOB_LABEL}={job_id}"), + ] + .into_iter() + .collect() +} + +/// How long after a job's own effective deadline its containers become sweepable. +/// +/// **One hour, and the size is the point.** The deadline is when the job must be finished, not when +/// its containers stop being legitimately in use: delivery, evidence capture and the teardown that +/// normally removes these containers all happen after it. A grace shorter than that work would have +/// the sweep racing the ordinary cleanup path for a container still in use — the one outcome worse +/// than the leak it exists to fix. An hour is far past any of it, and the cost of the margin is a +/// dead container occupying a name and no policy for at most that long. +pub const CLEANUP_GRACE_SECS: u64 = 3_600; + +/// The value for [`HOLDER_CLEANUP_AFTER_LABEL`]: this job's effective deadline plus the grace. +/// +/// Saturating, so a deadline near `u64::MAX` yields `u64::MAX` — a container that is never swept on +/// this path — rather than wrapping to zero and becoming instantly removable while its job runs. +/// Of the two failures available to arithmetic here, leaking is the one that does not destroy live +/// work. +#[must_use] +pub fn cleanup_after_unix(effective_deadline_unix: u64) -> u64 { + effective_deadline_unix.saturating_add(CLEANUP_GRACE_SECS) +} + +/// The cleanup stamp a launch writes onto its containers. +/// +/// **`job_deadline_unix` is the whole point (#996 D3).** When the caller can name the job's absolute +/// deadline, the stamp is a RESTATEMENT of it and `now_unix` is not consulted at all: the value +/// cannot then be moved by whatever the wall clock happens to say at the create. The previous +/// derivation — `now + remaining` — made the stamp a fresh measurement, so a clock that had stepped +/// backward between the caller computing `remaining` and this create produced a stamp EARLIER than +/// the deadline the job was actually running under, and the sweep would remove a container out from +/// under a job still inside its own deadline. +/// +/// `None` is for a launch with no job deadline to carry — a harness probe. There the remaining +/// window is the best available statement, and an unreadable clock (`now_unix == u64::MAX`) +/// saturates rather than wrapping, so it can never date a live container into the past. +#[must_use] +pub fn launch_cleanup_stamp( + job_deadline_unix: Option, + job_lifetime_secs: u64, + now_unix: u64, +) -> u64 { + cleanup_after_unix(match job_deadline_unix { + Some(deadline_unix) => deadline_unix, + None => now_unix.saturating_add(job_lifetime_secs), + }) +} + +/// How long any one `docker` invocation in this module may take before it is killed. A create or a +/// sidecar that never returns would otherwise hold the launch open indefinitely, and an unbounded +/// wait is the state in which cancellation leaves work nobody owns. +pub const DOCKER_DEADLINE: std::time::Duration = std::time::Duration::from_secs(120); + +/// How long a deadline-killed command waits for its plan writer to notice the closed pipe. +/// +/// Killing the child closes the read end, so a blocked `write_all` fails with `EPIPE` almost at +/// once. This grace exists so that the common case is JOINED rather than abandoned; a writer still +/// running after it is reported as outstanding, never waited on indefinitely. It is a bound on how +/// long this process will wait for that thread -- not a claim about how quickly any particular +/// writer unblocks. +const WRITER_EPIPE_GRACE: std::time::Duration = std::time::Duration::from_millis(50); + +/// The docker client one containment lifecycle spawns, carried **explicitly** by the code that uses +/// it. +/// +/// There is no environment variable, no global, and no configuration field behind this. The only +/// constructor a shipped build can reach is [`DockerCli::system`], which is the constant `docker` on +/// `PATH`; a test that needs a stand-in passes one in as an argument to the single call under test, +/// so two tests running in parallel cannot see or disturb each other's client. +/// +/// The earlier shapes of this seam were both wrong, in instructive ways. An environment variable let +/// anyone able to set a variable on the process redirect every containment command — create, +/// inspect, remove — at a binary of their choosing, in a shipped build. Moving it to a +/// `cfg(test)` process-global removed the shipped exposure but not the interference: a global is +/// still shared, still needs a lock every reader must remember to take, and any helper that forgot +/// — cleanup running from `Drop`, for instance — read whatever another test had installed. An +/// argument has neither failure mode. +#[derive(Clone, Debug)] +pub struct DockerCli { + program: std::sync::Arc, +} + +impl DockerCli { + /// The production client: `docker`, resolved on `PATH`. Nothing selects another. + #[must_use] + pub fn system() -> Self { + Self { program: std::sync::Arc::from("docker") } + } + + /// Test-only: an explicit stand-in, handed to one call. Not reachable from a shipped build. + #[cfg(test)] + fn stand_in(path: &std::path::Path) -> Self { + Self { program: std::sync::Arc::from(path.to_string_lossy().as_ref()) } + } + + fn program(&self) -> &str { + &self.program + } +} + +/// How long cleanup will wait for an in-flight create, and how long an owner keeps trying after it. +/// +/// A parameter rather than a constant so the delayed path can be exercised in under a second. The +/// production values are [`FenceBounds::production`]; nothing else constructs one outside tests. +#[derive(Clone, Copy, Debug)] +struct FenceBounds { + /// How long `Drop` itself blocks before handing off to an owner. + fast: std::time::Duration, + /// The outer bound on the handed-off owner, counted from when it takes over. + max: std::time::Duration, + /// How long the owner keeps asking the daemon to confirm the removal it issued. + confirm: std::time::Duration, + /// How much longer the owner KEEPS the job after `max` expires with the create still running. + /// + /// `max` is where an owner used to stop being an owner: it swept, printed a leak, and returned + /// while the create was still in flight, so a container landing one millisecond later had + /// nobody responsible for it. This is the window in which that container is still SOMEBODY'S — + /// the owner stays on the create's own schedule, removes what lands, and confirms it gone. + retain: std::time::Duration, +} + +impl FenceBounds { + /// The bound that matters is the create client's own: [`DOCKER_DEADLINE`] kills it at 120s, so a + /// blocking create closure cannot outlive that, and an owner waiting a margin past it waits for + /// an event that is guaranteed to have happened rather than for a guessed duration. + fn production() -> Self { + Self { + fast: NetnsHolder::CREATE_SETTLE_DEADLINE, + max: DOCKER_DEADLINE + std::time::Duration::from_secs(15), + confirm: std::time::Duration::from_secs(10), + // A create that has not settled by `max` is past its own client's kill, so this covers + // a daemon still working after the client it answered is gone — the case where the + // container appears with no client left to attribute it to. + retain: DOCKER_DEADLINE, + } + } +} + /// A running holder container, and the guarantee that it goes away. /// -/// Constructed the instant the container exists, so that every `?` after that point tears it down on -/// the way out — the holder is a resource with a lifetime, not a step in a procedure. +/// Constructed **before** the container does, so that every `?` — and every cancellation — after +/// that point tears it down on the way out. The holder is a resource with a lifetime, not a step in +/// a procedure. +/// +/// The guard also owns the **temporary containers joined to the namespace**. A sidecar is a joiner: +/// while it lives the namespace cannot go away, so removing the holder while an applier or a +/// readback is still running leaves the namespace pinned by a process nobody is tracking. Every +/// sidecar is therefore named, registered here for its lifetime, and force-removed before the holder +/// is. +/// Counts creates that may still be in flight **after** the future awaiting them is gone. +/// +/// Adoption alone was never a fence. It supplies a NAME to remove; it says nothing about WHEN the +/// container under that name comes into existence. A create runs on a blocking pool thread, and +/// cancelling the future above it does not stop that thread: the create can still be queued inside +/// the daemon, or half-finished, at the instant cleanup runs. Cleanup then asks docker to remove a +/// container that does not exist YET, is told "No such container" — which this module correctly +/// treats as benign — and returns satisfied. Moments later the create lands. The result is a +/// running container with a name nobody holds, which is the exact orphan the registry exists to +/// prevent, manufactured by the cleanup path. +/// +/// So a single early remove is not enough, and no ordering of removes fixes it: the remove has to +/// happen on the far side of the create SETTLING. This fence is that far side. Every create takes a +/// ticket before it is issued, the ticket is moved into the blocking closure, and it is released +/// when that closure ends — whether it succeeded, failed, was killed on the deadline, or ran on +/// past a cancelled future. Cleanup waits for the count to reach zero before it removes anything. +#[derive(Debug)] +struct CreationFence { + in_flight: std::sync::Mutex, + settled: std::sync::Condvar, + /// How many tickets this fence has EVER issued, which only ever grows. + /// + /// `in_flight` answers "is work outstanding now" and is therefore blind to work that started + /// and finished between two observations. This answers the different question "was anything + /// ever started under this fence at all", which is the only way to tell a create that was + /// refused before it began from one that was launched and instantly killed — those two look + /// identical from the outside, and exactly one of them leaves a container behind. + issued: std::sync::Mutex, + /// The owners this fence is RETAINING for jobs whose bounded owner ran out of wait. + /// + /// A bounded owner that reaches its limit with the create still in flight has exactly two + /// honest options: keep waiting (which only moves the edge), or hand the job to something that + /// outlives it. This is the something. It holds EVERY job handed to it — two owners that arrive + /// on the same fence are two obligations, and the second does not replace the first — and it is + /// drained the moment the create settles. It is still not a registry of containers: each entry + /// is one job for the one holder this fence exists to count. + retained: std::sync::Mutex>, +} + +impl Default for CreationFence { + fn default() -> Self { + Self::new() + } +} + +/// How many times a fence being DESTROYED re-runs an owner it still holds before giving up on it. +/// +/// A removal that could not be confirmed puts itself back, so the runner — not the job — is what +/// bounds the attempts. Destruction is the last moment THIS FENCE can act, and it is no longer the +/// last moment anything can: a name this fence could not confirm absent belongs to a container that +/// carries its own `cleanup-after` stamp, so the periodic [`sweep_expired`] pass removes it once its +/// job's deadline plus [`CLEANUP_GRACE_SECS`] has passed. This is a bound on IN-PROCESS effort, and +/// what replaces the retired process-lifetime custody thread: memory does not hold the obligation, +/// the container's own label does. +const RETAINED_FINAL_RUNS: usize = 3; + +/// Cleanup somebody holds on a holder's behalf after its bounded owner is gone. +/// +/// Data, not a closure. A closure that is dropped does nothing and leaves no trace, and it cannot +/// be reported on, merged, or re-scheduled by anyone but the code that built it. This carries the +/// names it is responsible for and everything needed to act on them, so any owner — the fence, the +/// supervisor, a test — can run one attempt and see exactly what is still outstanding afterwards. +#[derive(Clone)] +struct RetainedOwner { + holder: String, + names: Vec, + client: DockerCli, + bounds: FenceBounds, +} + +/// What every retained owner owes, and the only thing any of them owes since the per-job expiry +/// sweep replaced continuous custody: remove these names and confirm each one absent. +/// +/// It was one of two, the other being a watch on the docker event stream for a create whose answer +/// this process never read. That obligation had no end short of the process's own, which is exactly +/// the design Petar's revision superseded: the container now carries its own expiry, so a create +/// that lands late is removed by a later sweep instead of by a watcher this process must keep alive. +const OWED_LABEL: &str = "remove-and-confirm"; + +/// What a retained owner reports after one attempt. +/// +/// The point of returning this rather than `()` is that a cleanup which issued a removal and could +/// not confirm absence has NOT finished, and must not be able to end by returning quietly. It hands +/// back the job that is still owed, and the runner decides when it runs next — never whether. +enum Custody { + /// Every name is discharged on a daemon observation. Nothing is owed. + Discharged, + /// Some names are still owed. This is the job that owes them, narrowed to exactly those names. + StillOwed(RetainedOwner), +} + +/// What happened when a job was handed to a fence. +/// +/// A registration that silently does nothing is the failure this type exists to make impossible: +/// the caller cannot ignore the settled case, because the job comes back and must be run. +#[must_use = "an already-settled fence hands the job back, and it must be run or it is lost"] +enum Registration { + /// The fence took it; the create is still in flight, so a future last-ticket drop will run it. + Retained, + /// NOTHING was in flight at the instant of registration, so no future drop exists to run it. + /// The job is handed back rather than parked where it would never fire. + AlreadySettled(RetainedOwner), +} + +impl std::fmt::Debug for RetainedOwner { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "RetainedOwner({}, still owns {})", + OWED_LABEL, + self.names.join(", ") + ) + } +} + +impl RetainedOwner { + /// One bounded attempt at what this owner owes. Nothing here waits on a clock of its own: each + /// step is a removal or an inspect, all bounded by the existing deadlines. + fn attempt(self) -> Custody { + self.remove_and_confirm() + } + + /// Remove and confirm. A failed confirmation returns THIS owner again over exactly the names + /// that could not be confirmed, so responsibility narrows to what is actually outstanding + /// instead of being discarded at the first disappointment. + fn remove_and_confirm(self) -> Custody { + // A fresh fence deliberately: the create this would have waited for is the one that already + // ended, so there is nothing left to wait for and the owner goes straight to removal and + // confirmation. It keeps no `Arc` back to the fence that holds it — that would be a cycle. + let owner = HolderCleanup { + name: self.holder.clone(), + joiners: self.names.iter().filter(|name| **name != self.holder).cloned().collect(), + creation: std::sync::Arc::new(CreationFence::default()), + client: self.client.clone(), + bounds: self.bounds, + }; + owner.sweep(); + match owner.confirm_all_absent() { + Ok(()) => Custody::Discharged, + Err(mut pending) => { + pending.sort(); + pending.dedup(); + eprintln!( + "sandbox: a retained owner removed {} but could not confirm absence within {:?} \ + — it is NOT releasing them: the job is still owed", + pending.join(", "), + self.bounds.confirm + ); + Custody::StillOwed(RetainedOwner { names: pending, ..self }) + } + } + } + +} + +/// Build the owner that removes `names` under holder `name` and confirms each one absent. +fn retained_removal( + name: String, + names: Vec, + client: DockerCli, + bounds: FenceBounds, +) -> RetainedOwner { + RetainedOwner { holder: name, names, client, bounds } +} + +impl CreationFence { + /// A fence with nothing in flight and nothing retained. + /// + /// It carries no bounds of its own: every obligation it holds arrives already carrying the + /// bounds its owner ran under, and a fence no longer hands work to anything that would need to + /// choose new ones. + fn new() -> Self { + Self { + in_flight: std::sync::Mutex::new(0), + settled: std::sync::Condvar::new(), + issued: std::sync::Mutex::new(0), + retained: std::sync::Mutex::new(Vec::new()), + } + } + + /// Take custody of one create that is about to be issued. + fn begin(self: &std::sync::Arc) -> CreationTicket { + { + let mut in_flight = + self.in_flight.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + *in_flight += 1; + } + { + let mut issued = self.issued.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + *issued += 1; + } + CreationTicket { fence: std::sync::Arc::clone(self) } + } + + /// Install a job, or hand it back because there is nothing left to run it. + /// + /// ATOMIC WITH THE ZERO TRANSITION, and that is the entire point of the shape. The check and + /// the installation happen under ONE `in_flight` guard, and [`CreationTicket::drop`] takes the + /// slot while still holding that same guard. Without this, a real ordering loses the job + /// outright: the bounded owner's wait times out, the last ticket drops and finds the slot + /// empty, and only then does cleanup install a callback that no further drop will ever run. + /// Now the two cases are exhaustive — either a drop is still coming and the fence keeps the + /// job, or none is, and the caller is handed it back and must run it. + /// + /// EVERY job is kept. A second registration on an occupied fence is a second obligation, and it + /// is pushed alongside the first — never assigned over it. + fn register(&self, owner: RetainedOwner) -> Registration { + let in_flight = self.in_flight.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if *in_flight == 0 { + return Registration::AlreadySettled(owner); + } + self.retained.lock().unwrap_or_else(|poisoned| poisoned.into_inner()).push(owner); + Registration::Retained + } + + /// Take custody of a job: retained for the settlement that is coming, or RUN NOW if none is. + /// + /// Registering on an already-idle fence used to mean parking a closure that nothing would ever + /// fire, which reads exactly like ownership and behaves exactly like dropping it on the floor. + fn take_custody(&self, owner: RetainedOwner) { + match self.register(owner) { + Registration::Retained => {} + Registration::AlreadySettled(owner) => self.run_and_keep_if_still_owed(vec![owner]), + } + } + + /// Run each owner once and PUT BACK every one that is still owed. + /// + /// A removal whose absence could not be confirmed has not finished, so it returns to the fence: + /// a later settlement runs it again, and if no later settlement ever comes, this fence's own + /// destruction does — and hands it on from there. Nothing is dropped and nothing is overwritten: + /// the still-owed job is pushed next to whatever else the fence holds. + fn run_and_keep_if_still_owed(&self, owners: Vec) { + for owner in owners { + if let Custody::StillOwed(still_owed) = owner.attempt() { + self.retained + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .push(still_owed); + } + } + } + + /// How much work has ever been started under this fence. + #[cfg(test)] + fn tickets_issued(&self) -> usize { + *self.issued.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + /// Whether this fence is still holding somebody's cleanup. + /// + /// Exists so ownership can be ASSERTED rather than read out of a log line: a test can ask the + /// fence whether a job is still owned, which a message about ownership cannot answer. + #[cfg(test)] + fn holds_retained_owner(&self) -> bool { + self.retained + .lock() + .map(|retained| !retained.is_empty()) + .unwrap_or(false) + } + + /// Block until every in-flight create has ended, or the bound expires. + /// + /// Returns whether it settled. A timeout is reported by the caller rather than swallowed: a + /// create still running after this bound is a create whose container this process may never see, + /// and saying so is the difference between a known leak and a silent one. + fn wait_until_settled(&self, bound: std::time::Duration) -> bool { + let started = std::time::Instant::now(); + let mut in_flight = self.in_flight.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + while *in_flight > 0 { + let Some(left) = bound.checked_sub(started.elapsed()) else { + return false; + }; + let (guard, timeout) = self + .settled + .wait_timeout(in_flight, left) + .unwrap_or_else(|poisoned| poisoned.into_inner()); + in_flight = guard; + if timeout.timed_out() && *in_flight > 0 { + return false; + } + } + true + } +} + +/// One in-flight create. Releasing it is what "the create has settled" means. +/// +/// Held by the blocking closure itself, never by the future awaiting it — that is the whole point. +/// A cancelled future drops its side and the closure keeps this one until it genuinely ends. +#[derive(Debug)] +struct CreationTicket { + fence: std::sync::Arc, +} + +impl CreationTicket { + /// The fence this ticket belongs to, so work spawned underneath it can take its OWN ticket. + /// + /// Detached IO threads use this. A reader or writer holding a pipe endpoint is work this + /// process is still doing, and until it takes a ticket of its own it is work nobody is counted + /// for — the closure could return, release the only ticket, and let cleanup conclude while the + /// thread was still reading. + fn fence(&self) -> &std::sync::Arc { + &self.fence + } +} + +impl Drop for CreationTicket { + fn drop(&mut self) { + // The decrement and the claim on the retained jobs are ONE critical section, taken in the + // same order as `register`: `in_flight` first, then `retained`. Releasing the count before + // looking at the slot is what opened the missed-handoff window -- a registration could slip + // in after this drop had already decided there was nothing to run. + let owners = { + let mut in_flight = + self.fence.in_flight.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + *in_flight = in_flight.saturating_sub(1); + if *in_flight == 0 { + std::mem::take( + &mut *self.fence.retained.lock().unwrap_or_else(|poisoned| poisoned.into_inner()), + ) + } else { + Vec::new() + } + }; + self.fence.settled.notify_all(); + // THE HANDOFF LANDS HERE, on the thread that actually ended the create. + // + // A create whose answer this process never read is no longer watched from memory: the name + // is stamped with its job's own expiry at create time, so a container that lands after this + // point is discovered by a later sweep from the container itself. See `sweep_expired`. + // + // A bounded owner that gave up earlier left its job with the fence instead of dropping it. + // This is the event it was waiting for -- not a clock, the create's own end -- so the job + // runs now, however long "now" took to arrive. The lock is released before it runs: the + // job removes and confirms, and both talk to the daemon. + if !owners.is_empty() { + self.fence.run_and_keep_if_still_owed(owners); + } + } +} + +impl Drop for CreationFence { + /// THE LAST MOMENT THIS FENCE CAN ACT — and not the last moment this process can. + /// + /// The jobs still held run here, up to [`RETAINED_FINAL_RUNS`] rounds. What is still owed after + /// that is NAMED and left to the sweep: every container this fence could be owed is stamped + /// with its job's own `cleanup-after`, and [`sweep_expired`] runs every + /// [`SWEEP_INTERVAL_SECS`] for as long as the seller runs. So the obligation does not need an + /// in-memory owner that outlives the fence — which is the process-lifetime custody thread this + /// revision removed, per Petar's review of PR 996 — it needs the daemon to still be told, on a + /// schedule, about a container whose deadline has passed. + /// + /// The owners deliberately keep no `Arc` back to this fence (that would be a cycle, and the + /// fence would never be destroyed at all). + fn drop(&mut self) { + for _ in 0..RETAINED_FINAL_RUNS { + let owners = std::mem::take( + &mut *self.retained.lock().unwrap_or_else(|poisoned| poisoned.into_inner()), + ); + if owners.is_empty() { + break; + } + for owner in owners { + if let Custody::StillOwed(still_owed) = owner.attempt() { + self.retained + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .push(still_owed); + } + } + } + // Out of attempts HERE. Not out of owners. + let still_owed = std::mem::take( + &mut *self.retained.lock().unwrap_or_else(|poisoned| poisoned.into_inner()), + ); + for owner in still_owed { + eprintln!( + "sandbox: {} could not be confirmed absent in {} attempts by a fence that is being \ + destroyed — these names are left to the expiry sweep, which removes them once the \ + job's own deadline plus the cleanup grace has passed", + owner.names.join(", "), + RETAINED_FINAL_RUNS + ); + } + } +} + #[derive(Debug)] pub struct NetnsHolder { name: String, + sidecars: std::sync::Arc>>, + creation: std::sync::Arc, + client: DockerCli, + bounds: FenceBounds, + /// The `--label` arguments stamped onto every helper this holder runs, from + /// [`helper_label_args`]. Empty for a holder adopted outside a production launch — a test + /// fixture has no seat and no deadline to attribute a helper to, and an empty seat label would + /// be worse than none. + helper_labels: Vec, } impl NetnsHolder { - /// Adopt an already-created container as the holder. Private on purpose: a `NetnsHolder` that - /// does not correspond to a running container would promise a teardown it cannot perform. - fn adopt(name: String) -> Self { - Self { name } + /// Adopt a container name as the holder, whether or not the container exists yet. + /// + /// Private on purpose. Adoption happens **before** the create command is issued: the create is + /// an await, an await is a cancellation point, and a cancelled create can still complete inside + /// the blocking pool after the future is gone. Adopting afterwards left exactly that container + /// with no guard — running, joined to nothing, and invisible to this process. + /// + /// Adoption gives cleanup a name. [`CreationFence`] gives it a TIME. Both are required. + #[cfg(test)] + fn adopt(name: String, client: DockerCli) -> Self { + Self::adopt_bounded(name, client, FenceBounds::production()) + } + + /// As [`Self::adopt`], with the cleanup bounds named by the caller so the delayed path can be + /// exercised without waiting out the production ones. + fn adopt_bounded(name: String, client: DockerCli, bounds: FenceBounds) -> Self { + Self { + name, + sidecars: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), + creation: std::sync::Arc::new(CreationFence::default()), + client, + bounds, + helper_labels: Vec::new(), + } + } + + /// Stamp every helper this holder goes on to run with `labels` (from [`helper_label_args`]). + /// + /// Set once by [`establish_with`], which is the only place that knows the job, the seat and the + /// deadline at the same time. Applied centrally in [`run_sidecar_confirmed`] rather than in each + /// argv builder: every helper container in this module is created through that one funnel, so a + /// helper added later is stamped without its author having to remember to do it — and a forgotten + /// stamp is an unexpiring leftover, which is the failure this whole path exists to end. + fn label_helpers(&mut self, labels: Vec) { + self.helper_labels = labels; + } + + /// The helper stamp set by [`Self::label_helpers`]; empty when this holder was adopted outside a + /// production launch. + fn helper_labels(&self) -> &[String] { + &self.helper_labels + } + + /// The docker client this holder was built with. Cleanup uses it too, so a stand-in cannot be + /// half-applied: whatever created the container is what removes and confirms it. + fn client(&self) -> &DockerCli { + &self.client + } + + /// Take a ticket for a create about to be issued against this holder. + fn fence_creation(&self) -> CreationTicket { + self.creation.begin() + } + + /// Register a sidecar container name for the duration of one command. + fn watch_sidecar(&self, name: String) -> SidecarGuard { + if let Ok(mut names) = self.sidecars.lock() { + names.push(name.clone()); + } + SidecarGuard { name, registry: std::sync::Arc::clone(&self.sidecars), completed: false } + } + + /// Whether a failed `docker rm` says "there was nothing here" rather than "I could not do it". + /// + /// The only benign failure. Because the holder is adopted **before** its create is issued, a run + /// cancelled in that window tears down a container that never existed, and docker rightly + /// objects. Every other message is a container this process could not remove — a leak, which the + /// caller reports as a leak. An empty stderr is not benign: a removal that failed without saying + /// why is the one case where assuming success would be a silent orphan. + fn force_remove_stderr_is_benign(stderr: &str) -> bool { + stderr.contains("No such container") + } + + /// How long one `docker rm` may run before it is abandoned and reported as a leak. + /// + /// This runs inside `Drop`, on the thread that is tearing the job down, so it is a hard cap on + /// how long a wedged daemon can hold that thread. Long enough that an ordinary removal under + /// load is never cut short -- removals finish in well under a second -- and short enough that a + /// daemon which has stopped answering ends the job instead of pinning the caller forever. + const REMOVE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(20); + + /// How long cleanup waits for an in-flight create to settle before removing regardless. + /// + /// Bounded by the same reasoning as [`Self::REMOVE_DEADLINE`], and deliberately longer than it: + /// a create that has reached the daemon finishes in well under a second, while the thing this + /// guards against — removing BEFORE the container exists — is unrecoverable once it happens. + /// The create's own [`DOCKER_DEADLINE`] kills the client at 120s, so this can never wait for a + /// hung client indefinitely; it waits for the blocking closure to end, which that kill forces. + const CREATE_SETTLE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(30); + + /// Wait for one child, bounded. On the deadline the child is killed and reaped, and the wait is + /// reported as a failure rather than as a removal that succeeded. + /// + /// `Child::wait`, and the `output()` this replaced, have no timeout at all: a docker client + /// talking to a daemon that never answers blocks forever, which in `Drop` means teardown never + /// returns. The word "bounded" was in the comment above this function long before anything in + /// it bounded anything. + fn wait_bounded( + child: &mut std::process::Child, + deadline: std::time::Duration, + ) -> Result { + let expires = std::time::Instant::now() + deadline; + loop { + match child.try_wait() { + Ok(Some(status)) => return Ok(status), + Ok(None) => { + if std::time::Instant::now() >= expires { + // Killed AND reaped: leaving a zombie behind would be its own small leak, + // and the kill is what makes the bound real rather than advisory. + let _ = child.kill(); + let _ = child.wait(); + return Err(format!( + "docker rm did not finish within {}s and was abandoned -- the container \ + may still exist", + deadline.as_secs() + )); + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + Err(error) => return Err(format!("could not wait for docker rm: {error}")), + } + } + } + + /// Force-remove one container by name, bounded, and say what actually happened. + /// + /// `Ok(())` means docker reported the removal, or reported that there was nothing to remove. + fn force_remove(client: &DockerCli, name: &str) -> Result<(), String> { + let mut child = std::process::Command::new(client.program()) + .args(["rm", "--force", "--volumes", name]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn() + .map_err(|error| format!("could not run docker rm: {error}"))?; + let status = Self::wait_bounded(&mut child, Self::REMOVE_DEADLINE)?; + // Read after the wait returns. `docker rm` writes one short line at most, so this cannot + // deadlock on a full pipe the way a chatty child could. + let mut stderr_bytes = Vec::new(); + if let Some(mut pipe) = child.stderr.take() { + use std::io::Read as _; + let _ = pipe.read_to_end(&mut stderr_bytes); + } + if status.success() { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&stderr_bytes).trim().to_owned(); + // Removing something that was never created is the expected path when a create was + // cancelled before it started, and it is not a cleanup failure. + if Self::force_remove_stderr_is_benign(&stderr) { + Ok(()) + } else { + Err(if stderr.is_empty() { + "docker rm failed and said nothing".to_owned() + } else { + stderr + }) + } } /// The container name, for `docker` commands that address it directly. @@ -100,33 +866,314 @@ impl NetnsHolder { } } +/// One sidecar's registration, dropped when its command finishes however it finishes. +/// +/// On a normal return the container is already gone (`--rm`) and this only deregisters. On +/// cancellation the future is dropped mid-command, the name stays with the holder, and the holder's +/// own `Drop` force-removes it — which is the case that used to leave a joiner pinning a namespace +/// whose holder had just been removed. +#[derive(Debug)] +struct SidecarGuard { + name: String, + registry: std::sync::Arc>>, + /// Set only when the command returned. A guard dropped without this is a cancelled command. + completed: bool, +} + +impl SidecarGuard { + /// The command's client was **reaped with an exit status**. `docker run --rm` removes the + /// container when its client exits -- including on a nonzero exit -- so from here the name is no + /// longer a cleanup target, and keeping it would make the holder report a leak for something + /// already gone. + /// + /// Deliberately NOT called for a result that merely returned: a deadline kill, a signal, or a + /// failure before the wait leaves a container this process never saw finish. + fn completed(&mut self) { + self.completed = true; + } +} + +impl Drop for SidecarGuard { + /// Deregisters ONLY a command that finished. + /// + /// This used to deregister unconditionally, which quietly inverted the custody it was written + /// for. Cancellation drops the future mid-command, which drops this guard, which struck the + /// name from the registry -- and the holder's own `Drop`, reading that registry moments later, + /// then saw nothing to remove. The container the blocking docker client had already created + /// stayed joined to the namespace with no guard, no record and no remover: exactly the orphan + /// the registry exists to prevent, produced by the cleanup path itself. + /// + /// So a cancelled command leaves its name behind deliberately. The cost of keeping a name whose + /// container never got created is one `docker rm` answering "No such container", which + /// [`NetnsHolder::force_remove_stderr_is_benign`] already treats as success. The cost of + /// dropping a name whose container does exist is a pinned namespace nothing will ever clean up. + fn drop(&mut self) { + if !self.completed { + return; + } + if let Ok(mut names) = self.registry.lock() { + names.retain(|name| name != &self.name); + } + } +} + impl Drop for NetnsHolder { - /// Destroy the holder, **synchronously**. + /// Destroy the holder, **synchronously**, and everything joined to it first. /// /// Deliberately a blocking `std::process::Command` and not a spawned task: a task spawned from /// `Drop` can be discarded when the runtime shuts down, and runtime shutdown is exactly the path a /// panicking or aborted job takes. A leaked holder is a container pinned to a namespace nothing /// will ever clean up, so the ~100 ms block is the cheaper end of that trade. /// - /// Failure is logged, never propagated: `Drop` cannot return, and the reaper in - /// [`reap_orphans`] is the backstop for the case where this did not work. + /// **Sidecars go first.** A joiner still running when the holder is removed keeps the namespace + /// alive, and is precisely what a cancelled applier or readback leaves behind. Only names this + /// run registered are removed; nothing is matched by pattern, so a sibling job's containers are + /// never in scope. + /// + /// Failure is reported, never propagated and never implied away: `Drop` cannot return, so each + /// failure is printed as a failure — "could not remove", not "destroyed" — and + /// [`reap_orphans`] is the backstop. A cleanup that failed is a leak that is now on the record. fn drop(&mut self) { - let outcome = std::process::Command::new("docker") - .args(["rm", "--force", "--volumes", &self.name]) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::piped()) - .output(); - match outcome { - Ok(done) if done.status.success() => {} - Ok(done) => eprintln!( - "sandbox: could not remove netns holder {}: {}", - self.name, - String::from_utf8_lossy(&done.stderr).trim() + // FIRST, before a single remove is issued: let any in-flight create finish. + // + // Removing ahead of the create is worse than not removing at all, because "No such + // container" reads as success and closes the case on a container that is about to exist. + // Waiting here costs nothing in the ordinary path (nothing is in flight, the count is + // already zero) and is the only thing that makes the removes below meaningful in the + // cancelled path. + let cleanup = HolderCleanup { + name: self.name.clone(), + joiners: self.sidecars.lock().map(|names| names.clone()).unwrap_or_default(), + creation: std::sync::Arc::clone(&self.creation), + client: self.client.clone(), + bounds: self.bounds, + }; + if self.creation.wait_until_settled(self.bounds.fast) { + // Ordinary path: nothing was in flight, or it finished while we waited. `docker rm` + // returning success here IS the daemon's answer, so no second question is asked. + // + // A removal the daemon REFUSED is not answered by a log line alone. This path used to + // sweep, print "LEAKED" for whatever refused, and return, leaving nothing responsible. + // What refuses here is now NAMED and left to the expiry sweep: the container carries + // its own job's `cleanup-after` stamp, and the seller's periodic pass removes it once + // that deadline plus the grace has passed. `Drop` stays the ~100 ms it always was, and + // nothing is retried from memory for the life of the process. + let refused = cleanup.sweep(); + if !refused.is_empty() { + eprintln!( + "sandbox: {} refused removal in netns holder {}'s ordinary teardown — left to \ + the expiry sweep, which removes them once the job's deadline plus the cleanup \ + grace has passed", + refused.join(", "), + self.name + ); + } + return; + } + // Delayed path. The create is STILL running, and this is the case the previous version got + // wrong: it removed anyway, printed LEAKED, and returned — leaving nobody responsible for + // the container that was still on its way. "No such container" then read as success for an + // object about to exist. + // + // Removing now cannot be made safe by waiting longer, so cleanup is not removed — it is + // HANDED OVER. The owner below outlives this `Drop` and finishes the job on the create's own + // schedule: it waits for the ticket to actually settle, then removes, then keeps asking the + // daemon until absence is CONFIRMED. The wait is bounded by the create client's own + // `DOCKER_DEADLINE` kill plus a margin, so it waits for an event guaranteed to occur rather + // than for a duration someone guessed. + let name = self.name.clone(); + let spawned = std::thread::Builder::new() + .name("mx-holder-cleanup".to_owned()) + .spawn(move || cleanup.own_until_settled_or_confirmed()); + match spawned { + Ok(_owner) => eprintln!( + "sandbox: a create against netns holder {name} is still in flight after {:?} — \ + cleanup is NOT removing ahead of it; an owner has been retained and will remove \ + and confirm once the create settles", + self.bounds.fast ), + // No thread to hand it to: finish the job here rather than remove early. Blocking is + // the lesser harm; removing ahead of a live create is the one outcome with no recovery. Err(error) => { - eprintln!("sandbox: could not run docker rm for netns holder {}: {error}", self.name) + eprintln!( + "sandbox: could not retain a cleanup owner for netns holder {name} ({error}) — \ + completing the wait inline instead" + ); + let inline = HolderCleanup { + name: self.name.clone(), + joiners: self.sidecars.lock().map(|names| names.clone()).unwrap_or_default(), + creation: std::sync::Arc::clone(&self.creation), + client: self.client.clone(), + bounds: self.bounds, + }; + inline.own_until_settled_or_confirmed(); + } + } + } +} + +/// The cleanup that owns a holder's name once the holder itself is gone. +/// +/// Split out of `Drop` for one reason: `Drop` must not be the last thing that cares about the +/// container. When a create is still in flight, this outlives the holder and stays responsible until +/// the create settles or the daemon confirms the name is gone. +struct HolderCleanup { + name: String, + joiners: Vec, + creation: std::sync::Arc, + client: DockerCli, + bounds: FenceBounds, +} + +impl HolderCleanup { + /// Remove the joiners, then the holder. Sidecars first: a joiner still running pins the + /// namespace the holder is being torn down to release. + /// + /// Returns every name whose removal the daemon REFUSED (or did not answer), in removal order, so + /// the caller can keep owning them. Logging a refusal was never the same as owning it. + fn sweep(&self) -> Vec { + let mut refused = Vec::new(); + for joiner in &self.joiners { + if let Err(error) = NetnsHolder::force_remove(&self.client, joiner) { + eprintln!( + "sandbox: could not remove sidecar {joiner} joined to netns holder {}: {error} \ + — the namespace may still be pinned by it", + self.name + ); + refused.push(joiner.clone()); + } + } + if let Err(error) = NetnsHolder::force_remove(&self.client, &self.name) { + eprintln!( + "sandbox: could not remove netns holder {}: {error} — not destroyed; it stays owed \ + to whoever called this sweep", + self.name + ); + refused.push(self.name.clone()); + } + refused + } + + /// Ask the daemon, repeatedly, whether EVERY container this owner is responsible for is gone — + /// each joiner as well as the holder. + /// + /// A removal issued is not a removal observed. `Some(true)` is the only answer that retires a + /// name; "could not tell" is treated exactly like "still there", because the cost of asking + /// again is a bounded retry and the cost of believing it is an orphan nobody is looking for. + /// + /// Confirming the holder ALONE was not enough, and that was a real hole: [`Self::sweep`] only + /// LOGS a failed joiner removal, so a sidecar that refused to go on still pins the namespace + /// the holder was torn down to release. An owner ending on holder-absence announced a clean + /// release directly over the top of a container it owns and never asked about. + /// + /// Returns the names that could not be confirmed gone, so the caller can name them. + fn confirm_all_absent(&self) -> Result<(), Vec> { + let give_up = std::time::Instant::now() + self.bounds.confirm; + let mut pause = std::time::Duration::from_millis(20); + // Joiners first: the holder's namespace is not actually released while one of them pins it. + let mut pending: Vec = + self.joiners.iter().cloned().chain(std::iter::once(self.name.clone())).collect(); + loop { + pending.retain(|name| container_is_absent(&self.client, name) != Some(true)); + if pending.is_empty() { + return Ok(()); + } + if std::time::Instant::now() >= give_up { + return Err(pending); + } + std::thread::sleep(pause); + pause = (pause * 2).min(std::time::Duration::from_millis(500)); + } + } + + /// Wait for the create to genuinely settle, then remove, then confirm. + /// + /// Ends on ACTUAL settlement followed by CONFIRMED absence. If the create never settles within + /// the bound, the sweep still runs and the confirmation still decides the verdict: a container + /// that never landed is confirmed absent and the case closes honestly; one that cannot be + /// confirmed gone is reported as leaked, with the reason, rather than silently written off. + fn own_until_settled_or_confirmed(self) { + let mut settled = self.creation.wait_until_settled(self.bounds.max); + // Best effort either way: whatever HAS landed should go now. + self.sweep(); + if !settled { + // `max` expired with the create STILL RUNNING. This is where ownership used to end: it + // swept, printed a leak and returned, which handed the container that was still on its + // way to nobody. The sweep above cannot cover it — you cannot remove what has not + // appeared — so the only thing that keeps it owned is staying. + // + // So the job is KEPT for `retain` longer. If the create lands in that window it is + // swept again, by an owner that is still responsible for it, and then confirmed gone. + eprintln!( + "sandbox: a create against netns holder {} is STILL IN FLIGHT after {:?} — this \ + owner is NOT releasing it: custody is retained for a further {:?}, and anything \ + that lands in that window will be removed and confirmed by this owner", + self.name, self.bounds.max, self.bounds.retain + ); + settled = self.creation.wait_until_settled(self.bounds.retain); + if settled { + // It landed late, and it is still this owner's to remove. + self.sweep(); } } + if !settled { + // The wait is over and the create is STILL in flight. Ownership does NOT end here. + // + // Waiting longer was never the answer. Whatever the bound, the case that breaks is a + // container landing one millisecond past it, so a bigger window makes the orphan rarer + // without making it impossible -- it moves the edge, it does not remove it. The job is + // TRANSFERRED instead, to an owner the create's own fence retains. The fence is the one + // object that knows when this create genuinely ends, because the create's ticket is + // what releases it, so the handoff is to the settlement event itself rather than to + // another clock. + // + // This claims nothing about whether the daemon will finish. It is the narrower true + // thing: if the container ever lands, somebody still owns it. + self.creation.take_custody(retained_removal( + self.name.clone(), + self.joiners.clone(), + self.client.clone(), + self.bounds, + )); + eprintln!( + "sandbox: a create against netns holder {} was STILL IN FLIGHT after {:?} and did \ + not land within the further {:?} this owner retained it — custody is NOT being \ + released: it has been TRANSFERRED to an owner retained by the create's own fence, \ + which removes and confirms this holder and its {} joiner(s) when that create \ + settles, whenever that is", + self.name, + self.bounds.max, + self.bounds.retain, + self.joiners.len() + ); + return; + } + if let Err(pending) = self.confirm_all_absent() { + // A removal was ISSUED and the daemon never confirmed absence. Responsibility used to + // end on the log line below -- the names were named, and then let go, which is + // indistinguishable downstream from a clean release. An unconfirmed name is kept + // instead: the fence retains an owner holding exactly those names, so they remain OWNED + // rather than merely mentioned, and a later create settling on this holder runs them + // again. + // THIS FENCE IS ALREADY IDLE. The create settled -- that is why confirmation ran at + // all -- so there is no future ticket drop here to fire a parked callback. Handing the + // job over therefore has to mean RUN IT, which `take_custody` does, and if it still + // cannot confirm absence it goes back into the slot where this fence's destruction + // will run it again rather than discard it. + self.creation.take_custody(retained_removal( + self.name.clone(), + pending.clone(), + self.client.clone(), + self.bounds, + )); + eprintln!( + "sandbox: could not confirm {} absent within {:?} after the create settled — these \ + are NOT released: an owner for them was run again on this holder's fence and is \ + kept there until it confirms, and the boot reaper remains the backstop", + pending.join(", "), + self.bounds.confirm + ); + } } } @@ -137,6 +1184,10 @@ impl Drop for NetnsHolder { pub struct Containment { pub holder: NetnsHolder, pub proxy_host: String, + /// The link inside the namespace the egress filters were installed on, as measured — never a + /// guess like `eth0`. Carried so a caller, a log line or a test can name the interface that is + /// actually filtered rather than the one everybody assumes. + pub egress_dev: String, } /// The holder's container name for `job_id`. @@ -160,6 +1211,13 @@ pub fn holder_name(job_id: &str) -> String { /// `seat` is the owning seller's public key hex and goes on as a second label. It is what lets the /// boot reaper tell this seat's holders from another daemon's on a shared host; see /// [`HOLDER_SEAT_LABEL`]. +/// +/// `cleanup_after` is the absolute unix second from [`cleanup_after_unix`] — this job's own +/// effective deadline plus the grace. It goes on as a third label so the periodic sweep can judge +/// this container **without knowing anything about the job**, including after the process that +/// created it is gone. It is derived by the seller from the deadline it is itself enforcing; no part +/// of it comes from the buyer's payload, which is why a request cannot ask for a container that +/// never expires. pub fn holder_argv( name: &str, network: &str, @@ -168,6 +1226,7 @@ pub fn holder_argv( gid: u32, job_id: &str, seat: &str, + cleanup_after: u64, ) -> Vec { [ "docker", @@ -181,6 +1240,10 @@ pub fn holder_argv( &format!("{HOLDER_LABEL}={job_id}"), "--label", &format!("{HOLDER_SEAT_LABEL}={seat}"), + "--label", + &format!("{HOLDER_CLEANUP_AFTER_LABEL}={cleanup_after}"), + "--label", + &format!("{HOLDER_ROLE_LABEL}={ROLE_HOLDER}"), "--read-only", "--cap-drop", "ALL", @@ -386,12 +1449,379 @@ pub fn parse_holder_listing(stdout: &str) -> Vec { .collect() } -/// `docker` argv listing every container on the host by full id. -pub fn list_all_containers_argv() -> Vec { - ["docker", "ps", "--all", "--no-trunc", "--quiet"] - .into_iter() - .map(String::from) - .collect() +/// `docker` argv listing the containers **this seat owns**, with the metadata the expiry sweep +/// judges them by: full id, owning seat, cleanup-after stamp, and role. +/// +/// The `label=` filter is narrowing, exactly as in [`list_holders_argv`], and exactly as +/// there it is **not** the guard: [`expired_owned`] re-checks the seat in Rust with an exact string +/// comparison, because a filter that silently matched too much would be indistinguishable from one +/// that worked. The filter's only failure that matters is matching too little, which leaks a +/// container instead of removing a stranger's. +/// +/// `--all` because an expired container is usually not running: a holder whose job died is +/// `Exited`, and a helper that finished is `Exited` too. Listing only running containers would miss +/// precisely the leftovers this sweep exists to remove. +pub fn list_owned_argv(seat: &str) -> Vec { + [ + "docker", + "ps", + "--all", + "--no-trunc", + "--filter", + &format!("label={HOLDER_SEAT_LABEL}={seat}"), + "--format", + &format!( + "{{{{.ID}}}}\t{{{{.Label \"{HOLDER_SEAT_LABEL}\"}}}}\t{{{{.Label \"{HOLDER_CLEANUP_AFTER_LABEL}\"}}}}\t{{{{.Label \"{HOLDER_ROLE_LABEL}\"}}}}\t{{{{.Label \"{HELPER_JOB_LABEL}\"}}}}\t{{{{.Label \"{HOLDER_LABEL}\"}}}}" + ), + ] + .into_iter() + .map(String::from) + .collect() +} + +/// One container as the expiry sweep sees it. +/// +/// Every field after `id` is an `Option` because every one of them can be absent on a real host: a +/// container from a build older than these labels, a container whose labels were not applied +/// because the create died between docker accepting the argv and recording it, or simply a +/// container belonging to something else that happens to carry the seat label. Absence is never +/// read as a permission. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OwnedContainer { + /// Full container id, as the sweep will name it to `docker rm`. + pub id: String, + /// Owning seat from [`HOLDER_SEAT_LABEL`]; `None` when the label is absent or empty. + pub seat: Option, + /// Parsed [`HOLDER_CLEANUP_AFTER_LABEL`]; `None` when absent, empty, or not a unix second. + pub cleanup_after: Option, + /// Parsed [`HOLDER_ROLE_LABEL`]; `None` when absent or empty. + /// + /// **A removal criterion, as of #996.** It was previously parsed and then ignored, which let the + /// sweep act on any container wearing this seat's label whatever it was. + pub role: Option, + /// Raw [`HELPER_JOB_LABEL`] column — the job a HELPER names; `None` when absent or empty. + /// + /// **Held as read, never merged with the holder column.** Which column is authoritative is a + /// function of the row's role, so the two are carried separately and resolved by + /// [`OwnedContainer::resolve_job`]. + pub helper_job: Option, + /// Raw [`HOLDER_LABEL`] column — the job a HOLDER names; `None` when absent or empty. + /// + /// **Also a removal criterion**, through [`OwnedContainer::resolve_job`]. A container that + /// cannot name its job cannot be shown to have outlived one, and the stamp alone is then just a + /// number with no provenance. + pub holder_job: Option, +} + +impl OwnedContainer { + /// The job this container belongs to, read from the column ITS ROLE writes — or why it cannot + /// be read. + /// + /// **Resolution is per role, and the other role's column is never a fallback (#996 R3/3).** A + /// holder names its job in [`HOLDER_LABEL`] and never wears [`HELPER_JOB_LABEL`]; a helper does + /// the reverse. Trying one column and falling back to the other — `field().or_else(|| field())` + /// — accepted two shapes no production writer emits. A row carrying two DIFFERENT job ids was + /// accepted on whichever column came first, the disagreement discarded before any gate could + /// see it; and a job sitting in the other role's column read as valid provenance. Both describe + /// a container this module cannot attribute to a job, and an unattributable container is + /// precisely what the skip report exists to surface. + /// + /// Both columns present and AGREEING is not a defect: per-role resolution returns the same job + /// either way, so there is nothing the row fails to say. Only disagreement is refused. + /// + /// Refusals are returned, never dropped: [`partition_owned`] carries each into + /// [`OwnedSelection::skipped`] beside `MissingJob` and `UnreadableStamp`. + fn resolve_job(&self) -> Result<&str, SkipReason> { + let (own_role_column, other_role_column) = match self.role.as_deref() { + Some(ROLE_HOLDER) => (self.holder_job.as_deref(), self.helper_job.as_deref()), + Some(ROLE_HELPER) => (self.helper_job.as_deref(), self.holder_job.as_deref()), + // Unreachable from `partition_owned`, which refuses an unknown role first. Total here + // so the method can never be read as "any role resolves to something". + role => return Err(SkipReason::UnknownRole(role.map(str::to_owned))), + }; + match (own_role_column, other_role_column) { + (Some(job), Some(other)) if job != other => Err(SkipReason::ConflictingJobs { + own_role: job.to_owned(), + other_role: other.to_owned(), + }), + (Some(job), _) => Ok(job), + (None, Some(misplaced)) => Err(SkipReason::JobInWrongColumn { + role: self.role.clone().unwrap_or_default(), + job: misplaced.to_owned(), + }), + (None, None) => Err(SkipReason::MissingJob), + } + } +} + +/// Parse `docker ps --format '{{.ID}}\t{{.Label …}}…'` output into one record per container. +/// +/// **A malformed stamp parses to `None`, not to zero.** An absent label arrives from docker as an +/// empty field, and a corrupted one as arbitrary text; reading either as the number 0 would date the +/// container to 1970 and make it instantly sweepable. Every unreadable stamp therefore becomes +/// `None`, which [`expired_owned`] refuses to act on. The failure mode is a leak the operator can +/// see, never a removal nobody authorised. +pub fn parse_owned_listing(stdout: &str) -> Vec { + stdout + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(|line| { + let mut fields = line.split('\t'); + let id = fields.next().unwrap_or_default().trim().to_owned(); + let field = |fields: &mut std::str::Split<'_, char>| { + fields.next().map(str::trim).filter(|value| !value.is_empty()).map(str::to_owned) + }; + let seat = field(&mut fields); + let cleanup_after = field(&mut fields).and_then(|value| value.parse::().ok()); + let role = field(&mut fields); + // Each job column is read into its OWN field, in listing order, and collapsed nowhere + // here. Reading only the helper label made every production holder parse as jobless + // (the leak this sweep exists to close); collapsing the two with `or_else` then let a + // present helper column short-circuit the holder column, which accepted rows naming two + // different jobs and rows whose job sat in the wrong-role column. Role decides which + // column speaks — see [`OwnedContainer::resolve_job`]. + let helper_job = field(&mut fields); + let holder_job = field(&mut fields); + OwnedContainer { + id, + seat, + cleanup_after, + role, + helper_job, + holder_job, + } + }) + .filter(|container| !container.id.is_empty()) + .collect() +} + +/// The containers `seat` may remove right now: **owned by `seat`** and carrying a **readable** +/// cleanup stamp that `now_unix` has passed. +/// +/// **Both legs are required and neither is sufficient**, for the same reason the boot reaper needs +/// two. Ownership alone would remove a container whose job is still inside its deadline. Expiry +/// alone would remove a co-tenant seat's container on a shared docker socket — the exact accident +/// [`HOLDER_SEAT_LABEL`] was added to prevent. +/// +/// **Attachment is deliberately NOT consulted here**, and that is the one place this predicate +/// differs from [`reapable_holders`]. The boot reaper must not touch an attached holder, because a +/// live job is joined to it and `unattached` is its only evidence the job is gone. This sweep has +/// better evidence: the container's own stamp says its job's deadline passed more than +/// [`CLEANUP_GRACE_SECS`] ago. A container still attached at that point is attached to something +/// that outlived its own deadline by an hour, which is the leak — refusing to remove it would leave +/// precisely the case this exists for. The grace is sized so that ordinary post-deadline work has +/// long finished. +/// +/// An empty `seat` selects nothing: a caller that cannot name itself owns nothing to remove. +#[must_use] +pub fn expired_owned(containers: &[OwnedContainer], seat: &str, now_unix: u64) -> Vec { + partition_owned(containers, seat, now_unix).removable +} + +/// Why the sweep refused to act on a container that carries this seat's label. +/// +/// Every variant names a container the sweep SAW and left alone. They are carried out of the +/// selection instead of being dropped because an unreadable record is exactly the shape a leak +/// takes: a container nobody can prove is expired is also a container nobody will ever remove. +/// Filtering them silently — the behaviour before #996 — made a growing pile of malformed +/// containers indistinguishable from a clean host. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SkipReason { + /// The seat label came back absent or empty from a listing that filtered on it. + UnreadableSeat, + /// Absent, or a role this module never issues. Carries what was read, for the operator. + UnknownRole(Option), + /// No job label: the container cannot be tied to a job whose deadline could have passed. + MissingJob, + /// BOTH job columns name a job and they DISAGREE. The row cannot say which job its stamp was + /// derived from, so it names none: removing it would act on a job identity nothing establishes. + ConflictingJobs { + /// What the row's own role column said. + own_role: String, + /// What the other role's column said instead. + other_role: String, + }, + /// The only job id sits in the column belonging to the OTHER role — a helper's column on a + /// holder row, or the reverse. No production writer emits that, so the row's provenance is + /// unknown and its stamp unattributable. + JobInWrongColumn { + /// The role the row declared. + role: String, + /// The job id found in the wrong column, for the operator. + job: String, + }, + /// The cleanup stamp is absent, empty, or not a unix second. + UnreadableStamp, +} + +impl std::fmt::Display for SkipReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::UnreadableSeat => write!(f, "seat label absent or unreadable"), + Self::UnknownRole(Some(role)) => write!(f, "unrecognised role {role:?}"), + Self::UnknownRole(None) => write!(f, "role label absent"), + Self::MissingJob => write!(f, "job label absent"), + Self::ConflictingJobs { + own_role, + other_role, + } => write!( + f, + "job labels disagree: this role's column says {own_role:?}, the other role's says \ + {other_role:?}" + ), + Self::JobInWrongColumn { role, job } => write!( + f, + "job {job:?} sits in the wrong-role column for a {role:?} — no launch writes that" + ), + Self::UnreadableStamp => write!(f, "cleanup stamp absent or unparseable"), + } + } +} + +/// What one look at the listing decided: what may be removed, and what was refused and why. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct OwnedSelection { + /// Fully validated and past their stamp. These, and only these, may be handed to `docker rm`. + pub removable: Vec, + /// Containers wearing this seat's label that failed validation, each with its reason. + /// + /// Reported whether or not their stamp has passed: a record that cannot be validated now will + /// not become valid by ageing, so the operator needs it the first time it is seen. + pub skipped: Vec<(String, SkipReason)>, +} + +/// Split this seat's containers into what it may remove and what it refuses to touch. +/// +/// **Four things must all hold before an id reaches `removable`**, and the point of each is that +/// absence is never read as permission: +/// +/// * the seat label is present and is THIS seat — ownership, the one thing that makes this safe +/// on a docker socket shared with another seller daemon; +/// * the role is one this module issues ([`ROLE_HOLDER`] or [`ROLE_HELPER`]) — a container of +/// someone else's that happens to carry a matching seat label is not ours to remove; +/// * the job label is present — the stamp has to belong to a job, or it is an unattributable +/// number; +/// * the stamp parses and `now_unix` has passed it. +/// +/// A container of ours failing any of the middle three is REPORTED in `skipped` rather than +/// dropped. A container belonging to another seat is neither removed nor reported: it is simply +/// not our business, and reporting it would turn a co-tenant's normal operation into noise here. +/// +/// An empty `seat` selects nothing: a caller that cannot name itself owns nothing to remove. +#[must_use] +pub fn partition_owned(containers: &[OwnedContainer], seat: &str, now_unix: u64) -> OwnedSelection { + let mut selection = OwnedSelection::default(); + if seat.trim().is_empty() { + return selection; + } + for container in containers { + match container.seat.as_deref() { + // Someone else's container on a shared socket. Not ours to remove, not ours to report. + Some(owner) if owner != seat => continue, + Some(_) => {} + None => { + selection + .skipped + .push((container.id.clone(), SkipReason::UnreadableSeat)); + continue; + } + } + let role = container.role.as_deref(); + if role != Some(ROLE_HOLDER) && role != Some(ROLE_HELPER) { + selection.skipped.push(( + container.id.clone(), + SkipReason::UnknownRole(container.role.clone()), + )); + continue; + } + // Per role, and reported rather than dropped: a row naming two different jobs, or naming + // one in the other role's column, is refused here with the reason the operator needs. + if let Err(reason) = container.resolve_job() { + selection.skipped.push((container.id.clone(), reason)); + continue; + } + let Some(after) = container.cleanup_after else { + selection + .skipped + .push((container.id.clone(), SkipReason::UnreadableStamp)); + continue; + }; + if now_unix >= after { + selection.removable.push(container.id.clone()); + } + } + selection +} + +/// Choose this pass's attempts from `candidates`, resuming after `cursor` and wrapping. +/// +/// `candidates` must be sorted and deduplicated; the caller sorts so that the order this walks is a +/// property of the ids themselves and not of whatever order docker happened to list them in. +/// +/// **This is the anti-starvation rule.** Taking the first `cap` ids every pass — the behaviour +/// before #996 — means that when those `cap` removals keep failing, they are selected again on the +/// next pass, and again, and the `cap + 1`th container is never once asked about however long it +/// has been expired. Resuming after the last id attempted makes every candidate reachable within +/// `ceil(len / cap)` passes no matter how many removals fail, because the cursor advances past an +/// attempt whether it succeeded or not. +/// +/// Returns `(attempt, deferred)`, `deferred` in the order the next pass will reach it. +#[must_use] +pub fn select_pass( + candidates: &[String], + cursor: Option<&str>, + cap: usize, +) -> (Vec, Vec) { + if candidates.is_empty() || cap == 0 { + return (Vec::new(), candidates.to_vec()); + } + // First id strictly after the cursor. `unwrap_or(0)` wraps to the front when the cursor is at + // or past the end — including when the ids it named have since been removed. + let start = match cursor { + Some(cursor) => candidates + .iter() + .position(|id| id.as_str() > cursor) + .unwrap_or(0), + None => 0, + }; + let rotated: Vec = candidates[start..] + .iter() + .chain(candidates[..start].iter()) + .cloned() + .collect(); + let take = cap.min(rotated.len()); + (rotated[..take].to_vec(), rotated[take..].to_vec()) +} + +/// Where each seat's sweep left off, so the next pass resumes instead of restarting. +/// +/// Keyed by seat because the cursor is only meaningful against one seat's candidate set, and a +/// process that serves two seats must not let one seat's progress skip the other's containers. +static SWEEP_CURSOR: std::sync::OnceLock< + std::sync::Mutex>, +> = std::sync::OnceLock::new(); + +fn sweep_cursor_for(seat: &str) -> Option { + SWEEP_CURSOR + .get_or_init(Default::default) + .lock() + .ok() + .and_then(|cursors| cursors.get(seat).cloned()) +} + +fn record_sweep_cursor(seat: &str, last_attempted: &str) { + if let Ok(mut cursors) = SWEEP_CURSOR.get_or_init(Default::default).lock() { + cursors.insert(seat.to_owned(), last_attempted.to_owned()); + } +} + +/// `docker` argv listing every container on the host by full id. +pub fn list_all_containers_argv() -> Vec { + ["docker", "ps", "--all", "--no-trunc", "--quiet"] + .into_iter() + .map(String::from) + .collect() } /// `docker` argv printing one `` line per container in `ids`, in order. @@ -462,7 +1892,10 @@ pub async fn reapable_holders_live(seat: &str) -> Result, String> { if seat.trim().is_empty() { return Err("refusing to reap: no owning seat was named".to_owned()); } - let (listing, _) = run_docker(list_holders_argv(seat), None) + // The production client, named here and passed down. Nothing in this path reads an environment + // variable, a global, or a configuration field to decide what to spawn. + let client = DockerCli::system(); + let (listing, _) = run_docker(&client, list_holders_argv(seat), None) .await .map_err(|error| format!("could not list containment holders — {error}"))?; let holders = parse_holder_listing(&listing); @@ -470,11 +1903,11 @@ pub async fn reapable_holders_live(seat: &str) -> Result, String> { return Ok(Vec::new()); } - let (all, _) = run_docker(list_all_containers_argv(), None) + let (all, _) = run_docker(&client, list_all_containers_argv(), None) .await .map_err(|error| format!("could not list containers — {error}"))?; let all: Vec = all.lines().map(str::trim).filter(|id| !id.is_empty()).map(str::to_owned).collect(); - let (modes, _) = run_docker(network_modes_argv(&all), None) + let (modes, _) = run_docker(&client, network_modes_argv(&all), None) .await .map_err(|error| format!("could not read container network modes — {error}"))?; @@ -505,6 +1938,19 @@ pub struct ReapReport { pub removed: Vec, /// The holders the selection chose and `docker rm` refused, each with docker's own reason. pub failed: Vec<(String, String)>, + /// Expired containers this pass SELECTED BUT NEVER ASKED DOCKER ABOUT — the backlog beyond + /// [`MAX_SWEEP_REMOVALS`], and whatever was left when [`SWEEP_PASS_BUDGET`] ran out. + /// + /// Reported rather than dropped, so "this pass removed 32" can be read as "and 140 more are + /// still waiting" instead of as "the host is clean now". Every one of them is still expired on + /// the next tick. + pub deferred: Vec, + /// Containers wearing this seat's label that the sweep REFUSED to act on, each with its reason. + /// + /// Distinct from `failed`, which is docker refusing a removal this pass asked for. These were + /// never asked about: the record itself did not establish that removal was authorised. They are + /// surfaced because a malformed container is a leak that no future pass will clear on its own. + pub skipped: Vec<(String, SkipReason)>, } #[cfg(feature = "acp")] @@ -536,8 +1982,10 @@ impl ReapReport { #[cfg(feature = "acp")] pub async fn reap_orphans(seat: &str) -> Result { let mut report = ReapReport::default(); + let client = DockerCli::system(); for holder in reapable_holders_live(seat).await? { match run_docker( + &client, ["docker", "rm", "--force", "--volumes", holder.as_str()] .into_iter() .map(String::from) @@ -556,18 +2004,361 @@ pub async fn reap_orphans(seat: &str) -> Result { Ok(report) } +/// Per-command bound for the periodic sweep's docker calls. +/// +/// Shorter than [`DOCKER_DEADLINE`] on purpose. That bound sizes the calls a *job launch* depends +/// on, where waiting two minutes beats failing the job. The sweep depends on nothing and is retried +/// every tick, so a docker daemon that has stopped answering should cost this loop twenty seconds +/// and be tried again later, not hold the seller's cadence for two minutes to reach the same +/// conclusion. +#[cfg(feature = "acp")] +pub const SWEEP_DOCKER_DEADLINE: std::time::Duration = std::time::Duration::from_secs(20); + +/// How often the seller's run loop sweeps expired containers: **every five minutes**. +/// +/// Not gated on `acp`, because the run loop schedules the tick on every build and only the docker +/// work behind it needs the feature. +/// +/// **The cadence is the honest half of the tradeoff.** A container is removed no earlier than its +/// own job's deadline plus [`CLEANUP_GRACE_SECS`], and no later than that plus one interval — so the +/// worst case an operator should expect is deadline + 1 h + 5 min, and longer if docker or the +/// daemon itself was down, because a sweep that could not list is retried rather than assumed. Five +/// minutes is chosen against the grace it serves: a cadence near the grace would make the *interval* +/// the dominant term in how long a leftover survives, and a much tighter one would spend a `docker +/// ps` on an idle host every few seconds to discover, almost always, nothing. Against a 3600 s grace +/// this adds at most 8% to the wait and costs one listing per five minutes. +pub const SWEEP_INTERVAL_SECS: u64 = 300; + +/// How many expired containers a single sweep will remove before leaving the rest to the next one. +/// +/// **Bounded work, so the leak cannot become the outage.** A host that accumulated hundreds of +/// leftovers — a crash loop, a docker daemon down for a day — would otherwise hand this loop an +/// unbounded queue of removals on one tick, and the seller stops answering offers while it drains. +/// The remainder is not lost: it is still expired on the next tick, and the sweep is periodic. A +/// backlog clears across several minutes instead of blocking one. +#[cfg(feature = "acp")] +pub const MAX_SWEEP_REMOVALS: usize = 32; + +/// The WALL-CLOCK bound on one whole sweep pass, listing included. +/// +/// [`MAX_SWEEP_REMOVALS`] bounds the COUNT, and it is not by itself a bound on duration: the +/// removals run one after another, each with its own [`SWEEP_DOCKER_DEADLINE`], so a daemon that +/// accepts the connection and then hangs turns a 32-removal pass into 32 × 20 s = 640 s of work +/// — past two further ticks. That is the case this budget names. When it is spent the pass stops +/// starting removals and reports what it did not attempt as [`ReapReport::deferred`]; those +/// containers are still expired, so the next tick selects them again. +/// +/// 240 s is chosen under [`SWEEP_INTERVAL_SECS`] deliberately: a pass therefore ends before the +/// tick that follows it, so the pathological case degrades into "fewer removals per pass" rather +/// than into overlapping passes. The guaranteed floor is what one budget buys at the per-call +/// deadline — at least 11 removals per pass even when every single call burns its full 20 s, and +/// the usual 32 when they answer in milliseconds. +#[cfg(feature = "acp")] +pub const SWEEP_PASS_BUDGET: std::time::Duration = std::time::Duration::from_secs(240); + +/// Remove this seat's containers whose own cleanup stamp `now_unix` has passed, and report what +/// happened to each. +/// +/// This is the replacement for keeping an owner alive in memory until every container is confirmed +/// gone. Nothing here remembers a job: the stamp written at create time is the whole record, so a +/// container that appeared **after** the process that asked for it had exited is discovered by the +/// next sweep exactly like any other, and a seller that was `SIGKILL`ed mid-job cleans up after its +/// own restart. +/// +/// **A failed listing is never an empty one.** Both reads return `Err` rather than an empty +/// selection, because "docker did not answer" and "nothing is expired" are the same value to a +/// caller that only counts removals — and treating the first as the second is how a sweep reports +/// success for a host it never looked at. +/// +/// **A failed removal is retried by the NEXT sweep, not here.** The container stays expired, so the +/// following tick selects it again. Retrying in place would spend this tick's bounded budget on a +/// container docker has already refused once. +#[cfg(feature = "acp")] +pub async fn sweep_expired(seat: &str, now_unix: u64) -> Result { + sweep_expired_with(&DockerCli::system(), seat, now_unix).await +} + +/// [`sweep_expired`], with the docker client supplied by the caller. +/// +/// Private for the same reason [`establish_with`] is: a test hands in a stand-in as an ARGUMENT, so +/// the substitution is confined to the call under test and two such tests can run in parallel +/// without sharing any global. +#[cfg(feature = "acp")] +async fn sweep_expired_with( + client: &DockerCli, + seat: &str, + now_unix: u64, +) -> Result { + sweep_expired_within(client, seat, now_unix, SWEEP_PASS_BUDGET).await +} + +/// [`sweep_expired_with`], with the pass budget supplied by the caller so a test can spend it. +#[cfg(feature = "acp")] +async fn sweep_expired_within( + client: &DockerCli, + seat: &str, + now_unix: u64, + budget: std::time::Duration, +) -> Result { + // Refused rather than run on an identity we do not have: an empty seat would match every + // container whose seat label failed to parse. Same refusal, same reason, as + // `reapable_holders_live`. + if seat.trim().is_empty() { + return Err("refusing to sweep: no owning seat was named".to_owned()); + } + // Started BEFORE the listing, because the listing is part of the pass a hung daemon can stall: + // a budget measured from the first removal would let one stuck `docker ps` spend 20 s that the + // caller's cadence never accounted for. + let started = std::time::Instant::now(); + let mut report = ReapReport::default(); + let (listing, _) = run_bounded(client, list_owned_argv(seat), None, SWEEP_DOCKER_DEADLINE) + .await + .map_err(|error| format!("could not list this seat's containers — {error}"))?; + let selection = partition_owned(&parse_owned_listing(&listing), seat, now_unix); + // Carried out of the pass whatever else happens: these are the containers the sweep cannot + // authorise itself to remove, and they are invisible to the operator anywhere else. + report.skipped = selection.skipped; + let mut candidates = selection.removable; + // Sorted so the rotation below walks a stable order rather than docker's listing order, which + // is free to differ between passes and would make "resume after" meaningless. + candidates.sort(); + candidates.dedup(); + // The count bound, applied from where the LAST pass stopped rather than from the front. See + // `select_pass`: starting at the front every time is what let 32 permanently-failing removals + // monopolise every pass while the 33rd container waited forever. + let cursor = sweep_cursor_for(seat); + let (attempt, deferred) = select_pass(&candidates, cursor.as_deref(), MAX_SWEEP_REMOVALS); + report.deferred = deferred; + let mut last_attempted: Option = None; + let mut queue = attempt.into_iter(); + for id in queue.by_ref() { + // Checked before STARTING a removal, never mid-call: a `docker rm` this pass has already + // issued is left to its own deadline, because abandoning it would leave the pass unable to + // say whether the container was removed. + if started.elapsed() >= budget { + // In front of whatever the count bound already deferred: these were selected earlier, + // so they are older, and the next pass should reach them first. + let unattempted: Vec = std::iter::once(id).chain(queue).collect(); + report.deferred.splice(0..0, unattempted); + break; + } + // Recorded BEFORE the outcome is known, and kept whether docker accepts or refuses: a + // cursor that only advanced past successes would park on a container docker always refuses + // and reproduce the starvation this replaced. + last_attempted = Some(id.clone()); + match run_bounded( + client, + ["docker", "rm", "--force", "--volumes", id.as_str()] + .into_iter() + .map(String::from) + .collect(), + None, + SWEEP_DOCKER_DEADLINE, + ) + .await + { + Ok(_) => report.removed.push(id), + // Collected and carried past, exactly as the boot reaper does: one container docker + // refuses must not stop the rest, and the failure is returned rather than dropped so + // the caller can say so in its log. + Err(error) => report.failed.push((id, error)), + } + } + // Advanced only past what this pass ACTUALLY asked docker about — never past what the count + // bound or the budget deferred, which no pass has attempted yet. + if let Some(last) = last_attempted { + record_sweep_cursor(seat, &last); + } + Ok(report) +} + /// Run a `docker` argv to completion, optionally feeding `stdin`, and return `(stdout, stderr)`. /// /// `std::process::Command` on a blocking pool thread, not `tokio::process`: this crate's tokio is /// built without the `process` feature, and reaching for it would widen the dependency of every /// default build to enable three calls that happen once per job. #[cfg(feature = "acp")] -async fn run_docker(argv: Vec, stdin: Option) -> Result<(String, String), String> { - tokio::task::spawn_blocking(move || { - use std::io::Write; +async fn run_docker( + client: &DockerCli, + argv: Vec, + stdin: Option, +) -> Result<(String, String), String> { + run_bounded(client, argv, stdin, DOCKER_DEADLINE).await +} + +/// As [`run_docker`], but the create it issues is **fenced**: the ticket lives inside the blocking +/// closure, so cleanup cannot remove ahead of a create that outlived the future awaiting it. +/// +/// The ticket is deliberately not held by this future. Holding it here would release it on +/// cancellation — at precisely the moment the create is still running — which is the bug. +#[cfg(feature = "acp")] +async fn run_docker_fenced( + client: &DockerCli, + argv: Vec, + stdin: Option, + ticket: CreationTicket, +) -> Result<(String, String), String> { + let client = client.clone(); + // Taken HERE, on the caller's side of the queue. `spawn_blocking` hands work to a pool that can + // be saturated, and a clock started inside the closure cannot see the time spent waiting for a + // thread -- so a create could sit queued for longer than its own deadline and still be handed a + // full budget on arrival. The bound is measured from the moment the work was ASKED for. + let queued_at = std::time::Instant::now(); + let joined = tokio::task::spawn_blocking(move || { + // Moved in, and dropped only when this closure ends: killed on the deadline, failed, or + // finished. That drop is what "settled" means to `CreationFence::wait_until_settled`. + let _ticket = ticket; + let mut child_exited = false; + run_bounded_blocking( + &client, + argv, + stdin, + DOCKER_DEADLINE, + queued_at, + Some(_ticket.fence()), + &mut child_exited, + ) + }) + .await; + match joined { + Ok(outcome) => outcome, + Err(error) => Err(format!("docker task panicked: {error}")), + } +} + +/// Run an argv to completion with a **wall-clock bound**, optionally feeding `stdin`. +/// +/// The bound is the cancellation ownership this module was missing. A `docker` client that never +/// returns holds the launch open for as long as it likes, and while it is blocked in the pool the +/// future above it can be cancelled — leaving a command nobody is waiting for and a container nobody +/// is tracking. Past the deadline the child is killed and the caller gets a failure that names the +/// deadline rather than a hang that names nothing. +#[cfg(feature = "acp")] +async fn run_bounded( + client: &DockerCli, + argv: Vec, + stdin: Option, + deadline: std::time::Duration, +) -> Result<(String, String), String> { + run_bounded_tracked(client, argv, stdin, deadline).await.0 +} + +/// As [`run_bounded`], and also says whether the docker CLIENT was reaped with an exit status. +/// +/// **That flag is not a removal receipt, and nothing downstream may read it as one.** It says one +/// narrow thing: this process waited for the client and got a status back. It is `true` for a clean +/// exit, a nonzero exit, AND a signal-terminated client — every case where `try_wait` yields a +/// status — because all of them mean the same thing here, that the client is no longer running. +/// +/// What it deliberately does NOT mean is that the container is gone. `docker run --rm` asks the +/// daemon to remove the container on the container's own lifecycle; it is not discharged by this +/// process reaping a local client, and on an error path the removal may never have been reached. +/// Promoting "reaped" to "removed" here is what struck live containers off the registry that exists +/// to remove them. The only thing entitled to end custody is a daemon-side absence check — see +/// [`run_sidecar_confirmed`] and [`container_is_absent`]. +/// +/// A client that was never reaped at all (deadline kill before a status, a panicked task) yields +/// `false`, which is weaker still: not even worth asking the daemon about yet. +#[cfg(feature = "acp")] +async fn run_bounded_tracked( + client: &DockerCli, + argv: Vec, + stdin: Option, + deadline: std::time::Duration, +) -> (Result<(String, String), String>, bool) { + run_bounded_tracked_fenced(client, argv, stdin, deadline, None).await +} + +/// As [`run_bounded_tracked`], optionally holding a [`CreationTicket`] for the duration of the +/// blocking work. +/// +/// The ticket exists because registering a name is not the same as fencing a create. Registration +/// tells cleanup WHAT to remove; it says nothing about WHEN the container appears. A sidecar create +/// still in flight when the holder drops would be removed by name, answered "No such container" +/// because it does not exist yet, marked done — and would then land as an orphan pinning the very +/// namespace the holder was trying to tear down. +/// +/// As in [`run_docker_fenced`], the ticket is moved INTO the closure and never held by this future, +/// so cancelling the future cannot release it while the create is still running. +#[cfg(feature = "acp")] +async fn run_bounded_tracked_fenced( + client: &DockerCli, + argv: Vec, + stdin: Option, + deadline: std::time::Duration, + ticket: Option, +) -> (Result<(String, String), String>, bool) { + let client = client.clone(); + // As in [`run_docker_fenced`]: the clock starts before the queue, not after it. + let queued_at = std::time::Instant::now(); + let joined = tokio::task::spawn_blocking(move || { + let _ticket = ticket; + let mut child_exited = false; + let outcome = run_bounded_blocking( + &client, + argv, + stdin, + deadline, + queued_at, + _ticket.as_ref().map(CreationTicket::fence), + &mut child_exited, + ); + (outcome, child_exited) + }) + .await; + match joined { + Ok(pair) => pair, + // A panicked task establishes nothing about the container either. + Err(error) => (Err(format!("docker task panicked: {error}")), false), + } +} + +/// The blocking half of [`run_bounded_tracked`]. Sets `child_exited` the moment the child is reaped. +#[cfg(feature = "acp")] +fn run_bounded_blocking( + client: &DockerCli, + argv: Vec, + stdin: Option, + deadline: std::time::Duration, + queued_at: std::time::Instant, + fence: Option<&std::sync::Arc>, + child_exited: &mut bool, +) -> Result<(String, String), String> { + { + use std::io::{Read, Write}; use std::process::{Command, Stdio}; - let (program, args) = argv.split_first().expect("a docker argv is never empty"); + let (program, args) = argv.split_first().expect("an argv is never empty"); + // Substituted at the SPAWN site, not in the argv builders: every rendered argv still reads + // `docker ...`, so what the plan tests assert is what production runs. + let program = + if program == "docker" { client.program().to_owned() } else { program.clone() }; + let program = program.as_str(); + // The clock was started by the CALLER, before this work was queued, and every wait below is + // measured against it: queue time, spawn, plan write, child wait, output drain and writer + // join all spend the same budget. + // + // Anchoring it after the stdin write left that write outside the bound entirely, and + // anchoring it inside this closure left the queue wait outside it. What is bounded here is + // exactly this process's flow; it is NOT a statement about when the daemon finishes creating + // a container, which only a daemon-side absence check can settle. + let started = queued_at; + // REFUSED BEFORE IT IS ISSUED, not bounded after it. + // + // The budget can already be gone before this closure runs at all: it sat in the blocking + // pool's queue, and queue time spends the same clock as every wait below. Spawning anyway + // starts a create whose caller is ALREADY past its bound -- the container can land with + // nothing waiting on it, which is the orphan this module exists to prevent, issued + // knowingly. Bounding the wait afterwards cannot help: by then the create exists. The only + // correct answer at this point is to not start it. + if started.elapsed() >= deadline { + return Err(format!( + "`{program}` was NOT started: its {}s budget was already spent while the work sat \ + queued for a blocking thread, so no create was issued — starting one here would \ + launch a container whose caller is already past its bound", + deadline.as_secs(), + )); + } let mut child = Command::new(program) .args(args) .stdin(if stdin.is_some() { Stdio::piped() } else { Stdio::null() }) @@ -575,32 +2366,414 @@ async fn run_docker(argv: Vec, stdin: Option) -> Result<(String, .stderr(Stdio::piped()) .spawn() .map_err(|error| format!("could not run `{program}`: {error}"))?; - if let Some(plan) = stdin { - child - .stdin - .as_mut() - .ok_or_else(|| "docker stdin was not piped".to_string())? - .write_all(plan.as_bytes()) - .map_err(|error| format!("could not write the plan to the sidecar: {error}"))?; - // Dropped so the sidecar's `read` loop sees EOF; without this it waits forever and the - // job's launch hangs instead of failing. - drop(child.stdin.take()); - } - let done = child - .wait_with_output() - .map_err(|error| format!("could not wait for `{program}`: {error}"))?; + // How much of the budget is left, measured from the caller's pre-queue clock. Every wait + // below asks this rather than starting a fresh one, so no step can quietly extend the bound. + let remaining = || deadline.saturating_sub(started.elapsed()); + + // Written on its own thread so a blocked write cannot outrun the deadline, and its result + // comes back through a CHANNEL rather than a `JoinHandle`. + // + // `JoinHandle::join` has no timeout. The old code joined it unconditionally, reasoning that + // reaping the child closes the read end -- but a descendant started by the client inherits + // that end and can hold it open, so the join could block after the bounded wait had already + // returned. A channel can be waited on WITH the remaining budget; the thread itself cannot + // be killed (Rust has no such thing), so when it outlives the bound it is NAMED instead of + // being silently dropped. + let (wrote_tx, wrote_rx) = std::sync::mpsc::channel::>(); + let writing = match stdin { + Some(plan) => { + let mut pipe = + child.stdin.take().ok_or_else(|| "docker stdin was not piped".to_string())?; + // The writer takes a ticket of its OWN, and holds it until the write ends. + // + // Reporting that a writer is still running was never the same as owning it. The + // channel timeout below lets this CALL end; the thread keeps the pipe endpoint + // either way, and while it was ticketless the closure could return, release the + // only ticket, and let the fence read as settled with a write still in progress. + // Cleanup would then be free to remove against a create that had not finished + // being written to. Now the fence cannot reach zero while this thread exists. + let ticket = fence.map(|fence| fence.begin()); + std::thread::spawn(move || { + let _ticket = ticket; + let outcome = pipe.write_all(plan.as_bytes()).map_err(|error| { + format!("could not write the plan to the sidecar: {error}") + }); + let _ = wrote_tx.send(outcome); + }); + true + } + None => false, + }; + + // Poll rather than `wait_with_output`, so the deadline is enforceable at all. + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break status, + Ok(None) => {} + Err(error) => { + return Err(format!("could not wait for `{program}`: {error}")); + } + } + if started.elapsed() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + // The daemon's answer to this create was never read, so whether the request was + // applied is unknown to this process. Nothing here records that: the create carried + // this job's own expiry as a label, so a container that lands after this point is + // removed by a later `sweep_expired` on the strength of that label alone. + // The writer is settled HERE too, not abandoned. Killing the child closes the read + // end, so a blocked `write_all` fails with `EPIPE` and the thread ends on its own; + // this waits a short, explicit grace for exactly that and reports the writer as + // still running when it does not arrive. Dropping the handle instead is how this + // flow used to end "complete" while a write was still in progress. + // Whichever way it goes, the writer's disposition is STATED. The failure this + // replaces was silence: the handle was dropped on the way out, so a caller could + // not tell a writer that had finished from one still pushing a plan into a pipe. + // Both answers are legitimate; not having asked is not. + let writer = if !writing { + "; no plan was being written" + } else if wrote_rx.recv_timeout(WRITER_EPIPE_GRACE).is_ok() { + "; the thread writing its plan was settled after the kill" + } else { + "; the thread writing its plan is STILL RUNNING in this process and could not \ + be joined within the grace after the kill" + }; + return Err(format!( + "`{program}` did not finish within {}s and was killed — a command with no bound \ + is a launch that can hang and a container nobody is waiting for{writer}", + deadline.as_secs(), + )); + } + std::thread::sleep(std::time::Duration::from_millis(20)); + }; + // Reaped with a status — ANY status, including a signal termination, which `code()` reports + // as `None` below. This flag means only "the client is no longer running", never "the + // container is gone": `--rm` is discharged by the daemon on the container's lifecycle, not + // by this process waiting on a client. The caller must still confirm absence with the + // daemon before ending custody. + *child_exited = true; + // Reaping the child does NOT close its pipes. A descendant it started inherits the write + // ends and can hold them open indefinitely, and `read_to_end` returns at EOF -- precisely + // what such a descendant withholds. Draining on this thread therefore put an UNBOUNDED wait + // directly after the bounded one, which is the hole this replaces: the drains run on their + // own threads and are collected against the same budget as everything above. + // Each reader sends its RESULT, not a buffer: a read that failed is an output this process + // did not get, and it is recorded as unknown rather than as "the client said nothing". + let (drained_tx, drained_rx) = + std::sync::mpsc::channel::<(&'static str, std::io::Result>)>(); + let mut pending: Vec<&'static str> = Vec::new(); + // Each drain takes its OWN ticket, for the same reason the writer does: a descendant can + // hold these endpoints open long past the channel timeout below, and a reader still blocked + // on a pipe is work this process is still doing. Ticketless, it was work nobody was counted + // for -- the closure returned, the last ticket went with it, and the fence said settled + // while two threads still held the create's output. The ticket is released when the read + // ends, not when this call does. + if let Some(mut pipe) = child.stdout.take() { + let tx = drained_tx.clone(); + pending.push("stdout"); + let ticket = fence.map(|fence| fence.begin()); + std::thread::spawn(move || { + let _ticket = ticket; + let mut buffer = Vec::new(); + let outcome = pipe.read_to_end(&mut buffer).map(|_| buffer); + let _ = tx.send(("stdout", outcome)); + }); + } + if let Some(mut pipe) = child.stderr.take() { + let tx = drained_tx.clone(); + pending.push("stderr"); + let ticket = fence.map(|fence| fence.begin()); + std::thread::spawn(move || { + let _ticket = ticket; + let mut buffer = Vec::new(); + let outcome = pipe.read_to_end(&mut buffer).map(|_| buffer); + let _ = tx.send(("stderr", outcome)); + }); + } + drop(drained_tx); + let mut stdout = Vec::new(); + // `None` until stderr is read TO EOF WITHOUT ERROR. A stream still held by a descendant, or + // a read that failed, leaves this unknown — and an unknown stderr is never read as "the + // daemon refused". + let mut stderr_read: Option> = None; + while !pending.is_empty() { + match drained_rx.recv_timeout(remaining()) { + Ok((which, outcome)) => { + pending.retain(|name| *name != which); + match (which, outcome) { + ("stdout", Ok(buffer)) => stdout = buffer, + ("stdout", Err(error)) => { + eprintln!("sandbox: could not read `{program}` stdout: {error}"); + } + (_, Ok(buffer)) => stderr_read = Some(buffer), + (_, Err(error)) => { + eprintln!( + "sandbox: could not read `{program}` stderr: {error} — its answer, \ + if it gave one, is unknown to this process" + ); + } + } + } + Err(_) => break, + } + } + let stderr_known = stderr_read + .as_deref() + .map(|bytes| String::from_utf8_lossy(bytes).trim().to_owned()); + let done = std::process::Output { + status, + stdout, + stderr: stderr_read.clone().unwrap_or_default(), + }; let stdout = String::from_utf8_lossy(&done.stdout).trim().to_owned(); - let stderr = String::from_utf8_lossy(&done.stderr).trim().to_owned(); + let stderr = stderr_known.clone().unwrap_or_default(); + // CLASSIFIED HERE, before any early return below, and once. The client has ended; whether + // the daemon ANSWERED it — accepted, or refused — is decided from positive proof only: exit + // 0, the daemon's own refusal text in a stderr this process read to EOF, or a `run` whose + // exit code is the contained command's. Everything else — a signal, a client-side exit with + // no daemon text, a stderr still held by a descendant, a failed read — is an outcome this + // process does not know, and the name is recorded as unanswered so absence is not taken as + // proof for it. The previous flow returned on a pending drain BEFORE reaching its + // classification, so a reaped client whose pipe a descendant held was never recorded at all, + // and it recognised only eight stderr substrings as "lost the daemon", reading every other + // text as a refusal. + // A half-written plan is a sidecar that acted on a truncated instruction, so the write's own + // failure is reported -- but only when the child itself did not already fail, because the + // child's exit code names the refusal more precisely than a broken pipe does. Waited on with + // what is left of the budget, never unconditionally. + let mut writer_outstanding = false; + let wrote = if writing { + match wrote_rx.recv_timeout(remaining()) { + Ok(result) => result, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + writer_outstanding = true; + Err(format!( + "the plan was still being written to `{program}` when the {}s bound expired \ + -- the writing thread is still running in this process, so this call ends \ + on its bound rather than reporting a completed write", + deadline.as_secs() + )) + } + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + Err("the thread writing the plan to the sidecar panicked".to_string()) + } + } + } else { + Ok(()) + }; + // An output this process never finished reading is not an output it may report. Naming the + // stream and the still-running reader is the honest end; inventing a truncated success is + // how a caller comes to believe a create said something it never said. + if !pending.is_empty() { + return Err(format!( + "`{program}` was reaped, but its {} did not reach EOF within the {}s bound — a \ + descendant is holding the pipe open, so this call ends on its bound; the reading \ + thread(s) remain outstanding in this process{}", + pending.join(" and "), + deadline.as_secs(), + if writer_outstanding { + ", as does the plan writer" + } else { + "" + } + )); + } match done.status.code() { - Some(0) => Ok((stdout, stderr)), + Some(0) => match wrote { + Ok(()) => Ok((stdout, stderr)), + Err(error) => Err(error), + }, // The sidecar's codes are an interface; pass them through in the message so the caller's - // error names WHICH refusal happened rather than "it failed". - Some(code) => Err(format!("exit {code}: {}", if stderr.is_empty() { &stdout } else { &stderr })), + // error names WHICH refusal happened rather than "it failed". Whether the daemon + // answered was decided above, before the drain check, from positive proof. + Some(code) => { + Err(format!("exit {code}: {}", if stderr.is_empty() { &stdout } else { &stderr })) + } None => Err("killed by a signal".to_string()), } - }) - .await - .map_err(|error| format!("docker task panicked: {error}"))? + } +} + +/// A unique name for one temporary container joined to `holder`'s namespace. +/// +/// Unique per process and per call, so nothing here can address — or remove — a container belonging +/// to another run. +pub fn sidecar_name(holder: &str, verb: &str) -> String { + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT: AtomicU64 = AtomicU64::new(0); + let serial = NEXT.fetch_add(1, Ordering::Relaxed); + format!("{holder}-{verb}-{}-{serial}", std::process::id()) +} + +/// Give a `docker run` argv an explicit container name. +/// +/// An unnamed sidecar cannot be cleaned up after a cancellation: docker assigns it a random name +/// this process never learns, so the one container capable of pinning the namespace open is the one +/// container nothing can address. +pub fn with_container_name(mut argv: Vec, name: &str) -> Result, String> { + match argv.get(1).map(String::as_str) { + Some("run") => { + argv.splice(2..2, ["--name".to_owned(), name.to_owned()]); + Ok(argv) + } + other => Err(format!( + "refusing to name {other:?} as a container: this is not a `docker run` argv, and naming \ + the wrong command would register a cleanup target that does not exist" + )), + } +} + +/// Splice `labels` into a `docker run` argv, as [`with_container_name`] splices the name, and for +/// the same reason: the argv builders stay pure and independently testable, and the one place that +/// knows the owning job decorates what they produced. +/// +/// Refuses anything that is not a `docker run` argv. Labels on a `docker ps` would be read as +/// filters, and a cleanup path that quietly changed the meaning of a command is worse than one that +/// stops. An empty `labels` is returned unchanged: a fixture holder with no job to attribute to +/// stamps nothing rather than stamping emptiness. +pub fn with_helper_labels(mut argv: Vec, labels: &[String]) -> Result, String> { + if labels.is_empty() { + return Ok(argv); + } + match argv.get(1).map(String::as_str) { + Some("run") => { + argv.splice(2..2, labels.iter().cloned()); + Ok(argv) + } + other => Err(format!( + "refusing to label {other:?} as a job-owned helper: this is not a `docker run` argv, and \ + labels spliced into another verb would change what that command means" + )), + } +} + +/// Run one sidecar joined to the holder's namespace: named, registered for its lifetime, bounded. +#[cfg(feature = "acp")] +async fn run_sidecar( + holder: &NetnsHolder, + verb: &str, + argv: Vec, + stdin: Option, +) -> Result<(String, String), String> { + run_sidecar_with_deadline(holder, verb, argv, stdin, DOCKER_DEADLINE).await +} + +/// As [`run_sidecar`], with the bound named by the caller. +/// +/// The deadline is a parameter solely so the custody rule below can be measured offline. A test +/// cannot wait out the production bound, and a rule about what happens when the client is killed is +/// worth nothing if the only thing measured is the flag feeding it. +#[cfg(feature = "acp")] +async fn run_sidecar_with_deadline( + holder: &NetnsHolder, + verb: &str, + argv: Vec, + stdin: Option, + deadline: std::time::Duration, +) -> Result<(String, String), String> { + run_sidecar_confirmed(holder, verb, argv, stdin, deadline, container_is_absent).await +} + + +/// How custody asks whether a container is gone. `Some(true)` = confirmed absent, `Some(false)` = +/// confirmed present, `None` = could not be established. +/// +/// Injected so the rule below is measurable without a daemon. Only `Some(true)` releases custody, so +/// a confirmer that cannot tell is treated exactly like one that says "still there". +#[cfg(feature = "acp")] +type ConfirmAbsent = fn(&DockerCli, &str) -> Option; + +/// As [`run_sidecar_with_deadline`], with the absence check injected. +/// +/// **Custody ends on confirmed absence, and on nothing else.** +/// +/// The previous rule ended it on a reaped client, reasoning that `docker run --rm` removes the +/// container when its client exits. That reasoning describes the happy path and quietly covers the +/// failure paths with it. A client reaped with a nonzero status, a stdin write that failed before +/// the wait, a client killed on the deadline — each returns from the same call, and none of them is +/// the DAEMON confirming the container is gone. `--rm` is a request to the daemon, not a receipt +/// from it: removal can still be queued, in progress, or refused, and on an error path it may never +/// have been reached at all. Deregistering on the client's say-so struck live containers off the +/// registry that exists to remove them. +/// +/// So the client's exit is now only a reason to ASK. The answer comes from docker, and a confirmer +/// that cannot answer keeps the name a cleanup target — the cost of which is one `docker rm` +/// replying "No such container", which cleanup already treats as success. +#[cfg(feature = "acp")] +async fn run_sidecar_confirmed( + holder: &NetnsHolder, + verb: &str, + argv: Vec, + stdin: Option, + deadline: std::time::Duration, + confirm_absent: ConfirmAbsent, +) -> Result<(String, String), String> { + let name = sidecar_name(holder.name(), verb); + let argv = with_container_name(argv, &name)?; + // Stamped here, at the one funnel every helper in this module passes through, rather than in the + // individual argv builders. A helper missing the stamp is a container the sweep can never judge. + let argv = with_helper_labels(argv, holder.helper_labels())?; + // Registered BEFORE the command starts: a cancellation between these two lines must still leave + // a cleanup target behind, and registering afterwards would not. + let mut registration = holder.watch_sidecar(name.clone()); + // Registration says WHAT to remove; the ticket says WHEN it is safe to. Without it, a holder + // dropped while this create is in flight removes the name, is told "No such container" because + // the container does not exist YET, treats that as done — and the create then lands as an + // orphan pinning the namespace. Moved into the blocking closure, never held by this future. + let (outcome, child_exited) = run_bounded_tracked_fenced( + holder.client(), + argv, + stdin, + deadline, + Some(holder.fence_creation()), + ) + .await; + // Reaching this line at all proves the command is no longer in flight: a cancellation drops the + // future before it, so a cancelled command's name stays a cleanup target. + // + // A client that was never reaped is not even worth asking about — the daemon may still be + // creating or running that container — so custody is simply kept. + if child_exited { + let asked = name.clone(); + let client = holder.client().clone(); + let absent = tokio::task::spawn_blocking(move || confirm_absent(&client, &asked)) + .await + .unwrap_or(None); + if absent == Some(true) { + registration.completed(); + } + } + drop(registration); + outcome +} + +/// Ask docker whether a container name is gone. +/// +/// `Some(true)` only for docker saying the object does not exist. A successful inspect is +/// `Some(false)`: the container is still there. Anything else — docker missing, the daemon not +/// answering, an unrecognised error — is `None`, which keeps custody. +fn container_is_absent(client: &DockerCli, name: &str) -> Option { + let mut child = std::process::Command::new(client.program()) + .args(["inspect", "--type", "container", "--format", "{{.Id}}", name]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn() + .ok()?; + let status = NetnsHolder::wait_bounded(&mut child, NetnsHolder::REMOVE_DEADLINE).ok()?; + let mut stderr_bytes = Vec::new(); + if let Some(mut pipe) = child.stderr.take() { + use std::io::Read as _; + let _ = pipe.read_to_end(&mut stderr_bytes); + } + if status.success() { + return Some(false); + } + let stderr = String::from_utf8_lossy(&stderr_bytes); + if NetnsHolder::force_remove_stderr_is_benign(&stderr) || stderr.contains("No such object") { + Some(true) + } else { + None + } } /// Establish containment for one job: measure the proxy address, create the namespace holder, install @@ -629,38 +2802,102 @@ pub async fn establish( proxy_ports: Option, log_connections: bool, dns_resolvers: Vec, + cleanup_after: u64, ) -> Result { - // Measured BEFORE the holder exists, so a probe failure needs no cleanup. - let (probe_stdout, _) = run_docker(host_gateway_probe_argv(sidecar_image, proxy_alias), None) - .await - .map_err(|error| format!("could not resolve {proxy_alias} for the pinhole — {error}"))?; - let proxy_host = parse_getent_ipv4(&probe_stdout).ok_or_else(|| { - format!("resolving {proxy_alias} produced no IPv4 address (got {probe_stdout:?})") - })?; - - let name = holder_name(job_id); - run_docker(holder_argv(&name, network, holder_image, uid, gid, job_id, seat), None) - .await - .map_err(|error| format!("could not start the netns holder {name} — {error}"))?; - // From here on the container exists, so every early return must tear it down. Adopting it into - // the guard immediately is what makes that automatic rather than remembered. - let holder = NetnsHolder::adopt(name); - - // The resolvers arrive from the caller rather than being discovered here, and that is the one - // property that keeps the job's `/etc/resolv.conf` and this policy in agreement: the caller - // resolves once, writes that file from the result, and hands the same addresses here. Two - // discoveries could disagree and the job would be pointed at a resolver its own firewall drops. - let policy = NetPolicy { - gateway: proxy_host.clone(), + // The production client is named here, once, and threaded down. This is the ONLY constructor a + // shipped build can reach, and it takes no input: no environment variable, no config field, no + // global. A test that needs a stand-in calls `establish_with` and hands one in. + establish_with( + &DockerCli::system(), + FenceBounds::production(), + network, + holder_image, + sidecar_image, + proxy_alias, + job_id, + seat, + uid, + gid, proxy_ports, log_connections, dns_resolvers, - }; - let (plan, expected) = plan_stdin(&policy); - let (applied, _) = run_docker(sidecar_argv(&holder, sidecar_image), Some(plan)) - .await - .map_err(|error| format!("containment was not installed — {error}"))?; - + cleanup_after, + ) + .await +} + +/// [`establish`], with the docker client and the cleanup bounds supplied by the caller. +/// +/// Private, and the only way to supply either. Tests pass a stand-in here as an ARGUMENT, so the +/// substitution is confined to the one call under test: nothing is installed anywhere another test +/// could read it, no lock has to be remembered, and two such tests can run in parallel without +/// seeing each other. +#[cfg(feature = "acp")] +#[allow(clippy::too_many_arguments)] +async fn establish_with( + client: &DockerCli, + bounds: FenceBounds, + network: &str, + holder_image: &str, + sidecar_image: &str, + proxy_alias: &str, + job_id: &str, + seat: &str, + uid: u32, + gid: u32, + proxy_ports: Option, + log_connections: bool, + dns_resolvers: Vec, + cleanup_after: u64, +) -> Result { + // Measured BEFORE the holder exists, so a probe failure needs no cleanup. + let (probe_stdout, _) = + run_docker(client, host_gateway_probe_argv(sidecar_image, proxy_alias), None) + .await + .map_err(|error| format!("could not resolve {proxy_alias} for the pinhole — {error}"))?; + let proxy_host = parse_getent_ipv4(&probe_stdout).ok_or_else(|| { + format!("resolving {proxy_alias} produced no IPv4 address (got {probe_stdout:?})") + })?; + + let name = holder_name(job_id); + // Adopted BEFORE the create is issued, not after it returns. `run_docker` awaits, an await is a + // cancellation point, and the blocking create can complete after the future above it is gone: + // adopting afterwards left exactly that container running with no guard and no record. The guard + // costs one `docker rm` that reports "No such container" when the create never happened. + let mut holder = NetnsHolder::adopt_bounded(name.clone(), client.clone(), bounds); + // Every helper joined to this namespace carries the SAME seat and the SAME cleanup stamp as the + // holder created just below, so the sweep expires a job's containers as one set rather than + // leaving its joiners behind unattributable. + holder.label_helpers(helper_label_args(job_id, seat, cleanup_after)); + // Fenced, not merely adopted. The ticket is taken before the create is issued and travels into + // the blocking closure, so a cancellation here leaves cleanup waiting for the create to settle + // instead of racing it to a "No such container" that means "not yet". + let ticket = holder.fence_creation(); + run_docker_fenced( + client, + holder_argv(&name, network, holder_image, uid, gid, job_id, seat, cleanup_after), + None, + ticket, + ) + .await + .map_err(|error| format!("could not start the netns holder {name} — {error}"))?; + + // The resolvers arrive from the caller rather than being discovered here, and that is the one + // property that keeps the job's `/etc/resolv.conf` and this policy in agreement: the caller + // resolves once, writes that file from the result, and hands the same addresses here. Two + // discoveries could disagree and the job would be pointed at a resolver its own firewall drops. + let policy = NetPolicy { + gateway: proxy_host.clone(), + proxy_ports, + log_connections, + dns_resolvers, + }; + let (plan, expected) = plan_stdin(&policy); + let (applied, _) = + run_sidecar(&holder, "iptables", sidecar_argv(&holder, sidecar_image), Some(plan)) + .await + .map_err(|error| format!("containment was not installed — {error}"))?; + // The count cross-check. A truncated stdin applies cleanly and exits 0, so no exit code reveals // it; only comparing the sidecar's own total against what was rendered does. let applied: usize = applied @@ -680,17 +2917,113 @@ pub async fn establish( // Both families are checked, and a v6 failure is as fatal as a v4 one: an unfiltered address family // is the cheapest bypass there is. for family in [Family::V4, Family::V6] { - let (readback, _) = run_docker(readback_argv(holder.name(), sidecar_image, family), None) - .await - .map_err(|error| { - format!("could not read {} rules back from the namespace — {error}", family.binary()) - })?; + let (readback, _) = run_sidecar( + &holder, + "iptables-readback", + readback_argv(holder.name(), sidecar_image, family), + None, + ) + .await + .map_err(|error| { + format!("could not read {} rules back from the namespace — {error}", family.binary()) + })?; policy.verify_readback(family, &readback).map_err(|error| { format!("containment did not verify after installation — {error}") })?; } - Ok(Containment { holder, proxy_host }) + // ── The interface the packets actually leave by ─────────────────────────────────────────── + // + // Everything above installs and verifies rules on the host kernel's `OUTPUT` chain, and a gVisor + // payload never traverses it: `runsc` runs its own netstack and hands finished packets straight + // to the namespace's veth. The readback above is entirely honest and the job is still uncontained + // — measured on this repo's fixtures, both families, over TCP. + // + // So the same rendered policy is translated onto the veth itself, and unconditionally rather than + // only for a `runsc` job: `establish` is not told which runtime the caller will launch under, and + // "contained under one runtime" is exactly the state being closed here. Under `runc` the filters + // are redundant with the chain above, which costs one qdisc and a handful of filters per job. + // + // Same failure discipline as the chain above: no partial success, no retry. Every `?` from here + // leaves through the holder guard, which destroys the namespace on the way out. + let dev = egress_device(&holder, sidecar_image).await?; + let iface = crate::sandbox_iface::IfacePlan::derive(&dev, &policy) + .map_err(|error| format!("the egress filter plan for {dev} could not be rendered — {error}"))?; + let (iface_plan, iface_expected) = crate::sandbox_iface::plan_stdin(&iface); + let (iface_applied, _) = run_sidecar( + &holder, + "iface", + crate::sandbox_iface::iface_sidecar_argv(holder.name(), sidecar_image), + Some(iface_plan), + ) + .await + .map_err(|error| { + format!( + "egress filters were not installed on {dev} — {error} (the applier's exit 6 means this \ + sidecar image shipped without iproute2, so no job can be contained by this build; its \ + exit 3 means the interface is PARTIALLY filtered and the namespace is being destroyed \ + rather than retried)" + ) + })?; + + // The same count cross-check the chain above does, for the same reason: a truncated stdin applies + // perfectly and exits 0, and only comparing the applier's own total against what was rendered + // reveals it. + let iface_applied: usize = iface_applied.parse().map_err(|_| { + format!("the interface applier reported {iface_applied:?} filters applied, not a number") + })?; + if iface_applied != iface_expected { + return Err(format!( + "egress filtering is incomplete: {iface_applied} of {iface_expected} steps applied on \ + {dev} (the plan was truncated in transit)" + )); + } + + // The readback, from a different container running a different verb, because everything above is + // still the installer's own account of its work. `verify_readback` checks presence, order, both + // families, the exceptions' width and that no drop carries a protocol match — the TCP-only drop + // is the bug this closes, not the fix. + let (iface_readback, _) = run_sidecar( + &holder, + "iface-readback", + crate::sandbox_iface::filter_readback_argv(holder.name(), sidecar_image, &dev), + None, + ) + .await + .map_err(|error| format!("could not read the egress filters back from {dev} — {error}"))?; + iface.verify_readback(&iface_readback).map_err(|error| { + format!("egress filtering did not verify on {dev} after installation — {error}") + })?; + + Ok(Containment { holder, proxy_host, egress_dev: dev }) +} + +/// Which link inside the holder's namespace the job's packets leave by — measured from the +/// namespace's own link list, never assumed to be `eth0`. +/// +/// The probe is an **unprivileged** container (`--cap-drop ALL`, no `NET_ADMIN`): enumerating links +/// is a read, and the one container in this design that can change an interface must not also be the +/// thing that chooses which interface to change. +/// +/// [`crate::sandbox_iface::select_egress_link`] refuses anything that is not a job's own namespace — +/// a bridge among the links, no loopback, or more than one candidate — so a mis-aimed `--network` +/// fails the launch here instead of installing drops on something shared. +#[cfg(feature = "acp")] +async fn egress_device(holder: &NetnsHolder, sidecar_image: &str) -> Result { + let (links, _) = run_sidecar( + holder, + "link-probe", + crate::sandbox_iface::link_probe_argv(holder.name(), sidecar_image), + None, + ) + .await + .map_err(|error| format!("could not enumerate the links in the job's namespace — {error}"))?; + let parsed = crate::sandbox_iface::parse_links(&links).map_err(|error| { + format!("the job's namespace listed a link this build cannot read — {error}") + })?; + let link = crate::sandbox_iface::select_egress_link(&parsed) + .map_err(|error| format!("the job's egress interface could not be identified — {error}"))?; + Ok(link.name) } #[cfg(test)] @@ -800,13 +3133,13 @@ mod tests { #[test] fn the_job_joins_the_holders_namespace_and_never_names_a_network() { - let holder = NetnsHolder::adopt("maxplayer-netns-abc".into()); + let holder = NetnsHolder::adopt("maxplayer-netns-abc".into(), DockerCli::system()); assert_eq!(holder.network_mode(), "container:maxplayer-netns-abc"); } #[test] fn the_holder_runs_sleep_in_exec_form_with_no_shell() { - let argv = holder_argv("h", "net", "img", 1000, 1000, "abc", &seat_b()); + let argv = holder_argv("h", "net", "img", 1000, 1000, "abc", &seat_b(), 2_000_000_000); let tail = &argv[argv.len() - 4..]; assert_eq!(tail, ["--entrypoint", "sleep", "img", "infinity"]); // A shell anywhere in the argv would mean the holder runs something that parses a string. @@ -815,7 +3148,7 @@ mod tests { #[test] fn the_holder_is_locked_down_and_labelled_for_reaping() { - let argv = holder_argv("h", "net", "img", 1000, 1000, "abc", &seat_b()); + let argv = holder_argv("h", "net", "img", 1000, 1000, "abc", &seat_b(), 2_000_000_000); for expected in ["--read-only", "--cap-drop", "ALL", "no-new-privileges"] { assert!(argv.iter().any(|a| a == expected), "missing {expected} in {argv:?}"); } @@ -829,7 +3162,7 @@ mod tests { /// nothing to match and every holder is unattributable — a reaper that correctly reaps nothing. #[test] fn the_holder_carries_the_seat_that_created_it() { - let argv = holder_argv("h", "net", "img", 1000, 1000, "abc", &seat_b()); + let argv = holder_argv("h", "net", "img", 1000, 1000, "abc", &seat_b(), 2_000_000_000); assert!( argv.iter().any(|a| a == &format!("{HOLDER_SEAT_LABEL}={}", seat_b())), "{argv:?}" @@ -844,19 +3177,19 @@ mod tests { #[test] fn only_the_sidecar_is_granted_net_admin() { - let holder = NetnsHolder::adopt("h".into()); + let holder = NetnsHolder::adopt("h".into(), DockerCli::system()); let sidecar = sidecar_argv(&holder, "netfilter"); assert!(sidecar.windows(2).any(|w| w == ["--cap-add", "NET_ADMIN"]), "{sidecar:?}"); // …and it still drops everything else first, so the grant is exactly one capability. assert!(sidecar.windows(2).any(|w| w == ["--cap-drop", "ALL"]), "{sidecar:?}"); // The holder must never carry it: it shares its namespace with the job. - let holder_argv = holder_argv("h", "net", "img", 1000, 1000, "abc", &seat_b()); + let holder_argv = holder_argv("h", "net", "img", 1000, 1000, "abc", &seat_b(), 2_000_000_000); assert!(!holder_argv.iter().any(|a| a == "NET_ADMIN"), "{holder_argv:?}"); } #[test] fn the_sidecar_takes_the_plan_on_stdin_and_is_told_nothing_else() { - let holder = NetnsHolder::adopt("h".into()); + let holder = NetnsHolder::adopt("h".into(), DockerCli::system()); let sidecar = sidecar_argv(&holder, "netfilter"); assert!(sidecar.iter().any(|a| a == "--interactive"), "no stdin: {sidecar:?}"); // The image is the last word — no policy is passed as an argument. @@ -1002,8 +3335,2744 @@ mod tests { dns_resolvers: Vec::new(), }; let (stdin, _) = plan_stdin(&policy); - let accepts: Vec<&str> = stdin.lines().filter(|l| l.contains("ACCEPT")).collect(); - assert_eq!(accepts.len(), 1, "exactly one pinhole: {accepts:?}"); + // The pinhole is v4 — the proxy is reached at the namespace's v4 gateway. The v6 plan also + // carries ACCEPTs, and they are deliberately not pinholes: they are the two neighbour + // discovery exceptions, which name no host and open no port. Matching on "ACCEPT" alone + // would count them here and the assertion would be about arithmetic, not about the pinhole. + let accepts: Vec<&str> = stdin + .lines() + .filter(|l| l.starts_with("iptables ") && l.contains("ACCEPT")) + .collect(); + assert_eq!(accepts.len(), 1, "exactly one v4 pinhole: {accepts:?}"); assert!(accepts[0].contains(&measured), "the pinhole must name the measured host: {accepts:?}"); + let v6_accepts = stdin + .lines() + .filter(|l| l.starts_with("ip6tables ") && l.contains("ACCEPT")) + .count(); + assert_eq!(v6_accepts, 2, "v6 permits neighbour discovery and nothing else"); + } + + // ── Cancellation custody (F4) ───────────────────────────────────────────────────────────── + // + // A cancelled establish must leave nothing running that this process cannot name. These tests + // check the three properties that make that true without a daemon: the sidecar is addressable, + // its name is unique to this run, and the holder tracks it for exactly as long as it is alive. + + /// Every container joined to the namespace is named by us. An unnamed sidecar gets a random name + /// this process never learns, so a cancellation mid-command leaves the one container capable of + /// pinning the namespace open as the one container nothing can address. + #[test] + fn every_sidecar_is_named_so_a_cancelled_one_can_still_be_removed() { + let holder = NetnsHolder::adopt("maxplayer-netns-abc".into(), DockerCli::system()); + let name = sidecar_name(holder.name(), "iface"); + for argv in [ + sidecar_argv(&holder, "netfilter"), + readback_argv(holder.name(), "netfilter", Family::V4), + crate::sandbox_iface::iface_sidecar_argv(holder.name(), "netfilter"), + crate::sandbox_iface::filter_readback_argv(holder.name(), "netfilter", "eth0"), + crate::sandbox_iface::link_probe_argv(holder.name(), "netfilter"), + ] { + let named = with_container_name(argv, &name).expect("a docker run argv"); + assert!( + named.windows(2).any(|w| w == ["--name", name.as_str()]), + "an unnamed joiner cannot be cleaned up: {named:?}" + ); + // The name goes to the docker client, before the image and its command: appended at the + // end it would become an argument to the sidecar instead of a flag to `run`. + let at = named.iter().position(|a| a == "--name").expect("named"); + let image = named.iter().position(|a| a == "netfilter").expect("the image"); + assert!(at < image, "--name must precede the image: {named:?}"); + } + } + + /// The name must be unique per call. A deterministic sidecar name is a name two concurrent jobs + /// share, and cleaning up "the" sidecar would then remove a sibling's live container. + #[test] + fn sidecar_names_are_unique_per_call_so_cleanup_cannot_hit_a_sibling() { + let first = sidecar_name("maxplayer-netns-abc", "iface"); + let second = sidecar_name("maxplayer-netns-abc", "iface"); + assert_ne!(first, second, "two joiners of the same holder must not share a name"); + // Each is still attributable to its holder and its purpose, which is what makes an orphan + // readable to an operator rather than merely unique. + for name in [&first, &second] { + assert!(name.starts_with("maxplayer-netns-abc-iface-"), "{name}"); + } + // Different holders never collide either. + assert_ne!( + sidecar_name("maxplayer-netns-abc", "iface"), + sidecar_name("maxplayer-netns-def", "iface") + ); + } + + /// Naming is refused rather than misapplied. Splicing `--name` into something that is not a + /// `docker run` would register a cleanup target that does not exist, and a cleanup target that + /// does not exist reports success for a container still running. + #[test] + fn naming_a_non_run_argv_is_refused() { + let err = with_container_name(list_all_containers_argv(), "x") + .expect_err("`docker ps` takes no --name"); + assert!(err.contains("not a `docker run` argv"), "{err}"); + assert!(with_container_name(network_modes_argv(&["a".into()]), "x").is_err()); + // The positive control, so the refusal is not simply "always refuse". + let holder = NetnsHolder::adopt("h".into(), DockerCli::system()); + assert!(with_container_name(sidecar_argv(&holder, "img"), "x").is_ok()); + } + + /// A joiner is tracked for exactly its command's lifetime: registered before it starts (a + /// cancellation between registration and start must still leave a cleanup target) and dropped + /// when it finishes, so a completed sidecar is not removed twice or reported as an orphan. + #[test] + fn a_joiner_is_tracked_while_it_runs_and_forgotten_when_it_finishes() { + let holder = NetnsHolder::adopt("maxplayer-netns-abc".into(), DockerCli::system()); + let tracked = |holder: &NetnsHolder| -> Vec { + holder.sidecars.lock().expect("registry").clone() + }; + assert!(tracked(&holder).is_empty(), "nothing is joined before anything runs"); + + let mut first = holder.watch_sidecar(sidecar_name(holder.name(), "iface")); + let mut second = holder.watch_sidecar(sidecar_name(holder.name(), "iface-readback")); + assert_eq!(tracked(&holder).len(), 2, "both live joiners are cleanup targets"); + + // Finishing one deregisters only that one: the other is still running and still owned. + // `completed` is what makes this finishing rather than cancellation, and only a command + // that returned may claim it. + let second_name = second.name.clone(); + second.completed(); + drop(second); + assert_eq!(tracked(&holder), vec![first.name.clone()], "{second_name} must be forgotten"); + + first.completed(); + drop(first); + assert!(tracked(&holder).is_empty(), "a finished joiner is not an orphan"); + } + + /// A removal that never returns is abandoned on its deadline, killed, and reported as a + /// failure -- not waited on forever and not reported as a removal that worked. + /// + /// This is the property the word "bounded" claimed while the code called `output()`, which has + /// no timeout: a docker client talking to a wedged daemon blocked the teardown thread for as + /// long as the daemon stayed wedged. Exercised on a child that is guaranteed not to exit, so + /// the deadline is the only thing that can end the wait. + #[test] + fn a_removal_that_never_returns_is_abandoned_on_its_deadline() { + let mut child = std::process::Command::new("sleep") + .arg("60") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("spawn sleep"); + let started = std::time::Instant::now(); + let error = NetnsHolder::wait_bounded(&mut child, std::time::Duration::from_millis(250)) + .expect_err("a child that never exits must hit the deadline"); + let waited = started.elapsed(); + + assert!(error.contains("did not finish"), "{error}"); + assert!( + error.contains("may still exist"), + "an abandoned removal must be reported as a possible leak, not as success: {error}" + ); + assert!(waited < std::time::Duration::from_secs(10), "waited {waited:?}"); + // Killed AND reaped, so the bound is real rather than advisory: the child is already gone + // and this returns its status immediately rather than blocking for the remaining ~59s. + assert!( + child.try_wait().expect("reap").is_some(), + "the abandoned child must be killed, not left running" + ); + } + + /// A removal that answers promptly is NOT abandoned -- the control that keeps the test above + /// from passing on a deadline that fires unconditionally. + #[test] + fn a_removal_that_returns_is_not_abandoned() { + let mut child = std::process::Command::new("true").spawn().expect("spawn true"); + let status = NetnsHolder::wait_bounded(&mut child, std::time::Duration::from_secs(10)) + .expect("a child that exits at once must be waited on normally"); + assert!(status.success(), "{status:?}"); + } + + /// A **cancelled** sidecar command leaves its name with the holder. + /// + /// The sibling above covers the finishing path. This one covers the path that produced the + /// defect: the guard was struck from the registry by cancellation itself, so the holder's `Drop` + /// found an empty list and removed nothing, while the container the blocking docker client had + /// already created stayed joined to the namespace. + /// + /// Cancellation is performed here the way tokio performs it -- the future is polled once, so the + /// registration exists and the command is in flight, and then the future is dropped. No runtime + /// and no docker are involved, so this measures the custody rule itself. + #[test] + fn a_cancelled_joiner_stays_a_cleanup_target() { + use std::future::Future as _; + + let holder = NetnsHolder::adopt("maxplayer-netns-cancelled".into(), DockerCli::system()); + let name = sidecar_name(holder.name(), "iface"); + { + let mut command = Box::pin(async { + let mut registration = holder.watch_sidecar(name.clone()); + // Stands in for the docker command that never returns before the cancellation. + std::future::pending::<()>().await; + registration.completed(); + }); + let waker = std::task::Waker::noop(); + let mut cx = std::task::Context::from_waker(waker); + assert!( + command.as_mut().poll(&mut cx).is_pending(), + "the command must still be in flight when it is cancelled" + ); + assert_eq!( + holder.sidecars.lock().expect("registry").len(), + 1, + "the joiner is registered before its command starts" + ); + } + + assert_eq!( + holder.sidecars.lock().expect("registry").clone(), + vec![name], + "a cancelled command must leave its container as a cleanup target -- deregistering here \ + is what left an orphan pinning the namespace" + ); + } + + /// Cleanup reports what happened. "No such container" after a cancelled create is the expected + /// path and not a failure; anything else is a leak, and must be reported as one rather than + /// swallowed into a teardown that claims to have destroyed the namespace. + #[test] + fn removing_something_that_was_never_created_is_not_a_cleanup_failure() { + // The holder is adopted before the create is issued precisely so this case exists. + let name = holder_name("a-job-whose-create-was-cancelled"); + assert!(name.starts_with("maxplayer-netns-"), "{name}"); + // No daemon is touched here; the classification under test is the string one, and it is the + // only place a "nothing to remove" result is allowed to pass as success. + assert!( + NetnsHolder::force_remove_stderr_is_benign("Error: No such container: x"), + "a container that never existed is not a leak" + ); + for real in [ + "Error response from daemon: cannot remove a running container", + "permission denied while trying to connect to the Docker daemon socket", + "", + ] { + assert!( + !NetnsHolder::force_remove_stderr_is_benign(real), + "a failed removal must be reported as a leak, not as a teardown: {real:?}" + ); + } + } + + /// F4: **the bound itself, exercised.** Every other cancellation test in this module inspects + /// argv or drives `Drop` by hand; none of them ever let a command run long enough to be + /// stopped, so the deadline that owns cancellation was asserted only by reading it. + /// + /// `sleep 30` under a one-second bound needs no daemon and no docker: the property is that a + /// command which does not finish is **killed** and the caller gets a failure naming the + /// deadline — not a hang, and not a success. The elapsed-time assertion is the real one; an + /// implementation that returned the right error after waiting out the full thirty seconds would + /// satisfy the string check and still be the bug. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "current_thread")] + async fn a_command_that_outlives_its_deadline_is_killed_and_says_so() { + let started = std::time::Instant::now(); + let outcome = run_bounded( + &DockerCli::system(), + vec!["sleep".to_owned(), "30".to_owned()], + None, + std::time::Duration::from_secs(1), + ) + .await; + let elapsed = started.elapsed(); + + let error = outcome.expect_err("a command past its deadline must not report success"); + assert!( + error.contains("did not finish within 1s"), + "the failure must name the deadline it broke: {error}" + ); + assert!( + elapsed < std::time::Duration::from_secs(10), + "the bound returned after {elapsed:?} — a deadline that is only reported once the \ + command finishes on its own is not a bound at all" + ); + } + + /// F4: a program that cannot be started fails **by name**, immediately. + /// + /// The path that matters is the one where docker is absent or unexecutable: that must surface as + /// a named failure rather than as a deadline timeout thirty seconds later, and it must never be + /// confused with a container that was created. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "current_thread")] + async fn a_program_that_cannot_be_started_fails_by_name() { + let missing = "maxplayer-no-such-program-exists"; + let error = run_bounded( + &DockerCli::system(), + vec![missing.to_owned()], + None, + std::time::Duration::from_secs(30), + ) + .await + .expect_err("a program that cannot be run must not report success"); + assert!( + error.contains("could not run") && error.contains(missing), + "the failure must name the program it could not run: {error}" + ); + } + + /// F4 residual: the client returning is not the container being gone. + /// + /// `run_sidecar` used to call `registration.completed()` after **every** returned result, on the + /// stated grounds that `docker run --rm` has removed the container by then. That holds for a + /// client which was reaped with a status — including a nonzero one — and not otherwise. A client + /// killed on our own deadline, or one that failed before the wait, leaves a container the daemon + /// may still be creating or running; deregistering it struck the one cleanup target for a + /// container that outlived its client. + /// + /// Both halves are asserted here, because only the pair distinguishes the fix from "never + /// deregister", which would make every sidecar report a phantom leak. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "current_thread")] + async fn only_a_reaped_client_may_end_a_sidecars_custody() { + let (outcome, child_exited) = run_bounded_tracked( + &DockerCli::system(), + vec!["sh".to_owned(), "-c".to_owned(), "exit 7".to_owned()], + None, + std::time::Duration::from_secs(10), + ) + .await; + let error = outcome.expect_err("a nonzero exit is still a failure to the caller"); + assert!(error.contains("exit 7"), "the caller's error must name the code: {error}"); + assert!( + child_exited, + "a nonzero exit is a REAPED client: that is a reason to ASK docker whether the \ + container is gone. It is not itself an answer, and custody no longer ends on it \ + alone — see `a_reaped_client_whose_container_is_still_there_keeps_custody`" + ); + + let started = std::time::Instant::now(); + let (outcome, child_exited) = run_bounded_tracked( + &DockerCli::system(), + vec!["sleep".to_owned(), "30".to_owned()], + None, + std::time::Duration::from_millis(400), + ) + .await; + let error = outcome.expect_err("a command past its deadline must not report success"); + assert!(error.contains("did not finish within"), "{error}"); + assert!( + !child_exited, + "a client killed on the deadline has shown nothing about its container, so the name \ + must stay a cleanup target" + ); + assert!( + started.elapsed() < std::time::Duration::from_secs(10), + "the deadline must be the thing that returned, not the command finishing" + ); + } + + /// The custody rule at the site that applies it. + /// + /// The sibling above measures the flag; this one measures what `run_sidecar` DOES with it, which + /// is the part a reviewer cannot take on trust. Written after a negative control showed the + /// flag test alone stayed green while the decision was reverted to the defective one. + /// + /// No docker and no daemon: the argv names a script that ignores its arguments, which is all + /// `with_container_name` needs (it requires `argv[1] == "run"` and splices the name after it). + #[cfg(feature = "acp")] + #[tokio::test(flavor = "current_thread")] + async fn a_sidecar_whose_client_was_killed_stays_a_cleanup_target() { + use std::os::unix::fs::PermissionsExt as _; + + let dir = std::env::temp_dir().join(format!("mx-custody-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let slow = dir.join("slow"); + let quick = dir.join("quick"); + std::fs::write(&slow, "#!/bin/sh\nsleep 30\n").expect("write slow"); + std::fs::write(&quick, "#!/bin/sh\nexit 0\n").expect("write quick"); + for path in [&slow, &quick] { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + } + + let holder = NetnsHolder::adopt("maxplayer-netns-custody".into(), DockerCli::system()); + + // A client killed on the deadline: the daemon may still be creating or running the + // container, so the name has to survive as a cleanup target. + let killed = run_sidecar_with_deadline( + &holder, + "iface", + vec![slow.to_string_lossy().into_owned(), "run".to_owned()], + None, + std::time::Duration::from_millis(400), + ) + .await; + assert!(killed.is_err(), "a command past its deadline must not report success"); + assert_eq!( + holder.sidecars.lock().expect("registry").len(), + 1, + "a client killed on the deadline never showed its container gone, so its name must \ + stay a cleanup target" + ); + + // A client reaped normally AND docker confirming the container gone: only then may the name + // be struck. Without this half, "never deregister" would pass the assertion above. + // + // The confirmer is injected rather than real. This used to call the production path, which + // reached a live `docker inspect` from inside an offline unit test: the test passed only + // because a daemon happened to answer, which is a dependency an offline suite must not have. + fn confirmed_gone(_client: &DockerCli, _name: &str) -> Option { + Some(true) + } + run_sidecar_confirmed( + &holder, + "iface", + vec![quick.to_string_lossy().into_owned(), "run".to_owned()], + None, + std::time::Duration::from_secs(10), + confirmed_gone, + ) + .await + .expect("a script that exits 0 must succeed"); + assert_eq!( + holder.sidecars.lock().expect("registry").len(), + 1, + "a reaped client whose container docker confirms gone must be struck, leaving only the \ + killed one" + ); + + let _ = std::fs::remove_dir_all(&dir); + std::mem::forget(holder); + } + + /// The `Err`-path custody failure, reproduced. + /// + /// This is the defect the verdict names: the client is reaped — `child_exited` is true, with a + /// NONZERO exit, exactly the shape a deadline-killed, I/O-failed or refused `docker run` returns + /// — and the container it named is **still there**. The old rule ended custody on the client's + /// exit alone and struck the only cleanup target for a live container. + /// + /// Hermetic: the confirmer is a stub, so this asserts the DECISION, not a daemon's mood. Revert + /// the rule to `if child_exited { registration.completed(); }` and this test fails. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "current_thread")] + async fn a_reaped_client_whose_container_is_still_there_keeps_custody() { + use std::os::unix::fs::PermissionsExt as _; + + let dir = std::env::temp_dir().join(format!("mx-err-custody-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let failing = dir.join("failing"); + std::fs::write(&failing, "#!/bin/sh\nexit 7\n").expect("write failing"); + std::fs::set_permissions(&failing, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + + /// Docker answering "that container is still here". + fn still_present(_client: &DockerCli, _name: &str) -> Option { + Some(false) + } + /// Docker unable to answer at all — must be treated exactly like "still here". + fn cannot_tell(_client: &DockerCli, _name: &str) -> Option { + None + } + + for (confirm, label) in [ + (still_present as ConfirmAbsent, "docker says the container is still there"), + (cannot_tell as ConfirmAbsent, "docker cannot say whether it is there"), + ] { + let holder = NetnsHolder::adopt("maxplayer-netns-err-custody".into(), DockerCli::system()); + let outcome = run_sidecar_confirmed( + &holder, + "iface", + vec![failing.to_string_lossy().into_owned(), "run".to_owned()], + None, + std::time::Duration::from_secs(10), + confirm, + ) + .await; + + let error = outcome.expect_err("exit 7 is a failure"); + assert!(error.contains("exit 7"), "the caller still sees the real error: {error}"); + assert_eq!( + holder.sidecars.lock().expect("registry").len(), + 1, + "the client was REAPED with a nonzero exit, but {label}: custody must be held \ + until the container is CONFIRMED GONE, or cleanup has no target for a container \ + that outlived its client" + ); + std::mem::forget(holder); + } + + let _ = std::fs::remove_dir_all(&dir); + } + + /// A SIGNAL-terminated client is reaped too, and must still not end custody on its own. + /// + /// The R3 verdict named this case precisely: `try_wait` yields a status for a signalled child + /// just as it does for an ordinary exit, so `child_exited` is `true` here, while `code()` + /// returns `None` and the call reports "killed by a signal". The old comments promised that + /// signal failures retain custody; the old code did not deliver it, because the flag alone was + /// allowed to release the name. + /// + /// This is the sharpest form of "reaped is not removed": a client killed mid-flight tells us + /// nothing whatever about whether the daemon created, is running, or removed that container. + /// Custody is kept unless the daemon itself says the container is gone. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "current_thread")] + async fn a_signal_killed_client_does_not_end_custody_by_itself() { + use std::os::unix::fs::PermissionsExt as _; + + let dir = std::env::temp_dir().join(format!("mx-signal-custody-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let suicide = dir.join("suicide"); + // Kills ITSELF with SIGKILL: reaped with a status, but `code()` is None. + std::fs::write(&suicide, "#!/bin/sh\nkill -9 $$\n").expect("write suicide"); + std::fs::set_permissions(&suicide, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + + fn still_present(_client: &DockerCli, _name: &str) -> Option { + Some(false) + } + + let holder = NetnsHolder::adopt("maxplayer-netns-signal-custody".into(), DockerCli::system()); + let outcome = run_sidecar_confirmed( + &holder, + "iface", + vec![suicide.to_string_lossy().into_owned(), "run".to_owned()], + None, + std::time::Duration::from_secs(10), + still_present, + ) + .await; + + let error = outcome.expect_err("a signalled client is a failure"); + assert!( + error.contains("killed by a signal"), + "this must exercise the signal path, not an ordinary nonzero exit: {error}" + ); + assert_eq!( + holder.sidecars.lock().expect("registry").len(), + 1, + "a signal-killed client proves nothing about the container; custody must be kept" + ); + + std::mem::forget(holder); + let _ = std::fs::remove_dir_all(&dir); + } + + // ── The production path itself ──────────────────────────────────────────────────────────── + // + // Everything above tests a helper. These two drive `establish` — the function production calls, + // with its real ordering of adopt, fence, create, apply and cleanup — against a stand-in docker + // client, because the fault these close is precisely that a fixture was standing in for the + // production path and could agree with a bug the production path does not survive. + + // The stand-in client is passed to `establish_with` as an ARGUMENT. There is deliberately no + // lock and no shared cell here: the previous shape installed the client in a process-global, + // which meant every test that touched this path had to remember to take a mutex, any helper + // that ran outside one (cleanup from `Drop`, notably) read whatever another test had installed, + // and the tests below could not run in parallel. Passing it in removes the interference rather + // than serialising around it. + + /// A stand-in `docker` that answers `establish`'s sequence and records what it was asked. + /// + /// Writes the applier's stdin to `stdin.txt` and every removed name to `rm.log`, so a test can + /// assert on what production actually sent rather than on what it believes production sends. + #[cfg(feature = "acp")] + fn stand_in_docker(work: &std::path::Path, create_delay: &str) -> std::path::PathBuf { + use std::os::unix::fs::PermissionsExt as _; + + let script = work.join("docker"); + let body = r#"#!/bin/sh +WORK="__WORK__" +# The daemon is unreachable: every query fails, and none of them may be read as "absent". +if [ -f "$WORK/daemon-down" ]; then + echo "Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?" >&2 + exit 1 +fi +case "$*" in + *"--entrypoint getent"*) + echo "203.0.113.77 STREAM host.docker.internal" + exit 0 + ;; + *"inspect --type container"*) + for a in "$@"; do last="$a"; done + echo "inspect $last" >> "$WORK/events.log" + if [ -f "$WORK/present-$last" ]; then + echo "sha256:deadbeefcafe" + exit 0 + fi + # THE RACE, made deterministic: this inspect answers ABSENT, and the container lands the instant + # after that answer — before any event query the caller makes next. `race-NAME` is consumed so + # it fires exactly once; `raced-NAME` records that it fired. + if [ -f "$WORK/race-$last" ]; then + mv "$WORK/race-$last" "$WORK/present-$last" + : > "$WORK/raced-$last" + fi + echo "Error response from daemon: No such container" >&2 + exit 1 + ;; + *"rm --force --volumes"*) + for a in "$@"; do last="$a"; done + echo "$last" >> "$WORK/rm.log" + echo "rm $last" >> "$WORK/events.log" + if [ -f "$WORK/rmfail-$last" ]; then + echo "Error response from daemon: cannot remove container $last" >&2 + exit 1 + fi + # A removal that succeeds makes the container ABSENT, exactly as the daemon would: the presence + # marker is what `inspect` answers from, so a test can assert the container really went away + # instead of asserting that a removal was merely attempted. + rm -f "$WORK/present-$last" + exit 0 + ;; + *"events --since"*) + # The daemon's event log, in the production format `IDnameaction`: a container under + # NAME that is present now has a `create` and no `destroy`; one a test recorded as landed and + # since gone (`landed-NAME`) has both, for the same id. `evhang-NAME` leaves a descendant holding + # this query's stdout open after the client exits, as a client's child process can, for 20s; + # when it lets go it records `released-NAME`, so a test can assert ORDER against the hold. + for a in "$@"; do + case "$a" in + container=*) + n="${a#container=}" + echo "events $n" >> "$WORK/events.log" + if [ -f "$WORK/evhang-$n" ]; then + ( sleep 20; : > "$WORK/released-$n" ) & + exit 0 + fi + if [ -f "$WORK/present-$n" ]; then + printf 'deadbeefcafe\t%s\tcreate\n' "$n" + fi + if [ -f "$WORK/landed-$n" ]; then + printf 'feedfacef00d\t%s\tcreate\nfeedfacef00d\t%s\tdestroy\n' "$n" "$n" + fi + # THE OTHER RACE, made deterministic: a container lands the instant after this event query + # answered — before the caller's next inspect. Consumed so it fires once; `raced-after- + # events-NAME` records that it did. + if [ -f "$WORK/land-after-events-$n" ]; then + mv "$WORK/land-after-events-$n" "$WORK/present-$n" + : > "$WORK/raced-after-events-$n" + fi + ;; + esac + done + exit 0 + ;; + *--detach*) + : > "$WORK/creating" + echo "create-start" >> "$WORK/events.log" + __DELAY__ + echo "create-end" >> "$WORK/events.log" + echo deadbeefcafe + exit 0 + ;; +esac +cat > "$WORK/stdin.txt" +echo 0 +exit 0 +"# + .replace("__WORK__", &work.to_string_lossy()) + .replace("__DELAY__", create_delay); + std::fs::write(&script, body).expect("write stand-in docker"); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + script + } + + #[cfg(feature = "acp")] + fn stand_in_work_dir(tag: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!("mx-establish-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("work dir"); + dir + } + + /// A world the expiry sweep can be run against: `docker ps` answers from a listing file, and + /// `docker rm` edits that same file. + /// + /// The listing is the daemon's state, not a fixture the test re-states between calls, because + /// the properties under test are all about a SECOND sweep seeing what the first one did: a + /// removal that succeeded must be gone from the next listing, a removal that failed must still + /// be in it, and a container that appears afterwards must be found without anything remembering + /// it. A stand-in that replayed a canned listing could not tell any of those apart. + #[cfg(feature = "acp")] + fn stand_in_sweep_docker(work: &std::path::Path) -> std::path::PathBuf { + use std::os::unix::fs::PermissionsExt as _; + + let script = work.join("docker"); + let body = r#"#!/bin/sh +WORK="__WORK__" +if [ -f "$WORK/daemon-down" ]; then + echo "Cannot connect to the Docker daemon at unix:///var/run/docker.sock." >&2 + exit 1 +fi +case "$*" in + *"ps --all"*) + echo "ps" >> "$WORK/calls.log" + cat "$WORK/listing.tsv" 2>/dev/null + exit 0 + ;; + *"rm --force --volumes"*) + for a in "$@"; do last="$a"; done + echo "$last" >> "$WORK/rm.log" + if [ -f "$WORK/rmfail-$last" ]; then + echo "Error response from daemon: cannot remove container $last" >&2 + exit 1 + fi + grep -v "^$last " "$WORK/listing.tsv" > "$WORK/listing.next" 2>/dev/null + mv "$WORK/listing.next" "$WORK/listing.tsv" + exit 0 + ;; +esac +exit 0 +"# + .replace("__WORK__", &work.to_string_lossy()); + std::fs::write(&script, body).expect("write sweep stand-in docker"); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + script + } + + /// Write the daemon's container listing in the exact format [`list_owned_argv`] asks for. + #[cfg(feature = "acp")] + fn write_listing(work: &std::path::Path, rows: &[(&str, &str, &str, &str)]) { + let mut out = String::new(); + for (id, seat, cleanup_after, role) in rows { + // Every container this module creates carries a job label, so the default fixture does + // too — IN THE COLUMN ITS OWN ROLE WRITES. A holder names its job in `HOLDER_LABEL` and + // never wears the helper label; a helper does the reverse. Writing every row's job into + // the helper column (this fixture before #996 R3/3) gave holder rows a shape no + // production launch emits, and that is precisely how the wrong-role hole stayed + // invisible to these tests while a real holder went unswept. A row that needs the job + // ABSENT or a field malformed is written with `write_raw_listing`, which states the + // whole line. + let (helper_job, holder_job) = if *role == ROLE_HELPER { + (format!("job-{id}"), String::new()) + } else { + (String::new(), format!("job-{id}")) + }; + out.push_str(&format!( + "{id}\t{seat}\t{cleanup_after}\t{role}\t{helper_job}\t{holder_job}\n" + )); + } + std::fs::write(work.join("listing.tsv"), out).expect("listing"); + } + + /// A listing written verbatim, for rows whose whole point is a field docker returned unusable. + #[cfg(feature = "acp")] + fn write_raw_listing(work: &std::path::Path, listing: &str) { + std::fs::write(work.join("listing.tsv"), listing).expect("listing"); + } + + /// A seat of this test's own, because the sweep cursor is PROCESS-GLOBAL and keyed by seat. + /// + /// Two sweep rows sharing one seat string share a resume point: whichever runs first leaves a + /// cursor, and the second starts mid-queue for reasons nothing in its own body states. That is + /// a property of the cursor being real state rather than a test defect to paper over — so each + /// row takes a distinct seat, exactly as two seats on one host would. + fn sweep_seat(name: &str) -> String { + let mut seat: String = name.bytes().map(|b| format!("{b:02x}")).collect(); + seat.truncate(64); + while seat.len() < 64 { + seat.push('0'); + } + seat + } + + #[cfg(feature = "acp")] + fn rm_log(work: &std::path::Path) -> Vec { + std::fs::read_to_string(work.join("rm.log")) + .unwrap_or_default() + .lines() + .map(str::to_owned) + .collect() + } + + /// THE THRESHOLD IS THE JOB'S OWN DEADLINE PLUS THE GRACE, AND IT IS A FLOOR. + /// + /// One second before it the container is untouched; at it and after it, it is selected. The + /// boundary itself is asserted because `>` and `>=` are the same test everywhere else, and the + /// difference between them is a whole second in which a job's containers are either still its + /// own or already the sweep's. + #[test] + fn a_container_is_untouched_before_its_stamp_and_selected_at_it() { + let stamp = cleanup_after_unix(1_000); + assert_eq!(stamp, 1_000 + CLEANUP_GRACE_SECS, "the grace is the job's deadline plus an hour"); + let owned = vec![OwnedContainer { + id: "c1".to_owned(), + seat: Some(seat_b()), + cleanup_after: Some(stamp), + role: Some(ROLE_HOLDER.to_owned()), + helper_job: None, + holder_job: Some("job-c1".to_owned()), + }]; + assert!( + expired_owned(&owned, &seat_b(), stamp - 1).is_empty(), + "a container one second inside its own grace is still the job's" + ); + assert_eq!(expired_owned(&owned, &seat_b(), stamp), vec!["c1".to_owned()]); + assert_eq!(expired_owned(&owned, &seat_b(), stamp + 86_400), vec!["c1".to_owned()]); + } + + /// EACH JOB IS JUDGED AGAINST ITS OWN DEADLINE, NOT A SHARED AGE. + /// + /// The whole reason the expiry is written onto the container: a seat runs a ten-minute job and an + /// eight-hour one at the same time, and a single global age would either strangle the long job or + /// keep the short one's leftovers for the long one's lifetime. At one instant the short job's + /// container is expired and the long job's is not. + #[test] + fn each_job_is_judged_against_its_own_deadline_not_a_shared_age() { + let short = cleanup_after_unix(1_000 + 600); + let long = cleanup_after_unix(1_000 + 28_800); + let owned = vec![ + OwnedContainer { + id: "short-job".to_owned(), + seat: Some(seat_b()), + cleanup_after: Some(short), + role: Some(ROLE_HOLDER.to_owned()), + helper_job: None, + holder_job: Some("short".to_owned()), + }, + OwnedContainer { + id: "long-job".to_owned(), + seat: Some(seat_b()), + cleanup_after: Some(long), + role: Some(ROLE_HOLDER.to_owned()), + helper_job: None, + holder_job: Some("long".to_owned()), + }, + ]; + let now = short + 1; + assert_eq!( + expired_owned(&owned, &seat_b(), now), + vec!["short-job".to_owned()], + "the long job's container is inside ITS OWN deadline and must be left alone" + ); + assert_eq!(expired_owned(&owned, &seat_b(), long).len(), 2, "both, once both have passed"); + } + + /// AN UNREADABLE, ABSENT OR FOREIGN STAMP IS NEVER A PERMISSION. + /// + /// Four ways a container can fail to be this seat's expired one — no stamp, a corrupt stamp, + /// another seat's stamp, and a caller that cannot name its own seat — and none of them may be + /// read as "remove it". A corrupt stamp parsed as the number 0 would date the container to 1970 + /// and make it instantly sweepable, which is why the parse is asserted too. + #[test] + fn an_unreadable_or_foreign_stamp_is_never_a_permission_to_remove() { + let listing = format!( + "unstamped\t{seat}\t\t{ROLE_HOLDER}\tjob-u\n\ + mangled\t{seat}\tnot-a-number\t{ROLE_HELPER}\tjob-m\n\ + stranger\tffff\t1\t{ROLE_HOLDER}\tjob-s\nshared\t\t\t\t\n", + seat = seat_b() + ); + let owned = parse_owned_listing(&listing); + assert_eq!(owned.len(), 4, "every row is still reported: {owned:?}"); + assert_eq!(owned[0].cleanup_after, None, "an absent stamp is None, never 0"); + assert_eq!(owned[1].cleanup_after, None, "a corrupt stamp is None, never 0"); + assert!( + expired_owned(&owned, &seat_b(), u64::MAX).is_empty(), + "not even at the end of time: an unreadable stamp is not an expired one, a stranger's \ + container is not ours, and a shared container carries neither" + ); + let stamped = vec![OwnedContainer { + id: "ours".to_owned(), + seat: Some(seat_b()), + cleanup_after: Some(1), + role: Some(ROLE_HOLDER.to_owned()), + helper_job: None, + holder_job: Some("job-ours".to_owned()), + }]; + assert!( + expired_owned(&stamped, " ", u64::MAX).is_empty(), + "a caller that cannot name its seat owns nothing to remove" + ); + } + + /// EVERY CONTAINER THE SWEEP REFUSES IS REPORTED, AND ROLE AND JOB BOTH GATE REMOVAL. + /// + /// Before #996 D1 an unreadable stamp was filtered out in silence and `role` was parsed and then + /// ignored, so a container this seat could never remove looked exactly like a host with nothing + /// on it. Every row here is judged at `u64::MAX`, so no row is held back by its expiry: what + /// keeps each one out of `removable` is the validation, and every one comes back NAMED. + #[test] + fn a_container_the_sweep_cannot_validate_is_refused_and_reported() { + let stamp = 1_000_u64; + let listing = format!( + "good\t{seat}\t{stamp}\t{ROLE_HOLDER}\t\tjob-1\n\ + no-role\t{seat}\t{stamp}\t\tjob-2\t\n\ + odd-role\t{seat}\t{stamp}\tinterloper\tjob-3\t\n\ + no-job\t{seat}\t{stamp}\t{ROLE_HELPER}\t\t\n\ + bad-stamp\t{seat}\tnot-a-number\t{ROLE_HOLDER}\t\tjob-4\n\ + no-seat\t\t{stamp}\t{ROLE_HOLDER}\t\tjob-5\n\ + stranger\tffff\t{stamp}\t{ROLE_HOLDER}\t\tjob-6\n", + seat = seat_b() + ); + let owned = parse_owned_listing(&listing); + let selection = partition_owned(&owned, &seat_b(), u64::MAX); + assert_eq!( + selection.removable, + vec!["good".to_owned()], + "only a container that establishes seat, role, job AND a passed stamp may be removed" + ); + let reported: std::collections::BTreeMap<&str, String> = selection + .skipped + .iter() + .map(|(id, reason)| (id.as_str(), reason.to_string())) + .collect(); + assert_eq!( + reported.keys().copied().collect::>(), + vec!["bad-stamp", "no-job", "no-role", "no-seat", "odd-role"], + "every refused container of OURS is named rather than dropped: {reported:?}" + ); + // A co-tenant's container is neither removed nor reported: reporting it would turn another + // seat's ordinary operation into a permanent complaint in this seat's log. + assert!( + !reported.contains_key("stranger"), + "another seat's container is not ours to remove OR to report: {reported:?}" + ); + assert!( + reported["odd-role"].contains("interloper"), + "the reason carries what was actually read, so the operator can act on it: {reported:?}" + ); + assert!(reported["no-role"].contains("role"), "{reported:?}"); + assert!(reported["no-job"].contains("job"), "{reported:?}"); + assert!(reported["bad-stamp"].contains("stamp"), "{reported:?}"); + assert!(reported["no-seat"].contains("seat"), "{reported:?}"); + } + + /// A PRODUCTION HOLDER NAMES ITS JOB IN ITS OWN LABEL, AND THE SWEEP MUST READ IT THERE. + /// + /// Found by the live matrix at 647c457b, not by this module. Every holder production launches + /// carries its job in [`HOLDER_LABEL`] and NEVER wears [`HELPER_JOB_LABEL`], so a listing that + /// read the job only from the helper label parsed every real holder as jobless, and the D1 gate + /// then refused it as [`SkipReason::MissingJob`] on that pass and every later one. The holders + /// this sweep exists to reclaim would have been the one thing it could never remove. The unit + /// fixtures hid it because `write_listing` gives every row a helper job label; production does + /// not, which is why this row is written out in full. + #[test] + fn a_holder_stamped_the_way_production_stamps_it_is_swept_not_refused() { + let stamp = 1_000_u64; + let seat = sweep_seat("prod-shape"); + // Exactly what `list_owned_argv` hands back for a live holder: the helper-job column is + // empty and the holder's own label carries the job. + let listing = format!( + "holder\t{seat}\t{stamp}\t{ROLE_HOLDER}\t\tjob-live\n\ + helper\t{seat}\t{stamp}\t{ROLE_HELPER}\tjob-live\t\n\ + jobless\t{seat}\t{stamp}\t{ROLE_HOLDER}\t\t\n" + ); + let owned = parse_owned_listing(&listing); + assert_eq!( + owned[0].resolve_job().ok(), + Some("job-live"), + "a holder's job is read from its own label, not from a helper label it never wears" + ); + assert_eq!( + owned[1].resolve_job().ok(), + Some("job-live"), + "a helper still names its own job" + ); + assert_eq!( + owned[2].resolve_job().err(), + Some(SkipReason::MissingJob), + "a container naming no job in EITHER column names none" + ); + + let selection = partition_owned(&owned, &seat, u64::MAX); + assert!( + selection.removable.contains(&"holder".to_owned()), + "an expired production-stamped holder must be removable, not refused: {selection:?}" + ); + assert!( + selection.removable.contains(&"helper".to_owned()), + "and the helper alongside it: {selection:?}" + ); + assert_eq!( + selection.skipped, + vec![("jobless".to_owned(), SkipReason::MissingJob)], + "absence is still never permission: the row naming no job at all is the only refusal" + ); + } + + /// A ROW WHOSE TWO JOB COLUMNS DISAGREE NAMES NO JOB, AND IS REPORTED RATHER THAN REMOVED. + /// + /// The R3/3 remaining FAIL. `field(&mut fields).or_else(|| field(&mut fields))` returned + /// whichever column happened to be populated FIRST, so + /// `idseat1holderjob-Ajob-B` was accepted on `job-A` — the HELPER's + /// column — and removed at `now >= 1`, while the holder's own column naming a DIFFERENT job was + /// never read. A container whose own labels contradict each other cannot be attributed to any + /// job, and the contradiction will not age out: it has to be refused AND shown, or it is a + /// removal nobody authorised justified by metadata nobody can trust. + #[test] + fn a_row_whose_job_columns_disagree_is_skipped_and_reported() { + let seat = sweep_seat("conflict"); + // Column order is `list_owned_argv`'s: id, seat, cleanup-after, role, helper-job, holder-job. + let listing = format!( + "two-jobs-holder\t{seat}\t1\t{ROLE_HOLDER}\tjob-A\tjob-B\n\ + two-jobs-helper\t{seat}\t1\t{ROLE_HELPER}\tjob-A\tjob-B\n" + ); + let owned = parse_owned_listing(&listing); + assert_eq!( + owned[0].resolve_job().err(), + Some(SkipReason::ConflictingJobs { + own_role: "job-B".to_owned(), + other_role: "job-A".to_owned(), + }), + "a holder resolves from its own column, and a disagreeing helper column refuses the row" + ); + assert_eq!( + owned[1].resolve_job().err(), + Some(SkipReason::ConflictingJobs { + own_role: "job-A".to_owned(), + other_role: "job-B".to_owned(), + }), + "and a helper the same way round — the refusal is symmetric, not holder-only" + ); + + // `u64::MAX` is far past the stamp: without the refusal, both rows would be REMOVED here. + let selection = partition_owned(&owned, &seat, u64::MAX); + assert!( + selection.removable.is_empty(), + "a container whose labels contradict each other is never removable: {selection:?}" + ); + let reported: std::collections::BTreeMap<_, _> = selection + .skipped + .iter() + .map(|(id, reason)| (id.clone(), reason.to_string())) + .collect(); + assert_eq!( + reported.len(), + 2, + "both rows are carried out to the operator, not silently dropped: {reported:?}" + ); + assert!( + reported["two-jobs-holder"].contains("job-A") + && reported["two-jobs-holder"].contains("job-B"), + "the reason names BOTH job ids that disagree, so the operator can see which: {reported:?}" + ); + } + + /// A JOB SITTING IN THE OTHER ROLE'S COLUMN IS NOT PROVENANCE. + /// + /// The second half of the same FAIL: a helper carrying only [`HOLDER_LABEL`], or a holder + /// carrying only [`HELPER_JOB_LABEL`], was read as valid because the fallback did not care + /// which column answered. No launch in this module writes either shape — [`holder_argv`] writes + /// the holder label and [`helper_label_args`] the helper label — so a row like this came from + /// something that is not one of our launches, and its stamp cannot be attributed to a job of + /// ours. Refused and reported, for the same reason as a conflict. + #[test] + fn a_job_in_the_wrong_role_column_is_skipped_and_reported() { + let seat = sweep_seat("misplaced"); + let listing = format!( + "holder-with-helper-job\t{seat}\t1\t{ROLE_HOLDER}\tjob-x\t\n\ + helper-with-holder-job\t{seat}\t1\t{ROLE_HELPER}\t\tjob-y\n" + ); + let owned = parse_owned_listing(&listing); + assert_eq!( + owned[0].resolve_job().err(), + Some(SkipReason::JobInWrongColumn { + role: ROLE_HOLDER.to_owned(), + job: "job-x".to_owned(), + }), + "a holder does not inherit a job from the helper column it never wears" + ); + assert_eq!( + owned[1].resolve_job().err(), + Some(SkipReason::JobInWrongColumn { + role: ROLE_HELPER.to_owned(), + job: "job-y".to_owned(), + }), + "nor a helper from the holder column" + ); + + let selection = partition_owned(&owned, &seat, u64::MAX); + assert!( + selection.removable.is_empty(), + "neither misplaced row is removable however long past its stamp: {selection:?}" + ); + let reported: std::collections::BTreeMap<_, _> = selection + .skipped + .iter() + .map(|(id, reason)| (id.clone(), reason.to_string())) + .collect(); + assert_eq!(reported.len(), 2, "both are reported: {reported:?}"); + assert!( + reported["holder-with-helper-job"].contains("job-x"), + "the reason carries the job id actually found, for the operator: {reported:?}" + ); + } + + /// THE CONTROL THAT KEEPS THE TWO REFUSALS ABOVE HONEST: EVERY PRODUCTION SHAPE STILL RESOLVES. + /// + /// A gate that refused every row would pass both tests above and close the sweep entirely — + /// which is the failure mode this whole change exists to fix. So the two shapes production + /// actually writes must still parse and still be removable, and columns that AGREE are not a + /// contradiction: per-role resolution returns the same job either way. + #[test] + fn each_role_still_resolves_the_job_from_its_own_column() { + let seat = sweep_seat("per-role-ok"); + let listing = format!( + "holder\t{seat}\t1\t{ROLE_HOLDER}\t\tjob-h\n\ + helper\t{seat}\t1\t{ROLE_HELPER}\tjob-p\t\n\ + agreeing\t{seat}\t1\t{ROLE_HOLDER}\tjob-same\tjob-same\n" + ); + let owned = parse_owned_listing(&listing); + assert_eq!( + owned[0].resolve_job().ok(), + Some("job-h"), + "the production holder shape: job in HOLDER_LABEL, helper column empty" + ); + assert_eq!( + owned[1].resolve_job().ok(), + Some("job-p"), + "the production helper shape: job in HELPER_JOB_LABEL, holder column empty" + ); + assert_eq!( + owned[2].resolve_job().ok(), + Some("job-same"), + "columns that agree say one thing, and a row is not refused for saying it twice" + ); + + let selection = partition_owned(&owned, &seat, u64::MAX); + assert_eq!( + selection.removable.len(), + 3, + "all three resolve, so all three are removable past their stamp: {selection:?}" + ); + assert!( + selection.skipped.is_empty(), + "and nothing is refused: {selection:?}" + ); + } + + /// A PERSISTENTLY FAILING HEAD CANNOT STARVE THE TAIL OF THE QUEUE. + /// + /// The defect (#996 D2): the pass took the first [`MAX_SWEEP_REMOVALS`] candidates every time, + /// so when those removals kept failing they were selected again next pass, and again, and a + /// container behind them was never once asked about however long it had been expired. Resuming + /// after the last id ATTEMPTED — not after the last one that succeeded — bounds every + /// candidate's wait at `ceil(len / cap)` passes however many removals fail. + #[test] + fn a_failing_head_cannot_starve_the_tail_of_the_sweep_queue() { + let candidates: Vec = (0..40).map(|n| format!("c{n:02}")).collect(); + let cap = 32_usize; + + let (first, deferred) = select_pass(&candidates, None, cap); + assert_eq!(first.len(), cap, "the count bound still holds"); + assert_eq!(first[0], "c00"); + assert_eq!( + deferred.len(), + 8, + "the tail waits, and is reported as waiting" + ); + + // NOTHING was removed: every attempt failed, so the candidate set is unchanged. This is + // precisely the state the old selection could never escape. + let cursor = first + .last() + .cloned() + .expect("a pass that attempted something"); + let (second, _) = select_pass(&candidates, Some(&cursor), cap); + assert_eq!( + &second[..8], + &candidates[32..40], + "the pass after a wholly failing one must start where that one stopped: {second:?}" + ); + let reached: std::collections::BTreeSet<&String> = + first.iter().chain(second.iter()).collect(); + assert_eq!( + reached.len(), + candidates.len(), + "every candidate is attempted within 2 passes even though not one removal succeeded" + ); + } + + /// THE RESUME POINT WRAPS, AND SURVIVES THE IDS IT NAMED BEING REMOVED. + #[test] + fn the_sweep_resume_point_wraps_and_survives_its_ids_disappearing() { + let candidates: Vec = (0..5).map(|n| format!("c{n}")).collect(); + // A cursor past every remaining id — the ordinary case once the tail has been swept. + let (attempt, deferred) = select_pass(&candidates, Some("zzz"), 2); + assert_eq!( + attempt, + vec!["c0".to_owned(), "c1".to_owned()], + "it wraps to the front" + ); + assert_eq!(deferred.len(), 3); + // A cursor naming a container that has since been REMOVED: the next id after it is still + // well defined, so a pass that succeeded does not lose its place. + let survivors = vec!["c0".to_owned(), "c3".to_owned(), "c4".to_owned()]; + let (attempt, _) = select_pass(&survivors, Some("c2"), 2); + assert_eq!(attempt, vec!["c3".to_owned(), "c4".to_owned()]); + // A cap of zero attempts nothing and DEFERS everything, rather than quietly dropping it. + let (attempt, deferred) = select_pass(&survivors, None, 0); + assert!(attempt.is_empty()); + assert_eq!(deferred.len(), survivors.len()); + } + + /// THE STAMP IS A RESTATEMENT OF THE DEADLINE, NOT A MEASUREMENT TAKEN AT THE CREATE. + /// + /// #996 D3. The stamp a launch writes must be the same second whenever the create is issued; + /// the clock at the create is not evidence about the job's deadline. The backward step is the + /// case that mattered — it is the one that shortens the reconstruction. + #[test] + fn a_carried_deadline_stamps_the_same_second_whatever_the_clock_says() { + let deadline = 1_700_000_900_u64; + // The window the caller computed, at 1_700_000_000. + let lifetime = 900_u64; + let expected = deadline + CLEANUP_GRACE_SECS; + for now_at_create in [ + 1_700_000_000_u64, + 1_700_000_450, + 1_699_999_000, + 1_700_000_899, + ] { + assert_eq!( + launch_cleanup_stamp(Some(deadline), lifetime, now_at_create), + expected, + "a carried deadline must not move with the clock at the create ({now_at_create})" + ); + } + // What is being refused: reconstructing from a backward-stepped clock lands the stamp + // earlier than the job's real deadline. + assert!( + launch_cleanup_stamp(None, lifetime, 1_699_999_000) < expected, + "if the fallback ever stops being clock-dependent, this row has lost its subject" + ); + // And with no deadline to carry, the window is still the best statement available. + assert_eq!( + launch_cleanup_stamp(None, lifetime, 1_700_000_000), + expected + ); + // An unreadable clock saturates instead of wrapping into the past. + assert_eq!(launch_cleanup_stamp(None, lifetime, u64::MAX), u64::MAX); + } + + /// THE PRODUCTION DEADLINE REACHES THE CONTAINER AND DECIDES THE SWEEP. + /// + /// The one test that spans the whole path rather than a link of it: the remaining window + /// `prepare_launch` computes for a job (`unified_job_timeout` against the deadline + /// `seller::job_deadline_unix` chose) becomes the stamp on the create argv, and that same stamp, + /// read back off a listing, is what the sweep judges. A build that stamped a *guessed* global cap + /// would still pass every test above and fail this one. + /// + /// `wallet`-gated because [`crate::seller_exec`] is: the module carrying the production + /// arithmetic does not exist on an `acp`-only build, and a test that named it there would fail + /// to compile a real CI row (see the feature note on `seller_exec` in `lib.rs`). + #[cfg(feature = "wallet")] + #[test] + fn the_production_deadline_reaches_the_container_and_decides_the_sweep() { + let now = 1_700_000_000_u64; + // A seller-selected deadline of `now + 900`, as `job_deadline_unix` would return. + let deadline = now + 900; + let lifetime = crate::seller_exec::unified_job_timeout(deadline, now); + // The arithmetic `prepare_launch` performs since #996 D3: the job's ABSOLUTE deadline, + // carried down from the caller that chose it, plus the grace. No clock is read here. + let cleanup_after = cleanup_after_unix(deadline); + assert_eq!(cleanup_after, deadline + CLEANUP_GRACE_SECS, "the job's OWN deadline, plus grace"); + // The property the carried deadline has and the reconstruction did not: the stamp does not + // depend on WHEN the create is issued. A clock that stepped backward between the caller + // computing `lifetime` and this create lands the old derivation EARLIER than the job's real + // deadline — and the sweep would then remove a container out from under a job still inside + // it. The reconstruction is modelled here only to show what is being refused. + let reconstructed_after_a_backward_step = + cleanup_after_unix((now - 300).saturating_add(lifetime.as_secs())); + assert!( + reconstructed_after_a_backward_step < cleanup_after, + "a backward clock step is what shortened the old derivation; if that stops being true \ + the regression this row guards has changed shape and the row needs rewriting" + ); + + let argv = holder_argv("h", "net", "img", 1000, 1000, "job-1", &seat_b(), cleanup_after); + let label = format!("{HOLDER_CLEANUP_AFTER_LABEL}={cleanup_after}"); + assert!(argv.iter().any(|a| a == &label), "the create must carry the stamp: {argv:?}"); + + // …and what docker would report for that container is what the sweep judges. + let listing = format!( + "deadbeef\t{}\t{cleanup_after}\t{ROLE_HOLDER}\t\tjob-1\n", + seat_b() + ); + let owned = parse_owned_listing(&listing); + assert!( + expired_owned(&owned, &seat_b(), deadline + CLEANUP_GRACE_SECS - 1).is_empty(), + "still inside the hour after its real deadline" + ); + assert_eq!( + expired_owned(&owned, &seat_b(), deadline + CLEANUP_GRACE_SECS), + vec!["deadbeef".to_owned()] + ); + } + + /// A JOB'S HELPERS CARRY THE SAME SEAT AND THE SAME STAMP AS ITS HOLDER. + /// + /// A helper stamped differently from its holder would be a second, quieter deadline; one stamped + /// not at all is the leftover this work exists to remove. The helper's job id is deliberately + /// NOT `HOLDER_LABEL`, because that label is what the boot reaper selects holders by. + #[test] + fn a_job_owned_helper_carries_the_same_seat_and_stamp_as_its_holder() { + let stamp = cleanup_after_unix(2_000); + let labels = helper_label_args("job-1", &seat_b(), stamp); + for expected in [ + format!("{HOLDER_SEAT_LABEL}={}", seat_b()), + format!("{HOLDER_CLEANUP_AFTER_LABEL}={stamp}"), + format!("{HOLDER_ROLE_LABEL}={ROLE_HELPER}"), + format!("{HELPER_JOB_LABEL}=job-1"), + ] { + assert!(labels.iter().any(|l| l == &expected), "missing {expected} in {labels:?}"); + } + assert!( + !labels.iter().any(|l| l.starts_with(&format!("{HOLDER_LABEL}="))), + "a helper carrying the holder label would be reaped as a namespace holder: {labels:?}" + ); + + let argv = with_helper_labels(sidecar_argv(&NetnsHolder::adopt("h".into(), DockerCli::system()), "img"), &labels) + .expect("a docker run argv takes labels"); + assert_eq!(argv[1], "run", "the verb is untouched"); + assert!(argv.iter().any(|a| a == &format!("{HOLDER_CLEANUP_AFTER_LABEL}={stamp}")), "{argv:?}"); + // …and the same splice is refused on anything that is not a create. + with_helper_labels(list_owned_argv(&seat_b()), &labels) + .expect_err("labels spliced into `docker ps` would become filters"); + assert_eq!( + with_helper_labels(list_owned_argv(&seat_b()), &[]).expect("no labels, no change"), + list_owned_argv(&seat_b()), + "a holder with nothing to attribute stamps nothing" + ); + } + + #[cfg(feature = "acp")] + fn quick_bounds() -> FenceBounds { + FenceBounds { + fast: std::time::Duration::from_millis(10), + max: std::time::Duration::from_millis(60), + confirm: std::time::Duration::from_millis(300), + retain: std::time::Duration::from_millis(400), + } + } + + /// AGAINST A DAEMON: ONLY THIS SEAT'S EXPIRED CONTAINERS ARE REMOVED, HOLDER AND HELPER ALIKE. + /// + /// The selection above, now spent on a real `docker rm`: what the sweep reports removed is what + /// the daemon was actually asked to remove, and nothing else was touched. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "current_thread")] + async fn the_sweep_removes_only_this_seats_expired_containers() { + let work = stand_in_work_dir("sweep-basic"); + let client = DockerCli::stand_in(&stand_in_sweep_docker(&work)); + let seat = sweep_seat("sweep-basic"); + write_listing( + &work, + &[ + ("expired-holder", &seat, "1000", ROLE_HOLDER), + ("expired-helper", &seat, "1000", ROLE_HELPER), + ("live-holder", &seat, "9999", ROLE_HOLDER), + ("strangers", "ffff", "1", ROLE_HOLDER), + ("unstamped", &seat, "", ROLE_HOLDER), + ], + ); + + let report = sweep_expired_with(&client, &seat, 5_000).await.expect("the listing answered"); + + let mut removed = report.removed.clone(); + removed.sort(); + assert_eq!(removed, vec!["expired-helper".to_owned(), "expired-holder".to_owned()]); + assert!(report.failed.is_empty(), "{:?}", report.failed); + let mut asked = rm_log(&work); + asked.sort(); + assert_eq!( + asked, + vec!["expired-helper".to_owned(), "expired-holder".to_owned()], + "the daemon must not have been asked about the live, foreign or unstamped containers" + ); + let _ = std::fs::remove_dir_all(&work); + } + + /// A REMOVAL DOCKER REFUSED IS REPORTED, NOT SWALLOWED — AND THE NEXT SWEEP TRIES AGAIN. + /// + /// The retry is the property, and it is a property of the CADENCE, not of a loop inside one + /// pass: the container is still expired, so the next listing selects it again. Asserted by + /// running a second sweep against the daemon the first one left behind. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "current_thread")] + async fn a_removal_docker_refuses_is_retried_by_the_next_sweep() { + let work = stand_in_work_dir("sweep-retry"); + let client = DockerCli::stand_in(&stand_in_sweep_docker(&work)); + let seat = sweep_seat("sweep-retry"); + write_listing(&work, &[("stubborn", &seat, "1000", ROLE_HOLDER)]); + std::fs::write(work.join("rmfail-stubborn"), "").expect("marker"); + + let first = sweep_expired_with(&client, &seat, 5_000).await.expect("the listing answered"); + assert!(first.removed.is_empty(), "a refused removal is not a removal"); + assert_eq!(first.failed.len(), 1, "and it is reported: {:?}", first.failed); + assert_eq!(first.failed[0].0, "stubborn"); + + // The daemon relents; nothing re-registered the container anywhere. + std::fs::remove_file(work.join("rmfail-stubborn")).expect("marker"); + let second = sweep_expired_with(&client, &seat, 5_000).await.expect("the listing answered"); + assert_eq!(second.removed, vec!["stubborn".to_owned()], "the next sweep selects it again"); + assert_eq!(rm_log(&work), vec!["stubborn".to_owned(), "stubborn".to_owned()]); + let _ = std::fs::remove_dir_all(&work); + } + + /// A LISTING DOCKER COULD NOT ANSWER IS AN ERROR, NOT AN EMPTY SWEEP. + /// + /// "Docker did not answer" and "nothing is expired" are the same value to a caller that only + /// counts removals, and reporting the first as the second is how a sweep claims success for a + /// host it never looked at. The operator log distinguishes them because this returns `Err`. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "current_thread")] + async fn a_listing_docker_could_not_answer_is_an_error_not_an_empty_sweep() { + let work = stand_in_work_dir("sweep-down"); + let client = DockerCli::stand_in(&stand_in_sweep_docker(&work)); + write_listing(&work, &[("expired", &seat_b(), "1000", ROLE_HOLDER)]); + std::fs::write(work.join("daemon-down"), "").expect("marker"); + + let error = sweep_expired_with(&client, &seat_b(), 5_000) + .await + .expect_err("a daemon that cannot be reached has not shown that nothing is expired"); + assert!(error.contains("could not list"), "{error}"); + assert!(rm_log(&work).is_empty(), "nothing may be removed on an unread host"); + + // And an empty seat is refused before any call is made at all. + sweep_expired_with(&client, " ", 5_000).await.expect_err("no seat, no sweep"); + let _ = std::fs::remove_dir_all(&work); + } + + /// A CONTAINER THAT APPEARS AFTER A SWEEP IS REMOVED BY THE NEXT ONE — WITH NO REGISTRY. + /// + /// This is the whole replacement for continuous custody, and the reason the expiry lives on the + /// container instead of in this process. The first sweep sees an empty host. The container then + /// appears — a create the daemon materialised after the process that asked for it was gone, or + /// one made by a seller that has since been killed and restarted — and the next sweep removes it + /// on the strength of its own label alone. Nothing between the two calls remembers anything: a + /// fresh `sweep_expired_with` is exactly what a restarted seller's first tick performs. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "current_thread")] + async fn a_container_that_appears_after_a_sweep_is_removed_by_the_next_one() { + let work = stand_in_work_dir("sweep-late"); + let client = DockerCli::stand_in(&stand_in_sweep_docker(&work)); + let seat = sweep_seat("sweep-late"); + write_listing(&work, &[]); + + let first = sweep_expired_with(&client, &seat, 5_000).await.expect("the listing answered"); + assert!(first.removed.is_empty() && first.failed.is_empty(), "an empty host is not a leak"); + + // The late arrival, carrying the stamp written at ITS create. + write_listing(&work, &[("late-arrival", &seat, "1000", ROLE_HOLDER)]); + let second = sweep_expired_with(&client, &seat, 5_000).await.expect("the listing answered"); + assert_eq!(second.removed, vec!["late-arrival".to_owned()]); + + // …and it is really gone from the daemon, so a third sweep has nothing to do. + let third = sweep_expired_with(&client, &seat, 5_000).await.expect("the listing answered"); + assert!(third.removed.is_empty(), "a removed container must not be selected forever"); + let _ = std::fs::remove_dir_all(&work); + } + + /// ONE SWEEP REMOVES AT MOST ITS BOUND, AND THE BACKLOG CLEARS ACROSS LATER SWEEPS. + /// + /// A host that accumulated hundreds of leftovers must not hand this loop an unbounded queue of + /// removals on one tick — the seller stops answering offers while it drains. The remainder is not + /// lost: it is still expired on the next tick. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "current_thread")] + async fn one_sweep_removes_at_most_its_bound_and_the_rest_wait_for_the_next() { + let work = stand_in_work_dir("sweep-bound"); + let client = DockerCli::stand_in(&stand_in_sweep_docker(&work)); + let seat = sweep_seat("sweep-bound"); + let ids: Vec = (0..MAX_SWEEP_REMOVALS + 5).map(|n| format!("c{n}")).collect(); + let rows: Vec<(&str, &str, &str, &str)> = + ids.iter().map(|id| (id.as_str(), seat.as_str(), "1000", ROLE_HOLDER)).collect(); + write_listing(&work, &rows); + + let first = sweep_expired_with(&client, &seat, 5_000).await.expect("the listing answered"); + assert_eq!(first.removed.len(), MAX_SWEEP_REMOVALS, "one tick's bounded budget"); + // The remainder is REPORTED, not silently dropped: "removed 32" on a host with 37 expired + // containers must not read as "the host is clean now". + assert_eq!(first.deferred.len(), 5, "the backlog this pass did not attempt is named"); + assert!( + first.deferred.iter().all(|id| !first.removed.contains(id)), + "a container cannot be both removed and deferred" + ); + assert_eq!( + rm_log(&work).len(), + MAX_SWEEP_REMOVALS, + "the deferred ones were never even asked about" + ); + let second = sweep_expired_with(&client, &seat, 5_000).await.expect("the listing answered"); + assert_eq!(second.removed.len(), 5, "the backlog clears on the following ticks"); + assert!(second.deferred.is_empty(), "and nothing is left over from that pass"); + let third = sweep_expired_with(&client, &seat, 5_000).await.expect("the listing answered"); + assert!(third.removed.is_empty(), "and then there is nothing left to do"); + let _ = std::fs::remove_dir_all(&work); + } + + /// A PASS IS BOUNDED IN WALL CLOCK, NOT ONLY IN COUNT — AND SAYS WHAT IT DID NOT REACH. + /// + /// `MAX_SWEEP_REMOVALS` bounds how many containers a pass removes, which is not a bound on how + /// long the pass takes: the removals are serial, each carries `SWEEP_DOCKER_DEADLINE`, and a + /// daemon that accepts the connection and then hangs makes 32 of them 640 s of work — past two + /// further ticks. `SWEEP_PASS_BUDGET` is the bound on the pass itself. Spent, the pass stops + /// STARTING removals and reports the rest as deferred; nothing is lost, because every one of + /// them is still expired when the next tick selects it. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "current_thread")] + async fn a_pass_stops_at_its_wall_clock_budget_and_defers_what_it_did_not_start() { + let work = stand_in_work_dir("sweep-budget"); + let client = DockerCli::stand_in(&stand_in_sweep_docker(&work)); + let seat = sweep_seat("sweep-budget"); + let ids: Vec = (0..6).map(|n| format!("slow{n}")).collect(); + let rows: Vec<(&str, &str, &str, &str)> = + ids.iter().map(|id| (id.as_str(), seat.as_str(), "1000", ROLE_HOLDER)).collect(); + write_listing(&work, &rows); + + // A budget already spent when the pass reaches its first removal. Deterministic — no sleep, + // no timing window: the listing alone is enough elapsed time for a zero budget. + let spent = sweep_expired_within(&client, &seat, 5_000, std::time::Duration::ZERO) + .await + .expect("the listing still answered — the budget bounds removals, not the read"); + assert!(spent.removed.is_empty(), "a spent budget starts no removal"); + assert!(rm_log(&work).is_empty(), "and docker is never asked"); + assert_eq!(spent.deferred.len(), 6, "every selected container is accounted for"); + + // Nothing about that pass consumed the work: the next one, with a real budget, does it all. + let next = sweep_expired_within(&client, &seat, 5_000, SWEEP_PASS_BUDGET) + .await + .expect("the listing answered"); + assert_eq!(next.removed.len(), 6, "the deferred backlog is removed by the following pass"); + assert!(next.deferred.is_empty()); + let _ = std::fs::remove_dir_all(&work); + } + + /// THE DEFERRED BACKLOG IS OLDEST-FIRST, SO A PERMANENT BACKLOG STILL DRAINS. + /// + /// The count bound and the wall-clock bound defer different containers, and the ones the budget + /// stopped were selected EARLIER than the ones the count bound never looked at. Reporting them + /// in selection order is what makes the next pass reach the oldest leftovers first instead of + /// re-starting at whatever docker listed first this time. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "current_thread")] + async fn the_deferred_backlog_keeps_the_selection_order_the_next_pass_needs() { + let work = stand_in_work_dir("sweep-order"); + let client = DockerCli::stand_in(&stand_in_sweep_docker(&work)); + let seat = sweep_seat("sweep-order"); + let ids: Vec = (0..MAX_SWEEP_REMOVALS + 3).map(|n| format!("o{n:03}")).collect(); + let rows: Vec<(&str, &str, &str, &str)> = + ids.iter().map(|id| (id.as_str(), seat.as_str(), "1000", ROLE_HOLDER)).collect(); + write_listing(&work, &rows); + + let stalled = sweep_expired_within(&client, &seat, 5_000, std::time::Duration::ZERO) + .await + .expect("the listing answered"); + assert_eq!( + stalled.deferred, ids, + "budget-stopped names come first, count-bound names after, both in selection order" + ); + let _ = std::fs::remove_dir_all(&work); + } + + /// AGAINST A DAEMON: A PERSISTENTLY FAILING HEAD DOES NOT MONOPOLISE THE PASS. + /// + /// The #996 D2 defect, driven through the real `sweep_expired_with` rather than the selection + /// alone. Every removal here is refused, on every pass, so the candidate set never shrinks — + /// the state the old rule could not escape, because it re-selected the same first + /// [`MAX_SWEEP_REMOVALS`] ids each time and the tail was never once asked about. + /// + /// The assertion is deliberately about what the DAEMON WAS ASKED, not about what the report + /// says: `rm.log` is written by the stand-in when a removal is actually issued. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "current_thread")] + async fn a_permanently_failing_head_does_not_monopolise_the_sweep_pass() { + let work = stand_in_work_dir("sweep-starve"); + let client = DockerCli::stand_in(&stand_in_sweep_docker(&work)); + let seat = sweep_seat("sweep-starve"); + // Zero-padded so lexical order is numeric order, which is the order the pass walks. + let ids: Vec = (0..MAX_SWEEP_REMOVALS + 8) + .map(|n| format!("c{n:03}")) + .collect(); + let rows: Vec<(&str, &str, &str, &str)> = ids + .iter() + .map(|id| (id.as_str(), seat.as_str(), "1000", ROLE_HOLDER)) + .collect(); + write_listing(&work, &rows); + // EVERY removal is refused, permanently. + for id in &ids { + std::fs::write(work.join(format!("rmfail-{id}")), "").expect("marker"); + } + + let first = sweep_expired_with(&client, &seat, 5_000) + .await + .expect("the listing answered"); + assert!( + first.removed.is_empty(), + "nothing can be removed on this host" + ); + assert_eq!( + first.failed.len(), + MAX_SWEEP_REMOVALS, + "the whole pass was spent failing" + ); + assert_eq!(first.deferred.len(), 8, "and the tail is named as waiting"); + + let second = sweep_expired_with(&client, &seat, 5_000) + .await + .expect("the listing answered"); + assert!(second.removed.is_empty(), "still nothing removable"); + + // THE POINT: the tail was reached on the second pass, even though not one removal has ever + // succeeded and the head is still expired and still failing. Under the old rule the daemon + // would have been asked about `c000..c031` twice and about `c032..c039` never. + let asked: std::collections::BTreeSet = rm_log(&work).into_iter().collect(); + for id in &ids { + assert!( + asked.contains(id), + "every expired container must be attempted within ceil(40/32) = 2 passes; \ + {id} never was — a failing head is starving the tail again" + ); + } + let _ = std::fs::remove_dir_all(&work); + } + + /// AGAINST A DAEMON: WHAT THE SWEEP REFUSES TO TOUCH COMES BACK NAMED. + /// + /// #996 D1 through the real pass. Each unusable row is long past any stamp, so expiry is not + /// what holds it back — the validation is — and the daemon is asked about none of them. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "current_thread")] + async fn the_sweep_reports_every_container_it_refused_to_act_on() { + let work = stand_in_work_dir("sweep-skipped"); + let client = DockerCli::stand_in(&stand_in_sweep_docker(&work)); + let seat = sweep_seat("sweep-skipped"); + write_raw_listing( + &work, + &format!( + "good\t{seat}\t1000\t{ROLE_HOLDER}\t\tjob-1\n\ + bad-stamp\t{seat}\tnot-a-number\t{ROLE_HOLDER}\t\tjob-2\n\ + no-job\t{seat}\t1000\t{ROLE_HELPER}\t\t\n\ + odd-role\t{seat}\t1000\tinterloper\tjob-3\t\n" + ), + ); + + let report = sweep_expired_with(&client, &seat, 5_000) + .await + .expect("the listing answered"); + assert_eq!( + report.removed, + vec!["good".to_owned()], + "only the valid, expired one" + ); + assert_eq!( + rm_log(&work), + vec!["good".to_owned()], + "the daemon is asked about nothing else" + ); + + let named: std::collections::BTreeMap = report + .skipped + .iter() + .map(|(id, why)| (id.clone(), why.to_string())) + .collect(); + assert_eq!( + named.keys().cloned().collect::>(), + vec![ + "bad-stamp".to_owned(), + "no-job".to_owned(), + "odd-role".to_owned() + ], + "the leak has to be visible in the report, not only absent from the removals: {named:?}" + ); + assert!(named["bad-stamp"].contains("stamp"), "{named:?}"); + assert!(named["no-job"].contains("job"), "{named:?}"); + assert!(named["odd-role"].contains("interloper"), "{named:?}"); + let _ = std::fs::remove_dir_all(&work); + } + + /// Cleanup owns the JOINERS too, and must confirm each one is really gone. + /// + /// `sweep` only LOGS a failed sidecar removal, and confirmation inspected the holder alone. A + /// sidecar that refused removal and is still running pins the very namespace the holder was + /// torn down to release — so an owner that ends on holder-absence alone reports a clean release + /// on top of a container it owns and never looked at. The daemon has to be asked about every + /// owned name, not just the convenient one. + #[cfg(feature = "acp")] + #[test] + fn cleanup_confirms_every_owned_joiner_is_absent_not_only_the_holder() { + let work = stand_in_work_dir("joiner-confirm"); + let script = stand_in_docker(&work, ""); + // This sidecar refuses removal AND keeps answering "present": precisely the case that + // holder-only confirmation reports as clean. + std::fs::write(work.join("rmfail-side-1"), "").expect("marker"); + std::fs::write(work.join("present-side-1"), "").expect("marker"); + + let cleanup = HolderCleanup { + name: "holder-joiner-confirm".to_owned(), + joiners: vec!["side-1".to_owned()], + // Nothing in flight, so settlement is immediate and this test is only about custody. + creation: std::sync::Arc::new(CreationFence::default()), + client: DockerCli::stand_in(&script), + bounds: quick_bounds(), + }; + cleanup.own_until_settled_or_confirmed(); + + let log = std::fs::read_to_string(work.join("events.log")).unwrap_or_default(); + assert!( + log.contains("inspect side-1"), + "cleanup ended custody without ever asking the daemon whether the sidecar it owns is \ + gone. Its removal failed and it is still running, pinning the namespace, and this \ + owner reported a clean release anyway. Event log:\n{log}" + ); + let _ = std::fs::remove_dir_all(&work); + } + + /// An absence observed while a create is STILL IN FLIGHT is not proof of anything. + /// + /// This is success-shaped emptiness: "No such container" reads identically whether the create + /// never happened or has simply not landed yet. The previous owner waited out its bound, swept, + /// asked once, got "absent", and returned announcing that *nothing landed* — while the create + /// it was waiting on was still running and could land immediately afterwards, unowned. + #[cfg(feature = "acp")] + #[test] + fn cleanup_does_not_take_absence_as_proof_while_a_create_is_still_in_flight() { + let work = stand_in_work_dir("unsettled-confirm"); + let script = stand_in_docker(&work, ""); + let fence = std::sync::Arc::new(CreationFence::default()); + // Held for the whole test and never released: this create NEVER settles. + let _ticket = fence.begin(); + + let cleanup = HolderCleanup { + name: "holder-unsettled".to_owned(), + joiners: Vec::new(), + creation: std::sync::Arc::clone(&fence), + client: DockerCli::stand_in(&script), + bounds: quick_bounds(), + }; + cleanup.own_until_settled_or_confirmed(); + + let log = std::fs::read_to_string(work.join("events.log")).unwrap_or_default(); + assert!( + !log.contains("inspect holder-unsettled"), + "the create never settled, yet cleanup asked the daemon for an absence answer and ended \ + on it. That answer cannot distinguish \"nothing landed\" from \"has not landed yet\", \ + so resting a clean verdict on it is exactly the orphan this fence exists to prevent. \ + Event log:\n{log}" + ); + let _ = std::fs::remove_dir_all(&work); + } + + /// The deadline must bound the WHOLE create flow, stdin included. + /// + /// The timer used to start after the plan had already been written to the child. A client that + /// never reads its stdin fills the pipe and blocks that write forever, so the bound was never + /// armed and the launch hung with no deadline at all — the precise state in which cancellation + /// leaves work nobody owns. + #[cfg(feature = "acp")] + #[test] + fn the_deadline_bounds_the_whole_create_flow_including_the_stdin_write() { + use std::os::unix::fs::PermissionsExt as _; + + let work = stand_in_work_dir("stdin-bound"); + let script = work.join("docker"); + // Never reads stdin, so a large plan fills the pipe and the write blocks. + std::fs::write(&script, "#!/bin/sh\nsleep 30\n").expect("write stand-in"); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + + let deadline = std::time::Duration::from_millis(300); + let mut child_exited = false; + let started = std::time::Instant::now(); + let outcome = run_bounded_blocking( + &DockerCli::stand_in(&script), + vec!["docker".to_owned(), "create".to_owned()], + Some("x".repeat(4 * 1024 * 1024)), + deadline, + started, + None, + &mut child_exited, + ); + let elapsed = started.elapsed(); + + assert!( + elapsed < std::time::Duration::from_secs(10), + "the bound never armed: a client that refuses to read its stdin blocked the write for \ + {elapsed:?} against a {deadline:?} deadline. A create with no enforceable bound is a \ + launch that can hang and a container nobody is waiting for." + ); + assert!(outcome.is_err(), "a client killed on its deadline cannot report success"); + let _ = std::fs::remove_dir_all(&work); + } + + /// A container that lands AFTER the bound is still removed — by an owner that never left. + /// + /// This is the ownership hole, and it is not a reporting one: when `max` expired with the + /// create still running, the owner swept what it could see, printed a leak and RETURNED. The + /// sweep cannot touch a container that has not appeared yet, so the one case the fence exists + /// for — a create landing late — ended with no owner at all, and the container stayed up until + /// a boot reaper happened to find it. + /// + /// The assertion is therefore about the CONTAINER, not the log: the stand-in daemon answers + /// `inspect` from a presence marker and drops that marker when a removal succeeds, so this + /// passes only if the thing that landed late was actually removed, and only if the removal came + /// after it landed. + #[cfg(feature = "acp")] + #[test] + fn a_create_that_lands_after_the_bound_is_still_removed_by_its_retained_owner() { + use std::io::Write as _; + + let work = stand_in_work_dir("late-custody"); + let script = stand_in_docker(&work, ""); + let fence = std::sync::Arc::new(CreationFence::default()); + let ticket = fence.begin(); + + // The create lands strictly AFTER the owner's first sweep, and only then settles. + // + // Ordered on the observed sweep rather than on a sleep, deliberately: a wall-clock delay + // makes this test a race, and a lucky schedule where the pre-landing sweep happens to run + // late lets a dropped-custody build pass. Waiting for the removal to appear in the stand-in + // daemon's log pins the one ordering that matters — the owner has already swept, and the + // container arrives afterwards, which is exactly the case a sweep cannot cover. + let landing = work.clone(); + let lander = std::thread::spawn(move || { + let give_up = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < give_up { + let swept = std::fs::read_to_string(landing.join("rm.log")) + .map(|log| log.contains("holder-late")) + .unwrap_or(false); + if swept { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + std::fs::write(landing.join("present-holder-late"), "").expect("presence marker"); + let mut log = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(landing.join("events.log")) + .expect("events log"); + writeln!(log, "landed holder-late").expect("events log"); + drop(ticket); + }); + + let cleanup = HolderCleanup { + name: "holder-late".to_owned(), + joiners: Vec::new(), + creation: std::sync::Arc::clone(&fence), + client: DockerCli::stand_in(&script), + bounds: FenceBounds { + retain: std::time::Duration::from_secs(5), + ..quick_bounds() + }, + }; + cleanup.own_until_settled_or_confirmed(); + lander.join().expect("lander"); + + assert!( + !work.join("present-holder-late").exists(), + "the container landed after the owner's bound and is STILL RUNNING: custody was \ + dropped at `max` while the create was in flight, so nothing removed what arrived \ + afterwards. An owner that stops owning at a timeout is how this fence manufactures the \ + orphan it exists to prevent." + ); + let log = std::fs::read_to_string(work.join("events.log")).unwrap_or_default(); + let landed = log + .lines() + .position(|line| line.contains("landed holder-late")) + .expect("the stand-in create never landed, so this test proved nothing"); + assert!( + log.lines().skip(landed + 1).any(|line| line.contains("rm holder-late")), + "the only removal issued for this holder happened BEFORE it existed — a removal aimed \ + at a container that had not landed yet, which the daemon answers 'No such container' \ + and which proves nothing. Event log:\n{log}" + ); + let _ = std::fs::remove_dir_all(&work); + } + + /// A create still in flight when every bound expires is TRANSFERRED, not released. + /// + /// This is the termination path, and it is the one a longer wait cannot reach. The bounded + /// owner waited `max`, then kept the job a further `retain`, and then — with the create still + /// running — printed a leak and RETURNED. A container landing after that had no owner at all, + /// which is the same hole the retained window was added to close, one window further out. + /// + /// Ownership is asserted here as a FACT ABOUT THE FENCE, not as a sentence in a log: the fence + /// is holding somebody's cleanup, and when the create finally settles that owner removes the + /// container that landed. + #[cfg(feature = "acp")] + #[test] + fn a_create_still_in_flight_at_every_bound_is_transferred_to_a_retained_owner_that_removes_it() { + let work = stand_in_work_dir("retained-transfer"); + let script = stand_in_docker(&work, ""); + let fence = std::sync::Arc::new(CreationFence::default()); + // Taken and NOT released: this create is still in flight through every bound below. + let ticket = fence.begin(); + + let cleanup = HolderCleanup { + name: "holder-transfer".to_owned(), + joiners: Vec::new(), + creation: std::sync::Arc::clone(&fence), + client: DockerCli::stand_in(&script), + bounds: quick_bounds(), + }; + cleanup.own_until_settled_or_confirmed(); + + // The bounded owner is finished. Responsibility must NOT have ended with it. + assert!( + fence.holds_retained_owner(), + "the owner reached its last bound with the create still in flight and simply returned, \ + so nothing is responsible for a container that has not landed yet. Another window \ + would only move this same edge; the job has to belong to somebody once the wait is \ + over." + ); + + // The create lands LONG after every bound expired, and only now settles. + std::fs::write(work.join("present-holder-transfer"), "").expect("presence marker"); + drop(ticket); + + assert!( + !work.join("present-holder-transfer").exists(), + "the container landed after every bound expired and is STILL RUNNING. The retained \ + owner either never ran or never removed it — either way this is the orphan the fence \ + exists to prevent, arriving exactly where the bounded owner stopped looking." + ); + assert!( + !fence.holds_retained_owner(), + "the retained owner was never consumed, so the handoff did not actually run on \ + settlement" + ); + let _ = std::fs::remove_dir_all(&work); + } + + /// A name the daemon never confirmed absent stays OWNED. + /// + /// The removal was issued and the daemon would not say it was gone. That used to end in a log + /// line and a return: the names were named, then let go, which downstream is indistinguishable + /// from a clean release. An unconfirmed name is a container that may well still be running, so + /// ownership of it has to survive the failure to confirm it. + #[cfg(feature = "acp")] + #[test] + fn names_that_could_not_be_confirmed_absent_stay_owned_rather_than_released() { + let work = stand_in_work_dir("confirm-failed"); + let script = stand_in_docker(&work, ""); + // The container is present, and every removal against it FAILS, so the daemon keeps + // answering that it is still there and confirmation cannot succeed. + std::fs::write(work.join("present-holder-unconfirmed"), "").expect("presence marker"); + std::fs::write(work.join("rmfail-holder-unconfirmed"), "").expect("rm failure marker"); + let fence = std::sync::Arc::new(CreationFence::default()); + + let cleanup = HolderCleanup { + name: "holder-unconfirmed".to_owned(), + joiners: Vec::new(), + creation: std::sync::Arc::clone(&fence), + client: DockerCli::stand_in(&script), + bounds: quick_bounds(), + }; + cleanup.own_until_settled_or_confirmed(); + + assert!( + work.join("present-holder-unconfirmed").exists(), + "the fixture removed the container after all, so this test proved nothing about \ + unconfirmed names" + ); + assert!( + fence.holds_retained_owner(), + "the daemon never confirmed this name absent and the owner RELEASED it anyway. A \ + container that could not be confirmed gone is one that may still be running, and \ + naming it in a log is not the same as still owning it." + ); + let _ = std::fs::remove_dir_all(&work); + } + + /// A CREATE THAT SETTLED BEFORE CLEANUP REGISTERED still gets its container removed. + /// + /// This is the ordering the previous version lost outright: the bounded wait times out, the + /// last ticket drops and finds an empty slot, and only THEN does cleanup install its callback. + /// No later drop exists to run it, so the job sat in the slot until the fence died. The earlier + /// test could not catch it because it deliberately held the last ticket alive across + /// registration — the one ordering in which the bug cannot occur. + /// + /// What is asserted is the CONTAINER, not the slot: the presence marker the stand-in answers + /// `inspect` from is gone, and the removal is in the daemon's own log. + #[cfg(feature = "acp")] + #[test] + fn a_create_that_settled_before_cleanup_registered_is_still_removed() { + let work = stand_in_work_dir("settle-before-register"); + let script = stand_in_docker(&work, ""); + std::fs::write(work.join("present-holder-raced"), "").expect("presence marker"); + let fence = std::sync::Arc::new(CreationFence::default()); + + // The create ENDS FIRST: this is the last ticket, and it drops while the slot is empty. + drop(fence.begin()); + + // Only now does the bounded owner hand its job over — to a fence with nothing left in + // flight that could ever fire it. + fence.take_custody(retained_removal( + "holder-raced".to_owned(), + Vec::new(), + DockerCli::stand_in(&script), + quick_bounds(), + )); + + assert!( + !work.join("present-holder-raced").exists(), + "the create settled BEFORE cleanup registered, nothing ever ran the handed-over job, \ + and the container is still there. A registration that lands after the last ticket \ + drop has to run the job, not park it where no event can reach it." + ); + let removals = std::fs::read_to_string(work.join("rm.log")).unwrap_or_default(); + assert!( + removals.contains("holder-raced"), + "no removal was ever issued for a job handed to an already-settled fence: {removals:?}" + ); + let _ = std::fs::remove_dir_all(&work); + } + + /// THE LAST `Arc` TO THE FENCE GOING AWAY RUNS THE JOB — it does not discard it. + /// + /// The retained job is a `FnOnce`, and a `FnOnce` that is merely dropped does nothing at all. + /// The closure deliberately keeps no `Arc` back to its own fence (that would be a cycle, and + /// the fence would never be destroyed at all), so nothing held the slot alive: once the holder + /// and every ticket were gone, destruction threw the cleanup away in silence — the one outcome + /// downstream cannot tell apart from never having owned the container. + /// + /// The container is what is inspected, after every owner the test holds is gone. + #[cfg(feature = "acp")] + #[test] + fn cleanup_survives_the_loss_of_every_arc_to_the_fence_and_still_removes() { + let work = stand_in_work_dir("last-arc"); + let script = stand_in_docker(&work, ""); + std::fs::write(work.join("present-holder-lastarc"), "").expect("presence marker"); + // Removal FAILS at first, so the job cannot discharge and stays owed in the slot — the only + // state in which a fence can be destroyed while still holding cleanup. + std::fs::write(work.join("rmfail-holder-lastarc"), "").expect("rm failure marker"); + let fence = std::sync::Arc::new(CreationFence::default()); + + let cleanup = HolderCleanup { + name: "holder-lastarc".to_owned(), + joiners: Vec::new(), + creation: std::sync::Arc::clone(&fence), + client: DockerCli::stand_in(&script), + bounds: quick_bounds(), + }; + cleanup.own_until_settled_or_confirmed(); + assert!( + work.join("present-holder-lastarc").exists(), + "the fixture removed the container while removals were supposed to fail, so this test \ + proved nothing about destruction" + ); + + // The daemon stops refusing: from here a removal would genuinely succeed. + std::fs::remove_file(work.join("rmfail-holder-lastarc")).expect("clear the rm failure"); + // EVERY other owner is gone — the cleanup consumed itself — so this is the last `Arc`. + assert_eq!( + std::sync::Arc::strong_count(&fence), + 1, + "this test is only meaningful while it holds the LAST Arc to the fence" + ); + drop(fence); + + assert!( + !work.join("present-holder-lastarc").exists(), + "the last Arc to the fence was dropped while it still held cleanup, and the job went \ + with it: the container is STILL PRESENT. Destruction is the final moment this process \ + can act on a container it owns, so it has to act rather than drop a live obligation." + ); + let _ = std::fs::remove_dir_all(&work); + } + + /// A NAME THAT FAILED CONFIRMATION IS DISCHARGED LATER, BY THE OWNER THAT KEPT IT. + /// + /// Not "the slot is occupied" — that is the code's opinion of itself, and it reads the same + /// whether the owner is alive or inert. The claim under test is that the retained owner is + /// LIVE: when the container genuinely goes away later, this owner is what notices, and it stops + /// owing only then. Nothing notifies it; it has to look. + #[cfg(feature = "acp")] + #[test] + fn a_name_that_failed_confirmation_is_discharged_by_its_owner_on_the_later_real_removal() { + let work = stand_in_work_dir("confirm-later"); + let script = stand_in_docker(&work, ""); + std::fs::write(work.join("present-holder-later"), "").expect("presence marker"); + std::fs::write(work.join("rmfail-holder-later"), "").expect("rm failure marker"); + let fence = std::sync::Arc::new(CreationFence::default()); + + let cleanup = HolderCleanup { + name: "holder-later".to_owned(), + joiners: Vec::new(), + creation: std::sync::Arc::clone(&fence), + client: DockerCli::stand_in(&script), + bounds: quick_bounds(), + }; + cleanup.own_until_settled_or_confirmed(); + let looked_before = std::fs::read_to_string(work.join("events.log")) + .unwrap_or_default() + .lines() + .count(); + assert!( + work.join("present-holder-later").exists(), + "the fixture confirmed the name absent after all, so nothing was left owed" + ); + + // LATER, the container genuinely goes away: the daemon finally reaps what those failed + // removals were about, and removals start working again. + std::fs::remove_file(work.join("present-holder-later")).expect("the container goes away"); + std::fs::remove_file(work.join("rmfail-holder-later")).expect("removals work again"); + + // A later create settles on this holder's fence — the event the owner was kept for. + drop(fence.begin()); + + let looked_after = std::fs::read_to_string(work.join("events.log")) + .unwrap_or_default() + .lines() + .count(); + assert!( + looked_after > looked_before, + "the retained owner never ran again when a later create settled: it was not a live \ + owner, only a flag recording that something had once gone wrong" + ); + assert!( + !fence.holds_retained_owner(), + "the name is genuinely absent now and its owner did look, yet the job is still owed — \ + an owner that cannot discharge on the real removal never ends" + ); + let _ = std::fs::remove_dir_all(&work); + } + + // ---- Cleanup that ends: bounded in-process effort, then the expiry sweep ------------------- + // + // Petar's review of PR 996 replaced process-lifetime custody with a per-job deadline stamped on + // the container itself. Nothing below waits on an in-memory owner outliving its fence: a name + // that is still owed when the fence dies is left to `sweep_expired`, whose gates live with the + // sweep. Each gate here asserts on CONTAINER STATE (the presence marker the stand-in daemon + // answers `inspect` from) and on the daemon's own `rm.log`, never on a log string. + + // ---- Round-2 gates: F1..F4 ------------------------------------------------------------------ + + // ---- Live gates: the same paths against the real daemon on the approved VM ----------------- + // + // Run with `cargo test --features acp,wallet -p maxplayer-core --lib -- --ignored live_`. + // The client is real `docker`; the containers are real. Where a fault is injected it is injected + // at the CLIENT (a wrapper that refuses `rm` while a marker exists) and is labelled as such — the + // daemon's answers to inspect, create, events and the eventual removal are the real daemon's. + + /// Work whose budget expired IN THE QUEUE never starts a create at all. + /// + /// The clock starts before the work is queued, so a saturated blocking pool can consume the + /// entire budget before the closure runs. The previous flow spawned anyway and then bounded the + /// wait — but by then the create exists, and a container can land with its caller already past + /// its bound. That is an orphan issued knowingly. The only correct answer at that point is to + /// not start it, and what this asserts is that nothing was started. + #[cfg(feature = "acp")] + #[test] + fn work_whose_budget_expired_in_the_queue_is_refused_before_anything_is_spawned() { + let work = stand_in_work_dir("queue-expired"); + let script = stand_in_docker(&work, ""); + let deadline = std::time::Duration::from_millis(200); + // The caller's clock, taken before the work was queued. It waited three budgets for a + // thread. + let queued_at = std::time::Instant::now() - std::time::Duration::from_millis(600); + let fence = std::sync::Arc::new(CreationFence::default()); + let mut child_exited = false; + + let outcome = run_bounded_blocking( + &DockerCli::stand_in(&script), + vec!["docker".to_owned(), "run".to_owned(), "--detach".to_owned()], + Some("plan".to_owned()), + deadline, + queued_at, + Some(&fence), + &mut child_exited, + ); + + outcome.expect_err("work whose budget was spent before it began cannot report success"); + // THE ASSERTION IS THAT NOTHING WAS EVER STARTED UNDER THIS FENCE. + // + // Deliberately not the stand-in's side effects: a client that IS spawned with no budget + // left is killed within microseconds, long before a shell can reach its first line, so the + // markers it would have written are absent either way and prove nothing. (That is not a + // guess — with the pre-spawn refusal removed, the marker form of this gate stayed green.) + // + // Ticket issuance is the fact that survives that race. Spawning takes a ticket for the plan + // writer synchronously, on this thread, before any wait — so a fence that never issued one + // is a fence under which nothing was launched, whatever happened afterwards. + assert_eq!( + fence.tickets_issued(), + 0, + "work whose budget had already expired in the queue was STARTED anyway: a ticket was \ + issued under this fence, so a client was launched with nothing left to wait for it. \ + The container it creates can land with its caller already past its bound — an orphan \ + this module exists to prevent, issued knowingly. Bounding the wait afterwards is too \ + late, because by then the create exists." + ); + assert!( + !work.join("creating").exists(), + "a create against the stand-in daemon completed for work with no budget left" + ); + assert!(!child_exited, "no child can have been reaped when none was ever spawned"); + let _ = std::fs::remove_dir_all(&work); + } + + /// Reaping the client does not close its pipes: a DESCENDANT can hold them open. + /// + /// The bounded wait covered the child and stopped there. After the status came back the flow + /// ran `read_to_end` on stdout and stderr on this very thread, with no bound at all, on the + /// reasoning that a reaped child leaves closed pipes. It does not. Anything the client started + /// inherits the write ends, and `read_to_end` waits for an EOF that a living descendant never + /// sends — so the whole flow could block indefinitely immediately AFTER its deadline had been + /// satisfied. The client here exits at once and leaves a descendant holding stdout. + #[cfg(feature = "acp")] + #[test] + fn a_descendant_holding_the_output_pipe_cannot_outlast_the_bound() { + use std::os::unix::fs::PermissionsExt as _; + + let work = stand_in_work_dir("descendant-pipe"); + let script = work.join("docker"); + // SYNCHRONIZED TO THE BRANCH THIS GATE EXISTS FOR. + // + // The branch under test is the drain that runs AFTER the client has been reaped. The + // previous fixture gave the client a 400ms budget and assumed it would exit inside it; on + // an independent loaded machine it did not, the deadline killed the client before it ever + // exited, and the run took the deadline-kill path instead. The gate failed having never + // reached the code it was written to exercise, which proves nothing either way — a fixture + // that dies before its branch tests nothing. + // + // So the client exits AT ONCE with no work in front of it, the descendant announces that it + // is holding the pipe, and the budget is wide enough that arriving at the drain is not a + // race. `child_exited` is then checked FIRST, because it is the fact that says which branch + // actually ran. + // + // TWO CONSTANTS WERE REMOVED FROM THIS FIXTURE, because on a loaded machine each of them + // decided the result on its own: + // + // * The BUDGET is no longer a guess about how fast this host spawns a process. It is eight + // times a spawn measured on this host, seconds before, through this very function. A + // machine slow enough to miss that is a machine that cannot spawn at all. + // * The descendant no longer `sleep`s for a fixed span. It holds the pipe until this test + // RELEASES it, so "the drain returned while the pipe was still held" is a fact and not a + // race between two timers. That also repairs what the fixed sleep quietly cost: a six + // second hold is shorter than the four-deadline bound asserted below, so an unbounded + // drain would have finished inside the bound and this gate would have passed through the + // exact regression it exists for. Held until released, an unbounded drain runs into the + // descendant's own safety cap and the bound fails, as it must. + let trivial = work.join("docker-trivial"); + std::fs::write(&trivial, "#!/bin/sh\nexit 0\n").expect("write trivial stand-in"); + std::fs::set_permissions(&trivial, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + let measured = { + let fence = std::sync::Arc::new(CreationFence::default()); + let mut exited = false; + let at = std::time::Instant::now(); + let _ = run_bounded_blocking( + &DockerCli::stand_in(&trivial), + vec!["docker".to_owned(), "create".to_owned()], + None, + std::time::Duration::from_secs(60), + at, + Some(&fence), + &mut exited, + ); + assert!(exited, "this host could not spawn and reap a `#!/bin/sh exit 0` inside a minute"); + at.elapsed() + }; + let deadline = std::cmp::max(std::time::Duration::from_secs(2), measured * 8); + + // 0.05 s × 2400 = two minutes, the descendant's own safety cap: if an assertion below + // panics before the release, nothing is left holding a pipe on this machine indefinitely. + std::fs::write( + &script, + format!( + "#!/bin/sh\n( echo holding > \"{work}/holding\"\n i=0\n while [ ! -f \"{work}/release\" ] && [ $i -lt 2400 ]; do sleep 0.05; i=$((i+1)); done ) &\nexit 0\n", + work = work.to_string_lossy() + ), + ) + .expect("write stand-in"); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + + let fence = std::sync::Arc::new(CreationFence::default()); + let mut child_exited = false; + let started = std::time::Instant::now(); + let outcome = run_bounded_blocking( + &DockerCli::stand_in(&script), + vec!["docker".to_owned(), "create".to_owned()], + None, + deadline, + started, + Some(&fence), + &mut child_exited, + ); + let elapsed = started.elapsed(); + + assert!( + child_exited, + "the client was killed on its {deadline:?} deadline — eight times the {measured:?} this \ + host took to spawn and reap a no-op moments ago — before it ever exited, so the \ + post-exit drain this gate exists for never ran. That is the FIXTURE failing, not the \ + production code: it has to reach the branch under test before it can say anything \ + about it." + ); + assert!( + work.join("holding").exists(), + "the descendant never announced that it had the pipe, so nothing was holding stdout \ + open and the drain had nothing to be blocked by" + ); + assert!( + elapsed < deadline * 4, + "the flow ran unbounded AFTER the child was reaped: a descendant held stdout open and \ + the drain waited {elapsed:?} against a {deadline:?} bound. A create whose tail is \ + unbounded is a create nobody is waiting on." + ); + outcome.expect_err( + "an output this process never finished reading is not a result it may report", + ); + // OWNERSHIP, not wording. The call is over and a reader thread still holds the create's + // stdout. While it does, the fence must NOT read as settled: a settled fence is cleanup's + // permission to start removing, and that permission cannot be granted while this process is + // still doing the create's IO. The drain threads used to be ticketless, so the fence fell + // silent the instant this function returned and the reader became work nobody was counted + // for. + assert!( + !fence.wait_until_settled(std::time::Duration::from_millis(300)), + "the fence reported this create SETTLED while a reader thread still held its stdout \ + open. Cleanup is entitled to remove on that answer, so unowned IO work outlived the \ + ticket that was supposed to cover it." + ); + // The release is the test's own act, so what follows is measured from a known event rather + // than from a sleep that may or may not have elapsed yet. + std::fs::write(work.join("release"), "go").expect("release the descendant"); + // And it is not owned forever. When the descendant lets go, the read ends and the ticket + // goes with it: retained ownership means the lifecycle closes on the real event, not that + // it never closes. + assert!( + fence.wait_until_settled(std::time::Duration::from_secs(20)), + "the reader's ticket was never released even after the descendant exited and stdout \ + reached EOF, so this fence can never settle and cleanup could never run" + ); + let _ = std::fs::remove_dir_all(&work); + } + + /// A writer that outlives the deadline is NAMED, never silently dropped. + /// + /// On the deadline path the flow killed the child, returned, and dropped the writer handle on + /// the way out. Killing the direct client normally closes the read end and the blocked write + /// fails with `EPIPE` — but a descendant holding that end open defeats exactly that, leaving a + /// thread still writing a plan into a pipe while this call reports the command finished. The + /// returned error has to carry that outstanding custody instead of implying a settled flow. + #[cfg(feature = "acp")] + #[test] + fn a_plan_writer_still_running_after_the_deadline_is_reported_not_abandoned() { + use std::os::unix::fs::PermissionsExt as _; + + let work = stand_in_work_dir("writer-outstanding"); + let script = work.join("docker"); + // Never reads stdin, so a large plan fills the pipe and the write blocks; the descendant + // keeps the READ end open, so killing the client does NOT deliver `EPIPE` to the writer and + // the outstanding-writer case is reached every time rather than by luck. + // + // The read end is parked on fd 3 on purpose. A background job in a non-interactive shell + // has its STDIN redirected to /dev/null by POSIX, so the obvious `sleep 30 &` holds nothing + // — and `sleep 30 <&0 &` does not help either, because that default is applied before the + // redirection resolves, leaving it duplicating /dev/null. An unrelated descriptor is + // inherited untouched, so fd 3 keeps the pipe genuinely open. Written the obvious way this + // test raced: the writer took `EPIPE` instead, and whether it arrived inside the grace + // decided the result — it passed single-threaded and failed under parallel load. + std::fs::write(&script, "#!/bin/sh\nexec 3<&0\nsleep 30 &\nsleep 30\n") + .expect("write stand-in"); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + + let deadline = std::time::Duration::from_millis(300); + let mut child_exited = false; + let started = std::time::Instant::now(); + let outcome = run_bounded_blocking( + &DockerCli::stand_in(&script), + vec!["docker".to_owned(), "create".to_owned()], + Some("x".repeat(4 * 1024 * 1024)), + deadline, + started, + None, + &mut child_exited, + ); + let elapsed = started.elapsed(); + + assert!( + elapsed < std::time::Duration::from_secs(10), + "the bound never armed: {elapsed:?} against {deadline:?}." + ); + let error = outcome.expect_err("a client killed on its deadline cannot report success"); + // The assertion is that the writer was ACCOUNTED FOR, not which way it went. + // + // Both dispositions are correct: the kill normally closes the read end and the blocked + // write ends on `EPIPE`, while a descendant holding that end leaves the thread running and + // it has to be named. Which one happens depends on whether the stand-in reached its + // backgrounded holder before the kill, and under parallel load it sometimes does not — an + // earlier version of this test asserted the still-running branch and failed for that reason + // alone. What must never happen, and is what the production defect did, is ending the + // deadline path having said nothing about the writer at all. + assert!( + error.contains("the thread writing its plan") || error.contains("no plan was being written"), + "the deadline path ended without accounting for the thread writing the plan — that \ + handle was dropped, so a write into a descendant-held pipe can continue while this \ + call reads as finished. Got:\n{error}" + ); + let _ = std::fs::remove_dir_all(&work); + } + + /// Time spent WAITING FOR A THREAD is time spent against the bound. + /// + /// `spawn_blocking` hands work to a pool that can be saturated, and the clock used to start + /// inside the closure — after the queue. A create could therefore sit queued for longer than + /// its entire deadline and still be handed a full fresh budget when a thread finally freed up, + /// which is not a bound on the flow at all. The clock is now taken on the caller's side and + /// passed in; this hands in a budget already mostly spent and requires the remainder to be + /// honoured rather than restarted. + #[cfg(feature = "acp")] + #[test] + fn the_bound_counts_the_time_the_work_spent_queued_for_a_thread() { + use std::os::unix::fs::PermissionsExt as _; + + let work = stand_in_work_dir("queue-time"); + let script = work.join("docker"); + std::fs::write(&script, "#!/bin/sh\nsleep 30\n").expect("write stand-in"); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + + let deadline = std::time::Duration::from_millis(600); + let queued_for = std::time::Duration::from_millis(500); + // Stood for a call that waited `queued_for` on the pool before a thread took it. + let queued_at = std::time::Instant::now() - queued_for; + let mut child_exited = false; + let entered = std::time::Instant::now(); + let outcome = run_bounded_blocking( + &DockerCli::stand_in(&script), + vec!["docker".to_owned(), "create".to_owned()], + None, + deadline, + queued_at, + None, + &mut child_exited, + ); + let spent_here = entered.elapsed(); + + assert!(outcome.is_err(), "a client killed on its deadline cannot report success"); + // What is left of the budget, plus slack for a loaded machine. A flow that restarts its + // clock on arrival instead spends the WHOLE deadline here and lands well outside this. + let remainder = deadline - queued_for + std::time::Duration::from_millis(250); + assert!( + spent_here < remainder, + "the queue wait was not counted: this call had {queued_for:?} of a {deadline:?} budget \ + already spent before it started, so at most {remainder:?} remained — yet it ran a \ + further {spent_here:?}, a full fresh deadline granted on arrival. Work that waits \ + longer than its bound for a thread would never be cut off." + ); + let _ = std::fs::remove_dir_all(&work); + } + + /// The address `establish` MEASURES is the address its rendered policy pinholes. + /// + /// The single-source property was only ever asserted against a hand-built `NetPolicy`. That + /// cannot catch the failure that matters: `establish` measuring one address and rendering the + /// plan from another, which produces a job whose firewall permits a proxy it is not pointed at, + /// or points at a proxy its firewall drops. Here the measurement comes from the stand-in client + /// and the assertion is made on the bytes production actually sent to the applier. + /// + /// The run ends at the applier's count cross-check, which is the point of interest: reaching it + /// proves the probe, the fenced holder create and the plan render all ran in production order. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn the_address_establish_measures_is_the_address_its_plan_pinholes() { + let work = stand_in_work_dir("proxy"); + let script = stand_in_docker(&work, ""); + + let outcome = establish_with( + &DockerCli::stand_in(&script), + FenceBounds::production(), + "mx-scratch", + "holder:local", + "sidecar:local", + "host.docker.internal", + "proxy-route", + "seat", + 1000, + 1000, + Some(crate::sandbox_net::PortRange::new(9000, 9002).expect("valid range")), + false, + vec!["10.0.0.53".to_owned()], + 2_000_000_000, + ) + .await; + + let error = outcome.expect_err("the stand-in applier reports a short count"); + assert!( + error.contains("containment is incomplete"), + "the run must reach the applier's count cross-check, not fail earlier: {error}" + ); + + let plan = std::fs::read_to_string(work.join("stdin.txt")) + .expect("production sent a plan to the applier"); + assert!( + plan.contains("203.0.113.77"), + "the plan must pinhole the address establish measured, not some other one:\n{plan}" + ); + + let _ = std::fs::remove_dir_all(&work); + } + + /// A CANCELLED `establish` does not leave the container it created behind. + /// + /// Cancellation mid-create is the production shape of the delayed-create race: the future is + /// dropped while the blocking create is still running, and a cleanup that races it issues a + /// remove for a container that does not exist yet. The container then arrives, unowned, pinning + /// a namespace with nobody left to remove it. + /// + /// Two assertions, and both are needed: cleanup must OUTLAST the create (otherwise the removal + /// it issued named nothing), and it must actually name the holder (otherwise it waited and then + /// removed nothing). + #[cfg(feature = "acp")] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_cancelled_establish_outlasts_its_create_and_removes_the_holder() { + let work = stand_in_work_dir("cancel"); + let script = stand_in_docker(&work, "sleep 1"); + + // Bound to the test, not to the call expression: the future below outlives the statement + // that builds it, so the client it borrows has to as well. + let client = DockerCli::stand_in(&script); + let mut establishing = Box::pin(establish_with( + &client, + FenceBounds::production(), + "mx-scratch", + "holder:local", + "sidecar:local", + "host.docker.internal", + "cancelled-establish", + "seat", + 1000, + 1000, + None, + false, + vec!["10.0.0.53".to_owned()], + 2_000_000_000, + )); + + // Cancel on the CREATE ITSELF, not on a stopwatch. A fixed deadline raced the probe and + // cancelled before the create had begun, which measures nothing: the marker is written by + // the stand-in as the create starts, so the drop below always lands mid-create. + let marker = work.join("creating"); + let give_up = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !marker.exists() { + tokio::select! { + _ = establishing.as_mut() => panic!("establish cannot finish against this client"), + _ = tokio::time::sleep(std::time::Duration::from_millis(5)) => {} + } + assert!(std::time::Instant::now() < give_up, "the create never started"); + } + + let started = std::time::Instant::now(); + drop(establishing); // the cancellation under test; the holder's cleanup runs in here + let elapsed = started.elapsed(); + + assert!( + elapsed >= std::time::Duration::from_millis(700), + "cancellation returned in {elapsed:?}, while the create it had to outlast was still \ + running: the container arrives afterwards with nobody holding it" + ); + + let removed = std::fs::read_to_string(work.join("rm.log")).unwrap_or_default(); + assert!( + removed.contains("cancelled-establish"), + "a cancelled establish must remove the holder it created, got {removed:?}" + ); + + let _ = std::fs::remove_dir_all(&work); + } + + /// Cleanup whose wait for an in-flight create **times out** still removes only AFTER that create + /// has settled. + /// + /// This is the delayed path, and it is the one the previous version got wrong. It waited 30s for + /// a create the client itself allows 120s to run, and on expiry it removed anyway and printed + /// that the container might be LEAKED. Every part of that is the bug: the removal names a + /// container that does not exist yet, docker answers "No such container", cleanup treats that as + /// success, and the create then lands with nobody holding it. The log line did not make it safe; + /// it only made it documented. + /// + /// The bound is a parameter purely so this can be measured: `fast` expires here while the create + /// is still running, which is exactly the production shape at a scale a test can wait out. The + /// assertion is an ORDERING, not a duration — `rm` must appear after `create-end` in the client's + /// own event log — because the property under test is "never removes ahead of a live create", + /// and a stopwatch would pass for a version that simply slept longer before making the same + /// mistake. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn cleanup_that_outwaits_its_bound_removes_only_after_the_create_settles() { + let work = stand_in_work_dir("late"); + let script = stand_in_docker(&work, "sleep 1"); + let client = DockerCli::stand_in(&script); + // `fast` expires mid-create; `max` is generous enough that the owner waits the create out. + let bounds = FenceBounds { + fast: std::time::Duration::from_millis(50), + max: std::time::Duration::from_secs(30), + confirm: std::time::Duration::from_secs(10), + retain: std::time::Duration::from_secs(30), + }; + + let mut establishing = Box::pin(establish_with( + &client, + bounds, + "mx-scratch", + "holder:local", + "sidecar:local", + "host.docker.internal", + "late-cleanup", + "seat", + 1000, + 1000, + None, + false, + vec!["10.0.0.53".to_owned()], + 2_000_000_000, + )); + + // Cancel on the create itself, so the drop below always lands while it is in flight. + let marker = work.join("creating"); + let give_up = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !marker.exists() { + tokio::select! { + _ = establishing.as_mut() => panic!("establish cannot finish against this client"), + _ = tokio::time::sleep(std::time::Duration::from_millis(5)) => {} + } + assert!(std::time::Instant::now() < give_up, "the create never started"); + } + drop(establishing); + + // The owner runs past this scope, so the removal is awaited here rather than assumed. + let events = work.join("events.log"); + let give_up = std::time::Instant::now() + std::time::Duration::from_secs(20); + loop { + let log = std::fs::read_to_string(&events).unwrap_or_default(); + if log.contains("rm ") { + break; + } + assert!( + std::time::Instant::now() < give_up, + "cleanup never removed the holder at all; the event log was {log:?}" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + + let log = std::fs::read_to_string(&events).expect("the stand-in recorded its calls"); + let lines: Vec<&str> = log.lines().collect(); + let create_end = lines + .iter() + .position(|line| line.trim() == "create-end") + .expect("the create ran to completion"); + let removed = lines + .iter() + .position(|line| line.starts_with("rm ") && line.contains("late-cleanup")) + .expect("cleanup removed the holder it created"); + assert!( + removed > create_end, + "cleanup removed the holder at step {removed} but the create only settled at step \ + {create_end}: the remove was issued ahead of a live create, which docker answers \ + \"No such container\" and cleanup then treats as done — the container lands afterwards \ + unowned. Event log:\n{log}" + ); + + let _ = std::fs::remove_dir_all(&work); + } + + /// The same delayed-create failure, reproduced on the **sidecar** path rather than the holder's. + /// + /// This is the half that registration alone does not cover, and the distinction the R3 verdict + /// drew: pre-registering the name tells cleanup WHAT to remove and nothing about WHEN the + /// container appears. A sidecar create still in flight when the holder drops gets removed by + /// name, answered "No such container" because it does not exist yet, and marked done — then it + /// lands, pinning the namespace the holder was tearing down. + /// + /// The future is genuinely CANCELLED here (dropped by `timeout`) while its blocking work runs + /// on, which is the real shape of the bug: cancelling the future must not release the fence. + /// Remove the `Some(holder.fence_creation())` argument in `run_sidecar_confirmed` and this + /// fails, because `Drop` returns while the create is still running. + #[cfg(feature = "acp")] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_sidecar_create_still_in_flight_fences_cleanup() { + use std::os::unix::fs::PermissionsExt as _; + + let dir = std::env::temp_dir().join(format!("mx-sc-fence-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let slow = dir.join("slow"); + std::fs::write(&slow, "#!/bin/sh\nsleep 1\n").expect("write slow"); + std::fs::set_permissions(&slow, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + + fn never_asked(_client: &DockerCli, _name: &str) -> Option { + panic!("a cancelled create must not reach the absence check") + } + + let holder = NetnsHolder::adopt("maxplayer-netns-sidecar-fence".into(), DockerCli::system()); + + // Cancel the future ~100ms in, leaving roughly 900ms of blocking create still running. + let cancelled = tokio::time::timeout( + std::time::Duration::from_millis(100), + run_sidecar_confirmed( + &holder, + "iface", + vec![slow.to_string_lossy().into_owned(), "run".to_owned()], + None, + std::time::Duration::from_secs(10), + never_asked, + ), + ) + .await; + assert!(cancelled.is_err(), "the future must have been cancelled, not completed"); + assert_eq!( + holder.sidecars.lock().expect("registry").len(), + 1, + "a cancelled create must leave its name a cleanup target" + ); + + let started = std::time::Instant::now(); + drop(holder); + let waited = started.elapsed(); + + assert!( + waited >= std::time::Duration::from_millis(400), + "cleanup returned in {waited:?}, while the sidecar create it had to outlast was still \ + running: every remove it issued named a container that did not exist yet, and the one \ + that arrives afterwards pins the namespace with nobody holding it" + ); + + let _ = std::fs::remove_dir_all(&dir); + } + + /// The delayed-create timing failure, reproduced at the site that must fence it. + /// + /// The create runs on a blocking thread; cancelling the future above it does not stop that + /// thread. Cleanup used to run immediately, ask docker to remove a container that did not exist + /// YET, be told "No such container" — which is treated as success — and return satisfied, after + /// which the create landed and left an untracked container. + /// + /// Reproduced here by holding a creation ticket that is released 700ms from now, as an in-flight + /// create would be, and then dropping the holder. The assertion is not "the fence helper works": + /// it is that **`Drop` had not finished before the create settled**. Remove the + /// `wait_until_settled` call from `Drop` and this fails, because `Drop` returns while the flag + /// is still false. + /// + /// `Drop` does issue real `docker rm` calls, which on this path answer "No such container"; they + /// are not what makes this pass, and the elapsed-time assertion below is deliberately well under + /// the settle delay so a slow removal cannot substitute for the wait. + #[cfg(feature = "acp")] + #[test] + fn cleanup_does_not_remove_ahead_of_a_create_that_is_still_in_flight() { + use std::sync::atomic::{AtomicBool, Ordering}; + + let holder = NetnsHolder::adopt("maxplayer-netns-fence-probe".into(), DockerCli::system()); + let settled = std::sync::Arc::new(AtomicBool::new(false)); + + let ticket = holder.fence_creation(); + let flag = std::sync::Arc::clone(&settled); + let creating = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(700)); + // The create finishes: the container now exists, and only now is removing it sound. + flag.store(true, Ordering::SeqCst); + drop(ticket); + }); + + let started = std::time::Instant::now(); + drop(holder); + let waited = started.elapsed(); + + assert!( + settled.load(Ordering::SeqCst), + "cleanup finished while a create was still in flight: every remove it issued was \ + aimed at a container that did not exist yet, and the one that arrived afterwards is \ + an orphan no one holds" + ); + assert!( + waited >= std::time::Duration::from_millis(500), + "cleanup returned in {waited:?}, far sooner than the create it had to outlast — it \ + cannot have waited for the fence" + ); + creating.join().expect("the create thread"); } } diff --git a/crates/maxplayer-core/src/seller_exec.rs b/crates/maxplayer-core/src/seller_exec.rs index 74ff3e945..3737ad706 100644 --- a/crates/maxplayer-core/src/seller_exec.rs +++ b/crates/maxplayer-core/src/seller_exec.rs @@ -184,7 +184,18 @@ impl std::error::Error for ProbeRunError {} /// while a probe that cannot answer inside its own health-check limit still fails the probe. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum AgentRunTimeout { - JobDeadline(Duration), + /// A real job: the window still remaining, AND the absolute unix second it ends at. + /// + /// **Both, deliberately.** The remaining window is what the run is bounded by; the absolute + /// deadline is what any durable artefact of the run — notably a container's cleanup stamp — must + /// be derived from. Reconstructing the second from the first at some later instant (#996 F3) + /// makes it a function of whatever the wall clock said then, which is exactly the property a + /// deadline must not have. + JobDeadline { + remaining: Duration, + deadline_unix: u64, + }, + /// A harness probe, which has no job and therefore no job deadline — only its own limit. HarnessProbe(Duration), } @@ -192,7 +203,19 @@ impl AgentRunTimeout { #[cfg(feature = "acp")] fn duration(self) -> Duration { match self { - Self::JobDeadline(duration) | Self::HarnessProbe(duration) => duration, + Self::JobDeadline { remaining, .. } | Self::HarnessProbe(remaining) => remaining, + } + } + + /// The absolute deadline this run is bounded by, when it is a job's. + /// + /// `None` for a harness probe: there is no job deadline to carry, and inventing one from the + /// probe's limit would be the same reconstruction this exists to avoid. + #[cfg(feature = "acp")] + fn deadline_unix(self) -> Option { + match self { + Self::JobDeadline { deadline_unix, .. } => Some(deadline_unix), + Self::HarnessProbe(_) => None, } } } @@ -1963,7 +1986,7 @@ pub enum CleanupPolicy { /// an abandoned probe container leaks exactly the same way. pub fn cleanup_policy(timeout: AgentRunTimeout) -> CleanupPolicy { match timeout { - AgentRunTimeout::JobDeadline(_) => CleanupPolicy::CaptureThenRemove, + AgentRunTimeout::JobDeadline { .. } => CleanupPolicy::CaptureThenRemove, AgentRunTimeout::HarnessProbe(_) => CleanupPolicy::RemoveOnly, } } @@ -2672,7 +2695,15 @@ pub async fn run_agent_job_in_env( use crate::event::JobId; use crate::log::EventLog; - let prepared = prepare_launch(agent_command, policy, workdir, identity, timeout.duration()).await?; + let prepared = prepare_launch( + agent_command, + policy, + workdir, + identity, + timeout.duration(), + timeout.deadline_unix(), + ) + .await?; let mut session_mcp_servers = prepared.mcp_servers.clone(); session_mcp_servers.extend(attachments.mcp_servers); let job = JobLaunch { @@ -2808,6 +2839,7 @@ pub(crate) async fn prepare_launch( workdir: &Path, identity: &DeliveryAgentIdentity, job_lifetime: Duration, + job_deadline_unix: Option, ) -> Result { // Run the container/process as the seller's own uid/gid so a docker bind-mount's output is owned // by the seller and the delivery snapshot can read it. Ignored by the host executors. @@ -2878,6 +2910,29 @@ pub(crate) async fn prepare_launch( )) })?; job_resolv_conf = Some(resolv_path); + // The expiry this job's containers will be judged by: **the job's own absolute + // deadline**, carried down from the call site that chose it, with `cleanup_after_unix` + // adding the grace on top. + // + // It is NOT reconstructed as `now + remaining`. That reconstruction (#996 F3) read a + // fresh wall clock at create time, which made the stamp a measurement rather than a + // restatement: any backward clock step between the caller computing the remaining + // window and this create being issued lands the stamp EARLIER than the deadline the job + // is actually running under, and the sweep then removes a container out from under a + // job still inside its own deadline. Carrying the absolute value takes the clock out of + // the derivation altogether, so the stamp cannot be shortened by one. + // + // The `None` arm is reached only where there is no job deadline to carry — a harness + // probe, whose containers belong to no job. There the old derivation is still the best + // available, and an unreadable clock still yields `u64::MAX`, which is never swept, + // because an unreadable clock must not be able to date a live container into the past. + let cleanup_after = crate::sandbox_netns::launch_cleanup_stamp( + job_deadline_unix, + job_lifetime.as_secs(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(u64::MAX, |since| since.as_secs()), + ); let established = crate::sandbox_netns::establish( network, image, @@ -2892,6 +2947,7 @@ pub(crate) async fn prepare_launch( policy.proxy_ports(), true, resolvers.addresses().to_vec(), + cleanup_after, ) .await // Fail the job rather than run it uncontained. The whole point of moving containment into @@ -2984,6 +3040,62 @@ pub(crate) async fn prepare_launch( }) } +/// Run the **production** preparation and launch construction for one job, hand the resulting argv +/// to `run_payload`, and tear everything down afterwards. +/// +/// This is the entrypoint the live containment gate goes through, and it exists because the +/// alternative failed review: a gate that creates its own holder and installs its own plan proves +/// those filters *can* be installed while saying nothing about whether a real job is launched with +/// them. Here the same [`prepare_launch`] a seat calls establishes containment, the same +/// [`SandboxPolicy::launch`] builds the argv, and `netns` is wired from `holder_name` exactly as +/// [`run_agent_job_with_env`] wires it — one code path, exercised rather than re-implemented. +/// +/// `run_payload` receives the argv to execute and the holder name, and its return value is passed +/// back. The containment guard lives across the call and is dropped **after** it returns, so a +/// caller that measures cleanup can compare what it saw during the call with what survives after. +/// +/// `#[doc(hidden)]`: this is reachable so an integration test can exercise the real path, not an +/// interface for callers. Production code calls `run_agent_job*`. +#[cfg(feature = "acp")] +#[doc(hidden)] +pub async fn with_prepared_launch( + agent_command: &[String], + policy: &SandboxPolicy, + workdir: &Path, + identity: &DeliveryAgentIdentity, + job_lifetime: Duration, + job_deadline_unix: Option, + run_payload: impl FnOnce(&AgentLaunch, Option<&str>) -> R, +) -> Result { + let prepared = prepare_launch( + agent_command, + policy, + workdir, + identity, + job_lifetime, + job_deadline_unix, + ) + .await?; + let job = JobLaunch { + workdir, + env: &prepared.env, + uid: prepared.uid, + gid: prepared.gid, + netns: prepared.holder_name.as_deref(), + // The resolver the contained job is handed, exactly as `run_agent_job` hands it over + // (see the production call site). Omitting it here would launch the live containment legs + // with no `/etc/resolv.conf` mount while production launches with one, so the gates would + // measure a job that cannot resolve and call it contained. + resolv_conf: prepared.resolv_conf.as_deref(), + mcp_servers: &prepared.mcp_servers, + }; + let launch = policy.launch(&prepared.effective_command, &job)?; + let outcome = run_payload(&launch, prepared.holder_name.as_deref()); + // `prepared` drops here: proxy first, then the namespace, in the declared field order. + drop(prepared); + Ok(outcome) +} + /// Without the `acp` feature there is no containment path to prepare — fail closed. #[cfg(not(feature = "acp"))] pub(crate) async fn prepare_launch( @@ -2992,6 +3104,7 @@ pub(crate) async fn prepare_launch( _workdir: &Path, _identity: &DeliveryAgentIdentity, _job_lifetime: Duration, + _job_deadline_unix: Option, ) -> Result { Err(ExecError::AcpRequired) } @@ -3841,7 +3954,7 @@ fn classify_run_error(error: crate::engine::EngineError, timeout: AgentRunTimeou match (error, timeout) { ( crate::engine::EngineError::Driver(crate::driver::DriverError::ResponseTimeout { .. }), - AgentRunTimeout::JobDeadline(_), + AgentRunTimeout::JobDeadline { .. }, ) => ExecError::DeadlineExceeded, (error, _) => ExecError::Agent(error.to_string()), } @@ -4319,6 +4432,645 @@ mod tests { assert_ne!(job_argv, probe_argv); } + /// A REAL SELLER DEADLINE REACHES A REAL CONTAINER'S LABEL, AND DECIDES A REAL SWEEP. + /// + /// #996 R3/3 asked for exactly this, and the module did not have it. The unit test + /// `sandbox_netns::tests::the_production_deadline_reaches_the_container_and_decides_the_sweep` + /// builds the stamp BY HAND and feeds it to a hand-written listing, so it would still pass if + /// this call site stopped carrying the deadline altogether — it tests the arithmetic, not the + /// wiring. This drives the product's own path end to end: [`prepare_launch`] → + /// [`crate::sandbox_netns::launch_cleanup_stamp`] → `docker run --label` → + /// [`crate::sandbox_netns::list_owned_argv`] → [`crate::sandbox_netns::parse_owned_listing`] + /// → [`crate::sandbox_netns::partition_owned`], and reads the stamp back out of the container + /// a real daemon actually created. + /// + /// **The mutation control that makes it worth running:** the job lifetime (60s) and the job + /// deadline (+24h) are far apart ON PURPOSE, so dropping the deadline moves the recorded + /// stamp by nearly a day and reds the stamp assertion. A green that cannot go red is + /// decoration. + /// + /// **Which call site this guards — precisely, because the earlier wording did not.** This + /// test enters at [`prepare_launch`] with a deadline it constructs ITSELF, so it guards the + /// `launch_cleanup_stamp` argument at the `prepare_launch` end and nothing upstream of it. + /// It does NOT guard the forwarding edge inside [`run_agent_job_with_env`], where a real + /// seller deadline actually crosses into the launch as `timeout.deadline_unix()`: replacing + /// THAT argument with `None` leaves this test green, because this test supplies the very + /// value the mutation removes. Calling it "the production call site", singular, is how the + /// bypassed caller went unnoticed. The sibling below + /// (`a_seller_deadline_crosses_run_agent_job_with_env_into_the_daemon_label`) covers that + /// edge; both are kept, because they fail for different reasons. + /// + /// `#[ignore]` rather than an env-var early return, for the reason the probe test above states: + /// a test that returns early when its precondition is missing reports as PASSED. + #[cfg(feature = "acp")] + #[tokio::test] + #[ignore = "live: needs a docker daemon, MAXPLAYER_HOLDER_IMAGE and the netfilter sidecar image"] + async fn a_seller_deadline_reaches_the_daemon_label_and_decides_the_sweep() { + use crate::sandbox_netns::{ + cleanup_after_unix, list_owned_argv, parse_owned_listing, partition_owned, ROLE_HOLDER, + }; + + let image = std::env::var("MAXPLAYER_HOLDER_IMAGE").expect( + "set MAXPLAYER_HOLDER_IMAGE to a job image — this test measures a real container and \ + has nothing to say without one", + ); + // A seat THIS INVOCATION alone owns, and a scope that releases what it created even when + // an assertion panics. The shared constant that stood here made `list_owned_argv(&seat)` + // match every concurrent run on the daemon, so two runs with a same-second deadline could + // assert on — and then force-remove — each other's containers. + let scope = LiveScope::new("e2e"); + let seat = scope.seat.clone(); + let identity = DeliveryAgentIdentity::for_seller(&seat); + let tag = scope.network.clone(); + + // The workdir's last component IS the job id the launch derives its container names from. + let workdir = scope.workdir.clone(); + let job_id = job_id_of(&workdir); + + let policy = SandboxPolicy::docker(DockerPolicy { + image, + forward_env: Vec::new(), + runtime: std::env::var("MAXPLAYER_RUNSC_RUNTIME").ok(), + network: Some(tag.clone()), + proxy_ports: None, + file_credentials: Vec::new(), + dns_servers: Vec::new(), + container_delivery: None, + mcp_tools: Vec::new(), + }); + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("a clock") + .as_secs(); + let deadline = now + 86_400; + let lifetime = Duration::from_secs(60); + + let prepared = prepare_launch( + &["sh".to_owned()], + &policy, + &workdir, + &identity, + lifetime, + Some(deadline), + ) + .await + .expect("containment must establish"); + + // Read the labels back through the PRODUCTION listing argv, never a hand-written docker ps. + let argv = list_owned_argv(&seat); + let listed = std::process::Command::new(&argv[0]) + .args(&argv[1..]) + .output() + .expect("docker ps must run"); + let owned = parse_owned_listing(&String::from_utf8_lossy(&listed.stdout)); + assert!( + !owned.is_empty(), + "the launch's containers must be visible to the production listing argv" + ); + + let expected = cleanup_after_unix(deadline); + let if_dropped = cleanup_after_unix(now + lifetime.as_secs()); + for container in &owned { + assert_eq!( + container.cleanup_after, + Some(expected), + "the daemon recorded a stamp that is not this job's deadline plus the grace; \ + {if_dropped} would mean the call site stopped carrying Some(deadline)" + ); + } + + // Per-role provenance, asserted on labels a real daemon wrote rather than on a fixture. + let holder = owned + .iter() + .find(|container| container.role.as_deref() == Some(ROLE_HOLDER)) + .expect("the launch created a holder"); + assert_eq!( + holder.holder_job.as_deref(), + Some(job_id.as_str()), + "a production holder names its job in the holder column" + ); + assert_eq!( + holder.helper_job, None, + "and never wears the helper label — the shape the sweep must read per role" + ); + + // The decision itself, from those same records: kept inside the deadline, swept past it. + let inside = partition_owned(&owned, &seat, deadline); + assert!( + !inside.removable.contains(&holder.id), + "a holder inside its job's deadline is never removable: {inside:?}" + ); + assert!( + !inside.skipped.iter().any(|(id, _)| id == &holder.id), + "and it is readable, not refused: {inside:?}" + ); + let past = partition_owned(&owned, &seat, expected); + assert!( + past.removable.contains(&holder.id), + "past the deadline plus the grace the holder is swept: {past:?}" + ); + + drop(prepared); + // Containers, network and workdir are released by `scope` on the way out — including on + // the panic paths above, which this trailing block never reached. + } + + /// Everything one live launch owns on the daemon, released on the way out. + /// + /// The seat is the isolation boundary: `list_owned_argv` selects BY SEAT, so a seat this + /// invocation alone owns is what stops two concurrent runs from listing, asserting on, and + /// then force-removing each other's containers. Teardown lives in `Drop` because a trailing + /// block of `docker rm` calls is skipped entirely when an assertion panics — which is the + /// path a failing test takes, and therefore the path that most needs to clean up. + #[cfg(feature = "acp")] + /// What this invocation actually DID to a named resource. + /// + /// A name is not ownership. The guard holds both names from its first instant — that is what + /// closes the orphan window — but holding a name says nothing about whether this invocation + /// created the thing it names. Teardown that deletes by name alone will, after a failed + /// create, happily remove a resource that already belonged to somebody else. + /// + /// `Uncertain` is the honest third state: the attempt ran and its outcome could not be + /// established. It is REPORTED and never deleted, because deleting on a guess is exactly the + /// failure this distinction exists to prevent. + #[cfg(feature = "acp")] + #[derive(Clone, Copy)] + enum Held { + /// Named only. Nothing was created, so there is nothing to remove. + Proposed, + /// This invocation created it and saw it succeed. Teardown owns it. + Acquired, + /// The attempt ran and the result is not established. Report, never delete. + Uncertain, + } + + /// The three outcomes a docker invocation really has. + /// + /// Collapsing "the command ran and said no" into "docker could not be run" is what lets a + /// failed observation be read as a fact about the world. Teardown has to tell them apart. + #[cfg(feature = "acp")] + enum Ran { + /// Exited zero; payload is trimmed stdout. + Ok(String), + /// Ran and exited nonzero; payload is the diagnostic. + Failed(String), + /// Never ran at all; payload is why not. + Unavailable(String), + } + + #[cfg(feature = "acp")] + struct LiveScope { + seat: String, + network: String, + workdir: std::path::PathBuf, + /// Whether THIS invocation created the docker network named above. + network_state: Held, + /// Whether THIS invocation created the workdir named above. + workdir_state: Held, + /// The launch thread, owned by the scope rather than by the test body. A panic in an + /// assertion unwinds past every line after it — including a trailing `join` — so a handle + /// the body owns is a handle that gets DETACHED at exactly the moment teardown begins. + /// Owned here, it is finished and joined on both paths before a single container is + /// removed. + runner: Option>, + /// Asks a still-running launch to stop, so the unwind path does not have to wait out the + /// run's whole remaining window before it can tear down. + cancel: Option>, + } + + #[cfg(feature = "acp")] + static LIVE_SCOPE_SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + + #[cfg(feature = "acp")] + impl LiveScope { + /// Teardown's only reporting primitive. + /// + /// `eprintln!` PANICS if the write fails — documented behaviour, including a nonblocking + /// stderr returning `WouldBlock`. Reached from `Drop`, that panic either skips the cleanup + /// that follows it or double-panics during an assertion unwind and ABORTS the process, + /// destroying the very failure the test was reporting. Best-effort by construction. + fn report(line: &str) { + use std::io::Write as _; + let _ = writeln!(std::io::stderr(), "{line}"); + } + + /// Never panics. `Drop` calls this while an assertion may already be unwinding. + /// + /// The three-way result is the point: a nonzero exit means docker answered and refused, + /// while a spawn error means docker never ran. Those license different conclusions about + /// what exists, so they are not flattened into one boolean. + fn docker(args: &[&str]) -> Ran { + match std::process::Command::new("docker") + .args(args) + .stdin(std::process::Stdio::null()) + .output() + { + Ok(out) if out.status.success() => { + Ran::Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned()) + } + Ok(out) => { + let code = out + .status + .code() + .map(|c| c.to_string()) + .unwrap_or_else(|| "signal".to_owned()); + Ran::Failed(format!( + "exit {code}: {}", + String::from_utf8_lossy(&out.stderr).trim() + )) + } + Err(err) => Ran::Unavailable(format!("could not run `docker`: {err}")), + } + } + + /// 64 hex characters — the shape of a real seat — but unique to this invocation. The pid + /// alone repeats across hosts and after wrap, so the wall clock and a process-local + /// counter are mixed in; two tests in one binary must not collide either. + /// + /// The RESOURCE NAME carries its own token rather than a prefix of the seat. `&seat[..24]` + /// looked like it carried that entropy and did not: it is the 16 pid hex digits plus the + /// first 8 digits of `as_secs()`, and for a ~1.79e9 timestamp those 8 are ALL ZERO. The + /// name therefore collapsed to the pid, and pid reuse against leftovers reused the name. + /// The token below keeps the low seconds, the nanoseconds and the sequence — the fields + /// that actually differ between two invocations. + fn new(kind: &str) -> Self { + let since = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("a clock"); + let sequence = LIVE_SCOPE_SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let seat = format!( + "{:016x}{:016x}{:016x}{:016x}", + std::process::id(), + since.as_secs(), + since.subsec_nanos(), + sequence, + ); + let token = format!( + "{:08x}{:08x}{:08x}{:04x}", + std::process::id(), + (since.as_secs() & 0xffff_ffff) as u32, + since.subsec_nanos(), + (sequence & 0xffff) as u16, + ); + let network = format!("mx996-{kind}-{token}"); + let workdir = std::env::temp_dir().join(&network); + + // OWNERSHIP FIRST, but ownership is STATE, not a name. The guard exists before + // anything fallible runs, so no resource is ever created with nobody responsible for + // it; each resource is then marked acquired only when this invocation has SEEN it + // created. A panic below unwinds through `Drop`, which removes exactly what was + // acquired and reports the rest. + let mut scope = Self { + seat, + network, + workdir, + network_state: Held::Proposed, + workdir_state: Held::Proposed, + runner: None, + cancel: None, + }; + + match Self::docker(&["network", "create", &scope.network]) { + Ran::Ok(_) => scope.network_state = Held::Acquired, + Ran::Failed(why) => { + // docker answered and refused. An exit code alone cannot rule out a partial + // create, so the name is UNCERTAIN: teardown reports it and deletes nothing. + scope.network_state = Held::Uncertain; + panic!("could not create the test network {}: {why}", scope.network); + } + Ran::Unavailable(why) => { + // docker never ran, so nothing was created and the name stays PROPOSED. + panic!("could not create the test network {}: {why}", scope.network); + } + } + + // `create_dir_all` accepts an existing directory, which would let a path this + // invocation did NOT create be torn down as if it had. `create_dir` fails with + // `AlreadyExists` instead, so ownership is decided by this call rather than by + // whatever happens to be on the filesystem. + match std::fs::create_dir(&scope.workdir) { + Ok(()) => scope.workdir_state = Held::Acquired, + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { + // Not ours: left PROPOSED so teardown never removes it. The network above IS + // ours, and its cleanup still runs on the way out of this panic. + panic!( + "the workdir {} already exists and was not created by this invocation", + scope.workdir.display() + ); + } + Err(err) => { + scope.workdir_state = Held::Uncertain; + panic!("could not create the workdir {}: {err}", scope.workdir.display()); + } + } + + scope + } + + /// Hand the launch thread to the scope. After this the thread is owned on every exit + /// path, including an unwind out of a failing assertion. + fn own_runner( + &mut self, + runner: std::thread::JoinHandle<()>, + cancel: tokio::sync::oneshot::Sender<()>, + ) { + self.runner = Some(runner); + self.cancel = Some(cancel); + } + + /// Cancel-and-join. Idempotent, because `Drop` calls it too: the success path and the + /// unwind path therefore go through exactly the same ordering. + fn finish_runner(&mut self) { + if let Some(cancel) = self.cancel.take() { + // The receiver is gone if the launch already returned; that is a finished runner, + // not an error. + let _ = cancel.send(()); + } + if let Some(runner) = self.runner.take() { + if runner.join().is_err() { + // Fallible write: reporting the panic must not skip the cleanup that follows. + Self::report(&format!( + "live scope {}: the launch thread panicked", + self.network + )); + } + } + } + } + + #[cfg(feature = "acp")] + impl Drop for LiveScope { + fn drop(&mut self) { + // ORDERING IS THE CONTRACT: the runner is cancelled and JOINED before a single + // removal, so a launch cannot still be creating containers while teardown removes + // them. On a failing assertion this is the only place that join happens. + self.finish_runner(); + + let mut failures: Vec = Vec::new(); + // Two passes. The first removes what the run created; the second catches anything + // that appeared while the first was still running, which a single listing misses. + for _ in 0..2 { + let argv = crate::sandbox_netns::list_owned_argv(&self.seat); + match std::process::Command::new(&argv[0]).args(&argv[1..]).output() { + Ok(listed) if listed.status.success() => { + let owned = crate::sandbox_netns::parse_owned_listing( + &String::from_utf8_lossy(&listed.stdout), + ); + if owned.is_empty() { + break; + } + for container in &owned { + match Self::docker(&["rm", "--force", &container.id]) { + Ran::Ok(_) => {} + Ran::Failed(why) | Ran::Unavailable(why) => { + failures.push(format!("rm {}: {why}", container.id)); + } + } + } + } + Ok(listed) => { + // A FAILED OBSERVATION IS NOT AN EMPTY SET. `docker ps` exiting nonzero + // with empty stdout parses as "this seat owns nothing"; concluding that + // turns a broken instrument into a clean bill of health. Keep the status + // and stderr, report them, and stop — without claiming anything is absent. + let code = listed + .status + .code() + .map(|c| c.to_string()) + .unwrap_or_else(|| "signal".to_owned()); + failures.push(format!( + "could not list this seat's containers: exit {code}: {} \ + (ownership UNKNOWN, not empty)", + String::from_utf8_lossy(&listed.stderr).trim() + )); + break; + } + Err(err) => { + failures.push(format!( + "could not list this seat's containers: {err} \ + (ownership UNKNOWN, not empty)" + )); + break; + } + } + } + + // ONLY WHAT THIS INVOCATION ACQUIRED. After a failed create the name is not ours, and + // removing by name would delete a resource somebody else owns. + match self.network_state { + Held::Acquired => match Self::docker(&["network", "rm", &self.network]) { + Ran::Ok(_) => {} + Ran::Failed(why) | Ran::Unavailable(why) => { + failures.push(format!("network rm {}: {why}", self.network)); + } + }, + Held::Uncertain => failures.push(format!( + "network {}: creation outcome unverified; left in place, NOT removed", + self.network + )), + Held::Proposed => {} + } + + match self.workdir_state { + Held::Acquired => { + if let Err(err) = std::fs::remove_dir_all(&self.workdir) { + if err.kind() != std::io::ErrorKind::NotFound { + failures.push(format!("workdir {}: {err}", self.workdir.display())); + } + } + } + Held::Uncertain => failures.push(format!( + "workdir {}: creation outcome unverified; left in place, NOT removed", + self.workdir.display() + )), + Held::Proposed => {} + } + + // REPORTED, never discarded, and never via a macro that panics on a failed write. + // A silent cleanup failure is how a leak becomes somebody else's flake. + if !failures.is_empty() { + Self::report(&format!( + "live scope {} cleanup failures: {}", + self.network, + failures.join("; ") + )); + } + } + } + + /// The ordered edge — the one the test above does NOT reach. + /// + /// A seller's deadline does not begin at [`prepare_launch`]. It arrives at + /// [`run_agent_job_with_env`] inside an [`AgentRunTimeout::JobDeadline`] and is forwarded from + /// there as `timeout.deadline_unix()`. That forwarding argument is the wiring a regression + /// would break, and a test that enters at `prepare_launch` with its own `Some(deadline)` + /// cannot see it break, because it supplies the value the mutation removes. + /// + /// So this enters at `run_agent_job_with_env` — the same function + /// `delivery_orchestrator.rs` calls in production — and reads the labels a real daemon wrote + /// through the production listing, parser and selection. + /// + /// **Why it observes mid-flight.** The containment is released when the run ends, so the + /// labels must be read while the call is still in progress. `sh` never speaks ACP, so the + /// call is going to fail; that failure is immaterial and deliberately unasserted, because + /// containment is established BEFORE the agent handshake. Asserting on the run's result here + /// would only prove `sh` is not an ACP agent. + /// + /// **Mutation control:** replace `timeout.deadline_unix()` with `None` at the + /// `prepare_launch` call inside `run_agent_job_with_env`. The remaining window (60s) and the + /// deadline (+24h) are a day apart, so the recorded stamp moves by nearly a day and the stamp + /// assertion reds. + #[cfg(feature = "acp")] + #[tokio::test] + #[ignore = "live: needs a docker daemon, MAXPLAYER_HOLDER_IMAGE and the netfilter sidecar image"] + async fn a_seller_deadline_crosses_run_agent_job_with_env_into_the_daemon_label() { + use crate::sandbox_netns::{ + cleanup_after_unix, list_owned_argv, parse_owned_listing, partition_owned, ROLE_HOLDER, + }; + + let image = std::env::var("MAXPLAYER_HOLDER_IMAGE").expect( + "set MAXPLAYER_HOLDER_IMAGE to a job image — this test measures a real container and \ + has nothing to say without one", + ); + let mut scope = LiveScope::new("edge"); + let seat = scope.seat.clone(); + let workdir = scope.workdir.clone(); + let job_id = job_id_of(&workdir); + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("a clock") + .as_secs(); + let deadline = now + 86_400; + let remaining = Duration::from_secs(60); + + let task_image = image.clone(); + let task_network = scope.network.clone(); + let task_seat = seat.clone(); + let task_workdir = workdir.clone(); + // The production launch future is NOT `Send`: `engine::run_job` takes + // `sink: &mut dyn FnMut(RunEvent<'_>)`, so `tokio::spawn` will not accept it. Drive it on + // its own thread with a current-thread runtime — the same construction the orchestrator + // uses. Only plain owned values cross the boundary; the future is created and driven + // entirely over there, which is what makes the non-`Send` sink a non-issue. + let (cancel_tx, cancel_rx) = tokio::sync::oneshot::channel::<()>(); + let runner = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("a current-thread runtime"); + runtime.block_on(async move { + let policy = SandboxPolicy::docker(DockerPolicy { + image: task_image, + forward_env: Vec::new(), + runtime: std::env::var("MAXPLAYER_RUNSC_RUNTIME").ok(), + network: Some(task_network), + proxy_ports: None, + file_credentials: Vec::new(), + dns_servers: Vec::new(), + container_delivery: None, + mcp_tools: Vec::new(), + }); + let identity = DeliveryAgentIdentity::for_seller(&task_seat); + // EXACTLY what the orchestrator hands it: the remaining window AND the absolute + // second the job ends at. Nothing on this path passes a deadline to + // `prepare_launch` directly — that is the whole point of entering here. + let timeout = AgentRunTimeout::JobDeadline { + remaining, + deadline_unix: deadline, + }; + // Bound rather than passed as a temporary: the future is now pinned and polled + // across statements, so its argv has to outlive the call expression. + let command = ["sh".to_owned()]; + let launch = run_agent_job_with_env( + &command, + &policy, + "", + &task_workdir, + &identity, + timeout, + None, + ); + tokio::pin!(launch); + // The run's own result stays immaterial and unasserted — `sh` never speaks ACP. + // What this adds is a way for teardown to STOP the thread promptly instead of + // waiting out the remaining window while it holds up cleanup. + tokio::select! { + _ = &mut launch => {} + _ = cancel_rx => {} + } + }) + }); + // From here the scope owns the thread on every exit path, including an unwind. + scope.own_runner(runner, cancel_tx); + + let argv = list_owned_argv(&seat); + let mut owned = Vec::new(); + for _ in 0..240 { + let listed = std::process::Command::new(&argv[0]) + .args(&argv[1..]) + .output() + .expect("docker ps must run"); + owned = parse_owned_listing(&String::from_utf8_lossy(&listed.stdout)); + if !owned.is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + assert!( + !owned.is_empty(), + "run_agent_job_with_env must have created containers visible to the production \ + listing argv under this run's own seat" + ); + + let expected = cleanup_after_unix(deadline); + let if_dropped = cleanup_after_unix(now + remaining.as_secs()); + assert_ne!( + expected, if_dropped, + "the control only means something while the two stamps differ" + ); + for container in &owned { + assert_eq!( + container.cleanup_after, + Some(expected), + "the daemon recorded a stamp that is not this job's deadline plus the grace; \ + {if_dropped} would mean run_agent_job_with_env stopped forwarding \ + timeout.deadline_unix() into prepare_launch" + ); + } + + let holder = owned + .iter() + .find(|container| container.role.as_deref() == Some(ROLE_HOLDER)) + .expect("the launch created a holder"); + assert_eq!( + holder.holder_job.as_deref(), + Some(job_id.as_str()), + "a production holder names its job in the holder column" + ); + assert_eq!( + holder.helper_job, None, + "and never wears the helper label — the shape the sweep must read per role" + ); + + let inside = partition_owned(&owned, &seat, deadline); + assert!( + !inside.removable.contains(&holder.id), + "a holder inside its job's deadline is never removable: {inside:?}" + ); + let past = partition_owned(&owned, &seat, expected); + assert!( + past.removable.contains(&holder.id), + "past the deadline plus the grace the holder is swept: {past:?}" + ); + + // Cancel-and-join THROUGH THE SCOPE, so the success path uses the same ordering the + // unwind path uses: the launch is stopped and joined before anything is torn down. When + // an assertion above fails this line is never reached and `Drop` performs it instead. + scope.finish_runner(); + } + // The honest false, measured in a REAL container rather than argued from a Dockerfile. // // `#[ignore]` rather than an env-var early-return: a test that returns early when its @@ -5417,7 +6169,10 @@ mod tests { #[test] fn an_awarded_job_captures_on_both_a_successful_and_a_failed_exit() { assert_eq!( - cleanup_policy(AgentRunTimeout::JobDeadline(Duration::from_secs(60))), + cleanup_policy(AgentRunTimeout::JobDeadline { + remaining: Duration::from_secs(60), + deadline_unix: 1_000 + }), CleanupPolicy::CaptureThenRemove, "an awarded job's diagnostics are the ones a refund argument gets made from" ); @@ -7216,7 +7971,10 @@ mod tests { let deadline = classify_run_error( EngineError::Driver(DriverError::ResponseTimeout { request_id: 3 }), - AgentRunTimeout::JobDeadline(Duration::from_secs(60)), + AgentRunTimeout::JobDeadline { + remaining: Duration::from_secs(60), + deadline_unix: 1_000, + }, ); assert!(matches!(deadline, ExecError::DeadlineExceeded)); assert_eq!( @@ -7946,7 +8704,10 @@ mod tests { "task", Path::new("."), &identity, - AgentRunTimeout::JobDeadline(Duration::from_secs(1)), + AgentRunTimeout::JobDeadline { + remaining: Duration::from_secs(1), + deadline_unix: 1_000, + }, ) .await .expect_err("acp required"); @@ -8914,6 +9675,8 @@ mod mcp_tool_tests { &workdir, &identity, Duration::from_secs(300), + // A credential-bridge preparation, not a job: no deadline to stamp. + None, ) .await .expect("prepare the launch"); @@ -9168,7 +9931,16 @@ mod mcp_tool_tests { prompt, &workdir, &identity, - AgentRunTimeout::JobDeadline(Duration::from_secs(420)), + AgentRunTimeout::JobDeadline { + remaining: Duration::from_secs(420), + // The absolute second this window ends at, taken HERE at construction — the + // one instant at which `now + remaining` IS the deadline rather than a guess. + deadline_unix: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("a clock") + .as_secs() + + 420, + }, ) .await .expect("the agent turn completes"); diff --git a/crates/maxplayer-core/src/seller_node/run.rs b/crates/maxplayer-core/src/seller_node/run.rs index 6d50ee4af..cf0424a3c 100644 --- a/crates/maxplayer-core/src/seller_node/run.rs +++ b/crates/maxplayer-core/src/seller_node/run.rs @@ -4200,6 +4200,19 @@ pub(crate) fn rearm_deadline( deadline.max(boot_floor) } +/// Clears the expiry sweep's in-flight flag when the pass ends, however it ends. +/// +/// A `set(false)` at the end of the task body would be skipped by a panic, and the flag would then +/// suppress every later sweep for the life of the process — the sweep would stop silently, which is +/// exactly the failure mode the periodic design exists to avoid. +struct SweepGuard(std::rc::Rc>); + +impl Drop for SweepGuard { + fn drop(&mut self) { + self.0.set(false); + } +} + impl SellerNodeRunner { /// Boot the node and connect its authenticated relay client. /// @@ -4553,6 +4566,114 @@ impl SellerNodeRunner { self.seller_pubkey.to_hex() } + /// One pass of the expiry sweep: remove this seat's containers whose own cleanup stamp has + /// passed, and say what happened in the operator log. + /// + /// **Best-effort, never a gate, and never retried in place.** A leftover container owns a + /// namespace and carries no policy, so failing to remove one wastes a container rather than + /// opening anything — the same standing this loop already gives the boot reap. Whatever this + /// pass could not do is still expired on the next tick, which is why nothing here loops: the + /// cadence is the retry, and a docker daemon that has stopped answering must cost this loop one + /// bounded call rather than hold the seller's other work while it insists. + /// + /// A clock that cannot be read skips the pass entirely. Every removal decision here is a + /// comparison against `now`, and a `now` this process had to invent could only be wrong in the + /// direction that removes a live job's containers. + /// + /// **A free function, not a method, and awaited by NOBODY in the run loop.** It is spawned by + /// [`spawn_expiry_sweep`], so it borrows nothing from the runner and the loop's `select!` is + /// free to serve offers, awards and the shutdown signal while docker is still answering. + #[cfg(feature = "acp")] + async fn run_expiry_sweep(seat: &str) { + let Ok(now) = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH) else { + opline!( + "seller node: skipping the container expiry sweep — the system clock is before the \ + unix epoch, and no container can be judged expired against a time this process had \ + to guess" + ); + return; + }; + match crate::sandbox_netns::sweep_expired(seat, now.as_secs()).await { + Ok(report) => { + if !report.removed.is_empty() { + opline!( + "seller node: swept {} expired container(s) of this seat (past their own \ + job's deadline plus {}s)", + report.removed.len(), + crate::sandbox_netns::CLEANUP_GRACE_SECS + ); + } + for (container, error) in &report.failed { + opline!( + "seller node: could not remove expired container {container} ({error}) — \ + harmless now, and the next sweep will select it again" + ); + } + // The backlog, said out loud. "Removed 32" on a host with 400 leftovers reads as + // "the host is clean" unless the remainder is named with it. + if !report.deferred.is_empty() { + opline!( + "seller node: {} more expired container(s) were left for the next sweep \ + (this pass is bounded to {} removals and {}s of wall clock)", + report.deferred.len(), + crate::sandbox_netns::MAX_SWEEP_REMOVALS, + crate::sandbox_netns::SWEEP_PASS_BUDGET.as_secs() + ); + } + // Every container the sweep REFUSED to act on, named one by one. These are the ones + // no later pass clears on its own: an unreadable stamp does not become readable by + // ageing, and a container with no job label never acquires one. A count would leave + // the operator unable to find them, and silence — the behaviour before #996 — made + // a growing pile of them look exactly like a clean host. + for (container, reason) in &report.skipped { + opline!( + "seller node: expiry sweep skipped container {container} ({reason}) — it \ + carries this seat's label but does not establish that removing it is \ + authorised, so it is left in place and named again every pass" + ); + } + } + Err(error) => opline!( + "seller node: the container expiry sweep could not read docker ({error}) — nothing \ + was removed this pass, and expired containers stay until a later one succeeds" + ), + } + } + + /// The same entry point on a build without the docker runner, so the run loop below schedules + /// its tick unconditionally and only the work behind it is feature-gated. + #[cfg(not(feature = "acp"))] + #[allow(clippy::unused_async)] + async fn run_expiry_sweep(_seat: &str) {} + + /// Start one expiry sweep pass **as its own task** and return immediately. + /// + /// **This is what keeps the sweep off the run loop's critical path.** Awaiting the pass inside + /// the `select!` arm would stall every other arm — offers, awards, drains, the shutdown signal + /// — for as long as docker took to answer, which is bounded but not short: a listing plus up to + /// [`crate::sandbox_netns::MAX_SWEEP_REMOVALS`] serial removals, each with its own deadline. + /// Spawned, the loop's own cost is the spawn. + /// + /// `in_flight` is why a slow pass cannot stack with the next tick: a tick that arrives while a + /// pass is still running is DROPPED, not queued, so an unreachable daemon can never accumulate + /// one outstanding pass per five minutes. The flag is cleared by a guard, so a pass that panics + /// releases it too. + fn spawn_expiry_sweep(seat: String, in_flight: &std::rc::Rc>) { + if in_flight.get() { + opline!( + "seller node: skipping this container expiry sweep — the previous pass is still \ + running, and the containers it has not reached stay expired for the next one" + ); + return; + } + in_flight.set(true); + let done = SweepGuard(std::rc::Rc::clone(in_flight)); + tokio::task::spawn_local(async move { + Self::run_expiry_sweep(&seat).await; + drop(done); + }); + } + /// A handle asking this node to leave the selling role: the run loop stops, publishes its /// terminal `accepting=n` beat (#747), and [`Self::run`] returns `Ok(())`. /// @@ -5006,6 +5127,26 @@ impl SellerNodeRunner { } let mut drain_tick = tokio::time::interval(DRAIN_INTERVAL); + // The container expiry sweep rides THIS loop, for the same reason the heartbeat does: a + // side-thread would need its own shutdown, its own clock and its own reason to exist, and + // this loop already stops when the node stops. + // + // `interval` fires immediately on first poll, and here that is the point: the first sweep + // happens at startup. A seller that was `SIGKILL`ed mid-job, or one whose containers the + // daemon only materialised after it was gone, therefore rediscovers those leftovers from + // their own labels on the next boot with no memory of the jobs that made them — the boot + // reaper covers the attached-holder case, this covers everything the stamp can judge. + let mut sweep_tick = + tokio::time::interval(Duration::from_secs(crate::sandbox_netns::SWEEP_INTERVAL_SECS)); + let sweep_seat = self.seller_pubkey(); + // One pass at a time, for the life of the loop. See `spawn_expiry_sweep`. + let sweep_in_flight = std::rc::Rc::new(std::cell::Cell::new(false)); + // Only when this node actually runs contained jobs. A seat with no sandbox network creates + // no holders and no helpers, and a sweep there would spend a `docker ps` every five minutes + // to look for containers this build never creates — on a host that may not even run docker. + let sweep_enabled = SandboxPolicy::from_config(self.node.home().config.sandbox.as_ref()) + .map(|sandbox| sandbox.sandbox_network().is_some()) + .unwrap_or(false); let wrap_backfill_interval_secs = resolve_wrap_backfill_interval_secs(); let mut wrap_backfill_tick = tokio::time::interval(Duration::from_secs(wrap_backfill_interval_secs)); @@ -5096,6 +5237,15 @@ impl SellerNodeRunner { opline!("seller node: shutdown requested ({reason}); retracting the seat and ending the loop"); break; } + // Expired-container sweep. SPAWNED, never awaited here: the pass is bounded by + // count (`MAX_SWEEP_REMOVALS`), by call (`SWEEP_DOCKER_DEADLINE`) and by wall clock + // (`SWEEP_PASS_BUDGET`), but even its bounded worst case is minutes, and this loop + // must keep serving offers, awards and shutdown throughout. A tick that lands while + // the previous pass still runs is dropped rather than queued. + _ = sweep_tick.tick(), if sweep_enabled => { + Self::spawn_expiry_sweep(sweep_seat.clone(), &sweep_in_flight); + continue; + } _ = drain_tick.tick() => { self.sweep_lapsed_claims(); self.reconsider_capacity_skips().await; @@ -7493,7 +7643,10 @@ impl SellerNodeRunner { &prompt, &workdir, &identity, - AgentRunTimeout::JobDeadline(job_timeout), + AgentRunTimeout::JobDeadline { + remaining: job_timeout, + deadline_unix: deadline, + }, None, attachments.clone(), ) @@ -7930,8 +8083,18 @@ impl SellerNodeRunner { // placeholders must outlive the push, hence the margin on the lifetime. let job_lifetime = unified_job_timeout(deadline, now_unix().max(0) as u64) + Duration::from_secs(orch::PUSH_MARGIN_SECS); - let prepared = prepare_launch(agent_command, &sandbox, workdir, identity, job_lifetime) - .await + let prepared = prepare_launch( + agent_command, + &sandbox, + workdir, + identity, + job_lifetime, + // This launch legitimately outlives the job deadline by the push margin, so the margin + // is part of ITS effective deadline — the same total `job_lifetime` is measured to, but + // stated absolutely rather than re-derived from a clock read at create time. + Some(deadline.saturating_add(orch::PUSH_MARGIN_SECS)), + ) + .await .map_err(|error| Fail::Setup(format!("container launch preparation failed ({error})")))?; // The push token source. A public/anonymous https remote takes no header (as on the host). diff --git a/crates/maxplayer-core/tests/sandbox_netns_live.rs b/crates/maxplayer-core/tests/sandbox_netns_live.rs index 6b894fae2..54a4cd0a7 100644 --- a/crates/maxplayer-core/tests/sandbox_netns_live.rs +++ b/crates/maxplayer-core/tests/sandbox_netns_live.rs @@ -30,6 +30,10 @@ use std::process::Command; +use maxplayer_core::sandbox_iface::{ + filter_readback_argv, iface_sidecar_argv, link_probe_argv, parse_links, select_egress_link, + IfacePlan, +}; use maxplayer_core::sandbox_net::{Family, NetPolicy, PortRange}; use maxplayer_core::sandbox_netns::{plan_stdin, readback_argv}; @@ -76,6 +80,75 @@ fn docker(args: &[&str], stdin: Option<&str>) -> (bool, String, String) { ) } +// ── Fixture ownership ───────────────────────────────────────────────────────────────────────── +// +// Every resource these tests create is unique to this run and carries a label saying so, and +// nothing is ever removed unless that label is read back off it first. +// +// The rule exists because the alternative was in this file: fixtures named deterministically +// (`mx-runsc-net`, `mx-reap-idle`, …) and a `docker rm --force` of those names at setup, to clear +// whatever a previous run had left. That start-by-deleting step is indistinguishable from deleting +// somebody else's container — a concurrent run of this same suite, or an operator's box where the +// name happens to be taken — and it destroys the evidence of the leak it is papering over. A unique +// name needs no pre-delete, and an ownership check makes teardown provably ours. + +/// The label every fixture resource carries, with this run's token as its value. +const FIXTURE_OWNER_LABEL: &str = "ai.maxplayer.live-fixture-owner"; + +/// This run's ownership token: pid plus process start-unique nanoseconds, so two concurrent runs on +/// one host — and a rerun after a crash — never share it. +fn owner_token() -> &'static str { + static TOKEN: std::sync::OnceLock = std::sync::OnceLock::new(); + TOKEN.get_or_init(|| { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.subsec_nanos() as u64 + d.as_secs().wrapping_mul(1_000_000_000)) + .unwrap_or(0); + format!("{}-{nanos:x}", std::process::id()) + }) +} + +/// A resource name this run owns and nothing else can be using. +fn owned_name(kind: &str) -> String { + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT: AtomicU64 = AtomicU64::new(0); + format!("mx-live-{kind}-{}-{}", owner_token(), NEXT.fetch_add(1, Ordering::Relaxed)) +} + +/// The `--label` argument that stamps this run's ownership. +fn owner_label() -> String { + format!("{FIXTURE_OWNER_LABEL}={}", owner_token()) +} + +/// Whether a resource carries **this run's** ownership token. Read off the daemon, never assumed +/// from the name: the name is what a collision would reproduce, the label is what it would not. +fn owned_by_this_run(kind: &str, name: &str) -> bool { + let format = format!("{{{{index .{} \"{FIXTURE_OWNER_LABEL}\"}}}}", match kind { + "network" => "Labels", + _ => "Config.Labels", + }); + let args: Vec<&str> = match kind { + "network" => vec!["network", "inspect", "--format", &format, name], + _ => vec!["inspect", "--format", &format, name], + }; + let (ok, out, _) = docker(&args, None); + ok && out.trim() == owner_token() +} + +/// Remove a container this run created, and only if it still says it is ours. +fn remove_owned_container(name: &str) { + if owned_by_this_run("container", name) { + docker(&["rm", "--force", "--volumes", name], None); + } +} + +/// Remove a network this run created, and only if it still says it is ours. +fn remove_owned_network(name: &str) { + if owned_by_this_run("network", name) { + docker(&["network", "rm", name], None); + } +} + /// A namespace holder plus the network it sits on, torn down on drop however the test exits. struct Fixture { network: String, @@ -84,9 +157,11 @@ struct Fixture { impl Fixture { fn new(tag: &str) -> Self { - let network = format!("mx-live-net-{tag}"); - let holder = format!("mx-live-holder-{tag}"); - let (ok, _, err) = docker(&["network", "create", &network], None); + let network = owned_name(&format!("net-{tag}")); + let holder = owned_name(&format!("holder-{tag}")); + // No pre-delete: the names above did not exist a microsecond ago, so there is nothing of + // anyone's to clear, and a create that fails is a real failure rather than a stale leftover. + let (ok, _, err) = docker(&["network", "create", "--label", &owner_label(), &network], None); assert!(ok, "could not create the test network: {err}"); let (ok, _, err) = docker( &[ @@ -94,6 +169,8 @@ impl Fixture { "--detach", "--name", &holder, + "--label", + &owner_label(), "--network", &network, "--read-only", @@ -145,8 +222,8 @@ impl Fixture { impl Drop for Fixture { fn drop(&mut self) { - docker(&["rm", "--force", "--volumes", &self.holder], None); - docker(&["network", "rm", &self.network], None); + remove_owned_container(&self.holder); + remove_owned_network(&self.network); } } @@ -200,23 +277,36 @@ fn a_truncated_plan_leaves_a_namespace_the_readback_refuses() { let policy = policy("172.17.0.1"); let (plan, expected) = plan_stdin(&policy); + // The cut is expressed in terms of the v4 rule count, not as a fixed number of lines from the + // end, and that is a correctness fix rather than a tidy-up. The plan is every v4 rule followed + // by every v6 rule, so `expected - 4` silently meant "drop the v6 rules and one v4 rule" only + // while v6 had three. The moment v6 grew the two neighbour-discovery exceptions, the same cut + // removed four v6 lines and no v4 line at all: v4 verified, and the test that exists to catch a + // partial install passed while installing a complete v4 policy. + let v4_rules = policy.rule_count(Family::V4); + let truncated_to = v4_rules - 1; let short: String = plan .lines() - .take(expected - 4) + .take(truncated_to) .map(|line| format!("{line}\n")) .collect(); + assert!(truncated_to < expected, "the plan must actually be truncated"); let (ok, applied, err) = fixture.apply(&short); assert!(ok, "a short plan still applies cleanly, which is the point: {err}"); - assert_eq!(applied.parse::().expect("a count"), expected - 4); + assert_eq!(applied.parse::().expect("a count"), truncated_to); - let readback = fixture.readback(Family::V4); - let refusal = policy - .verify_readback(Family::V4, &readback) - .expect_err("a partially contained namespace must not verify"); - assert!( - refusal.contains("rules in OUTPUT"), - "the refusal must name the count it measured: {refusal}" - ); + // Both families refuse, and for different reasons: v4 is short by exactly one rule, v6 was + // never reached at all. + for family in [Family::V4, Family::V6] { + let readback = fixture.readback(family); + let refusal = policy + .verify_readback(family, &readback) + .expect_err("a partially contained namespace must not verify"); + assert!( + refusal.contains("rules in OUTPUT"), + "the refusal must name the count it measured: {refusal}" + ); + } } /// A namespace missing exactly the metadata drop must be refused, and the refusal must name it. @@ -415,15 +505,17 @@ fn reaping_removes_an_unattached_holder_and_spares_a_busy_one_and_another_seats( // Two synthetic seats. `MINE` boots and reaps; `FOREIGN` is a co-tenant that must be left alone. const MINE: &str = "1111111111111111111111111111111111111111111111111111111111111111"; const FOREIGN: &str = "2222222222222222222222222222222222222222222222222222222222222222"; - let idle = "mx-reap-idle"; - let busy = "mx-reap-busy"; - let job = "mx-reap-job"; + // Unique per run, and stamped as ours. The seats below are synthetic, so no OTHER run of this + // suite can be reaping them; that is exactly why these names must not be the same two runs + // apart, and why nothing is force-removed here before it is created. + let idle = owned_name("reap-idle"); + let busy = owned_name("reap-busy"); + let job = owned_name("reap-job"); // Deliberately unattached, like a holder in its pre-attach window: the co-tenant case that the // host-wide reaper destroyed and attachment state cannot distinguish. - let foreign = "mx-reap-cotenant"; - for name in [idle, busy, job, foreign] { - docker(&["rm", "--force", "--volumes", name], None); - } + let foreign = owned_name("reap-cotenant"); + let (idle, busy, job, foreign) = + (idle.as_str(), busy.as_str(), job.as_str(), foreign.as_str()); // Three holders carrying the real label — two mine, one another seat's — and a job joined to // exactly one of mine. @@ -435,6 +527,8 @@ fn reaping_removes_an_unattached_holder_and_spares_a_busy_one_and_another_seats( "--name", name, "--label", + &owner_label(), + "--label", &format!("{}=jobfor-{name}", maxplayer_core::sandbox_netns::HOLDER_LABEL), "--label", &format!("{}={seat}", maxplayer_core::sandbox_netns::HOLDER_SEAT_LABEL), @@ -453,6 +547,8 @@ fn reaping_removes_an_unattached_holder_and_spares_a_busy_one_and_another_seats( "--detach", "--name", job, + "--label", + &owner_label(), "--network", &format!("container:{busy}"), "--entrypoint", @@ -477,9 +573,10 @@ fn reaping_removes_an_unattached_holder_and_spares_a_busy_one_and_another_seats( let busy_survived = still_there(busy); let foreign_survived = still_there(foreign); - // Clean up before asserting, so a failure does not leak containers. + // Clean up before asserting, so a failure does not leak containers. `idle` is expected to be + // gone already — the reaper removed it — and the ownership check simply finds nothing to do. for name in [idle, busy, job, foreign] { - docker(&["rm", "--force", "--volumes", name], None); + remove_owned_container(name); } assert!(!idle_survived, "my own unattached holder should have been reaped; reaped={reaped:?}"); @@ -518,7 +615,6 @@ fn reaping_removes_an_unattached_holder_and_spares_a_busy_one_and_another_seats( fn a_job_launched_through_the_policy_is_contained_and_an_uncontained_one_is_not() { use maxplayer_core::home::{SandboxConfig, SandboxMode}; use maxplayer_core::seller_exec::{JobLaunch, SandboxPolicy}; - use std::path::Path; let canary = Canary::new("203.0.113.0/24", "198.18.7.0/24"); let denied = canary.denied_ip.clone(); @@ -562,6 +658,21 @@ fn a_job_launched_through_the_policy_is_contained_and_an_uncontained_one_is_not( .map(String::from) .collect(); + // One owned workdir per launch, and never `/tmp`. + // + // Production names the job container after the workdir's basename and does not pass `--rm`, so + // a shared `/tmp` meant a single fixed name, `maxplayer-job-tmp`, for every job this file ever + // launched. It worked once on a clean daemon and then collided forever: this control passed on + // the first real run (2026-09-14) and failed on the next three with nothing changed, on + // `docker: Error response from daemon: Conflict. The container name "/maxplayer-job-tmp" is + // already in use`. Two workdirs, because the control and the contained job are two containers + // and would otherwise collide with each other inside this one test. + let control_workdir = std::env::temp_dir().join(owned_name("workdir-control")); + let contained_workdir = std::env::temp_dir().join(owned_name("workdir-contained")); + for dir in [&control_workdir, &contained_workdir] { + std::fs::create_dir_all(dir).expect("a workdir"); + } + // Control first, while the namespace has no rules: an UNCONTAINED job reaches the address. This also // proves the argv itself works — image, mount, user and all — so a later failure is attributable to // containment rather than to a malformed launch. @@ -569,7 +680,7 @@ fn a_job_launched_through_the_policy_is_contained_and_an_uncontained_one_is_not( .launch( &agent_command, &JobLaunch { - workdir: Path::new("/tmp"), + workdir: &control_workdir, env: &[], uid: 0, gid: 0, @@ -595,7 +706,7 @@ fn a_job_launched_through_the_policy_is_contained_and_an_uncontained_one_is_not( .launch( &agent_command, &JobLaunch { - workdir: Path::new("/tmp"), + workdir: &contained_workdir, env: &[], uid: 0, gid: 0, @@ -686,21 +797,21 @@ impl Canary { const OTHER_PORT: &'static str = "9998"; fn new(allowed_subnet: &str, denied_subnet: &str) -> Self { - let allowed_net = "mx-canary-allowed".to_owned(); - let denied_net = "mx-canary-denied".to_owned(); - // Setup can panic before the guard exists, which leaks whatever was created first. Clearing - // our own names up front makes a rerun idempotent instead of failing on the previous run's - // debris. Nothing here can touch a name we did not create. - for name in ["mx-canary-listener-allowed", "mx-canary-listener-denied", "mx-live-holder-canary"] { - docker(&["rm", "--force", "--volumes", name], None); - } - for net in [&allowed_net, &denied_net, &"mx-live-net-canary".to_owned()] { - docker(&["network", "rm", net], None); - } + // Unique per run and stamped as ours, so setup creates and never clears. + // + // These names used to be fixed, with a `docker rm --force` of each at the top to make a + // rerun idempotent over the previous run's debris. "Nothing here can touch a name we did not + // create" was wrong twice: a second concurrent run of this suite creates exactly these + // names, and on any host an operator may already hold one. Deleting first also destroys the + // leak it was hiding, so a fixture that leaks on panic now stays visible and attributable to + // the run that leaked it. + let allowed_net = owned_name("canary-allowed"); + let denied_net = owned_name("canary-denied"); let fixture = Fixture::new("canary"); for (net, subnet) in [(&allowed_net, allowed_subnet), (&denied_net, denied_subnet)] { - let (ok, _, err) = docker(&["network", "create", "--subnet", subnet, net], None); + let (ok, _, err) = + docker(&["network", "create", "--label", &owner_label(), "--subnet", subnet, net], None); assert!( ok, "could not create {net} on {subnet}: {err}\n\ @@ -709,8 +820,8 @@ impl Canary { ); } - let allowed_listener = "mx-canary-listener-allowed".to_owned(); - let denied_listener = "mx-canary-listener-denied".to_owned(); + let allowed_listener = owned_name("canary-listener-allowed"); + let denied_listener = owned_name("canary-listener-denied"); let mut ips = Vec::new(); for (name, net) in [(&allowed_listener, &allowed_net), (&denied_listener, &denied_net)] { // Two ports, because the pinhole test needs one address that is reachable on one port and @@ -721,6 +832,8 @@ impl Canary { "--detach", "--name", name, + "--label", + &owner_label(), "--network", net, "--entrypoint", @@ -761,7 +874,7 @@ impl Canary { assert!(ok, "could not attach {net} to the holder: {err}"); } - Self { + let canary = Self { fixture, allowed_net, denied_net, @@ -769,7 +882,25 @@ impl Canary { denied_listener, allowed_ip: ips[0].clone(), denied_ip: ips[1].clone(), + }; + + // Readiness, not assumption. `docker run --detach` returns once the container has *started*; + // the shell inside still has to add its second address and reach `nc -l`. Three runs of this + // file on the same daemon disagreed about the very first control for exactly that reason — + // it passed on the first run and failed on the next two, with nothing changed. Probing from + // outside the contained namespace is the same discriminator the legs use, so a listener that + // never comes up still fails, here rather than as a false containment three asserts later. + for (net, ip) in [ + (canary.allowed_net.clone(), canary.allowed_ip.clone()), + (canary.denied_net.clone(), canary.denied_ip.clone()), + ] { + assert!( + wait_until(20, || canary.can_reach_from_outside(&net, &ip)), + "the listener on {ip} never answered from {net} — every leg below would have read \ + that silence as containment" + ); } + canary } /// Open a real TCP connection from inside the contained namespace. `true` iff it connected. @@ -803,12 +934,12 @@ impl Drop for Canary { // A network cannot be removed while a container is attached, and the holder is attached to // both. `Fixture`'s Drop runs after this one, so the holder has to go first here — its own // removal then finds nothing, which is harmless. - docker(&["rm", "--force", "--volumes", &self.fixture.holder], None); + remove_owned_container(&self.fixture.holder); for name in [&self.allowed_listener, &self.denied_listener] { - docker(&["rm", "--force", "--volumes", name], None); + remove_owned_container(name); } for net in [&self.allowed_net, &self.denied_net] { - docker(&["network", "rm", net], None); + remove_owned_network(net); } } } @@ -818,8 +949,9 @@ impl Drop for Canary { #[test] #[ignore = "needs docker and the netfilter image"] fn establish_contains_a_namespace_and_tears_it_down_on_drop() { - let network = "mx-live-net-establish"; - let (ok, _, err) = docker(&["network", "create", network], None); + let network = owned_name("net-establish"); + let network = network.as_str(); + let (ok, _, err) = docker(&["network", "create", "--label", &owner_label(), network], None); assert!(ok, "could not create the test network: {err}"); let runtime = tokio::runtime::Runtime::new().expect("a runtime"); @@ -838,6 +970,7 @@ fn establish_contains_a_namespace_and_tears_it_down_on_drop() { // unit tests, and adding one here would open port 53 to an address this fixture never // measured. Vec::new(), + 2_000_000_000, )); let holder_name = match outcome { @@ -856,16 +989,2573 @@ fn establish_contains_a_namespace_and_tears_it_down_on_drop() { name } Err(error) => { - docker(&["network", "rm", network], None); + remove_owned_network(network); panic!("establish failed: {error}"); } }; // The guard's Drop is synchronous, so by here the holder must be gone. let (_, listed, _) = docker(&["ps", "--all", "--quiet", "--filter", &format!("name={holder_name}")], None); - docker(&["network", "rm", network], None); + remove_owned_network(network); assert!( listed.is_empty(), "dropping the containment must remove the holder, but {holder_name} is still listed" ); } + +/// The expiry stamp is on the **real** container, read back off the daemon. +/// +/// The unit gates prove `holder_argv` contains the label. That is an argument vector, not a +/// container: it cannot show that docker accepted the label, stored it, and will hand it back to a +/// later sweep in a different process. This asks the daemon. +#[test] +#[ignore = "needs docker and the netfilter image"] +fn a_real_holder_carries_its_own_expiry_stamp_on_the_daemon() { + use maxplayer_core::sandbox_netns::{ + HOLDER_CLEANUP_AFTER_LABEL, HOLDER_ROLE_LABEL, HOLDER_SEAT_LABEL, ROLE_HOLDER, + }; + + let network = owned_name("net-stamp"); + let network = network.as_str(); + let (ok, _, err) = docker(&["network", "create", "--label", &owner_label(), network], None); + assert!(ok, "could not create the test network: {err}"); + + // A seat unique to this run. The sweep selects by seat, and a shared seat would let this test + // reach containers belonging to another test or another seat entirely. + let seat = format!("{:0<64}", format!("stamp{}", owner_token())); + let stamp: u64 = 2_000_000_000; + + let runtime = tokio::runtime::Runtime::new().expect("a runtime"); + let outcome = runtime.block_on(maxplayer_core::sandbox_netns::establish( + network, + &holder_image(), + &netfilter_image(), + "host.docker.internal", + "live-stamp", + &seat, + 1000, + 1000, + Some(PortRange::new(49300, 49399).expect("valid range")), + true, + Vec::new(), + stamp, + )); + + let containment = match outcome { + Ok(containment) => containment, + Err(error) => { + remove_owned_network(network); + panic!("establish failed: {error}"); + } + }; + let holder_name = containment.holder.name().to_owned(); + + let label_of = |key: &str| -> String { + let format = format!("{{{{index .Config.Labels \"{key}\"}}}}"); + let (ok, out, err) = docker(&["inspect", "-f", &format, &holder_name], None); + assert!(ok, "docker inspect failed for {holder_name}: {err}"); + out.trim().to_owned() + }; + + let seen_stamp = label_of(HOLDER_CLEANUP_AFTER_LABEL); + let seen_role = label_of(HOLDER_ROLE_LABEL); + let seen_seat = label_of(HOLDER_SEAT_LABEL); + + drop(containment); + remove_owned_network(network); + + assert_eq!( + seen_stamp, + stamp.to_string(), + "the daemon must hand back the exact expiry the job was stamped with" + ); + assert_eq!(seen_role, ROLE_HOLDER, "the holder must be stamped with its role"); + assert_eq!(seen_seat, seat, "the holder must be stamped with the seat that owns it"); +} + +/// A real sweep against a real daemon: the expired holder goes, the live one stays. +/// +/// This is the leg the whole redesign rests on, and no unit test can reach it: the stand-in docker +/// in the unit gates returns listings I wrote. Here the listing, the label filter, the parse and the +/// removal all go through docker itself. +/// +/// Both holders are deliberately left un-dropped until after the sweep. The sweep removing a +/// container out from under a live guard is exactly the production situation -- a previous process's +/// holder -- and the guard's own drop is best-effort, so the double removal is harmless. +#[test] +#[ignore = "needs docker and the netfilter image"] +fn the_sweep_removes_an_expired_holder_and_leaves_one_inside_its_deadline() { + let network = owned_name("net-sweep"); + let network = network.as_str(); + let (ok, _, err) = docker(&["network", "create", "--label", &owner_label(), network], None); + assert!(ok, "could not create the test network: {err}"); + + let seat = format!("{:0<64}", format!("sweep{}", owner_token())); + let runtime = tokio::runtime::Runtime::new().expect("a runtime"); + + let establish_one = |job: &str, port_lo: u16, cleanup_after: u64| { + runtime.block_on(maxplayer_core::sandbox_netns::establish( + network, + &holder_image(), + &netfilter_image(), + "host.docker.internal", + job, + &seat, + 1000, + 1000, + Some(PortRange::new(port_lo, port_lo + 99).expect("valid range")), + true, + Vec::new(), + cleanup_after, + )) + }; + + // One whose deadline plus its grace is long past, one still far inside it. + let expired = establish_one("live-sweep-expired", 49400, 1); + let live = establish_one("live-sweep-live", 49500, 2_000_000_000); + + let (expired, live) = match (expired, live) { + (Ok(a), Ok(b)) => (a, b), + (a, b) => { + remove_owned_network(network); + panic!("establish failed: expired={:?} live={:?}", a.err(), b.err()); + } + }; + let expired_name = expired.holder.name().to_owned(); + let live_name = live.holder.name().to_owned(); + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("a clock after 1970") + .as_secs(); + let report = runtime + .block_on(maxplayer_core::sandbox_netns::sweep_expired(&seat, now)) + .expect("the sweep must reach the daemon"); + + // Ask the daemon what survived, rather than trusting the report. + let still_listed = |name: &str| -> bool { + let (_, out, _) = docker(&["ps", "--all", "--quiet", "--filter", &format!("name={name}")], None); + !out.trim().is_empty() + }; + let expired_survived = still_listed(&expired_name); + let live_survived = still_listed(&live_name); + + drop(expired); + drop(live); + remove_owned_network(network); + + assert!( + !expired_survived, + "the sweep must remove the holder whose stamp has passed, but {expired_name} is still listed" + ); + assert!( + live_survived, + "the sweep must leave the holder still inside its deadline, but {live_name} was removed" + ); + assert_eq!( + report.selected(), + 1, + "the sweep must select exactly the expired holder, not the live one: {report:?}" + ); + assert!(report.failed.is_empty(), "docker refused a removal: {:?}", report.failed); +} + +/// A HELPER STAMPED BY THE PRODUCTION LABEL BUILDER, PAST ITS EXPIRY, AGAINST THE REAL DAEMON. +/// +/// The row neither branch ran. The sweep only expires what carries the stamp, so a helper the +/// production path forgot to label is a container no sweep can ever judge — it pins the job's +/// namespace and outlives every deadline in the system. Two separate things have to be true, and +/// only a real daemon can show both: that [`helper_label_args`] produces labels docker ACCEPTS and +/// hands back unaltered, and that the sweep's own listing filter then SELECTS a container carrying +/// them. +/// +/// The labels here come from `helper_label_args` itself — the exact function `establish` hands to +/// the sidecar funnel — and not from strings this test wrote. A test that spells the labels out by +/// hand passes while production writes something else entirely, which is the failure mode that +/// matters: this is the level at which impl3's stand-in gates could not distinguish the two. +/// +/// The helper is `sleep infinity` with no `--rm`, because production's own helpers are short-lived +/// and self-deleting. That is deliberate: the container this test needs is the one that did NOT go +/// away — a helper whose `--rm` never fired because its daemon or its seller died mid-job — and +/// that is precisely the leftover the periodic sweep exists to collect. +#[test] +#[ignore = "needs docker and the netfilter image"] +fn a_production_stamped_helper_past_its_expiry_is_swept_against_the_real_daemon() { + use maxplayer_core::sandbox_netns::{ + HELPER_JOB_LABEL, HOLDER_CLEANUP_AFTER_LABEL, HOLDER_ROLE_LABEL, HOLDER_SEAT_LABEL, + ROLE_HELPER, helper_label_args, + }; + + let network = owned_name("net-helper-sweep"); + let network = network.as_str(); + let (ok, _, err) = docker(&["network", "create", "--label", &owner_label(), network], None); + assert!(ok, "could not create the test network: {err}"); + + let seat = format!("{:0<64}", format!("helper{}", owner_token())); + let job = "live-helper-sweep"; + // Long past: the stamp is the job's deadline plus its grace, already written into the past, so + // the very next sweep judges every container carrying it expired. + let stamp: u64 = 1; + let runtime = tokio::runtime::Runtime::new().expect("a runtime"); + + let outcome = runtime.block_on(maxplayer_core::sandbox_netns::establish( + network, + &holder_image(), + &netfilter_image(), + "host.docker.internal", + job, + &seat, + 1000, + 1000, + Some(PortRange::new(49700, 49799).expect("valid range")), + true, + Vec::new(), + stamp, + )); + let containment = match outcome { + Ok(containment) => containment, + Err(error) => { + remove_owned_network(network); + panic!("establish failed: {error}"); + } + }; + let holder_name = containment.holder.name().to_owned(); + let network_mode = containment.holder.network_mode(); + + // PRODUCTION's labels, for a helper joined to this job's namespace exactly as a real sidecar is. + let helper_name = owned_name("helper-left"); + let owner = owner_label(); + let image = holder_image(); + let mut argv: Vec = [ + "run", + "--detach", + "--name", + helper_name.as_str(), + "--label", + owner.as_str(), + "--network", + network_mode.as_str(), + ] + .into_iter() + .map(String::from) + .collect(); + argv.extend(helper_label_args(job, &seat, stamp)); + argv.extend( + ["--entrypoint", "sleep", image.as_str(), "infinity"].into_iter().map(String::from), + ); + let argv: Vec<&str> = argv.iter().map(String::as_str).collect(); + let (ok, _, err) = docker(&argv, None); + if !ok { + drop(containment); + remove_owned_network(network); + panic!("could not start the stamped helper {helper_name}: {err}"); + } + + // Read the stamp back off the daemon, not out of the argv this test just built. + let label_of = |key: &str| -> String { + let format = format!("{{{{index .Config.Labels \"{key}\"}}}}"); + let (ok, out, err) = docker(&["inspect", "-f", &format, &helper_name], None); + assert!(ok, "docker inspect failed for {helper_name}: {err}"); + out.trim().to_owned() + }; + let seen_seat = label_of(HOLDER_SEAT_LABEL); + let seen_stamp = label_of(HOLDER_CLEANUP_AFTER_LABEL); + let seen_role = label_of(HOLDER_ROLE_LABEL); + let seen_job = label_of(HELPER_JOB_LABEL); + + // PRODUCTION's own listing argv, run here before the sweep, so a red says whether the daemon + // failed to show the sweep this helper or the sweep saw it and left it. + let listing_argv = maxplayer_core::sandbox_netns::list_owned_argv(&seat); + let listing_argv: Vec<&str> = listing_argv.iter().skip(1).map(String::as_str).collect(); + let (_, seen_by_sweep, _) = docker(&listing_argv, None); + let seen_by_sweep = seen_by_sweep.replace('\t', "|").replace('\n', " ;; "); + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("a clock after 1970") + .as_secs(); + let report = runtime + .block_on(maxplayer_core::sandbox_netns::sweep_expired(&seat, now)) + .expect("the sweep must reach the daemon"); + + // The daemon decides what survived, not the report. + let still_listed = |name: &str| -> bool { + let (_, out, _) = + docker(&["ps", "--all", "--quiet", "--filter", &format!("name={name}")], None); + !out.trim().is_empty() + }; + let helper_survived = still_listed(&helper_name); + let holder_survived = still_listed(&holder_name); + // Captured BEFORE the cleanup below destroys the evidence: what the daemon still holds for this + // seat, so a failure names the surviving containers instead of only the one this test looked up. + let (_, seat_listing, _) = docker( + &[ + "ps", + "--all", + "--filter", + &format!("label={}={seat}", maxplayer_core::sandbox_netns::HOLDER_SEAT_LABEL), + "--format", + "{{.Names}}/{{.State}}", + ], + None, + ); + let seat_listing = seat_listing.split_whitespace().collect::>().join(" "); + + drop(containment); + let _ = docker(&["rm", "--force", "--volumes", &helper_name], None); + remove_owned_network(network); + + assert_eq!(seen_seat, seat, "production must stamp the helper with the seat that owns it"); + assert_eq!( + seen_stamp, + stamp.to_string(), + "the helper must carry the SAME expiry as its holder, handed back by the daemon unaltered" + ); + assert_eq!(seen_role, ROLE_HELPER, "the helper must be stamped as a helper, not as a holder"); + assert_eq!(seen_job, job, "the helper must name the job it belongs to"); + assert!( + !helper_survived, + "the sweep must remove the expired helper, but {helper_name} is still listed \ + (report {report:?}; the sweep's own listing saw [{seen_by_sweep}]; \ + seat still holds [{seat_listing}])" + ); + assert!( + !holder_survived, + "the sweep must remove the expired holder too, but {holder_name} is still listed \ + (report {report:?}; seat still holds [{seat_listing}])" + ); + assert_eq!( + report.selected(), + 2, + "the sweep must select the job's whole expired set — holder and helper: {report:?}" + ); + assert!(report.failed.is_empty(), "docker refused a removal: {:?}", report.failed); + assert!(report.deferred.is_empty(), "two containers are far inside one pass's budget"); +} + +// --------------------------------------------------------------------------------------------- +// The interface layer: the filters on the veth the packets actually leave by +// --------------------------------------------------------------------------------------------- + +/// The container runtime whose payloads do **not** traverse the host's `OUTPUT` chain, named by the +/// operator rather than guessed. Required, like [`netfilter_image`]: a default of `runsc` would let +/// this test quietly measure `runc` on a host without gVisor and report the leak as closed by rules +/// that never had to stop anything. +fn runsc_runtime() -> String { + std::env::var("MAXPLAYER_RUNSC_RUNTIME").expect( + "set MAXPLAYER_RUNSC_RUNTIME (e.g. `runsc`) — the whole point of this test is the runtime \ + that bypasses the OUTPUT chain, so it refuses to guess which one that is", + ) +} + +/// What a probe launched inside the namespace actually established. +/// +/// A bare boolean could not tell a DENIED packet from a probe that never ran. "the `docker run` +/// exited non-zero" is true when the filters dropped the packet, and equally true when the image is +/// missing, the runtime is not installed, or `nc` is not on the image — and a failure that never +/// reached the path proves nothing whatsoever about the path. Read as a denial, such a failure +/// reports containment that was never exercised. +#[derive(Debug)] +enum Reach { + /// `nc` connected. + Connected, + /// `nc` ran, reached the path, and did not get through: a real denial. + Denied, + /// The probe never ran. A broken fixture, not a containment result. + ToolFailure(String), +} + +impl Reach { + fn connected(&self) -> bool { + matches!(self, Reach::Connected) + } + + /// A denial that is REALLY a denial — or a loud failure. Never a silent "not connected". + fn denied(&self, what: &str) -> bool { + match self { + Reach::Denied => true, + Reach::Connected => false, + Reach::ToolFailure(why) => panic!( + "the probe for {what} never ran ({why}), so this run establishes nothing about \ + containment: a tool failure is not a denial" + ), + } + } +} + +/// Run docker and report the child's EXIT CODE, not merely success. +/// +/// The code is what separates "the packet was denied" from "the probe never ran": docker reserves +/// 125 for its own failure, 126 for a command it cannot execute and 127 for one it cannot find, +/// while any other non-zero code is the payload itself speaking. +fn docker_exit(args: &[&str]) -> (Option, String) { + let out = Command::new("docker") + .args(args) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .output() + .expect("docker must be on PATH for a live containment test"); + (out.status.code(), String::from_utf8_lossy(&out.stderr).trim().to_owned()) +} + +/// Connect from a container started under an explicit `--runtime`. +fn connect_under(runtime: &str, network: &str, ip: &str, port: &str) -> Reach { + let image = netfilter_image(); + let (code, stderr) = docker_exit(&[ + "run", + "--rm", + "--runtime", + runtime, + "--network", + network, + "--entrypoint", + "nc", + &image, + "-w", + "2", + ip, + port, + ]); + match code { + Some(0) => Reach::Connected, + // docker's own reserved codes: the container never got as far as running the probe. + Some(code @ (125 | 126 | 127)) => { + Reach::ToolFailure(format!("docker exited {code}: {stderr}")) + } + // `nc` ran and reported that it could not connect. + Some(_) => Reach::Denied, + None => Reach::ToolFailure(format!("the probe was killed by a signal: {stderr}")), + } +} + +/// Run a daemon-built argv verbatim. Every helper below goes through this rather than assembling its +/// own docker command, so what the tests exercise is the argv the product ships. +fn run_argv(argv: &[String], stdin: Option<&str>) -> (bool, String, String) { + let args: Vec<&str> = argv[1..].iter().map(String::as_str).collect(); + docker(&args, stdin) +} + +/// The job's egress link, measured inside the namespace through the daemon's own probe argv. +fn egress_dev(holder: &str) -> String { + let (ok, stdout, err) = run_argv(&link_probe_argv(holder, &netfilter_image()), None); + assert!(ok, "could not enumerate the namespace's links: {err}"); + // `parse_links` refuses a record it cannot read rather than skipping it, so an unreadable line + // fails the test here instead of silently shrinking the list the selector then judges. + let links = parse_links(&stdout).unwrap_or_else(|error| { + panic!("the namespace's link list could not be read: {error}\n{stdout}") + }); + select_egress_link(&links) + .expect("a job holder's namespace has exactly one non-loopback link") + .name +} + +fn iface_readback(holder: &str, dev: &str) -> String { + let (ok, stdout, err) = run_argv(&filter_readback_argv(holder, &netfilter_image(), dev), None); + assert!(ok, "reading the egress filters back from {dev} failed: {err}"); + stdout +} + +/// `establish` installs the egress filters too, and the kernel is asked — not the return value. +/// +/// The interface leg is inside `establish` rather than beside it deliberately: a test-only installer +/// would prove that these filters *can* be installed while every real job launched without them. +#[test] +#[ignore = "needs docker and the netfilter image"] +fn establish_filters_the_veth_the_packets_actually_leave_by() { + let network = owned_name("net-iface"); + let network = network.as_str(); + let (ok, _, err) = docker(&["network", "create", "--label", &owner_label(), network], None); + assert!(ok, "could not create the test network: {err}"); + + let runtime = tokio::runtime::Runtime::new().expect("a runtime"); + let outcome = runtime.block_on(maxplayer_core::sandbox_netns::establish( + network, + &holder_image(), + &netfilter_image(), + "host.docker.internal", + "live-iface", + "4444444444444444444444444444444444444444444444444444444444444444", + 1000, + 1000, + Some(PortRange::new(49200, 49299).expect("valid range")), + true, + // No resolver exception here either, for the same reason as the establish case above: this + // test asserts the veth filters mirror the rendered policy exactly, so an exception the + // fixture never measured would be an exception it cannot check. + Vec::new(), + 2_000_000_000, + )); + + let containment = match outcome { + Ok(containment) => containment, + Err(error) => { + remove_owned_network(network); + panic!("establish failed: {error}"); + } + }; + + // The device it filtered is a measured veth, not loopback and not a name anybody assumed. + assert_ne!(containment.egress_dev, "lo", "establish filtered loopback, not the job's egress link"); + assert!(!containment.egress_dev.is_empty(), "establish named no egress device at all"); + + // The plan the daemon must have installed, re-derived from the address IT measured, and checked + // against what the kernel in that namespace actually holds. + let policy = NetPolicy { + gateway: containment.proxy_host.clone(), + proxy_ports: Some(PortRange::new(49200, 49299).expect("valid range")), + log_connections: true, + // Matches the `Vec::new()` handed to `establish` above: this must be the policy the daemon + // actually installed, so a resolver here that the call never passed would fail the + // comparison for the wrong reason. + dns_resolvers: Vec::new(), + }; + let plan = IfacePlan::derive(&containment.egress_dev, &policy).expect("the plan renders"); + let readback = iface_readback(containment.holder.name(), &containment.egress_dev); + assert_eq!( + plan.verify_readback(&readback), + Ok(()), + "the namespace establish() blessed does not hold the egress filters:\n{readback}" + ); + // Both families present as drops, stated here as well as inside the verifier: an unfiltered + // address family is the cheapest bypass there is, and this assert fails by name if the verifier + // is ever loosened. + for protocol in ["ip", "ipv6"] { + assert!( + readback.lines().any(|line| line.contains(protocol)), + "no {protocol} filter in the live readback:\n{readback}" + ); + } + + let holder_name = containment.holder.name().to_owned(); + drop(containment); + let (_, listed, _) = + docker(&["ps", "--all", "--quiet", "--filter", &format!("name={holder_name}")], None); + remove_owned_network(network); + assert!(listed.is_empty(), "the holder {holder_name} outlived its containment"); +} + +/// The red-prove for the egress readback: remove ONE filter from a live namespace and the verifier +/// must refuse it. +/// +/// Without this the verifier could `Ok(())` unconditionally and every other test here would still be +/// green — the failure mode that makes a readback worthless is the readback that cannot fail. +#[test] +#[ignore = "needs docker and the netfilter image"] +fn a_namespace_missing_one_egress_filter_is_refused() { + let fixture = Fixture::new("iface-missing"); + let policy = policy("172.17.0.1"); + let dev = egress_dev(&fixture.holder); + let plan = IfacePlan::derive(&dev, &policy).expect("the plan renders"); + let (expected_stdin, expected) = maxplayer_core::sandbox_iface::plan_stdin(&plan); + + let (ok, applied, err) = + run_argv(&iface_sidecar_argv(&fixture.holder, &netfilter_image()), Some(&expected_stdin)); + assert!(ok, "the interface applier refused the plan: {err}"); + assert_eq!(applied.parse::().expect("a count"), expected, "every step must reach the kernel"); + assert_eq!( + plan.verify_readback(&iface_readback(&fixture.holder, &dev)), + Ok(()), + "control: the untouched namespace must verify, or the refusal below proves nothing" + ); + + // Delete the LAST filter, so what is left is a prefix of the plan: order, protocol and every + // remaining match key are still perfect, and only the absence is wrong. + let victim = plan.filters.last().expect("a plan has filters"); + let pref = victim.pref.to_string(); + let protocol = maxplayer_core::sandbox_iface::tc_protocol(victim.family); + let (ok, _, err) = docker( + &[ + "run", + "--rm", + "--network", + &format!("container:{}", fixture.holder), + "--cap-drop", + "ALL", + "--cap-add", + "NET_ADMIN", + "--entrypoint", + "tc", + &netfilter_image(), + "filter", + "del", + "dev", + &dev, + // The hook spelling the daemon's own argv uses. `clsact egress` is what `filter show` + // accepts and `filter del` does not: measured on iproute2 in the gvisor-repro VM, which + // answers `Unknown filter "clsact", hence option "egress" is unparsable`. + maxplayer_core::sandbox_iface::EGRESS_HOOK, + "pref", + &pref, + "protocol", + protocol, + ], + None, + ); + assert!(ok, "could not remove a filter to break containment with: {err}"); + + let broken = iface_readback(&fixture.holder, &dev); + let refusal = plan + .verify_readback(&broken) + .expect_err("a namespace missing an egress filter must be refused"); + assert!( + refusal.contains("expected"), + "the refusal must say what is missing, got: {refusal}" + ); +} + +/// **The regression gate.** The `OUTPUT` chain alone does not contain a `runsc` job; the veth filters +/// do. One namespace, one destination, one payload runtime — the only thing that changes between the +/// two measurements is whether the interface plan is installed. +/// +/// Leg 1 is the leak, and it is asserted as a *success*: with the iptables policy installed and +/// verified, a job under gVisor still reaches a destination the policy denies, while a `runc` job in +/// an identically prepared namespace is refused. That pair is what makes this a runtime property +/// rather than a broken fixture. +/// +/// Leg 2 installs the same rendered policy on the veth and the same connection is refused, with two +/// live positive controls so a refusal cannot be environmental: an allowed destination stays reachable +/// from inside, and the denied listener stays reachable from outside the namespace. +/// +/// **One namespace per gVisor payload, always.** `runsc` claims the namespace's links when it starts, +/// and a SECOND joiner into the same namespace gets `ENETUNREACH` — a refusal no rule caused, which +/// reads exactly like containment. Measured twice: by the prototype matrix ("single-use namespace") +/// and here, where reusing one holder made a *control* fail before any rule existed. So every probe +/// below builds its own holder through [`Payload`], and the listeners are what persist. +#[test] +#[ignore = "needs docker, the netfilter image and a runsc runtime"] +fn the_output_chain_alone_lets_a_runsc_job_out_and_the_veth_filters_stop_it() { + let runsc = runsc_runtime(); + let net = RunscNet::new(); + let policy = policy("172.17.0.1"); + + // CONTROL — with no rules anywhere, a gVisor payload reaches both addresses. A "denied" address + // that was never reachable proves nothing later. + assert!( + Payload::new(&net, "c1").reach(&runsc, RunscNet::DENIED_IP), + "control: {} must be reachable under {runsc} before any rules exist", + RunscNet::DENIED_IP + ); + assert!( + Payload::new(&net, "c2").reach(&runsc, &net.allowed_ip), + "control: {} must be reachable under {runsc} before any rules exist", + net.allowed_ip + ); + + // LEG 1 — the shipped iptables policy alone, installed and verified in each namespace. + // + // The discriminator first: the same policy, prepared the same way, contains a runc job. Without + // it a leak below could just as well be a policy that never applied. + assert!( + !Payload::new(&net, "l1runc") + .with_output_policy(&policy) + .reach("runc", RunscNet::DENIED_IP), + "control: the OUTPUT chain must contain a runc job, or leg 1 measures a broken policy rather \ + than a runtime bypass" + ); + assert!( + Payload::new(&net, "l1").with_output_policy(&policy).reach(&runsc, RunscNet::DENIED_IP), + "THE LEAK this change exists to close did not reproduce: a {runsc} job failed to reach {} \ + with only the OUTPUT chain installed. Do not read that as containment — read it as this \ + gate no longer measuring what it claims.", + RunscNet::DENIED_IP + ); + + // LEG 2 — the same rendered policy, also translated onto the veth. + assert!( + !Payload::new(&net, "l2") + .with_output_policy(&policy) + .with_veth_filters(&policy) + .reach(&runsc, RunscNet::DENIED_IP), + "a {runsc} job still reached the denied {} with the veth filters in force", + RunscNet::DENIED_IP + ); + assert!( + Payload::new(&net, "l2ok") + .with_output_policy(&policy) + .with_veth_filters(&policy) + .reach(&runsc, &net.allowed_ip), + "positive control: the allowed destination {} must stay reachable under the veth filters — a \ + filter that denies everything is not containment", + net.allowed_ip + ); + assert!( + net.reachable_from_outside(RunscNet::DENIED_IP), + "positive control: the denied listener must still answer from outside every contained \ + namespace, or the refusal above was a dead listener" + ); +} + +/// One network, one listener, two addresses — the topology a **contained job actually gets**. +/// +/// The canary fixture above attaches its holder to three networks, which is right for the `OUTPUT` +/// chain and wrong here: a job holder in production is created on exactly one network, so it has +/// exactly one veth, and `select_egress_link` refuses to filter one of several links rather than +/// leave the others open. Measured: the three-network holder made this gate fail with *"expected +/// exactly one non-loopback link, found 3"*, which is the product being right and the fixture being +/// unrealistic. +/// +/// So the denied destination is a **second address on the same listener**, inside a denied prefix, +/// with an on-link route added in each namespace. One veth, two destinations, and the only thing that +/// decides reachability is the policy. +struct RunscNet { + network: String, + listener: String, + /// Measured, never assumed: docker's IPAM picks it inside 203.0.113.0/24, which no policy rule + /// denies. + allowed_ip: String, +} + +impl RunscNet { + /// Inside the denied 198.18.0.0/15 (RFC 2544 benchmarking space), which ordinary networks do not + /// use and this repo's policy drops. + const DENIED_IP: &'static str = "198.18.7.2"; + + fn new() -> Self { + // Unique per run and stamped as ours. The fixed `mx-runsc-net` / `mx-runsc-listener` pair + // this replaces was force-removed at setup to clear a prior run, which is the same command + // whether the name is a leftover of ours or a resource somebody else owns. + let network = owned_name("runsc-net"); + let listener = owned_name("runsc-listener"); + let (ok, _, err) = docker( + &["network", "create", "--label", &owner_label(), "--subnet", "203.0.113.0/24", &network], + None, + ); + assert!( + ok, + "could not create {network}: {err}\n\ + If this says the pool overlaps, another network on this host already holds \ + 203.0.113.0/24 — pick a free prefix rather than deleting whatever holds it." + ); + + // One process, both addresses: `nc -l` binds every local address, so a refusal can never be + // "that one was not listening". + let (ok, _, err) = docker( + &[ + "run", + "--detach", + "--name", + &listener, + "--label", + &owner_label(), + "--network", + &network, + "--cap-add", + "NET_ADMIN", + "--entrypoint", + "sh", + &netfilter_image(), + "-c", + &format!( + "ip addr add {}/32 dev eth0 && \ + while :; do nc -l -p {} >/dev/null 2>&1; done & \ + while :; do nc -l -p {} >/dev/null 2>&1; done", + Self::DENIED_IP, + Canary::OTHER_PORT, + Canary::PORT + ), + ], + None, + ); + assert!(ok, "could not start the listener: {err}"); + + let (ok, allowed_ip, err) = docker( + &[ + "inspect", + "--format", + &format!("{{{{(index .NetworkSettings.Networks \"{network}\").IPAddress}}}}"), + &listener, + ], + None, + ); + assert!(ok && !allowed_ip.is_empty(), "could not read the listener's address: {err}"); + let net = Self { network, listener, allowed_ip }; + net.await_listener(); + net + } + + /// Block until the listener actually answers. + /// + /// `docker run --detach` returns when the container has *started*, not when the process inside + /// it has added its second address and reached `nc -l`. Two runs of this file on the same + /// daemon disagreed about the very first control for exactly that reason, so readiness is now + /// the fixture's job rather than a race every leg re-runs. This probes the same destination the + /// tests do: a listener that never comes up still fails, here instead of three legs later. + fn await_listener(&self) { + assert!( + wait_until(20, || self.reachable_from_outside(Self::DENIED_IP)), + "the listener never answered on {} — every leg below would have read that as \ + containment", + Self::DENIED_IP + ); + // The neighbouring port is a control, so it carries the same readiness bar as the first. + // A leg that asserts the allowed destination stays reachable on OTHER_PORT is measuring + // policy only if something is listening there; unchecked, a slow second `nc -l` would read + // as a port-scoped denial and the control would pass for the wrong reason. + assert!( + wait_until(20, || self.reachable_on(&self.allowed_ip, Canary::OTHER_PORT)), + "the listener never answered on {}:{} — the neighbouring-port control cannot tell a \ + policy denial from an absent listener", + self.allowed_ip, + Canary::OTHER_PORT + ); + } + + /// The same destination, from a container on the network but **outside** every contained + /// namespace. A success proves the listener is alive, which is the one thing a refusal inside + /// cannot distinguish itself from. + fn reachable_from_outside(&self, ip: &str) -> bool { + self.reachable_on(ip, Canary::PORT) + } + + /// The same probe, on a named port. + /// + /// Split out because the neighbouring-port control needs to establish that `OTHER_PORT` answers + /// from outside every contained namespace — the only thing that makes a refusal *inside* one + /// attributable to policy rather than to an absent listener. + fn reachable_on(&self, ip: &str, port: &str) -> bool { + let (ok, _, _) = docker( + &[ + "run", + "--rm", + "--network", + &self.network, + "--cap-add", + "NET_ADMIN", + "--entrypoint", + "sh", + &netfilter_image(), + "-c", + &format!("ip route add {ip}/32 dev eth0 && nc -w 2 {ip} {port}"), + ], + None, + ); + ok + } +} + +/// Can **this process**, on the VM host and in no namespace at all, reach `ip:port`? +/// +/// Every other probe in this file runs inside a container. That is the right instrument for asking +/// what a contained job can do, and it is the wrong one for asking whether containment leaked into +/// the host: a job's rules could be installed host-globally, break the host's own egress, and every +/// container-side leg here would still read exactly the same. +/// +/// So this one connects directly. A short timeout, because the failure being guarded against is a +/// DROP, which does not answer at all rather than refusing. +fn host_can_reach(ip: &str, port: &str) -> bool { + let address = format!("{ip}:{port}"); + let Ok(mut addresses) = std::net::ToSocketAddrs::to_socket_addrs(&address) else { + return false; + }; + let Some(address) = addresses.next() else { + return false; + }; + std::net::TcpStream::connect_timeout(&address, std::time::Duration::from_secs(2)).is_ok() +} + +/// Retry `probe` once a second until it holds, up to `attempts` times. +/// +/// For fixture startup only — a container that has been *started* is not yet a container whose +/// process is listening. It never softens an assertion: the probe is the same one the caller would +/// have run once, and a destination that never answers still returns `false`. +fn wait_until(attempts: u32, probe: impl Fn() -> bool) -> bool { + for attempt in 0..attempts { + if probe() { + return true; + } + if attempt + 1 < attempts { + std::thread::sleep(std::time::Duration::from_secs(1)); + } + } + false +} + +impl Drop for RunscNet { + fn drop(&mut self) { + remove_owned_container(&self.listener); + remove_owned_network(&self.network); + } +} + +/// A namespace for **one** payload: its own holder on the one network, carrying whichever containment +/// layers the case under test installs. Removed on drop however the test exits. +/// +/// It exists because a gVisor payload cannot share a namespace with an earlier one (see the gate +/// above), which is also why production gives every job a fresh holder. +struct Payload { + holder: String, +} + +impl Payload { + fn new(net: &RunscNet, tag: &str) -> Self { + let holder = owned_name(&format!("payload-{tag}")); + let (ok, _, err) = docker( + &[ + "run", + "--detach", + "--name", + &holder, + "--label", + &owner_label(), + "--network", + &net.network, + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--entrypoint", + "sleep", + &holder_image(), + "infinity", + ], + None, + ); + assert!(ok, "could not start the payload holder {holder}: {err}"); + // The denied address has to be ROUTABLE from this namespace before any rule exists, or a + // later refusal would be "no route" rather than "dropped" — and the two fail identically. + // On-link on the job's own veth, which is the interface the filters go on. + let dev = egress_dev(&holder); + let (ok, _, err) = docker( + &[ + "run", + "--rm", + "--network", + &format!("container:{holder}"), + "--cap-drop", + "ALL", + "--cap-add", + "NET_ADMIN", + "--entrypoint", + "ip", + &netfilter_image(), + "route", + "add", + &format!("{}/32", RunscNet::DENIED_IP), + "dev", + &dev, + ], + None, + ); + assert!(ok, "could not route {} into {holder}: {err}", RunscNet::DENIED_IP); + Self { holder } + } + + /// The shipped iptables policy, through the real sidecar, verified per family before any payload. + fn with_output_policy(self, policy: &NetPolicy) -> Self { + let (plan, expected) = plan_stdin(policy); + let (ok, applied, err) = docker( + &[ + "run", + "--rm", + "--interactive", + "--network", + &format!("container:{}", self.holder), + "--cap-drop", + "ALL", + "--cap-add", + "NET_ADMIN", + "--security-opt", + "no-new-privileges", + &netfilter_image(), + ], + Some(&plan), + ); + assert!(ok, "the sidecar refused the policy: {err}"); + assert_eq!(applied.parse::().expect("a count"), expected); + for family in [Family::V4, Family::V6] { + let argv = readback_argv(&self.holder, &netfilter_image(), family); + let (ok, readback, err) = run_argv(&argv, None); + assert!(ok, "policy readback failed: {err}"); + assert_eq!( + policy.verify_readback(family, &readback), + Ok(()), + "{} policy readback did not verify:\n{readback}", + family.binary() + ); + } + self + } + + /// The same rendered policy on the veth, through the real applier, read back and verified. + fn with_veth_filters(self, policy: &NetPolicy) -> Self { + let dev = egress_dev(&self.holder); + let iface = IfacePlan::derive(&dev, policy).expect("the plan renders"); + let (stdin, expected) = maxplayer_core::sandbox_iface::plan_stdin(&iface); + let (ok, applied, err) = + run_argv(&iface_sidecar_argv(&self.holder, &netfilter_image()), Some(&stdin)); + assert!(ok, "the interface applier refused the plan: {err}"); + assert_eq!(applied.parse::().expect("a count"), expected); + let readback = iface_readback(&self.holder, &dev); + assert_eq!( + iface.verify_readback(&readback), + Ok(()), + "the egress filters did not verify on {dev}:\n{readback}" + ); + self + } + + /// Run the one payload this namespace gets, under `runtime`, and report whether it connected. + /// + /// A probe that never ran is raised here rather than folded into `false`: "did not connect" + /// because the image is missing is not the same measurement as "did not connect" because the + /// filters stopped it, and only one of them says anything about containment. + fn reach(&self, runtime: &str, ip: &str) -> bool { + let probe = connect_under(runtime, &format!("container:{}", self.holder), ip, Canary::PORT); + if let Reach::ToolFailure(why) = &probe { + panic!( + "the canary probe never ran ({why}): a tool failure is not a containment result" + ); + } + probe.connected() + } +} + +impl Drop for Payload { + fn drop(&mut self) { + remove_owned_container(&self.holder); + } +} + +// ================================================================================================= +// The integrated launch matrix +// ================================================================================================= +// +// Everything above this line prepares a namespace by hand and then joins something to it. That +// proves these filters CAN be installed; it does not prove a real job is launched with them, and a +// gate that installs its own plan would stay green if `prepare_launch` stopped calling `establish` +// altogether. +// +// So this section goes through `seller_exec::with_prepared_launch`: the production `prepare_launch` +// establishes containment, the production `SandboxPolicy::launch` builds the argv, `netns` is wired +// from `holder_name` exactly as `run_agent_job_with_env` wires it, and the argv is executed +// verbatim. The separate OUTPUT-only reproduction above is kept deliberately — it is the baseline +// arm, and it must not be folded into this one. +// +// **The oracle.** A refusal and a launch that never happened are the same exit code, and reading +// docker's status alone lets a broken image, a missing mount or an OOM be scored as containment. +// Every payload here therefore prints a start marker before it tries anything and a result marker +// carrying the connection's own exit code, and the outcome is read from those markers. A payload +// that did not print the start marker is `NeverStarted` and is never counted as a denial. + +/// Printed by the payload before it attempts anything, so "the job ran" is observable separately +/// from "the job's connection failed". +const STARTED_MARKER: &str = "MX-PAYLOAD-STARTED"; + +/// Printed after the connection attempt, carrying `nc`'s own exit code. +const RESULT_MARKER: &str = "MX-CONNECT-RC="; + +/// What a payload actually did. The distinction between the last two variants is the whole point: +/// only `Refused` is evidence of containment. +#[derive(Debug, PartialEq, Eq)] +enum PayloadOutcome { + /// The payload's own process never reached its first statement. A docker, image, mount or + /// runtime failure — never containment evidence, in either direction. + NeverStarted, + /// The payload ran and its connection succeeded. + Connected, + /// The payload ran and its connection was refused or timed out. + Refused, + /// The payload ran, but what exited was the TOOL rather than the connection: `nc` missing, + /// not executable, or rejecting its own arguments. Never containment evidence. + /// + /// Split out of `Refused`, which used to absorb every nonzero status. An image without `nc` + /// exits 127 through `sh`, and 127 is not zero, so a leg measuring nothing at all scored as a + /// denial -- the single most flattering way this matrix could be wrong, since a fixture that + /// silently lost its payload would have reported perfect containment on every denied leg. + ToolFailed(i32), +} + +/// The agent command for one connection attempt, bracketed by markers. +/// +/// `sh -c` rather than `nc` directly, because the markers have to come from the payload's own +/// process: a wrapper outside the container would print "started" for a container that never did. +fn payload_command(ip: &str, port: &str) -> Vec { + vec![ + "sh".to_owned(), + "-c".to_owned(), + format!("echo {STARTED_MARKER}; nc -w 4 {ip} {port} /dev/null 2>&1; echo {RESULT_MARKER}$?"), + ] +} + +/// Classify a payload's own output. Docker's exit status is deliberately not consulted. +fn classify_payload(stdout: &str, stderr: &str) -> PayloadOutcome { + let combined = format!("{stdout}\n{stderr}"); + if !combined.contains(STARTED_MARKER) { + return PayloadOutcome::NeverStarted; + } + match combined + .lines() + .find_map(|line| line.trim().strip_prefix(RESULT_MARKER)) + .and_then(|code| code.trim().parse::().ok()) + { + Some(0) => PayloadOutcome::Connected, + // BusyBox `nc`, which is what the Alpine fixture carries, exits 1 for a refused connection + // and 1 for the `-w` timeout. Those are the two shapes containment takes here, and they + // are the only statuses allowed to mean it. + Some(1) => PayloadOutcome::Refused, + // 127 not found, 126 not executable, 2 usage, 128+n killed by a signal. Every one of these + // is the tool failing rather than the network answering. + Some(other) => PayloadOutcome::ToolFailed(other), + // Started, but never reported a result: killed mid-attempt. Not a denial. + None => PayloadOutcome::NeverStarted, + } +} + +/// The denial oracle only calls a connection refused when the connection was refused. +/// +/// Offline, because it is a property of the classifier rather than of any network, and because the +/// whole matrix rests on it: every denied leg in the saved record is this function's verdict. The +/// failure it guards against is the flattering one -- an image that lost `nc` exits 127, and while +/// any nonzero status counted as a denial, a leg that measured nothing reported containment. +#[test] +fn only_a_connection_failure_is_scored_as_a_denial() { + let started = |code: &str| format!("{STARTED_MARKER}\n{RESULT_MARKER}{code}\n"); + + assert_eq!(classify_payload(&started("0"), ""), PayloadOutcome::Connected); + assert_eq!( + classify_payload(&started("1"), ""), + PayloadOutcome::Refused, + "BusyBox nc exits 1 for a refused connection and for the -w timeout" + ); + for broken in [127, 126, 2, 137] { + assert_eq!( + classify_payload(&started(&broken.to_string()), ""), + PayloadOutcome::ToolFailed(broken), + "exit {broken} is the tool failing, not the network refusing" + ); + } + // And the pre-existing boundaries still hold: no start marker, and a start with no result. + assert_eq!(classify_payload("", ""), PayloadOutcome::NeverStarted); + assert_eq!(classify_payload(STARTED_MARKER, ""), PayloadOutcome::NeverStarted); +} + +/// Free the deterministic container name a launch is about to use. +/// +/// Production names a job container from its workdir and does not pass `--rm`, so the container +/// survives its own exit and the *same* launch cannot be run twice: the second attempt dies on +/// `Conflict. The container name ... is already in use` before the payload exists, which arrives as +/// `NeverStarted` and looks exactly like containment. Only the legs that deliberately run one job +/// more than once call this, and only between attempts — it is the reaper's job done by hand, not a +/// change to what any assertion measures. +fn free_job_name(launch: &maxplayer_core::seller_exec::AgentLaunch) { + if let Some(name) = + launch.args.windows(2).find(|pair| pair[0] == "--name").map(|pair| pair[1].clone()) + { + let _ = docker(&["rm", "-f", &name], None); + } +} + +/// Execute a production-built `AgentLaunch` verbatim and read the payload's own markers back. +fn run_launch_attributably(launch: &maxplayer_core::seller_exec::AgentLaunch) -> PayloadOutcome { + let out = Command::new(&launch.program) + .args(&launch.args) + .stdin(std::process::Stdio::null()) + .output() + .expect("the launch program must be runnable"); + let stdout = String::from_utf8_lossy(&out.stdout); + let stderr = String::from_utf8_lossy(&out.stderr); + let outcome = classify_payload(&stdout, &stderr); + if matches!(outcome, PayloadOutcome::ToolFailed(_)) { + // Loud for the same reason `NeverStarted` is loud, and louder in one respect: this leg's + // payload DID start, so the leg looks like it measured something. + eprintln!( + "ToolFailed ({outcome:?}): the payload's connection tool failed rather than being \ + refused, so this leg measured nothing: {} {:?}\n stdout: {}\n stderr: {}", + launch.program, + launch.args, + stdout.trim(), + stderr.trim() + ); + } + if outcome == PayloadOutcome::NeverStarted { + // "NeverStarted" is the one outcome that says nothing about containment and everything + // about the launch, so it must not be silent: the first real run of this file reported it + // for the sibling leg with no way to tell a refused `docker run` from a killed payload. + eprintln!( + "NeverStarted: {} {:?}\n stdout: {}\n stderr: {}", + launch.program, + launch.args, + stdout.trim(), + stderr.trim() + ); + } + outcome +} + +/// The seat identity the gate launches as. Synthetic, and distinct from the reaper test's seats so a +/// concurrent reap cannot touch this run's holders. +fn gate_identity() -> maxplayer_core::seller_git::DeliveryAgentIdentity { + maxplayer_core::seller_git::DeliveryAgentIdentity::for_seller( + "5555555555555555555555555555555555555555555555555555555555555555", + ) +} + +/// The resolver every gate in this file configures. +/// +/// TEST-NET-1 (RFC 5737): reserved for documentation and routed nowhere. Named explicitly rather +/// than left empty, because empty does not mean "no DNS" -- `sandbox_dns::resolve` falls back to +/// the host's `resolv.conf` and then to `resolvectl`, and refuses a loopback address, which is +/// exactly what a systemd host presents at `127.0.0.53`. Left empty, these legs would depend on the +/// DNS configuration of whichever machine ran them. +/// +/// **This address answers nothing, and nothing in this file proves DNS works.** It is a +/// deterministic fixture for *containment* legs: it makes the rendered rule set predictable so the +/// pinhole leg can check the SHAPE of what production installed. A rendered `--dport 53 -j ACCEPT` +/// and an enforced one produce the same green here. +/// +/// Functioning resolution is a different claim and is measured elsewhere, against a resolver that +/// actually answers: [`crate::sandbox_dns_live`] runs a real dnsmasq fixture through the same +/// production path, covering v4/v6 resolution through the written `resolv.conf`, UDP truncation +/// falling back to TCP 53, host-stub discovery, and the denied-neighbour controls (another private +/// address, and a non-53 port on the resolver itself). Do not cite this constant, or this file's +/// readback, as evidence that a contained job can resolve a name. +const GATE_DNS_RESOLVER: &str = "192.0.2.53"; + +/// The `[sandbox]` section an operator writes, resolved through the same call a booting seat makes. +fn gate_config(network: &str) -> maxplayer_core::home::SandboxConfig { + maxplayer_core::home::SandboxConfig { + mode: maxplayer_core::home::SandboxMode::Docker, + launcher: Vec::new(), + // This matrix measures network containment only, so it declares no tools. Both lists + // are required since `SandboxConfig` gained them on main. + mcp_tools: Vec::new(), + held_tools: Vec::new(), + // Carries `sh` and `nc`, and declares no entrypoint, so the agent command is the payload. + image: Some(holder_image()), + forward_env: Vec::new(), + runtime: None, + network: Some(network.to_owned()), + // No pinhole: this matrix measures denial and allowance, and a proxy range would add a + // second reason for a leg to differ from its control. The pinhole has its own coverage. + proxy_port_range: None, + // An explicit resolver, and deliberately not an empty list. Empty does not mean "no DNS": + // `sandbox_dns::resolve` falls back to the HOST's resolv.conf and then to resolvectl, and + // refuses a loopback address — which is exactly what a systemd host presents at + // `127.0.0.53`. Left empty, every integrated leg here would depend on the DNS configuration + // of whatever machine the matrix runs on, and would fail preparation on a perfectly healthy + // one. So the gate names its own. + // + // TEST-NET-1 (RFC 5737), which is reserved for documentation and routed nowhere. It + // exercises 995's real path — the resolver file is written and the port-53 exception is + // rendered and read back — while opening reach to nothing that exists. The payloads here + // dial numeric addresses and resolve nothing, so no leg depends on it answering. + dns_servers: vec![GATE_DNS_RESOLVER.to_owned()], + file_credentials: Vec::new(), + codex_chatgpt: None, + container_delivery: None, + container_delivery_token: None, + container_delivery_token_cap_secs: None, + } +} + +/// Production `prepare_launch` hardcodes [`DEFAULT_NETFILTER_IMAGE`] — it does NOT read +/// `MAXPLAYER_NETFILTER_IMAGE`, which only the hand-built fixtures above use. So an integrated run +/// measures whatever is tagged as that image on this host. +/// +/// This asserts it exists, and says what to do about it, because the alternative is the failure this +/// whole change is about: a gate reporting containment from an image that has no `tc` in it. +fn require_default_netfilter_image() { + let image = maxplayer_core::sandbox_netns::DEFAULT_NETFILTER_IMAGE; + let (ok, _, err) = docker(&["image", "inspect", "--format", "{{.Id}}", image], None); + assert!( + ok, + "the integrated gate goes through production `prepare_launch`, which hardcodes {image} and \ + ignores MAXPLAYER_NETFILTER_IMAGE. Tag the locally built sidecar as that image before \ + running this gate:\n docker build -t {image} docker/maxplayer-netfilter\nThe published \ + v0.5.8 image does NOT contain tc, so an integrated run against it is expected to refuse \ + every launch rather than contain anything.\ndocker said: {err}" + ); +} + +/// Make `ip` reachable on-link inside the holder's namespace, so a later refusal is the filters and +/// not the absence of a route. Run after preparation, which is the only moment the namespace exists +/// and the payload has not started. +fn route_on_link(holder: &str, ip: &str) { + let (ok, _, err) = docker( + &[ + "run", + "--rm", + "--network", + &format!("container:{holder}"), + "--cap-drop", + "ALL", + "--cap-add", + "NET_ADMIN", + "--entrypoint", + "ip", + &netfilter_image(), + "route", + "add", + &format!("{ip}/32"), + "dev", + "eth0", + ], + None, + ); + assert!(ok, "could not make {ip} routable inside {holder}: {err}"); +} + +/// Run one integrated leg: production preparation, production launch argv, attributable payload. +/// +/// `before_payload` runs inside the prepared namespace after containment is installed and before the +/// payload starts — the window a route injection has to use. +fn integrated_leg( + network: &str, + ip: &str, + port: &str, + before_payload: impl FnOnce(&str), +) -> Result { + integrated_leg_with(gate_config(network), ip, port, before_payload) +} + +/// [`integrated_leg`], for a leg that needs a `[sandbox]` section other than the default one — a +/// configured pinhole, or a named runtime. The path through production is identical. +/// The absolute unix second a gate's launch is bounded by, `within` seconds from now. +/// +/// The live gates state their deadline ABSOLUTELY, exactly as production does since #996 F3, so +/// these rows exercise the carried-deadline path and not the `None` fallback, which only a +/// deadline-less harness probe takes. +fn gate_deadline_unix(within: u64) -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(u64::MAX, |since| since.as_secs()) + .saturating_add(within) +} + +fn integrated_leg_with( + config: maxplayer_core::home::SandboxConfig, + ip: &str, + port: &str, + before_payload: impl FnOnce(&str), +) -> Result { + let policy = maxplayer_core::seller_exec::SandboxPolicy::from_config(Some(&config)) + .expect("a docker policy"); + let workdir = std::env::temp_dir().join(owned_name("workdir")); + std::fs::create_dir_all(&workdir).expect("a workdir"); + let runtime = tokio::runtime::Runtime::new().expect("a runtime"); + let outcome = runtime.block_on(maxplayer_core::seller_exec::with_prepared_launch( + &payload_command(ip, port), + &policy, + &workdir, + &gate_identity(), + std::time::Duration::from_secs(120), + Some(gate_deadline_unix(120)), + |launch, holder| { + let holder = holder.expect( + "a docker policy with a configured network must establish containment — a `None` \ + holder here means the job would run uncontained", + ); + assert!( + launch.args.iter().any(|arg| arg == &format!("container:{holder}")), + "the production launch must join the holder's namespace: {:?}", + launch.args + ); + before_payload(holder); + run_launch_attributably(launch) + }, + )); + let _ = std::fs::remove_dir_all(&workdir); + outcome.map_err(|error| error.to_string()) +} + +/// **The integrated gate.** One preparation path, three ordered legs, each scored from the payload's +/// own markers. +/// +/// The allowed leg is not a formality: a filter set that denies everything would pass the denied leg +/// and is not containment. The outside control is the discriminator for the denied leg — from +/// outside the namespace the policy does not apply, so a success there proves the listener was alive +/// and the refusal inside was the rules. +#[test] +#[ignore = "needs docker and the production-tagged netfilter image"] +fn a_job_prepared_and_launched_by_production_is_contained_on_its_veth() { + require_default_netfilter_image(); + let net = RunscNet::new(); + + // CONTROL, outside every namespace: both destinations answer. + assert!( + net.reachable_from_outside(RunscNet::DENIED_IP), + "control: {} must answer from outside, or the denied leg below proves nothing", + RunscNet::DENIED_IP + ); + assert!( + net.reachable_from_outside(&net.allowed_ip), + "control: {} must answer from outside", + net.allowed_ip + ); + + // LEG 1 — a destination the shipped policy denies, through the production launch path. + let denied = integrated_leg(&net.network, RunscNet::DENIED_IP, Canary::PORT, |holder| { + route_on_link(holder, RunscNet::DENIED_IP) + }) + .expect("preparation must succeed"); + assert_eq!( + denied, + PayloadOutcome::Refused, + "a job prepared and launched by production reached the denied {} \ + (NeverStarted here would mean the payload never ran, which is not containment either)", + RunscNet::DENIED_IP + ); + + // LEG 2 — the allowed destination, same path, same image, same user. + let allowed = integrated_leg(&net.network, &net.allowed_ip, Canary::PORT, |_| {}) + .expect("preparation must succeed"); + assert_eq!( + allowed, + PayloadOutcome::Connected, + "positive control: the allowed {} must stay reachable through the production launch path — \ + a policy that denies everything is not containment", + net.allowed_ip + ); + + // LEG 3 — a neighbouring port on the SAME allowed address. The policy's denials are not + // port-scoped, so the allowed address stays allowed; what this leg rules out is a filter that + // happened to match on one port number. + let other_port = integrated_leg(&net.network, &net.allowed_ip, Canary::OTHER_PORT, |_| {}) + .expect("preparation must succeed"); + // Asserted as Connected, which is what this leg's own comment claims and what the saved record + // states. `!= NeverStarted` passed on Refused too -- so a filter that DID silently become + // port-scoped, the exact failure this leg exists to rule out, satisfied it. A leg that accepts + // both answers to its own question is not a control. + assert_eq!( + other_port, + PayloadOutcome::Connected, + "the allowed {} must stay reachable on the neighbouring port {}: the policy's denials are \ + not port-scoped, and a leg that also accepted Refused would not have noticed if they were", + net.allowed_ip, + Canary::OTHER_PORT + ); +} + +/// **The oracle's own red-prove.** A payload that cannot start must not be scored as a denial. +/// +/// Without this the matrix above would pass with every leg broken: a bad image, a missing mount or a +/// runtime failure exits non-zero exactly like a refused connection, and round 1's oracle — docker's +/// status alone — could not tell them apart. +#[test] +#[ignore = "needs docker and the production-tagged netfilter image"] +fn a_payload_that_never_ran_is_not_scored_as_a_denial() { + // The classifier first, on captured shapes, so the rule is stated independently of any daemon. + assert_eq!( + classify_payload("", "docker: Error response from daemon: no such image"), + PayloadOutcome::NeverStarted, + "a docker failure must never be read as containment" + ); + assert_eq!( + classify_payload(&format!("{STARTED_MARKER}\n{RESULT_MARKER}1\n"), ""), + PayloadOutcome::Refused + ); + assert_eq!( + classify_payload(&format!("{STARTED_MARKER}\n{RESULT_MARKER}0\n"), ""), + PayloadOutcome::Connected + ); + assert_eq!( + classify_payload(&format!("{STARTED_MARKER}\n"), ""), + PayloadOutcome::NeverStarted, + "started but killed before reporting is not a denial" + ); + + // Then live: a real production launch whose command does not exist. Containment is established + // and correct; the payload still never runs, and the outcome must say so. + require_default_netfilter_image(); + let net = RunscNet::new(); + let config = gate_config(&net.network); + let policy = maxplayer_core::seller_exec::SandboxPolicy::from_config(Some(&config)) + .expect("a docker policy"); + let workdir = std::env::temp_dir().join(owned_name("workdir-nostart")); + std::fs::create_dir_all(&workdir).expect("a workdir"); + let runtime = tokio::runtime::Runtime::new().expect("a runtime"); + let outcome = runtime + .block_on(maxplayer_core::seller_exec::with_prepared_launch( + &["mx-no-such-binary".to_owned()], + &policy, + &workdir, + &gate_identity(), + std::time::Duration::from_secs(60), + Some(gate_deadline_unix(60)), + |launch, _| run_launch_attributably(launch), + )) + .expect("preparation must succeed — it is the payload that cannot start"); + let _ = std::fs::remove_dir_all(&workdir); + assert_eq!( + outcome, + PayloadOutcome::NeverStarted, + "a payload that could not start was scored as something other than NeverStarted" + ); +} + +/// **Fail-closed at preparation.** When containment cannot be established the launch is refused, no +/// payload is started, and nothing is left running. +/// +/// The trigger is a configured network that does not exist, which is the shape of every preparation +/// failure that matters: the seat is configured for containment and the daemon cannot deliver it. +/// The alternative behaviour — running the job on whatever networking is available — is exactly what +/// "configured but not enforced" means, and it must not be representable. +#[test] +#[ignore = "needs docker"] +fn containment_that_cannot_be_established_refuses_the_launch_and_leaves_nothing_behind() { + let missing = owned_name("net-that-does-not-exist"); + let config = gate_config(&missing); + let policy = maxplayer_core::seller_exec::SandboxPolicy::from_config(Some(&config)) + .expect("a docker policy"); + let workdir = std::env::temp_dir().join(owned_name("workdir-failclosed")); + std::fs::create_dir_all(&workdir).expect("a workdir"); + + let started = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let observed = std::sync::Arc::clone(&started); + let runtime = tokio::runtime::Runtime::new().expect("a runtime"); + let outcome = runtime.block_on(maxplayer_core::seller_exec::with_prepared_launch( + &payload_command("203.0.113.9", Canary::PORT), + &policy, + &workdir, + &gate_identity(), + std::time::Duration::from_secs(60), + Some(gate_deadline_unix(60)), + move |_, _| { + observed.store(true, std::sync::atomic::Ordering::SeqCst); + }, + )); + let _ = std::fs::remove_dir_all(&workdir); + + let error = outcome + .err() + .map(|error| error.to_string()) + .expect("a job whose containment cannot be established must not launch"); + assert!( + error.contains("egress containment not established"), + "the refusal must name what failed: {error}" + ); + assert!( + !started.load(std::sync::atomic::Ordering::SeqCst), + "the payload closure ran despite containment failing — the job would have started uncontained" + ); + // Nothing survives. The holder is named from the job id, which is derived from the workdir, so + // this looks for any holder still carrying this run's workdir name. + let (_, listed, _) = docker( + &["ps", "--all", "--quiet", "--filter", "label=ai.maxplayer.netns-holder"], + None, + ); + for id in listed.lines().filter(|line| !line.trim().is_empty()) { + let (_, name, _) = docker(&["inspect", "--format", "{{.Name}}", id], None); + assert!( + !name.contains("failclosed"), + "a holder from the failed preparation is still running: {name}" + ); + } +} + +/// **Sibling isolation across cleanup.** One contained job's teardown must not disturb another's. +/// +/// Two jobs are prepared through the production path on the same network. The first is torn down — +/// its holder removed, its namespace destroyed — while the second is still running, and the second +/// must still reach its allowed destination afterwards. This is the property a cleanup implemented +/// as "delete the tc filters" or "remove the containers matching our prefix" would break, and +/// neither failure is visible from a single-job test. +#[test] +#[ignore = "needs docker and the production-tagged netfilter image"] +fn one_jobs_cleanup_leaves_a_sibling_job_contained_and_running() { + require_default_netfilter_image(); + let net = RunscNet::new(); + + // The sibling: prepared, measured, and kept alive across the other job's whole lifetime. Its + // holder name is captured so its survival can be asserted rather than assumed. + let config = gate_config(&net.network); + let policy = maxplayer_core::seller_exec::SandboxPolicy::from_config(Some(&config)) + .expect("a docker policy"); + let workdir = std::env::temp_dir().join(owned_name("workdir-sibling")); + std::fs::create_dir_all(&workdir).expect("a workdir"); + let runtime = tokio::runtime::Runtime::new().expect("a runtime"); + + let survived = runtime.block_on(maxplayer_core::seller_exec::with_prepared_launch( + &payload_command(&net.allowed_ip, Canary::PORT), + &policy, + &workdir, + &gate_identity(), + std::time::Duration::from_secs(180), + Some(gate_deadline_unix(180)), + |launch, holder| { + let sibling_holder = holder.expect("containment").to_owned(); + // Before: the sibling reaches its allowed destination. + assert_eq!( + run_launch_attributably(launch), + PayloadOutcome::Connected, + "the sibling could not reach {} before the other job existed", + net.allowed_ip + ); + + // A whole second job, prepared and torn down inside this window. + // + // On its own thread, because this closure is already being driven by the sibling's + // runtime and `integrated_leg` builds one of its own: tokio refuses to start a runtime + // from inside a runtime, and the first real run of this file panicked here. The second + // job genuinely is a separate job, so giving it a separate thread is the shape the test + // was describing all along — not a workaround for the assertion. + let other = std::thread::scope(|scope| { + scope + .spawn(|| { + integrated_leg(&net.network, RunscNet::DENIED_IP, Canary::PORT, |h| { + route_on_link(h, RunscNet::DENIED_IP) + }) + }) + .join() + .expect("the second job's thread must not panic") + }) + .expect("the second job must prepare"); + assert_eq!(other, PayloadOutcome::Refused, "the second job was not contained"); + + // After the other job's guard dropped: the sibling's holder is still there… + let (_, listed, _) = docker( + &["ps", "--quiet", "--filter", &format!("name={sibling_holder}")], + None, + ); + assert!( + !listed.is_empty(), + "the sibling's holder {sibling_holder} was removed by another job's cleanup" + ); + // …and it is still contained and still working. + ( + { + // The sibling's first probe left a finished container holding this exact name. + free_job_name(launch); + run_launch_attributably(launch) + }, + { + let denied_probe = + prepared_launch_for(&policy, &workdir, RunscNet::DENIED_IP, &sibling_holder); + free_job_name(&denied_probe); + run_launch_attributably(&denied_probe) + }, + ) + }, + )); + let _ = std::fs::remove_dir_all(&workdir); + let (allowed_after, denied_after) = survived.expect("preparation must succeed"); + assert_eq!( + allowed_after, + PayloadOutcome::Connected, + "the sibling lost its allowed destination after another job's cleanup" + ); + assert_eq!( + denied_after, + PayloadOutcome::Refused, + "the sibling lost its containment after another job's cleanup — its veth filters were \ + deleted by a teardown that was not scoped to the job that owned them" + ); +} + +/// **The host's own egress is not collateral.** `host.unaffected.during-cleanup`. +/// +/// This leg was REQUIRED by `sandbox_evidence::REQUIRED_CASES` and, until now, asserted by nothing. +/// The round-2 record scored it `connected` on the strength of the sibling test running nearby; +/// that test probes from inside containers, so it could not have observed the host's egress in +/// either direction. The row was a claim, not a measurement, and the review was right to say so. +/// +/// What is measured here: the host reaches the canary directly, in its own namespace, at three +/// points — before a contained job exists, while one is prepared and running, and after its +/// teardown. A job whose containment mutated host-global state (an `OUTPUT` rule that was not +/// scoped to the veth, a teardown that flushed a shared chain) breaks one of the three. +/// +/// The negative control is not decoration: a probe that returned `true` unconditionally would pass +/// all three legs. A port nobody listens on must come back unreachable, from the same function, on +/// the same address. +#[test] +#[ignore = "needs docker and the production-tagged netfilter image"] +fn the_hosts_own_egress_is_unaffected_before_during_and_after_a_jobs_cleanup() { + require_default_netfilter_image(); + let net = RunscNet::new(); + + // CONTROL: the instrument can say "no". 9997 is neither PORT nor OTHER_PORT, so nothing in the + // canary is listening on it. + assert!( + !host_can_reach(&net.allowed_ip, "9997"), + "the host probe reported a port nobody listens on as reachable — it cannot distinguish \ + anything, and the three legs below would pass without measuring" + ); + + // BEFORE: no contained job has existed on this network yet. + assert!( + wait_until(20, || host_can_reach(&net.allowed_ip, Canary::PORT)), + "the host could not reach the canary at {}:{} before any job was prepared — the fixture, \ + not the containment, is what this leg would otherwise blame", + net.allowed_ip, + Canary::PORT + ); + + let config = gate_config(&net.network); + let policy = maxplayer_core::seller_exec::SandboxPolicy::from_config(Some(&config)) + .expect("a docker policy"); + let workdir = std::env::temp_dir().join(owned_name("workdir-host")); + std::fs::create_dir_all(&workdir).expect("a workdir"); + let runtime = tokio::runtime::Runtime::new().expect("a runtime"); + + // DURING: inside the window where the job is contained and its payload has run. The job itself + // is asserted as contained, so this is not measuring an empty namespace. + let during = runtime.block_on(maxplayer_core::seller_exec::with_prepared_launch( + &payload_command(RunscNet::DENIED_IP, Canary::PORT), + &policy, + &workdir, + &gate_identity(), + std::time::Duration::from_secs(120), + Some(gate_deadline_unix(120)), + |launch, holder| { + let holder = holder.expect("containment"); + route_on_link(holder, RunscNet::DENIED_IP); + let contained = run_launch_attributably(launch); + (contained, host_can_reach(&net.allowed_ip, Canary::PORT)) + }, + )); + let _ = std::fs::remove_dir_all(&workdir); + let (contained, host_during) = during.expect("preparation must succeed"); + + assert_eq!( + contained, + PayloadOutcome::Refused, + "the job was not contained, so the host leg beside it measured nothing about containment" + ); + assert!( + host_during, + "the host lost its own egress to {}:{} while a job was contained — the job's rules are not \ + scoped to its veth", + net.allowed_ip, + Canary::PORT + ); + + // AFTER: the guard has dropped, so teardown has run. + assert!( + host_can_reach(&net.allowed_ip, Canary::PORT), + "the host lost its own egress to {}:{} after a job's teardown — cleanup deleted something \ + it did not own", + net.allowed_ip, + Canary::PORT + ); +} + +// ============================================================================================ +// F2 — the legs round 1 named missing: IPv6, both registered runtimes, and the proxy pinhole. +// +// Authored under a hold that forbade running them; **first executed 2026-09-14** against a real +// daemon in the `gvisor-repro` VM, with both runtimes registered and the production tag resolved +// locally. The prediction written here at authoring time — that the first real run would need +// fixture-addressing adjustment — is what happened: the v6 leg's control read `docker run --detach` +// as readiness and probed a listener that had not yet reached `nc -l`. That is fixed by retrying +// the identical probe. The containment assertions themselves were not touched. +// ============================================================================================ + +/// The `[sandbox]` section of [`gate_config`] plus the pinhole an operator configures. Kept apart +/// from `gate_config` because the pinhole adds a second reason for a leg to differ from its +/// control, which is exactly what the base matrix is built to avoid. +fn gate_config_with_pinhole(network: &str, range: &str) -> maxplayer_core::home::SandboxConfig { + maxplayer_core::home::SandboxConfig { + proxy_port_range: Some(range.to_owned()), + ..gate_config(network) + } +} + +/// [`gate_config`] pinned to one named container runtime. +fn gate_config_with_runtime( + network: &str, + runtime: &str, +) -> maxplayer_core::home::SandboxConfig { + maxplayer_core::home::SandboxConfig { + runtime: Some(runtime.to_owned()), + ..gate_config(network) + } +} + +/// **The pinhole, as production installs it.** +/// +/// Round 1's matrix configured no proxy range at all, so nothing measured the one rule whose job is +/// to let traffic *through* a denied range. The hand-built fixture earlier in this file covers the +/// pinhole's semantics; what was missing is that **production's own preparation path** puts it on +/// the veth the packets leave by, with the range the operator wrote and no other. +/// +/// This leg reads the prepared namespace's own `tc` output back in the window between containment +/// and payload start, and checks the pinhole against the configured range rather than against +/// anything this file rendered. The payload leg that follows is the discriminator: a pinhole wide +/// enough to be useless would still satisfy a readback that only counted rules. +/// +/// **Scope.** This establishes that the pinhole production installs is no wider than the +/// configuration -- including that the resolver exceptions go to the configured resolver and +/// nowhere else. It does NOT establish that resolution works through them; see +/// [`GATE_DNS_RESOLVER`] for where that is measured. +#[test] +#[ignore = "needs docker and the production-tagged netfilter image"] +fn the_pinhole_production_installs_is_the_one_the_policy_names() { + require_default_netfilter_image(); + let net = RunscNet::new(); + const RANGE: &str = "49200-49299"; + const TC_RANGE: &str = "49200-49299"; + + // Port paired with destination: a port-53 rule is only admissible if it goes to the resolver + // the gate configured, and that is unanswerable from the port alone. + let seen_range = std::sync::Arc::new(std::sync::Mutex::new(Vec::<(String, String)>::new())); + let recorder = std::sync::Arc::clone(&seen_range); + + // The payload still goes to a denied destination: the pinhole must not become a hole. + let denied = integrated_leg_with( + gate_config_with_pinhole(&net.network, RANGE), + RunscNet::DENIED_IP, + Canary::PORT, + move |holder| { + route_on_link(holder, RunscNet::DENIED_IP); + let dev = egress_dev(holder); + let readback = iface_readback(holder, &dev); + let filters = maxplayer_core::sandbox_iface::parse_filters(&readback) + .expect("the prepared namespace's own tc output must parse"); + let ports: Vec<(String, String)> = filters + .iter() + .filter(|filter| filter.actions == vec!["pass".to_owned()]) + .filter_map(|filter| { + filter.key("dst_port").map(|port| { + (port.to_owned(), filter.key("dst_ip").unwrap_or("any").to_owned()) + }) + }) + .collect(); + *recorder.lock().expect("the recorder") = ports; + }, + ) + .expect("preparation must succeed"); + + let ports = seen_range.lock().expect("the recorder").clone(); + assert!( + ports.iter().any(|(port, _)| port == TC_RANGE), + "production installed no pass rule for the configured proxy range {RANGE} — the pinhole the \ + operator wrote is not on the veth the packets leave by. Pass rules carried ports: {ports:?}" + ); + // Every pass rule is one of exactly two things the configuration asked for: the proxy range, or + // a resolver pinhole on port 53 to a resolver the gate NAMED. The second admits the DNS + // exceptions without widening the check -- the destination is pinned to the configured address, + // so a port-53 rule to anywhere else still fails here, and so does any other port. + // + // Both intents, in one assertion. "No pinhole wider than its configuration" is the whole point + // of this leg, and a job that cannot resolve a name is useless: the resolver exception exists + // and is bounded, rather than being either banned or waved through. + let stray: Vec<&(String, String)> = ports + .iter() + .filter(|(port, dst)| { + let proxy = port == TC_RANGE; + let resolver = port == "53" && dst.starts_with(GATE_DNS_RESOLVER); + !proxy && !resolver + }) + .collect(); + assert!( + stray.is_empty(), + "production installed a pass rule the operator did not write: {stray:?} (all pass rules: \ + {ports:?}) — a pinhole wider than its configuration is a hole. Only the proxy range \ + {TC_RANGE} and port 53 to the configured resolver {GATE_DNS_RESOLVER} are configured here" + ); + assert!( + ports.iter().any(|(port, dst)| port == "53" && dst.starts_with(GATE_DNS_RESOLVER)), + "production installed no resolver pinhole for the configured {GATE_DNS_RESOLVER} — a job \ + that cannot reach its own resolver resolves nothing. Pass rules carried: {ports:?}" + ); + assert_eq!( + denied, + PayloadOutcome::Refused, + "a configured pinhole must not make an unrelated denied destination reachable" + ); +} + +/// **Both registered runtimes, one production path.** +/// +/// Round 1 proved containment under whichever runtime docker happens to default to. gVisor is the +/// reason this work exists and `runsc` reimplements the network stack, so "contained under runc" +/// and "contained under runsc" are two claims, not one. Running the identical leg under each is the +/// only thing that tells them apart. +#[test] +#[ignore = "needs docker, the production-tagged image and a runsc runtime"] +fn both_registered_runtimes_are_contained_by_the_same_production_path() { + require_default_netfilter_image(); + let net = RunscNet::new(); + assert!( + net.reachable_from_outside(RunscNet::DENIED_IP), + "control: {} must answer from outside, or every refusal below proves nothing", + RunscNet::DENIED_IP + ); + + for runtime in ["runc".to_owned(), runsc_runtime()] { + let denied = integrated_leg_with( + gate_config_with_runtime(&net.network, &runtime), + RunscNet::DENIED_IP, + Canary::PORT, + |holder| route_on_link(holder, RunscNet::DENIED_IP), + ) + .unwrap_or_else(|error| panic!("preparation must succeed under {runtime}: {error}")); + assert_eq!( + denied, + PayloadOutcome::Refused, + "under runtime {runtime} a job prepared and launched by production reached the denied {}", + RunscNet::DENIED_IP + ); + + let allowed = integrated_leg_with( + gate_config_with_runtime(&net.network, &runtime), + &net.allowed_ip, + Canary::PORT, + |_| {}, + ) + .unwrap_or_else(|error| panic!("preparation must succeed under {runtime}: {error}")); + assert_eq!( + allowed, + PayloadOutcome::Connected, + "positive control under runtime {runtime}: the allowed {} must stay reachable — a \ + runtime whose networking is broken denies everything and looks contained", + net.allowed_ip + ); + } +} + +/// An IPv6-capable network with a listener answering on **both** an allowed and a denied v6 address. +/// +/// The whole point of the v6 leg: the policy renders v6 drops, and until something dials a v6 +/// address through the production path, an unfiltered second address family is the cheapest bypass +/// on the box — and the one a v4-only matrix cannot see. +struct V6Net { + network: String, + listener: String, +} + +impl V6Net { + /// Documentation prefix (RFC 3849). No policy rule denies it, so it is the allowed control. + const SUBNET: &'static str = "2001:db8:ff::/64"; + const ALLOWED_IP: &'static str = "2001:db8:ff::2"; + /// Unique-local (RFC 4193), inside the `fc00::/7` this repo's policy drops. + const DENIED_IP: &'static str = "fd00:dead:beef::2"; + + fn new() -> Self { + let network = owned_name("v6-net"); + let listener = owned_name("v6-listener"); + let (ok, _, err) = docker( + &[ + "network", + "create", + "--ipv6", + "--label", + &owner_label(), + "--subnet", + Self::SUBNET, + &network, + ], + None, + ); + assert!( + ok, + "could not create the v6 network {network}: {err}\n\ + If this says IPv6 is not enabled, the daemon needs `\"ipv6\": true` — a daemon-level \ + change, which is a decision to name rather than to make from a test." + ); + + // One process, both addresses, exactly as the v4 fixture does it. + let (ok, _, err) = docker( + &[ + "run", + "--detach", + "--name", + &listener, + "--label", + &owner_label(), + "--network", + &network, + "--cap-add", + "NET_ADMIN", + "--entrypoint", + "sh", + &netfilter_image(), + "-c", + &format!( + "ip -6 addr add {}/128 dev eth0 && while :; do nc -l -p {} >/dev/null 2>&1; done", + Self::DENIED_IP, + Canary::PORT + ), + ], + None, + ); + assert!(ok, "could not start the v6 listener: {err}"); + let net = Self { network, listener }; + assert!( + wait_until(20, || net.reachable_from_outside(Self::DENIED_IP)), + "the v6 listener never answered on {} — see `RunscNet::await_listener`", + Self::DENIED_IP + ); + net + } + + /// The listener's own link-local address, as the kernel assigned it. + /// + /// Discovered rather than constructed: it is derived from the interface's MAC, so computing it + /// here would be a second implementation of SLAAC and would silently drift from the address + /// the listener actually answers on. + fn listener_link_local(&self) -> String { + let (ok, out, err) = docker( + &[ + "exec", + &self.listener, + "ip", + "-6", + "-oneline", + "addr", + "show", + "dev", + "eth0", + "scope", + "link", + ], + None, + ); + assert!(ok, "could not read the v6 listener's link-local address: {err}"); + let address = out + .split_whitespace() + .skip_while(|token| *token != "inet6") + .nth(1) + .and_then(|cidr| cidr.split('/').next()) + .unwrap_or_default() + .to_owned(); + assert!( + address.starts_with("fe80:"), + "expected a link-local address on the listener, read {address:?} from {out:?}" + ); + address + } + + /// [`Self::reachable_from_outside`], allowing the listener time to come up. + /// + /// `docker run --detach` returns when the container is *started*, not when the process inside + /// it has added its second address and reached `nc -l`. The first real run of this file probed + /// immediately and read that startup gap as a dead listener. This retries the identical probe, + /// so a destination that never answers still fails — the wait buys the fixture time, it does + /// not soften what the control proves. + fn reachable_from_outside_within(&self, ip: &str, attempts: u32) -> bool { + for attempt in 0..attempts { + if self.reachable_from_outside(ip) { + return true; + } + if attempt + 1 < attempts { + std::thread::sleep(std::time::Duration::from_secs(1)); + } + } + false + } + + /// From a container on the network but outside every contained namespace — the discriminator a + /// refusal inside cannot supply for itself. + fn reachable_from_outside(&self, ip: &str) -> bool { + let (ok, _, _) = docker( + &[ + "run", + "--rm", + "--network", + &self.network, + "--cap-add", + "NET_ADMIN", + "--entrypoint", + "sh", + &netfilter_image(), + "-c", + &format!("ip -6 route add {ip}/128 dev eth0; nc -w 2 {ip} {}", Canary::PORT), + ], + None, + ); + ok + } +} + +impl Drop for V6Net { + fn drop(&mut self) { + remove_owned_container(&self.listener); + remove_owned_network(&self.network); + } +} + +/// Make a v6 address reachable on-link inside the holder, so a refusal is the filters and not a +/// missing route. +fn route6_on_link(holder: &str, ip: &str) { + let (ok, _, err) = docker( + &[ + "run", + "--rm", + "--network", + &format!("container:{holder}"), + "--cap-drop", + "ALL", + "--cap-add", + "NET_ADMIN", + "--entrypoint", + "ip", + &netfilter_image(), + "-6", + "route", + "add", + &format!("{ip}/128"), + "dev", + "eth0", + ], + None, + ); + assert!(ok, "could not make {ip} routable inside {holder}: {err}"); +} + +/// **The second address family, through the production path.** +/// +/// Same three-leg shape as the v4 matrix, and for the same reasons: an outside control so a refusal +/// is not a dead listener, a denied leg, and an allowed leg so "denies everything" cannot pass as +/// containment. +#[test] +#[ignore = "needs docker, an IPv6-enabled daemon and the production image"] +fn the_denied_v6_prefix_is_denied_through_the_production_path() { + require_default_netfilter_image(); + let net = V6Net::new(); + + assert!( + net.reachable_from_outside_within(V6Net::DENIED_IP, 10), + "control: {} must answer from outside, or the denied leg below proves nothing", + V6Net::DENIED_IP + ); + + let denied = integrated_leg(&net.network, V6Net::DENIED_IP, Canary::PORT, |holder| { + route6_on_link(holder, V6Net::DENIED_IP) + }) + .expect("preparation must succeed"); + assert_eq!( + denied, + PayloadOutcome::Refused, + "a job prepared and launched by production reached the denied v6 {} — an unfiltered second \ + address family is the cheapest bypass there is", + V6Net::DENIED_IP + ); + + // The positive control, and what makes the denial above mean something. + // + // On its first real run (2026-09-14) this control failed, and the cause was in the policy, not + // in the test: the ALLOWED v6 address — in none of `DENIED_DESTINATIONS_V6` — was refused too. + // IPv6 Neighbour Solicitation goes to a solicited-node MULTICAST address, so the `ff00::/8` + // drop starved neighbour discovery and the namespace lost every v6 destination, denied and + // allowed alike. A denial measured on a dead v6 stack proves nothing about `fc00::/7`. + // + // `sandbox_net` now permits exactly the two ND control messages — solicitation to + // `ff02::1:ff00:0/104` and advertisement, both at hop limit 255 — and nothing else in + // `ff00::/8`. This assertion is the live proof that the narrowing works: the allowed address + // connects, which means ND resolved, which means the refusal above was the destination rule + // doing its job. If it ever reads `Refused` again the two legs must be read together, because + // a starved stack refuses both. + let allowed = integrated_leg(&net.network, V6Net::ALLOWED_IP, Canary::PORT, |_| {}) + .expect("preparation must succeed"); + assert_eq!( + allowed, + PayloadOutcome::Connected, + "the allowed v6 {} was refused by a job production contained — if neighbour discovery is \ + starved again then the denied leg above is denial by a dead stack and proves nothing", + V6Net::ALLOWED_IP + ); +} + +/// Permitting neighbour discovery must not have permitted link-local **traffic**. +/// +/// This is the counter-control for the ND exception, and it is the leg that would catch the lazy +/// fix. Making `integrated.allowed.v6` pass by widening `fe80::/10` — or by accepting all ICMPv6, +/// or all multicast — would light up the positive control just as well, and this leg is what tells +/// the two apart: the listener answers on its own link-local address, an unfiltered joiner reaches +/// it, and a contained job must not. +/// +/// `fe80::/10` is the range neighbour ADVERTISEMENT is sent into, so it is the one an over-broad ND +/// exception opens first. +#[test] +#[ignore = "needs docker, an IPv6-enabled daemon and the production image"] +fn permitting_neighbour_discovery_did_not_permit_link_local_traffic() { + require_default_netfilter_image(); + let net = V6Net::new(); + let link_local = net.listener_link_local(); + // A link-local destination is only meaningful with the interface it is scoped to. + let scoped = format!("{link_local}%eth0"); + + assert!( + net.reachable_from_outside_within(&scoped, 10), + "control: the listener must answer on its own link-local {scoped} from an UNFILTERED \ + joiner, or a refusal below proves nothing about the filters" + ); + + let outcome = integrated_leg(&net.network, &scoped, Canary::PORT, |_| {}) + .expect("preparation must succeed"); + assert_eq!( + outcome, + PayloadOutcome::Refused, + "a contained job reached the link-local {scoped} — the neighbour-discovery exception has \ + been widened past the two ICMPv6 control messages into ordinary link-local traffic" + ); +} + +/// Build a launch into an EXISTING holder, for the sibling check's second probe. Goes through the +/// production argv builder, and names the namespace explicitly rather than preparing a new one. +fn prepared_launch_for( + policy: &maxplayer_core::seller_exec::SandboxPolicy, + workdir: &std::path::Path, + ip: &str, + holder: &str, +) -> maxplayer_core::seller_exec::AgentLaunch { + policy + .launch( + &payload_command(ip, Canary::PORT), + &maxplayer_core::seller_exec::JobLaunch { + workdir, + env: &[], + uid: 0, + gid: 0, + netns: Some(holder), + // The canary payload is a plain command; the argv builder reads this list only + // to decide whether any server needs a mount, and none does here. + mcp_servers: &[], + // The canary payload dials a numeric address and resolves nothing, so it is handed + // no `/etc/resolv.conf` mount. Containment is what this launch measures. + resolv_conf: None, + }, + ) + .expect("the policy must build a launch") +} + +// --------------------------------------------------------------------------------------------- +// The two legs the renewed round-1 verdict found missing: a proxy connection that actually +// SUCCEEDS, and a cancellation resolved by the real daemon rather than by a stand-in. +// --------------------------------------------------------------------------------------------- + +/// A real TCP listener on the host, accepting until the test drops it. +/// +/// A listener in the test process rather than a container, because the leg under test is precisely +/// container-to-HOST: a listener living on the docker network would prove the pinhole reaches +/// another container, which is not where the credential proxy runs. +struct HostListener { + stop: std::sync::Arc, + handle: Option>, +} + +impl HostListener { + fn bind(port: u16) -> Self { + let listener = std::net::TcpListener::bind(("0.0.0.0", port)) + .unwrap_or_else(|error| panic!("could not bind the host listener on {port}: {error}")); + listener.set_nonblocking(true).expect("nonblocking"); + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let flag = std::sync::Arc::clone(&stop); + let handle = std::thread::spawn(move || { + while !flag.load(std::sync::atomic::Ordering::Relaxed) { + match listener.accept() { + Ok((stream, _)) => { + let _ = stream.shutdown(std::net::Shutdown::Both); + } + Err(_) => std::thread::sleep(std::time::Duration::from_millis(20)), + } + } + }); + Self { stop, handle: Some(handle) } + } +} + +impl Drop for HostListener { + fn drop(&mut self) { + self.stop.store(true, std::sync::atomic::Ordering::Relaxed); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} + +/// A contained job **actually reaches the credential proxy on the host**, through the pinhole +/// production installed, under the gVisor runtime. +/// +/// Every earlier live proxy test measured a DENIAL. Denial is the cheap half: a policy that drops +/// everything passes all of them, and the job it contains cannot do its work. The half that was +/// missing — and that the renewed round-1 verdict called out — is that the one address the job is +/// supposed to reach is actually reachable. That is what this asserts first. +/// +/// Both ports have a live listener on the host, and only one is inside `proxy_ports`. That is the +/// control built into the run: a refused connection here cannot be blamed on an absent listener, +/// and a permitted one cannot be blamed on a blanket allow, because the two legs differ only in +/// whether the pinhole names the port. +#[test] +#[ignore = "needs docker, gVisor and the netfilter image"] +fn a_contained_job_actually_connects_to_the_host_proxy_through_the_pinhole() { + let runtime_name = runsc_runtime(); + let network = owned_name("net-proxy-reach"); + let network = network.as_str(); + let (ok, _, err) = docker(&["network", "create", "--label", &owner_label(), network], None); + assert!(ok, "could not create the test network: {err}"); + + // A multi-port range, which is the shape production ships and every other tc-path test uses. + // The one-port shape is measured on its own by + // [`a_single_port_pinhole_establishes_and_the_job_reaches_only_that_port`]: it used to render + // `dst_port N-N` and be refused by the tc flower classifier outright. Keeping the two shapes in + // separate gates means neither can cover for a regression in the other. + let pinhole = PortRange::new(49220, 49229).expect("valid range"); + let allowed_port: u16 = 49221; // inside the pinhole + let denied_port: u16 = 49401; // outside it + let _allowed_listener = HostListener::bind(allowed_port); + let _denied_listener = HostListener::bind(denied_port); + + let rt = tokio::runtime::Runtime::new().expect("a runtime"); + let outcome = rt.block_on(maxplayer_core::sandbox_netns::establish( + network, + &holder_image(), + &netfilter_image(), + "host.docker.internal", + "live-proxy-reach", + "3333333333333333333333333333333333333333333333333333333333333333", + 1000, + 1000, + Some(pinhole), + true, + Vec::new(), + 2_000_000_000, + )); + + let containment = match outcome { + Ok(containment) => containment, + Err(error) => { + remove_owned_network(network); + panic!("establish failed: {error}"); + } + }; + let holder = containment.holder.name().to_owned(); + let proxy_host = containment.proxy_host.clone(); + let netns = format!("container:{holder}"); + + let reached = connect_under(&runtime_name, &netns, &proxy_host, &allowed_port.to_string()); + let refused = connect_under(&runtime_name, &netns, &proxy_host, &denied_port.to_string()); + + drop(containment); + remove_owned_network(network); + + assert!( + reached.connected(), + "the contained job could NOT reach the proxy at {proxy_host}:{allowed_port}, the one \ + address the pinhole exists to permit — a job under this policy cannot do its work \ + ({reached:?})" + ); + // `denied` refuses to read a tool failure as a denial: if the probe never ran, this panics + // rather than crediting containment that was never exercised. + assert!( + refused.denied(&format!("{proxy_host}:{denied_port}")), + "the contained job reached {proxy_host}:{denied_port}, which is OUTSIDE the pinhole: the \ + permit is not confined to the port the policy names" + ); +} + +/// The SINGLE-PORT pinhole, end to end against a real kernel. +/// +/// A one-port proxy range is supported configuration — `PortRange::new` accepts equal endpoints and +/// the parser documents a bare `"49200"` — and until this round it could not run at all. iptables +/// renders it `49221:49221`, the tc translation turned that into `dst_port 49221-49221`, and the +/// flower classifier refused it outright, so the filters never installed and establishment refused +/// the launch. Fail-closed, but a supported configuration that cannot start is still broken. +/// +/// Rendering it as the bare port is what the offline gates assert. Only a real `tc` can say whether +/// that rendering is one it ACCEPTS, and that is what this measures — the half no amount of string +/// assertion can reach. +/// +/// The control is the same as the range gate's, and tighter: both ports carry a live host listener, +/// and the denied one sits directly ABOVE the single permitted port, so a permit that quietly +/// widened by even one port fails here. +#[test] +#[ignore = "needs docker, gVisor and the netfilter image"] +fn a_single_port_pinhole_establishes_and_the_job_reaches_only_that_port() { + let runtime_name = runsc_runtime(); + let network = owned_name("net-proxy-singleton"); + let network = network.as_str(); + let (ok, _, err) = docker(&["network", "create", "--label", &owner_label(), network], None); + assert!(ok, "could not create the test network: {err}"); + + // start == end: the exact shape that could not be installed before this round. + let pinhole = PortRange::new(49221, 49221).expect("a single port is a valid range"); + let allowed_port: u16 = 49221; + let denied_port: u16 = 49222; + let _allowed_listener = HostListener::bind(allowed_port); + let _denied_listener = HostListener::bind(denied_port); + + let rt = tokio::runtime::Runtime::new().expect("a runtime"); + let outcome = rt.block_on(maxplayer_core::sandbox_netns::establish( + network, + &holder_image(), + &netfilter_image(), + "host.docker.internal", + "live-proxy-singleton", + "3333333333333333333333333333333333333333333333333333333333333333", + 1000, + 1000, + Some(pinhole), + true, + Vec::new(), + 2_000_000_000, + )); + + let containment = match outcome { + Ok(containment) => containment, + Err(error) => { + remove_owned_network(network); + // Exactly the failure the defect produced: tc refuses the filter, so containment cannot + // be established and the job never launches. + panic!( + "establish FAILED for a single-port pinhole: {error} — a supported one-port proxy \ + range must install like any other" + ); + } + }; + let holder = containment.holder.name().to_owned(); + let proxy_host = containment.proxy_host.clone(); + let netns = format!("container:{holder}"); + + let reached = connect_under(&runtime_name, &netns, &proxy_host, &allowed_port.to_string()); + let refused = connect_under(&runtime_name, &netns, &proxy_host, &denied_port.to_string()); + + drop(containment); + remove_owned_network(network); + + assert!( + reached.connected(), + "the contained job could NOT reach {proxy_host}:{allowed_port}, the single port its own \ + pinhole names ({reached:?}) — a one-port range must be installable, not merely accepted \ + by the type" + ); + assert!( + refused.denied(&format!("{proxy_host}:{denied_port}")), + "the contained job reached {proxy_host}:{denied_port}, one port ABOVE its single-port \ + pinhole: collapsing an equal-endpoint range to a bare port must not widen what it permits" + ); +} + +/// A cancelled `establish` leaves no holder behind **against the real docker daemon**. +/// +/// The offline cancellation gate drives a stand-in client, which earns ordering credit and nothing +/// more: a stand-in cannot show what a real daemon does with a create that was still running when +/// its caller went away. Here the daemon is real, the create is real, and the question is answered +/// by asking docker what containers exist afterwards. +/// +/// The cancellation walks across the create window rather than firing once, because the window is +/// short and a single fixed delay can miss it entirely — and a run that never cancelled mid-create +/// would pass no matter what cleanup did. A leak in ANY attempt fails the gate. +#[test] +#[ignore = "needs docker and the netfilter image"] +fn a_cancelled_establish_leaves_no_holder_behind_against_the_real_daemon() { + let network = owned_name("net-cancel-live"); + let network = network.as_str(); + let (ok, _, err) = docker(&["network", "create", "--label", &owner_label(), network], None); + assert!(ok, "could not create the test network: {err}"); + + let rt = tokio::runtime::Runtime::new().expect("a runtime"); + let mut leaked: Vec = Vec::new(); + // How many attempts actually cancelled an establish that was still running. Nothing in the + // previous version required this to be above zero, so a walk that never crossed the create + // window would have reported a clean pass having cancelled nothing at all. + let mut cancelled_in_flight = 0u32; + // Bound outside the future: it is polled to cancellation below, so anything it borrows has to + // outlive the statement that builds it. + let holder_image = holder_image(); + let netfilter_image = netfilter_image(); + + for attempt in 0..5u32 { + let job = format!("live-cancel-{attempt}"); + let holder = format!("maxplayer-netns-{job}"); + let delay = std::time::Duration::from_millis(150 + u64::from(attempt) * 120); + // Whether the cancellation landed on a still-running establish, rather than after one that + // had already finished. + let mut hit_creation = false; + + rt.block_on(async { + let mut establishing = Box::pin(maxplayer_core::sandbox_netns::establish( + network, + &holder_image, + &netfilter_image, + "host.docker.internal", + &job, + "3333333333333333333333333333333333333333333333333333333333333333", + 1000, + 1000, + None, + true, + Vec::new(), + 2_000_000_000, + )); + tokio::select! { + // establish won the race: this attempt exercised cleanup after SUCCESS. That must + // still not leak, but it says nothing about cancellation, so it is not counted as + // one. + _ = establishing.as_mut() => {} + _ = tokio::time::sleep(delay) => hit_creation = true, + } + drop(establishing); // the cancellation under test + }); + if hit_creation { + cancelled_in_flight += 1; + } + + // Absence, asked so that only a real absence can answer it. Three things together, because + // any one of them alone is satisfiable by a run that measured nothing: + // + // * a SUCCESSFUL daemon query. `docker ps` that fails prints nothing on stdout, and an + // empty stdout is exactly what a clean daemon prints too — success-shaped emptiness that + // reads identically whether the oracle worked or never ran at all. + // * the HOLDER AND ITS JOINERS. Sidecars are named `---`, so + // this substring filter covers them; a surviving sidecar pins the namespace the holder + // was torn down to release, and checking holder names alone would miss it entirely. + // * absence that is STABLE across consecutive answers rather than the first one seen. A + // create that outlived its cancellation can still land, so an early empty listing is a + // container that has not appeared YET, not one that never will. + const STABLE_ANSWERS: u32 = 5; + let mut consecutive_absent = 0u32; + let mut answered = 0u32; + let mut survivors: Vec = Vec::new(); + let mut last_error = String::new(); + let give_up = std::time::Instant::now() + std::time::Duration::from_secs(30); + loop { + let (ok, listed, err) = docker( + &["ps", "--all", "--format", "{{.Names}}", "--filter", &format!("name={holder}")], + None, + ); + if ok { + answered += 1; + let names: Vec = listed + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(str::to_owned) + .collect(); + if names.is_empty() { + consecutive_absent += 1; + } else { + consecutive_absent = 0; + survivors = names; + } + } else { + // A daemon that cannot answer is not a daemon reporting "clean". + consecutive_absent = 0; + last_error = err; + } + if consecutive_absent >= STABLE_ANSWERS || std::time::Instant::now() >= give_up { + break; + } + std::thread::sleep(std::time::Duration::from_millis(500)); + } + + assert!( + answered > 0, + "the daemon never successfully answered what exists under {holder} (last error: \ + {last_error:?}); with no answer at all there is nothing to conclude, and concluding \ + \"absent\" from a failed query is the whole defect this gate exists to catch" + ); + if consecutive_absent < STABLE_ANSWERS { + leaked.push(if survivors.is_empty() { holder.clone() } else { survivors.join(", ") }); + for name in survivors.iter().chain(std::iter::once(&holder)) { + remove_owned_container(name); + } + } + } + + remove_owned_network(network); + assert!( + cancelled_in_flight > 0, + "not one of the attempts cancelled an establish that was still running — every one of them \ + finished first, so this run measured cleanup after success and never exercised \ + cancellation at all. A green here would be a green for a property nothing tested." + ); + assert!( + leaked.is_empty(), + "a cancelled establish left {leaked:?} running against the real daemon: the create landed \ + after cleanup had already given up on it, and nothing owns those namespaces now" + ); +} diff --git a/docker/maxplayer-netfilter/Dockerfile b/docker/maxplayer-netfilter/Dockerfile index 48a0cf03f..154485709 100644 --- a/docker/maxplayer-netfilter/Dockerfile +++ b/docker/maxplayer-netfilter/Dockerfile @@ -38,11 +38,25 @@ FROM alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943 # `iptables` provides BOTH iptables and ip6tables. The policy denies the v6 LAN-equivalents as well, # and an image with only the v4 binary would fail those rules at apply time rather than silently skip # them — but there is no reason to rely on that. -RUN apk add --no-cache iptables \ +# +# `iproute2` provides `tc` and `ip`, and they are as load-bearing as iptables is. A gVisor payload +# writes its packets straight onto the namespace's veth without traversing the host's `OUTPUT` chain, +# so the iptables plan alone contains a `runc` job and lets a `runsc` one through — measured, both +# families, TCP. The egress filter that closes that hole lives on the veth itself, and `tc` is the +# only tool that installs it. `ip` is here for the link enumeration the daemon validates the +# interface's identity from; both are read back by a separate container. +# +# The `test -x` assertions are the gate, not the `apk add`: a base image that moved `tc` into a +# subpackage would otherwise produce an image that builds clean and cannot filter, and the first +# thing that noticed would be a job running uncontained. +RUN apk add --no-cache iptables iproute2 \ && test -x /usr/sbin/iptables \ - && test -x /usr/sbin/ip6tables + && test -x /usr/sbin/ip6tables \ + && test -x /sbin/tc \ + && test -x /sbin/ip COPY apply-policy.sh /usr/local/bin/apply-policy -RUN chmod 0755 /usr/local/bin/apply-policy +COPY apply-iface.sh /usr/local/bin/apply-iface +RUN chmod 0755 /usr/local/bin/apply-policy /usr/local/bin/apply-iface ENTRYPOINT ["/usr/local/bin/apply-policy"] diff --git a/docker/maxplayer-netfilter/apply-iface.sh b/docker/maxplayer-netfilter/apply-iface.sh new file mode 100644 index 000000000..51358ef44 --- /dev/null +++ b/docker/maxplayer-netfilter/apply-iface.sh @@ -0,0 +1,80 @@ +#!/bin/sh +# Applies a rendered EGRESS INTERFACE plan inside the job's network namespace (#797 follow-on). +# +# WHY THIS EXISTS ALONGSIDE apply-policy +# `apply-policy` installs the iptables plan. That plan contains a `runc` job and does not contain a +# `runsc` (gVisor) one: gVisor's netstack writes the payload's packets onto the namespace's veth +# without traversing the host's `OUTPUT` chain, so the denied destinations stay reachable — measured +# on both families over TCP. The filter that closes it has to sit on the veth itself, which means +# `tc`, which means a second plan and a second applier. +# +# Input: the plan on stdin, one rule per line, as `tc ` — exactly the argv rendered by +# `maxplayer-core::sandbox_iface::IfacePlan`. This script holds no policy of its own, and in +# particular it does not choose the interface, the prefixes or the order: all three are rendered and +# unit-tested in Rust, and read back out of the kernel by a different container before any payload +# starts. A dumb applier cannot get containment subtly wrong in a way the renderer's tests do not see. +# +# Exit codes are what the daemon branches on, so they are specific rather than a bare 1: +# +# 0 every rule applied, and there was at least one +# 3 a rule failed. The namespace is now PARTIALLY filtered. The caller must DESTROY THE HOLDER +# rather than retry: the qdisc and the filters that did land are still there, and a retry stacks +# a second copy on top of them at the same priorities, leaving a ruleset whose order nobody +# rendered. +# 4 the plan was empty — refusing to report success for an unfiltered interface +# 5 a line named a binary other than tc +# 6 `tc` is not in this image at all. Distinct from 3 because it is a BUILD fault, not a runtime +# one: it means the sidecar image shipped without iproute2, and no amount of retrying or +# re-rendering will fix it. +set -u + +# Checked once, before anything is read, so the error names the real fault instead of surfacing as +# rule 1 of N failing with "not found". An image without `tc` can still apply an iptables plan +# perfectly, so this is exactly the skew that would otherwise ship quietly. +command -v tc >/dev/null 2>&1 || { + echo "apply-iface: no 'tc' in this image — the sidecar was built without iproute2" >&2 + exit 6 +} + +applied=0 + +while IFS= read -r line; do + [ -n "$line" ] || continue + + # The first field names the binary. Anything else is refused rather than executed: this container + # is one of the two things in the design that hold CAP_NET_ADMIN, so it must never become a + # general-purpose exec surface for whatever can reach its stdin. `ip` is deliberately NOT allowed + # here even though the image carries it — enumerating links is a read the daemon does from an + # unprivileged container, and an applier that can also run `ip link` can move an interface. + binary=${line%% *} + case "$binary" in + tc) ;; + *) + echo "apply-iface: refusing to run '$binary' — an interface plan may only name tc" >&2 + exit 5 + ;; + esac + + # Intentionally unquoted: POSIX sh word-splits, and the fields ARE the argv. + # (Porting note: zsh does NOT word-split, so the same line there executes as one command NAME and + # fails with "command not found" naming the entire rule.) + # shellcheck disable=SC2086 + if ! $line; then + echo "apply-iface: rule $((applied + 1)) failed: $line" >&2 + echo "apply-iface: namespace is PARTIALLY filtered — destroy the holder, do not retry" >&2 + exit 3 + fi + applied=$((applied + 1)) +done + +# An empty plan applies cleanly and proves nothing. Reporting success here would hand the daemon a +# green for an interface with no egress filter at all — the failure mode being avoided is not "the +# filters were wrong" but "there were none and everything said OK". +if [ "$applied" -eq 0 ]; then + echo "apply-iface: empty plan — refusing to report success for an unfiltered interface" >&2 + exit 4 +fi + +# The count is echoed so the caller can cross-check it against the number of rules it rendered. A +# mismatch means stdin was truncated in transit, which no exit code would otherwise reveal. +echo "$applied" diff --git a/scripts/sandbox-acceptance.sh b/scripts/sandbox-acceptance.sh new file mode 100755 index 000000000..78b168a6c --- /dev/null +++ b/scripts/sandbox-acceptance.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# +# The ONE offline acceptance command for job-local veth containment (#996). +# +# It exists because the obvious command lies. The live containment matrix lives in +# `crates/maxplayer-core/tests/sandbox_netns_live.rs`, every case `#[ignore]`d because each one needs +# a docker daemon, a gVisor runtime and a built sidecar image. So `cargo test -p maxplayer-core +# --features acp,wallet` prints `0 passed; N ignored` and exits 0 on a machine that has never run a +# single containment case — the same green as a machine where every case passed. Accepting on that +# output is accepting nothing. +# +# This entrypoint splits the two claims apart and makes both of them fail-able: +# +# 1. SOURCE — the offline suite (unit + non-live integration) compiles and passes here, now. +# 2. MATRIX — a SAVED record of a live run is validated by `maxplayer_core::sandbox_evidence`: +# every required case present exactly once with the outcome the matrix requires, no unscored +# outcome, no unknown id, and identity headers naming the commit, binary and host that produced +# it. That is the leg `cargo test` alone cannot make. +# +# ★ Replay is NOT here, on purpose. Re-running the live cases needs the VM, the runtime and the +# image; a validator that shelled out to docker would be the live gate wearing a false beard, and +# would turn "is this record complete" into "is the daemon up today". Live replay is the separate +# command documented in the PR body. +# +# ★ A missing record is a FAILURE, never a skip. Refusing to run without one is the whole point: the +# silent skip is the defect this script was written against. +# +# Usage: +# +# scripts/sandbox-acceptance.sh path/to/live-matrix.txt +# MAXPLAYER_LIVE_EVIDENCE=path/to/live-matrix.txt scripts/sandbox-acceptance.sh +# +# Exit 0 means: this source passed its offline suite, and the named record is a complete live matrix +# attributable to a commit contained in this branch. It does NOT mean the live run happened today, +# and it does not re-measure one packet. + +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +RECORD="${1:-${MAXPLAYER_LIVE_EVIDENCE:-}}" + +if [[ -z "${RECORD}" ]]; then + cat >&2 <<'EOF' +sandbox-acceptance: no saved live matrix named. + +Pass one as the first argument, or set MAXPLAYER_LIVE_EVIDENCE. This is deliberately fatal: the +offline suite alone reports the live containment cases as *ignored*, so running without a record +would report success for a matrix nobody measured. + + scripts/sandbox-acceptance.sh evidence//live-matrix.txt +EOF + exit 2 +fi + +if [[ ! -r "${RECORD}" ]]; then + echo "sandbox-acceptance: saved live matrix '${RECORD}' is not readable." >&2 + echo "A named record that is not there is a missing gate, not an absent one." >&2 + exit 2 +fi + +RECORD_ABS="$(cd "$(dirname "${RECORD}")" && pwd)/$(basename "${RECORD}")" + +echo "== sandbox acceptance ==" +echo "repo HEAD : $(git rev-parse HEAD)" +echo "record : ${RECORD_ABS}" +echo "record sha: $(shasum -a 256 "${RECORD_ABS}" | awk '{print $1}')" + +# --- leg 0: the record's source identity belongs to this branch ------------------------------- +# +# `sandbox_evidence::validate` checks that the record NAMES a commit; it cannot check that the +# commit is one of ours. A complete matrix measured on someone else's source is a complete matrix +# about someone else's source, and it would pass the module's checks unchallenged. +SOURCE_HEAD="$(awk -F= '/^source_head=/{print $2; exit}' "${RECORD_ABS}" | tr -d '[:space:]')" +if [[ -z "${SOURCE_HEAD}" ]]; then + echo "sandbox-acceptance: record names no source_head." >&2 + exit 1 +fi +if ! git cat-file -e "${SOURCE_HEAD}^{commit}" 2>/dev/null; then + echo "sandbox-acceptance: record's source_head ${SOURCE_HEAD} is not a commit in this repo." >&2 + exit 1 +fi +if ! git merge-base --is-ancestor "${SOURCE_HEAD}" HEAD; then + echo "sandbox-acceptance: record's source_head ${SOURCE_HEAD} is NOT an ancestor of HEAD." >&2 + echo "The matrix describes source this branch does not contain; re-run the live matrix." >&2 + exit 1 +fi +echo "source_head ${SOURCE_HEAD} is contained in HEAD ($(git rev-list --count "${SOURCE_HEAD}..HEAD") commit(s) since)." + +# --- leg 1: the offline suite, on this source ------------------------------------------------- +# +# `--all-targets` so the live test file is type-checked even though it will not run, and the full +# `acp,wallet` set because `seller_exec` — the caller that hands a job to the contained runtime — is +# wallet-gated and is otherwise compiled out of the run. +echo +echo "-- cargo check (acp,wallet, all targets) --" +cargo check -p maxplayer-core --features acp,wallet --all-targets + +echo +echo "-- cargo test (acp,wallet) --" +cargo test -p maxplayer-core --features acp,wallet + +# --- leg 2: the saved matrix ------------------------------------------------------------------ +# +# The validating test returns early when the variable is unset, so a filter typo would make it +# "pass" having checked nothing. Hence the explicit `1 passed` assertion below: the gate has to prove +# it executed, not merely that nothing complained. +echo +echo "-- saved live matrix --" +MATRIX_LOG="$(mktemp -t sandbox-acceptance-matrix)" +trap 'rm -f "${MATRIX_LOG}"' EXIT +set +e +MAXPLAYER_LIVE_EVIDENCE="${RECORD_ABS}" cargo test -p maxplayer-core --features acp,wallet --lib -- \ + --exact sandbox_evidence::tests::the_named_saved_matrix_validates --nocapture 2>&1 | tee "${MATRIX_LOG}" +MATRIX_STATUS="${PIPESTATUS[0]}" +set -e +if [[ "${MATRIX_STATUS}" -ne 0 ]]; then + echo "sandbox-acceptance: the saved live matrix did not validate." >&2 + exit 1 +fi +if ! grep -qE 'test result: ok\. 1 passed' "${MATRIX_LOG}"; then + echo "sandbox-acceptance: the matrix test did not run (expected exactly 1 passed)." >&2 + echo "A filter that selects no test exits 0 and proves nothing." >&2 + exit 1 +fi + +echo +echo "ACCEPTED (offline): source suite green, saved matrix complete for ${SOURCE_HEAD}." +echo "NOT claimed: any live packet was measured by this command, or that the record is recent."