diff --git a/crates/maxplayer-core/examples/render_host_plan.rs b/crates/maxplayer-core/examples/render_host_plan.rs new file mode 100644 index 000000000..fc91666f4 --- /dev/null +++ b/crates/maxplayer-core/examples/render_host_plan.rs @@ -0,0 +1,42 @@ +//! Prints the **host-side** iptables plan [`HostPolicy`] would hand its applier, for gate evidence. +//! +//! Sibling of `render_net_plan`, and it exists for the same reason: a gate script must install the +//! rules the PRODUCT renders, never rules a script author transcribed. A transcription drifts the +//! moment the policy changes, and the gate then keeps passing against a firewall the product no +//! longer builds — the exact failure these gates exist to catch. +//! +//! It matters more here than for the namespace plan. The host plan lands in chains that are shared +//! with every other container on the daemon, so a hand-written approximation of it in a script is +//! not just drift, it is a rule keyed to the wrong source touching someone else's traffic. +//! +//! ```text +//! cargo run -p maxplayer-core --example render_host_plan -- 172.18.0.2 +//! cargo run -p maxplayer-core --example render_host_plan -- 172.18.0.2 --teardown +//! ``` +//! +//! Arguments: the job namespace's address, then optionally `--teardown` for the exact inverse plan. + +use maxplayer_core::sandbox_net::HostPolicy; +use maxplayer_core::sandbox_netns::{host_install_stdin, host_teardown_stdin}; + +fn main() { + let mut args = std::env::args().skip(1); + let Some(job_addr) = args.next() else { + eprintln!("usage: render_host_plan [--teardown]"); + std::process::exit(2); + }; + let teardown = match args.next().as_deref() { + None => false, + Some("--teardown") => true, + Some(other) => { + eprintln!("unknown argument {other:?} — expected --teardown or nothing"); + std::process::exit(2); + } + }; + + let policy = HostPolicy { job_addr }; + let (plan, count) = + if teardown { host_teardown_stdin(&policy) } else { host_install_stdin(&policy) }; + eprintln!("# {count} rules ({})", if teardown { "teardown" } else { "install" }); + print!("{plan}"); +} diff --git a/crates/maxplayer-core/examples/render_net_plan.rs b/crates/maxplayer-core/examples/render_net_plan.rs new file mode 100644 index 000000000..1df031f9f --- /dev/null +++ b/crates/maxplayer-core/examples/render_net_plan.rs @@ -0,0 +1,33 @@ +//! Prints the iptables plan [`NetPolicy`] would hand its sidecar, for gate evidence. +//! +//! This exists so the gVisor gate scripts install the rules the PRODUCT renders rather than rules a +//! script author transcribed by hand. A transcription drifts the moment the policy changes and the +//! gate keeps passing against a firewall the product no longer builds — which is the exact failure +//! the gates are supposed to catch. +//! +//! ```text +//! cargo run -p maxplayer-core --example render_net_plan -- 172.18.0.1 1.1.1.1 +//! ``` +//! +//! Arguments: the namespace gateway, then every resolver the job is allowed to reach on port 53. + +use maxplayer_core::sandbox_net::{NetPolicy, PortRange}; +use maxplayer_core::sandbox_netns::plan_stdin; + +fn main() { + let mut args = std::env::args().skip(1); + let Some(gateway) = args.next() else { + eprintln!("usage: render_net_plan [resolver ...]"); + std::process::exit(2); + }; + let dns_resolvers: Vec = args.collect(); + let policy = NetPolicy { + gateway, + proxy_ports: Some(PortRange::new(49200, 49299).expect("a valid fixed range")), + log_connections: true, + dns_resolvers, + }; + let (plan, count) = plan_stdin(&policy); + eprintln!("# {count} rules"); + print!("{plan}"); +} diff --git a/crates/maxplayer-core/src/home.rs b/crates/maxplayer-core/src/home.rs index a63551da9..924fb30d4 100644 --- a/crates/maxplayer-core/src/home.rs +++ b/crates/maxplayer-core/src/home.rs @@ -320,6 +320,20 @@ pub struct SandboxConfig { /// not name, or to carry a gateway base-URL. Unused under `launcher` mode. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub forward_env: Vec, + /// `docker` mode: the DNS resolver ADDRESSES a contained job's `/etc/resolv.conf` names. + /// Omitted ⇒ the host's own upstream resolvers are discovered and used; a host that names none + /// refuses to run jobs rather than picking a public resolver nobody chose. + /// + /// This exists because docker's embedded resolver at `127.0.0.11` is unreachable from a gVisor + /// sandbox — measured, with a runc control that succeeds on the identical image and network — + /// and because `docker run --dns` does not change what the daemon writes into a container on a + /// user-defined network. Addresses only, never hostnames: resolving the resolver is the problem + /// being fixed. A loopback address is refused for the same reason `127.0.0.11` fails. + /// + /// Each address named here is opened by the job's egress policy on port 53 and nothing else, + /// as a single host (`/32`, or `/128` for v6) — never a subnet. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dns_servers: Vec, /// `docker` mode: the container runtime to run the job under (`docker run --runtime `). /// Omitted ⇒ the daemon's default runtime (`runc`). The v1 sandbox posture sets this to `runsc` /// on Linux, where the default container shares the host kernel and gVisor is the primary @@ -329,24 +343,33 @@ pub struct SandboxConfig { /// Unused under `launcher` mode. #[serde(default, skip_serializing_if = "Option::is_none")] pub runtime: Option, - /// `docker` mode: the dedicated docker network a job's container joins (`docker run --network`). - /// Omitted ⇒ the daemon default (the shared `bridge` network). + /// `docker` mode: the **name prefix** for the docker network each job gets to itself. + /// Omitted ⇒ no containment, and the daemon default bridge. /// /// **Setting this is what turns #797 egress containment on for this seat.** A job launched under /// it runs in a network namespace whose rules were installed before the job process existed, and /// a job whose containment cannot be established FAILS rather than running exposed. There is no /// second step: no root command, and nothing to reinstall after a reboot. /// - /// Two reasons a *named* network rather than the default bridge: + /// **It names, it does not share.** This value is not one bridge every job sits on: each job gets + /// `-job-`, created before its holder and removed after it. The name still does the + /// job it always did — telling this seat's networks from a co-tenant daemon's — and no longer + /// puts two jobs on one wire. That changed because of a measurement: with jobs sharing a bridge, + /// a gVisor job REACHED a live listener inside another job's namespace and no host rule stopped + /// it, two containers on one bridge being switched rather than routed, and switched frames + /// entering no iptables chain at all on a host without `br_netfilter`. A network per job leaves + /// the job no on-link peer but its own gateway, which is also what makes every other destination + /// routed, and therefore visible to the host-side policy that binds a gVisor job at all. + /// + /// A *named* network rather than the default bridge, for a reason that predates all of that: + /// **the seller's own services are not on it.** The rules deny by destination, and the seat's LAN + /// and host addresses fall inside those denies, so a job must not share the bridge every other + /// container on the box uses. /// - /// * **The seller's own services are not on it.** The rules deny by destination, and the seat's - /// LAN and host addresses fall inside those denies. A dedicated network keeps a job's traffic - /// off the bridge every other container on the box shares. - /// * **DNS keeps working.** On a user-defined network a container resolves through docker's - /// embedded resolver at `127.0.0.11` inside its own netns, so no packet crosses to a host or LAN - /// resolver. On the shared default bridge docker copies the host's `resolv.conf` instead, and if - /// that names a LAN or host resolver then denying the LAN also denies DNS — which presents as - /// "the internet is broken" rather than as a firewall rule. + /// DNS does not come from the network. A contained job is handed a generated read-only + /// `/etc/resolv.conf` naming real upstream resolvers, because docker's embedded resolver at + /// `127.0.0.11` is a daemon-side socket reached by NAT inside the netns and a gVisor sandbox + /// terminates loopback in its own netstack, where it never answers. See [`crate::sandbox_dns`]. /// /// See [`crate::sandbox_net`] for what the rules are and [`crate::sandbox_netns`] for how they are /// put in force. diff --git a/crates/maxplayer-core/src/lib.rs b/crates/maxplayer-core/src/lib.rs index 3e8676b00..3fe452ac6 100644 --- a/crates/maxplayer-core/src/lib.rs +++ b/crates/maxplayer-core/src/lib.rs @@ -75,6 +75,10 @@ pub mod runtime_guard; /// `wallet`-only, so a default-features test run cannot execute a line of them — the policy is the /// part that decides what a stranger's job can reach, and it is compiled and tested on every build /// rather than only on the money-path one. +/// The resolver a contained job can reach, and the `resolv.conf` that names it. Ungated for the +/// same reason as `sandbox_net`: under gVisor a job that cannot resolve cannot deliver, so the +/// decision about where its lookups go is policy, and it is compiled and tested on every build. +pub mod sandbox_dns; pub mod sandbox_net; /// Putting `sandbox_net`'s policy in force: the holder container that owns the job's network /// namespace, and the sidecar that installs the rules into it before the job exists. Unconditional diff --git a/crates/maxplayer-core/src/sandbox_dns.rs b/crates/maxplayer-core/src/sandbox_dns.rs new file mode 100644 index 000000000..daacd57c8 --- /dev/null +++ b/crates/maxplayer-core/src/sandbox_dns.rs @@ -0,0 +1,420 @@ +//! The resolver a contained job can actually reach, and the `/etc/resolv.conf` that names it. +//! +//! ## Why this module exists at all +//! +//! Docker's embedded DNS resolver answers at `127.0.0.11` inside a container on any *user-defined* +//! network. It is not a process in the container: it is a socket the daemon binds inside that +//! network namespace, reached through NAT rules installed in the same namespace. +//! +//! Under gVisor (`--runtime runsc`) the sandbox runs its own network stack and terminates loopback +//! inside the sentry, so a packet a job sends to `127.0.0.11:53` never reaches those rules or that +//! socket. Measured on Ubuntu 24.04 with runsc release-20260817.0, image +//! `maxplayer-sandbox:v0.5.8`, one named bridge, identical container flags, only `--runtime` +//! differing: +//! +//! ```text +//! runsc: dns.lookup("relay.maxplayer.ai") -> EAI_AGAIN raw udp to 127.0.0.11:53 -> no answer +//! runc: dns.lookup("relay.maxplayer.ai") -> 34.225.223.145 +//! ``` +//! +//! `docker run --dns ` does **not** move it: on a user-defined network the daemon still writes +//! `nameserver 127.0.0.11` into the container and merely forwards upstream from its own side. So the +//! only lever that reaches the job is the file itself — the job is handed a `/etc/resolv.conf` +//! naming real upstream resolvers, read-only, and [`crate::sandbox_net::NetPolicy`] opens port 53 to +//! exactly those addresses and nothing wider. +//! +//! ## What is deliberately refused +//! +//! A loopback resolver (`127.0.0.0/8`, `::1`) is refused rather than written. On a systemd host +//! `/etc/resolv.conf` names the local stub `127.0.0.53`, which is unreachable from inside the +//! sandbox for the very same reason `127.0.0.11` is — writing it would reproduce the bug with a +//! different address and a more confusing error. +//! +//! When no resolver can be established, this module returns an error. It never falls back to a +//! public resolver of its own choosing: that would be a silent host-side decision about where a +//! stranger's job sends its lookups, and an operator who intended a specific resolver would never +//! learn it was ignored. + +use std::fmt; +use std::net::IpAddr; + +/// Where a resolver list came from, carried so an operator reading a doctor line or a job failure +/// knows which knob moved it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ResolverSource { + /// `[sandbox] dns_servers` named them explicitly. + Configured, + /// Discovered from the host's own resolver configuration. + HostResolvConf, + /// Discovered from `resolvectl status`, because the host's `/etc/resolv.conf` named only a local + /// stub. + HostResolvectl, +} + +impl fmt::Display for ResolverSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let text = match self { + Self::Configured => "[sandbox] dns_servers", + Self::HostResolvConf => "the host's /etc/resolv.conf", + Self::HostResolvectl => "resolvectl status (the host file named only a local stub)", + }; + f.write_str(text) + } +} + +/// A resolver set a contained job can use, and where it came from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Resolvers { + addresses: Vec, + source: ResolverSource, +} + +impl Resolvers { + /// The addresses, in the order they will be written and opened. + pub fn addresses(&self) -> &[String] { + &self.addresses + } + + /// Where they came from. + pub fn source(&self) -> ResolverSource { + self.source + } + + /// The `/etc/resolv.conf` body a job receives. + /// + /// `options ndots:0` is deliberate: without it a lookup of a dotted public name is first tried + /// against every entry of a `search` list, and this file names no search domain at all, so the + /// option states what the absent list already implies rather than leaving it to resolver + /// defaults that differ between libc and musl images. + pub fn render_resolv_conf(&self) -> String { + // The header deliberately carries no resolver address of its own. Someone debugging a + // broken job greps this file for the address it is using, and a commented-out address would + // answer that question wrongly. + let mut body = String::from( + "# Written by maxplayer for a contained job. Docker's embedded resolver is unreachable\n\ + # from a gVisor sandbox, so this file names upstream resolvers directly and the job's\n\ + # egress policy opens port 53 to exactly these addresses.\n", + ); + for address in &self.addresses { + body.push_str("nameserver "); + body.push_str(address); + body.push('\n'); + } + body.push_str("options ndots:0\n"); + body + } +} + +/// Why no usable resolver could be established. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ResolverError { + /// An address in `[sandbox] dns_servers` is not an IP address. + NotAnAddress(String), + /// An address is a loopback address, which no sandbox can reach. + Loopback(String), + /// Nothing usable was configured and nothing usable was discovered. + NoneFound(String), +} + +impl fmt::Display for ResolverError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotAnAddress(value) => write!( + f, + "[sandbox] dns_servers: {value:?} is not an IP address — a resolver must be \ + addressed, not named, because resolving the resolver is the problem being fixed" + ), + Self::Loopback(value) => write!( + f, + "[sandbox] dns_servers: {value:?} is a loopback address, which is unreachable from \ + inside the job's sandbox — that is exactly why docker's own 127.0.0.11 fails under \ + gVisor; name the upstream resolver itself" + ), + Self::NoneFound(detail) => write!( + f, + "no usable DNS resolver for contained jobs: {detail}. Set `[sandbox] dns_servers` to \ + the resolver addresses this host's jobs should use — jobs are refused rather than \ + pointed at a resolver nobody chose" + ), + } + } +} + +/// Validate operator-named resolvers. Empty input ⇒ `Ok(None)`, meaning "nothing configured", not +/// "nothing usable". +pub fn from_config(configured: &[String]) -> Result, ResolverError> { + let named: Vec<&str> = configured + .iter() + .map(|value| value.trim()) + .filter(|value| !value.is_empty()) + .collect(); + if named.is_empty() { + return Ok(None); + } + let mut addresses = Vec::new(); + for value in named { + let parsed: IpAddr = value + .parse() + .map_err(|_| ResolverError::NotAnAddress(value.to_owned()))?; + if parsed.is_loopback() { + return Err(ResolverError::Loopback(value.to_owned())); + } + let canonical = parsed.to_string(); + if !addresses.contains(&canonical) { + addresses.push(canonical); + } + } + Ok(Some(Resolvers { + addresses, + source: ResolverSource::Configured, + })) +} + +/// The usable `nameserver` lines of a `resolv.conf` body: parsed, de-duplicated, and stripped of +/// loopback stubs. +/// +/// Returned separately from the stub count so a caller can tell "this host names no resolver" from +/// "this host names only a stub" — the second is the systemd case that has an answer, and reporting +/// it as the first would send an operator to fix DNS that is working. +pub fn parse_resolv_conf(body: &str) -> (Vec, usize) { + let mut addresses = Vec::new(); + let mut stubs = 0usize; + for line in body.lines() { + let line = line.split('#').next().unwrap_or("").trim(); + let Some(rest) = line.strip_prefix("nameserver") else { + continue; + }; + let Ok(parsed) = rest.trim().parse::() else { + continue; + }; + if parsed.is_loopback() { + stubs += 1; + continue; + } + let canonical = parsed.to_string(); + if !addresses.contains(&canonical) { + addresses.push(canonical); + } + } + (addresses, stubs) +} + +/// The upstream resolvers in `resolvectl status` output — the "DNS Servers:" entries, which is where +/// a systemd host keeps the addresses its `127.0.0.53` stub forwards to. +pub fn parse_resolvectl(stdout: &str) -> Vec { + let mut addresses = Vec::new(); + let mut in_block = false; + for line in stdout.lines() { + let trimmed = line.trim(); + if let Some(rest) = trimmed.strip_prefix("DNS Servers:") { + in_block = true; + push_addresses(rest, &mut addresses); + continue; + } + if in_block { + // Continuation lines are indented and carry nothing but addresses; anything with a + // colon-terminated label starts a new field and ends the block. + let is_continuation = line.starts_with(char::is_whitespace) + && !trimmed.is_empty() + && trimmed.split_whitespace().all(|token| token.parse::().is_ok()); + if is_continuation { + push_addresses(trimmed, &mut addresses); + continue; + } + in_block = false; + } + } + addresses +} + +fn push_addresses(text: &str, into: &mut Vec) { + for token in text.split_whitespace() { + let Ok(parsed) = token.parse::() else { + continue; + }; + if parsed.is_loopback() { + continue; + } + let canonical = parsed.to_string(); + if !into.contains(&canonical) { + into.push(canonical); + } + } +} + +/// Resolve the resolver set for this seat: configuration first, then the host's own resolvers, then +/// an error. Never a guessed public resolver. +/// +/// The two host readers are injected so every branch — including "the host names only a stub and +/// `resolvectl` is absent" — is testable on a machine that is none of those things. +pub fn resolve( + configured: &[String], + read_resolv_conf: impl FnOnce() -> Option, + read_resolvectl: impl FnOnce() -> Option, +) -> Result { + if let Some(resolvers) = from_config(configured)? { + return Ok(resolvers); + } + let host_body = read_resolv_conf(); + let (host_addresses, stubs) = host_body + .as_deref() + .map(parse_resolv_conf) + .unwrap_or_else(|| (Vec::new(), 0)); + if !host_addresses.is_empty() { + return Ok(Resolvers { + addresses: host_addresses, + source: ResolverSource::HostResolvConf, + }); + } + if stubs > 0 { + if let Some(stdout) = read_resolvectl() { + let upstreams = parse_resolvectl(&stdout); + if !upstreams.is_empty() { + return Ok(Resolvers { + addresses: upstreams, + source: ResolverSource::HostResolvectl, + }); + } + } + return Err(ResolverError::NoneFound( + "this host's /etc/resolv.conf names only a local stub (systemd-resolved), and \ + `resolvectl status` reported no upstream address" + .to_owned(), + )); + } + Err(ResolverError::NoneFound( + "this host's /etc/resolv.conf names no resolver at all".to_owned(), + )) +} + +/// Read the host's `/etc/resolv.conf`, if it can be read. +pub fn host_resolv_conf() -> Option { + std::fs::read_to_string("/etc/resolv.conf").ok() +} + +/// Run `resolvectl status` and return its stdout, if the command exists and succeeds. +pub fn host_resolvectl() -> Option { + let output = std::process::Command::new("resolvectl") + .arg("status") + .output() + .ok()?; + if !output.status.success() { + return None; + } + Some(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_configured_resolver_wins_and_is_never_read_from_the_host() { + let resolvers = resolve( + &["9.9.9.9".to_owned()], + || panic!("the host must not be consulted when an operator named a resolver"), + || panic!("resolvectl must not run when an operator named a resolver"), + ) + .expect("configured resolvers resolve"); + assert_eq!(resolvers.addresses(), ["9.9.9.9"]); + assert_eq!(resolvers.source(), ResolverSource::Configured); + } + + #[test] + fn a_loopback_resolver_is_refused_rather_than_written() { + // The whole bug is that a loopback resolver is unreachable from the sandbox. Accepting one + // here would reproduce it with a different address. + let error = from_config(&["127.0.0.53".to_owned()]).expect_err("loopback is refused"); + assert_eq!(error, ResolverError::Loopback("127.0.0.53".to_owned())); + assert!(error.to_string().contains("unreachable from inside the job's sandbox")); + } + + #[test] + fn a_named_resolver_is_refused_because_resolving_it_is_the_problem() { + let error = + from_config(&["dns.example.com".to_owned()]).expect_err("a hostname is refused"); + assert!(matches!(error, ResolverError::NotAnAddress(_))); + } + + #[test] + fn the_hosts_real_resolvers_are_used_when_it_has_any() { + let resolvers = resolve( + &[], + || Some("nameserver 10.0.0.2\nnameserver 10.0.0.3\n".to_owned()), + || panic!("resolvectl must not run when the host file already names real resolvers"), + ) + .expect("host resolvers resolve"); + assert_eq!(resolvers.addresses(), ["10.0.0.2", "10.0.0.3"]); + assert_eq!(resolvers.source(), ResolverSource::HostResolvConf); + } + + #[test] + fn a_systemd_stub_falls_through_to_the_upstreams_resolvectl_reports() { + // Exactly the shape of the host in the reported failure: /etc/resolv.conf names 127.0.0.53 + // and nothing else, so the answer lives in resolvectl. + let resolvers = resolve( + &[], + || Some("nameserver 127.0.0.53\noptions edns0\n".to_owned()), + || { + Some( + "Global\n Protocols: -LLMNR\n DNS Servers: 1.1.1.1 1.0.0.1\n\ + \n Link 2 (eth0)\n Current Scopes: DNS\n" + .to_owned(), + ) + }, + ) + .expect("upstreams resolve"); + assert_eq!(resolvers.addresses(), ["1.1.1.1", "1.0.0.1"]); + assert_eq!(resolvers.source(), ResolverSource::HostResolvectl); + } + + #[test] + fn a_stub_with_no_discoverable_upstream_fails_rather_than_guessing() { + // The refusal this whole module exists to make: no public resolver is invented here. + let error = resolve( + &[], + || Some("nameserver 127.0.0.53\n".to_owned()), + || None, + ) + .expect_err("no upstream is an error"); + let text = error.to_string(); + assert!(text.contains("local stub"), "{text}"); + assert!(text.contains("dns_servers"), "{text}"); + assert!(!text.contains("8.8.8.8"), "no resolver may be guessed: {text}"); + } + + #[test] + fn a_host_with_no_resolver_at_all_fails_with_its_own_reason() { + let error = resolve(&[], || Some(String::new()), || None).expect_err("nothing to use"); + assert!(error.to_string().contains("names no resolver at all")); + } + + #[test] + fn resolvectl_continuation_lines_are_read_and_labels_end_the_block() { + let addresses = parse_resolvectl( + " DNS Servers: 1.1.1.1\n 1.0.0.1\n DNS Domain: lan\n", + ); + assert_eq!(addresses, ["1.1.1.1", "1.0.0.1"]); + } + + #[test] + fn the_rendered_file_names_every_resolver_and_no_search_domain() { + let resolvers = from_config(&["1.1.1.1".to_owned(), "9.9.9.9".to_owned()]) + .expect("valid") + .expect("configured"); + let body = resolvers.render_resolv_conf(); + assert!(body.contains("nameserver 1.1.1.1\n"), "{body}"); + assert!(body.contains("nameserver 9.9.9.9\n"), "{body}"); + assert!(body.contains("options ndots:0"), "{body}"); + assert!(!body.contains("127.0.0.11"), "the unreachable resolver must not appear: {body}"); + assert!(!body.contains("search "), "a search domain would change lookup shape: {body}"); + } + + #[test] + fn duplicate_resolvers_collapse_so_the_policy_opens_one_pinhole_pair() { + let resolvers = from_config(&["1.1.1.1".to_owned(), "1.1.1.1".to_owned()]) + .expect("valid") + .expect("configured"); + assert_eq!(resolvers.addresses(), ["1.1.1.1"]); + } +} diff --git a/crates/maxplayer-core/src/sandbox_net.rs b/crates/maxplayer-core/src/sandbox_net.rs index 6800b1f58..057cc1b26 100644 --- a/crates/maxplayer-core/src/sandbox_net.rs +++ b/crates/maxplayer-core/src/sandbox_net.rs @@ -297,6 +297,17 @@ fn arg_value<'a, S: AsRef>(args: &'a [S], flag: &str) -> Option<&'a str> { .map(AsRef::as_ref) } +/// Which family an address literal belongs to. A colon is the only thing that distinguishes them +/// here, and it is sufficient: these are addresses an operator configured or the host printed, not +/// hostnames — a resolver named rather than addressed is refused before it reaches a policy. +fn resolver_family(address: &str) -> Family { + if address.contains(':') { + Family::V6 + } else { + Family::V4 + } +} + /// An address as iptables prints it: a bare host address gains an explicit prefix length. /// /// Measured, not assumed — `-d 172.17.0.1` reads back as `-d 172.17.0.1/32`. Comparing the two @@ -361,6 +372,22 @@ pub struct NetPolicy { /// Connection and DNS logging (#797 requirement 3). Worth having with or without an allowlist: /// it is how anyone notices a job probing the LAN. pub log_connections: bool, + /// The upstream resolvers the job's `/etc/resolv.conf` names, each opened on port 53 and nothing + /// else. Empty ⇒ no DNS pinhole at all, which is correct only for a seat whose jobs need no name + /// resolution. + /// + /// **Why this field exists at all.** Docker's embedded resolver at `127.0.0.11` is a daemon-side + /// socket reached through NAT rules inside the container's namespace. Under gVisor the sandbox + /// terminates loopback in its own network stack, so those packets never arrive and every lookup + /// fails `EAI_AGAIN` — measured, with a runc control that succeeds on the identical image and + /// network, and with a bare UDP datagram to `127.0.0.11:53` timing out. `docker run --dns` does + /// not help: on a user-defined network the daemon writes `nameserver 127.0.0.11` regardless. So a + /// gVisor job is handed real upstream resolvers, and those resolvers need to be reachable through + /// a policy that otherwise denies the private ranges wholesale. + /// + /// Each address is opened as a single host (`/32`, or `/128` for v6) on port 53 only. Never a + /// subnet: an operator whose resolver is a LAN address gets that one address, not their LAN. + pub dns_resolvers: Vec, } impl NetPolicy { @@ -427,6 +454,32 @@ impl NetPolicy { )); } + // The DNS pinholes, also BEFORE the range drops and for the same reason: a resolver on a + // private address is inside a denied range, and a job that cannot resolve cannot deliver. + // One rule per transport, because a truncated UDP answer is retried over TCP and a seat that + // opened only UDP fails on exactly the large answers (DNSSEC, long CNAME chains) that are + // hardest to attribute later. + for resolver in &self.dns_resolvers { + let family = resolver_family(resolver); + let destination = with_prefix_len(resolver, family); + for protocol in ["udp", "tcp"] { + rules.push(Rule::new( + family, + vec![ + "-p", + protocol, + "-d", + destination.as_str(), + "--dport", + "53", + "-j", + "ACCEPT", + ], + "the sandbox's own resolver — docker's embedded one is unreachable under gVisor", + )); + } + } + for denied in DENIED_DESTINATIONS { if self.log_connections { rules.push(Rule::new( @@ -549,7 +602,12 @@ impl NetPolicy { .enumerate() .filter(|(_, rule)| rule.target.as_deref() == Some("ACCEPT")) .collect(); - let wanted = usize::from(self.proxy_ports.is_some()); + // Two ACCEPTs per v4 resolver (udp and tcp), plus the proxy pinhole if this seat has one. + // Counted rather than assumed: an ACCEPT this policy did not ask for is an egress hole + // whatever its destination, and the count is what catches one that carries a plausible + // address. + let dns_accepts = self.dns_pinhole_count(Family::V4); + let wanted = usize::from(self.proxy_ports.is_some()) + dns_accepts; if accepts.len() != wanted { return Err(format!( "the live namespace has {} ACCEPT rules, expected {wanted} — an unexpected ACCEPT \ @@ -558,16 +616,52 @@ impl NetPolicy { )); } - if let Some(ports) = self.proxy_ports { - let (accept_at, pinhole) = accepts[0]; - let gateway = with_prefix_len(&self.gateway, Family::V4); - if pinhole.destination.as_deref() != Some(gateway.as_str()) { + // Every resolver this policy named must actually be open on 53, and every ACCEPT that is + // not the proxy pinhole must be one of those resolvers. The first half catches a job that + // cannot resolve; the second catches a hole wearing a resolver's clothes. + for resolver in self.dns_resolvers.iter().filter(|r| resolver_family(r) == Family::V4) { + let destination = with_prefix_len(resolver, Family::V4); + let open = accepts + .iter() + .filter(|(_, rule)| { + rule.destination.as_deref() == Some(destination.as_str()) + && rule.dport.as_deref() == Some("53") + }) + .count(); + if open != 2 { + return Err(format!( + "{destination} has {open} port-53 ACCEPTs in the live namespace, expected 2 \ + (udp and tcp) — the job cannot resolve names, so it cannot deliver" + )); + } + } + + // No ACCEPT may sit above the metadata DROP, resolver or not. + for (accept_at, rule) in &accepts { + if *accept_at < metadata_dropped_at { return Err(format!( - "the pinhole points at {:?}, not the measured proxy address {gateway} — the \ - job cannot reach its model, or something else can", - pinhole.destination + "an ACCEPT for {:?} is at index {accept_at}, above the metadata DROP at \ + {metadata_dropped_at} — an ACCEPT above that drop reopens {METADATA_ENDPOINT}", + rule.destination )); } + } + + if let Some(ports) = self.proxy_ports { + let gateway_destination = with_prefix_len(&self.gateway, Family::V4); + let pinhole = accepts + .iter() + .find(|(_, rule)| { + rule.destination.as_deref() == Some(gateway_destination.as_str()) + && rule.dport.as_deref() != Some("53") + }) + .copied(); + let Some((accept_at, pinhole)) = pinhole else { + return Err(format!( + "no ACCEPT points at the measured proxy address {gateway_destination} — the \ + job cannot reach its model" + )); + }; // iptables collapses a single-port range to a bare port, so both spellings of the // same range must be accepted; anything wider is a hole. Derived from `to_match` // rather than from `Display`, which spells a range `start-end` — a form iptables @@ -587,17 +681,147 @@ impl NetPolicy { ports.to_match() )); } - if accept_at < metadata_dropped_at { - return Err(format!( - "the pinhole ACCEPT is at index {accept_at}, above the metadata DROP at \ - {metadata_dropped_at} — an ACCEPT above that drop reopens {METADATA_ENDPOINT}" - )); - } + let _ = accept_at; } } Ok(()) } + + /// How many ACCEPT rules this policy's resolvers install for one family: two per resolver, one + /// per transport. + pub fn dns_pinhole_count(&self, family: Family) -> usize { + self.dns_resolvers + .iter() + .filter(|resolver| resolver_family(resolver) == family) + .count() + * 2 + } +} + +/// Docker's documented hook in the root namespace's FORWARD path. Docker jumps to it before its +/// own rules, and it survives docker rewriting the rest of the chain — which a bare `-I FORWARD` +/// does not. +pub const DOCKER_USER_CHAIN: &str = "DOCKER-USER"; + +/// The root namespace's INPUT chain: where packets addressed to the host itself land. +pub const INPUT_CHAIN: &str = "INPUT"; + +/// The containment a gVisor job cannot step around, installed on the **host** side of the veth. +/// +/// # Why this exists at all +/// +/// [`NetPolicy`] renders rules for the job's own namespace, and for a runc job that is the whole +/// story. For a **gVisor** job it is not. Measured in `docs/gvisor-dns-delivery` (aarch64, runsc +/// release-20260817.0): with the full 26-rule plan installed and read back in the namespace, a +/// runsc job REACHED a live listener inside `-d 172.16.0.0/12 -j DROP`, while a runc job in that +/// same namespace got `timeout`. gVisor terminates the network inside the sandbox and writes +/// frames to the veth itself, so the host kernel's OUTPUT chain in that namespace — which only +/// ever sees packets from host sockets — never sees the job's. +/// +/// So the netns plan stays (it is what binds a runc job, and it costs nothing as defence in depth) +/// and this is added beside it, where the host kernel handles the packet whatever produced it. +/// +/// # Why two chains and not one +/// +/// Also measured, same evidence directory, against live listeners: +/// +/// | destination | `DOCKER-USER` | `INPUT` | +/// | --- | --- | --- | +/// | the host's own LAN address | `REACHED` — useless | `timeout` — binds | +/// | `169.254.169.254` (routed via the gateway) | binds | — | +/// +/// Packets addressed to the host are delivered locally and never traverse FORWARD, so DOCKER-USER +/// cannot see them. Packets routed onward do traverse it. Neither chain covers the other, which is +/// why both are rendered. +/// +/// # What this deliberately does not try to cover +/// +/// A peer on the job's **own bridge** is reached by switching, not routing, and on a host without +/// `br_netfilter` those frames enter no iptables chain at all — measured `REACHED` with the +/// DOCKER-USER rule installed. No host rule fixes that. A **per-job network** does, by leaving the +/// job no on-link peer but its gateway; see `sandbox_netns::establish`. +/// +/// IPv6 is not rendered here. `ip6tables` has a `DOCKER-USER` chain only when the daemon has IPv6 +/// enabled, and a missing chain is an install failure that would fail every job launch on a v4-only +/// host. The netns plan still carries [`DENIED_DESTINATIONS_V6`]; host-side v6 containment is +/// UNMEASURED and named as such in the runlog rather than rendered on faith. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostPolicy { + /// The job namespace's own address. Every rule is keyed to it as `-s`, so the policy denies + /// this job and nothing else on the host. + pub job_addr: String, +} + +/// One host-side rule: the chain it belongs in, what it denies, and why. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostRule { + pub chain: &'static str, + pub destination: String, + pub why: &'static str, +} + +impl HostPolicy { + /// Every rule this policy installs, in install order. + pub fn rules(&self) -> Vec { + let mut rules = vec![HostRule { + chain: DOCKER_USER_CHAIN, + destination: METADATA_ENDPOINT.to_owned(), + why: "instance credentials, reached by route through the gateway", + }]; + for denied in DENIED_DESTINATIONS { + rules.push(HostRule { + chain: DOCKER_USER_CHAIN, + destination: (*denied).to_owned(), + why: "a private destination the job reaches by route", + }); + } + for denied in DENIED_DESTINATIONS { + rules.push(HostRule { + chain: INPUT_CHAIN, + destination: (*denied).to_owned(), + why: "the same range addressed to the host itself, which never enters FORWARD", + }); + } + rules + } + + /// The `iptables` argv that installs the policy. + /// + /// `-I` rather than `-A`: DOCKER-USER is a shared chain and docker appends its own rules to it, + /// so appending would put this policy behind whatever is already there. + pub fn install_argv(&self) -> Vec> { + self.rules().iter().map(|rule| self.argv("-I", rule)).collect() + } + + /// The argv that removes it, exactly inverting [`Self::install_argv`]. + /// + /// This is not optional housekeeping. These rules are keyed to one job's address in a chain + /// that outlives the job; without the teardown the chain grows by one ruleset per job until the + /// host is a linear scan, and a recycled address inherits a dead job's policy. + pub fn teardown_argv(&self) -> Vec> { + let mut argv: Vec> = + self.rules().iter().map(|rule| self.argv("-D", rule)).collect(); + argv.reverse(); + argv + } + + fn argv(&self, op: &str, rule: &HostRule) -> Vec { + [ + Family::V4.binary(), + op, + rule.chain, + "-s", + &format!("{}/32", self.job_addr), + "-d", + &rule.destination, + "-j", + "DROP", + ] + .into_iter() + .map(String::from) + .collect() + } } #[cfg(test)] @@ -609,6 +833,7 @@ mod tests { gateway: "172.31.0.1".to_owned(), proxy_ports: Some(PortRange::new(49200, 49299).unwrap()), log_connections: true, + dns_resolvers: Vec::new(), } } @@ -902,6 +1127,7 @@ mod tests { gateway: "172.17.0.1".to_owned(), proxy_ports: Some(PortRange::new(49200, 49299).unwrap()), log_connections: true, + dns_resolvers: Vec::new(), } } @@ -1026,6 +1252,7 @@ mod tests { gateway: "172.17.0.1".to_owned(), proxy_ports: None, log_connections: true, + dns_resolvers: Vec::new(), }; // Its own readback is the measured one minus the pinhole. let without_pinhole = MEASURED_V4.replace( @@ -1057,6 +1284,7 @@ mod tests { gateway: "172.17.0.1".to_owned(), proxy_ports: Some(PortRange::new(49200, 49200).unwrap()), log_connections: false, + dns_resolvers: Vec::new(), }; let bare = policy .rules() @@ -1093,6 +1321,7 @@ mod tests { gateway: "172.17.0.1".to_owned(), proxy_ports, log_connections, + dns_resolvers: vec!["1.1.1.1".to_owned()], }; for rule in policy.rules() { for arg in &rule.args { @@ -1123,4 +1352,205 @@ mod tests { argument that broke containment" ); } + + fn host_policy() -> HostPolicy { + HostPolicy { job_addr: "172.31.19.2".into() } + } + + /// The rule that makes this policy safe to install on a shared host. Every rule is keyed to one + /// job's address; a rule that lost its `-s` would deny the range to the WHOLE host, including + /// the seller's own traffic and every other job. + #[test] + fn every_host_rule_is_keyed_to_this_job_and_nothing_else() { + let policy = host_policy(); + for argv in policy.install_argv().iter().chain(policy.teardown_argv().iter()) { + let source = argv.windows(2).find(|pair| pair[0] == "-s").map(|pair| &pair[1]); + assert_eq!( + source.map(String::as_str), + Some("172.31.19.2/32"), + "a host-side rule without this job's source key denies the range host-wide: {argv:?}" + ); + } + } + + /// Two chains, because neither covers the other: DOCKER-USER never sees a packet addressed to + /// the host, and INPUT never sees one routed onward. Both results are measured. + #[test] + fn the_host_policy_covers_the_routed_path_and_the_host_itself() { + let rules = host_policy().rules(); + for denied in DENIED_DESTINATIONS { + for chain in [DOCKER_USER_CHAIN, INPUT_CHAIN] { + assert!( + rules + .iter() + .any(|rule| rule.chain == chain && rule.destination == *denied), + "{denied} has no DROP in {chain}" + ); + } + } + assert!( + rules + .iter() + .any(|rule| rule.chain == DOCKER_USER_CHAIN + && rule.destination == METADATA_ENDPOINT), + "the metadata endpoint has no host-side DROP on the routed path" + ); + } + + /// The metadata drop goes in FIRST, so no rule this policy adds can precede it. + #[test] + fn the_metadata_drop_is_the_first_rule_rendered() { + assert_eq!(host_policy().rules()[0].destination, METADATA_ENDPOINT); + } + + /// Install renders `-I`, never `-A`: DOCKER-USER is shared and docker appends to it, so an + /// appended policy sits behind whatever is already there. + #[test] + fn the_host_policy_inserts_rather_than_appends() { + for argv in host_policy().install_argv() { + assert_eq!(argv[0], "iptables"); + assert_eq!(argv[1], "-I", "appending puts this policy behind docker's own rules"); + } + } + + /// Teardown is the exact inverse of install, reversed. These rules outlive the job's container + /// in a chain nothing else cleans up: a teardown that misses one leaks a rule per job, and a + /// recycled address inherits a dead job's policy. + #[test] + fn teardown_is_the_exact_inverse_of_install_in_reverse_order() { + let policy = host_policy(); + let install = policy.install_argv(); + let teardown = policy.teardown_argv(); + assert_eq!(install.len(), teardown.len()); + for (installed, removed) in install.iter().rev().zip(teardown.iter()) { + let mut expected = installed.clone(); + expected[1] = "-D".into(); + assert_eq!(&expected, removed); + } + } + + /// The exact argv, once, so a silent change to the shape of these rules has to be deliberate. + #[test] + fn the_first_host_rule_renders_exactly() { + assert_eq!( + host_policy().install_argv()[0], + vec![ + "iptables", + "-I", + "DOCKER-USER", + "-s", + "172.31.19.2/32", + "-d", + "169.254.169.254/32", + "-j", + "DROP" + ] + ); + } + + /// The host policy reuses [`DENIED_DESTINATIONS`] rather than carrying its own copy. A second + /// list is a second thing to forget: the range added to one and not the other is reachable. + #[test] + fn the_host_policy_denies_every_range_the_namespace_plan_denies() { + let host = host_policy().rules(); + let covered: Vec<&str> = DENIED_DESTINATIONS + .iter() + .copied() + .filter(|denied| host.iter().any(|rule| rule.destination == *denied)) + .collect(); + assert_eq!( + covered.len(), + DENIED_DESTINATIONS.len(), + "the host policy and the namespace plan disagree about what is denied" + ); + } + + /// Teardown must remove exactly what install added, in reverse. + /// + /// Gate 5h proved this on a live host: a recycled address inherited zero stale rules. That + /// was one measurement on one machine. This is the invariant, checked on every build — if + /// the two plans ever drift apart, teardown leaks rules into a shared chain and the next job + /// to be handed this address inherits a dead job's firewall. + #[test] + fn the_host_teardown_exactly_inverts_the_install() { + let policy = HostPolicy { job_addr: "172.18.0.2".to_owned() }; + let install = policy.install_argv(); + let teardown = policy.teardown_argv(); + assert_eq!( + install.len(), + teardown.len(), + "install and teardown must be the same length or teardown leaves rules behind" + ); + for (i, up) in install.iter().enumerate() { + let down = &teardown[teardown.len() - 1 - i]; + assert_eq!(up[1], "-I", "install must insert"); + assert_eq!(down[1], "-D", "teardown must delete"); + assert_eq!( + up[2..], + down[2..], + "teardown rule {i} does not match the install rule it is meant to remove" + ); + } + } + + /// Every rule, both directions, must carry this job's `/32` source key. + /// + /// A host rule without `-s` is not this job's policy — it is a deny for the whole range on a + /// chain shared with every container on the daemon. + #[test] + fn every_host_rule_is_keyed_to_the_job_address() { + let policy = HostPolicy { job_addr: "172.18.0.2".to_owned() }; + for argv in policy.install_argv().iter().chain(policy.teardown_argv().iter()) { + let at = argv + .iter() + .position(|arg| arg == "-s") + .unwrap_or_else(|| panic!("a host rule with no source key: {argv:?}")); + assert_eq!( + argv[at + 1], + "172.18.0.2/32", + "a host rule keyed to something other than this job: {argv:?}" + ); + } + } + + /// The count `establish()` cross-checks must equal the number of rules actually rendered. + /// + /// The applier reports a number and the caller compares it against this one; if the rendered + /// count and the plan's line count could disagree, a truncated plan would pass the check. + #[test] + fn the_rendered_host_plan_counts_exactly_what_it_renders() { + let policy = HostPolicy { job_addr: "172.18.0.2".to_owned() }; + let (plan, count) = crate::sandbox_netns::host_install_stdin(&policy); + assert_eq!(count, policy.install_argv().len(), "the install count is not the rule count"); + assert_eq!( + plan.lines().filter(|line| !line.trim().is_empty()).count(), + count, + "the install plan has a different number of lines than it claims rules" + ); + let (teardown, teardown_count) = crate::sandbox_netns::host_teardown_stdin(&policy); + assert_eq!(teardown_count, count, "teardown claims a different rule count than install"); + assert_eq!( + teardown.lines().filter(|line| !line.trim().is_empty()).count(), + teardown_count, + "the teardown plan has a different number of lines than it claims rules" + ); + } + + /// Why `establish()` refuses an empty address rather than rendering with it. + /// + /// This test asserts the hazard, not the fix: with no address the source key renders as bare + /// `/32`, which is not a host. Such a rule does not scope the deny to this job, so the guard + /// in `establish()` is load-bearing and must not be relaxed into a warning. + #[test] + fn an_empty_job_address_renders_a_source_key_that_is_not_a_host() { + let policy = HostPolicy { job_addr: String::new() }; + let argv = policy.install_argv(); + let first = &argv[0]; + let at = first.iter().position(|arg| arg == "-s").expect("a source key"); + assert_eq!( + first[at + 1], + "/32", + "an empty address must render an obviously-invalid key, which establish() then refuses" + ); + } } diff --git a/crates/maxplayer-core/src/sandbox_netns.rs b/crates/maxplayer-core/src/sandbox_netns.rs index 2373c5e16..bf6e89052 100644 --- a/crates/maxplayer-core/src/sandbox_netns.rs +++ b/crates/maxplayer-core/src/sandbox_netns.rs @@ -131,7 +131,13 @@ impl Drop for NetnsHolder { /// cannot disagree. #[derive(Debug)] pub struct Containment { + /// Declared first so it is DROPPED first: docker refuses to remove a network that still has an + /// endpoint attached, so the holder must be gone before [`Self::network`] is. pub holder: NetnsHolder, + /// The host-side rules keyed to this job's address. Dropping them removes the rules. + pub host_rules: HostRules, + /// This job's own network, dropped after the holder that sits on it. + pub network: JobNetwork, pub proxy_host: String, } @@ -205,13 +211,24 @@ pub fn holder_argv( /// `--rm` is safe here specifically because the caller captures stdout and stderr before the container /// is removed; the evidence is in hand before the container is gone. pub fn sidecar_argv(holder: &NetnsHolder, image: &str) -> Vec { + sidecar_argv_for(holder.name(), image) +} + +/// [`sidecar_argv`] for a namespace addressed by NAME. +/// +/// Exists for the doctor's delivery-route preflight, which builds and tears down its own throwaway +/// namespace and so holds a name rather than a [`NetnsHolder`] guard. Taking the guard there would +/// hand out a `Drop` that destroys a namespace the caller did not create. The argv is the same one +/// the awarded-job path uses because it IS that argv — a preflight rendering its own would test a +/// sidecar production never runs. +pub fn sidecar_argv_for(holder: &str, image: &str) -> Vec { [ "docker", "run", "--rm", "--interactive", "--network", - &holder.network_mode(), + &NetnsHolder::network_mode_for(holder), "--cap-drop", "ALL", "--cap-add", @@ -279,6 +296,256 @@ pub fn plan_stdin(policy: &NetPolicy) -> (String, usize) { (out, plan.len()) } +/// The per-job network's name, under the operator's configured network name as a prefix. +/// +/// The configured `[sandbox] network` used to name one shared bridge every job sat on. It keeps +/// its naming role — that is how a seat's networks are told from another daemon's on a shared host +/// — and loses its sharing role. The job id is appended for the same reason [`holder_name`] uses +/// it: a leaked network can be attributed to the job that leaked it. +pub fn job_network_name(configured: &str, job_id: &str) -> String { + format!("{configured}-job-{job_id}") +} + +/// `docker network create` argv for one job's own network. +/// +/// # Why a job gets a network to itself +/// +/// Measured in `docs/gvisor-dns-delivery` (aarch64, runsc release-20260817.0). With every job on +/// one shared bridge, a gVisor job REACHED a live listener in another job's namespace, and no host +/// rule stopped it: two containers on one bridge are **switched**, not routed, and on a host +/// without `br_netfilter` those frames enter no iptables chain at all — `DOCKER-USER` with the +/// right source key still read `REACHED`. Moving the neighbour to its own network changed the +/// result to `timeout`. +/// +/// So the per-job network is not tidiness. It is what leaves the job no on-link peer but its own +/// gateway, which makes every other destination **routed** — and routed packets from a gVisor +/// sandbox do traverse the host's chains, where [`crate::sandbox_net::HostPolicy`] can bind them. +pub fn network_create_argv(name: &str) -> Vec { + ["docker", "network", "create", "--driver", "bridge", name] + .into_iter() + .map(String::from) + .collect() +} + +/// `docker network rm` argv. Only succeeds once no container is attached, which is why the holder +/// is dropped before the network is. +pub fn network_rm_argv(name: &str) -> Vec { + ["docker", "network", "rm", name].into_iter().map(String::from).collect() +} + +/// `docker inspect` argv that reads the holder's address on `network`. +/// +/// The host-side policy is keyed to this address, so it is read from docker rather than computed +/// from the subnet: a policy keyed to a guess denies some other container and leaves this job open. +pub fn holder_address_argv(holder_name: &str, network: &str) -> Vec { + [ + "docker", + "inspect", + "--format", + &format!("{{{{(index .NetworkSettings.Networks \"{network}\").IPAddress}}}}"), + holder_name, + ] + .into_iter() + .map(String::from) + .collect() +} + +/// `docker run` argv for the container that applies the **host-side** policy. +/// +/// `--network host` is the whole point and the whole risk: these rules must land in the ROOT +/// namespace's chains, because that is the only place a gVisor job's packets can be seen. It runs +/// the same applier image as the sidecar, reading the same ` ` plan on stdin, so +/// there is one applier in this design and not two. +/// +/// It is safe to hand this container the host's network only because nothing untrusted is ever in +/// it: it runs our own image, for milliseconds, on a plan rendered in Rust from +/// [`crate::sandbox_net::HostPolicy`], and it is gone before the job starts. The job itself never +/// comes near `--network host`. +pub fn host_rules_argv(image: &str) -> Vec { + [ + "docker", + "run", + "--rm", + "--interactive", + "--network", + "host", + "--cap-drop", + "ALL", + "--cap-add", + "NET_ADMIN", + "--security-opt", + "no-new-privileges", + image, + ] + .into_iter() + .map(String::from) + .collect() +} + +/// One host-side plan as the applier reads it, plus its rule count for the same truncation +/// cross-check [`plan_stdin`] exists for. +fn host_stdin(argv: &[Vec]) -> (String, usize) { + let mut out = String::new(); + for rule in argv { + out.push_str(&rule.join(" ")); + out.push('\n'); + } + (out, argv.len()) +} + +/// The host-side plan that installs [`crate::sandbox_net::HostPolicy`]. +pub fn host_install_stdin(policy: &crate::sandbox_net::HostPolicy) -> (String, usize) { + host_stdin(&policy.install_argv()) +} + +/// The host-side plan that removes it again. +pub fn host_teardown_stdin(policy: &crate::sandbox_net::HostPolicy) -> (String, usize) { + host_stdin(&policy.teardown_argv()) +} + +/// A created per-job network, and the guarantee that it goes away. +/// +/// Same bargain as [`NetnsHolder`]: constructed the moment the network exists, so every later `?` +/// removes it on the way out. Dropped AFTER the holder — docker refuses to remove a network that +/// still has an endpoint attached — which the field order of [`Containment`] is what enforces. +#[derive(Debug)] +pub struct JobNetwork { + name: String, +} + +impl JobNetwork { + /// Gated to the feature that contains the one caller, so a build without `acp` does not carry + /// a constructor nothing can reach. + #[cfg(feature = "acp")] + fn adopt(name: String) -> Self { + Self { name } + } + + pub fn name(&self) -> &str { + &self.name + } +} + +impl Drop for JobNetwork { + fn drop(&mut self) { + let outcome = std::process::Command::new("docker") + .args(["network", "rm", &self.name]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .output(); + match outcome { + Ok(done) if done.status.success() => {} + Ok(done) => eprintln!( + "sandbox: could not remove job network {}: {}", + self.name, + String::from_utf8_lossy(&done.stderr).trim() + ), + Err(error) => { + eprintln!("sandbox: could not run docker network rm for {}: {error}", self.name) + } + } + } +} + +/// Installed host-side rules, and the guarantee that they are removed. +/// +/// The most important guard of the three, because its resource is invisible. A leaked holder is a +/// container someone will notice; a leaked host rule is a line in a shared chain that outlives the +/// job silently. Without this, `DOCKER-USER` grows by one ruleset per job forever, and a recycled +/// address inherits a dead job's policy. +#[derive(Debug)] +pub struct HostRules { + policy: crate::sandbox_net::HostPolicy, + image: String, + /// Whether the install was **verified complete** — the applier exited 0 and its count matched + /// what was rendered. Set after that cross-check, never at construction. + /// + /// It decides how teardown is run, and gate 5i is why it exists. `apply-policy` aborts on the + /// first rule that fails, and the teardown plan is the exact inverse in reverse order. After a + /// PARTIAL install that inverse begins with a rule which was never created, so the applier + /// aborts on its first line and removes **nothing** — measured: 9 rules installed, 9 rules + /// still in `DOCKER-USER` afterwards. + /// + /// The applier's exit-3 contract tells the caller to destroy the holder, and for the namespace + /// plan that is a complete remedy because those rules die with the netns. These do not: they + /// are in the root netns, in chains shared with every container on the daemon. + complete: bool, +} + +impl HostRules { + /// Adopt rules that may be only partly installed. Always the first thing done with them. + fn adopt(policy: crate::sandbox_net::HostPolicy, image: String) -> Self { + Self { policy, image, complete: false } + } + + /// Record that every rendered rule is in the kernel, which licenses the one-shot teardown. + fn mark_complete(&mut self) { + self.complete = true; + } +} + +/// Feeds one plan to the host-rule applier and reports whether it applied cleanly. +fn run_host_plan(image: &str, plan: &str) -> Result<(), String> { + use std::io::Write; + let argv = host_rules_argv(image); + let (program, args) = argv.split_first().expect("a docker argv is never empty"); + let mut child = std::process::Command::new(program) + .args(args) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn() + .map_err(|error| error.to_string())?; + if let Some(stdin) = child.stdin.as_mut() { + let _ = stdin.write_all(plan.as_bytes()); + } + drop(child.stdin.take()); + match child.wait_with_output() { + Ok(done) if done.status.success() => Ok(()), + Ok(done) => Err(String::from_utf8_lossy(&done.stderr).trim().to_owned()), + Err(error) => Err(error.to_string()), + } +} + +impl Drop for HostRules { + /// Synchronous, like [`NetnsHolder::drop`] and for the same reason: a task spawned here is + /// discarded on runtime shutdown, which is the path an aborted job takes. + fn drop(&mut self) { + if self.complete { + // Every rule is present, so the inverse plan matches it rule for rule and one + // invocation is both correct and cheapest. + let (plan, _) = host_teardown_stdin(&self.policy); + if let Err(error) = run_host_plan(&self.image, &plan) { + eprintln!( + "sandbox: host-rule teardown for {} failed: {error} — \ + DOCKER-USER and INPUT still carry this job's rules", + self.policy.job_addr + ); + } + return; + } + + // The install did not complete, so an unknown prefix of the plan is in the kernel and the + // rest never existed. One invocation would abort on the first absent rule and strand the + // present ones. Each rule gets its own invocation: a failure then means only "that rule was + // not there", which on this path is the expected case and not an error. + let rules = self.policy.teardown_argv(); + let total = rules.len(); + let mut removed = 0usize; + for rule in &rules { + let (plan, _) = host_stdin(std::slice::from_ref(rule)); + if run_host_plan(&self.image, &plan).is_ok() { + removed += 1; + } + } + eprintln!( + "sandbox: host-rule install for {} did not complete; removed {removed} of {total} \ + rules one at a time", + self.policy.job_addr + ); + } +} + /// `docker run` argv that asks **docker** what `host-gateway` means on this platform, by resolving /// `alias` inside a throwaway container that is allowed to carry `--add-host`. /// @@ -566,6 +833,7 @@ pub async fn establish( gid: u32, proxy_ports: Option, log_connections: bool, + dns_resolvers: Vec, ) -> Result { // Measured BEFORE the holder exists, so a probe failure needs no cleanup. let (probe_stdout, _) = run_docker(host_gateway_probe_argv(sidecar_image, proxy_alias), None) @@ -575,18 +843,63 @@ pub async fn establish( format!("resolving {proxy_alias} produced no IPv4 address (got {probe_stdout:?})") })?; + // This job's own network, created before the holder that will sit on it. Adopted into a guard + // immediately, for the same reason the holder is: every `?` below must remove it. + let net_name = job_network_name(network, job_id); + run_docker(network_create_argv(&net_name), None) + .await + .map_err(|error| format!("could not create the job network {net_name} — {error}"))?; + let job_network = JobNetwork::adopt(net_name.clone()); + let name = holder_name(job_id); - run_docker(holder_argv(&name, network, holder_image, uid, gid, job_id, seat), None) + run_docker(holder_argv(&name, &net_name, holder_image, uid, gid, job_id, seat), None) .await .map_err(|error| format!("could not start the netns holder {name} — {error}"))?; // From here on the container exists, so every early return must tear it down. Adopting it into // the guard immediately is what makes that automatic rather than remembered. let holder = NetnsHolder::adopt(name); + // The host-side policy, keyed to the address docker actually gave the holder. Read, never + // computed: a policy keyed to a guessed address denies some other container and leaves this + // job wide open. + let (job_addr, _) = run_docker(holder_address_argv(holder.name(), &net_name), None) + .await + .map_err(|error| format!("could not read the job namespace's address — {error}"))?; + let job_addr = job_addr.trim().to_owned(); + if job_addr.is_empty() { + return Err(format!( + "docker reported no address for {} on {net_name}, so the host-side policy would have \ + no source key and would deny the range host-wide", + holder.name() + )); + } + let host_policy = crate::sandbox_net::HostPolicy { job_addr }; + let (host_plan, host_expected) = host_install_stdin(&host_policy); + let host_applied = run_docker(host_rules_argv(sidecar_image), Some(host_plan)).await; + // Adopted before the result is examined: a plan that failed part-way has already installed + // rules, and those rules must come out whichever way this returns. + let mut host_rules = HostRules::adopt(host_policy, sidecar_image.to_owned()); + let (host_applied, _) = host_applied + .map_err(|error| format!("host-side containment was not installed — {error}"))?; + let host_applied: usize = host_applied + .parse() + .map_err(|_| format!("the host-rule applier reported {host_applied:?}, not a number"))?; + if host_applied != host_expected { + return Err(format!( + "host-side containment is incomplete: {host_applied} of {host_expected} rules applied \ + (the plan was truncated in transit)" + )); + } + // Only now is the one-shot inverse teardown known to match what is in the kernel. Before this + // line every early return unwinds rule-by-rule instead, which is the only way a partial + // install comes back out (gate 5i). + host_rules.mark_complete(); + let policy = NetPolicy { gateway: proxy_host.clone(), proxy_ports, log_connections, + dns_resolvers, }; let (plan, expected) = plan_stdin(&policy); let (applied, _) = run_docker(sidecar_argv(&holder, sidecar_image), Some(plan)) @@ -622,7 +935,7 @@ pub async fn establish( })?; } - Ok(Containment { holder, proxy_host }) + Ok(Containment { holder, host_rules, network: job_network, proxy_host }) } #[cfg(test)] @@ -635,6 +948,7 @@ mod tests { gateway: "172.17.0.1".into(), proxy_ports: Some(PortRange::new(9000, 9002).expect("valid range")), log_connections: true, + dns_resolvers: Vec::new(), } } @@ -785,6 +1099,34 @@ mod tests { assert!(!holder_argv.iter().any(|a| a == "NET_ADMIN"), "{holder_argv:?}"); } + /// The containment plane runs on the daemon's own runtime, never the job's. + /// + /// Measured in the gVisor repro (aarch64, runsc release-20260817.0, evidence + /// `docs/gvisor-dns-delivery/evidence/gate2a-runsc-holder-FAIL-*.txt`): a runsc container + /// joining a **runsc** holder's namespace sees `lo` only — no eth0, no route, every lookup + /// `EAI_AGAIN` — because a gVisor sandbox's netstack lives inside that sandbox and a second one + /// cannot enter it. Joining a **runc** holder, the same runsc job gets the holder's interface + /// and address, so the host kernel's rules govern its traffic. `iptables-nft` also refuses to + /// initialise inside gVisor, so a sandboxed sidecar could not install the plan even if the + /// namespace were shared. + /// + /// So this is not a default anyone may "improve" by threading the seat's runtime through: doing + /// that returns a job with no network at all, and a policy nothing enforces. + #[test] + fn the_containment_plane_never_carries_the_jobs_runtime() { + let holder = holder_argv("h", "net", "img", 1000, 1000, "abc", &seat_b()); + assert!( + !holder.iter().any(|a| a == "--runtime"), + "the holder must run on the daemon runtime, or the job cannot join its namespace: \ + {holder:?}" + ); + let sidecar = sidecar_argv(&NetnsHolder::adopt("h".into()), "netfilter"); + assert!( + !sidecar.iter().any(|a| a == "--runtime"), + "the sidecar must run on the daemon runtime, or iptables cannot initialise: {sidecar:?}" + ); + } + #[test] fn the_sidecar_takes_the_plan_on_stdin_and_is_told_nothing_else() { let holder = NetnsHolder::adopt("h".into()); @@ -930,10 +1272,47 @@ mod tests { gateway: measured.clone(), proxy_ports: Some(PortRange::new(9000, 9000).expect("valid range")), log_connections: false, + dns_resolvers: Vec::new(), }; let (stdin, _) = plan_stdin(&policy); let accepts: Vec<&str> = stdin.lines().filter(|l| l.contains("ACCEPT")).collect(); assert_eq!(accepts.len(), 1, "exactly one pinhole: {accepts:?}"); assert!(accepts[0].contains(&measured), "the pinhole must name the measured host: {accepts:?}"); } + + /// Adoption must assume the install is partial; the one-shot teardown has to be earned. + /// + /// Gate 5i measured what happens when it is not: a 9-of-17 install torn down by the inverse + /// plan removed **nothing**, because the applier aborts on the first rule that was never + /// created. So `complete` starts false, and only the count cross-check sets it. + #[test] + fn adopted_host_rules_are_not_complete_until_the_count_check_passes() { + let policy = crate::sandbox_net::HostPolicy { job_addr: "172.18.0.2".to_owned() }; + let mut rules = HostRules::adopt(policy, "image:tag".to_owned()); + assert!(!rules.complete, "adoption must assume a partial install"); + rules.mark_complete(); + assert!(rules.complete, "the count cross-check is what licenses the one-shot teardown"); + // Drop shells out to docker; this test is about the flag, not the teardown. + std::mem::forget(rules); + } + + /// Each rule of the rule-by-rule teardown must stand alone as one valid delete. + /// + /// This is the path a partial install unwinds through, and the applier refuses an empty plan + /// (exit 4) and anything that is not iptables (exit 5), so every single-rule plan must be one + /// line, a delete, and still keyed to this job. + #[test] + fn the_per_rule_teardown_renders_one_valid_delete_per_rule() { + let policy = crate::sandbox_net::HostPolicy { job_addr: "172.18.0.2".to_owned() }; + let rules = policy.teardown_argv(); + assert!(!rules.is_empty(), "there is nothing to tear down"); + for rule in &rules { + let (plan, count) = host_stdin(std::slice::from_ref(rule)); + assert_eq!(count, 1, "a per-rule plan must carry exactly one rule"); + assert_eq!(plan.lines().count(), 1, "a per-rule plan must be one line: {plan:?}"); + let line = plan.lines().next().expect("one line"); + assert!(line.starts_with("iptables -D "), "must be a delete: {line}"); + assert!(line.contains("172.18.0.2/32"), "must stay keyed to this job: {line}"); + } + } } diff --git a/crates/maxplayer-core/src/seller_exec.rs b/crates/maxplayer-core/src/seller_exec.rs index 7d12136e2..3b1b96f95 100644 --- a/crates/maxplayer-core/src/seller_exec.rs +++ b/crates/maxplayer-core/src/seller_exec.rs @@ -261,6 +261,12 @@ pub struct DockerPolicy { /// same reason as `proxy_ports`: the containment path that reads them is the one that builds the /// launch, so both come from one config value rather than being written down twice. file_credentials: Vec, + /// Operator-named resolver addresses for contained jobs, resolved from + /// [`crate::home::SandboxConfig::dns_servers`]. Empty ⇒ discover the host's own upstreams at + /// launch. Carried on the policy for the same reason as `proxy_ports`: the argv that mounts the + /// job's `resolv.conf` and the policy that opens port 53 to those addresses must name the same + /// resolvers, or the job is handed a resolver its own firewall drops. + dns_servers: Vec, } /// The agent-auth environment carried from the daemon into the container. @@ -307,6 +313,13 @@ pub struct JobLaunch<'a> { /// rules live in the namespace this names, and they were installed before this job's process /// existed. A `Some` here is therefore a containment claim, not a networking preference. pub netns: Option<&'a str>, + /// A host file to bind-mount read-only at `/etc/resolv.conf`, when this job needs a resolver + /// docker will not give it. `None` ⇒ the container keeps whatever the daemon wrote. + /// + /// Present for gVisor jobs and measured, not assumed: docker's embedded resolver at + /// `127.0.0.11` never answers inside a runsc sandbox, and `--dns` does not change what the + /// daemon writes on a user-defined network, so the file is the only lever that reaches the job. + pub resolv_conf: Option<&'a Path>, } /// What the ACP driver spawns: the process `program` + `args`, and the `cwd` the ACP session runs @@ -490,6 +503,12 @@ impl SandboxPolicy { ))); } } + // Validated HERE, at config resolution, for the same reason as the port range: a + // resolver that is a hostname or a loopback stub cannot serve a sandboxed job, and + // discovering that at job time would fail every job with an error that names the + // symptom rather than the config key. + crate::sandbox_dns::from_config(&config.dns_servers) + .map_err(|error| ExecError::Config(error.to_string()))?; let mut policy = Self::docker(DockerPolicy { image, forward_env: config.forward_env.clone(), @@ -497,6 +516,7 @@ impl SandboxPolicy { network, proxy_ports, file_credentials: config.file_credentials.clone(), + dns_servers: config.dns_servers.clone(), }); policy.codex_chatgpt = config.codex_chatgpt.clone(); Ok(policy) @@ -555,6 +575,14 @@ impl SandboxPolicy { /// port. Read by the containment path so the proxy's bind and the firewall's pinhole name the /// same ports — two artifacts that must agree, derived from one config value rather than /// written down twice. + /// The operator-named resolver addresses, empty when none were configured. + pub fn dns_servers(&self) -> &[String] { + match &self.kind { + PolicyKind::Docker(docker) => &docker.dns_servers, + _ => &[], + } + } + pub fn proxy_ports(&self) -> Option { match &self.kind { PolicyKind::Docker(policy) => policy.proxy_ports, @@ -730,15 +758,31 @@ impl DockerPolicy { "-w".into(), CONTAINER_WORKDIR.into(), ]); + // The job's resolver, read-only, when containment wrote one. + // + // ⛔ Not a preference and not a convenience: under gVisor docker's embedded resolver at + // `127.0.0.11` never answers, so without this file every lookup inside the job fails + // `EAI_AGAIN` while the identical container under runc resolves fine (measured, with the raw + // UDP datagram to `127.0.0.11:53` timing out). `--dns` cannot substitute — on a user-defined + // network the daemon writes `nameserver 127.0.0.11` whatever it is told — so the file is the + // lever, and `:ro` keeps a stranger's job from rewriting where its own lookups go. + if let Some(resolv_conf) = job.resolv_conf { + argv.push("-v".into()); + argv.push(format!("{}:/etc/resolv.conf:ro", resolv_conf.display())); + } // Egress containment (#797): join the namespace a holder container already owns, where the // rendered policy is in force BEFORE this process exists — the rules are not applied to the // job, the job is started into them. `crate::sandbox_netns` establishes that; `None` here // means it was not established, and the job falls back to the configured network (or, unset, // to the daemon default — exactly the behaviour before any of this existed). // - // Name resolution survives the swap: a container joining a namespace still gets its own - // /etc/resolv.conf pointing at docker's embedded resolver on 127.0.0.11 (measured), which is - // why `sandbox_net` must never deny loopback. + // Name resolution does NOT survive the swap on its own. A container joining a namespace gets + // its own /etc/resolv.conf pointing at docker's embedded resolver on 127.0.0.11, and under + // gVisor that resolver is unreachable: the sandbox terminates loopback in its own network + // stack, so the packet never reaches the NAT rules or the daemon socket behind them + // (measured — runsc EAI_AGAIN, runc OK, identical image and network). That is what the + // `resolv_conf` mount above exists to fix, and `sandbox_net` still never denies loopback + // because a runc seat continues to rely on exactly that resolver. match job.netns { Some(holder) => { argv.push("--network".into()); @@ -901,6 +945,7 @@ pub fn probe_launch_argv( uid, gid, netns: None, + resolv_conf: None, }; let launch = policy.launch(probe_command, &job)?; let mut argv = Vec::with_capacity(launch.args.len() + 1); @@ -2193,8 +2238,33 @@ pub async fn run_agent_job( // Established only for a docker policy with a configured network. No network ⇒ no containment, // which is the behaviour a seat had before any of this existed; it is not silently claimed. let _containment; + // The resolver file this job is handed, when it gets one. Declared out here so the argv built + // further down can name it: the file is written by the containment path, and only that path + // opens port 53 to the addresses inside it. + let mut job_resolv_conf: Option = None; let holder = match (policy.docker_image(), policy.sandbox_network()) { (Some(image), Some(network)) => { + // Resolvers FIRST, before the namespace exists, so a seat with no usable resolver fails + // with that reason and leaves nothing to tear down. Docker's embedded resolver is not an + // option here — under gVisor it never answers — so a job that cannot be given a real + // resolver is refused rather than launched to fail EAI_AGAIN with no explanation. + let resolvers = crate::sandbox_dns::resolve( + policy.dns_servers(), + crate::sandbox_dns::host_resolv_conf, + crate::sandbox_dns::host_resolvectl, + ) + .map_err(|error| ExecError::Policy(format!("[sandbox] {error}")))?; + let resolv_path = workdir + .parent() + .unwrap_or(workdir) + .join(format!("resolv-{}.conf", job_id_of(workdir))); + std::fs::write(&resolv_path, resolvers.render_resolv_conf()).map_err(|error| { + ExecError::Policy(format!( + "[sandbox] could not write the job's resolver file {}: {error}", + resolv_path.display() + )) + })?; + job_resolv_conf = Some(resolv_path); let established = crate::sandbox_netns::establish( network, image, @@ -2208,6 +2278,7 @@ pub async fn run_agent_job( gid, policy.proxy_ports(), true, + resolvers.addresses().to_vec(), ) .await // Fail the job rather than run it uncontained. The whole point of moving containment into @@ -2281,6 +2352,10 @@ pub async fn run_agent_job( uid, gid, netns: holder.as_ref().map(|(name, _)| name.as_str()), + // Present exactly when containment was established, because that is the only path that + // wrote a resolver file and opened port 53 to the addresses in it. Handing a job this file + // without those pinholes would point it at a resolver its own firewall drops. + resolv_conf: job_resolv_conf.as_deref(), }; let launch = policy.launch(&effective_command, &job)?; // The ACP idle/response timeout IS the unified job timeout — never a hardcoded 300s that could @@ -2962,6 +3037,7 @@ mod tests { uid: 1000, gid: 1000, netns: None, + resolv_conf: None, } } @@ -3065,6 +3141,7 @@ mod tests { fn docker_policy_mounts_only_the_job_workdir() { let agent_command = argv(&["claude-agent-acp"]); let policy = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "maxplayer-sandbox:latest".into(), forward_env: Vec::new(), runtime: None, @@ -3130,6 +3207,7 @@ mod tests { fn docker_policy_for_probe() -> SandboxPolicy { SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "maxplayer-sandbox:probe".into(), forward_env: Vec::new(), runtime: None, @@ -3288,7 +3366,7 @@ mod tests { let job_launch = policy .launch( &job_command, - &JobLaunch { workdir, env: &[], uid, gid, netns: None }, + &JobLaunch { workdir, env: &[], uid, gid, netns: None, resolv_conf: None }, ) .expect("a job renders"); let job_argv: Vec = @@ -3331,6 +3409,7 @@ mod tests { container and has nothing to say without one", ); let policy = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image, forward_env: Vec::new(), runtime: None, @@ -4019,6 +4098,7 @@ mod tests { #[test] fn docker_policy_keeps_the_container_alive_for_its_own_diagnostics() { let policy = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "maxplayer-sandbox:latest".into(), forward_env: Vec::new(), runtime: None, @@ -4059,6 +4139,7 @@ mod tests { #[test] fn docker_policy_hardens_against_the_strangers_code_it_runs() { let policy = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "maxplayer-sandbox:latest".into(), forward_env: Vec::new(), runtime: None, @@ -4114,6 +4195,7 @@ mod tests { #[test] fn job_container_carries_a_deterministic_name_and_label() { let policy = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "maxplayer-sandbox:latest".into(), forward_env: Vec::new(), runtime: None, @@ -4421,6 +4503,7 @@ mod tests { fn docker_runtime_is_named_only_when_configured_and_precedes_the_image() { // Unset: no --runtime anywhere. let default_rt = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "maxplayer-sandbox:latest".into(), forward_env: Vec::new(), runtime: None, @@ -4439,6 +4522,7 @@ mod tests { // Set: --runtime runsc, before the image. let gvisor = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "maxplayer-sandbox:latest".into(), forward_env: Vec::new(), runtime: Some("runsc".into()), @@ -4475,6 +4559,7 @@ mod tests { #[test] fn a_configured_sandbox_network_reaches_the_argv_and_an_unset_one_emits_no_flag() { let unset = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "maxplayer-sandbox:latest".into(), forward_env: Vec::new(), runtime: None, @@ -4492,6 +4577,7 @@ mod tests { ); let joined = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "maxplayer-sandbox:latest".into(), forward_env: Vec::new(), runtime: None, @@ -4860,6 +4946,7 @@ mod tests { } }; let docker = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "maxplayer-sandbox:latest".into(), forward_env: Vec::new(), runtime: None, @@ -5057,6 +5144,7 @@ mod tests { fn the_uncontained_audit_counts_both_registries_not_just_the_table() { let cred = file_cred(); let policy = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "maxplayer-sandbox:latest".into(), forward_env: vec!["CURSOR_AUTH_TOKEN".into(), "MY_AGENT_TOKEN".into()], runtime: None, @@ -5194,6 +5282,7 @@ mod tests { _ => None, }; let policy = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "img".into(), forward_env: vec!["MY_AGENT_TOKEN".into(), "ANTHROPIC_API_KEY".into()], runtime: None, @@ -5216,6 +5305,7 @@ mod tests { #[test] fn forwarded_env_reaches_the_container_argv() { let policy = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "maxplayer-sandbox:latest".into(), forward_env: Vec::new(), runtime: None, @@ -5264,6 +5354,7 @@ mod tests { #[test] fn todays_forwarding_leaks_every_real_credential_into_the_container_view() { let policy = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "maxplayer-sandbox:latest".into(), forward_env: Vec::new(), runtime: None, @@ -5294,6 +5385,7 @@ mod tests { #[test] fn contained_launch_keeps_every_real_credential_out_of_the_container_view() { let policy = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "maxplayer-sandbox:latest".into(), forward_env: Vec::new(), runtime: None, @@ -5384,6 +5476,7 @@ mod tests { #[test] fn docker_launch_without_containment_opens_no_pinhole() { let policy = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "img".into(), forward_env: Vec::new(), runtime: None, @@ -5408,6 +5501,7 @@ mod tests { fn uncontained_forwarded_credentials_flags_only_unrecognized_operator_vars() { let docker = |forward_env: Vec| { SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image: "img".into(), forward_env, runtime: None, @@ -5520,6 +5614,7 @@ mod tests { let agent_command = argv(&["sh", "-c", &probe]); let policy = SandboxPolicy::docker(DockerPolicy { + dns_servers: Vec::new(), image, forward_env: Vec::new(), runtime: None, diff --git a/crates/maxplayer-core/tests/sandbox_netns_live.rs b/crates/maxplayer-core/tests/sandbox_netns_live.rs index a9402dade..6d774f71f 100644 --- a/crates/maxplayer-core/tests/sandbox_netns_live.rs +++ b/crates/maxplayer-core/tests/sandbox_netns_live.rs @@ -155,6 +155,7 @@ fn policy(gateway: &str) -> NetPolicy { gateway: gateway.to_owned(), proxy_ports: Some(PortRange::new(49200, 49299).expect("valid range")), log_connections: true, + dns_resolvers: Vec::new(), } } @@ -371,6 +372,7 @@ fn the_pinhole_opens_one_port_and_the_rest_of_that_range_stays_denied() { gateway: canary.denied_ip.clone(), proxy_ports: Some(PortRange::new(port, port).expect("valid range")), log_connections: true, + dns_resolvers: Vec::new(), }; let (plan, expected) = plan_stdin(&policy); let (ok, applied, err) = canary.fixture.apply(&plan); @@ -552,6 +554,7 @@ fn a_job_launched_through_the_policy_is_contained_and_an_uncontained_one_is_not( uid: 0, gid: 0, netns: None, + resolv_conf: None, }, ) .expect("the policy must build a launch"); @@ -576,6 +579,7 @@ fn a_job_launched_through_the_policy_is_contained_and_an_uncontained_one_is_not( uid: 0, gid: 0, netns: Some(&canary.fixture.holder), + resolv_conf: None, }, ) .expect("the policy must build a launch"); @@ -605,6 +609,7 @@ fn policy_for(gateway: &str) -> NetPolicy { // No pinhole: this test wants the denied address denied, not excepted. proxy_ports: Some(PortRange::new(port + 1, port + 1).expect("valid range")), log_connections: true, + dns_resolvers: Vec::new(), } } diff --git a/crates/maxplayer/src/doctor.rs b/crates/maxplayer/src/doctor.rs index a74652e3f..1342aaed7 100644 --- a/crates/maxplayer/src/doctor.rs +++ b/crates/maxplayer/src/doctor.rs @@ -836,6 +836,10 @@ mod checks { gateway: "172.17.0.1".into(), proxy_ports: policy.proxy_ports(), log_connections: true, + // Placeholder alongside the gateway above, and for the same reason: only the + // COUNT is read here. What resolvers a job actually gets is decided per launch + // and proved by the sandbox DNS/TLS preflight, not by this render. + dns_resolvers: Vec::new(), } .install_plan() .len(); @@ -875,6 +879,430 @@ mod checks { } } + const DELIVERY_ROUTE_CHECK: &str = "sandbox delivery route"; + + /// The directory the route preflight runs its probe in, under the seat's real job tree — same + /// argument [`crate::sandbox_probe`] makes for its own paths: a launcher is configured for where + /// jobs run, so a probe somewhere else measures a route no job takes. + const ROUTE_WORKDIR_NAME: &str = ".route-preflight"; + + /// What exercising the real job route produced. + /// + /// Deliberately four outcomes rather than a bool: "the name never resolved", "it resolved but + /// TLS did not complete", and "the route could not be built at all" send an operator to three + /// different places, and collapsing them is how a doctor becomes something people skip. + #[derive(Debug, Clone, PartialEq, Eq)] + pub(super) enum RouteProbe { + /// Resolved a name AND completed a certificate-VERIFIED TLS handshake, from inside the + /// namespace an awarded job gets. + Delivered { resolver: String, address: String, subject: String }, + /// The lookup failed inside the job. The gVisor case this check was built for. + NoDns(String), + /// The name resolved, but TLS did not complete or its chain was not verified. + NoTls(String), + /// The route could not be built or the probe never reported. NOT a pass: a route that cannot + /// be measured has not been shown to work. + Unbuildable(String), + /// The instrument is absent — no docker on PATH — so there is nothing to measure and no + /// finding to report. Distinct from [`RouteProbe::Unbuildable`], which means the route WAS + /// asked and did not answer; the launcher check owns the missing-docker verdict and this one + /// must not double-report it. + Unmeasurable(String), + } + + /// #? gVisor DNS delivery: does a job on THIS seat actually reach the network it is supposed to? + /// + /// Every other network check here asks the question from the HOST. That is precisely the hole: + /// the measured failure is a seat whose host resolves and fetches perfectly while every job it + /// runs dies on `EAI_AGAIN`, because docker's embedded resolver at `127.0.0.11` is unreachable + /// from inside a gVisor sandbox. A host-side probe reports that seat READY and it then fails + /// every job it wins. So this one runs the real thing: the seat's own image, in the namespace + /// [`maxplayer_core::sandbox_netns`] builds, under the seat's own runtime, uid, dropped + /// capabilities and `no-new-privileges`, with the resolver file a job is handed — and it must + /// both resolve a name and complete a VERIFIED TLS handshake. + /// + /// It BLOCKS. A `Fail` here keeps [`readiness_ok`] false, because advertising a seat whose jobs + /// cannot deliver is the exact outcome the gate exists to prevent. It is `transient`, so the + /// boot gate's bounded retry gives a daemon or a resolver a moment to come back before the seat + /// is refused — and if it is still broken after that, refused is correct. + /// + /// The advisory `check_sandbox_egress` above is untouched (petar, 2026-08-18): that one reports + /// whether the network EXISTS and never blocks. This one asks whether the route WORKS. + pub(super) fn check_sandbox_delivery_route( + sandbox: Option, + home_root: std::path::PathBuf, + ) -> Check { + check_sandbox_delivery_route_in(sandbox, discover_resolvers, |policy, resolvers| { + // Ordering rule shared with check_sandbox_image and the Engine floor: ask only once + // docker itself resolves, or the spawn ENOENTs and a missing daemon is misreported as a + // broken route. A missing docker is the launcher check's verdict, not this one's. It + // lives in the REAL probe rather than in the injectable core so the unit tests below + // measure the verdicts and not the machine they run on. + if !argv0_resolvable("docker") { + return RouteProbe::Unmeasurable( + "docker not resolvable; the job route was not measured (see sandbox launcher)" + .to_owned(), + ); + } + run_delivery_route(policy, resolvers, &home_root) + }) + } + + /// The resolvers a job would be handed, by the product's own selection order. + fn discover_resolvers(configured: &[String]) -> Result, String> { + maxplayer_core::sandbox_dns::resolve( + configured, + maxplayer_core::sandbox_dns::host_resolv_conf, + maxplayer_core::sandbox_dns::host_resolvectl, + ) + .map(|resolvers| resolvers.addresses().to_vec()) + .map_err(|error| error.to_string()) + } + + /// [`check_sandbox_delivery_route`] over injected resolver discovery and an injected route + /// probe, so every verdict is testable on a host with no docker daemon at all — including the + /// one that matters most: host connectivity fine, job route broken, result still `Fail`. + pub(super) fn check_sandbox_delivery_route_in( + sandbox: Option, + resolvers: impl Fn(&[String]) -> Result, String>, + probe: impl Fn(&SandboxPolicy, &[String]) -> RouteProbe, + ) -> Check { + let policy = match SandboxPolicy::from_config(sandbox.as_ref()) { + // Already FAILed by the launcher check; do not double-report. + Err(_) => return Check::pass(DELIVERY_ROUTE_CHECK, "no resolvable docker executor"), + Ok(policy) => policy, + }; + if policy.docker_image().is_none() { + return Check::pass( + DELIVERY_ROUTE_CHECK, + "not a docker executor; jobs use this host's own network", + ); + } + let resolvers = match resolvers(policy.dns_servers()) { + Ok(resolvers) => resolvers, + // NOT transient: no retry discovers a resolver a misconfigured box does not have. + Err(error) => { + return Check::fail( + DELIVERY_ROUTE_CHECK, + format!("no resolver can be given to a job: {error}"), + "set `[sandbox] dns_servers` to one or more resolver ADDRESSES the seat can \ + reach (for example `dns_servers = [\"1.1.1.1\"]`)", + ) + } + }; + match probe(&policy, &resolvers) { + RouteProbe::Delivered { resolver, address, subject } => Check::pass( + DELIVERY_ROUTE_CHECK, + format!( + "a job resolved via {resolver} to {address} and completed a verified TLS \ + handshake with {subject}" + ), + ), + RouteProbe::NoDns(detail) => Check::fail_transient( + DELIVERY_ROUTE_CHECK, + format!("a job in the sandbox could not resolve names: {detail}"), + "the host resolving is not enough — the JOB must. Check `[sandbox] dns_servers` \ + and that port 53 to those addresses survives the job's egress policy", + ), + RouteProbe::NoTls(detail) => Check::fail_transient( + DELIVERY_ROUTE_CHECK, + format!("a job resolved, but could not complete a verified TLS handshake: {detail}"), + "check the job's egress policy allows 443 to the public internet and that the \ + sandbox image carries current CA certificates", + ), + RouteProbe::Unbuildable(detail) => Check::fail_transient( + DELIVERY_ROUTE_CHECK, + format!("the job's network route could not be measured: {detail}"), + "a route that cannot be measured has not been shown to work; check the docker \ + daemon, `[sandbox] network`, and that the sandbox image can run the probe", + ), + RouteProbe::Unmeasurable(detail) => Check::pass(DELIVERY_ROUTE_CHECK, detail), + } + } + + /// The host the route probe resolves and shakes hands with: the relay, because that is where a + /// job's answer is delivered, so a route that cannot reach it cannot earn anything. Kept in step + /// with [`maxplayer_core::home::DEFAULT_RELAY_URL`] by the test below. + pub(super) const ROUTE_PROBE_HOST: &str = "relay.maxplayer.ai"; + + /// The markers the in-container payload prints. Parsed rather than trusted to an exit code, + /// because "the payload never ran" and "the payload ran and failed" must not be one outcome. + const ROUTE_DNS_OK: &str = "route-dns-ok"; + const ROUTE_DNS_FAIL: &str = "route-dns-fail"; + const ROUTE_TLS_OK: &str = "route-tls-ok"; + const ROUTE_TLS_FAIL: &str = "route-tls-fail"; + + /// Build the real namespace, run the probe inside it, tear it down. + /// + /// Every container here comes from the PRODUCT's argv builders — `holder_argv`, + /// `sidecar_argv_for`, `plan_stdin`, and `SandboxPolicy::launch` — so this measures the route an + /// awarded job takes. A preflight that rendered its own argv would be a test of itself. + fn run_delivery_route( + policy: &SandboxPolicy, + resolvers: &[String], + home_root: &std::path::Path, + ) -> RouteProbe { + let workdir = home_root.join("seller-jobs").join(ROUTE_WORKDIR_NAME); + if let Err(error) = std::fs::create_dir_all(&workdir) { + return RouteProbe::Unbuildable(format!( + "cannot create the probe workdir {} ({error})", + workdir.display() + )); + } + let Some((uid, gid)) = owner_uid_gid(&workdir) else { + return RouteProbe::Unbuildable( + "cannot read the probe workdir's owner, so the probe could not run as the uid an \ + awarded job would get" + .to_owned(), + ); + }; + let resolv_path = workdir.join("resolv.conf"); + // Re-validated rather than trusted: these addresses came from discovery, and the file a job + // reads must be built by the same code that builds a job's real one. + let rendered = match maxplayer_core::sandbox_dns::from_config(resolvers) { + Ok(Some(resolvers)) => resolvers.render_resolv_conf(), + Ok(None) => { + return RouteProbe::Unbuildable( + "resolver discovery returned no addresses at all".to_owned(), + ) + } + Err(error) => return RouteProbe::Unbuildable(format!("resolvers rejected: {error}")), + }; + if let Err(error) = std::fs::write(&resolv_path, rendered) { + return RouteProbe::Unbuildable(format!( + "cannot write the probe resolver file {} ({error})", + resolv_path.display() + )); + } + + // No `[sandbox] network` ⇒ no namespace is established for a job either, so the honest route + // to measure is the daemon default the seat actually uses — with no resolver file, because + // nothing opened port 53 for one. + let Some(network) = policy.sandbox_network() else { + return job_leg(policy, &workdir, uid, gid, None, None, "the daemon's default network"); + }; + + let gateway = match network_gateway(network) { + Ok(gateway) => gateway, + Err(error) => { + return RouteProbe::Unbuildable(format!( + "cannot read the gateway of `[sandbox] network` '{network}': {error}" + )) + } + }; + let image = policy.docker_image().unwrap_or_default().to_owned(); + let holder_name = format!("maxplayer-route-preflight-{}", std::process::id()); + // Torn down on EVERY exit below, including the early returns — a preflight that leaks a + // holder container leaves the seat holding a namespace nothing will ever reap. + let _ = docker_rm_f(&holder_name); + let holder = maxplayer_core::sandbox_netns::holder_argv( + &holder_name, + network, + &image, + uid, + gid, + "route-preflight", + "route-preflight", + ); + if let Err(error) = run_docker_argv(&holder, None) { + let _ = docker_rm_f(&holder_name); + return RouteProbe::Unbuildable(format!("the namespace holder would not start: {error}")); + } + + let net_policy = maxplayer_core::sandbox_net::NetPolicy { + gateway, + proxy_ports: None, + log_connections: false, + dns_resolvers: resolvers.to_vec(), + }; + let (plan, _rules) = maxplayer_core::sandbox_netns::plan_stdin(&net_policy); + let sidecar = maxplayer_core::sandbox_netns::sidecar_argv_for(&holder_name, &image); + if let Err(error) = run_docker_argv(&sidecar, Some(plan)) { + let _ = docker_rm_f(&holder_name); + return RouteProbe::Unbuildable(format!("the egress policy would not install: {error}")); + } + + let outcome = job_leg( + policy, + &workdir, + uid, + gid, + Some(holder_name.as_str()), + Some(resolv_path.as_path()), + "the job namespace", + ); + let _ = docker_rm_f(&holder_name); + outcome + } + + /// The job leg: the seat's own image, launched by [`SandboxPolicy::launch`] exactly as an + /// awarded job is, reporting the two legs that matter. + fn job_leg( + policy: &SandboxPolicy, + workdir: &std::path::Path, + uid: u32, + gid: u32, + netns: Option<&str>, + resolv_conf: Option<&std::path::Path>, + where_: &str, + ) -> RouteProbe { + let job = maxplayer_core::seller_exec::JobLaunch { + workdir, + env: &[], + uid, + gid, + netns, + resolv_conf, + }; + let launch = match policy.launch(&route_payload(), &job) { + Ok(launch) => launch, + Err(error) => { + return RouteProbe::Unbuildable(format!("cannot build the probe launch: {error}")) + } + }; + let mut argv = Vec::with_capacity(launch.args.len() + 1); + argv.push(launch.program); + argv.extend(launch.args); + let output = match run_docker_argv_capturing(&argv) { + Ok(output) => output, + Err(error) => { + return RouteProbe::Unbuildable(format!("the probe container did not run: {error}")) + } + }; + read_route_markers(&output, where_) + } + + /// Judge the payload's own words. Kept separate from the spawning so the verdicts are unit- + /// testable without a daemon, and so "neither marker appeared" stays a distinct outcome from + /// "the DNS marker said it failed". + pub(super) fn read_route_markers(output: &str, where_: &str) -> RouteProbe { + let marker = |name: &str| { + output.lines().find_map(|line| line.trim().strip_prefix(name).map(str::trim)) + }; + if let Some(detail) = marker(ROUTE_DNS_FAIL) { + return RouteProbe::NoDns(format!("{detail} (from inside {where_})")); + } + let Some(address) = marker(ROUTE_DNS_OK) else { + return RouteProbe::Unbuildable(format!( + "the probe in {where_} reported neither success nor failure; it likely never ran" + )); + }; + if let Some(detail) = marker(ROUTE_TLS_FAIL) { + return RouteProbe::NoTls(format!("{detail} (from inside {where_})")); + } + match marker(ROUTE_TLS_OK) { + Some(subject) => RouteProbe::Delivered { + resolver: address.split_whitespace().nth(1).unwrap_or("?").to_owned(), + address: address.split_whitespace().next().unwrap_or("?").to_owned(), + subject: subject.to_owned(), + }, + None => RouteProbe::NoTls(format!( + "the lookup succeeded in {where_} but the handshake reported nothing" + )), + } + } + + /// The payload, in the image's own node: resolve, then complete a TLS request whose certificate + /// chain is VERIFIED. + /// + /// `rejectUnauthorized` is left at its default and `socket.authorized` is REPORTED, so a pass + /// cannot be a handshake that skipped verification — which is the failure mode a preflight for + /// delivery would be worst at catching. + fn route_payload() -> Vec { + let script = format!( + "const dns=require('dns'),https=require('https');const h='{ROUTE_PROBE_HOST}';\ + dns.lookup(h,(e,a)=>{{if(e){{console.log('{ROUTE_DNS_FAIL} '+e.code);process.exit(0);}}\ + console.log('{ROUTE_DNS_OK} '+a);\ + const r=https.request({{host:h,port:443,path:'/',method:'HEAD',timeout:15000}},(res)=>{{\ + const c=res.socket.getPeerCertificate();\ + if(res.socket.authorized){{console.log('{ROUTE_TLS_OK} '+((c&&c.subject&&c.subject.CN)||h));}}\ + else{{console.log('{ROUTE_TLS_FAIL} certificate chain not verified');}}process.exit(0);}});\ + r.on('timeout',()=>{{console.log('{ROUTE_TLS_FAIL} timeout');process.exit(0);}});\ + r.on('error',(err)=>{{console.log('{ROUTE_TLS_FAIL} '+err.code);process.exit(0);}});r.end();}});" + ); + vec!["node".to_owned(), "-e".to_owned(), script] + } + + /// The uid/gid owning `path` — the uid an awarded job's container runs as, read from the + /// filesystem rather than through a `libc` dependency this crate does not otherwise carry. + fn owner_uid_gid(path: &std::path::Path) -> Option<(u32, u32)> { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + std::fs::metadata(path).ok().map(|m| (m.uid(), m.gid())) + } + #[cfg(not(unix))] + { + let _ = path; + None + } + } + + /// The gateway of a docker network, asked of the daemon rather than computed — the same trap + /// [`maxplayer_core::sandbox_netns`] documents: a computed gateway puts the pinhole on an + /// address nothing listens on while every rendering test stays green. + fn network_gateway(network: &str) -> Result { + let output = std::process::Command::new("docker") + .args([ + "network", + "inspect", + network, + "--format", + "{{(index .IPAM.Config 0).Gateway}}", + ]) + .output() + .map_err(|error| error.to_string())?; + if !output.status.success() { + return Err(String::from_utf8_lossy(&output.stderr).trim().to_owned()); + } + let gateway = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + if gateway.is_empty() { + return Err("the daemon reported no gateway for it".to_owned()); + } + Ok(gateway) + } + + fn docker_rm_f(name: &str) -> std::io::Result { + std::process::Command::new("docker").args(["rm", "-f", name]).output() + } + + /// Run a `docker ...` argv (element 0 is the program), optionally feeding it stdin. + fn run_docker_argv(argv: &[String], stdin: Option) -> Result { + let (program, args) = argv.split_first().ok_or("empty argv")?; + let mut command = std::process::Command::new(program); + command.args(args); + command.stdin(match stdin { + Some(_) => std::process::Stdio::piped(), + None => std::process::Stdio::null(), + }); + command.stdout(std::process::Stdio::piped()); + command.stderr(std::process::Stdio::piped()); + let mut child = command.spawn().map_err(|error| error.to_string())?; + if let Some(stdin) = stdin { + use std::io::Write as _; + let mut pipe = child.stdin.take().ok_or("no stdin pipe")?; + pipe.write_all(stdin.as_bytes()).map_err(|error| error.to_string())?; + } + let output = child.wait_with_output().map_err(|error| error.to_string())?; + if !output.status.success() { + return Err(String::from_utf8_lossy(&output.stderr).trim().to_owned()); + } + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) + } + + /// Like [`run_docker_argv`], but the payload's OUTPUT is the evidence, so a non-zero exit still + /// yields what it said. + fn run_docker_argv_capturing(argv: &[String]) -> Result { + let (program, args) = argv.split_first().ok_or("empty argv")?; + let output = std::process::Command::new(program) + .args(args) + .output() + .map_err(|error| error.to_string())?; + let mut combined = String::from_utf8_lossy(&output.stdout).into_owned(); + combined.push_str(&String::from_utf8_lossy(&output.stderr)); + Ok(combined) + } + const CREDENTIAL_CONTAINMENT_CHECK: &str = "sandbox credential containment"; /// The #647 credential proxy keeps out of a docker container every model-credential variable named @@ -1738,8 +2166,10 @@ fn build_checks( let sandbox_for_image = sandbox.clone(); let sandbox_for_engine = sandbox.clone(); let sandbox_for_egress = sandbox.clone(); + let sandbox_for_route = sandbox.clone(); // The probe runs in the seat's OWN home, because that is where a launcher's config points. let home_root = home.root.clone(); + let route_home_root = home.root.clone(); // Open-pool claiming is the exposure the containment gate is about: it is what makes this box // run code from a counterparty nobody chose. Off by default (#357), so an unconfigured seat is // targeted-only and stays advisory. @@ -1782,6 +2212,14 @@ fn build_checks( // one who can be told, so this WARNs. Advisory — never blocks boot, because turning a working // docker seat red on upgrade is a behaviour change, not a doctor's call. checks.push(Box::new(move || checks::check_sandbox_egress(sandbox_for_egress))); + // The route a JOB takes, measured from inside a job's own namespace. Unlike every other network + // check here it does not ask the host anything: a seat whose host resolves perfectly while its + // gVisor jobs die on EAI_AGAIN is exactly the state that produced this check, and a host-side + // probe calls that seat ready. BLOCKING, and transient so the boot gate's bounded retry gives a + // daemon a moment to come back before the seat is refused. + checks.push(Box::new(move || { + checks::check_sandbox_delivery_route(sandbox_for_route, route_home_root) + })); // #792 phase 3: under mode=docker, the image the seat runs jobs in must be present or pullable, // or the first awarded job stalls. On absence this prints the exact `docker pull` command. A // non-docker policy is a no-op Pass. Placed after the launcher (docker-resolves) check. @@ -2521,6 +2959,158 @@ mod tests { ); } + /// RED-PROVE for the gVisor delivery failure: a seat whose HOST resolves and fetches perfectly + /// while its jobs cannot must be refused, and refused with words that send the operator to the + /// job's route rather than to their own network. + /// + /// This is the whole point of the check. Every other network row here is answered by the host, + /// and the measured failure — docker's embedded resolver at `127.0.0.11` being unreachable from + /// inside a gVisor sandbox — is invisible from there: the seat looks healthy, advertises, wins a + /// job, and fails it. So the probe result is INJECTED here and the host's own connectivity is + /// never consulted: these assertions hold on a laptop with perfect internet. + #[test] + fn doctor_delivery_route_fails_when_the_job_route_is_broken_however_healthy_the_host_is() { + use checks::RouteProbe; + use maxplayer_core::home::{SandboxConfig, SandboxMode}; + let docker = || { + Some(SandboxConfig { + mode: SandboxMode::Docker, + image: Some("maxplayer-sandbox:latest".into()), + network: Some("sbx".into()), + ..Default::default() + }) + }; + let resolvers_ok = |_: &[String]| Ok(vec!["1.1.1.1".to_owned()]); + + // The gVisor case itself: resolvers were found, the namespace was built, and the JOB still + // could not resolve. Blocking, and retryable. + let no_dns = checks::check_sandbox_delivery_route_in(docker(), resolvers_ok, |_, _| { + RouteProbe::NoDns("EAI_AGAIN (from inside the job namespace)".to_owned()) + }); + assert_eq!(no_dns.status, Status::Fail, "{}", no_dns.render()); + assert!(no_dns.transient, "a resolver blip deserves the bounded retry: {}", no_dns.render()); + assert!( + !readiness_ok(&[no_dns.clone()]), + "a seat whose jobs cannot resolve must not be advertised as ready: {}", + no_dns.render() + ); + assert!( + no_dns.render().contains("JOB") && no_dns.render().contains("dns_servers"), + "the remedy must point at the job's route and the key that fixes it, not at the host: \ + {}", + no_dns.render() + ); + + // Resolved, but the handshake never completed or its chain was not verified. Also blocking: + // a job that cannot complete verified TLS cannot deliver an answer. + let no_tls = checks::check_sandbox_delivery_route_in(docker(), resolvers_ok, |_, _| { + RouteProbe::NoTls("certificate chain not verified".to_owned()) + }); + assert_eq!(no_tls.status, Status::Fail, "{}", no_tls.render()); + assert!(!readiness_ok(&[no_tls.clone()]), "{}", no_tls.render()); + + // Could not be measured at all. NOT a pass — "I could not ask" is not "it works", and this + // is the arm a future edit is most likely to soften into a Warn. + let unbuildable = checks::check_sandbox_delivery_route_in(docker(), resolvers_ok, |_, _| { + RouteProbe::Unbuildable("the namespace holder would not start".to_owned()) + }); + assert_eq!(unbuildable.status, Status::Fail, "{}", unbuildable.render()); + assert!(!readiness_ok(&[unbuildable])); + + // No resolver can be given to a job at all. Blocking and NOT transient: no retry discovers + // a resolver the box does not have, so burning the backoff budget only delays the same + // refusal. + let no_resolver = checks::check_sandbox_delivery_route_in( + docker(), + |_| Err("the host names only the systemd stub and resolvectl reported none".to_owned()), + |_, _| panic!("the probe must not run when no resolver can be handed to a job"), + ); + assert_eq!(no_resolver.status, Status::Fail, "{}", no_resolver.render()); + assert!( + !no_resolver.transient, + "retrying cannot conjure a resolver; refuse immediately: {}", + no_resolver.render() + ); + + // The healthy route passes, and says what it actually proved — a resolver, an address, and + // a VERIFIED peer — so a green row cannot be read as "docker looked fine". + let ok = checks::check_sandbox_delivery_route_in(docker(), resolvers_ok, |_, _| { + RouteProbe::Delivered { + resolver: "1.1.1.1".to_owned(), + address: "34.225.223.145".to_owned(), + subject: "relay.maxplayer.ai".to_owned(), + } + }); + assert_eq!(ok.status, Status::Pass, "{}", ok.render()); + assert!( + ok.detail.contains("verified") && ok.detail.contains("1.1.1.1"), + "a pass must name the resolver it used and that the handshake was verified: {}", + ok.detail + ); + assert!(readiness_ok(&[ok])); + + // A host executor has no container route to measure ⇒ never a spurious failure. + assert_eq!( + checks::check_sandbox_delivery_route_in(None, resolvers_ok, |_, _| panic!( + "a host executor has no job container to probe" + )) + .status, + Status::Pass, + ); + + // No docker on PATH is the launcher check's verdict, not this one's: reporting it here too + // would refuse a box twice for one fault and bury the row that names the real fix. + let unmeasurable = + checks::check_sandbox_delivery_route_in(docker(), resolvers_ok, |_, _| { + RouteProbe::Unmeasurable("docker not resolvable".to_owned()) + }); + assert_eq!(unmeasurable.status, Status::Pass, "{}", unmeasurable.render()); + } + + /// The payload's own words are judged, and silence is not consent: a container that printed + /// nothing (no node in the image, an entrypoint that swallowed the payload) must never read as a + /// working route. + #[test] + fn a_silent_route_probe_is_not_a_passing_route() { + use checks::RouteProbe; + assert!(matches!( + checks::read_route_markers("", "the job namespace"), + RouteProbe::Unbuildable(_) + )); + assert!(matches!( + checks::read_route_markers("sh: node: not found", "the job namespace"), + RouteProbe::Unbuildable(_) + )); + // Resolved, then nothing about the handshake: not a pass either. + assert!(matches!( + checks::read_route_markers("route-dns-ok 34.225.223.145", "the job namespace"), + RouteProbe::NoTls(_) + )); + assert!(matches!( + checks::read_route_markers("route-dns-fail EAI_AGAIN", "the job namespace"), + RouteProbe::NoDns(_) + )); + assert!(matches!( + checks::read_route_markers( + "route-dns-ok 34.225.223.145\nroute-tls-ok relay.maxplayer.ai", + "the job namespace" + ), + RouteProbe::Delivered { .. } + )); + } + + /// The host the probe shakes hands with is the relay the seat actually delivers to. Pinned so a + /// relay move cannot leave the preflight proving reachability to an address nothing uses. + #[test] + fn the_route_probe_targets_the_configured_relay_host() { + assert!( + maxplayer_core::home::DEFAULT_RELAY_URL.contains(checks::ROUTE_PROBE_HOST), + "probe host {} is not the relay in DEFAULT_RELAY_URL {}", + checks::ROUTE_PROBE_HOST, + maxplayer_core::home::DEFAULT_RELAY_URL + ); + } + // RED-PROVE (#792 phase 3): an absent docker sandbox image is flagged with the ACTIONABLE // `docker pull ` command, not a raw failure — the operator can act without reading source. // A present image passes; a pullable one warns and still prints the pre-pull command. @@ -2808,6 +3398,9 @@ mod tests { // and a file-sourced credential is a containment concern that would only add a second // reason for the check to move. file_credentials: Vec::new(), + // Same decision, same reason: empty means "discover the host's own resolvers", and the + // engine floor is measured from the daemon's version string, which no resolver touches. + dns_servers: Vec::new(), // Same decision and the same reason: a host ChatGPT session is a containment concern, // and reading one here would give the check a second reason to move. codex_chatgpt: None, diff --git a/crates/maxplayer/src/sandbox_probe.rs b/crates/maxplayer/src/sandbox_probe.rs index da7bc67aa..0faf9edd8 100644 --- a/crates/maxplayer/src/sandbox_probe.rs +++ b/crates/maxplayer/src/sandbox_probe.rs @@ -417,6 +417,9 @@ fn run_in_container(policy: &SandboxPolicy, canary: &Path, workdir: &Path) -> Co // The probe launches its payload with no containment established, so it must not claim one. // The behavioural egress canary that DOES run inside a contained namespace is separate work. netns: None, + // No containment, so no resolver file either: this probe asks about the filesystem, and a + // resolver it was not given pinholes for would only add a confusing failure mode. + resolv_conf: None, }; let launch = match policy.launch(&payload, &job) { Ok(launch) => launch, diff --git a/docs/gvisor-dns-delivery/DESIGN.md b/docs/gvisor-dns-delivery/DESIGN.md new file mode 100644 index 000000000..4189935e0 --- /dev/null +++ b/docs/gvisor-dns-delivery/DESIGN.md @@ -0,0 +1,94 @@ +# Fix design — gVisor named-network DNS, and a delivery preflight that can fail + +Status: **drafted, not yet executed as code.** The measurements it rests on are +real (`RUNLOG.md`, `evidence/gate1-runsc-vs-runc-20260910T0046Z.txt`); the code +below is not written until a disposable Linux host is ruled, because an +unexecuted fix is not a fix. + +## What has to change, and why each piece exists + +### 1. The job sandbox gets a resolver it can actually reach + +Measured: under runsc, `127.0.0.11:53` answers nothing at all, and `--dns` does +not move it — docker writes `nameserver 127.0.0.11` on any user-defined network +regardless. So the only lever that works from outside the daemon is the file +itself: mount the job's `/etc/resolv.conf` read-only, naming real upstream +resolvers. + +Lands in `seller_exec.rs::run_argv` (the same argv that already carries +`--runtime`, `--cap-drop ALL`, `--user`, `--security-opt no-new-privileges`) as +one more `-v :/etc/resolv.conf:ro`. The generated file is per-seat, +not per-job — it holds no job data — and is written under the seller home with +mode 0444. + +Resolver selection, in order, with **no silent fallback**: + +1. explicit config (`[sandbox] dns_servers`), if set — an operator on a VPS with + a mandated resolver needs this and it is the only way to express it; +2. otherwise the host's real upstream resolvers, discovered by reading + `/etc/resolv.conf` and, when that names only a local stub (`127.0.0.53`, + systemd-resolved — exactly what Bob's VPS shows), `resolvectl status` for the + actual upstreams; +3. otherwise **fail loudly at boot**. Guessing `8.8.8.8` here would be a silent + host-side fallback and is precisely what the brief forbids. + +A stub address (`127.0.0.0/8`) is never written into the sandbox file: it is +unreachable from inside the sandbox by construction, and writing it would +reproduce the bug with a different address. + +### 2. The egress policy opens port 53 to exactly those resolvers + +`NetPolicy` currently carries `gateway`, `proxy_ports`, `log_connections`, and +denies the private ranges wholesale (`DENIED_DESTINATIONS`) while never denying +loopback. DNS to an upstream resolver is new traffic that the deny ranges may +shadow, so the policy grows one field: the resolver addresses. + +Rules added in `NetPolicy::rules()`, before the range denies for the same reason +the proxy pinhole is: + +- `-p udp -d /32 --dport 53 -j ACCEPT` and the tcp counterpart, one + pair per resolver, **/32 (or /128) only** — never a subnet, never "port 53 + anywhere in the private range". If an operator's resolver is itself a private + address, this opens that single host and nothing else, and the check in §3 + proves what it opened. +- `verify_readback` learns the same rules, so a namespace missing them is + reported rather than assumed. + +That keeps gate 5 intact: every other private, loopback, link-local and metadata +destination stays denied, and a DNS name resolving to a denied address still +dies at the deny rules because resolution and reachability are separate rules. + +### 3. Doctor/readiness runs the real route, and a failure blocks ready + +Today `check_sandbox_egress` answers "can a namespace be built" (docker network +exists) and is deliberately `Warn`-only (petar, 2026-08-18: automate first, then +require). That check is not touched — it answers a different question, and its +advisory status was a ruling, not an oversight. + +The new check is a different thing: **it launches the actual contained sandbox +and makes it resolve and complete a certificate-validated TLS handshake to the +delivery host**, under the production runtime, user, cap-drop and namespace. +Host connectivity is never consulted, so a host that can reach the internet +while the sandbox cannot produces a FAIL, which is the whole point. + +- Status `Fail`, so `readiness_ok` (any `Fail` ⇒ not ready) refuses the seat. +- `transient: true`, so a genuine network blip is retried on the existing + bounded schedule (5 attempts, 20s/40s/60s/80s) and a still-broken route is + then refused. Transient-retry is not transient-forgiveness. +- The failure message names the resolver it used, the runtime, and the exact + docker command to reproduce, because "DNS failed" sends nobody anywhere. + +### 4. Delivery is proven from inside the sandbox, not beside it + +Gate 4 requires a real container-side git push whose remote hash matches. The +preflight in §3 proves DNS+TLS; the delivery gate proves the actual `git push` +path from the same contained sandbox to a disposable remote, with the remote's +hash read back afterwards. A host-side upload passing while the sandbox path is +broken is the exact false-ready this work exists to kill. + +## What is deliberately NOT done + +- No `--network=host`, no runsc network passthrough, no runc fallback: the + measured fix needs none of them. +- No broad private-range allowance; resolver pinholes are single addresses. +- No relaxation of `check_sandbox_egress`'s existing ruling in either direction. diff --git a/docs/gvisor-dns-delivery/RUNLOG.md b/docs/gvisor-dns-delivery/RUNLOG.md new file mode 100644 index 000000000..11d069724 --- /dev/null +++ b/docs/gvisor-dns-delivery/RUNLOG.md @@ -0,0 +1,814 @@ +# gVisor named-network DNS + container delivery — run log + +Lane: `w-gvisor-dns-delivery-r2` · ordering seat: maxie (requested by Bob) +Brief: `/Users/forge/forge/v2/maxie/runs/gvisor-dns-delivery-brief-20260909.md` +(3,851 B, sha256 `3b0a20feab6ae107e9ddc28cb70b30b1a73656c15fa95c72abc62edd25430812`, verified 2026-09-09) + +Six acceptance gates. Nothing below is claimed as passing unless the command +output is recorded here or in `evidence/`. + +## Environment reality (2026-09-09, recorded before any test) + +| Fact | Value | +| --- | --- | +| Forge host | macOS 26.4.1, arm64 (Apple silicon) | +| Reported failing host (Bob) | Ubuntu 24.04, kernel 6.8.0-124, x86_64, Docker 29.8.0, runsc release-20260831.0 | +| Shared container runtime on this host | colima `default`, aarch64, Docker 29.7.2 client / 29.5.2 server — **off limits**, other lanes use it, no runsc install there | +| Disposable Linux for gates | lima VM `gvisor-repro`, Ubuntu 24.04 (noble release-20260705), aarch64, vz driver, 2 CPU / 4 GiB / 20 GiB | + +**Architecture divergence is named, not papered over.** The reproduction host +available to this lane is aarch64; the reported evidence is x86_64. Any gate +result carries the arch it was executed on. If a failure mode proves x86-only, +that is reported as a limitation, not as a pass. + +## Source map (read before edits) + +- `crates/maxplayer-core/src/sandbox_net.rs` (1126 lines) — renders the egress + policy. Already documents that loopback must never be denied because docker's + embedded DNS answers at `127.0.0.11` inside the namespace, and carries a + load-bearing test for it. +- `crates/maxplayer-core/src/sandbox_netns.rs` (939 lines) — puts the policy in + force via holder → sidecar → job, all sharing one network namespace. Header + asserts "name resolution is unaffected: a container joining the namespace + still gets its own `/etc/resolv.conf` pointing at docker's embedded resolver + on `127.0.0.11`". That assertion is exactly what the reported failure + contradicts under runsc, so it is the first thing to test rather than trust. +- `crates/maxplayer-core/tests/sandbox_netns_live.rs` (840 lines) — existing + live tests. + +## Timeline + +- 17:39 PDT — brief read, hash verified, worktree + branch created off + immutable commit `b45f8651dc9cab5b71c962eadd7b84840f5791061dc9cab5b71c962eadd7b84840f579106`. +- 17:41 PDT — disposable lima VM `gvisor-repro` creation started (Ubuntu 24.04 + cloud image, arm64). Shared colima VM deliberately untouched. +- 17:46 PDT — **failure reproduced** (`evidence/gate1-runsc-vs-runc-20260910T0046Z.txt`). +- 17:52 PDT — root cause isolated and a containment-preserving fix validated by + hand before any code was written. + +## Gate 1 result — reproduced (aarch64) + +Exact run recorded in `evidence/gate1-runsc-vs-runc-20260910T0046Z.txt`. +Identical image, identical named bridge, identical container security settings; +only `--runtime` differs: + +| Probe | Result | +| --- | --- | +| `dns.lookup(relay.maxplayer.ai)` under `--runtime runsc` | `ERR EAI_AGAIN`, exit 1 | +| same under `--runtime runc` (control) | `OK 34.225.223.145`, exit 0 | +| raw UDP datagram to `127.0.0.11:53` under runsc | **TIMEOUT — no answer at all** | + +Both containers were handed the *same* `/etc/resolv.conf` +(`nameserver 127.0.0.11`, docker's embedded resolver, `ExtServers: +[host(127.0.0.53)]`). Digests: image +`sha256:1c50e46a35dfe91fcdbbba11876bff312a95567bda98d6dcb7f675c884777412` +(arm64/linux), docker 29.1.3, runsc release-20260817.0, kernel 6.8.0-134, +network subnet 172.18.0.0/16. + +**Causal source.** The failure is not resolver policy, not the allowlist, and +not name-specific: a bare UDP packet to `127.0.0.11:53` gets no reply inside the +sandbox. Docker's embedded DNS on a *user-defined* network is a socket bound by +the daemon inside the container's network namespace on `127.0.0.11:`, +reached through NAT rules installed in that namespace. Under runsc the sandbox +runs its own network stack and terminates loopback inside the sentry, so those +packets never reach the namespace-side rules or the daemon's socket. Under runc +the container shares the host kernel's stack, so they do. That is the whole +delta, and it explains why the shared job namespace fails identically — the +holder's namespace has exactly the same embedded resolver. + +**`--dns` does not fix it** (measured): with `--dns 1.1.1.1` on a user-defined +network docker *still* writes `nameserver 127.0.0.11` and merely forwards +upstream from the daemon side, so the container still fails `EAI_AGAIN`. Any +fix that only sets docker DNS flags is theatre. + +**Validated fix direction** (measured, same runsc runtime, same named network, +same `--user 65534:65534 --cap-drop ALL --security-opt no-new-privileges`): +supply the sandbox its own `/etc/resolv.conf` naming real upstream resolvers, +read-only, instead of the unreachable embedded one. Result: `lookup: OK +34.225.223.145`, `tls: 200 cert-verified`. Containment is untouched — still the +named bridge, no host networking, no runc, no added capability. The egress +policy must then explicitly permit port 53 to exactly those resolver addresses +and nothing wider. + +## Gate status + +| Gate | State | +| --- | --- | +| 1 repro + runc control, digests, causal evidence | **done (aarch64), evidence committed** | +| 2 DNS + TLS from real shared job namespace, fresh + recreated | not started | +| 3 doctor/readiness on the real sandbox route + regression tests | not started | +| 4 real container-side Git delivery, remote hash match | not started | +| 5 private/metadata denial + concurrent public success | not started | +| 6 bounded gate script, executed test counts, PR | not started | + +## 2026-09-10 — gate 5 turned up a hole bigger than the one I was sent for + +Gate 5's cross-job leg failed: from a runsc job in a namespace carrying the full +26-rule plan, a container at `172.31.12.3:8080` — inside `-d 172.16.0.0/12 -j DROP` — +was **REACHED**. + +`gate5b-does-the-plan-bind-a-runsc-job.sh` isolates it. Same namespace, same plan +(read back from the netns: the DROP is rule 11/12 and it is there), same probe, +one variable — the job's runtime: + +| job runtime | result to a listener inside a DROPped range | +| --- | --- | +| runc | `timeout` — the DROP is enforced | +| runsc | `REACHED` — the DROP is not | + +**The per-job egress plan does not bind the runtime it was written for.** gVisor's +netstack terminates the network inside the sandbox and writes frames to the veth +itself; the host kernel's OUTPUT chain in that netns only sees packets from host +sockets, so it never sees the job's. The chain is installed, correct, verified by +readback — and irrelevant to a gVisor job. + +This is not a regression from the DNS work; it predates this branch. The DNS +change opens port 53 to a `/32` in a chain that was already not constraining the +job. + +### It also devalues part of my own gate-2 evidence +Gate 2 recorded `metadata: denied (ENETUNREACH)`. I read that as policy. It is +not evidence of policy: nothing listens on `169.254.169.254` in this VM, and +**absence is indistinguishable from enforcement** unless the denied destination +has a live listener. Every denial leg in gate 5 that "passed" against a dead +address proves nothing. Only the neighbour leg — a real listener inside a real +DROP range — was a valid test, and it failed. + +Rule for the remaining gates: a denial is only proven against a destination that +answers when it is allowed to. + +### Where containment has to live instead +Not in the netns OUTPUT chain. The candidate that gVisor cannot bypass is the +host side of the veth: FORWARD-chain rules in the root netns keyed to the job +namespace's source address, and/or a per-job network rather than one shared +`maxplayer-sbx` bridge (all jobs currently share it, which is why job A could see +job B at all). Both need measuring before either goes in. + +## 2026-09-10 — the shared job namespace is SINGLE-USE for gVisor + +`gate5c` tried to measure the two candidate enforcement sites and returned +`ENETUNREACH` for everything late in the run — including the **runc control**, +which had worked minutes earlier in that same namespace, and including DNS to +`1.1.1.1`, which no rule under test touched. A control that dies is not a +control, so gate5c's verdicts are **void**. + +`gate5d` settles why. One namespace, read with `os.networkInterfaces()`: + +| moment | interfaces | dns | +| --- | --- | --- | +| before any gVisor container | `lo=127.0.0.1 eth0=172.31.16.2` | `34.225.223.145` | +| during the runsc job | `lo=127.0.0.1 eth0=172.31.16.2` | `34.225.223.145` | +| after it exits, via runc | `lo=127.0.0.1` | `EAI_AGAIN` | +| after it exits, via a second runsc job | `lo=127.0.0.1` | `EAI_AGAIN` | + +**A gVisor container takes the namespace's addresses into its netstack and does +not give them back when it exits.** The namespace is usable exactly once. The +second container to enter it — whatever runtime — finds a namespace with nothing +but loopback. + +### What this voids, and what survives +- **VOID**: gate5c, both candidates. Measured against a dead namespace. +- **VOID**: gate 5's git leg (`Could not resolve host: github.com`). It was the + second runsc container in that namespace, not a DNS bug. +- **STANDS**: gate5b. Its runc control ran FIRST, while the namespace was + healthy, and was correctly dropped; the runsc probe was the first gVisor + container in that namespace. The finding holds: the plan binds runc and not runsc. +- **STANDS**: gate 4 and gate 2 — one runsc container per namespace in each. +- **STANDS**: the gate-2 retraction, for the separate dead-address reason. + +### Rule for every remaining measurement +One gVisor container per namespace, and a health check of the namespace +immediately before any leg whose result is meant to be evidence. Re-run gate5c +under that rule before either enforcement site is chosen. + +## 2026-09-10 — gate5c, re-run soundly: DOCKER-USER does not bind a bridged gVisor job; a per-job network does + +Rewritten under the one-gVisor-container-per-namespace rule, with a health check +printed beside every leg. Every leg below ran in a namespace verified HEALTHY +(address present, DNS resolving) immediately before the probe, against a LIVE +listener. + +| leg | runc | runsc | +| --- | --- | --- | +| live neighbour, same bridge, no host rule | `timeout` | **`REACHED`** | +| live neighbour, same bridge, `DOCKER-USER -s /32 -d 172.16.0.0/12 -j DROP` | `timeout` | **`REACHED`** | +| live neighbour, **other bridge** | — | `timeout` | + +With that DOCKER-USER rule in place the public route the job must keep is +untouched: `PUBLIC-PASS dns=34.225.223.145 tls=200 verified=true` and +`PUBLIC-PASS git 7fd1a60b…`. + +**(a) DOCKER-USER does not bind a gVisor job talking to a peer on its own bridge.** +The mechanism is in the environment line: `br_netfilter=absent`. Without +`bridge-nf-call-iptables`, frames switched between two containers on the same +bridge never enter iptables' FORWARD path at all, so DOCKER-USER cannot see them. +The runc leg's `timeout` in that row is its own netns OUTPUT plan doing the work, +not DOCKER-USER — which is exactly why a control that only ever shows "denied" +proves nothing about the site under test. + +**(b) A per-job network does bind it.** Cross-bridge traffic is routed rather than +switched, so it meets DOCKER-ISOLATION in the root netns and is dropped, gVisor +or not. + +### The shape of the fix this implies +A per-job network is not merely nicer isolation, it is what makes host-side +enforcement possible at all: once the job's only on-link peer is its gateway, +every other destination is ROUTED, and routed packets from a gVisor sandbox do +traverse the host's FORWARD path where a source-keyed policy can bind them. +Shared bridge + gVisor is the combination with no enforcement point. + +Still to measure before any of this is written into product code: with a per-job +network, does a source-keyed DOCKER-USER policy actually deny a runsc job a +ROUTED private destination that docker isolation does not already block, and +does it deny the metadata address? Host-directed traffic (the VM's own +`192.168.5.15`) lands in INPUT, not FORWARD, and needs its own answer. + +## 2026-09-10 — gate5e: the enforcement sites, named + +Same discipline: fresh holder+plan namespace per probe, one gVisor container in +each, health printed beside every leg, live listeners. + +**1. The host itself (`192.168.5.15:49252`, a real listener, reached by route)** + +| leg | result | +| --- | --- | +| runc, bare | `timeout` — its netns plan drops 192.168/16 | +| runsc, bare | **`REACHED`** | +| runsc, `DOCKER-USER -s /32 -d 192.168.0.0/16 -j DROP` | **`REACHED`** | +| runsc, `INPUT -s /32 -d 192.168.0.0/16 -j DROP` | `timeout` | + +A gVisor job can reach the host's own LAN address today, and DOCKER-USER cannot +stop it: host-directed packets are delivered locally, so they land in **INPUT** +and never traverse FORWARD. INPUT binds it. + +**2. The metadata address.** Nothing listens on it here, so only the difference +between runs is evidence — and there is one: bare `ECONNREFUSED`, with +`DOCKER-USER -s /32 -d 169.254.169.254/32 -j DROP` `timeout`. The rule +changes the outcome, so DOCKER-USER does bind the metadata address for a runsc +job. It is reached by route through the gateway, which is why FORWARD sees it +and the same chain was useless for the host and for a same-bridge peer. + +**3. Cross-job, each job on its own network:** runsc → job 2's live listener, +`timeout`. Per-job networks are the cross-job answer. + +**4. The public route survives all of it:** with both rules installed, +`PUBLIC-PASS dns=34.225.223.145 tls=200 verified=true` and +`PUBLIC-PASS git 7fd1a60b…`. + +### The fix, now fully specified by measurement +Three parts, none of which replaces the others: +1. **A per-job network**, not the one shared `maxplayer-sbx` bridge. It contains + job-to-job traffic and it converts everything else into routed traffic that + host chains can see at all. +2. **A host-side, source-keyed policy** applied daemon-side for the job + namespace's address: DOCKER-USER for routed destinations (metadata, private + ranges off-link) and INPUT for host-directed ones. +3. **The existing netns OUTPUT plan, kept** — it is what binds a runc job, and + it costs nothing to leave in place as defence in depth. + +## Gate 5f — the product's own host policy, measured against the hole it was built for + +`scripts/gate5f-product-host-rules-bind-runsc.sh`, evidence in +`evidence/gate5f-evidence.txt`. aarch64, kernel 6.8.0-134, docker 29.1.3, +runsc release-20260817.0, `br_netfilter=absent`, daemon `default-runtime=runc`. + +The rules under test are **rendered by the product**, not written by the script: +`cargo run -p maxplayer-core --example render_host_plan -- 172.31.21.10`, from +`HostPolicy`, 17 rules (`evidence/gate5f-host-install-rendered.txt`), plus the +exact inverse for teardown. The probe namespace's address is pinned with +`--ip` and the script **refuses to run** unless the rendered plan carries +`-s 172.31.21.10/32`; a plan keyed to the wrong address would deny some other +container while leaving this job open, and that mistake would otherwise be +invisible. They are installed by piping the plan into the **same applier image +the sidecar uses**, in a `--network host` container — the product's path, so +the image's ability to write the root namespace's chains is itself under test. + +### What changed, and what did not +| leg | before | after | what actually binds it | +|---|---|---|---| +| runsc → host `192.168.5.15:49253` (live) | REACHED | timeout | **the new INPUT rules** | +| runc → host `192.168.5.15:49253` (live) | timeout | timeout | the netns OUTPUT plan, already | +| runsc → own-network neighbour (live) | timeout | timeout | the per-job network + DOCKER-ISOLATION | +| runsc → same-bridge neighbour (live) | REACHED | REACHED | **nothing — and that is the point** | + +Only the first row is a delta produced by the host policy, and it is the exact +hole gate5e found: a gVisor job reaching the host itself, which DOCKER-USER +cannot stop because host-directed packets never traverse FORWARD. Gate5e +supplies the matching delta for DOCKER-USER (metadata: bare `ECONNREFUSED` vs +ruled `timeout`). Together the two chains of the rendered plan are each caught +working, on evidence, against a runsc job. + +The rows that did not change are recorded deliberately. The own-network row was +already denied before the policy was installed, so **this gate does not prove +the host rules deny cross-job traffic** — the per-job network does that, and +claiming otherwise would repeat the gate-2 error of reading an absence as +enforcement. The same-bridge row stays REACHED because switched frames enter no +chain on a host without `br_netfilter`. That is not a defect in the policy; it +is why the product gives every job its own network instead of trying to rule +its way out of a shared one, and it is measured here so that reason stays +evidenced rather than asserted. + +### The route survives +With the full policy installed: `PUBLIC-PASS dns=34.225.223.145 tls=200 +verified=true` and `PUBLIC-PASS git 7fd1a60b01f91b314f59955a4e4d4e80d8edf11d`. + +### Teardown leaves no trace +The rendered teardown plan removed all 17 rules; host readback for the job +address went 9/8 → 0/0, and both chains returned to depth 1, the depth they +had before the run. This is the failure mode `HostRules`' drop guard exists +to prevent: a leaked container gets noticed, a leaked rule in a shared chain +does not, and a recycled address would inherit a dead job's policy. + +## Gate 5 — denial holds, and three concurrent jobs still deliver + +`scripts/gate5-denial-and-concurrent-success.sh` (rewritten), +evidence `evidence/gate5-denial-and-concurrent-success-PASS-20260910T0311Z.txt`. +The first version's FAIL is kept beside it as +`…-20260910T0245Z.txt`. aarch64, kernel 6.8.0-134, docker 29.1.3, runsc +release-20260817.0, `br_netfilter=absent`, daemon `default-runtime=runc`. +Five namespaces, each on its own network, each with rules rendered by the +product and guarded against a stale plan by source key. **Verdict: PASS, +0 denial legs failing.** + +### Denial, before and after the host policy +| destination | before | after | attributable to | +|---|---|---|---| +| host `192.168.5.15:49254`, **live** | REACHED | timeout | **the host policy (INPUT)** | +| `denied-lan.maxplayer.test:49254` → same live listener, **by name** | REACHED | timeout | **the host policy (INPUT)** | +| neighbour job `172.31.34.10:8080`, **live** | timeout | timeout | the per-job network, not the policy | +| `169.254.169.254:80`, nothing listens | ECONNREFUSED | timeout | **the host policy (DOCKER-USER)** — a real difference | + +Two legs are new evidence and two are careful non-claims. The by-name leg is +coverage no earlier gate had: a job that reaches a denied address through a +NAME is denied exactly as one that dials the address, which matters because +every real exfiltration attempt is a hostname. It uses a mounted hosts file +rather than a third-party wildcard DNS service so the leg cannot pass or fail +for an unrelated reason. + +The metadata leg is evidence only because bare and ruled runs DIFFER +(`ECONNREFUSED` → `timeout`). With nothing listening there, an identical +result in both sections would prove nothing, and this is the same reasoning +that made me retract gate 2's "metadata denied" line rather than defend it. + +The neighbour leg was already denied before the policy went in, so the host +policy is **not** credited with it. The per-job network plus docker's own +isolation does that, and gate5c is where it was isolated. + +### IPv6 — measured, and left unproven on purpose +The job namespace has no global IPv6 address at all (`NO-V6` under both +runtimes), so there was nothing here to deny and nothing was proved. The +host-side plan deliberately renders no ip6tables rules, because `DOCKER-USER` +is not guaranteed to exist in the v6 table. **Host-side IPv6 denial for a +gVisor job is UNPROVEN and must not be claimed.** The netns plan does cover +v6, and its readback is verified per family at install time, but gate5b showed +the netns plan does not bind a runsc job — so on a host where the job DOES get +a global v6 address, this is an open hole and is listed as such in the +limitations. + +### Concurrent delivery +Three jobs at once, each with its own network, holder, netns plan and +host policy installed simultaneously (17 rules each, all reading back on the +host kernel). Every one delivered: + +``` +c1: OK dns=34.225.223.145 tls=200 verified=true OK git 7fd1a60b… +c2: OK dns=34.225.223.145 tls=200 verified=true OK git 7fd1a60b… +c3: OK dns=34.225.223.145 tls=200 verified=true OK git 7fd1a60b… +``` + +Denial and delivery are therefore not in tension: the policy that blocks the +host, the metadata address and a denied name by name is the same policy under +which three jobs resolved, verified a certificate and cloned a repository at +the same time. + +### No leaks +All four policies torn down by their rendered inverse; the host kernel went +from 17 rules each to 0, and `DOCKER-USER`/`INPUT` returned to depth 1, where +they started. + +## Gate 6 — reproducibility, and what this branch does NOT prove + +`scripts/run-all-gates.sh` runs the set bounded, writes each gate's output to +its own evidence file, and prints a verdict per gate. Proof run (aarch64, +gate1 + gate5f): `ALL GATES: PASS` in 61s — +`evidence/run-all-gates-proof-summary-20260910T0316Z.txt`. I checked the +per-gate logs rather than the summary line, because 61s looked too fast for a +suite that had taken minutes before; both are complete runs, the speed being +warm images. A full five-gate run through the runner was not executed in one +sitting; gates 1, 2, 4, 5 and 5f each have their own full-run evidence file +from a direct run, and the runner is proved on two of them. **Saying which is +the point of this section.** + +### Limitations — each one a thing a reader should not assume +1. **x86_64 is OUTSTANDING.** Every result here is aarch64, kernel 6.8.0-134, + docker 29.1.3, runsc **release-20260817.0**. The host is arm64 and runsc + release-20260831.0 ships no aarch64 artifact, so neither a newer runsc nor + a different architecture was tested. gVisor's netstack behaviour is the + whole subject of this branch, and it is exactly the kind of thing that can + differ per platform. +2. **`br_netfilter` is ABSENT on this host**, and that shaped the measurements. + It is why a same-bridge peer is reachable and unbindable here. Where it is + enabled, switched frames do enter the chains and that leg may read + differently. The per-job network makes the product correct either way — with + no on-link peer, the case does not arise — but the MEASUREMENT is + host-specific and should not be quoted as universal. +3. **Host-side IPv6 is not rendered, and v6 denial for a gVisor job is + UNPROVEN.** `HostPolicy` deliberately emits no ip6tables rules because + `DOCKER-USER` is not guaranteed to exist in the v6 table. The netns plan does + cover v6 and its readback is verified per family — but gate5b showed the + netns plan does not bind a runsc job. In this VM the job namespace has no + global v6 address, so nothing was denied and nothing was proved. **On a host + whose jobs do get one, this is an open hole.** +4. **The container-side git push uses an unauthenticated disposable remote, by + design.** A credentialed push would mean putting a secret inside a + stranger's sandbox. Gate 4 therefore proves the network path for a write, + not an authenticated push. +5. **The runtime boundary is baseline, not new — but it deserves review.** + Holder, sidecar and the host-rule applier carry no `--runtime` and inherit + the daemon default; only the JOB carries the configured runtime. That is + true of that baseline commit too: `sandbox_netns.rs` there has no `--runtime` in + `holder_argv`/`sidecar_argv`, and `seller_exec.rs` `run_argv` (lines + 681–687) emits it for the job alone. This branch adds the test that pins it + (`the_containment_plane_never_carries_the_jobs_runtime`). An operator who + sets `default-runtime=runsc` gets a runsc holder; measured, that fails + CLOSED (the job sees `lo` only). None of the three helpers executes any + seller- or task-controlled input: the holder is `--entrypoint sleep … infinity`, + the other two read a plan rendered in Rust. +6. **The `--network host` rule applier is the one privileged surface this + branch adds.** It must be `--network host` because the rules have to land in + the root namespace's chains, which is the only place a gVisor job's packets + can be seen. It runs our own image, on a Rust-rendered plan, for + milliseconds, and it is gone before the job starts; the job never touches + it. That is my judgement and it should not be only mine — **advisor review + is requested on this specifically.** + +### Retractions kept in the record +Gate 2's "metadata denied (ENETUNREACH)" line is **withdrawn**: nothing listens +there, so absence was read as enforcement. Gate5c's first run is **VOID** +(namespace reuse under gVisor) and its file is kept marked VOID. The first +gate 5 script FAILED and that evidence is kept beside the passing rewrite. + +## Gate 5g — the baseline vulnerability, reproduced on that baseline commit + +Maxie's ruling: *"Prior-baseline vulnerability claim needs a baseline +reproduction/source citation."* Correct demand — the branch had been asserting +"this predates the fix" from source reading alone. Here is the reproduction. +`scripts/gate5g-baseline-vulnerability-repro.sh`, evidence +`evidence/gate5g-baseline-vulnerability-CONFIRMED-20260910T0330Z.txt`. + +Built to be unfair to my own claim: +* **Baseline rules.** Rendered from immutable commit `b45f8651dc9cab5b71c962eadd7b84840f5791061dc9cab5b71c962eadd7b84840f579106` by a throwaway + example compiled against BASELINE's `NetPolicy` (three fields, no + `dns_resolvers`) — 24 rules, kept as + `evidence/gate5g-baseline-plan-from-origin-main.txt`. A renderer built + against the fix would reproduce the fix, not the bug. The script refuses to + run if the staged plan carries resolver pinholes, which only the fixed plan has. +* **Baseline arrangement.** ONE shared network, as a single `[sandbox] network` + produced, and **no host-side rules at all** — that baseline commit has no + `HostPolicy` (grep: 0 occurrences). The script installs none. +* **Positive controls.** `runc` runs the identical probe. Had the plan failed to + bind runc too, the finding would be "the plan was never installed". +* **Readback.** Every attacker namespace prints `applier=24/24` and a kernel + readback of the `172.16.0.0/12` rules, so a REACHED cannot mean "no rules". + +### Result on `b45f8651dc9cab5b71c962eadd7b84840f579106` +| leg | runc (control) | runsc | | +|---|---|---|---| +| cross-job → live victim `172.31.40.20:8080` | timeout | **REACHED** | containment failure | +| private egress → live host `192.168.5.15:49255` | timeout | **REACHED** | containment failure | + +**BASELINE-VULNERABLE: CONFIRMED, 2 findings.** The netns plan binds a runc job +and not a gVisor one, and baseline has no second line of defence. The branch +does not introduce this; it repairs it. + +### A confounded first run, kept +The first attempt read `timeout` on the cross-job leg and looked like baseline +containment working. It was nothing of the kind. The victim carries the +baseline plan too, and its listener was running under **runc** — so the plan +bound the *victim*, and its own OUTPUT rules dropped its replies to a +172.16/12 peer. The probe measured a victim that could not answer, not an +attacker that could not reach. + +Fixed by running the victim's listener under **runsc**, which is the real +arrangement on a baseline seat (every job is a gVisor job), and by printing a +`VICTIM-SERVING` liveness check from inside the victim's own namespace before +any conclusion is drawn from a timeout. The confounded run is kept as +`evidence/gate5g-baseline-repro-CONFOUNDED-runc-victim-20260910T0325Z.txt`. + +It is the same failure this branch has now hit three times in different +clothes — gate 2's metadata line, gate5c's voided run, and this — and the +lesson is identical each time: **a timeout is only evidence when something was +proved able to answer.** + +## Gate 5h — lifecycle: teardown, recycled address, recreation + +Maxie: *"Prove selected enforcement path handles runsc, lifecycle +cleanup/recreation and fail-closed setup/readiness."* Gate 5 proved the rules +**deny**. It never proved they **go away**, and that gap is dangerous in both +directions: a rule keyed to job A's address does not stop existing when A does, +and docker hands addresses back — so the next job to get `172.31.55.2` would +inherit a firewall written for a stranger (traffic denied that should be +allowed, or an ACCEPT pinhole that was A's proxy and is now someone else's open +door). A network that fails to delete wedges the next job with the same id. + +`scripts/gate5h-lifecycle-and-recycled-address.sh`, evidence +`evidence/gate5h-lifecycle-recycled-address-PASS-20260910T0332Z.txt`, plans +`evidence/gate5h-plans/` (17 install / 17 teardown, rendered by +`render_host_plan`, never transcribed). + +**GATE 5h: PASS — 0 failing checks** (aarch64, runsc release-20260817.0): + +| check | result | +|---|---| +| A establishes → host rules appear | 0 → **17** rules keyed to `172.31.55.2` | +| A contained while installed | runsc → live host `timeout` | +| A tears down → rules gone | **0** rules survive | +| A tears down → network gone | removed | +| B **recycles A's address** `172.31.55.2` | genuinely recycled | +| B inherits stale firewall? | **0** stale rules | +| B bare (control) → live host | **REACHED** | +| B after its own rules → live host | **ENETUNREACH** | +| same job id twice | both came up, no "already exists" wedge | +| chains returned to start depth | 2 → 2, nothing leaked | + +### What each leg is worth +The recycled-address leg is the strong one, and it is the reason the gate +exists: a **bare-vs-ruled difference measured against a live listener** +(`REACHED` → `ENETUNREACH`), on an address that a previous job had owned. It +rules out both "the listener was never reachable" and "the old rules were doing +the work". + +Leg 1 proves less and should be read that way: it measures job A only with its +rules installed, with no bare control, so on its own it is consistent with the +host simply being unreachable from that namespace. It is corroboration, not +proof; the bare control in leg 3 and gate5f's `REACHED → timeout` carry the +weight. + +Note the two denials read differently — `timeout` for A, `ENETUNREACH` for B. +Both are denials, and the difference is not yet explained; it is most likely +DROP versus an unreachable route at the moment of probe. Recorded as an +observation, not a claim. + +## Fail-closed setup — what the product does, and what now holds it + +Maxie: *"…and fail-closed setup/readiness."* Read back from +`sandbox_netns.rs::establish()`, the host-side path refuses the job at **four** +distinct points rather than warning and continuing: + +1. the holder's address cannot be read → `could not read the job namespace's address`; +2. the address comes back **empty** → refused, because a policy with no source + key `would deny the range host-wide`; +3. the host-rule applier fails → `host-side containment was not installed`; +4. the applier's count ≠ the rendered count → `host-side containment is + incomplete … the plan was truncated in transit`. + +`HostRules` is adopted **before** the applier's result is examined, so a plan +that failed part-way still has its rules removed on the way out. A namespace +readback (#797 R1) then asks the kernel directly, because everything above it +is the installer's own account of its work. + +### The gap that was there +All four guards were held by **reading the source**, not by tests. Gate 5h +measured the teardown once, on one host. Four rendering tests now lock the +invariants on every build (`sandbox_net.rs`): + +* `the_host_teardown_exactly_inverts_the_install` — same length, `-I`↔`-D`, + reverse order, every field otherwise identical. If these two plans drift, + teardown leaks rules into a shared chain and a recycled address inherits a + dead job's firewall. +* `every_host_rule_is_keyed_to_the_job_address` — both directions; a rule + without `-s` is a deny for the whole range on a chain shared with every + container on the daemon. +* `the_rendered_host_plan_counts_exactly_what_it_renders` — the count guard 4 + relies on; if rendered count and line count could disagree, a truncated plan + would pass the cross-check. +* `an_empty_job_address_renders_a_source_key_that_is_not_a_host` — asserts the + *hazard* (bare `/32`), documenting why guard 2 must never be relaxed. + +### Test counts, corrected and verified +Earlier notes said "60 sandbox + 55 doctor". The doctor tests are **not** in +`maxplayer-core` — that crate has a `doctor` module with zero `#[test]`. They +live in the `maxplayer` **binary** target. Verified this run: + +| suite | command | result | +|---|---|---| +| core lib (all) | `cargo test -p maxplayer-core --lib` | **387 passed, 0 failed** | +| sandbox filter | `cargo test -p maxplayer-core --lib sandbox` | **65 passed, 0 failed** | +| doctor | `cargo test -p maxplayer --bins doctor` | **55 passed, 0 failed** (73 filtered) | + +The four new tests were confirmed **by name** in the output, not inferred from +the total moving. + +## Gate 5i — fail-closed, and a defect this branch introduced + +`scripts/gate5i-fail-closed-live.sh`, evidence +`evidence/gate5i-fail-closed-FAIL-teardown-leak-20260910T0339Z.txt`. +**GATE 5i: FAIL, 2 failing checks.** Kept as a FAIL, unfixed in this commit, +because it found something real. + +### What passed +* **A — applier cannot start** (bad image tag): 0 rules installed. `establish()` + maps this to a hard error. +* **B — applier without `NET_ADMIN`**: `apply-policy: namespace is PARTIALLY + configured — destroy the holder, do not retry`, **0** rules installed. +* **C — truncated plan**: the code comment claimed a truncated plan "applies + cleanly and exits 0, so no exit code reveals it". **Confirmed, measured**: + exit `0`, applier reported `9`, kernel held **9 of 17**. The count + cross-check is genuinely the only thing that catches it. And the question + that matters — with 9 of 17 rules, `runsc` → live host `192.168.5.15:49257` + came back **REACHED**. A partially-installed policy leaves the job + **uncontained**, so refusing the job is the only safe response. + +### What failed — and it is mine +* **D — teardown of a partial install left all 9 rules in place.** Chain depth + went **2 → 11**. The gate leaked rules into a chain shared with every + container on the daemon. + +Mechanism, confirmed directly against the image rather than inferred: + +``` +apply-policy: rule 1 failed: iptables -D DOCKER-USER -s 10.99.99.99/32 ... +apply-policy: namespace is PARTIALLY configured — destroy the holder, do not retry +``` +…and the *next* rule in that probe plan never ran. The applier aborts on the +first failure, by design. + +Teardown is the exact inverse — 17 `-D` in reverse order — so it begins with +rule 17, which a 9-rule partial install never created. That first delete fails, +the applier aborts, and **none of the 9 real rules come out**. + +### Why this is a real defect and not a script artefact +`apply-policy`'s exit-3 contract says *destroy the holder*. For the **namespace** +plan that is a complete remedy: the rules live in the holder's netns and die +with it. My **host-side** plan puts rules in the **root netns**, in `DOCKER-USER` +and `INPUT`. Destroying the holder removes none of them. The host path reuses an +applier whose failure contract assumes namespace-scoped rules. + +This defeats the intent documented at the adoption site — *"a plan that failed +part-way has already installed rules, and those rules must come out whichever +way this returns."* `HostRules` is adopted correctly; the teardown it runs is +what cannot do the job. + +Consequence if shipped: any partial host-rule install strands rules keyed to a +job address in a shared chain, and gate 5h showed those addresses get recycled. +Gate 5h passed only because its install was **complete**, so its teardown +matched rule-for-rule. + +**Not fixed in this commit.** The failing gate and its evidence land first. + +## Gate 5i, after the fix — PASS + +`evidence/gate5i-fail-closed-PASS-after-fix-20260910T0347Z.txt`. The FAIL run +is kept alongside it, not overwritten. + +**The fix** (`sandbox_netns.rs`): `HostRules` now carries `complete`, set only +after the applier's count cross-check passes — never at construction. Teardown +branches on it: + +* **complete** → the one-shot inverse plan, exactly as before. Every rule is + present, so the inverse matches rule-for-rule and one invocation is correct + and cheapest. +* **not complete** → **one applier invocation per rule**. A failure then means + only "that rule was not there", which on this path is expected rather than an + error, so a rule that was never created can no longer strand the ones that were. + +Adoption still happens **before** the result is examined; that was always right. +What changed is that the teardown it runs can now cope with the state adoption +exists to clean up. + +Held by two unit tests: `adopted_host_rules_are_not_complete_until_the_count_check_passes` +(the default must be the rule-by-rule path; the fast path is earned) and +`the_per_rule_teardown_renders_one_valid_delete_per_rule` (each single-rule plan +is one line, a delete, still keyed to this job's `/32` — the applier refuses an +empty plan with exit 4 and a non-iptables binary with exit 5). + +**Both strategies now run back to back on the same partial install**, because +the difference between them *is* the fix: + +| leg | result | +|---|---| +| C — truncated plan | applier exit `0`, reported `9`, kernel held **9 of 17** | +| C — runsc → live host with 9 of 17 rules | **REACHED** — a partial policy is no policy | +| D1 — one-shot inverse (pre-fix) | **9 of 9 remain**, applier aborts on the first absent rule | +| D2 — per-rule (post-fix) | **removed 9, 0 remain** | +| chain depth | **2 → 2** | + +**GATE 5i: PASS, 0 failing checks.** + +D1 is kept deliberately as a live regression witness rather than deleted: if +the applier's abort-on-first-failure behaviour ever changes, this gate will say +so instead of quietly preserving a workaround nobody needs any more. + +### What this leg does and does not prove +The Rust branch in `HostRules::drop` is covered by the two unit tests above. +What the shell gate proves is narrower and worth stating plainly: that the +per-rule **strategy** clears real rules from real chains where the one-shot +inverse plan cannot. The gate does not drive `establish()` end to end. + +## Gate 5j — IPv6: measured, and the code's stated reason does not hold here + +Gate 5 left IPv6 as `NO-V6`/UNPROVEN. Maxie asked for "applicable IPv6". +Evidence: `evidence/gate5j-ipv6-finding-20260910T0347Z.txt`. + +### What is true by construction +`HostPolicy::argv` uses `Family::V4.binary()` unconditionally, so **no +host-side IPv6 rule is rendered at all**. The netns plan still carries +`DENIED_DESTINATIONS_V6`. This is deliberate and documented at +`sandbox_net.rs`. + +### What was measured (aarch64, docker 29.1.3, gvisor-repro) +* `/etc/docker/daemon.json` declares only the `runsc` runtime — **no IPv6**. +* `docker info` reports no ipv6 key. +* Inside a job-shaped holder on a per-job network: `ip -6 addr` shows only + `::1/128 scope host`. **Zero global IPv6 addresses.** + +So on this host a job has no IPv6 egress path, and host-side v6 containment is +not exercised. **IPv6 is not-applicable here** — and that is a statement about +this host, not a safety property of the branch. + +### The part that contradicts the code comment +The doc comment justifies omitting v6 rules like this: *"`ip6tables` has a +`DOCKER-USER` chain only when the daemon has IPv6 enabled, and a missing chain +is an install failure that would fail every job launch on a v4-only host."* + +Measured on this v4-only host: + +``` +$ sudo ip6tables -S DOCKER-USER +-N DOCKER-USER +``` + +**The chain exists**, with IPv6 disabled on the daemon. The stated premise does +not hold here, so it is not the reason v6 rules are safe to omit. The honest +reason is the one gate 5 already gave: host-side v6 containment is +**UNMEASURED**. The comment should be corrected rather than relied on, and I +have not rewritten it in this branch because changing v6 behaviour is outside +the minimum repair maxie scoped — it is named here and in the PR instead. + +### The hole, stated plainly +A host **with IPv6 enabled** is **not covered** by this branch. If jobs there +receive global v6 addresses, they have an egress path with no host-side +containment in front of it, and the per-job network does not help for +destinations reached by routing. That is an open hole in an unshipped +configuration — not a proof of safety, and not something this branch fixes. + +## Baseline pinning (maxie's ruling, 9 Sep 2026) + +Maxie: *"Pin baseline source evidence to a full immutable hash, not moving +`origin/main`."* Every baseline citation in this RUNLOG and in the gate5g +script now names the full commit + + b45f8651dc9cab5b71c962eadd7b84840f579106 + +and no longer identifies the baseline by a branch ref, which can move under a +reader and would make the reproduction unverifiable later. + +The evidence FILES under `evidence/` still print the abbreviated `b45f865`, +and that is deliberate: they are verbatim records of runs that printed it. +Editing a recorded run's output to look tidier would falsify the record. The +abbreviation resolves to the full commit above, which a reviewer can confirm +with `git rev-parse b45f865`. + +## Gate 5k — an unsupported daemon default runtime fails CLOSED + +Maxie: *"Missing `--runtime` means daemon default, not guaranteed runc; test +unsupported defaults fail closed."* The correction lands on the product. +`holder_argv()` deliberately passes **no `--runtime`**, and a test locks it +there for a sound reason — a runsc holder's namespace is unusable, a job +joining it sees `lo` only (gate2a). So the design **assumes the daemon default +is runc**, and nothing in the product verifies that assumption. + +Evidence: `evidence/gate5k-v3-PASS-daemon-default-20260910T0345Z.txt`. + +| daemon default | listener | holder runtime | job | +|---|---|---|---| +| `runc` (supported) | SERVING | runc | `HEALTHY 172.18.0.3` → **REACHED** | +| `runsc` (unsupported) | SERVING | runsc | `SICK no-address` → **ENETUNREACH** | + +`failing_checks=0`: both legs had a *serving* listener, so leg 2's denial is a +measurement and not a silence. Leg 1 is the positive control — it proves the +harness can observe reachability at all. `/etc/docker/daemon.json` was restored +and the default verified back to `runc` by the EXIT trap. + +**Verdict: fails CLOSED.** With an unsupported default the job has no address +and no route, so there is no egress path to contain. That is an availability +failure, not a containment bypass — the product breaks loudly rather than +running jobs uncontained. + +**The gap that remains, named not fixed:** nothing detects this at startup. A +seat whose daemon default is not runc will fail every job with no explanatory +signal. The honest fix is a doctor/readiness row asserting +`docker info --format '{{.DefaultRuntime}}' == runc`, which is a product change +outside the minimum containment repair maxie scoped, so it is recorded here for +the follow-up rather than smuggled into this branch. + +### Three attempts, two failures kept + +v1 (`gate5k-CONFOUNDED-harness-defect-*.txt`) started the listener with +`docker exec` into a runsc holder, which runsc refuses; the target never served +in either leg, including the control. v2 +(`gate5k-v2-daemon-default-runtime-*.txt`) found the root cause: the sandbox +image has **no `nc` and no `wget`**, so every probe was doomed before it ran. +Both are committed. They are also the reason the earlier gates were re-checked: +gate5f/5g/5h/5i never use those tools — they probe with `--entrypoint node` and +inline JS — so their SERVING and REACHED readings stand. diff --git a/docs/gvisor-dns-delivery/evidence/gate1-runsc-vs-runc-20260910T0046Z.txt b/docs/gvisor-dns-delivery/evidence/gate1-runsc-vs-runc-20260910T0046Z.txt new file mode 100644 index 000000000..f502d6c6e --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate1-runsc-vs-runc-20260910T0046Z.txt @@ -0,0 +1,26 @@ +=== gate1: environment === +utc=2026-09-10T00:46:01Z +kernel=6.8.0-134-generic arch=aarch64 +os=Ubuntu 24.04.4 LTS +docker=29.1.3 +runsc=runsc version release-20260817.0 +image=ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8 +image_digest=ghcr.io/makeprisms/maxplayer-sandbox@sha256:1c50e46a35dfe91fcdbbba11876bff312a95567bda98d6dcb7f675c884777412 +image_arch=arm64/linux +network=maxplayer-dns-repro subnet=172.18.0.0/16 + +=== gate1: dns lookup under --runtime runsc === +resolv.conf: # Generated by Docker Engine.|# This file can be edited; Docker Engine will not make further changes once it|# has been modified.||nameserver 127.0.0.11|search lan|options edns0 trust-ad ndots:0||# Based on host file: '/etc/resolv.conf' (internal resolver)|# ExtServers: [host(127.0.0.53)]|# Overrides: []|# Option ndots from: internal +lookup: ERR EAI_AGAIN +exit=1 + +=== gate1: dns lookup under --runtime runc === +resolv.conf: # Generated by Docker Engine.|# This file can be edited; Docker Engine will not make further changes once it|# has been modified.||nameserver 127.0.0.11|search lan|options edns0 trust-ad ndots:0||# Based on host file: '/etc/resolv.conf' (internal resolver)|# ExtServers: [host(127.0.0.53)]|# Overrides: []|# Option ndots from: internal +lookup: OK 34.225.223.145 +exit=0 + +=== gate1: raw udp/53 to the embedded resolver under runsc === +udp53: TIMEOUT (no answer from 127.0.0.11) +exit=1 + +=== gate1: done === diff --git a/docs/gvisor-dns-delivery/evidence/gate2-plan-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate2-plan-rendered.txt new file mode 100644 index 000000000..aa16b5b11 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate2-plan-rendered.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.7.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate2a-runsc-holder-FAIL-20260910T0145Z.txt b/docs/gvisor-dns-delivery/evidence/gate2a-runsc-holder-FAIL-20260910T0145Z.txt new file mode 100644 index 000000000..d53a1af45 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate2a-runsc-holder-FAIL-20260910T0145Z.txt @@ -0,0 +1,66 @@ +=== gate2: environment === +utc=2026-09-10T01:45:37Z +kernel=6.8.0-134-generic arch=aarch64 +os=Ubuntu 24.04.4 LTS +docker=29.1.3 +runsc=runsc version release-20260817.0 +image=ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8 +image_digest=ghcr.io/makeprisms/maxplayer-sandbox@sha256:1c50e46a35dfe91fcdbbba11876bff312a95567bda98d6dcb7f675c884777412 +netfilter_digest=ghcr.io/makeprisms/maxplayer-netfilter@sha256:913985269261c7169b4f7b14a5db69843b9a3421aa483145a26d6ac38db3da3a +resolver=1.1.1.1 +resolv_conf=/home/forge.guest/gate2-resolv.conf sha256=24c2b7ed40eef08c2661229c666dce74d955b8e365b7ec598456cbf042bece91 +plan=/home/forge.guest/gate2-plan.txt rules=26 sha256=eb0fc310b18db6104445fd2f08553d1f0d068f3747eb58d97c434f7670360f43 +network=maxplayer-dns-gate2 subnet=172.31.7.0/24 gateway=172.31.7.1 + +=== gate2/fresh: establish the shared job namespace === +holder=gate2-holder started=0 +--- sidecar applies the rendered plan --- +iptables: Failed to initialize nft: Protocol not supported +apply-policy: rule 1 failed: iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +apply-policy: namespace is PARTIALLY configured — destroy the holder, do not retry +sidecar_exit=3 +--- readback (iptables -S) --- +iptables: Failed to initialize nft: Protocol not supported +readback_exit=1 + +=== gate2/fresh: job in the shared namespace — dns + verified tls === +resolv.conf: nameserver 1.1.1.1|options timeout:2 attempts:2 +lookup: ERR EAI_AGAIN +job_exit=1 +--- containment still holds: metadata address --- +metadata: denied (ENETUNREACH) +metadata_exit=0 + +=== gate2: destroy the namespace and rebuild it === + +=== gate2/recreated: establish the shared job namespace === +holder=gate2-holder started=0 +--- sidecar applies the rendered plan --- +iptables: Failed to initialize nft: Protocol not supported +apply-policy: rule 1 failed: iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +apply-policy: namespace is PARTIALLY configured — destroy the holder, do not retry +sidecar_exit=3 +--- readback (iptables -S) --- +iptables: Failed to initialize nft: Protocol not supported +readback_exit=1 + +=== gate2/recreated: job in the shared namespace — dns + verified tls === +resolv.conf: nameserver 1.1.1.1|options timeout:2 attempts:2 +lookup: ERR EAI_AGAIN +job_exit=1 +--- containment still holds: metadata address --- +metadata: denied (ENETUNREACH) +metadata_exit=0 + +=== gate2: verdict === +GATE2: FAIL + +=== gate2 diagnostic: interfaces a container sees, by runtime and network mode === +(probe: node os.networkInterfaces(); the sandbox image ships no iproute2) +A runsc job joining a RUNSC holder netns: lo +B runsc job joining a RUNC holder netns: lo,eth0 +C runsc job on the bridge DIRECTLY: lo,eth0 +D runc job joining the RUNC holder: lo +=== iptables backend available to the sidecar, by runtime === +runc sidecar on runc holder: -P OUTPUT ACCEPT +runsc sidecar on runsc holder: iptables: Failed to initialize nft: Protocol not supported diff --git a/docs/gvisor-dns-delivery/evidence/gate2b-runc-plane-runsc-job-PASS-20260910T0210Z.txt b/docs/gvisor-dns-delivery/evidence/gate2b-runc-plane-runsc-job-PASS-20260910T0210Z.txt new file mode 100644 index 000000000..161b52916 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate2b-runc-plane-runsc-job-PASS-20260910T0210Z.txt @@ -0,0 +1,102 @@ +=== gate2: environment === +utc=2026-09-10T01:48:54Z +kernel=6.8.0-134-generic arch=aarch64 +os=Ubuntu 24.04.4 LTS +docker=29.1.3 +runsc=runsc version release-20260817.0 +holder_runtime=runc job_runtime=runsc +image=ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8 +image_digest=ghcr.io/makeprisms/maxplayer-sandbox@sha256:1c50e46a35dfe91fcdbbba11876bff312a95567bda98d6dcb7f675c884777412 +netfilter_digest=ghcr.io/makeprisms/maxplayer-netfilter@sha256:913985269261c7169b4f7b14a5db69843b9a3421aa483145a26d6ac38db3da3a +resolver=1.1.1.1 +/Users/forge/forge/v2/wt/w-gvisor-dns-delivery-r2/docs/gvisor-dns-delivery/scripts/gate2-namespace-dns-tls.sh: line 55: /home/forge.guest/gate2-resolv.conf: Permission denied +resolv_conf=/home/forge.guest/gate2-resolv.conf sha256=24c2b7ed40eef08c2661229c666dce74d955b8e365b7ec598456cbf042bece91 +plan=/home/forge.guest/gate2-plan.txt rules=26 sha256=eb0fc310b18db6104445fd2f08553d1f0d068f3747eb58d97c434f7670360f43 +network=maxplayer-dns-gate2 subnet=172.31.7.0/24 gateway=172.31.7.1 + +=== gate2/fresh: establish the shared job namespace === +holder=gate2-holder started=0 +--- sidecar applies the rendered plan --- +26 +sidecar_exit=0 +--- readback (iptables -S) --- +-P OUTPUT ACCEPT +-A OUTPUT -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-conn:" +-A OUTPUT -p udp -m udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-dns:" +-A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny-metadata:" +-A OUTPUT -d 169.254.169.254/32 -j DROP +-A OUTPUT -d 172.31.7.1/32 -p tcp -m tcp --dport 49200:49299 -j ACCEPT +-A OUTPUT -d 1.1.1.1/32 -p udp -m udp --dport 53 -j ACCEPT +-A OUTPUT -d 1.1.1.1/32 -p tcp -m tcp --dport 53 -j ACCEPT +-A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 10.0.0.0/8 -j DROP +-A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 172.16.0.0/12 -j DROP +-A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 192.168.0.0/16 -j DROP +-A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 169.254.0.0/16 -j DROP +-A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 100.64.0.0/10 -j DROP +-A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 198.18.0.0/15 -j DROP +-A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 224.0.0.0/4 -j DROP +-A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 240.0.0.0/4 -j DROP +readback_exit=0 + +=== gate2/fresh: job in the shared namespace — dns + verified tls === +resolv.conf: nameserver 1.1.1.1|options timeout:2 attempts:2 +lookup: OK 34.225.223.145 +tls: 200 cert-verified subject=relay.maxplayer.ai issuer=YE1 authorized=true +job_exit=0 +--- containment still holds: metadata address --- +metadata: denied (ENETUNREACH) +metadata_exit=0 + +=== gate2: destroy the namespace and rebuild it === + +=== gate2/recreated: establish the shared job namespace === +holder=gate2-holder started=0 +--- sidecar applies the rendered plan --- +26 +sidecar_exit=0 +--- readback (iptables -S) --- +-P OUTPUT ACCEPT +-A OUTPUT -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-conn:" +-A OUTPUT -p udp -m udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-dns:" +-A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny-metadata:" +-A OUTPUT -d 169.254.169.254/32 -j DROP +-A OUTPUT -d 172.31.7.1/32 -p tcp -m tcp --dport 49200:49299 -j ACCEPT +-A OUTPUT -d 1.1.1.1/32 -p udp -m udp --dport 53 -j ACCEPT +-A OUTPUT -d 1.1.1.1/32 -p tcp -m tcp --dport 53 -j ACCEPT +-A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 10.0.0.0/8 -j DROP +-A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 172.16.0.0/12 -j DROP +-A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 192.168.0.0/16 -j DROP +-A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 169.254.0.0/16 -j DROP +-A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 100.64.0.0/10 -j DROP +-A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 198.18.0.0/15 -j DROP +-A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 224.0.0.0/4 -j DROP +-A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 240.0.0.0/4 -j DROP +readback_exit=0 + +=== gate2/recreated: job in the shared namespace — dns + verified tls === +resolv.conf: nameserver 1.1.1.1|options timeout:2 attempts:2 +lookup: OK 34.225.223.145 +tls: 200 cert-verified subject=relay.maxplayer.ai issuer=YE1 authorized=true +job_exit=0 +--- containment still holds: metadata address --- +metadata: denied (ENETUNREACH) +metadata_exit=0 + +=== gate2: verdict === +GATE2: PASS (fresh and recreated namespaces both resolved and completed verified TLS) diff --git a/docs/gvisor-dns-delivery/evidence/gate4-container-git-delivery-PASS-20260910T0330Z.txt b/docs/gvisor-dns-delivery/evidence/gate4-container-git-delivery-PASS-20260910T0330Z.txt new file mode 100644 index 000000000..8c045cadb --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate4-container-git-delivery-PASS-20260910T0330Z.txt @@ -0,0 +1,42 @@ +=== gate4: environment === +utc=2026-09-10T02:21:28Z +kernel=6.8.0-134-generic arch=aarch64 +os=Ubuntu 24.04.4 LTS +docker=29.1.3 +runsc=runsc version release-20260817.0 +holder_runtime=runc job_runtime=runsc +image_digest=ghcr.io/makeprisms/maxplayer-sandbox@sha256:1c50e46a35dfe91fcdbbba11876bff312a95567bda98d6dcb7f675c884777412 +container_git=git version 2.39.5 +public_repo=https://github.com/octocat/Hello-World.git +/Users/forge/forge/v2/wt/w-gvisor-dns-delivery-r2/docs/gvisor-dns-delivery/scripts/gate4-container-git-delivery.sh: line 65: /home/forge.guest/gate4-resolv.conf: Permission denied +plan=/home/forge.guest/gate4-plan.txt rules=26 sha256=70ab9b4e65dfe6a1a48911ea734e0fecf6ca84f9d0879a890d460e84830e3d78 +network=maxplayer-dns-gate4 subnet=172.31.8.0/24 gateway=172.31.8.1 + +=== gate4: control — the remote's real HEAD, read from the VM (outside any container) === +control_head=7fd1a60b01f91b314f59955a4e4d4e80d8edf11d + +=== gate4: disposable write remote === +write_remote=git://172.31.8.1:49250/answer.git (disposable, unauthenticated, torn down on exit) +remote_refs_before=0 + +=== gate4: establish the shared job namespace === +holder_started=0 +26 +sidecar_exit=0 + +=== gate4: job container (runsc, non-root, cap-drop ALL) === +--- read leg: clone over https from inside the sandbox --- +delivered_head=7fd1a60b01f91b314f59955a4e4d4e80d8edf11d +--- write leg: commit here, push to the remote outside this container --- +answer_commit=a4a883b087273c86480ea1269f7e50ff965296b2 +push_exit=0 +job_exit=0 + +=== gate4: verdict === +control_head=7fd1a60b01f91b314f59955a4e4d4e80d8edf11d +delivered_head=7fd1a60b01f91b314f59955a4e4d4e80d8edf11d +READ: PASS — the sandbox delivered the remote's real HEAD +answer_commit=a4a883b087273c86480ea1269f7e50ff965296b2 +remote_hash=a4a883b087273c86480ea1269f7e50ff965296b2 +WRITE: PASS — the commit made inside the sandbox reached the remote, hash matches +GATE4: PASS diff --git a/docs/gvisor-dns-delivery/evidence/gate4-plan-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate4-plan-rendered.txt new file mode 100644 index 000000000..1c9eb3de5 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate4-plan-rendered.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.8.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-denial-and-concurrent-success-20260910T0245Z.txt b/docs/gvisor-dns-delivery/evidence/gate5-denial-and-concurrent-success-20260910T0245Z.txt new file mode 100644 index 000000000..b62d905d5 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-denial-and-concurrent-success-20260910T0245Z.txt @@ -0,0 +1,46 @@ +=== gate5: environment === +utc=2026-09-10T02:29:10Z +kernel=6.8.0-134-generic arch=aarch64 +os=Ubuntu 24.04.4 LTS +docker=29.1.3 +runsc=runsc version release-20260817.0 +holder_runtime=runc job_runtime=runsc +image_digest=ghcr.io/makeprisms/maxplayer-sandbox@sha256:1c50e46a35dfe91fcdbbba11876bff312a95567bda98d6dcb7f675c884777412 +/Users/forge/forge/v2/wt/w-gvisor-dns-delivery-r2/docs/gvisor-dns-delivery/scripts/gate5-denial-and-concurrent-success.sh: line 58: /home/forge.guest/gate5-resolv.conf: Permission denied +plan=/home/forge.guest/gate5-plan.txt rules=26 sha256=2d312d2e5c59f974a4aa857fecb943b03d081e5c9ef2ff2de23dbb539e66d59d +network=maxplayer-dns-gate5 subnet=172.31.12.0/24 gateway=172.31.12.1 +host_loopback_listener=127.0.0.1:49251 + +=== gate5: two job namespaces, both contained === +established=gate5-holder-a sidecar_exit=0 +established=gate5-holder-b sidecar_exit=0 +job_b_address=172.31.12.3 +neighbour_listening=true + +=== gate5: job A — denial legs and public legs, one namespace, one run === +DENY-PASS ipv6 link-local (fe80::1:80) -> ENETUNREACH +DENY-PASS ipv6 unique-local (fc00::1:80) -> ENETUNREACH +DENY-PASS host loopback via 127.0.0.1 (127.0.0.1:49251) -> ECONNREFUSED +DENY-PASS ipv6 loopback (::1:80) -> ECONNREFUSED +DENY-PASS metadata (direct ip) (169.254.169.254:80) -> ECONNREFUSED +DENY-PASS link-local (direct ip) (169.254.1.1:80) -> ECONNREFUSED +DENY-FAIL another job namespace (172.31.12.3:8080) -> REACHED +DENY-PASS metadata BY NAME (169.254.169.254.nip.io:80) -> ECONNREFUSED +PUBLIC-PASS dns relay.maxplayer.ai -> 34.225.223.145 +PUBLIC-PASS tls 200 verified relay.maxplayer.ai +DENY-PASS rfc1918 10/8 (10.0.0.1:80) -> timeout +DENY-PASS rfc1918 192.168/16 (192.168.1.1:80) -> timeout +DENY-PASS rfc1918 BY NAME (10.0.0.1.nip.io:80) -> timeout +denial_failures=1 +PUBLIC-FAIL tls timeout + +=== gate5: job A — git clone over https, same namespace === +fatal: unable to access 'https://github.com/octocat/Hello-World.git/': Could not resolve host: github.com +git_exit=128 + +=== gate5: verdict === +denial_failures=1 +denials_proven=10 +DENIAL: FAIL — something a contained job must not reach was reachable +PUBLIC: FAIL — the sandbox did not demonstrate the public route it is supposed to keep (dns_ok=1 tls_ok=1 git_exit=128) +GATE5: FAIL diff --git a/docs/gvisor-dns-delivery/evidence/gate5-denial-and-concurrent-success-PASS-20260910T0311Z.txt b/docs/gvisor-dns-delivery/evidence/gate5-denial-and-concurrent-success-PASS-20260910T0311Z.txt new file mode 100644 index 000000000..18abb1cf5 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-denial-and-concurrent-success-PASS-20260910T0311Z.txt @@ -0,0 +1,51 @@ +=== gate5: environment === +utc=2026-09-10T03:11:45Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 runsc=runsc version release-20260817.0 +docker default-runtime=runc · br_netfilter=absent +chain depth before anything: DOCKER-USER=1 INPUT=1 +live listeners: host 192.168.5.15:49254 · neighbour job 172.31.34.10:8080 · name denied-lan.maxplayer.test -> 192.168.5.15 + +=== gate5: 1. denial, BEFORE the host policy (so each leg has a baseline) === + runsc -> host 192.168.5.15:49254 (live): REACHED + runsc -> denied-lan.maxplayer.test:49254 (live, by name): REACHED + runsc -> neighbour job 172.31.34.10:8080 (live): timeout + runsc -> 169.254.169.254:80 (nothing listens): ECONNREFUSED + +=== gate5: 2. the product's host policy, installed for the probe namespace === + host policy install for 172.31.30.10: applier=17/17, host kernel now carries 17 + +=== gate5: 3. denial, AFTER — every one of these MUST be denied === + runsc -> host 192.168.5.15:49254 (live): timeout (denied) + runsc -> denied-lan.maxplayer.test:49254 (live, by name): timeout (denied) + runsc -> neighbour job 172.31.34.10:8080 (live): timeout (denied) + runsc -> 169.254.169.254:80 (nothing listens): timeout (denied) + ^ read the metadata leg ONLY as the difference against section 1; with no listener, + an identical result in both sections is NO EVIDENCE either way. + +=== gate5: 4. IPv6, measured rather than assumed === + namespace v6: NO-V6 + runsc v6: NO-V6 + The host-side plan renders NO ip6tables rules (DOCKER-USER may not exist there). + Where the job namespace has no global IPv6 there is nothing to deny; where it has, + host-side v6 denial for a runsc job is UNPROVEN and must not be claimed. + +=== gate5: 5. concurrent delivery, three jobs at once, all policies installed === + host policy install for 172.31.31.10: applier=17/17, host kernel now carries 17 + host policy install for 172.31.32.10: applier=17/17, host kernel now carries 17 + host policy install for 172.31.33.10: applier=17/17, host kernel now carries 17 + c1: OK dns=34.225.223.145 tls=200 verified=true OK git 7fd1a60b01f91b314f59955a4e4d4e80d8edf11d + c3: OK dns=34.225.223.145 tls=200 verified=true OK git 7fd1a60b01f91b314f59955a4e4d4e80d8edf11d + c2: OK dns=34.225.223.145 tls=200 verified=true OK git 7fd1a60b01f91b314f59955a4e4d4e80d8edf11d + host policy teardown for 172.31.31.10: applier=17/17, host kernel now carries 0 + host policy teardown for 172.31.32.10: applier=17/17, host kernel now carries 0 + host policy teardown for 172.31.33.10: applier=17/17, host kernel now carries 0 + +=== gate5: 6. teardown leaves the shared chains as it found them === + host policy teardown for 172.31.30.10: applier=17/17, host kernel now carries 0 + DOCKER-USER=1 (was 1) · INPUT=1 (was 1) + CLEAN + +=== gate5: verdict === +denial legs failing: 0 +A concurrent line counts as delivery only if it reads OK dns=… tls=200 verified=true AND OK git . +GATE5-DENIAL: PASS diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plan-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate5-plan-rendered.txt new file mode 100644 index 000000000..fa81a0198 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plan-rendered.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.12.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c1-teardown.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c1-teardown.txt new file mode 100644 index 000000000..c5c077bb8 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c1-teardown.txt @@ -0,0 +1,17 @@ +iptables -D INPUT -s 172.31.31.10/32 -d 240.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.31.10/32 -d 224.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.31.10/32 -d 198.18.0.0/15 -j DROP +iptables -D INPUT -s 172.31.31.10/32 -d 100.64.0.0/10 -j DROP +iptables -D INPUT -s 172.31.31.10/32 -d 169.254.0.0/16 -j DROP +iptables -D INPUT -s 172.31.31.10/32 -d 192.168.0.0/16 -j DROP +iptables -D INPUT -s 172.31.31.10/32 -d 172.16.0.0/12 -j DROP +iptables -D INPUT -s 172.31.31.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 240.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 224.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 198.18.0.0/15 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 100.64.0.0/10 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 169.254.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 192.168.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 172.16.0.0/12 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 169.254.169.254/32 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c1.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c1.txt new file mode 100644 index 000000000..684f92229 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c1.txt @@ -0,0 +1,17 @@ +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 169.254.169.254/32 -j DROP +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 10.0.0.0/8 -j DROP +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 172.16.0.0/12 -j DROP +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 192.168.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 169.254.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 100.64.0.0/10 -j DROP +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 198.18.0.0/15 -j DROP +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 224.0.0.0/4 -j DROP +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 240.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.31.10/32 -d 10.0.0.0/8 -j DROP +iptables -I INPUT -s 172.31.31.10/32 -d 172.16.0.0/12 -j DROP +iptables -I INPUT -s 172.31.31.10/32 -d 192.168.0.0/16 -j DROP +iptables -I INPUT -s 172.31.31.10/32 -d 169.254.0.0/16 -j DROP +iptables -I INPUT -s 172.31.31.10/32 -d 100.64.0.0/10 -j DROP +iptables -I INPUT -s 172.31.31.10/32 -d 198.18.0.0/15 -j DROP +iptables -I INPUT -s 172.31.31.10/32 -d 224.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.31.10/32 -d 240.0.0.0/4 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c2-teardown.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c2-teardown.txt new file mode 100644 index 000000000..0044ef780 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c2-teardown.txt @@ -0,0 +1,17 @@ +iptables -D INPUT -s 172.31.32.10/32 -d 240.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.32.10/32 -d 224.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.32.10/32 -d 198.18.0.0/15 -j DROP +iptables -D INPUT -s 172.31.32.10/32 -d 100.64.0.0/10 -j DROP +iptables -D INPUT -s 172.31.32.10/32 -d 169.254.0.0/16 -j DROP +iptables -D INPUT -s 172.31.32.10/32 -d 192.168.0.0/16 -j DROP +iptables -D INPUT -s 172.31.32.10/32 -d 172.16.0.0/12 -j DROP +iptables -D INPUT -s 172.31.32.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 240.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 224.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 198.18.0.0/15 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 100.64.0.0/10 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 169.254.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 192.168.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 172.16.0.0/12 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 169.254.169.254/32 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c2.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c2.txt new file mode 100644 index 000000000..d5d279f2c --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c2.txt @@ -0,0 +1,17 @@ +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 169.254.169.254/32 -j DROP +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 10.0.0.0/8 -j DROP +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 172.16.0.0/12 -j DROP +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 192.168.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 169.254.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 100.64.0.0/10 -j DROP +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 198.18.0.0/15 -j DROP +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 224.0.0.0/4 -j DROP +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 240.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.32.10/32 -d 10.0.0.0/8 -j DROP +iptables -I INPUT -s 172.31.32.10/32 -d 172.16.0.0/12 -j DROP +iptables -I INPUT -s 172.31.32.10/32 -d 192.168.0.0/16 -j DROP +iptables -I INPUT -s 172.31.32.10/32 -d 169.254.0.0/16 -j DROP +iptables -I INPUT -s 172.31.32.10/32 -d 100.64.0.0/10 -j DROP +iptables -I INPUT -s 172.31.32.10/32 -d 198.18.0.0/15 -j DROP +iptables -I INPUT -s 172.31.32.10/32 -d 224.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.32.10/32 -d 240.0.0.0/4 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c3-teardown.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c3-teardown.txt new file mode 100644 index 000000000..4989b3f8c --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c3-teardown.txt @@ -0,0 +1,17 @@ +iptables -D INPUT -s 172.31.33.10/32 -d 240.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.33.10/32 -d 224.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.33.10/32 -d 198.18.0.0/15 -j DROP +iptables -D INPUT -s 172.31.33.10/32 -d 100.64.0.0/10 -j DROP +iptables -D INPUT -s 172.31.33.10/32 -d 169.254.0.0/16 -j DROP +iptables -D INPUT -s 172.31.33.10/32 -d 192.168.0.0/16 -j DROP +iptables -D INPUT -s 172.31.33.10/32 -d 172.16.0.0/12 -j DROP +iptables -D INPUT -s 172.31.33.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 240.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 224.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 198.18.0.0/15 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 100.64.0.0/10 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 169.254.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 192.168.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 172.16.0.0/12 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 169.254.169.254/32 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c3.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c3.txt new file mode 100644 index 000000000..2672b1810 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c3.txt @@ -0,0 +1,17 @@ +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 169.254.169.254/32 -j DROP +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 10.0.0.0/8 -j DROP +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 172.16.0.0/12 -j DROP +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 192.168.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 169.254.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 100.64.0.0/10 -j DROP +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 198.18.0.0/15 -j DROP +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 224.0.0.0/4 -j DROP +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 240.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.33.10/32 -d 10.0.0.0/8 -j DROP +iptables -I INPUT -s 172.31.33.10/32 -d 172.16.0.0/12 -j DROP +iptables -I INPUT -s 172.31.33.10/32 -d 192.168.0.0/16 -j DROP +iptables -I INPUT -s 172.31.33.10/32 -d 169.254.0.0/16 -j DROP +iptables -I INPUT -s 172.31.33.10/32 -d 100.64.0.0/10 -j DROP +iptables -I INPUT -s 172.31.33.10/32 -d 198.18.0.0/15 -j DROP +iptables -I INPUT -s 172.31.33.10/32 -d 224.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.33.10/32 -d 240.0.0.0/4 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-d-teardown.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-d-teardown.txt new file mode 100644 index 000000000..f33c795fc --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-d-teardown.txt @@ -0,0 +1,17 @@ +iptables -D INPUT -s 172.31.30.10/32 -d 240.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.30.10/32 -d 224.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.30.10/32 -d 198.18.0.0/15 -j DROP +iptables -D INPUT -s 172.31.30.10/32 -d 100.64.0.0/10 -j DROP +iptables -D INPUT -s 172.31.30.10/32 -d 169.254.0.0/16 -j DROP +iptables -D INPUT -s 172.31.30.10/32 -d 192.168.0.0/16 -j DROP +iptables -D INPUT -s 172.31.30.10/32 -d 172.16.0.0/12 -j DROP +iptables -D INPUT -s 172.31.30.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 240.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 224.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 198.18.0.0/15 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 100.64.0.0/10 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 169.254.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 192.168.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 172.16.0.0/12 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 169.254.169.254/32 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-d.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-d.txt new file mode 100644 index 000000000..c1aeca8f7 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-d.txt @@ -0,0 +1,17 @@ +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 169.254.169.254/32 -j DROP +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 10.0.0.0/8 -j DROP +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 172.16.0.0/12 -j DROP +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 192.168.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 169.254.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 100.64.0.0/10 -j DROP +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 198.18.0.0/15 -j DROP +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 224.0.0.0/4 -j DROP +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 240.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.30.10/32 -d 10.0.0.0/8 -j DROP +iptables -I INPUT -s 172.31.30.10/32 -d 172.16.0.0/12 -j DROP +iptables -I INPUT -s 172.31.30.10/32 -d 192.168.0.0/16 -j DROP +iptables -I INPUT -s 172.31.30.10/32 -d 169.254.0.0/16 -j DROP +iptables -I INPUT -s 172.31.30.10/32 -d 100.64.0.0/10 -j DROP +iptables -I INPUT -s 172.31.30.10/32 -d 198.18.0.0/15 -j DROP +iptables -I INPUT -s 172.31.30.10/32 -d 224.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.30.10/32 -d 240.0.0.0/4 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-n-teardown.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-n-teardown.txt new file mode 100644 index 000000000..a4f1ef81a --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-n-teardown.txt @@ -0,0 +1,17 @@ +iptables -D INPUT -s 172.31.34.10/32 -d 240.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.34.10/32 -d 224.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.34.10/32 -d 198.18.0.0/15 -j DROP +iptables -D INPUT -s 172.31.34.10/32 -d 100.64.0.0/10 -j DROP +iptables -D INPUT -s 172.31.34.10/32 -d 169.254.0.0/16 -j DROP +iptables -D INPUT -s 172.31.34.10/32 -d 192.168.0.0/16 -j DROP +iptables -D INPUT -s 172.31.34.10/32 -d 172.16.0.0/12 -j DROP +iptables -D INPUT -s 172.31.34.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 240.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 224.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 198.18.0.0/15 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 100.64.0.0/10 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 169.254.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 192.168.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 172.16.0.0/12 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 169.254.169.254/32 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-n.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-n.txt new file mode 100644 index 000000000..8f81a1b9b --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-n.txt @@ -0,0 +1,17 @@ +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 169.254.169.254/32 -j DROP +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 10.0.0.0/8 -j DROP +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 172.16.0.0/12 -j DROP +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 192.168.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 169.254.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 100.64.0.0/10 -j DROP +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 198.18.0.0/15 -j DROP +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 224.0.0.0/4 -j DROP +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 240.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.34.10/32 -d 10.0.0.0/8 -j DROP +iptables -I INPUT -s 172.31.34.10/32 -d 172.16.0.0/12 -j DROP +iptables -I INPUT -s 172.31.34.10/32 -d 192.168.0.0/16 -j DROP +iptables -I INPUT -s 172.31.34.10/32 -d 169.254.0.0/16 -j DROP +iptables -I INPUT -s 172.31.34.10/32 -d 100.64.0.0/10 -j DROP +iptables -I INPUT -s 172.31.34.10/32 -d 198.18.0.0/15 -j DROP +iptables -I INPUT -s 172.31.34.10/32 -d 224.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.34.10/32 -d 240.0.0.0/4 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c1.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c1.txt new file mode 100644 index 000000000..74158933d --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c1.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.31.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c2.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c2.txt new file mode 100644 index 000000000..82827ad86 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c2.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.32.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c3.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c3.txt new file mode 100644 index 000000000..5995217f2 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c3.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.33.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-d.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-d.txt new file mode 100644 index 000000000..d7477903b --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-d.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.30.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-n.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-n.txt new file mode 100644 index 000000000..03ededd9f --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-n.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.34.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5b-plan-does-not-bind-runsc-FINDING-20260910T0250Z.txt b/docs/gvisor-dns-delivery/evidence/gate5b-plan-does-not-bind-runsc-FINDING-20260910T0250Z.txt new file mode 100644 index 000000000..7524ba481 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5b-plan-does-not-bind-runsc-FINDING-20260910T0250Z.txt @@ -0,0 +1,25 @@ +=== gate5b: environment === +utc=2026-09-10T02:33:28Z +kernel=6.8.0-134-generic arch=aarch64 +docker=29.1.3 runsc=runsc version release-20260817.0 +network=maxplayer-dns-gate5b subnet=172.31.13.0/24 plan_rules=26 +26 +plan_applied_rules_reported=0 +neighbour=172.31.13.3:8080 running=true +neighbour_is_inside_a_dropped_range=172.16.0.0/12 + +=== gate5b: the same probe, the same namespace, two runtimes === +job_runtime=runc RESULT=timeout +job_runtime=runsc RESULT=REACHED + +=== gate5b: the rule as installed, read back from the namespace === +11:-A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +12:-A OUTPUT -d 172.16.0.0/12 -j DROP + +=== gate5b: verdict === +runc_result=RESULT=timeout +runsc_result=RESULT=REACHED +FINDING: the plan binds a runc job and DOES NOT BIND a runsc job. +gVisor's netstack emits packets to the veth itself; the host kernel's OUTPUT chain +in that netns never sees them, so per-job egress policy is not enforced for the job +it is written for. Containment for gVisor jobs cannot live in the netns OUTPUT chain. diff --git a/docs/gvisor-dns-delivery/evidence/gate5b-plan-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate5b-plan-rendered.txt new file mode 100644 index 000000000..b443c2876 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5b-plan-rendered.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.13.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5c-plan-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate5c-plan-rendered.txt new file mode 100644 index 000000000..cb5f616af --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5c-plan-rendered.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.17.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5c-where-does-containment-bind-20260910T0244Z.txt b/docs/gvisor-dns-delivery/evidence/gate5c-where-does-containment-bind-20260910T0244Z.txt new file mode 100644 index 000000000..351991af0 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5c-where-does-containment-bind-20260910T0244Z.txt @@ -0,0 +1,28 @@ +=== gate5c: environment === +utc=2026-09-10T02:44:06Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 runsc=runsc version release-20260817.0 +br_netfilter=absent +rule: one gVisor container per namespace; health check before every evidential leg +/Users/forge/forge/v2/wt/w-gvisor-dns-delivery-r2/docs/gvisor-dns-delivery/scripts/gate5c-where-does-containment-bind.sh: line 68: /home/forge.guest/gate5c-resolv.conf: Permission denied +live neighbours: same_bridge=172.31.17.2:8080 other_bridge=172.31.18.2:8080 + +=== gate5c: baseline — today's behaviour, no host-side rule === + runc -> live neighbour, same bridge: timeout [ns=172.31.17.3 rule=none health=HEALTHY 172.31.17.3 dns=34.225.223.145] + runsc -> live neighbour, same bridge: REACHED [ns=172.31.17.3 rule=none health=HEALTHY 172.31.17.3 dns=34.225.223.145] + +=== gate5c: (b) per-job network — the neighbour is on a DIFFERENT bridge === + runsc -> live neighbour, other bridge: timeout [ns=172.31.17.3 rule=none health=HEALTHY 172.31.17.3 dns=34.225.223.145] + +=== gate5c: (a) DOCKER-USER keyed to the namespace's own address === + runc -> live neighbour, same bridge: timeout [ns=172.31.17.3 rule=-s 172.31.17.3/32 -d 172.16.0.0/12 -j DROP health=HEALTHY 172.31.17.3 dns=34.225.223.145] + runsc -> live neighbour, same bridge: REACHED [ns=172.31.17.3 rule=-s 172.31.17.3/32 -d 172.16.0.0/12 -j DROP health=HEALTHY 172.31.17.3 dns=34.225.223.145] + +=== gate5c: does (a) cost the public route the job must keep? === + health before the public leg: HEALTHY 172.31.17.3 dns=34.225.223.145 + PUBLIC-PASS dns=34.225.223.145 tls=200 verified=true + health before the git leg: HEALTHY 172.31.17.3 dns=34.225.223.145 + PUBLIC-PASS git 7fd1a60b01f91b314f59955a4e4d4e80d8edf11d + +=== gate5c: how to read this === +(a) binds runsc only if BOTH rule-legs are denied AND both public legs still pass. +(b) binds runsc only if the other-bridge leg is denied from a HEALTHY namespace. diff --git a/docs/gvisor-dns-delivery/evidence/gate5c-where-does-containment-bind-VOID-20260910T0310Z.txt b/docs/gvisor-dns-delivery/evidence/gate5c-where-does-containment-bind-VOID-20260910T0310Z.txt new file mode 100644 index 000000000..3549f480c --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5c-where-does-containment-bind-VOID-20260910T0310Z.txt @@ -0,0 +1,27 @@ +=== gate5c: environment === +utc=2026-09-10T02:37:36Z +kernel=6.8.0-134-generic arch=aarch64 +docker=29.1.3 runsc=runsc version release-20260817.0 +br_netfilter=absent +net_a=maxplayer-dns-gate5c-a 172.31.14.0/24 | net_b=maxplayer-dns-gate5c-b 172.31.15.0/24 +job_a=172.31.14.2 | live_neighbour_same_network=172.31.14.3:8080 | live_neighbour_other_network=172.31.15.2:8080 + +=== gate5c: baseline — no host-side rule (this is today's behaviour) === +runc -> live neighbour, same bridge: timeout +runsc -> live neighbour, same bridge: REACHED + +=== gate5c: (b) per-job network — the neighbour is on a DIFFERENT bridge === +runsc -> live neighbour, other bridge: ENETUNREACH + +=== gate5c: (a) DOCKER-USER, keyed to the job namespace's source address === +installed: iptables -I DOCKER-USER -s 172.31.14.2/32 -d 172.16.0.0/12 -j DROP +runc -> live neighbour, same bridge: ENETUNREACH +runsc -> live neighbour, same bridge: ENETUNREACH + +=== gate5c: does that rule cost the public route the job must keep? === +PUBLIC-FAIL dns EAI_AGAIN +fatal: unable to access 'https://github.com/octocat/Hello-World.git/': Could not resolve host: github.com + +=== gate5c: read the finding off the table above === +(a) binds runsc if the DOCKER-USER runs show 'timeout' for BOTH runtimes +(b) binds runsc if the other-bridge probe is not REACHED diff --git a/docs/gvisor-dns-delivery/evidence/gate5d-namespace-is-single-use-for-gvisor-FINDING-20260910T0310Z.txt b/docs/gvisor-dns-delivery/evidence/gate5d-namespace-is-single-use-for-gvisor-FINDING-20260910T0310Z.txt new file mode 100644 index 000000000..ab3b4627f --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5d-namespace-is-single-use-for-gvisor-FINDING-20260910T0310Z.txt @@ -0,0 +1,18 @@ +=== gate5d: environment === +utc=2026-09-10T02:40:40Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 runsc=runsc version release-20260817.0 +namespace established, plan applied (26 rules) + +=== gate5d: the namespace before any gVisor container has touched it === + [before via runc] interfaces: lo=127.0.0.1 eth0=172.31.16.2 dns: 34.225.223.145 + +=== gate5d: one runsc container runs and exits === + [the gVisor job itself via runsc] interfaces: lo=127.0.0.1 eth0=172.31.16.2 dns: 34.225.223.145 + +=== gate5d: the same namespace afterwards === + [after, host runtime via runc] interfaces: lo=127.0.0.1 dns: EAI_AGAIN + [after, a second gVisor job via runsc] interfaces: lo=127.0.0.1 dns: EAI_AGAIN + +=== gate5d: read it off the two 'after' lines === +If 'before' has an eth0 address and 'after' says NONE, the namespace is single-use +for gVisor, and gate5c's ENETUNREACH verdicts are void — a dead control, not a policy. diff --git a/docs/gvisor-dns-delivery/evidence/gate5d-plan-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate5d-plan-rendered.txt new file mode 100644 index 000000000..9b57d9ff0 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5d-plan-rendered.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.16.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5e-plan-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate5e-plan-rendered.txt new file mode 100644 index 000000000..f7729072e --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5e-plan-rendered.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.19.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5e-routed-enforcement-under-runsc-20260910T0305Z.txt b/docs/gvisor-dns-delivery/evidence/gate5e-routed-enforcement-under-runsc-20260910T0305Z.txt new file mode 100644 index 000000000..2b7de26a2 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5e-routed-enforcement-under-runsc-20260910T0305Z.txt @@ -0,0 +1,33 @@ +=== gate5e: environment === +utc=2026-09-10T02:48:54Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 runsc=runsc version release-20260817.0 +br_netfilter=absent +host_lan=192.168.5.15 (the VM's own eth0 — host-directed traffic lands in INPUT, not FORWARD) +per-job networks: job1=172.31.19.0/24 job2=172.31.20.0/24 +live host listener: 192.168.5.15:49252 (bare tcp, answers every connection) +live neighbour in job 2's own namespace: 172.31.20.2:8080 + +=== gate5e: 1. the host itself, a routed private destination with a LIVE listener === + runc -> 192.168.5.15:49252, bare: timeout [ns=172.31.19.2 rules=none] + runsc -> 192.168.5.15:49252, bare: REACHED [ns=172.31.19.2 rules=none] + runsc -> 192.168.5.15:49252, DOCKER-USER: REACHED [ns=172.31.19.2 rules=DOCKER-USER|-s 172.31.19.2/32 -d 192.168.0.0/16 -j DROP] + runsc -> 192.168.5.15:49252, INPUT: timeout [ns=172.31.19.2 rules=INPUT|-s 172.31.19.2/32 -d 192.168.0.0/16 -j DROP] + +=== gate5e: 2. the metadata address (no listener anywhere — read only the DIFFERENCE) === + runsc -> 169.254.169.254:80, bare: ECONNREFUSED [ns=172.31.19.2 rules=none] + runsc -> 169.254.169.254:80, DOCKER-USER: timeout [ns=172.31.19.2 rules=DOCKER-USER|-s 172.31.19.2/32 -d 169.254.169.254/32 -j DROP] + +=== gate5e: 3. cross-job, each job on its OWN network (the regression proof) === + runsc -> job 2's live listener: timeout [ns=172.31.19.2 rules=none] + +=== gate5e: 4. the public route, with the candidate rules installed === + health: HEALTHY 172.31.19.2 rules=DOCKER-USER|-s 172.31.19.2/32 -d 169.254.169.254/32 -j DROP INPUT|-s 172.31.19.2/32 -d 192.168.0.0/16 -j DROP + PUBLIC-PASS dns=34.225.223.145 tls=200 verified=true + health: HEALTHY 172.31.19.2 + PUBLIC-PASS git 7fd1a60b01f91b314f59955a4e4d4e80d8edf11d + +=== gate5e: how to read this === +leg 1 names the chain that binds a routed, host-directed destination for runsc. +leg 2 is evidence ONLY if bare and ruled differ; identical timeouts prove nothing. +leg 3 must be denied for per-job networks to be the cross-job answer. +leg 4 must pass, or the candidate costs the job the route it exists to have. diff --git a/docs/gvisor-dns-delivery/evidence/gate5f-host-install-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate5f-host-install-rendered.txt new file mode 100644 index 000000000..9444d3d31 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5f-host-install-rendered.txt @@ -0,0 +1,17 @@ +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 169.254.169.254/32 -j DROP +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 10.0.0.0/8 -j DROP +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 172.16.0.0/12 -j DROP +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 192.168.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 169.254.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 100.64.0.0/10 -j DROP +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 198.18.0.0/15 -j DROP +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 224.0.0.0/4 -j DROP +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 240.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.21.10/32 -d 10.0.0.0/8 -j DROP +iptables -I INPUT -s 172.31.21.10/32 -d 172.16.0.0/12 -j DROP +iptables -I INPUT -s 172.31.21.10/32 -d 192.168.0.0/16 -j DROP +iptables -I INPUT -s 172.31.21.10/32 -d 169.254.0.0/16 -j DROP +iptables -I INPUT -s 172.31.21.10/32 -d 100.64.0.0/10 -j DROP +iptables -I INPUT -s 172.31.21.10/32 -d 198.18.0.0/15 -j DROP +iptables -I INPUT -s 172.31.21.10/32 -d 224.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.21.10/32 -d 240.0.0.0/4 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5f-host-teardown-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate5f-host-teardown-rendered.txt new file mode 100644 index 000000000..3b9a79a91 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5f-host-teardown-rendered.txt @@ -0,0 +1,17 @@ +iptables -D INPUT -s 172.31.21.10/32 -d 240.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.21.10/32 -d 224.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.21.10/32 -d 198.18.0.0/15 -j DROP +iptables -D INPUT -s 172.31.21.10/32 -d 100.64.0.0/10 -j DROP +iptables -D INPUT -s 172.31.21.10/32 -d 169.254.0.0/16 -j DROP +iptables -D INPUT -s 172.31.21.10/32 -d 192.168.0.0/16 -j DROP +iptables -D INPUT -s 172.31.21.10/32 -d 172.16.0.0/12 -j DROP +iptables -D INPUT -s 172.31.21.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 240.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 224.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 198.18.0.0/15 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 100.64.0.0/10 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 169.254.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 192.168.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 172.16.0.0/12 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 169.254.169.254/32 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5f-plan-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate5f-plan-rendered.txt new file mode 100644 index 000000000..66bca772f --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5f-plan-rendered.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.21.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5f-product-host-rules-bind-runsc-20260910T0305Z.txt b/docs/gvisor-dns-delivery/evidence/gate5f-product-host-rules-bind-runsc-20260910T0305Z.txt new file mode 100644 index 000000000..2d36601b0 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5f-product-host-rules-bind-runsc-20260910T0305Z.txt @@ -0,0 +1,43 @@ +=== gate5f: environment === +utc=2026-09-10T03:05:46Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 runsc=runsc version release-20260817.0 +docker default-runtime=runc (helpers inherit this; the JOB always carries --runtime runsc) +br_netfilter=absent +host plan rendered by the product for job_addr=172.31.21.10: 17 rules +chain depth before anything: DOCKER-USER=1 INPUT=1 +live listeners: same-bridge 172.31.21.20:8080 · own-network 172.31.22.10:8080 · host 192.168.5.15:49253 + +=== gate5f: 1. BEFORE — the hole, with no host rules === + runsc -> same-bridge neighbour 172.31.21.20:8080: REACHED + runsc -> own-network neighbour 172.31.22.10:8080: timeout + runsc -> host 192.168.5.15:49253: REACHED + runc -> host 192.168.5.15:49253: timeout + +=== gate5f: 2. installing the product's rendered host policy === + host-rule install: applier reported 17, plan had 17 rules + readback from the host kernel: DOCKER-USER carries 9, INPUT carries 8 rules for 172.31.21.10 + +=== gate5f: 3. AFTER — the same probes, unchanged === + runsc -> own-network neighbour 172.31.22.10:8080 (MUST be denied): timeout + runsc -> host 192.168.5.15:49253 (MUST be denied): timeout + runc -> host 192.168.5.15:49253 (MUST be denied): timeout + runsc -> same-bridge neighbour 172.31.21.20:8080 (known unbindable): REACHED + +=== gate5f: 4. the public route must survive the policy === + health: HEALTHY 172.31.21.10 + PUBLIC-PASS dns=34.225.223.145 tls=200 verified=true + health: HEALTHY 172.31.21.10 + PUBLIC-PASS git 7fd1a60b01f91b314f59955a4e4d4e80d8edf11d + +=== gate5f: 5. teardown must leave the shared chains exactly as it found them === + host-rule teardown: applier reported 17, plan had 17 rules + readback from the host kernel: DOCKER-USER carries 0, INPUT carries 0 rules for 172.31.21.10 + chain depth after teardown: DOCKER-USER=1 (was 1) INPUT=1 (was 1) + CLEAN — no rule leaked (this is the failure mode the HostRules drop guard exists to prevent) + +=== gate5f: how to read this === +The fix is proved by section 3 differing from section 1 on the own-network and host legs. +The same-bridge leg is expected to stay REACHED: switched frames enter no chain on a host +without br_netfilter, which is WHY the product gives every job its own network rather than +trying to rule its way out of a shared one. It is measured here so that reason stays evidenced. +Section 4 must pass, or the policy costs the job the delivery route it exists to have. diff --git a/docs/gvisor-dns-delivery/evidence/gate5g-baseline-plan-from-origin-main.txt b/docs/gvisor-dns-delivery/evidence/gate5g-baseline-plan-from-origin-main.txt new file mode 100644 index 000000000..51b0d9de6 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5g-baseline-plan-from-origin-main.txt @@ -0,0 +1,24 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.40.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5g-baseline-repro-CONFOUNDED-runc-victim-20260910T0325Z.txt b/docs/gvisor-dns-delivery/evidence/gate5g-baseline-repro-CONFOUNDED-runc-victim-20260910T0325Z.txt new file mode 100644 index 000000000..ebc503838 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5g-baseline-repro-CONFOUNDED-runc-victim-20260910T0325Z.txt @@ -0,0 +1,25 @@ +=== gate5g: environment === +utc=2026-09-10T03:25:09Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 runsc=runsc version release-20260817.0 +baseline commit=b45f865 (origin/main) · plan rules=24 +host rules installed by this script: NONE (origin/main has no HostPolicy — grep says 0) +arrangement: ONE shared network 172.31.40.0/24, as a single [sandbox] network produced +chain depth (untouched by this gate): DOCKER-USER=1 INPUT=1 +live listeners: victim job 172.31.40.20:8080 (inside 172.16.0.0/12, which the baseline plan DROPs) · host 192.168.5.15:49255 + +=== gate5g: 1. cross-job on the baseline shared network, live victim === + [ns gate5g-attacker-1: applier=24/24 rules, kernel readback shows 2 rule(s) for 172.16.0.0/12] + runc -> victim 172.31.40.20:8080 (positive control: the plan MUST bind runc): timeout + [ns gate5g-attacker-2: applier=24/24 rules, kernel readback shows 2 rule(s) for 172.16.0.0/12] + runsc -> victim 172.31.40.20:8080 (the vulnerability): timeout + +=== gate5g: 2. private egress to the host itself, live listener === + [ns gate5g-attacker-3: applier=24/24 rules, kernel readback shows 2 rule(s) for 172.16.0.0/12] + runc -> host 192.168.5.15:49255 (positive control): timeout + [ns gate5g-attacker-4: applier=24/24 rules, kernel readback shows 2 rule(s) for 172.16.0.0/12] + runsc -> host 192.168.5.15:49255 (the vulnerability): REACHED + +=== gate5g: verdict === +baseline REACHED count (each is a containment failure on origin/main): 1 +BASELINE-VULNERABLE: CONFIRMED on b45f865 — the netns plan binds runc and not runsc, +with no host-side enforcement present to catch it. The branch does not introduce this. diff --git a/docs/gvisor-dns-delivery/evidence/gate5g-baseline-vulnerability-CONFIRMED-20260910T0330Z.txt b/docs/gvisor-dns-delivery/evidence/gate5g-baseline-vulnerability-CONFIRMED-20260910T0330Z.txt new file mode 100644 index 000000000..f432fc0be --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5g-baseline-vulnerability-CONFIRMED-20260910T0330Z.txt @@ -0,0 +1,27 @@ +=== gate5g: environment === +utc=2026-09-10T03:27:15Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 runsc=runsc version release-20260817.0 +baseline commit=b45f865 (origin/main) · plan rules=24 +host rules installed by this script: NONE (origin/main has no HostPolicy — grep says 0) +arrangement: ONE shared network 172.31.40.0/24, as a single [sandbox] network produced +chain depth (untouched by this gate): DOCKER-USER=1 INPUT=1 +victim liveness (from inside its own namespace): VICTIM-DEAD ECONNREFUSED +live listeners: victim job 172.31.40.20:8080 (inside 172.16.0.0/12, which the baseline plan DROPs) · host 192.168.5.15:49255 + +=== gate5g: 1. cross-job on the baseline shared network, live victim === + (valid only if the victim is SERVING above; a dead victim makes every timeout meaningless) + [ns gate5g-attacker-1: applier=24/24 rules, kernel readback shows 2 rule(s) for 172.16.0.0/12] + runc -> victim 172.31.40.20:8080 (positive control: the plan MUST bind runc): timeout + [ns gate5g-attacker-2: applier=24/24 rules, kernel readback shows 2 rule(s) for 172.16.0.0/12] + runsc -> victim 172.31.40.20:8080 (the vulnerability): REACHED + +=== gate5g: 2. private egress to the host itself, live listener === + [ns gate5g-attacker-3: applier=24/24 rules, kernel readback shows 2 rule(s) for 172.16.0.0/12] + runc -> host 192.168.5.15:49255 (positive control): timeout + [ns gate5g-attacker-4: applier=24/24 rules, kernel readback shows 2 rule(s) for 172.16.0.0/12] + runsc -> host 192.168.5.15:49255 (the vulnerability): REACHED + +=== gate5g: verdict === +baseline REACHED count (each is a containment failure on origin/main): 2 +BASELINE-VULNERABLE: CONFIRMED on b45f865 — the netns plan binds runc and not runsc, +with no host-side enforcement present to catch it. The branch does not introduce this. diff --git a/docs/gvisor-dns-delivery/evidence/gate5h-lifecycle-recycled-address-PASS-20260910T0332Z.txt b/docs/gvisor-dns-delivery/evidence/gate5h-lifecycle-recycled-address-PASS-20260910T0332Z.txt new file mode 100644 index 000000000..1fd69a6cf --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5h-lifecycle-recycled-address-PASS-20260910T0332Z.txt @@ -0,0 +1,38 @@ +=== gate5h: environment === +utc=2026-09-10T03:32:21Z +kernel=6.8.0-134-generic arch=aarch64 runsc=runsc version release-20260817.0 +chain depth at start (DOCKER-USER + INPUT): 2 +live host listener: 192.168.5.15:49256 + +=== gate5h: 1. job A establishes, rules appear, containment holds === + job A address (read from docker inspect): 172.31.55.2 + rules keyed to 172.31.55.2: before=0 after-install=17 + ok: install added 17 host rules + namespace health: HEALTHY 172.31.55.2 + runsc -> live host 192.168.5.15:49256: timeout + ok: A contained while its rules are installed + +=== gate5h: 2. job A tears down — rules must be GONE, network must be GONE === + rules keyed to 172.31.55.2 after teardown: 0 + ok: no leftover host rules + ok: network removed + +=== gate5h: 3. RECYCLED address — job B takes A's old address === + job B address: 172.31.55.2 (A's was 172.31.55.2) + ok: address genuinely recycled — this is the case that matters + rules keyed to 172.31.55.2 BEFORE B installs its own: 0 + ok: B inherits no stale firewall + runsc -> live host BEFORE B's rules (bare control, expect REACHED): REACHED + runsc -> live host AFTER B's rules: ENETUNREACH + ok: B contained by its own rules + +=== gate5h: 4. RECREATION — the same job id twice must not wedge === + first incarnation address: 172.31.55.2 + second incarnation address: 172.31.55.2 + ok: same job id came up twice, no 'network already exists' wedge + +=== gate5h: verdict === +chain depth: start=2 end=2 + ok: chains returned to starting depth — the gate left nothing behind +failing checks: 0 +GATE 5h: PASS diff --git a/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.55.2-install.txt b/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.55.2-install.txt new file mode 100644 index 000000000..608555190 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.55.2-install.txt @@ -0,0 +1,17 @@ +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 169.254.169.254/32 -j DROP +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 10.0.0.0/8 -j DROP +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 172.16.0.0/12 -j DROP +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 192.168.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 169.254.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 100.64.0.0/10 -j DROP +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 198.18.0.0/15 -j DROP +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 224.0.0.0/4 -j DROP +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 240.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.55.2/32 -d 10.0.0.0/8 -j DROP +iptables -I INPUT -s 172.31.55.2/32 -d 172.16.0.0/12 -j DROP +iptables -I INPUT -s 172.31.55.2/32 -d 192.168.0.0/16 -j DROP +iptables -I INPUT -s 172.31.55.2/32 -d 169.254.0.0/16 -j DROP +iptables -I INPUT -s 172.31.55.2/32 -d 100.64.0.0/10 -j DROP +iptables -I INPUT -s 172.31.55.2/32 -d 198.18.0.0/15 -j DROP +iptables -I INPUT -s 172.31.55.2/32 -d 224.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.55.2/32 -d 240.0.0.0/4 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.55.2-teardown.txt b/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.55.2-teardown.txt new file mode 100644 index 000000000..32a3befd0 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.55.2-teardown.txt @@ -0,0 +1,17 @@ +iptables -D INPUT -s 172.31.55.2/32 -d 240.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.55.2/32 -d 224.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.55.2/32 -d 198.18.0.0/15 -j DROP +iptables -D INPUT -s 172.31.55.2/32 -d 100.64.0.0/10 -j DROP +iptables -D INPUT -s 172.31.55.2/32 -d 169.254.0.0/16 -j DROP +iptables -D INPUT -s 172.31.55.2/32 -d 192.168.0.0/16 -j DROP +iptables -D INPUT -s 172.31.55.2/32 -d 172.16.0.0/12 -j DROP +iptables -D INPUT -s 172.31.55.2/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 240.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 224.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 198.18.0.0/15 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 100.64.0.0/10 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 169.254.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 192.168.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 172.16.0.0/12 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 169.254.169.254/32 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.56.2-install.txt b/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.56.2-install.txt new file mode 100644 index 000000000..3c9e64315 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.56.2-install.txt @@ -0,0 +1,17 @@ +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 169.254.169.254/32 -j DROP +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 10.0.0.0/8 -j DROP +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 172.16.0.0/12 -j DROP +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 192.168.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 169.254.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 100.64.0.0/10 -j DROP +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 198.18.0.0/15 -j DROP +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 224.0.0.0/4 -j DROP +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 240.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.56.2/32 -d 10.0.0.0/8 -j DROP +iptables -I INPUT -s 172.31.56.2/32 -d 172.16.0.0/12 -j DROP +iptables -I INPUT -s 172.31.56.2/32 -d 192.168.0.0/16 -j DROP +iptables -I INPUT -s 172.31.56.2/32 -d 169.254.0.0/16 -j DROP +iptables -I INPUT -s 172.31.56.2/32 -d 100.64.0.0/10 -j DROP +iptables -I INPUT -s 172.31.56.2/32 -d 198.18.0.0/15 -j DROP +iptables -I INPUT -s 172.31.56.2/32 -d 224.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.56.2/32 -d 240.0.0.0/4 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.56.2-teardown.txt b/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.56.2-teardown.txt new file mode 100644 index 000000000..330586ec8 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.56.2-teardown.txt @@ -0,0 +1,17 @@ +iptables -D INPUT -s 172.31.56.2/32 -d 240.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.56.2/32 -d 224.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.56.2/32 -d 198.18.0.0/15 -j DROP +iptables -D INPUT -s 172.31.56.2/32 -d 100.64.0.0/10 -j DROP +iptables -D INPUT -s 172.31.56.2/32 -d 169.254.0.0/16 -j DROP +iptables -D INPUT -s 172.31.56.2/32 -d 192.168.0.0/16 -j DROP +iptables -D INPUT -s 172.31.56.2/32 -d 172.16.0.0/12 -j DROP +iptables -D INPUT -s 172.31.56.2/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 240.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 224.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 198.18.0.0/15 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 100.64.0.0/10 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 169.254.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 192.168.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 172.16.0.0/12 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 169.254.169.254/32 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5i-fail-closed-FAIL-teardown-leak-20260910T0339Z.txt b/docs/gvisor-dns-delivery/evidence/gate5i-fail-closed-FAIL-teardown-leak-20260910T0339Z.txt new file mode 100644 index 000000000..2d17ff65e --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5i-fail-closed-FAIL-teardown-leak-20260910T0339Z.txt @@ -0,0 +1,36 @@ +=== gate5i: environment === +utc=2026-09-10T03:39:18Z +kernel=6.8.0-134-generic arch=aarch64 runsc=runsc version release-20260817.0 +chain depth at start: 2 +live host listener: 192.168.5.15:49257 +job address: 172.31.56.2 +rendered plan: 17 rules + +=== gate5i: A. applier cannot start (bad image) — nothing may be installed === + applier said: Run 'docker run --help' for more information + rules keyed to 172.31.56.2: 0 + ok: a non-starting applier installs nothing (establish() maps this to a hard error) + +=== gate5i: B. applier without NET_ADMIN — must not silently succeed === + applier said: apply-policy: namespace is PARTIALLY configured — destroy the holder, do not retry + rules keyed to 172.31.56.2: 0 + ok: no capability, no rules + +=== gate5i: C. TRUNCATED plan — does it apply cleanly and lie? === + feeding 17 rendered rules as a 9-line plan + applier exit=0 said=9 + rules actually in the kernel: 9 of 17 rendered + ok: CONFIRMED: a truncated plan exits 0 while under-installing — only the count reveals it + ok: count cross-check would refuse this job (9 != 17) + runsc -> live host 192.168.5.15:49257 with a PARTIAL firewall: REACHED + ok: a partially-installed policy leaves the job UNCONTAINED — refusing the job is the only safe move + +=== gate5i: D. teardown unwinds the partial install === + rules keyed to 172.31.56.2 after teardown: 9 + FAIL: 9 rule(s) survived teardown of a partial install + +=== gate5i: verdict === +chain depth: start=2 end=11 + FAIL: chains changed 2 -> 11 +failing checks: 2 +GATE 5i: FAIL diff --git a/docs/gvisor-dns-delivery/evidence/gate5i-fail-closed-PASS-after-fix-20260910T0347Z.txt b/docs/gvisor-dns-delivery/evidence/gate5i-fail-closed-PASS-after-fix-20260910T0347Z.txt new file mode 100644 index 000000000..e90305887 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5i-fail-closed-PASS-after-fix-20260910T0347Z.txt @@ -0,0 +1,41 @@ +=== gate5i: environment === +utc=2026-09-10T03:44:36Z +kernel=6.8.0-134-generic arch=aarch64 runsc=runsc version release-20260817.0 +chain depth at start: 2 +./gate5i.sh: line 59: /home/forge.guest/gate5i-resolv.conf: Permission denied +live host listener: 192.168.5.15:49257 +job address: 172.31.56.2 +rendered plan: 17 rules + +=== gate5i: A. applier cannot start (bad image) — nothing may be installed === + applier said: Run 'docker run --help' for more information + rules keyed to 172.31.56.2: 0 + ok: a non-starting applier installs nothing (establish() maps this to a hard error) + +=== gate5i: B. applier without NET_ADMIN — must not silently succeed === + applier said: apply-policy: namespace is PARTIALLY configured — destroy the holder, do not retry + rules keyed to 172.31.56.2: 0 + ok: no capability, no rules + +=== gate5i: C. TRUNCATED plan — does it apply cleanly and lie? === + feeding 17 rendered rules as a 9-line plan + applier exit=0 said=9 + rules actually in the kernel: 9 of 17 rendered + ok: CONFIRMED: a truncated plan exits 0 while under-installing — only the count reveals it + ok: count cross-check would refuse this job (9 != 17) + runsc -> live host 192.168.5.15:49257 with a PARTIAL firewall: REACHED + ok: a partially-installed policy leaves the job UNCONTAINED — refusing the job is the only safe move + +=== gate5i: D. teardown of a PARTIAL install === + D1: one-shot inverse plan (the pre-fix behaviour) + rules remaining: 9 (of 9 installed) + as expected: the applier aborts on the first never-created rule and removes nothing + D2: per-rule teardown (what HostRules::drop now does on the partial path) + removed 9 rule(s) one at a time; 0 remain + ok: the partial install came out — a missing rule no longer strands the present ones + +=== gate5i: verdict === +chain depth: start=2 end=2 + ok: chains returned to starting depth +failing checks: 0 +GATE 5i: PASS diff --git a/docs/gvisor-dns-delivery/evidence/gate5j-ipv6-finding-20260910T0347Z.txt b/docs/gvisor-dns-delivery/evidence/gate5j-ipv6-finding-20260910T0347Z.txt new file mode 100644 index 000000000..cdc68e005 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5j-ipv6-finding-20260910T0347Z.txt @@ -0,0 +1,22 @@ +utc=2026-09-10T03:47:21Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 +=== daemon.json === +{ + "runtimes": { + "runsc": { + "path": "/usr/local/bin/runsc" + } + } +}=== daemon ipv6 setting === +(docker info reports no ipv6 key) +=== a job-shaped holder on a per-job network === +--- ip -6 addr inside the holder namespace --- +1: lo: mtu 65536 state UNKNOWN qlen 1000 + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever +--- global (non-link-local) v6 addresses seen --- +0 +0 +--- does ip6tables even have a DOCKER-USER chain here? --- +-N DOCKER-USER +=== end === diff --git a/docs/gvisor-dns-delivery/evidence/gate5k-CONFOUNDED-harness-defect-20260910T0310Z.txt b/docs/gvisor-dns-delivery/evidence/gate5k-CONFOUNDED-harness-defect-20260910T0310Z.txt new file mode 100644 index 000000000..6decd5bba --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5k-CONFOUNDED-harness-defect-20260910T0310Z.txt @@ -0,0 +1,32 @@ +gate5k — unsupported daemon default runtime +kernel: Linux 6.8.0-134-generic aarch64 +docker: 29.1.3 +runsc: runsc version release-20260817.0 +ORIGINAL default runtime: runc + +listener address: 172.18.0.2 + +---- LEG 1 — control: daemon default is runc (the supported configuration) +default runtime now: runc +listener liveness: NOT-SERVING + holder runtime: runc + holder ifaces: exec failed + job: + job: <= no-answer + +---- LEG 2 — daemon default switched to runsc (the UNSUPPORTED configuration) +default runtime now: runsc +listener liveness: NOT-SERVING +listener down after the daemon restart — leg 2 is NO EVIDENCE + holder runtime: runsc + holder ifaces: executing processes for container + job: + job: <= no-answer + +---- VERDICT +Read leg 2's job line: a job that shows only 'lo' and cannot reach the live +listener FAILED CLOSED (no egress, no containment bypass). A job that prints +REACHED under the unsupported default FAILED OPEN and is a security defect. +failing_checks=2 +---- restoring /etc/docker/daemon.json +restored default runtime = runc diff --git a/docs/gvisor-dns-delivery/evidence/gate5k-v2-daemon-default-runtime-20260910T0325Z.txt b/docs/gvisor-dns-delivery/evidence/gate5k-v2-daemon-default-runtime-20260910T0325Z.txt new file mode 100644 index 000000000..177269f7e --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5k-v2-daemon-default-runtime-20260910T0325Z.txt @@ -0,0 +1,27 @@ +gate5k v2 — unsupported daemon default runtime +kernel: Linux 6.8.0-134-generic aarch64 +docker: 29.1.3 +runsc: runsc version release-20260817.0 +ORIGINAL default runtime: runc + +---- LEG 1 — control: daemon default is runc (the supported configuration) +default runtime now: runc +listener 172.18.0.2 liveness: NOT-SERVING +leg 1 target is dead — NO EVIDENCE + holder runtime: runc + job: ifaces= lo eth0 | routes=eth0 eth0 | target=no-answer + +---- LEG 2 — daemon default switched to runsc (the UNSUPPORTED configuration) +default runtime now: runsc +listener 172.18.0.2 liveness: NOT-SERVING +leg 2 target is dead — NO EVIDENCE, not a fail-closed result + holder runtime: runsc + job: ifaces= lo | routes=| target=no-answer + +---- VERDICT +Leg 2 with a SERVING listener: a job showing only 'lo', or refusing to run, +FAILED CLOSED — no egress and no containment bypass. A job printing REACHED +under the unsupported default FAILED OPEN and is a security defect. +failing_checks=2 (a non-zero count means legs above proved nothing) +---- restoring /etc/docker/daemon.json +restored default runtime = runc diff --git a/docs/gvisor-dns-delivery/evidence/gate5k-v3-PASS-daemon-default-20260910T0345Z.txt b/docs/gvisor-dns-delivery/evidence/gate5k-v3-PASS-daemon-default-20260910T0345Z.txt new file mode 100644 index 000000000..5e18161fc --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5k-v3-PASS-daemon-default-20260910T0345Z.txt @@ -0,0 +1,29 @@ +gate5k v3 — unsupported daemon default runtime +kernel: Linux 6.8.0-134-generic aarch64 +docker: 29.1.3 +runsc: runsc version release-20260817.0 +ORIGINAL default runtime: runc + +---- LEG 1 — control: daemon default is runc (the supported configuration) +default runtime now: runc +listener 172.18.0.2 liveness: SERVING + holder runtime: runc + job: HEALTHY 172.18.0.3 | target=REACHED + ^ expected here: HEALTHY and target=REACHED. This leg is the + positive control: it proves the harness CAN observe reachability, so + leg 2's silence means something. + +---- LEG 2 — daemon default switched to runsc (the UNSUPPORTED configuration) +default runtime now: runsc +listener 172.18.0.2 liveness: SERVING + holder runtime: runsc + job: SICK no-address | target=ENETUNREACH + +---- VERDICT +With a SERVING listener in both legs: leg 1 REACHED and leg 2 SICK/denied +means the unsupported default FAILS CLOSED — no egress path to contain, an +availability failure rather than a containment bypass. Leg 2 printing +target=REACHED would mean it FAILS OPEN and is a security defect. +failing_checks=0 (non-zero means a leg above proved nothing) +---- restoring /etc/docker/daemon.json +restored default runtime = runc diff --git a/docs/gvisor-dns-delivery/evidence/run-all-gates-proof-run-20260910T0316Z.txt b/docs/gvisor-dns-delivery/evidence/run-all-gates-proof-run-20260910T0316Z.txt new file mode 100644 index 000000000..acffc1254 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/run-all-gates-proof-run-20260910T0316Z.txt @@ -0,0 +1,15 @@ +=== gVisor DNS/delivery gates === +utc=2026-09-10T03:15:13Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 runsc=runsc version release-20260817.0 +per-gate timeout=420s total budget=800s evidence=/home/forge.guest/gate-evidence +NOTE: results are labelled by architecture on purpose — x86_64 is NOT covered by this run. + +--- gate1: running (timeout 420s) -> /home/forge.guest/gate-evidence/gate1-20260910T031513Z.log +--- gate5f: running (timeout 420s) -> /home/forge.guest/gate-evidence/gate5f-20260910T031522Z.log +=== summary === +utc=2026-09-10T03:16:14Z +arch=aarch64 runsc=runsc version release-20260817.0 +gate1 PASS exit=0 log=gate1-20260910T031513Z.log +gate5f PASS exit=0 log=gate5f-20260910T031522Z.log +total=61s gates=2 not-passing=0 +ALL GATES: PASS diff --git a/docs/gvisor-dns-delivery/evidence/run-all-gates-proof-summary-20260910T0316Z.txt b/docs/gvisor-dns-delivery/evidence/run-all-gates-proof-summary-20260910T0316Z.txt new file mode 100644 index 000000000..54cf7acaa --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/run-all-gates-proof-summary-20260910T0316Z.txt @@ -0,0 +1,7 @@ +=== summary === +utc=2026-09-10T03:16:14Z +arch=aarch64 runsc=runsc version release-20260817.0 +gate1 PASS exit=0 log=gate1-20260910T031513Z.log +gate5f PASS exit=0 log=gate5f-20260910T031522Z.log +total=61s gates=2 not-passing=0 +ALL GATES: PASS diff --git a/docs/gvisor-dns-delivery/scripts/gate1-repro.sh b/docs/gvisor-dns-delivery/scripts/gate1-repro.sh new file mode 100755 index 000000000..8c0fa4528 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate1-repro.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Gate 1: reproduce the named-bridge DNS failure under runsc, with runc as a +# DIAGNOSTIC CONTROL only. Runs inside the disposable VM. Bounded: every docker +# run carries a timeout and the network is removed on exit. +# +# runc appears here to isolate the variable. It is never a fallback path for +# production jobs. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NET="${NET:-maxplayer-dns-repro}" +HOST_TARGET="${HOST_TARGET:-relay.maxplayer.ai}" +OUT="${OUT:-$HOME/gate1-evidence.txt}" + +exec > >(tee "${OUT}") 2>&1 + +echo "=== gate1: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m)" +. /etc/os-release && echo "os=${PRETTY_NAME}" +echo "docker=$(sudo docker version --format '{{.Server.Version}}')" +echo "runsc=$(runsc --version | head -1)" +echo "image=${IMAGE}" +echo "image_digest=$(sudo docker image inspect "${IMAGE}" --format '{{index .RepoDigests 0}}')" +echo "image_arch=$(sudo docker image inspect "${IMAGE}" --format '{{.Architecture}}/{{.Os}}')" + +cleanup() { sudo docker network rm "${NET}" >/dev/null 2>&1 || true; } +trap cleanup EXIT +sudo docker network rm "${NET}" >/dev/null 2>&1 || true +sudo docker network create "${NET}" >/dev/null +echo "network=${NET} subnet=$(sudo docker network inspect "${NET}" --format '{{(index .IPAM.Config 0).Subnet}}')" + +# One probe body, run identically under both runtimes: resolve, then report the +# resolver the container was actually handed. +PROBE='const dns=require("dns");const fs=require("fs"); +console.log("resolv.conf:", fs.readFileSync("/etc/resolv.conf","utf8").trim().replace(/\n/g,"|")); +dns.lookup(process.argv[1],(e,a)=>{console.log("lookup:", e?("ERR "+e.code):("OK "+a));process.exitCode=e?1:0});' + +run_probe() { + local runtime="$1" + echo + echo "=== gate1: dns lookup under --runtime ${runtime} ===" + sudo timeout 90 docker run --rm --runtime "${runtime}" --network "${NET}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + --entrypoint node "${IMAGE}" -e "${PROBE}" "${HOST_TARGET}" + echo "exit=$?" +} + +run_probe runsc +run_probe runc + +echo +echo "=== gate1: raw udp/53 to the embedded resolver under runsc ===" +sudo timeout 90 docker run --rm --runtime runsc --network "${NET}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + --entrypoint node "${IMAGE}" -e ' +const dgram=require("dgram");const s=dgram.createSocket("udp4"); +const q=Buffer.from("abcd01000001000000000000057265" + + "6c6179096d6178706c61796572026169000001" + "0001","hex"); +const t=setTimeout(()=>{console.log("udp53: TIMEOUT (no answer from 127.0.0.11)");s.close();process.exitCode=1;},8000); +s.on("message",(m)=>{clearTimeout(t);console.log("udp53: ANSWER "+m.length+" bytes");s.close();}); +s.on("error",(e)=>{clearTimeout(t);console.log("udp53: ERR "+e.code);s.close();process.exitCode=1;}); +s.send(q,53,"127.0.0.11");' +echo "exit=$?" + +echo +echo "=== gate1: done ===" diff --git a/docs/gvisor-dns-delivery/scripts/gate2-namespace-dns-tls.sh b/docs/gvisor-dns-delivery/scripts/gate2-namespace-dns-tls.sh new file mode 100755 index 000000000..fbddccb46 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate2-namespace-dns-tls.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Gate 2: DNS **and** certificate-validated TLS from inside the REAL shared job +# namespace — holder + sidecar-applied policy + job — under runsc, non-root, +# cap-drop ALL, no-new-privileges. Proven on a fresh namespace and again after +# the namespace is destroyed and recreated. +# +# Runs inside the disposable gvisor-repro VM. Bounded: every docker run carries a +# timeout, and every container and the network are removed on exit. +# +# The iptables plan is NOT transcribed here. It is rendered by the product's own +# `NetPolicy` (cargo run -p maxplayer-core --example render_net_plan) and passed +# in via PLAN_FILE, so this gate cannot pass against a firewall the product no +# longer builds. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +NET="${NET:-maxplayer-dns-gate2}" +HOST_TARGET="${HOST_TARGET:-relay.maxplayer.ai}" +RESOLVER="${RESOLVER:-1.1.1.1}" +SUBNET="${SUBNET:-172.31.7.0/24}" +GATEWAY="${GATEWAY:-172.31.7.1}" +PLAN_FILE="${PLAN_FILE:-$HOME/gate2-plan.txt}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate2-resolv.conf}" +OUT="${OUT:-$HOME/gate2-evidence.txt}" +HOLDER="gate2-holder" +# The containment plane (holder + sidecar) runs on the HOST runtime; only the job runs +# under gVisor. Measured, not preference: a runsc container joining a runsc holder's +# network namespace sees `lo` ONLY — no eth0, no route, every lookup EAI_AGAIN — because +# a gVisor sandbox's netstack lives inside that sandbox and cannot be entered by a second +# one. Joining a runc holder, a runsc job gets the holder's own interface and address +# (172.31.11.2 in both, measured), so the host kernel's rules govern its traffic. The +# sidecar needs the host runtime for a second reason: iptables-nft inside gVisor fails +# `Failed to initialize nft: Protocol not supported`. +HOLDER_RUNTIME="${HOLDER_RUNTIME:-runc}" +JOB_RUNTIME="${JOB_RUNTIME:-runsc}" + +exec > >(tee "${OUT}") 2>&1 + +fail=0 + +echo "=== gate2: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m)" +. /etc/os-release && echo "os=${PRETTY_NAME}" +echo "docker=$(sudo docker version --format '{{.Server.Version}}')" +echo "runsc=$(runsc --version | head -1)" +echo "holder_runtime=${HOLDER_RUNTIME} job_runtime=${JOB_RUNTIME}" +echo "image=${IMAGE}" +echo "image_digest=$(sudo docker image inspect "${IMAGE}" --format '{{index .RepoDigests 0}}')" +echo "netfilter_digest=$(sudo docker image inspect "${NETFILTER_IMAGE}" --format '{{index .RepoDigests 0}}')" +echo "resolver=${RESOLVER}" + +# The resolver file the product would write, in the product's format. +cat > "${RESOLV_FILE}" </dev/null 2>&1 || true +} +cleanup() { + teardown_ns + sudo docker network rm "${NET}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +# A FIXED subnet, so the gateway in the rendered plan and the gateway of the network +# the job actually joins are the same address. Letting docker pick would render the +# proxy pinhole for one address while the job reaches the host at another — rules +# that look right in every log and route nothing. +sudo docker network rm "${NET}" >/dev/null 2>&1 || true +sudo docker network create --subnet "${SUBNET}" --gateway "${GATEWAY}" "${NET}" >/dev/null +echo "network=${NET} subnet=$(sudo docker network inspect "${NET}" --format '{{(index .IPAM.Config 0).Subnet}}') gateway=${GATEWAY}" + +# The job probe: resolve, then complete a TLS handshake whose certificate chain is +# VERIFIED against the image's own trust store. `rejectUnauthorized` stays default +# (true) and the peer certificate is printed, so a passing gate cannot be a +# handshake that skipped verification. +PROBE='const dns=require("dns"),https=require("https"),fs=require("fs"); +const host=process.argv[1]; +console.log("resolv.conf:", fs.readFileSync("/etc/resolv.conf","utf8").trim().split("\n").filter(l=>!l.startsWith("#")).join("|")); +dns.lookup(host,(e,a)=>{ + if(e){console.log("lookup: ERR "+e.code);process.exit(1);} + console.log("lookup: OK "+a); + const req=https.request({host,port:443,path:"/",method:"HEAD",timeout:15000},(res)=>{ + const c=res.socket.getPeerCertificate(); + console.log("tls: "+res.statusCode+" cert-verified subject="+(c&&c.subject&&c.subject.CN)+" issuer="+(c&&c.issuer&&c.issuer.CN)+" authorized="+res.socket.authorized); + process.exit(res.socket.authorized?0:1); + }); + req.on("timeout",()=>{console.log("tls: TIMEOUT");process.exit(1);}); + req.on("error",(err)=>{console.log("tls: ERR "+err.code+" "+err.message);process.exit(1);}); + req.end(); +});' + +# A second probe proving the containment the DNS pinhole must not have widened: +# the cloud metadata address stays denied while public egress works. +DENY_PROBE='const net=require("net"); +const s=net.connect({host:"169.254.169.254",port:80,timeout:6000}); +s.on("connect",()=>{console.log("metadata: REACHED (containment broken)");process.exit(1);}); +s.on("timeout",()=>{console.log("metadata: denied (timeout)");process.exit(0);}); +s.on("error",(e)=>{console.log("metadata: denied ("+e.code+")");process.exit(0);});' + +establish_namespace() { + local label="$1" + echo + echo "=== gate2/${label}: establish the shared job namespace ===" + # Holder: owns the namespace, holds no capability, runs as nobody, read-only. + sudo timeout 120 docker run --detach --name "${HOLDER}" --runtime "${HOLDER_RUNTIME}" \ + --network "${NET}" --read-only --cap-drop ALL --security-opt no-new-privileges \ + --user 65534:65534 --entrypoint sleep "${IMAGE}" infinity >/dev/null + echo "holder=${HOLDER} started=$?" + + # Sidecar: the ONLY container handed NET_ADMIN, scoped to the holder's namespace, + # gone before the job starts. Same runtime as the holder, so it writes into the netns + # the job will actually join. + echo "--- sidecar applies the rendered plan ---" + sudo timeout 120 docker run --rm --interactive --runtime "${HOLDER_RUNTIME}" \ + --network "container:${HOLDER}" --cap-drop ALL --cap-add NET_ADMIN \ + --security-opt no-new-privileges "${NETFILTER_IMAGE}" < "${PLAN_FILE}" + echo "sidecar_exit=$?" + + # Read the rules back out of the namespace with a DIFFERENT container running a + # DIFFERENT verb, because the question is what the netstack holds and not whether + # the installer believes it succeeded. + echo "--- readback (iptables -S) ---" + sudo timeout 60 docker run --rm --runtime "${HOLDER_RUNTIME}" --network "container:${HOLDER}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + --entrypoint iptables "${NETFILTER_IMAGE}" -S OUTPUT + echo "readback_exit=$?" +} + +run_job() { + local label="$1" + echo + echo "=== gate2/${label}: job in the shared namespace — dns + verified tls ===" + sudo timeout 120 docker run --rm --runtime "${JOB_RUNTIME}" --network "container:${HOLDER}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" \ + --entrypoint node "${IMAGE}" -e "${PROBE}" "${HOST_TARGET}" + local rc=$? + echo "job_exit=${rc}" + [ "${rc}" -eq 0 ] || fail=1 + + echo "--- containment still holds: metadata address ---" + sudo timeout 60 docker run --rm --runtime "${JOB_RUNTIME}" --network "container:${HOLDER}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" \ + --entrypoint node "${IMAGE}" -e "${DENY_PROBE}" + local drc=$? + echo "metadata_exit=${drc}" + [ "${drc}" -eq 0 ] || fail=1 +} + +establish_namespace fresh +run_job fresh + +echo +echo "=== gate2: destroy the namespace and rebuild it ===" +teardown_ns +sleep 2 +establish_namespace recreated +run_job recreated + +echo +echo "=== gate2: verdict ===" +if [ "${fail}" -eq 0 ]; then + echo "GATE2: PASS (fresh and recreated namespaces both resolved and completed verified TLS)" +else + echo "GATE2: FAIL" +fi +exit "${fail}" diff --git a/docs/gvisor-dns-delivery/scripts/gate4-container-git-delivery.sh b/docs/gvisor-dns-delivery/scripts/gate4-container-git-delivery.sh new file mode 100755 index 000000000..ba5f971fa --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate4-container-git-delivery.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# Gate 4: REAL git delivery originating INSIDE the job sandbox — not a mock, not a +# host-side upload. Two legs, both from the job container under runsc, non-root, +# cap-drop ALL, no-new-privileges, inside the shared job namespace: +# +# READ — clone a real public repository over HTTPS, and prove the delivered +# commit is the remote's real one by comparing against `git ls-remote` +# taken independently OUTSIDE the sandbox. +# WRITE — commit in the container and PUSH to a disposable bare remote that +# lives outside the container, then prove the remote's ref now holds +# exactly the hash the container produced. +# +# The write remote is reached at the namespace gateway on a port inside the +# policy's proxy pinhole — the one host-facing hole the design already opens — +# so the push crosses the job's egress policy rather than sidestepping it. +# +# ⛔ No credential is used, needed, or logged. A container-side push to a +# credentialed remote would require putting a secret inside a stranger's +# sandbox, which is the one thing the whole containment design exists to +# prevent; the write leg therefore uses an unauthenticated disposable remote. +# Named as a limitation in the runlog rather than papered over. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +NET="${NET:-maxplayer-dns-gate4}" +SUBNET="${SUBNET:-172.31.8.0/24}" +GATEWAY="${GATEWAY:-172.31.8.1}" +RESOLVER="${RESOLVER:-1.1.1.1}" +# Inside the proxy pinhole range the rendered plan opens (49200-49299). +REMOTE_PORT="${REMOTE_PORT:-49250}" +PUBLIC_REPO="${PUBLIC_REPO:-https://github.com/octocat/Hello-World.git}" +PLAN_FILE="${PLAN_FILE:-$HOME/gate4-plan.txt}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate4-resolv.conf}" +REMOTE_ROOT="${REMOTE_ROOT:-$HOME/gate4-remote}" +OUT="${OUT:-$HOME/gate4-evidence.txt}" +HOLDER="gate4-holder" +HOLDER_RUNTIME="${HOLDER_RUNTIME:-runc}" +JOB_RUNTIME="${JOB_RUNTIME:-runsc}" + +exec > >(tee "${OUT}") 2>&1 +fail=0 + +echo "=== gate4: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m)" +. /etc/os-release && echo "os=${PRETTY_NAME}" +echo "docker=$(sudo docker version --format '{{.Server.Version}}')" +echo "runsc=$(runsc --version | head -1)" +echo "holder_runtime=${HOLDER_RUNTIME} job_runtime=${JOB_RUNTIME}" +echo "image_digest=$(sudo docker image inspect "${IMAGE}" --format '{{index .RepoDigests 0}}')" +echo "container_git=$(sudo docker run --rm --entrypoint git "${IMAGE}" --version)" +echo "public_repo=${PUBLIC_REPO}" + +cleanup() { + sudo docker rm -f "${HOLDER}" >/dev/null 2>&1 || true + sudo docker network rm "${NET}" >/dev/null 2>&1 || true + pkill -f "git-daemon.*${REMOTE_PORT}" >/dev/null 2>&1 || true + pkill -f "git daemon.*${REMOTE_PORT}" >/dev/null 2>&1 || true + rm -rf "${REMOTE_ROOT}" +} +trap cleanup EXIT +cleanup + +cat > "${RESOLV_FILE}" </dev/null +echo "network=${NET} subnet=${SUBNET} gateway=${GATEWAY}" + +# The CONTROL for the read leg, taken outside the sandbox: what the remote really holds. +echo +echo "=== gate4: control — the remote's real HEAD, read from the VM (outside any container) ===" +CONTROL_HEAD="$(git ls-remote "${PUBLIC_REPO}" HEAD | awk '{print $1}')" +echo "control_head=${CONTROL_HEAD}" +[ -n "${CONTROL_HEAD}" ] || { echo "GATE4: FAIL (no control hash)"; exit 1; } + +# The disposable write remote: a bare repo outside every container, served on the +# gateway address at a port inside the policy's proxy pinhole. +echo +echo "=== gate4: disposable write remote ===" +mkdir -p "${REMOTE_ROOT}" +git init --bare --quiet "${REMOTE_ROOT}/answer.git" +git --git-dir="${REMOTE_ROOT}/answer.git" config http.receivepack true +setsid git daemon --reuseaddr --listen="${GATEWAY}" --port="${REMOTE_PORT}" \ + --base-path="${REMOTE_ROOT}" --export-all --enable=receive-pack \ + >/dev/null 2>&1 /dev/null +echo "holder_started=$?" +sudo timeout 120 docker run --rm --interactive --runtime "${HOLDER_RUNTIME}" \ + --network "container:${HOLDER}" --cap-drop ALL --cap-add NET_ADMIN \ + --security-opt no-new-privileges "${NETFILTER_IMAGE}" < "${PLAN_FILE}" +echo "sidecar_exit=$?" + +# One payload, both legs, run as the job: clone over https, commit, push to the +# remote outside the container, and print the hashes for comparison. +PAYLOAD=' +set -e +export HOME=/tmp GIT_TERMINAL_PROMPT=0 +cd /tmp +echo "--- read leg: clone over https from inside the sandbox ---" +# A FULL clone, not --depth 1: a shallow history cannot be pushed on ("shallow update +# not allowed"), and the write leg is half the gate. The repository is deliberately tiny. +git clone --quiet "$1" work +cd work +echo "delivered_head=$(git rev-parse HEAD)" +echo "--- write leg: commit here, push to the remote outside this container ---" +git config user.email job@sandbox.invalid +git config user.name "sandbox job" +date -u +%s > answer.txt +git add answer.txt +git commit --quiet -m "answer from the sandboxed job" +echo "answer_commit=$(git rev-parse HEAD)" +git push --quiet "$2" HEAD:refs/heads/answer +echo "push_exit=$?" +' + +echo +echo "=== gate4: job container (runsc, non-root, cap-drop ALL) ===" +JOB_OUT="$(sudo timeout 180 docker run --rm --runtime "${JOB_RUNTIME}" \ + --network "container:${HOLDER}" --user 65534:65534 --cap-drop ALL \ + --security-opt no-new-privileges -v "${RESOLV_FILE}:/etc/resolv.conf:ro" \ + --entrypoint sh "${IMAGE}" -c "${PAYLOAD}" gate4 \ + "${PUBLIC_REPO}" "git://${GATEWAY}:${REMOTE_PORT}/answer.git" 2>&1)" +echo "${JOB_OUT}" +echo "job_exit=$?" + +DELIVERED_HEAD="$(printf '%s\n' "${JOB_OUT}" | sed -n 's/^delivered_head=//p')" +ANSWER_COMMIT="$(printf '%s\n' "${JOB_OUT}" | sed -n 's/^answer_commit=//p')" + +echo +echo "=== gate4: verdict ===" +echo "control_head=${CONTROL_HEAD}" +echo "delivered_head=${DELIVERED_HEAD}" +if [ -n "${DELIVERED_HEAD}" ] && [ "${DELIVERED_HEAD}" = "${CONTROL_HEAD}" ]; then + echo "READ: PASS — the sandbox delivered the remote's real HEAD" +else + echo "READ: FAIL — delivered hash does not match the remote's real HEAD" + fail=1 +fi + +# --verify, so a MISSING ref is empty rather than the string "refs/heads/answer" — which +# would otherwise be compared against a hash and merely look like a mismatch. +REMOTE_HASH="$(git --git-dir="${REMOTE_ROOT}/answer.git" rev-parse --verify -q refs/heads/answer 2>/dev/null)" +echo "answer_commit=${ANSWER_COMMIT}" +echo "remote_hash=${REMOTE_HASH}" +if [ -n "${ANSWER_COMMIT}" ] && [ "${ANSWER_COMMIT}" = "${REMOTE_HASH}" ]; then + echo "WRITE: PASS — the commit made inside the sandbox reached the remote, hash matches" +else + echo "WRITE: FAIL — the remote does not hold the container's commit" + fail=1 +fi + +if [ "${fail}" -eq 0 ]; then + echo "GATE4: PASS" +else + echo "GATE4: FAIL" +fi +exit "${fail}" diff --git a/docs/gvisor-dns-delivery/scripts/gate5-denial-and-concurrent-success.sh b/docs/gvisor-dns-delivery/scripts/gate5-denial-and-concurrent-success.sh new file mode 100755 index 000000000..ee045caa1 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5-denial-and-concurrent-success.sh @@ -0,0 +1,269 @@ +#!/usr/bin/env bash +# Gate 5: denial holds, and concurrent jobs still deliver. +# +# This is a REWRITE. The first version of this script produced a FAIL that is kept in +# evidence/, and it was unsound in three ways that the sub-experiments (gate5b–gate5f) +# then exposed. All three are fixed here, and naming them is part of the gate: +# +# 1. It probed addresses where NOTHING LISTENED and read the resulting timeouts as +# denial. Absence and enforcement are indistinguishable that way — the same error +# cost gate 2 its "metadata denied" line. Every denial leg below is either aimed at +# a LIVE listener, or reported as NO EVIDENCE unless bare and ruled runs DIFFER. +# 2. It reused one namespace across gVisor probes. The namespace is single-use for +# gVisor: runsc takes the addresses into its netstack and never gives them back, so +# every later leg ran in a namespace with `lo` only and "passed" by being broken. +# One gVisor container per namespace here, with a health check beside every leg. +# 3. It double-counted a request timeout and scored a SUCCESS as PUBLIC-FAIL. +# +# The rules under test are rendered by the PRODUCT (`--example render_net_plan` and +# `--example render_host_plan`), never transcribed here. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +PLAN_DIR="${PLAN_DIR:-$HOME/gate5-plans}" +RESOLVER="${RESOLVER:-1.1.1.1}" +HOST_LAN="${HOST_LAN:-192.168.5.15}" +HOST_PORT="${HOST_PORT:-49254}" +PUBLIC_REPO="${PUBLIC_REPO:-https://github.com/octocat/Hello-World.git}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate5-resolv.conf}" +HOSTS_FILE="${HOSTS_FILE:-$HOME/gate5-hosts}" +OUT="${OUT:-$HOME/gate5-evidence.txt}" +DENIED_NAME="denied-lan.maxplayer.test" + +exec > >(tee "${OUT}") 2>&1 +CUR_NS="" +NS_SEQ=0 +declare -a RULES_UP=() +FAILURES=0 +note_fail() { FAILURES=$((FAILURES + 1)); } + +for tag in d n c1 c2 c3; do + for f in "plan-${tag}.txt" "host-${tag}.txt" "host-${tag}-teardown.txt"; do + [ -s "${PLAN_DIR}/${f}" ] || { echo "MISSING rendered plan ${PLAN_DIR}/${f}"; exit 2; } + done +done + +# tag -> network, subnet, gateway, pinned address +net_of() { echo "maxplayer-dns-gate5-$1"; } +subnet_of() { case "$1" in d) echo 172.31.30.0/24;; n) echo 172.31.34.0/24;; c1) echo 172.31.31.0/24;; c2) echo 172.31.32.0/24;; c3) echo 172.31.33.0/24;; esac; } +gw_of() { case "$1" in d) echo 172.31.30.1;; n) echo 172.31.34.1;; c1) echo 172.31.31.1;; c2) echo 172.31.32.1;; c3) echo 172.31.33.1;; esac; } +addr_of() { case "$1" in d) echo 172.31.30.10;; n) echo 172.31.34.10;; c1) echo 172.31.31.10;; c2) echo 172.31.32.10;; c3) echo 172.31.33.10;; esac; } + +# The stale-plan guard from gate5f: a host plan keyed to the wrong address denies some +# other container and leaves this job open, and nothing in the run would show it. +for tag in d n c1 c2 c3; do + grep -q -- "-s $(addr_of "${tag}")/32" "${PLAN_DIR}/host-${tag}.txt" || + { echo "REFUSING TO RUN: host-${tag}.txt is not keyed to $(addr_of "${tag}")"; exit 2; } +done + +echo "=== gate5: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) docker=$(sudo docker version --format '{{.Server.Version}}') runsc=$(runsc --version | head -1)" +echo "docker default-runtime=$(sudo docker info --format '{{.DefaultRuntime}}') · br_netfilter=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables 2>/dev/null || echo absent)" +BASE_DU="$(sudo iptables -S DOCKER-USER | wc -l)"; BASE_IN="$(sudo iptables -S INPUT | wc -l)" +echo "chain depth before anything: DOCKER-USER=${BASE_DU} INPUT=${BASE_IN}" + +apply_host() { # tag install|teardown + local tag="$1" verb="$2" file="${PLAN_DIR}/host-$1.txt" + [ "${verb}" = teardown ] && file="${PLAN_DIR}/host-$1-teardown.txt" + local applied expected + applied="$(sudo timeout 120 docker run --rm --interactive --network host \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${file}" 2>&1 | tr -d '\r' | tail -1)" + expected="$(wc -l < "${file}" | tr -d ' ')" + local landed; landed="$(( $(sudo iptables -S DOCKER-USER | grep -c -- "-s $(addr_of "${tag}")/32") + $(sudo iptables -S INPUT | grep -c -- "-s $(addr_of "${tag}")/32") ))" + echo " host policy ${verb} for $(addr_of "${tag}"): applier=${applied}/${expected}, host kernel now carries ${landed}" + if [ "${verb}" = install ]; then RULES_UP+=("${tag}"); else + local keep=(); local t; for t in "${RULES_UP[@]:-}"; do [ -n "${t}" ] && [ "${t}" != "${tag}" ] && keep+=("${t}"); done + RULES_UP=("${keep[@]:-}") + fi +} +drop_ns() { [ -n "${CUR_NS}" ] && { sudo docker rm -f "${CUR_NS}" >/dev/null 2>&1; CUR_NS=""; }; return 0; } +cleanup() { + drop_ns + local t; for t in "${RULES_UP[@]:-}"; do [ -n "${t}" ] && apply_host "${t}" teardown >/dev/null 2>&1; done + sudo docker rm -f gate5-neigh-holder gate5-neigh $(sudo docker ps -aq --filter "name=gate5-") >/dev/null 2>&1 || true + for t in d n c1 c2 c3; do sudo docker network rm "$(net_of "${t}")" >/dev/null 2>&1; done + pkill -f "gate5-host-listener" >/dev/null 2>&1 || true +} +trap cleanup EXIT +cleanup + +rm -f "${RESOLV_FILE}" "${HOSTS_FILE}" +printf 'nameserver %s\noptions timeout:2 attempts:2\n' "${RESOLVER}" > "${RESOLV_FILE}" +# A name that resolves INTO a denied range, without depending on a third-party wildcard +# DNS service being up. The point of the leg is that reaching a denied address BY NAME is +# denied too, which no earlier gate measured. +printf '127.0.0.1 localhost\n%s %s\n' "${HOST_LAN}" "${DENIED_NAME}" > "${HOSTS_FILE}" +chmod 0444 "${RESOLV_FILE}" "${HOSTS_FILE}" + +for t in d n c1 c2 c3; do + sudo docker network create --subnet "$(subnet_of "${t}")" --gateway "$(gw_of "${t}")" "$(net_of "${t}")" >/dev/null +done + +setsid python3 -c " +import socket +s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) +s.bind(('0.0.0.0',${HOST_PORT})); s.listen(16) # gate5-host-listener +while True: + c,_=s.accept(); c.sendall(b'REACHED'); c.close() +" >/dev/null 2>&1 /dev/null +sudo timeout 60 docker run --detach --name gate5-neigh --network "container:gate5-neigh-holder" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges --entrypoint node "${IMAGE}" \ + -e "require('http').createServer((q,r)=>r.end('REACHED')).listen(8080,'0.0.0.0')" >/dev/null +sleep 3 +echo "live listeners: host ${HOST_LAN}:${HOST_PORT} · neighbour job $(addr_of n):8080 · name ${DENIED_NAME} -> ${HOST_LAN}" + +in_ns() { # runtime script args... + local rt="$1"; shift; local script="$1"; shift + sudo timeout 180 docker run --rm --runtime "${rt}" --network "container:${CUR_NS}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" -v "${HOSTS_FILE}:/etc/hosts:ro" \ + --entrypoint node "${IMAGE}" -e "${script}" "$@" 2>&1 | tr -d '\r' | tail -1 +} +HEALTH=' +const os=require("os"),dns=require("dns"); +const v4=Object.values(os.networkInterfaces()).flat().filter(x=>x&&x.family==="IPv4"&&!x.internal); +if(!v4.length){console.log("SICK no-address");process.exit(0);} +dns.lookup("relay.maxplayer.ai",(e,a)=>console.log(e?("SICK dns-"+e.code):("HEALTHY "+v4[0].address))); +' +PROBE=' +const net=require("net"); +const s=net.connect({host:process.argv[1],port:Number(process.argv[2]),timeout:8000}); +let said=false; const say=(w)=>{ if(!said){said=true;console.log(w);} s.destroy(); }; +s.on("connect",()=>say("REACHED")); +s.on("timeout",()=>say("timeout")); +s.on("error",(e)=>say(e.code)); +' +V6=' +const os=require("os"); +const v6=Object.values(os.networkInterfaces()).flat().filter(x=>x&&x.family==="IPv6"&&!x.internal); +console.log(v6.length?("HAS-V6 "+v6.map(x=>x.address).join(",")):"NO-V6"); +' +fresh_ns() { # tag + drop_ns + NS_SEQ=$((NS_SEQ + 1)); CUR_NS="gate5-ns-$1-${NS_SEQ}" + sudo timeout 120 docker run --detach --name "${CUR_NS}" --network "$(net_of "$1")" \ + --ip "$(addr_of "$1")" --read-only --cap-drop ALL --security-opt no-new-privileges \ + --user 65534:65534 --entrypoint sleep "${IMAGE}" infinity >/dev/null + sudo timeout 120 docker run --rm --interactive --network "container:${CUR_NS}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${PLAN_DIR}/plan-$1.txt" >/dev/null +} +probe_leg() { # label expect host port + local label="$1" expect="$2" host="$3" port="$4" + fresh_ns d + local h; h="$(in_ns runc "${HEALTH}")" + case "${h}" in + HEALTHY*) ;; + *) echo " ${label}: UNSOUND — ${h}"; note_fail; return ;; + esac + local got; got="$(in_ns runsc "${PROBE}" "${host}" "${port}")" + if [ "${expect}" = denied ]; then + case "${got}" in + REACHED) echo " ${label}: ${got} *** FAIL, expected denial ***"; note_fail ;; + *) echo " ${label}: ${got} (denied)" ;; + esac + else + echo " ${label}: ${got}" + fi +} + +echo +echo "=== gate5: 1. denial, BEFORE the host policy (so each leg has a baseline) ===" +probe_leg "runsc -> host ${HOST_LAN}:${HOST_PORT} (live)" measure "${HOST_LAN}" "${HOST_PORT}" +probe_leg "runsc -> ${DENIED_NAME}:${HOST_PORT} (live, by name)" measure "${DENIED_NAME}" "${HOST_PORT}" +probe_leg "runsc -> neighbour job $(addr_of n):8080 (live)" measure "$(addr_of n)" 8080 +probe_leg "runsc -> 169.254.169.254:80 (nothing listens)" measure 169.254.169.254 80 + +echo +echo "=== gate5: 2. the product's host policy, installed for the probe namespace ===" +apply_host d install + +echo +echo "=== gate5: 3. denial, AFTER — every one of these MUST be denied ===" +probe_leg "runsc -> host ${HOST_LAN}:${HOST_PORT} (live)" denied "${HOST_LAN}" "${HOST_PORT}" +probe_leg "runsc -> ${DENIED_NAME}:${HOST_PORT} (live, by name)" denied "${DENIED_NAME}" "${HOST_PORT}" +probe_leg "runsc -> neighbour job $(addr_of n):8080 (live)" denied "$(addr_of n)" 8080 +probe_leg "runsc -> 169.254.169.254:80 (nothing listens)" denied 169.254.169.254 80 +echo " ^ read the metadata leg ONLY as the difference against section 1; with no listener," +echo " an identical result in both sections is NO EVIDENCE either way." + +echo +echo "=== gate5: 4. IPv6, measured rather than assumed ===" +fresh_ns d +echo " namespace v6: $(in_ns runc "${V6}")" +echo " runsc v6: $(in_ns runsc "${V6}")" +echo " The host-side plan renders NO ip6tables rules (DOCKER-USER may not exist there)." +echo " Where the job namespace has no global IPv6 there is nothing to deny; where it has," +echo " host-side v6 denial for a runsc job is UNPROVEN and must not be claimed." + +echo +echo "=== gate5: 5. concurrent delivery, three jobs at once, all policies installed ===" +for t in c1 c2 c3; do apply_host "${t}" install; done +declare -a PIDS=() +for t in c1 c2 c3; do + ( + ns="gate5-ns-${t}" + sudo timeout 120 docker run --detach --name "${ns}" --network "$(net_of "${t}")" \ + --ip "$(addr_of "${t}")" --read-only --cap-drop ALL --security-opt no-new-privileges \ + --user 65534:65534 --entrypoint sleep "${IMAGE}" infinity >/dev/null + sudo timeout 120 docker run --rm --interactive --network "container:${ns}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${PLAN_DIR}/plan-${t}.txt" >/dev/null + # ONE gVisor container in this namespace, doing all three things, because a second one + # would find the namespace already emptied into the first one's netstack. + out="$(sudo timeout 240 docker run --rm --runtime runsc --network "container:${ns}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint sh "${IMAGE}" -c ' + export HOME=/tmp GIT_TERMINAL_PROMPT=0 + node -e '"'"' + const dns=require("dns"),https=require("https"); + dns.lookup("relay.maxplayer.ai",(e,a)=>{ + if(e){console.log("FAIL dns "+e.code);process.exit(0);} + let done=false; const say=(w)=>{if(!done){done=true;console.log(w);}}; + const r=https.request({host:"relay.maxplayer.ai",port:443,path:"/",method:"HEAD",timeout:20000},(res)=>{ + say((res.socket.authorized?"OK":"FAIL")+" dns="+a+" tls="+res.statusCode+" verified="+res.socket.authorized); + res.resume(); r.destroy(); + }); + r.on("timeout",()=>{say("FAIL tls timeout");r.destroy();}); + r.on("error",(x)=>say("FAIL tls "+x.code)); + r.end(); + }); + '"'"' + cd /tmp && git clone --quiet "$1" repo >/dev/null 2>&1 && + echo "OK git $(git -C repo rev-parse HEAD)" || echo "FAIL git" + ' gate5 "${PUBLIC_REPO}" 2>&1 | tr -d '\r')" + echo " ${t}: $(echo "${out}" | paste -sd' | ' -)" + sudo docker rm -f "${ns}" >/dev/null 2>&1 + ) & + PIDS+=($!) +done +for p in "${PIDS[@]}"; do wait "${p}"; done +for t in c1 c2 c3; do apply_host "${t}" teardown; done + +echo +echo "=== gate5: 6. teardown leaves the shared chains as it found them ===" +drop_ns +apply_host d teardown +AFTER_DU="$(sudo iptables -S DOCKER-USER | wc -l)"; AFTER_IN="$(sudo iptables -S INPUT | wc -l)" +echo " DOCKER-USER=${AFTER_DU} (was ${BASE_DU}) · INPUT=${AFTER_IN} (was ${BASE_IN})" +if [ "${AFTER_DU}" = "${BASE_DU}" ] && [ "${AFTER_IN}" = "${BASE_IN}" ]; then + echo " CLEAN" +else + echo " LEAKED"; note_fail +fi + +echo +echo "=== gate5: verdict ===" +echo "denial legs failing: ${FAILURES}" +echo "A concurrent line counts as delivery only if it reads OK dns=… tls=200 verified=true AND OK git ." +[ "${FAILURES}" -eq 0 ] && echo "GATE5-DENIAL: PASS" || echo "GATE5-DENIAL: FAIL" diff --git a/docs/gvisor-dns-delivery/scripts/gate5b-does-the-plan-bind-a-runsc-job.sh b/docs/gvisor-dns-delivery/scripts/gate5b-does-the-plan-bind-a-runsc-job.sh new file mode 100755 index 000000000..960f1a611 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5b-does-the-plan-bind-a-runsc-job.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# Gate 5b: does the egress plan actually BIND a gVisor job? +# +# Gate 5 turned up something that cannot be waved away: from a runsc job inside a +# namespace carrying the full 26-rule plan, a container at 172.31.x.x — squarely +# inside the `172.16.0.0/12 -j DROP` rule — was REACHED. +# +# Every other "denial" in gate 5 was a timeout or a refusal to an address where +# NOTHING IS LISTENING. Absence looks exactly like enforcement. So the only honest +# test is a destination that is (a) inside a DROPped range and (b) has a real +# listener, approached from the same namespace by two runtimes: +# +# runsc job -> neighbour vs runc job -> neighbour +# +# If runc is dropped and runsc gets through, the plan is not binding the gVisor job +# and the containment story for this branch is wrong as written. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +NET="${NET:-maxplayer-dns-gate5b}" +SUBNET="${SUBNET:-172.31.13.0/24}" +GATEWAY="${GATEWAY:-172.31.13.1}" +RESOLVER="${RESOLVER:-1.1.1.1}" +PLAN_FILE="${PLAN_FILE:-$HOME/gate5b-plan.txt}" +OUT="${OUT:-$HOME/gate5b-evidence.txt}" +HOLDER_A="gate5b-holder-a" +HOLDER_B="gate5b-holder-b" +NEIGHBOUR="gate5b-neighbour" + +exec > >(tee "${OUT}") 2>&1 + +echo "=== gate5b: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m)" +echo "docker=$(sudo docker version --format '{{.Server.Version}}') runsc=$(runsc --version | head -1)" + +cleanup() { + sudo docker rm -f "${HOLDER_A}" "${HOLDER_B}" "${NEIGHBOUR}" >/dev/null 2>&1 || true + sudo docker network rm "${NET}" >/dev/null 2>&1 || true +} +trap cleanup EXIT +cleanup + +sudo docker network create --subnet "${SUBNET}" --gateway "${GATEWAY}" "${NET}" >/dev/null +echo "network=${NET} subnet=${SUBNET} plan_rules=$(grep -c . "${PLAN_FILE}")" + +for holder in "${HOLDER_A}" "${HOLDER_B}"; do + sudo timeout 120 docker run --detach --name "${holder}" --runtime runc --network "${NET}" \ + --read-only --cap-drop ALL --security-opt no-new-privileges --user 65534:65534 \ + --entrypoint sleep "${IMAGE}" infinity >/dev/null +done +# Only namespace A carries the plan; B is just where the neighbour lives. +sudo timeout 120 docker run --rm --interactive --runtime runc \ + --network "container:${HOLDER_A}" --cap-drop ALL --cap-add NET_ADMIN \ + --security-opt no-new-privileges "${NETFILTER_IMAGE}" < "${PLAN_FILE}" +echo "plan_applied_rules_reported=$?" + +B_ADDR="$(sudo docker inspect "${HOLDER_B}" --format "{{(index .NetworkSettings.Networks \"${NET}\").IPAddress}}")" +sudo timeout 60 docker run --detach --name "${NEIGHBOUR}" --runtime runc \ + --network "container:${HOLDER_B}" --user 65534:65534 --cap-drop ALL \ + --security-opt no-new-privileges --entrypoint node "${IMAGE}" \ + -e "require('http').createServer((q,r)=>r.end('REACHED')).listen(8080,'0.0.0.0')" >/dev/null +sleep 3 +echo "neighbour=${B_ADDR}:8080 running=$(sudo docker inspect -f '{{.State.Running}}' "${NEIGHBOUR}")" +echo "neighbour_is_inside_a_dropped_range=172.16.0.0/12" + +PROBE=' +const net=require("net"); +const s=net.connect({host:process.argv[1],port:8080,timeout:8000}); +s.on("connect",()=>{console.log("RESULT=REACHED");s.destroy();}); +s.on("timeout",()=>{console.log("RESULT=timeout");s.destroy();}); +s.on("error",(e)=>{console.log("RESULT="+e.code);s.destroy();}); +' + +echo +echo "=== gate5b: the same probe, the same namespace, two runtimes ===" +for rt in runc runsc; do + r="$(sudo timeout 60 docker run --rm --runtime "${rt}" --network "container:${HOLDER_A}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + --entrypoint node "${IMAGE}" -e "${PROBE}" "${B_ADDR}" 2>&1 | tr -d '\r')" + echo "job_runtime=${rt} ${r}" + eval "res_${rt}=\"${r}\"" +done + +# Is the rule even visible from inside the namespace? Ask the host runtime, which can see it. +echo +echo "=== gate5b: the rule as installed, read back from the namespace ===" +sudo timeout 60 docker run --rm --runtime runc --network "container:${HOLDER_A}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + --entrypoint iptables "${NETFILTER_IMAGE}" -S OUTPUT 2>&1 | grep -n "172.16.0.0/12" || true + +echo +echo "=== gate5b: verdict ===" +echo "runc_result=${res_runc}" +echo "runsc_result=${res_runsc}" +if [ "${res_runc}" = "RESULT=REACHED" ]; then + echo "INCONCLUSIVE: the plan did not stop the host runtime either — the harness, not the runtime, is wrong" + exit 2 +fi +if [ "${res_runsc}" = "RESULT=REACHED" ]; then + echo "FINDING: the plan binds a runc job and DOES NOT BIND a runsc job." + echo "gVisor's netstack emits packets to the veth itself; the host kernel's OUTPUT chain" + echo "in that netns never sees them, so per-job egress policy is not enforced for the job" + echo "it is written for. Containment for gVisor jobs cannot live in the netns OUTPUT chain." + exit 1 +fi +echo "NO FINDING: both runtimes were denied; gate 5's REACHED was a harness artifact" +exit 0 diff --git a/docs/gvisor-dns-delivery/scripts/gate5c-where-does-containment-bind.sh b/docs/gvisor-dns-delivery/scripts/gate5c-where-does-containment-bind.sh new file mode 100755 index 000000000..f5cdc69ae --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5c-where-does-containment-bind.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +# Gate 5c (rewritten): if the netns OUTPUT chain does not bind a gVisor job, what does? +# +# The first version of this script measured a dead namespace and had to be thrown +# away: gate5d showed a runsc container takes the namespace's addresses into its +# netstack and never returns them, so the namespace is usable exactly ONCE and +# every leg after the first gVisor container was reading a corpse. +# +# This version obeys the rule that finding forced: +# * ONE gVisor container per namespace — every probe gets a FRESH holder+plan; +# * a health check immediately before any leg meant to be evidence, and a leg +# whose namespace was already sick is reported UNSOUND, never as a denial; +# * every denial targets a LIVE listener, because a timeout to an address where +# nothing listens is indistinguishable from enforcement. +# +# Candidates, both on the far side of the veth where the host kernel handles the +# packet whatever produced it: +# (a) DOCKER-USER (root-netns FORWARD path) keyed to the namespace's address; +# (b) a per-job network instead of the one shared bridge all jobs share today. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +NET_A="${NET_A:-maxplayer-dns-gate5c-a}" +NET_B="${NET_B:-maxplayer-dns-gate5c-b}" +SUBNET_A="${SUBNET_A:-172.31.17.0/24}" +GATEWAY_A="${GATEWAY_A:-172.31.17.1}" +SUBNET_B="${SUBNET_B:-172.31.18.0/24}" +GATEWAY_B="${GATEWAY_B:-172.31.18.1}" +RESOLVER="${RESOLVER:-1.1.1.1}" +PUBLIC_REPO="${PUBLIC_REPO:-https://github.com/octocat/Hello-World.git}" +PLAN_FILE="${PLAN_FILE:-$HOME/gate5c-plan.txt}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate5c-resolv.conf}" +OUT="${OUT:-$HOME/gate5c-evidence.txt}" +HOLDER_SAME="gate5c-holder-same" +HOLDER_FAR="gate5c-holder-far" +NEIGH_SAME="gate5c-neighbour-same" +NEIGH_FAR="gate5c-neighbour-far" + +exec > >(tee "${OUT}") 2>&1 +NS_SEQ=0 +CUR_NS="" +CUR_RULE="" + +echo "=== gate5c: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) docker=$(sudo docker version --format '{{.Server.Version}}') runsc=$(runsc --version | head -1)" +echo "br_netfilter=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables 2>/dev/null || echo absent)" +echo "rule: one gVisor container per namespace; health check before every evidential leg" + +drop_ns() { + [ -n "${CUR_RULE}" ] && { sudo iptables -D DOCKER-USER ${CUR_RULE} >/dev/null 2>&1; CUR_RULE=""; } + [ -n "${CUR_NS}" ] && { sudo docker rm -f "${CUR_NS}" >/dev/null 2>&1; CUR_NS=""; } + return 0 +} +cleanup() { + drop_ns + sudo docker rm -f "${HOLDER_SAME}" "${HOLDER_FAR}" "${NEIGH_SAME}" "${NEIGH_FAR}" >/dev/null 2>&1 || true + sudo docker network rm "${NET_A}" "${NET_B}" >/dev/null 2>&1 || true + # Belt and braces: any stray rule this script could have left in DOCKER-USER. + while sudo iptables -S DOCKER-USER 2>/dev/null | grep -q -- "-d 172.16.0.0/12 -j DROP"; do + sudo iptables -D DOCKER-USER $(sudo iptables -S DOCKER-USER | grep -m1 -- "-d 172.16.0.0/12 -j DROP" | sed 's/^-A DOCKER-USER //') || break + done +} +trap cleanup EXIT +cleanup + +printf 'nameserver %s\noptions timeout:2 attempts:2\n' "${RESOLVER}" > "${RESOLV_FILE}" +chmod 0444 "${RESOLV_FILE}" +sudo docker network create --subnet "${SUBNET_A}" --gateway "${GATEWAY_A}" "${NET_A}" >/dev/null +sudo docker network create --subnet "${SUBNET_B}" --gateway "${GATEWAY_B}" "${NET_B}" >/dev/null + +holder_on() { # name network + sudo timeout 120 docker run --detach --name "$1" --runtime runc --network "$2" \ + --read-only --cap-drop ALL --security-opt no-new-privileges --user 65534:65534 \ + --entrypoint sleep "${IMAGE}" infinity >/dev/null +} +addr_of() { sudo docker inspect "$1" --format "{{(index .NetworkSettings.Networks \"$2\").IPAddress}}"; } + +# The live neighbours. Both are runc-only, so no gVisor container ever enters their +# namespaces and they stay alive for the whole run. +holder_on "${HOLDER_SAME}" "${NET_A}" +holder_on "${HOLDER_FAR}" "${NET_B}" +for pair in "${NEIGH_SAME} ${HOLDER_SAME}" "${NEIGH_FAR} ${HOLDER_FAR}"; do + set -- ${pair} + sudo timeout 60 docker run --detach --name "$1" --runtime runc --network "container:$2" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + --entrypoint node "${IMAGE}" \ + -e "require('http').createServer((q,r)=>r.end('REACHED')).listen(8080,'0.0.0.0')" >/dev/null +done +sleep 3 +SAME_ADDR="$(addr_of "${HOLDER_SAME}" "${NET_A}")" +FAR_ADDR="$(addr_of "${HOLDER_FAR}" "${NET_B}")" +echo "live neighbours: same_bridge=${SAME_ADDR}:8080 other_bridge=${FAR_ADDR}:8080" + +in_ns() { # runtime script args... + local rt="$1"; shift + local script="$1"; shift + sudo timeout 180 docker run --rm --runtime "${rt}" --network "container:${CUR_NS}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint node "${IMAGE}" \ + -e "${script}" "$@" 2>&1 | tr -d '\r' | tail -1 +} + +HEALTH=' +const os=require("os"),dns=require("dns"); +const v4=Object.values(os.networkInterfaces()).flat().filter(x=>x&&x.family==="IPv4"&&!x.internal); +if(!v4.length){console.log("SICK no-address");process.exit(0);} +dns.lookup("relay.maxplayer.ai",(e,a)=>console.log(e?("SICK dns-"+e.code):("HEALTHY "+v4[0].address+" dns="+a))); +' +PROBE=' +const net=require("net"); +const s=net.connect({host:process.argv[1],port:Number(process.argv[2]),timeout:8000}); +let said=false; const say=(w)=>{ if(!said){said=true;console.log(w);} s.destroy(); }; +s.on("connect",()=>say("REACHED")); +s.on("timeout",()=>say("timeout")); +s.on("error",(e)=>say(e.code)); +' + +fresh_ns() { # with_rule("rule"|"") + drop_ns + NS_SEQ=$((NS_SEQ + 1)) + CUR_NS="gate5c-ns-${NS_SEQ}" + holder_on "${CUR_NS}" "${NET_A}" + sudo timeout 120 docker run --rm --interactive --runtime runc \ + --network "container:${CUR_NS}" --cap-drop ALL --cap-add NET_ADMIN \ + --security-opt no-new-privileges "${NETFILTER_IMAGE}" < "${PLAN_FILE}" >/dev/null + NS_ADDR="$(addr_of "${CUR_NS}" "${NET_A}")" + if [ "$1" = "rule" ]; then + CUR_RULE="-s ${NS_ADDR}/32 -d 172.16.0.0/12 -j DROP" + sudo iptables -I DOCKER-USER ${CUR_RULE} + fi +} + +leg() { # label with_rule runtime host port + fresh_ns "$2" + local h; h="$(in_ns runc "${HEALTH}")" + case "${h}" in + HEALTHY*) echo " ${1}: $(in_ns "$3" "${PROBE}" "$4" "$5") [ns=${NS_ADDR} rule=${CUR_RULE:-none} health=${h}]" ;; + *) echo " ${1}: UNSOUND — namespace was already sick before the probe (${h})" ;; + esac +} + +echo +echo "=== gate5c: baseline — today's behaviour, no host-side rule ===" +leg "runc -> live neighbour, same bridge" "" runc "${SAME_ADDR}" 8080 +leg "runsc -> live neighbour, same bridge" "" runsc "${SAME_ADDR}" 8080 + +echo +echo "=== gate5c: (b) per-job network — the neighbour is on a DIFFERENT bridge ===" +leg "runsc -> live neighbour, other bridge" "" runsc "${FAR_ADDR}" 8080 + +echo +echo "=== gate5c: (a) DOCKER-USER keyed to the namespace's own address ===" +leg "runc -> live neighbour, same bridge" rule runc "${SAME_ADDR}" 8080 +leg "runsc -> live neighbour, same bridge" rule runsc "${SAME_ADDR}" 8080 + +echo +echo "=== gate5c: does (a) cost the public route the job must keep? ===" +fresh_ns rule +H="$(in_ns runc "${HEALTH}")" +echo " health before the public leg: ${H}" +in_ns runsc ' +const dns=require("dns"),https=require("https"); +dns.lookup("relay.maxplayer.ai",(e,a)=>{ + if(e) return console.log("PUBLIC-FAIL dns "+e.code); + const r=https.request({host:"relay.maxplayer.ai",port:443,path:"/",method:"HEAD",timeout:15000},(res)=>{ + console.log((res.socket.authorized?"PUBLIC-PASS":"PUBLIC-FAIL")+" dns="+a+" tls="+res.statusCode+" verified="+res.socket.authorized); + res.resume(); r.destroy(); + }); + r.on("timeout",()=>{console.log("PUBLIC-FAIL tls timeout");r.destroy();}); + r.on("error",(x)=>console.log("PUBLIC-FAIL tls "+x.code)); + r.end(); +}); +' | sed 's/^/ /' + +fresh_ns rule +echo " health before the git leg: $(in_ns runc "${HEALTH}")" +sudo timeout 180 docker run --rm --runtime runsc --network "container:${CUR_NS}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint sh "${IMAGE}" -c ' + export HOME=/tmp GIT_TERMINAL_PROMPT=0 + cd /tmp && git clone --quiet "$1" repo 2>&1 && echo "PUBLIC-PASS git $(git -C repo rev-parse HEAD)" + ' gate5c "${PUBLIC_REPO}" 2>&1 | tr -d '\r' | sed 's/^/ /' + +echo +echo "=== gate5c: how to read this ===" +echo "(a) binds runsc only if BOTH rule-legs are denied AND both public legs still pass." +echo "(b) binds runsc only if the other-bridge leg is denied from a HEALTHY namespace." diff --git a/docs/gvisor-dns-delivery/scripts/gate5d-is-the-namespace-single-use.sh b/docs/gvisor-dns-delivery/scripts/gate5d-is-the-namespace-single-use.sh new file mode 100755 index 000000000..ddfce701a --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5d-is-the-namespace-single-use.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Gate 5d: does a runsc container leave the shared namespace usable behind it? +# +# gate5c's later legs all returned ENETUNREACH — including the runc CONTROL, which +# had worked minutes earlier in the same namespace, and including DNS to 1.1.1.1 +# which no rule under test touched. A control that dies is not a control, so +# gate5c's verdicts on both candidate enforcement sites are void until this is +# settled. The same signature appeared in gate 5: the first runsc container +# resolved fine, the second could not resolve at all. +# +# The suspicion: runsc's netstack takes the veth's addresses and routes INTO the +# sandbox, and does not put them back when it exits — leaving the namespace +# stripped for whatever runs next. If so, the shared job namespace is SINGLE-USE +# for gVisor, and every multi-container measurement in this branch has to be read +# again with that in mind. +# +# The image has no iproute2, so the namespace is read with node's +# os.networkInterfaces() — the same way the earlier probes read it. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +NET="${NET:-maxplayer-dns-gate5d}" +SUBNET="${SUBNET:-172.31.16.0/24}" +GATEWAY="${GATEWAY:-172.31.16.1}" +RESOLVER="${RESOLVER:-1.1.1.1}" +PLAN_FILE="${PLAN_FILE:-$HOME/gate5d-plan.txt}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate5d-resolv.conf}" +OUT="${OUT:-$HOME/gate5d-evidence.txt}" +HOLDER="gate5d-holder" + +exec > >(tee "${OUT}") 2>&1 + +echo "=== gate5d: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) docker=$(sudo docker version --format '{{.Server.Version}}') runsc=$(runsc --version | head -1)" + +cleanup() { + sudo docker rm -f "${HOLDER}" >/dev/null 2>&1 || true + sudo docker network rm "${NET}" >/dev/null 2>&1 || true +} +trap cleanup EXIT +cleanup + +printf 'nameserver %s\noptions timeout:2 attempts:2\n' "${RESOLVER}" > "${RESOLV_FILE}" +chmod 0444 "${RESOLV_FILE}" +sudo docker network create --subnet "${SUBNET}" --gateway "${GATEWAY}" "${NET}" >/dev/null +sudo timeout 120 docker run --detach --name "${HOLDER}" --runtime runc --network "${NET}" \ + --read-only --cap-drop ALL --security-opt no-new-privileges --user 65534:65534 \ + --entrypoint sleep "${IMAGE}" infinity >/dev/null +sudo timeout 120 docker run --rm --interactive --runtime runc --network "container:${HOLDER}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${PLAN_FILE}" >/dev/null +echo "namespace established, plan applied ($(grep -c . "${PLAN_FILE}") rules)" + +READ_NS=' +const os=require("os"),dns=require("dns"); +const ifs=os.networkInterfaces(); +const v4=Object.entries(ifs).flatMap(([n,a])=>(a||[]).filter(x=>x.family==="IPv4").map(x=>n+"="+x.address)); +console.log("interfaces: "+(v4.join(" ")||"NONE")); +dns.lookup("relay.maxplayer.ai",(e,a)=>console.log("dns: "+(e?e.code:a))); +' +look() { # runtime label + echo " [$2 via $1] $(sudo timeout 60 docker run --rm --runtime "$1" --network "container:${HOLDER}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint node "${IMAGE}" -e "${READ_NS}" 2>&1 | tr -d '\r' | tr '\n' ' ')" +} + +echo +echo "=== gate5d: the namespace before any gVisor container has touched it ===" +look runc "before" + +echo +echo "=== gate5d: one runsc container runs and exits ===" +look runsc "the gVisor job itself" + +echo +echo "=== gate5d: the same namespace afterwards ===" +look runc "after, host runtime" +look runsc "after, a second gVisor job" + +echo +echo "=== gate5d: read it off the two 'after' lines ===" +echo "If 'before' has an eth0 address and 'after' says NONE, the namespace is single-use" +echo "for gVisor, and gate5c's ENETUNREACH verdicts are void — a dead control, not a policy." diff --git a/docs/gvisor-dns-delivery/scripts/gate5e-routed-enforcement-under-runsc.sh b/docs/gvisor-dns-delivery/scripts/gate5e-routed-enforcement-under-runsc.sh new file mode 100755 index 000000000..76c993317 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5e-routed-enforcement-under-runsc.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# Gate 5e: with a per-job network, which host-side chain actually binds a gVisor job? +# +# gate5c settled two things: a same-bridge peer is unreachable to iptables (this host +# has no br_netfilter, so switched frames never enter FORWARD), and a per-job network +# turns cross-job traffic into ROUTED traffic that DOCKER-ISOLATION already drops. +# What is still unmeasured is everything a job reaches by ROUTE rather than by switch: +# the host itself, and the metadata address. +# +# Discipline carried over, because it is what made the last two results trustworthy: +# * a FRESH holder+plan namespace for every probe, ONE gVisor container in each; +# * a health check printed beside every leg, and UNSOUND rather than a verdict if +# the namespace was already sick; +# * live listeners. Where a live listener is impossible (the metadata address), the +# leg is reported as NO EVIDENCE unless bare and ruled runs DIFFER. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +NET_J1="${NET_J1:-maxplayer-dns-gate5e-job1}" +NET_J2="${NET_J2:-maxplayer-dns-gate5e-job2}" +SUBNET_J1="${SUBNET_J1:-172.31.19.0/24}" +GATEWAY_J1="${GATEWAY_J1:-172.31.19.1}" +SUBNET_J2="${SUBNET_J2:-172.31.20.0/24}" +GATEWAY_J2="${GATEWAY_J2:-172.31.20.1}" +RESOLVER="${RESOLVER:-1.1.1.1}" +HOST_LAN="${HOST_LAN:-192.168.5.15}" +HOST_PORT="${HOST_PORT:-49252}" +PUBLIC_REPO="${PUBLIC_REPO:-https://github.com/octocat/Hello-World.git}" +PLAN_FILE="${PLAN_FILE:-$HOME/gate5e-plan.txt}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate5e-resolv.conf}" +OUT="${OUT:-$HOME/gate5e-evidence.txt}" +HOLDER_J2="gate5e-holder-job2" +NEIGH_J2="gate5e-neighbour-job2" + +exec > >(tee "${OUT}") 2>&1 +NS_SEQ=0 +CUR_NS="" +NS_ADDR="" +declare -a ADDED_RULES=() + +echo "=== gate5e: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) docker=$(sudo docker version --format '{{.Server.Version}}') runsc=$(runsc --version | head -1)" +echo "br_netfilter=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables 2>/dev/null || echo absent)" +echo "host_lan=${HOST_LAN} (the VM's own eth0 — host-directed traffic lands in INPUT, not FORWARD)" + +undo_rules() { + local i + for ((i = ${#ADDED_RULES[@]} - 1; i >= 0; i--)); do + local chain="${ADDED_RULES[i]%%|*}" spec="${ADDED_RULES[i]#*|}" + sudo iptables -D "${chain}" ${spec} >/dev/null 2>&1 + done + ADDED_RULES=() +} +drop_ns() { + undo_rules + [ -n "${CUR_NS}" ] && { sudo docker rm -f "${CUR_NS}" >/dev/null 2>&1; CUR_NS=""; } + return 0 +} +cleanup() { + drop_ns + sudo docker rm -f "${HOLDER_J2}" "${NEIGH_J2}" >/dev/null 2>&1 || true + sudo docker network rm "${NET_J1}" "${NET_J2}" >/dev/null 2>&1 || true + pkill -f "gate5e-host-listener" >/dev/null 2>&1 || true +} +trap cleanup EXIT +cleanup + +# chmod 0444 below makes the file unwritable, so a second run of this script fails +# to rewrite it unless it is removed first. That cost a run's worth of noise once. +rm -f "${RESOLV_FILE}" +printf 'nameserver %s\noptions timeout:2 attempts:2\n' "${RESOLVER}" > "${RESOLV_FILE}" +chmod 0444 "${RESOLV_FILE}" + +sudo docker network create --subnet "${SUBNET_J1}" --gateway "${GATEWAY_J1}" "${NET_J1}" >/dev/null +sudo docker network create --subnet "${SUBNET_J2}" --gateway "${GATEWAY_J2}" "${NET_J2}" >/dev/null +echo "per-job networks: job1=${SUBNET_J1} job2=${SUBNET_J2}" + +# A live listener on the VM's own LAN address: a private destination reached by ROUTE, +# which docker isolation does not block, and which no rule blocks yet. +setsid python3 -c " +import socket,sys +s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) +s.bind(('0.0.0.0',${HOST_PORT})); s.listen(8) # gate5e-host-listener +while True: + c,_=s.accept(); c.sendall(b'REACHED'); c.close() +" >/dev/null 2>&1 /dev/null +sudo timeout 60 docker run --detach --name "${NEIGH_J2}" --runtime runc \ + --network "container:${HOLDER_J2}" --user 65534:65534 --cap-drop ALL \ + --security-opt no-new-privileges --entrypoint node "${IMAGE}" \ + -e "require('http').createServer((q,r)=>r.end('REACHED')).listen(8080,'0.0.0.0')" >/dev/null +sleep 3 +J2_ADDR="$(sudo docker inspect "${HOLDER_J2}" --format "{{(index .NetworkSettings.Networks \"${NET_J2}\").IPAddress}}")" +echo "live neighbour in job 2's own namespace: ${J2_ADDR}:8080" + +in_ns() { # runtime script args... + local rt="$1"; shift + local script="$1"; shift + sudo timeout 180 docker run --rm --runtime "${rt}" --network "container:${CUR_NS}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint node "${IMAGE}" \ + -e "${script}" "$@" 2>&1 | tr -d '\r' | tail -1 +} +HEALTH=' +const os=require("os"),dns=require("dns"); +const v4=Object.values(os.networkInterfaces()).flat().filter(x=>x&&x.family==="IPv4"&&!x.internal); +if(!v4.length){console.log("SICK no-address");process.exit(0);} +dns.lookup("relay.maxplayer.ai",(e,a)=>console.log(e?("SICK dns-"+e.code):("HEALTHY "+v4[0].address))); +' +PROBE=' +const net=require("net"); +const s=net.connect({host:process.argv[1],port:Number(process.argv[2]),timeout:8000}); +let said=false; const say=(w)=>{ if(!said){said=true;console.log(w);} s.destroy(); }; +s.on("connect",()=>say("REACHED")); +s.on("timeout",()=>say("timeout")); +s.on("error",(e)=>say(e.code)); +' + +fresh_ns() { # rules... each "CHAIN|spec with %NS% for this namespace's address" + drop_ns + NS_SEQ=$((NS_SEQ + 1)) + CUR_NS="gate5e-ns-${NS_SEQ}" + sudo timeout 120 docker run --detach --name "${CUR_NS}" --runtime runc --network "${NET_J1}" \ + --read-only --cap-drop ALL --security-opt no-new-privileges --user 65534:65534 \ + --entrypoint sleep "${IMAGE}" infinity >/dev/null + sudo timeout 120 docker run --rm --interactive --runtime runc --network "container:${CUR_NS}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${PLAN_FILE}" >/dev/null + NS_ADDR="$(sudo docker inspect "${CUR_NS}" --format "{{(index .NetworkSettings.Networks \"${NET_J1}\").IPAddress}}")" + local r + for r in "$@"; do + [ -z "${r}" ] && continue + local chain="${r%%|*}" spec="${r#*|}" + spec="${spec//%NS%/${NS_ADDR}}" + sudo iptables -I "${chain}" ${spec} + ADDED_RULES+=("${chain}|${spec}") + done +} +leg() { # label runtime host port rules... + local label="$1" rt="$2" host="$3" port="$4"; shift 4 + fresh_ns "$@" + local h; h="$(in_ns runc "${HEALTH}")" + local shown="none"; [ ${#ADDED_RULES[@]} -gt 0 ] && shown="${ADDED_RULES[*]}" + case "${h}" in + HEALTHY*) echo " ${label}: $(in_ns "${rt}" "${PROBE}" "${host}" "${port}") [ns=${NS_ADDR} rules=${shown}]" ;; + *) echo " ${label}: UNSOUND — namespace sick before the probe (${h})" ;; + esac +} + +echo +echo "=== gate5e: 1. the host itself, a routed private destination with a LIVE listener ===" +leg "runc -> ${HOST_LAN}:${HOST_PORT}, bare" runc "${HOST_LAN}" "${HOST_PORT}" +leg "runsc -> ${HOST_LAN}:${HOST_PORT}, bare" runsc "${HOST_LAN}" "${HOST_PORT}" +leg "runsc -> ${HOST_LAN}:${HOST_PORT}, DOCKER-USER" runsc "${HOST_LAN}" "${HOST_PORT}" \ + "DOCKER-USER|-s %NS%/32 -d 192.168.0.0/16 -j DROP" +leg "runsc -> ${HOST_LAN}:${HOST_PORT}, INPUT" runsc "${HOST_LAN}" "${HOST_PORT}" \ + "INPUT|-s %NS%/32 -d 192.168.0.0/16 -j DROP" + +echo +echo "=== gate5e: 2. the metadata address (no listener anywhere — read only the DIFFERENCE) ===" +leg "runsc -> 169.254.169.254:80, bare" runsc "169.254.169.254" 80 +leg "runsc -> 169.254.169.254:80, DOCKER-USER" runsc "169.254.169.254" 80 \ + "DOCKER-USER|-s %NS%/32 -d 169.254.169.254/32 -j DROP" + +echo +echo "=== gate5e: 3. cross-job, each job on its OWN network (the regression proof) ===" +leg "runsc -> job 2's live listener" runsc "${J2_ADDR}" 8080 + +echo +echo "=== gate5e: 4. the public route, with the candidate rules installed ===" +fresh_ns "DOCKER-USER|-s %NS%/32 -d 169.254.169.254/32 -j DROP" \ + "INPUT|-s %NS%/32 -d 192.168.0.0/16 -j DROP" +echo " health: $(in_ns runc "${HEALTH}") rules=${ADDED_RULES[*]}" +in_ns runsc ' +const dns=require("dns"),https=require("https"); +dns.lookup("relay.maxplayer.ai",(e,a)=>{ + if(e) return console.log("PUBLIC-FAIL dns "+e.code); + const r=https.request({host:"relay.maxplayer.ai",port:443,path:"/",method:"HEAD",timeout:15000},(res)=>{ + console.log((res.socket.authorized?"PUBLIC-PASS":"PUBLIC-FAIL")+" dns="+a+" tls="+res.statusCode+" verified="+res.socket.authorized); + res.resume(); r.destroy(); + }); + r.on("timeout",()=>{console.log("PUBLIC-FAIL tls timeout");r.destroy();}); + r.on("error",(x)=>console.log("PUBLIC-FAIL tls "+x.code)); + r.end(); +}); +' | sed 's/^/ /' + +fresh_ns "DOCKER-USER|-s %NS%/32 -d 169.254.169.254/32 -j DROP" \ + "INPUT|-s %NS%/32 -d 192.168.0.0/16 -j DROP" +echo " health: $(in_ns runc "${HEALTH}")" +sudo timeout 180 docker run --rm --runtime runsc --network "container:${CUR_NS}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint sh "${IMAGE}" -c ' + export HOME=/tmp GIT_TERMINAL_PROMPT=0 + cd /tmp && git clone --quiet "$1" repo 2>&1 && echo "PUBLIC-PASS git $(git -C repo rev-parse HEAD)" + ' gate5e "${PUBLIC_REPO}" 2>&1 | tr -d '\r' | sed 's/^/ /' + +echo +echo "=== gate5e: how to read this ===" +echo "leg 1 names the chain that binds a routed, host-directed destination for runsc." +echo "leg 2 is evidence ONLY if bare and ruled differ; identical timeouts prove nothing." +echo "leg 3 must be denied for per-job networks to be the cross-job answer." +echo "leg 4 must pass, or the candidate costs the job the route it exists to have." diff --git a/docs/gvisor-dns-delivery/scripts/gate5f-product-host-rules-bind-runsc.sh b/docs/gvisor-dns-delivery/scripts/gate5f-product-host-rules-bind-runsc.sh new file mode 100755 index 000000000..ed189fc44 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5f-product-host-rules-bind-runsc.sh @@ -0,0 +1,258 @@ +#!/usr/bin/env bash +# Gate 5f: does the PRODUCT's own host-side policy bind a gVisor job? +# +# gate5b found the hole: the namespace OUTPUT plan does not bind a runsc job, because +# gVisor's netstack writes frames straight to the veth and the host OUTPUT chain in that +# namespace only ever sees host sockets. gate5c and gate5e then found where a runsc job +# CAN be bound: routed destinations, in DOCKER-USER, and host-directed ones in INPUT — +# while a same-bridge peer is bound by neither, this host having no br_netfilter. +# +# This is the regression proof for the fix built on those findings, and the rules under +# test are rendered by the product (`--example render_host_plan`, from HostPolicy), never +# transcribed here. A transcription would drift from the policy and this gate would keep +# passing against a firewall the product no longer builds. +# +# They are also installed the way the product installs them: piped into the SAME applier +# image the sidecar uses, in a --network host container, which is itself under test — if +# that image cannot write the root namespace's chains, this gate must fail, not paper over it. +# +# Discipline from gate5c/5d, which is what made those results trustworthy: +# * a FRESH holder+plan namespace per probe, ONE gVisor container in each (the namespace +# is single-use for gVisor: runsc takes the addresses into its netstack and never +# gives them back, which voided an earlier run of gate5c); +# * a health check printed beside every leg, UNSOUND instead of a verdict if it is sick; +# * LIVE listeners. A timeout against an address where nothing listens is NO EVIDENCE: +# absence and enforcement are indistinguishable. That mistake is why part of gate 2's +# evidence was retracted. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +NET_J1="${NET_J1:-maxplayer-dns-gate5f-job1}" +NET_J2="${NET_J2:-maxplayer-dns-gate5f-job2}" +SUBNET_J1="${SUBNET_J1:-172.31.21.0/24}" +GATEWAY_J1="${GATEWAY_J1:-172.31.21.1}" +SUBNET_J2="${SUBNET_J2:-172.31.22.0/24}" +GATEWAY_J2="${GATEWAY_J2:-172.31.22.1}" +# The probe namespace's address is PINNED, because the host plan is rendered ahead of time +# for exactly this source key and a plan keyed to the wrong address would deny some other +# container while leaving this one open. The script refuses to run if they disagree. +JOB_ADDR="${JOB_ADDR:-172.31.21.10}" +SAME_BRIDGE_ADDR="${SAME_BRIDGE_ADDR:-172.31.21.20}" +J2_ADDR="${J2_ADDR:-172.31.22.10}" +RESOLVER="${RESOLVER:-1.1.1.1}" +HOST_LAN="${HOST_LAN:-192.168.5.15}" +HOST_PORT="${HOST_PORT:-49253}" +PUBLIC_REPO="${PUBLIC_REPO:-https://github.com/octocat/Hello-World.git}" +PLAN_FILE="${PLAN_FILE:-$HOME/gate5f-plan.txt}" +HOST_INSTALL="${HOST_INSTALL:-$HOME/gate5f-host-install.txt}" +HOST_TEARDOWN="${HOST_TEARDOWN:-$HOME/gate5f-host-teardown.txt}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate5f-resolv.conf}" +OUT="${OUT:-$HOME/gate5f-evidence.txt}" +HOLDER_SAME="gate5f-holder-same-bridge" +NEIGH_SAME="gate5f-neighbour-same-bridge" +HOLDER_J2="gate5f-holder-job2" +NEIGH_J2="gate5f-neighbour-job2" + +exec > >(tee "${OUT}") 2>&1 +NS_SEQ=0 +CUR_NS="" +HOST_RULES_UP=0 + +for f in "${PLAN_FILE}" "${HOST_INSTALL}" "${HOST_TEARDOWN}"; do + [ -s "${f}" ] || { echo "MISSING rendered plan ${f} — render it on the host first"; exit 2; } +done +# The stale-plan guard. Cheap, and it is the difference between measuring this job's policy +# and measuring a rule that belongs to nothing. +grep -q -- "-s ${JOB_ADDR}/32" "${HOST_INSTALL}" || { + echo "REFUSING TO RUN: ${HOST_INSTALL} carries no -s ${JOB_ADDR}/32 source key"; exit 2; } + +echo "=== gate5f: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) docker=$(sudo docker version --format '{{.Server.Version}}') runsc=$(runsc --version | head -1)" +echo "docker default-runtime=$(sudo docker info --format '{{.DefaultRuntime}}') (helpers inherit this; the JOB always carries --runtime runsc)" +echo "br_netfilter=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables 2>/dev/null || echo absent)" +echo "host plan rendered by the product for job_addr=${JOB_ADDR}: $(wc -l < "${HOST_INSTALL}") rules" + +BASE_DOCKER_USER="$(sudo iptables -S DOCKER-USER | wc -l)" +BASE_INPUT="$(sudo iptables -S INPUT | wc -l)" +echo "chain depth before anything: DOCKER-USER=${BASE_DOCKER_USER} INPUT=${BASE_INPUT}" + +host_rules() { # install|teardown + local file="${HOST_INSTALL}" verb="install" + [ "$1" = teardown ] && { file="${HOST_TEARDOWN}"; verb="teardown"; } + # The product's applier, on the product's plan, in the root namespace. Same image as the + # sidecar: one applier in this design, not two. + local applied + applied="$(sudo timeout 120 docker run --rm --interactive --network host \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${file}" 2>&1 | tr -d '\r' | tail -1)" + local expected; expected="$(wc -l < "${file}" | tr -d ' ')" + echo " host-rule ${verb}: applier reported ${applied}, plan had ${expected} rules" + [ "${verb}" = install ] && HOST_RULES_UP=1 || HOST_RULES_UP=0 + # The applier's own account is not proof it reached the ROOT namespace's chains, so the + # host kernel is asked directly. This readback is the whole reason the leg is trustworthy. + local landed; landed="$(sudo iptables -S DOCKER-USER | grep -c -- "-s ${JOB_ADDR}/32")" + local landed_in; landed_in="$(sudo iptables -S INPUT | grep -c -- "-s ${JOB_ADDR}/32")" + echo " readback from the host kernel: DOCKER-USER carries ${landed}, INPUT carries ${landed_in} rules for ${JOB_ADDR}" +} + +drop_ns() { + [ -n "${CUR_NS}" ] && { sudo docker rm -f "${CUR_NS}" >/dev/null 2>&1; CUR_NS=""; } + return 0 +} +cleanup() { + drop_ns + [ "${HOST_RULES_UP}" = 1 ] && host_rules teardown >/dev/null 2>&1 + sudo docker rm -f "${HOLDER_SAME}" "${NEIGH_SAME}" "${HOLDER_J2}" "${NEIGH_J2}" >/dev/null 2>&1 || true + sudo docker network rm "${NET_J1}" "${NET_J2}" >/dev/null 2>&1 || true + pkill -f "gate5f-host-listener" >/dev/null 2>&1 || true +} +trap cleanup EXIT +cleanup + +rm -f "${RESOLV_FILE}" +printf 'nameserver %s\noptions timeout:2 attempts:2\n' "${RESOLVER}" > "${RESOLV_FILE}" +chmod 0444 "${RESOLV_FILE}" + +sudo docker network create --subnet "${SUBNET_J1}" --gateway "${GATEWAY_J1}" "${NET_J1}" >/dev/null +sudo docker network create --subnet "${SUBNET_J2}" --gateway "${GATEWAY_J2}" "${NET_J2}" >/dev/null + +# A live listener on the VM's own LAN address: private, reached by ROUTE, and the one +# destination in this gate where the INPUT half of the plan can be caught working. +setsid python3 -c " +import socket +s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) +s.bind(('0.0.0.0',${HOST_PORT})); s.listen(8) # gate5f-host-listener +while True: + c,_=s.accept(); c.sendall(b'REACHED'); c.close() +" >/dev/null 2>&1 /dev/null +sudo timeout 60 docker run --detach --name "${NEIGH_SAME}" --network "container:${HOLDER_SAME}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges --entrypoint node "${IMAGE}" \ + -e "require('http').createServer((q,r)=>r.end('REACHED')).listen(8080,'0.0.0.0')" >/dev/null +sudo timeout 120 docker run --detach --name "${HOLDER_J2}" --network "${NET_J2}" \ + --ip "${J2_ADDR}" --read-only --cap-drop ALL --security-opt no-new-privileges \ + --user 65534:65534 --entrypoint sleep "${IMAGE}" infinity >/dev/null +sudo timeout 60 docker run --detach --name "${NEIGH_J2}" --network "container:${HOLDER_J2}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges --entrypoint node "${IMAGE}" \ + -e "require('http').createServer((q,r)=>r.end('REACHED')).listen(8080,'0.0.0.0')" >/dev/null +sleep 3 +echo "live listeners: same-bridge ${SAME_BRIDGE_ADDR}:8080 · own-network ${J2_ADDR}:8080 · host ${HOST_LAN}:${HOST_PORT}" + +in_ns() { # runtime script args... + local rt="$1"; shift + local script="$1"; shift + sudo timeout 180 docker run --rm --runtime "${rt}" --network "container:${CUR_NS}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint node "${IMAGE}" \ + -e "${script}" "$@" 2>&1 | tr -d '\r' | tail -1 +} +HEALTH=' +const os=require("os"),dns=require("dns"); +const v4=Object.values(os.networkInterfaces()).flat().filter(x=>x&&x.family==="IPv4"&&!x.internal); +if(!v4.length){console.log("SICK no-address");process.exit(0);} +dns.lookup("relay.maxplayer.ai",(e,a)=>console.log(e?("SICK dns-"+e.code):("HEALTHY "+v4[0].address))); +' +PROBE=' +const net=require("net"); +const s=net.connect({host:process.argv[1],port:Number(process.argv[2]),timeout:8000}); +let said=false; const say=(w)=>{ if(!said){said=true;console.log(w);} s.destroy(); }; +s.on("connect",()=>say("REACHED")); +s.on("timeout",()=>say("timeout")); +s.on("error",(e)=>say(e.code)); +' + +fresh_ns() { + drop_ns + NS_SEQ=$((NS_SEQ + 1)) + CUR_NS="gate5f-ns-${NS_SEQ}" + sudo timeout 120 docker run --detach --name "${CUR_NS}" --network "${NET_J1}" \ + --ip "${JOB_ADDR}" --read-only --cap-drop ALL --security-opt no-new-privileges \ + --user 65534:65534 --entrypoint sleep "${IMAGE}" infinity >/dev/null + sudo timeout 120 docker run --rm --interactive --network "container:${CUR_NS}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${PLAN_FILE}" >/dev/null + local got; got="$(sudo docker inspect "${CUR_NS}" --format "{{(index .NetworkSettings.Networks \"${NET_J1}\").IPAddress}}")" + [ "${got}" = "${JOB_ADDR}" ] || echo " WARNING: namespace came up as ${got}, not the ${JOB_ADDR} the host plan is keyed to" +} +leg() { # label runtime host port + local label="$1" rt="$2" host="$3" port="$4" + fresh_ns + local h; h="$(in_ns runc "${HEALTH}")" + case "${h}" in + HEALTHY*) echo " ${label}: $(in_ns "${rt}" "${PROBE}" "${host}" "${port}")" ;; + *) echo " ${label}: UNSOUND — namespace sick before the probe (${h})" ;; + esac +} + +echo +echo "=== gate5f: 1. BEFORE — the hole, with no host rules ===" +leg "runsc -> same-bridge neighbour ${SAME_BRIDGE_ADDR}:8080" runsc "${SAME_BRIDGE_ADDR}" 8080 +leg "runsc -> own-network neighbour ${J2_ADDR}:8080" runsc "${J2_ADDR}" 8080 +leg "runsc -> host ${HOST_LAN}:${HOST_PORT}" runsc "${HOST_LAN}" "${HOST_PORT}" +leg "runc -> host ${HOST_LAN}:${HOST_PORT}" runc "${HOST_LAN}" "${HOST_PORT}" + +echo +echo "=== gate5f: 2. installing the product's rendered host policy ===" +host_rules install + +echo +echo "=== gate5f: 3. AFTER — the same probes, unchanged ===" +leg "runsc -> own-network neighbour ${J2_ADDR}:8080 (MUST be denied)" runsc "${J2_ADDR}" 8080 +leg "runsc -> host ${HOST_LAN}:${HOST_PORT} (MUST be denied)" runsc "${HOST_LAN}" "${HOST_PORT}" +leg "runc -> host ${HOST_LAN}:${HOST_PORT} (MUST be denied)" runc "${HOST_LAN}" "${HOST_PORT}" +leg "runsc -> same-bridge neighbour ${SAME_BRIDGE_ADDR}:8080 (known unbindable)" runsc "${SAME_BRIDGE_ADDR}" 8080 + +echo +echo "=== gate5f: 4. the public route must survive the policy ===" +fresh_ns +echo " health: $(in_ns runc "${HEALTH}")" +in_ns runsc ' +const dns=require("dns"),https=require("https"); +dns.lookup("relay.maxplayer.ai",(e,a)=>{ + if(e) return console.log("PUBLIC-FAIL dns "+e.code); + const r=https.request({host:"relay.maxplayer.ai",port:443,path:"/",method:"HEAD",timeout:15000},(res)=>{ + console.log((res.socket.authorized?"PUBLIC-PASS":"PUBLIC-FAIL")+" dns="+a+" tls="+res.statusCode+" verified="+res.socket.authorized); + res.resume(); r.destroy(); + }); + r.on("timeout",()=>{console.log("PUBLIC-FAIL tls timeout");r.destroy();}); + r.on("error",(x)=>console.log("PUBLIC-FAIL tls "+x.code)); + r.end(); +}); +' | sed 's/^/ /' +fresh_ns +echo " health: $(in_ns runc "${HEALTH}")" +sudo timeout 180 docker run --rm --runtime runsc --network "container:${CUR_NS}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint sh "${IMAGE}" -c ' + export HOME=/tmp GIT_TERMINAL_PROMPT=0 + cd /tmp && git clone --quiet "$1" repo 2>&1 && echo "PUBLIC-PASS git $(git -C repo rev-parse HEAD)" + ' gate5f "${PUBLIC_REPO}" 2>&1 | tr -d '\r' | sed 's/^/ /' + +echo +echo "=== gate5f: 5. teardown must leave the shared chains exactly as it found them ===" +drop_ns +host_rules teardown +AFTER_DOCKER_USER="$(sudo iptables -S DOCKER-USER | wc -l)" +AFTER_INPUT="$(sudo iptables -S INPUT | wc -l)" +echo " chain depth after teardown: DOCKER-USER=${AFTER_DOCKER_USER} (was ${BASE_DOCKER_USER}) INPUT=${AFTER_INPUT} (was ${BASE_INPUT})" +if [ "${AFTER_DOCKER_USER}" = "${BASE_DOCKER_USER}" ] && [ "${AFTER_INPUT}" = "${BASE_INPUT}" ]; then + echo " CLEAN — no rule leaked (this is the failure mode the HostRules drop guard exists to prevent)" +else + echo " LEAKED — the shared chains kept rules after teardown" +fi + +echo +echo "=== gate5f: how to read this ===" +echo "The fix is proved by section 3 differing from section 1 on the own-network and host legs." +echo "The same-bridge leg is expected to stay REACHED: switched frames enter no chain on a host" +echo "without br_netfilter, which is WHY the product gives every job its own network rather than" +echo "trying to rule its way out of a shared one. It is measured here so that reason stays evidenced." +echo "Section 4 must pass, or the policy costs the job the delivery route it exists to have." diff --git a/docs/gvisor-dns-delivery/scripts/gate5g-baseline-vulnerability-repro.sh b/docs/gvisor-dns-delivery/scripts/gate5g-baseline-vulnerability-repro.sh new file mode 100755 index 000000000..24607ef8d --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5g-baseline-vulnerability-repro.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# Gate 5g: reproduce the containment failure ON THE UNFIXED BASELINE. +# +# The branch claims the netns plan does not bind a gVisor job and that this PREDATES the +# fix. A claim like that is not settled by reading source — a reviewer is entitled to a +# reproduction on the code as it ships today. This is that reproduction, and it is built +# to be unfair to my own claim: +# +# * The rules come from that baseline commit (b45f8651dc9cab5b71c962eadd7b84840f579106), rendered by a throwaway example that +# compiles against BASELINE's NetPolicy — three fields, no dns_resolvers. A renderer +# built against the fix would be reproducing the fix, not the bug. 24 rules. +# * The ARRANGEMENT is baseline's too: ONE shared network for every job, which is what a +# single `[sandbox] network` setting produced, and NO host-side rules, because +# that baseline commit has no HostPolicy at all (grep: 0 occurrences). +# * Every denial leg targets a LIVE listener. A timeout against a dead address would +# prove nothing here, and reading absence as enforcement is the exact error that cost +# gate 2 its metadata line. +# * runc runs the identical probe as the positive control. If the plan failed to bind +# runc too, the finding would be "the plan was never installed", not "runsc escapes". +# +# Runs only in the disposable gvisor-repro VM. It installs no host rules whatsoever, so it +# cannot touch shared infrastructure even by accident. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +NET="${NET:-maxplayer-dns-gate5g-shared}" +SUBNET="${SUBNET:-172.31.40.0/24}" +GATEWAY="${GATEWAY:-172.31.40.1}" +VICTIM_ADDR="${VICTIM_ADDR:-172.31.40.20}" +RESOLVER="${RESOLVER:-1.1.1.1}" +HOST_LAN="${HOST_LAN:-192.168.5.15}" +HOST_PORT="${HOST_PORT:-49255}" +PLAN_FILE="${PLAN_FILE:-$HOME/gate5g-baseline-plan.txt}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate5g-resolv.conf}" +OUT="${OUT:-$HOME/gate5g-evidence.txt}" + +exec > >(tee "${OUT}") 2>&1 +CUR_NS=""; NS_SEQ=0; FINDINGS=0 + +[ -s "${PLAN_FILE}" ] || { echo "MISSING ${PLAN_FILE} — render it from that baseline commit first"; exit 2; } +# Guard: this must be the BASELINE plan. The fix's plan carries port-53 resolver pinholes; +# baseline's does not. If this file has them, the wrong plan was staged and the whole +# reproduction would be meaningless. +if grep -q -- "--dport 53 -d" "${PLAN_FILE}"; then + echo "REFUSING TO RUN: ${PLAN_FILE} carries resolver pinholes — that is the FIXED plan, not baseline"; exit 2 +fi + +echo "=== gate5g: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) docker=$(sudo docker version --format '{{.Server.Version}}') runsc=$(runsc --version | head -1)" +echo "baseline commit=b45f8651dc9cab5b71c962eadd7b84840f579106 (immutable commit, not a moving ref) · plan rules=$(wc -l < "${PLAN_FILE}")" +echo "host rules installed by this script: NONE (that baseline commit has no HostPolicy — grep says 0)" +echo "arrangement: ONE shared network ${SUBNET}, as a single [sandbox] network produced" +echo "chain depth (untouched by this gate): DOCKER-USER=$(sudo iptables -S DOCKER-USER | wc -l) INPUT=$(sudo iptables -S INPUT | wc -l)" + +drop_ns() { [ -n "${CUR_NS}" ] && { sudo docker rm -f "${CUR_NS}" >/dev/null 2>&1; CUR_NS=""; }; return 0; } +cleanup() { + drop_ns + sudo docker rm -f gate5g-victim-holder gate5g-victim $(sudo docker ps -aq --filter "name=gate5g-") >/dev/null 2>&1 || true + sudo docker network rm "${NET}" >/dev/null 2>&1 || true + pkill -f "gate5g-host-listener" >/dev/null 2>&1 || true +} +trap cleanup EXIT +cleanup + +rm -f "${RESOLV_FILE}" +printf 'nameserver %s\noptions timeout:2 attempts:2\n' "${RESOLVER}" > "${RESOLV_FILE}" +chmod 0444 "${RESOLV_FILE}" +sudo docker network create --subnet "${SUBNET}" --gateway "${GATEWAY}" "${NET}" >/dev/null + +setsid python3 -c " +import socket +s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) +s.bind(('0.0.0.0',${HOST_PORT})); s.listen(8) # gate5g-host-listener +while True: + c,_=s.accept(); c.sendall(b'REACHED'); c.close() +" >/dev/null 2>&1 /dev/null +sudo timeout 120 docker run --rm --interactive --network "container:gate5g-victim-holder" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${PLAN_FILE}" >/dev/null +# The victim's listener runs under **runsc**, and that detail decides whether this leg is +# evidence at all. A first run used a runc listener and read `timeout` — which looked like +# containment and was nothing of the kind: the victim carries the baseline plan too, the +# plan DOES bind a runc process, so the victim's own OUTPUT rules dropped its replies to a +# 172.16/12 peer. The probe measured a victim that could not answer, not an attacker that +# could not reach. Under runsc the victim is unbound exactly as the attacker is, which is +# the real arrangement on a baseline seat: every job is a gVisor job. +sudo timeout 60 docker run --detach --name gate5g-victim --runtime runsc \ + --network "container:gate5g-victim-holder" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges --entrypoint node "${IMAGE}" \ + -e "require('http').createServer((q,r)=>r.end('REACHED')).listen(8080,'0.0.0.0')" >/dev/null +sleep 4 +# Proof the victim is actually serving before any conclusion is drawn from a timeout. +VICTIM_UP="$(sudo timeout 60 docker run --rm --network "container:gate5g-victim-holder" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges --entrypoint node "${IMAGE}" \ + -e 'const n=require("net");const s=n.connect({host:"127.0.0.1",port:8080,timeout:5000}); + s.on("connect",()=>{console.log("VICTIM-SERVING");s.destroy();}); + s.on("timeout",()=>{console.log("VICTIM-DEAD timeout");s.destroy();}); + s.on("error",e=>console.log("VICTIM-DEAD "+e.code));' 2>&1 | tr -d '\r' | tail -1)" +echo "victim liveness (from inside its own namespace): ${VICTIM_UP}" +echo "live listeners: victim job ${VICTIM_ADDR}:8080 (inside 172.16.0.0/12, which the baseline plan DROPs) · host ${HOST_LAN}:${HOST_PORT}" + +in_ns() { local rt="$1"; shift; local script="$1"; shift + sudo timeout 180 docker run --rm --runtime "${rt}" --network "container:${CUR_NS}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint node "${IMAGE}" -e "${script}" "$@" 2>&1 | tr -d '\r' | tail -1 +} +HEALTH=' +const os=require("os"); +const v4=Object.values(os.networkInterfaces()).flat().filter(x=>x&&x.family==="IPv4"&&!x.internal); +console.log(v4.length?("HEALTHY "+v4[0].address):"SICK no-address"); +' +PROBE=' +const net=require("net"); +const s=net.connect({host:process.argv[1],port:Number(process.argv[2]),timeout:8000}); +let said=false; const say=(w)=>{ if(!said){said=true;console.log(w);} s.destroy(); }; +s.on("connect",()=>say("REACHED")); +s.on("timeout",()=>say("timeout")); +s.on("error",(e)=>say(e.code)); +' +# A fresh attacker namespace carrying the baseline plan, verified installed before use. +fresh_ns() { + drop_ns + NS_SEQ=$((NS_SEQ+1)); CUR_NS="gate5g-attacker-${NS_SEQ}" + sudo timeout 120 docker run --detach --name "${CUR_NS}" --network "${NET}" \ + --read-only --cap-drop ALL --security-opt no-new-privileges --user 65534:65534 \ + --entrypoint sleep "${IMAGE}" infinity >/dev/null + local applied + applied="$(sudo timeout 120 docker run --rm --interactive --network "container:${CUR_NS}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${PLAN_FILE}" 2>&1 | tr -d '\r' | tail -1)" + # Readback: the plan must be IN FORCE, or a REACHED below would only mean "no rules". + local readback + readback="$(sudo timeout 60 docker run --rm --network "container:${CUR_NS}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + --entrypoint iptables "${NETFILTER_IMAGE}" -S 2>&1 | grep -c "172.16.0.0/12")" + echo " [ns ${CUR_NS}: applier=${applied}/$(wc -l < "${PLAN_FILE}") rules, kernel readback shows ${readback} rule(s) for 172.16.0.0/12]" +} +leg() { # label runtime host port + local label="$1" rt="$2" host="$3" port="$4" + fresh_ns + local h; h="$(in_ns runc "${HEALTH}")" + case "${h}" in HEALTHY*) ;; *) echo " ${label}: UNSOUND — ${h}"; return ;; esac + local got; got="$(in_ns "${rt}" "${PROBE}" "${host}" "${port}")" + echo " ${label}: ${got}" + [ "${got}" = REACHED ] && FINDINGS=$((FINDINGS+1)) + return 0 +} + +echo +echo "=== gate5g: 1. cross-job on the baseline shared network, live victim ===" +echo " (valid only if the victim is SERVING above; a dead victim makes every timeout meaningless)" +leg "runc -> victim ${VICTIM_ADDR}:8080 (positive control: the plan MUST bind runc)" runc "${VICTIM_ADDR}" 8080 +leg "runsc -> victim ${VICTIM_ADDR}:8080 (the vulnerability)" runsc "${VICTIM_ADDR}" 8080 + +echo +echo "=== gate5g: 2. private egress to the host itself, live listener ===" +leg "runc -> host ${HOST_LAN}:${HOST_PORT} (positive control)" runc "${HOST_LAN}" "${HOST_PORT}" +leg "runsc -> host ${HOST_LAN}:${HOST_PORT} (the vulnerability)" runsc "${HOST_LAN}" "${HOST_PORT}" + +echo +echo "=== gate5g: verdict ===" +echo "baseline REACHED count (each is a containment failure at that baseline commit): ${FINDINGS}" +if [ "${FINDINGS}" -gt 0 ]; then + echo "BASELINE-VULNERABLE: CONFIRMED on b45f8651dc9cab5b71c962eadd7b84840f579106 — the netns plan binds runc and not runsc," + echo "with no host-side enforcement present to catch it. The branch does not introduce this." +else + echo "BASELINE-VULNERABLE: NOT REPRODUCED — the 'predates the branch' claim must be WITHDRAWN." +fi diff --git a/docs/gvisor-dns-delivery/scripts/gate5h-lifecycle-and-recycled-address.sh b/docs/gvisor-dns-delivery/scripts/gate5h-lifecycle-and-recycled-address.sh new file mode 100644 index 000000000..8901fd603 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5h-lifecycle-and-recycled-address.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# Gate 5h: lifecycle — teardown leaves nothing behind, and a RECYCLED address is safe. +# +# Maxie: "Prove selected enforcement path handles runsc, lifecycle cleanup/recreation and +# fail-closed setup/readiness." Gate 5 proved the rules DENY. It never proved they GO AWAY. +# +# That gap is the dangerous one, and in both directions: +# * A leftover rule keyed to job A's address does not stop existing when A does. Docker +# hands addresses back. The next job to get 172.x.0.2 inherits a firewall written for +# someone else — denied traffic it should be allowed, or worse, an ACCEPT pinhole that +# was A's proxy and is now a stranger's open door. +# * A network that fails to delete wedges the next job with the same id at +# "network already exists", which is a fail-OPEN if anything in the caller shrugs at it. +# +# So this gate refuses to accept "teardown ran" as teardown. It counts rules keyed to the +# address in the real chains before and after, recycles the address deliberately, and makes +# the recycled job prove containment on its OWN rules against a LIVE listener with runc as +# the positive control. +# +# Every rule installed here is rendered by the PRODUCT (`render_host_plan`), never +# transcribed. Runs only in the disposable gvisor-repro VM; the trap removes every rule and +# network it created, and the final check asserts the chains are back to their start depth. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +WT="${WT:-/Users/forge/forge/v2/wt/w-gvisor-dns-delivery-r2}" +PLANDIR="${PLANDIR:-$HOME/gate5h-plans}" +SUBNET="${SUBNET:-172.31.55.0/24}" +HOST_LAN="${HOST_LAN:-192.168.5.15}" +HOST_PORT="${HOST_PORT:-49256}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate5h-resolv.conf}" +OUT="${OUT:-$HOME/gate5h-evidence.txt}" + +exec > >(tee "${OUT}") 2>&1 +FAIL=0 +CREATED_NETS=() +CUR_NS="" +NS_SEQ=0 + +fail() { echo " FAIL: $*"; FAIL=$((FAIL+1)); } +ok() { echo " ok: $*"; } + +# --- rule accounting ------------------------------------------------------------------- +# Counts rules keyed to an address in the two chains the product writes to. This is the +# measurement the whole gate turns on, so it reads the KERNEL, never a script variable. +rules_for() { local addr="$1" + local d i + d=$(sudo iptables -S DOCKER-USER 2>/dev/null | grep -c -- "${addr}") + i=$(sudo iptables -S INPUT 2>/dev/null | grep -c -- "${addr}") + echo $((d + i)) +} +chain_depth() { echo "$(( $(sudo iptables -S DOCKER-USER | wc -l) + $(sudo iptables -S INPUT | wc -l) ))"; } + +apply_plan() { # plan-file -> applies on the HOST netns, exactly as the product's applier does + sudo timeout 120 docker run --rm --interactive --network host \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "$1" >/dev/null 2>&1 +} + +drop_ns() { [ -n "${CUR_NS}" ] && { sudo docker rm -f "${CUR_NS}" >/dev/null 2>&1; CUR_NS=""; }; return 0; } + +cleanup() { + drop_ns + sudo docker rm -f $(sudo docker ps -aq --filter "name=gate5h-") >/dev/null 2>&1 || true + for f in "${PLANDIR}"/*-teardown.txt; do [ -f "$f" ] && apply_plan "$f"; done + for n in "${CREATED_NETS[@]:-}"; do [ -n "$n" ] && sudo docker network rm "$n" >/dev/null 2>&1; done + pkill -f "gate5h-host-listener" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +echo "=== gate5h: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) runsc=$(runsc --version | head -1)" +[ -d "${PLANDIR}" ] || { echo "MISSING ${PLANDIR} — render product plans first"; exit 2; } +START_DEPTH="$(chain_depth)" +echo "chain depth at start (DOCKER-USER + INPUT): ${START_DEPTH}" + +printf 'nameserver 1.1.1.1\noptions timeout:2 attempts:2\n' > "${RESOLV_FILE}"; chmod 0444 "${RESOLV_FILE}" + +setsid python3 -c " +import socket +s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) +s.bind(('0.0.0.0',${HOST_PORT})); s.listen(8) # gate5h-host-listener +while True: + c,_=s.accept(); c.sendall(b'REACHED'); c.close() +" >/dev/null 2>&1 prints addr + local net="$1" holder="$2" + sudo docker network create --driver bridge --subnet "${SUBNET}" "${net}" >/dev/null 2>&1 + CREATED_NETS+=("${net}") + sudo timeout 120 docker run --detach --name "${holder}" --network "${net}" \ + --read-only --cap-drop ALL --security-opt no-new-privileges --user 65534:65534 \ + --entrypoint sleep "${IMAGE}" infinity >/dev/null 2>&1 + sudo docker inspect --format "{{(index .NetworkSettings.Networks \"${net}\").IPAddress}}" "${holder}" 2>/dev/null +} + +probe_in() { # runtime holder script args... -> last line + local rt="$1" holder="$2" script="$3"; shift 3 + sudo timeout 180 docker run --rm --runtime "${rt}" --network "container:${holder}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint node "${IMAGE}" -e "${script}" "$@" 2>&1 | tr -d '\r' | tail -1 +} + +# --------------------------------------------------------------------------------------- +echo +echo "=== gate5h: 1. job A establishes, rules appear, containment holds ===" +NET_A="maxplayer-dns-gate5h-job-alpha" +ADDR_A="$(establish_holder "${NET_A}" gate5h-holder-a)" +[ -n "${ADDR_A}" ] || { echo "FATAL: empty address for A"; exit 2; } +echo " job A address (read from docker inspect): ${ADDR_A}" +BEFORE_A="$(rules_for "${ADDR_A}")" +apply_plan "${PLANDIR}/${ADDR_A}-install.txt" +AFTER_A="$(rules_for "${ADDR_A}")" +echo " rules keyed to ${ADDR_A}: before=${BEFORE_A} after-install=${AFTER_A}" +[ "${AFTER_A}" -gt "${BEFORE_A}" ] && ok "install added ${AFTER_A} host rules" || fail "install added no rules" +H="$(probe_in runc gate5h-holder-a "${HEALTH}")"; echo " namespace health: ${H}" +GOT="$(probe_in runsc gate5h-holder-a "${PROBE}" "${HOST_LAN}" "${HOST_PORT}")" +echo " runsc -> live host ${HOST_LAN}:${HOST_PORT}: ${GOT}" +[ "${GOT}" = REACHED ] && fail "A not contained" || ok "A contained while its rules are installed" + +# --------------------------------------------------------------------------------------- +echo +echo "=== gate5h: 2. job A tears down — rules must be GONE, network must be GONE ===" +sudo docker rm -f gate5h-holder-a >/dev/null 2>&1 +apply_plan "${PLANDIR}/${ADDR_A}-teardown.txt" +LEFT_A="$(rules_for "${ADDR_A}")" +echo " rules keyed to ${ADDR_A} after teardown: ${LEFT_A}" +[ "${LEFT_A}" -eq 0 ] && ok "no leftover host rules" || fail "${LEFT_A} rule(s) survived teardown" +sudo docker network rm "${NET_A}" >/dev/null 2>&1 +sudo docker network inspect "${NET_A}" >/dev/null 2>&1 && fail "network ${NET_A} survived" || ok "network removed" + +# --------------------------------------------------------------------------------------- +echo +echo "=== gate5h: 3. RECYCLED address — job B takes A's old address ===" +NET_B="maxplayer-dns-gate5h-job-bravo" +ADDR_B="$(establish_holder "${NET_B}" gate5h-holder-b)" +echo " job B address: ${ADDR_B} (A's was ${ADDR_A})" +if [ "${ADDR_B}" = "${ADDR_A}" ]; then + ok "address genuinely recycled — this is the case that matters" +else + echo " NOTE: docker handed a different address; recycling not exercised this run" +fi +STALE_B="$(rules_for "${ADDR_B}")" +echo " rules keyed to ${ADDR_B} BEFORE B installs its own: ${STALE_B}" +[ "${STALE_B}" -eq 0 ] && ok "B inherits no stale firewall" || fail "B inherited ${STALE_B} stale rule(s)" +# B must be contained by B's OWN rules, proven against a live listener with a runc control. +GOT_BARE="$(probe_in runsc gate5h-holder-b "${PROBE}" "${HOST_LAN}" "${HOST_PORT}")" +echo " runsc -> live host BEFORE B's rules (bare control, expect REACHED): ${GOT_BARE}" +[ "${GOT_BARE}" = REACHED ] || echo " NOTE: bare leg did not reach; the denial below proves less than intended" +apply_plan "${PLANDIR}/${ADDR_B}-install.txt" +GOT_B="$(probe_in runsc gate5h-holder-b "${PROBE}" "${HOST_LAN}" "${HOST_PORT}")" +echo " runsc -> live host AFTER B's rules: ${GOT_B}" +[ "${GOT_B}" = REACHED ] && fail "recycled job B not contained" || ok "B contained by its own rules" +apply_plan "${PLANDIR}/${ADDR_B}-teardown.txt" +sudo docker rm -f gate5h-holder-b >/dev/null 2>&1 +sudo docker network rm "${NET_B}" >/dev/null 2>&1 + +# --------------------------------------------------------------------------------------- +echo +echo "=== gate5h: 4. RECREATION — the same job id twice must not wedge ===" +NET_R="maxplayer-dns-gate5h-job-repeat" +A1="$(establish_holder "${NET_R}" gate5h-holder-r1)" +echo " first incarnation address: ${A1:-}" +sudo docker rm -f gate5h-holder-r1 >/dev/null 2>&1 +sudo docker network rm "${NET_R}" >/dev/null 2>&1 +A2="$(establish_holder "${NET_R}" gate5h-holder-r2)" +echo " second incarnation address: ${A2:-}" +if [ -n "${A1}" ] && [ -n "${A2}" ]; then ok "same job id came up twice, no 'network already exists' wedge" +else fail "recreation wedged (empty address on one incarnation)"; fi +sudo docker rm -f gate5h-holder-r2 >/dev/null 2>&1 +sudo docker network rm "${NET_R}" >/dev/null 2>&1 + +# --------------------------------------------------------------------------------------- +echo +echo "=== gate5h: verdict ===" +END_DEPTH="$(chain_depth)" +echo "chain depth: start=${START_DEPTH} end=${END_DEPTH}" +[ "${END_DEPTH}" -eq "${START_DEPTH}" ] && ok "chains returned to starting depth — the gate left nothing behind" \ + || fail "chains changed depth ${START_DEPTH} -> ${END_DEPTH}: this gate leaked rules" +echo "failing checks: ${FAIL}" +[ "${FAIL}" -eq 0 ] && echo "GATE 5h: PASS" || echo "GATE 5h: FAIL" diff --git a/docs/gvisor-dns-delivery/scripts/gate5i-fail-closed-live.sh b/docs/gvisor-dns-delivery/scripts/gate5i-fail-closed-live.sh new file mode 100644 index 000000000..1b301d32c --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5i-fail-closed-live.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +# Gate 5i: fail-closed — a job must not run on a network the product could not contain. +# +# Maxie: "Prove selected enforcement path handles runsc, lifecycle cleanup/recreation and +# fail-closed setup/readiness." +# +# `establish()` refuses at four points and the rendering tests now lock the plan invariants. +# What neither covers is the claim the code makes about the WORLD, in this comment: +# +# "A truncated stdin applies cleanly and exits 0, so no exit code reveals it; only +# comparing the sidecar's own total against what was rendered does." +# +# If that is wrong — if a truncated plan failed loudly — the count cross-check would be +# belt-and-braces. If it is right, the cross-check is the ONLY thing standing between a +# half-installed firewall and a job that believes it is contained. That is worth measuring +# rather than asserting, so leg C measures it, and then asks the question that actually +# matters: with a partial firewall, is the job still contained? Against a LIVE listener. +# +# Product-rendered plans only. Runs in the disposable gvisor-repro VM. The trap removes +# every rule it created and the verdict asserts the chains came back to their start depth. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +PLANDIR="${PLANDIR:-$HOME/gate5h-plans}" +SUBNET="${SUBNET:-172.31.56.0/24}" +NET="${NET:-maxplayer-dns-gate5i-job}" +HOST_LAN="${HOST_LAN:-192.168.5.15}" +HOST_PORT="${HOST_PORT:-49257}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate5i-resolv.conf}" +OUT="${OUT:-$HOME/gate5i-evidence.txt}" + +exec > >(tee "${OUT}") 2>&1 +FAIL=0 +fail() { echo " FAIL: $*"; FAIL=$((FAIL+1)); } +ok() { echo " ok: $*"; } + +rules_for() { local a="$1" + echo $(( $(sudo iptables -S DOCKER-USER 2>/dev/null | grep -c -- "${a}") \ + + $(sudo iptables -S INPUT 2>/dev/null | grep -c -- "${a}") )); } +chain_depth() { echo "$(( $(sudo iptables -S DOCKER-USER | wc -l) + $(sudo iptables -S INPUT | wc -l) ))"; } + +cleanup() { + sudo docker rm -f $(sudo docker ps -aq --filter "name=gate5i-") >/dev/null 2>&1 || true + [ -n "${ADDR:-}" ] && [ -f "${PLANDIR}/${ADDR}-teardown.txt" ] && \ + sudo timeout 120 docker run --rm --interactive --network host --cap-drop ALL \ + --cap-add NET_ADMIN --security-opt no-new-privileges "${NETFILTER_IMAGE}" \ + < "${PLANDIR}/${ADDR}-teardown.txt" >/dev/null 2>&1 + sudo docker network rm "${NET}" >/dev/null 2>&1 || true + pkill -f "gate5i-host-listener" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +echo "=== gate5i: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) runsc=$(runsc --version | head -1)" +START_DEPTH="$(chain_depth)"; echo "chain depth at start: ${START_DEPTH}" + +printf 'nameserver 1.1.1.1\noptions timeout:2 attempts:2\n' > "${RESOLV_FILE}"; chmod 0444 "${RESOLV_FILE}" +setsid python3 -c " +import socket +s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) +s.bind(('0.0.0.0',${HOST_PORT})); s.listen(8) # gate5i-host-listener +while True: + c,_=s.accept(); c.sendall(b'REACHED'); c.close() +" >/dev/null 2>&1 /dev/null 2>&1 +sudo timeout 120 docker run --detach --name gate5i-holder --network "${NET}" \ + --read-only --cap-drop ALL --security-opt no-new-privileges --user 65534:65534 \ + --entrypoint sleep "${IMAGE}" infinity >/dev/null 2>&1 +ADDR="$(sudo docker inspect --format "{{(index .NetworkSettings.Networks \"${NET}\").IPAddress}}" gate5i-holder)" +echo "job address: ${ADDR}" +PLAN="${PLANDIR}/${ADDR}-install.txt" +[ -s "${PLAN}" ] || { echo "MISSING ${PLAN} — render it with render_host_plan"; exit 2; } +EXPECTED="$(grep -c . "${PLAN}")" +echo "rendered plan: ${EXPECTED} rules" + +PROBE=' +const net=require("net"); +const s=net.connect({host:process.argv[1],port:Number(process.argv[2]),timeout:8000}); +let said=false; const say=(w)=>{ if(!said){said=true;console.log(w);} s.destroy(); }; +s.on("connect",()=>say("REACHED")); s.on("timeout",()=>say("timeout")); s.on("error",e=>say(e.code)); +' +probe() { sudo timeout 180 docker run --rm --runtime "$1" --network "container:gate5i-holder" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint node "${IMAGE}" -e "${PROBE}" \ + "${HOST_LAN}" "${HOST_PORT}" 2>&1 | tr -d '\r' | tail -1; } + +# --------------------------------------------------------------------------------------- +echo +echo "=== gate5i: A. applier cannot start (bad image) — nothing may be installed ===" +OUT_A="$(sudo timeout 120 docker run --rm --interactive --network host --cap-drop ALL \ + --cap-add NET_ADMIN --security-opt no-new-privileges \ + "ghcr.io/makeprisms/maxplayer-netfilter:definitely-not-a-tag" < "${PLAN}" 2>&1 | tail -1)" +RC_A=$? +echo " applier said: $(echo "${OUT_A}" | cut -c1-90)" +LEFT_A="$(rules_for "${ADDR}")" +echo " rules keyed to ${ADDR}: ${LEFT_A}" +[ "${LEFT_A}" -eq 0 ] && ok "a non-starting applier installs nothing (establish() maps this to a hard error)" \ + || fail "${LEFT_A} rules present after the applier failed to start" + +echo +echo "=== gate5i: B. applier without NET_ADMIN — must not silently succeed ===" +OUT_B="$(sudo timeout 120 docker run --rm --interactive --network host --cap-drop ALL \ + --security-opt no-new-privileges "${NETFILTER_IMAGE}" < "${PLAN}" 2>&1 | tail -1)" +echo " applier said: $(echo "${OUT_B}" | cut -c1-90)" +LEFT_B="$(rules_for "${ADDR}")" +echo " rules keyed to ${ADDR}: ${LEFT_B}" +if [ "${LEFT_B}" -eq "${EXPECTED}" ]; then + fail "the full policy installed without NET_ADMIN — the capability is not what gates this" +elif [ "${LEFT_B}" -eq 0 ]; then + ok "no capability, no rules" +else + echo " PARTIAL: ${LEFT_B} of ${EXPECTED} — exactly the half-installed state the count check exists for" +fi + +echo +echo "=== gate5i: C. TRUNCATED plan — does it apply cleanly and lie? ===" +TRUNC="$HOME/gate5i-truncated.txt" +head -9 "${PLAN}" > "${TRUNC}" +echo " feeding ${EXPECTED} rendered rules as a $(grep -c . "${TRUNC}")-line plan" +OUT_C="$(sudo timeout 120 docker run --rm --interactive --network host --cap-drop ALL \ + --cap-add NET_ADMIN --security-opt no-new-privileges "${NETFILTER_IMAGE}" < "${TRUNC}" 2>&1 | tail -1)" +RC_C=$? +echo " applier exit=${RC_C} said=$(echo "${OUT_C}" | cut -c1-60)" +INSTALLED_C="$(rules_for "${ADDR}")" +echo " rules actually in the kernel: ${INSTALLED_C} of ${EXPECTED} rendered" +if [ "${RC_C}" -eq 0 ] && [ "${INSTALLED_C}" -lt "${EXPECTED}" ]; then + ok "CONFIRMED: a truncated plan exits 0 while under-installing — only the count reveals it" +else + echo " NOTE: truncation did not present as a clean exit; the code comment overstates the risk" +fi +[ "${INSTALLED_C}" -ne "${EXPECTED}" ] && ok "count cross-check would refuse this job (${INSTALLED_C} != ${EXPECTED})" \ + || fail "count check could not distinguish the truncated plan" + +# The question that matters: a partial firewall is not a firewall. Measure it. +GOT_C="$(probe runsc)" +echo " runsc -> live host ${HOST_LAN}:${HOST_PORT} with a PARTIAL firewall: ${GOT_C}" +if [ "${GOT_C}" = REACHED ]; then + ok "a partially-installed policy leaves the job UNCONTAINED — refusing the job is the only safe move" +else + echo " NOTE: this particular truncation still denied the probed destination; another would not" +fi + +echo +echo "=== gate5i: D. teardown of a PARTIAL install ===" +# Two strategies, run back to back on the same partial install, because the difference between +# them IS the fix. D1 is what HostRules::drop did before gate 5i; it is kept as a live regression +# witness rather than deleted, so that if the applier's abort-on-first-failure behaviour ever +# changes, this gate says so instead of silently keeping a workaround nobody needs. +# +# Note honestly what this leg is: the Rust branch in HostRules::drop is covered by unit tests +# (`adopted_host_rules_are_not_complete_until_the_count_check_passes`, +# `the_per_rule_teardown_renders_one_valid_delete_per_rule`). What a shell gate can prove, and +# what these two legs do prove, is that the per-rule STRATEGY actually clears real rules from +# real chains where the one-shot inverse plan cannot. +echo " D1: one-shot inverse plan (the pre-fix behaviour)" +sudo timeout 120 docker run --rm --interactive --network host --cap-drop ALL \ + --cap-add NET_ADMIN --security-opt no-new-privileges "${NETFILTER_IMAGE}" \ + < "${PLANDIR}/${ADDR}-teardown.txt" >/dev/null 2>&1 +LEFT_D1="$(rules_for "${ADDR}")" +echo " rules remaining: ${LEFT_D1} (of ${INSTALLED_C} installed)" +if [ "${LEFT_D1}" -gt 0 ]; then + echo " as expected: the applier aborts on the first never-created rule and removes nothing" +else + echo " NOTE: the one-shot plan cleared it — the applier no longer aborts; revisit the fix" +fi + +echo " D2: per-rule teardown (what HostRules::drop now does on the partial path)" +REMOVED=0 +while IFS= read -r line; do + [ -n "${line}" ] || continue + if printf '%s\n' "${line}" | sudo timeout 60 docker run --rm --interactive --network host \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" >/dev/null 2>&1; then + REMOVED=$((REMOVED+1)) + fi +done < "${PLANDIR}/${ADDR}-teardown.txt" +LEFT_D="$(rules_for "${ADDR}")" +echo " removed ${REMOVED} rule(s) one at a time; ${LEFT_D} remain" +[ "${LEFT_D}" -eq 0 ] && ok "the partial install came out — a missing rule no longer strands the present ones" \ + || fail "${LEFT_D} rule(s) survived even the per-rule teardown" + +echo +echo "=== gate5i: verdict ===" +END_DEPTH="$(chain_depth)" +echo "chain depth: start=${START_DEPTH} end=${END_DEPTH}" +[ "${END_DEPTH}" -eq "${START_DEPTH}" ] && ok "chains returned to starting depth" \ + || fail "chains changed ${START_DEPTH} -> ${END_DEPTH}" +echo "failing checks: ${FAIL}" +[ "${FAIL}" -eq 0 ] && echo "GATE 5i: PASS" || echo "GATE 5i: FAIL" diff --git a/docs/gvisor-dns-delivery/scripts/gate5k-unsupported-daemon-default.sh b/docs/gvisor-dns-delivery/scripts/gate5k-unsupported-daemon-default.sh new file mode 100644 index 000000000..87e779ca9 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5k-unsupported-daemon-default.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# gate5k — does an UNSUPPORTED daemon default runtime fail CLOSED? +# +# Maxie's ruling (9 Sep 2026): "Missing --runtime means daemon default, not +# guaranteed runc; test unsupported defaults fail closed." +# +# The correction lands on the product. `holder_argv()` deliberately passes NO +# `--runtime`, and `the_containment_plane_never_carries_the_jobs_runtime` locks +# it there, because a runsc holder's namespace is unusable: a job joining it +# sees `lo` only (gate2a). The design therefore ASSUMES the daemon default is +# runc, and nothing in the product verifies that assumption. +# +# Fail CLOSED = the job gets no usable egress, or never starts. +# Fail OPEN = the job runs and REACHES a live listener with no containment. +# Only the second is a security defect; the first is an availability failure. +# +# v3. Two earlier drafts are committed as failures and kept: +# v1 gate5k-CONFOUNDED-harness-defect-*.txt — listener started with `docker +# exec` into a runsc holder, which runsc refuses; target never served. +# v2 gate5k-v2-daemon-default-runtime-*.txt — root cause found: the sandbox +# image has NO nc and NO wget, so every probe was doomed before it ran. +# This version uses the primitives the other gates already proved work in this +# image: `--entrypoint node` with inline JS. gate5f/5g/5h/5i never used nc or +# wget, which is why their SERVING/REACHED readings were real. +# +# Discipline: ONE gVisor container per namespace (gate5d: namespaces are +# single-use for gVisor), a fresh namespace per probe, a LIVE listener, and a +# liveness reading printed beside every leg so a silence is never mistaken for +# containment. +# +# Runs inside the disposable gvisor-repro VM ONLY: it edits +# /etc/docker/daemon.json and restarts docker. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +DAEMON_JSON=/etc/docker/daemon.json +BACKUP="/home/forge.guest/daemon.json.gate5k.bak" +NET="gate5k-net" +FAIL=0 +LISTEN_ADDR="" + +say() { printf '%s\n' "$*"; } +hr() { printf -- '---- %s\n' "$*"; } + +[ -f "${DAEMON_JSON}" ] || { say "ABORT: no ${DAEMON_JSON}"; exit 2; } +if command -v limactl >/dev/null 2>&1; then + say "ABORT: limactl present — this looks like the HOST, not the disposable VM"; exit 2 +fi + +SERVER_JS='require("http").createServer((q,r)=>r.end("REACHED")).listen(8080,"0.0.0.0");' + +LIVENESS_JS='const n=require("net");const s=n.connect({host:process.argv[1],port:8080,timeout:5000}); +s.on("connect",()=>{console.log("SERVING");s.destroy();}); +s.on("timeout",()=>{console.log("DEAD timeout");s.destroy();}); +s.on("error",e=>console.log("DEAD "+e.code));' + +PROBE_JS='const os=require("os"),http=require("http"); +const v4=Object.values(os.networkInterfaces()).flat().filter(x=>x&&x.family==="IPv4"&&!x.internal); +process.stdout.write(v4.length?("HEALTHY "+v4[0].address):"SICK no-address"); +const r=http.get({host:process.argv[1],port:8080,timeout:8000},res=>{let d=""; + res.on("data",c=>d+=c);res.on("end",()=>console.log(" | target="+d.trim()));}); +r.on("timeout",()=>{console.log(" | target=TIMEOUT");r.destroy();}); +r.on("error",e=>console.log(" | target="+e.code));' + +default_runtime() { sudo docker info --format '{{.DefaultRuntime}}' 2>/dev/null; } + +restore() { + hr "restoring ${DAEMON_JSON}" + if [ -f "${BACKUP}" ]; then + sudo cp "${BACKUP}" "${DAEMON_JSON}" + sudo systemctl restart docker || true + sleep 5 + say "restored default runtime = $(default_runtime)" + fi + sudo docker rm -f gate5k-listener gate5k-ns1 gate5k-ns2 >/dev/null 2>&1 + sudo docker network rm "${NET}" >/dev/null 2>&1 +} +trap restore EXIT + +sudo cp "${DAEMON_JSON}" "${BACKUP}" +say "gate5k v3 — unsupported daemon default runtime" +say "kernel: $(uname -srm)" +say "docker: $(sudo docker version --format '{{.Server.Version}}')" +say "runsc: $(runsc --version 2>/dev/null | head -1)" +say "ORIGINAL default runtime: $(default_runtime)" +say "" + +sudo docker network create --driver bridge "${NET}" >/dev/null 2>&1 + +# Test infrastructure, pinned to runc so it keeps working when the default flips +# underneath it. gate5g's confound (a listener's own OUTPUT rules dropping its +# replies) cannot arise here: gate5k installs no netns plan at all. +sudo docker rm -f gate5k-listener >/dev/null 2>&1 +sudo timeout 120 docker run --detach --name gate5k-listener --runtime runc \ + --network "${NET}" --entrypoint node "${IMAGE}" -e "${SERVER_JS}" >/dev/null 2>&1 + +serve() { # a daemon restart stops containers; revive and re-read the address + sudo docker start gate5k-listener >/dev/null 2>&1 + sleep 4 + LISTEN_ADDR="$(sudo docker inspect -f "{{(index .NetworkSettings.Networks \"${NET}\").IPAddress}}" gate5k-listener 2>/dev/null)" +} + +liveness() { + sudo timeout 60 docker run --rm --runtime runc --network "${NET}" \ + --entrypoint node "${IMAGE}" -e "${LIVENESS_JS}" "${LISTEN_ADDR}" 2>&1 | tr -d '\r' | tail -1 +} + +probe() { # ns_name — product-SHAPED holder carrying NO --runtime, then ONE gVisor job + local ns="$1" + sudo docker rm -f "${ns}" >/dev/null 2>&1 + sudo timeout 120 docker run --detach --name "${ns}" --network "${NET}" \ + --entrypoint sleep "${IMAGE}" infinity >/dev/null 2>&1 + local started=$? + sleep 4 + if ! sudo docker ps --format '{{.Names}}' | grep -qx "${ns}"; then + say " holder: DID NOT START (docker run exit=${started}) -> no job can run" + return 3 + fi + say " holder runtime: $(sudo docker inspect -f '{{.HostConfig.Runtime}}' "${ns}" 2>/dev/null)" + local out + out="$(sudo timeout 120 docker run --rm --runtime runsc --network "container:${ns}" \ + --entrypoint node "${IMAGE}" -e "${PROBE_JS}" "${LISTEN_ADDR}" 2>&1 | tr -d '\r' | tail -2 | tr '\n' ' ')" + if [ -z "${out}" ]; then + say " job: PRODUCED NO OUTPUT -> could not run in this namespace (fail closed by refusal)" + else + say " job: ${out}" + fi +} + +hr "LEG 1 — control: daemon default is runc (the supported configuration)" +say "default runtime now: $(default_runtime)" +serve +L1="$(liveness)" +say "listener ${LISTEN_ADDR} liveness: ${L1}" +if [ "${L1}" != "SERVING" ]; then + say "leg 1 target is dead — NO EVIDENCE"; FAIL=$((FAIL + 1)) +fi +probe gate5k-ns1 +say " ^ expected here: HEALTHY and target=REACHED. This leg is the" +say " positive control: it proves the harness CAN observe reachability, so" +say " leg 2's silence means something." +say "" + +hr "LEG 2 — daemon default switched to runsc (the UNSUPPORTED configuration)" +printf '{\n "default-runtime": "runsc",\n "runtimes": { "runsc": { "path": "/usr/local/bin/runsc" } }\n}\n' \ + | sudo tee "${DAEMON_JSON}" >/dev/null +sudo systemctl restart docker +sleep 7 +NOW="$(default_runtime)" +say "default runtime now: ${NOW}" +if [ "${NOW}" != "runsc" ]; then + say "could not switch the default runtime — leg 2 is NO EVIDENCE" + FAIL=$((FAIL + 1)) +else + serve + L2="$(liveness)" + say "listener ${LISTEN_ADDR} liveness: ${L2}" + if [ "${L2}" != "SERVING" ]; then + say "leg 2 target is dead — NO EVIDENCE, not a fail-closed result" + FAIL=$((FAIL + 1)) + fi + probe gate5k-ns2 +fi +say "" + +hr "VERDICT" +say "With a SERVING listener in both legs: leg 1 REACHED and leg 2 SICK/denied" +say "means the unsupported default FAILS CLOSED — no egress path to contain, an" +say "availability failure rather than a containment bypass. Leg 2 printing" +say "target=REACHED would mean it FAILS OPEN and is a security defect." +say "failing_checks=${FAIL} (non-zero means a leg above proved nothing)" +exit 0 diff --git a/docs/gvisor-dns-delivery/scripts/provision-repro-vm.sh b/docs/gvisor-dns-delivery/scripts/provision-repro-vm.sh new file mode 100755 index 000000000..09405e4e0 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/provision-repro-vm.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Provision a DISPOSABLE Linux VM for the gVisor DNS reproduction. +# Runs INSIDE the throwaway lima VM `gvisor-repro`. Never run on the forge host, +# and never against the shared colima VM: other lanes depend on that daemon. +# +# Installs: docker engine, gVisor runsc, registered as the `runsc` docker runtime. +# Everything it writes lives inside the VM and dies with `limactl delete gvisor-repro`. +set -euo pipefail + +RUNSC_RELEASE="${RUNSC_RELEASE:-20260831.0}" +ARCH="$(uname -m)" +LOG=/var/log/gvisor-provision.log + +log() { echo "[provision] $*"; } + +log "arch=${ARCH} kernel=$(uname -r) os=$(. /etc/os-release && echo "${PRETTY_NAME}")" + +export DEBIAN_FRONTEND=noninteractive +sudo -E apt-get update -qq +sudo -E apt-get install -y -qq docker.io curl ca-certificates git iproute2 dnsutils >/dev/null +sudo systemctl enable --now docker + +log "docker: $(docker --version)" + +# gVisor. Try the release named in the brief first; fall back to the current +# release only if that exact one has no artifact for this arch, and say so loudly +# so no report can silently claim the briefed version. +install_runsc() { + local rel="$1" base url tmp + base="https://storage.googleapis.com/gvisor/releases/release/${rel}/${ARCH}" + tmp="$(mktemp -d)" + for f in runsc containerd-shim-runsc-v1; do + url="${base}/${f}" + if ! curl -fsSL "${url}" -o "${tmp}/${f}"; then + echo "MISS ${url}" >&2 + rm -rf "${tmp}" + return 1 + fi + curl -fsSL "${url}.sha512" -o "${tmp}/${f}.sha512" || true + done + ( cd "${tmp}" && sha512sum -c ./*.sha512 ) || { echo "checksum failed for ${rel}" >&2; rm -rf "${tmp}"; return 1; } + sudo install -m 0755 -t /usr/local/bin "${tmp}/runsc" "${tmp}/containerd-shim-runsc-v1" + rm -rf "${tmp}" + echo "${rel}" | sudo tee /etc/gvisor-installed-release >/dev/null + return 0 +} + +if install_runsc "${RUNSC_RELEASE}"; then + log "runsc installed from briefed release ${RUNSC_RELEASE}" +else + log "WARNING: briefed release ${RUNSC_RELEASE} has no ${ARCH} artifact; falling back to 'latest'" + install_runsc latest + log "runsc installed from 'latest' — every result must record this substitution" +fi + +log "runsc: $(runsc --version | tr '\n' ' ')" + +sudo runsc install +sudo systemctl restart docker +sleep 3 +docker info --format 'runtimes={{.Runtimes}}' | tee -a "${LOG}" 2>/dev/null || docker info | grep -i runtime + +log "provision complete" diff --git a/docs/gvisor-dns-delivery/scripts/run-all-gates.sh b/docs/gvisor-dns-delivery/scripts/run-all-gates.sh new file mode 100755 index 000000000..52691435d --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/run-all-gates.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Runs the gVisor DNS/delivery gates in order, bounded, and reports a verdict per gate. +# +# The gates are the deliverable, but a gate nobody can re-run is a claim, not evidence. +# This is how someone else reproduces the whole set on a fresh VM without knowing which +# script needs which rendered plan. +# +# Two rules it will not bend: +# +# * **BOUNDED.** Every gate gets a `timeout`, and the run gets a total budget. A gate +# that hangs is a FAIL with a reason, never a run that sits there until a watchdog +# kills the session and leaves no evidence at all. +# * **A GATE THAT DID NOT RUN IS NOT A PASS.** Missing prerequisites print SKIPPED with +# the reason and count against the run. The failure mode this exists to prevent is a +# green summary produced by a suite that quietly executed nothing — which is exactly +# the class of bug gates 2 and 5 were caught making about their own probes. +# +# Prerequisites: the repro VM from provision-repro-vm.sh, and the rendered plans copied to +# $HOME (see PLANS below). Plans are rendered on the REPO host, because rendering them here +# would mean this script decides what the rules are — and then the gates would be testing +# the script instead of the product. +set -uo pipefail + +EVIDENCE_DIR="${EVIDENCE_DIR:-$HOME/gate-evidence}" +PER_GATE_TIMEOUT="${PER_GATE_TIMEOUT:-900}" +TOTAL_BUDGET="${TOTAL_BUDGET:-3600}" +SUMMARY="${SUMMARY:-$EVIDENCE_DIR/summary.txt}" +STARTED=$(date +%s) + +mkdir -p "${EVIDENCE_DIR}" +declare -a NAMES=() VERDICTS=() REASONS=() + +record() { NAMES+=("$1"); VERDICTS+=("$2"); REASONS+=("$3"); } + +# gate -> the files it needs in $HOME before it can say anything true +plans_for() { + case "$1" in + gate1) echo "" ;; + gate2) echo "gate2-plan.txt" ;; + gate4) echo "gate4-plan.txt" ;; + gate5) echo "gate5-plans/plan-d.txt gate5-plans/host-d.txt gate5-plans/host-d-teardown.txt" ;; + gate5f) echo "gate5f-plan.txt gate5f-host-install.txt gate5f-host-teardown.txt" ;; + esac +} +script_for() { + case "$1" in + gate1) echo "gate1-repro.sh" ;; + gate2) echo "gate2-namespace-dns-tls.sh" ;; + gate4) echo "gate4-container-git-delivery.sh" ;; + gate5) echo "gate5-denial-and-concurrent-success.sh" ;; + gate5f) echo "gate5f-product-host-rules-bind-runsc.sh" ;; + esac +} +# The line in a gate's own output that decides it. Grepping the gate's verdict rather than +# trusting its exit code is deliberate: several of these scripts run under `set -uo pipefail` +# without `-e` and exit 0 while reporting a failure inside. +verdict_of() { # gate logfile + local gate="$1" log="$2" + case "${gate}" in + gate1) + grep -q "EAI_AGAIN" "${log}" && grep -qi "runc" "${log}" && echo PASS || echo FAIL ;; + gate2) + grep -q "verified=true\|cert-verified" "${log}" && echo PASS || echo FAIL ;; + gate4) + grep -q "GATE4: PASS" "${log}" && echo PASS || echo FAIL ;; + gate5) + grep -q "GATE5-DENIAL: PASS" "${log}" && echo PASS || echo FAIL ;; + gate5f) + # No self-verdict line: it passes when the host-directed leg changed under the policy + # and the public route survived, and when nothing leaked. + grep -q "CLEAN" "${log}" && grep -q "PUBLIC-PASS git" "${log}" && echo PASS || echo FAIL ;; + esac +} + +run_gate() { # gate + local gate="$1" + local script; script="$(script_for "${gate}")" + local log="${EVIDENCE_DIR}/${gate}-$(date -u +%Y%m%dT%H%M%SZ).log" + + local elapsed=$(( $(date +%s) - STARTED )) + if [ "${elapsed}" -ge "${TOTAL_BUDGET}" ]; then + record "${gate}" SKIPPED "total budget ${TOTAL_BUDGET}s exhausted" + return + fi + if [ ! -x "${HOME}/${script}" ]; then + record "${gate}" SKIPPED "missing ${HOME}/${script}" + return + fi + local missing="" p + for p in $(plans_for "${gate}"); do + [ -s "${HOME}/${p}" ] || missing="${missing} ${p}" + done + if [ -n "${missing}" ]; then + record "${gate}" SKIPPED "missing rendered plan(s):${missing}" + return + fi + + echo "--- ${gate}: running (timeout ${PER_GATE_TIMEOUT}s) -> ${log}" + local budget_left=$(( TOTAL_BUDGET - elapsed )) + local limit=$(( PER_GATE_TIMEOUT < budget_left ? PER_GATE_TIMEOUT : budget_left )) + timeout "${limit}" "${HOME}/${script}" > "${log}" 2>&1 + local rc=$? + if [ "${rc}" -eq 124 ]; then + record "${gate}" FAIL "timed out after ${limit}s (partial log kept)" + return + fi + record "${gate}" "$(verdict_of "${gate}" "${log}")" "exit=${rc} log=$(basename "${log}")" +} + +echo "=== gVisor DNS/delivery gates ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) docker=$(sudo docker version --format '{{.Server.Version}}' 2>/dev/null) runsc=$(runsc --version 2>/dev/null | head -1)" +echo "per-gate timeout=${PER_GATE_TIMEOUT}s total budget=${TOTAL_BUDGET}s evidence=${EVIDENCE_DIR}" +echo "NOTE: results are labelled by architecture on purpose — x86_64 is NOT covered by this run." +echo + +GATES="${GATES:-gate1 gate2 gate4 gate5 gate5f}" +for g in ${GATES}; do run_gate "${g}"; done + +{ + echo "=== summary ===" + date -u +"utc=%Y-%m-%dT%H:%M:%SZ" + echo "arch=$(uname -m) runsc=$(runsc --version 2>/dev/null | head -1)" + fails=0 + for i in "${!NAMES[@]}"; do + printf '%-8s %-8s %s\n' "${NAMES[i]}" "${VERDICTS[i]}" "${REASONS[i]}" + [ "${VERDICTS[i]}" = PASS ] || fails=$((fails + 1)) + done + echo "total=$(( $(date +%s) - STARTED ))s gates=${#NAMES[@]} not-passing=${fails}" + # SKIPPED counts against the run. A suite that ran nothing must never read green. + [ "${fails}" -eq 0 ] && echo "ALL GATES: PASS" || echo "ALL GATES: FAIL (${fails} not passing)" +} | tee "${SUMMARY}"