From 081b3bc187bf8edfd5d2ab12339b3c11e7ff1cb5 Mon Sep 17 00:00:00 2001 From: w-pr-991-host-plane-recut Date: Mon, 14 Sep 2026 02:37:06 -0700 Subject: [PATCH] sandbox: contain a gVisor job on the host side, give each job its own network, and block readiness on the route a job takes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-cut of #991 onto post-#995 main. Everything #995 landed — the resolver selection in sandbox_dns, the DNS pinholes, the multiset verify_readback, the container-delivery call site — is dropped from this branch and taken from main unchanged. What is left is the host plane #995 does not touch. The host plane. NetPolicy renders rules for the job's own namespace, which binds a runc job completely and a gVisor job not at all: gVisor terminates the network inside the sandbox and writes frames to the veth, so the host kernel's OUTPUT chain in that namespace never sees the job's packets. Measured with the full 26-rule plan installed and read back, a runsc job REACHED a live listener inside -d 172.16.0.0/12 -j DROP while a runc job in that same namespace timed out. HostPolicy installs the same denies on the host side of the veth, in DOCKER-USER for what is routed onward and INPUT for what is addressed to the host itself — neither chain covers the other, also measured. Every rule is keyed -s /32, so it denies this job and nothing else on the box. A network per job. Two containers on one bridge are switched, not routed, and on a host without br_netfilter those frames enter no iptables chain at all: a gVisor job REACHED a listener in another job's namespace with the host rule installed. Per-job network leaves the job no on-link peer but its gateway, which is what makes every other destination routed and therefore visible to the host plane. [sandbox] network keeps its naming role and loses its sharing role. Lifecycle. JobNetwork and HostRules are Drop guards adopted the moment their resource exists. HostRules starts complete=false and only the count cross-check sets it, because the applier aborts on the first rule that was never created: a partial install torn down by the inverse plan removed nothing (9 installed, 9 still present), so an incomplete install unwinds rule-by-rule instead. Doctor. A blocking, transient `sandbox delivery route` row that asks the question from inside the namespace an awarded job gets, because a seat whose host resolves perfectly while its jobs die on EAI_AGAIN reads as READY to every host-side check. Tests: 12 host-policy unit tests, 3 netns lifecycle tests, 3 doctor route tests, and one combined-plane test asserting the host plane and main's namespace verifier agree on the same job, with a negative control. --- .../examples/render_host_plan.rs | 42 + crates/maxplayer-core/src/home.rs | 26 +- crates/maxplayer-core/src/sandbox_net.rs | 369 ++++++++ crates/maxplayer-core/src/sandbox_netns.rs | 384 ++++++++- crates/maxplayer/src/doctor.rs | 586 +++++++++++++ docs/gvisor-dns-delivery/DESIGN.md | 94 ++ docs/gvisor-dns-delivery/RUNLOG.md | 814 ++++++++++++++++++ .../gate1-runsc-vs-runc-20260910T0046Z.txt | 26 + .../evidence/gate2-plan-rendered.txt | 26 + ...ate2a-runsc-holder-FAIL-20260910T0145Z.txt | 66 ++ ...nc-plane-runsc-job-PASS-20260910T0210Z.txt | 102 +++ ...ainer-git-delivery-PASS-20260910T0330Z.txt | 42 + .../evidence/gate4-plan-rendered.txt | 26 + ...-and-concurrent-success-20260910T0245Z.txt | 46 + ...concurrent-success-PASS-20260910T0311Z.txt | 51 ++ .../evidence/gate5-plan-rendered.txt | 26 + .../evidence/gate5-plans/host-c1-teardown.txt | 17 + .../evidence/gate5-plans/host-c1.txt | 17 + .../evidence/gate5-plans/host-c2-teardown.txt | 17 + .../evidence/gate5-plans/host-c2.txt | 17 + .../evidence/gate5-plans/host-c3-teardown.txt | 17 + .../evidence/gate5-plans/host-c3.txt | 17 + .../evidence/gate5-plans/host-d-teardown.txt | 17 + .../evidence/gate5-plans/host-d.txt | 17 + .../evidence/gate5-plans/host-n-teardown.txt | 17 + .../evidence/gate5-plans/host-n.txt | 17 + .../evidence/gate5-plans/plan-c1.txt | 26 + .../evidence/gate5-plans/plan-c2.txt | 26 + .../evidence/gate5-plans/plan-c3.txt | 26 + .../evidence/gate5-plans/plan-d.txt | 26 + .../evidence/gate5-plans/plan-n.txt | 26 + ...-not-bind-runsc-FINDING-20260910T0250Z.txt | 25 + .../evidence/gate5b-plan-rendered.txt | 26 + .../evidence/gate5c-plan-rendered.txt | 26 + ...e-does-containment-bind-20260910T0244Z.txt | 28 + ...s-containment-bind-VOID-20260910T0310Z.txt | 27 + ...-use-for-gvisor-FINDING-20260910T0310Z.txt | 18 + .../evidence/gate5d-plan-rendered.txt | 26 + .../evidence/gate5e-plan-rendered.txt | 26 + ...enforcement-under-runsc-20260910T0305Z.txt | 33 + .../evidence/gate5f-host-install-rendered.txt | 17 + .../gate5f-host-teardown-rendered.txt | 17 + .../evidence/gate5f-plan-rendered.txt | 26 + ...t-host-rules-bind-runsc-20260910T0305Z.txt | 43 + .../gate5g-baseline-plan-from-origin-main.txt | 24 + ...-CONFOUNDED-runc-victim-20260910T0325Z.txt | 25 + ...vulnerability-CONFIRMED-20260910T0330Z.txt | 27 + ...e-recycled-address-PASS-20260910T0332Z.txt | 38 + .../gate5h-plans/172.31.55.2-install.txt | 17 + .../gate5h-plans/172.31.55.2-teardown.txt | 17 + .../gate5h-plans/172.31.56.2-install.txt | 17 + .../gate5h-plans/172.31.56.2-teardown.txt | 17 + ...osed-FAIL-teardown-leak-20260910T0339Z.txt | 36 + ...l-closed-PASS-after-fix-20260910T0347Z.txt | 41 + .../gate5j-ipv6-finding-20260910T0347Z.txt | 22 + ...NFOUNDED-harness-defect-20260910T0310Z.txt | 32 + ...-daemon-default-runtime-20260910T0325Z.txt | 27 + ...-v3-PASS-daemon-default-20260910T0345Z.txt | 29 + ...run-all-gates-proof-run-20260910T0316Z.txt | 15 + ...all-gates-proof-summary-20260910T0316Z.txt | 7 + .../scripts/gate1-repro.sh | 67 ++ .../scripts/gate2-namespace-dns-tls.sh | 179 ++++ .../scripts/gate4-container-git-delivery.sh | 174 ++++ .../gate5-denial-and-concurrent-success.sh | 269 ++++++ .../gate5b-does-the-plan-bind-a-runsc-job.sh | 109 +++ .../gate5c-where-does-containment-bind.sh | 189 ++++ .../gate5d-is-the-namespace-single-use.sh | 85 ++ .../gate5e-routed-enforcement-under-runsc.sh | 211 +++++ .../gate5f-product-host-rules-bind-runsc.sh | 258 ++++++ .../gate5g-baseline-vulnerability-repro.sh | 177 ++++ .../gate5h-lifecycle-and-recycled-address.sh | 201 +++++ .../scripts/gate5i-fail-closed-live.sh | 194 +++++ .../gate5k-unsupported-daemon-default.sh | 172 ++++ .../scripts/provision-repro-vm.sh | 63 ++ .../scripts/run-all-gates.sh | 132 +++ 75 files changed, 6221 insertions(+), 14 deletions(-) create mode 100644 crates/maxplayer-core/examples/render_host_plan.rs create mode 100644 docs/gvisor-dns-delivery/DESIGN.md create mode 100644 docs/gvisor-dns-delivery/RUNLOG.md create mode 100644 docs/gvisor-dns-delivery/evidence/gate1-runsc-vs-runc-20260910T0046Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate2-plan-rendered.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate2a-runsc-holder-FAIL-20260910T0145Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate2b-runc-plane-runsc-job-PASS-20260910T0210Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate4-container-git-delivery-PASS-20260910T0330Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate4-plan-rendered.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-denial-and-concurrent-success-20260910T0245Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-denial-and-concurrent-success-PASS-20260910T0311Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-plan-rendered.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-plans/host-c1-teardown.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-plans/host-c1.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-plans/host-c2-teardown.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-plans/host-c2.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-plans/host-c3-teardown.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-plans/host-c3.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-plans/host-d-teardown.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-plans/host-d.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-plans/host-n-teardown.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-plans/host-n.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c1.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c2.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c3.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-plans/plan-d.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5-plans/plan-n.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5b-plan-does-not-bind-runsc-FINDING-20260910T0250Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5b-plan-rendered.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5c-plan-rendered.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5c-where-does-containment-bind-20260910T0244Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5c-where-does-containment-bind-VOID-20260910T0310Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5d-namespace-is-single-use-for-gvisor-FINDING-20260910T0310Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5d-plan-rendered.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5e-plan-rendered.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5e-routed-enforcement-under-runsc-20260910T0305Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5f-host-install-rendered.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5f-host-teardown-rendered.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5f-plan-rendered.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5f-product-host-rules-bind-runsc-20260910T0305Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5g-baseline-plan-from-origin-main.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5g-baseline-repro-CONFOUNDED-runc-victim-20260910T0325Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5g-baseline-vulnerability-CONFIRMED-20260910T0330Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5h-lifecycle-recycled-address-PASS-20260910T0332Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.55.2-install.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.55.2-teardown.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.56.2-install.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.56.2-teardown.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5i-fail-closed-FAIL-teardown-leak-20260910T0339Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5i-fail-closed-PASS-after-fix-20260910T0347Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5j-ipv6-finding-20260910T0347Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5k-CONFOUNDED-harness-defect-20260910T0310Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5k-v2-daemon-default-runtime-20260910T0325Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/gate5k-v3-PASS-daemon-default-20260910T0345Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/run-all-gates-proof-run-20260910T0316Z.txt create mode 100644 docs/gvisor-dns-delivery/evidence/run-all-gates-proof-summary-20260910T0316Z.txt create mode 100755 docs/gvisor-dns-delivery/scripts/gate1-repro.sh create mode 100755 docs/gvisor-dns-delivery/scripts/gate2-namespace-dns-tls.sh create mode 100755 docs/gvisor-dns-delivery/scripts/gate4-container-git-delivery.sh create mode 100755 docs/gvisor-dns-delivery/scripts/gate5-denial-and-concurrent-success.sh create mode 100755 docs/gvisor-dns-delivery/scripts/gate5b-does-the-plan-bind-a-runsc-job.sh create mode 100755 docs/gvisor-dns-delivery/scripts/gate5c-where-does-containment-bind.sh create mode 100755 docs/gvisor-dns-delivery/scripts/gate5d-is-the-namespace-single-use.sh create mode 100755 docs/gvisor-dns-delivery/scripts/gate5e-routed-enforcement-under-runsc.sh create mode 100755 docs/gvisor-dns-delivery/scripts/gate5f-product-host-rules-bind-runsc.sh create mode 100755 docs/gvisor-dns-delivery/scripts/gate5g-baseline-vulnerability-repro.sh create mode 100644 docs/gvisor-dns-delivery/scripts/gate5h-lifecycle-and-recycled-address.sh create mode 100644 docs/gvisor-dns-delivery/scripts/gate5i-fail-closed-live.sh create mode 100644 docs/gvisor-dns-delivery/scripts/gate5k-unsupported-daemon-default.sh create mode 100755 docs/gvisor-dns-delivery/scripts/provision-repro-vm.sh create mode 100755 docs/gvisor-dns-delivery/scripts/run-all-gates.sh diff --git a/crates/maxplayer-core/examples/render_host_plan.rs b/crates/maxplayer-core/examples/render_host_plan.rs new file mode 100644 index 00000000..fc91666f --- /dev/null +++ b/crates/maxplayer-core/examples/render_host_plan.rs @@ -0,0 +1,42 @@ +//! Prints the **host-side** iptables plan [`HostPolicy`] would hand its applier, for gate evidence. +//! +//! Sibling of `render_net_plan`, and it exists for the same reason: a gate script must install the +//! rules the PRODUCT renders, never rules a script author transcribed. A transcription drifts the +//! moment the policy changes, and the gate then keeps passing against a firewall the product no +//! longer builds — the exact failure these gates exist to catch. +//! +//! It matters more here than for the namespace plan. The host plan lands in chains that are shared +//! with every other container on the daemon, so a hand-written approximation of it in a script is +//! not just drift, it is a rule keyed to the wrong source touching someone else's traffic. +//! +//! ```text +//! cargo run -p maxplayer-core --example render_host_plan -- 172.18.0.2 +//! cargo run -p maxplayer-core --example render_host_plan -- 172.18.0.2 --teardown +//! ``` +//! +//! Arguments: the job namespace's address, then optionally `--teardown` for the exact inverse plan. + +use maxplayer_core::sandbox_net::HostPolicy; +use maxplayer_core::sandbox_netns::{host_install_stdin, host_teardown_stdin}; + +fn main() { + let mut args = std::env::args().skip(1); + let Some(job_addr) = args.next() else { + eprintln!("usage: render_host_plan [--teardown]"); + std::process::exit(2); + }; + let teardown = match args.next().as_deref() { + None => false, + Some("--teardown") => true, + Some(other) => { + eprintln!("unknown argument {other:?} — expected --teardown or nothing"); + std::process::exit(2); + } + }; + + let policy = HostPolicy { job_addr }; + let (plan, count) = + if teardown { host_teardown_stdin(&policy) } else { host_install_stdin(&policy) }; + eprintln!("# {count} rules ({})", if teardown { "teardown" } else { "install" }); + print!("{plan}"); +} diff --git a/crates/maxplayer-core/src/home.rs b/crates/maxplayer-core/src/home.rs index 295948b6..d1c997d8 100644 --- a/crates/maxplayer-core/src/home.rs +++ b/crates/maxplayer-core/src/home.rs @@ -607,24 +607,28 @@ pub struct SandboxConfig { /// Unused under `launcher` mode. #[serde(default, skip_serializing_if = "Option::is_none")] pub runtime: Option, - /// `docker` mode: the dedicated docker network a job's container joins (`docker run --network`). - /// Omitted ⇒ the daemon default (the shared `bridge` network). + /// `docker` mode: the **name prefix** for the docker network each job gets to itself. + /// Omitted ⇒ no containment, and the daemon default bridge. /// /// **Setting this is what turns #797 egress containment on for this seat.** A job launched under /// it runs in a network namespace whose rules were installed before the job process existed, and /// a job whose containment cannot be established FAILS rather than running exposed. There is no /// second step: no root command, and nothing to reinstall after a reboot. /// - /// Two reasons a *named* network rather than the default bridge: + /// **It names, it does not share.** This value is not one bridge every job sits on: each job gets + /// `-job-`, created before its holder and removed after it. The name still does the + /// job it always did — telling this seat's networks from a co-tenant daemon's — and no longer + /// puts two jobs on one wire. That changed because of a measurement: with jobs sharing a bridge, + /// a gVisor job REACHED a live listener inside another job's namespace and no host rule stopped + /// it, two containers on one bridge being switched rather than routed, and switched frames + /// entering no iptables chain at all on a host without `br_netfilter`. A network per job leaves + /// the job no on-link peer but its own gateway, which is also what makes every other destination + /// routed, and therefore visible to the host-side policy that binds a gVisor job at all. /// - /// * **The seller's own services are not on it.** The rules deny by destination, and the seat's - /// LAN and host addresses fall inside those denies. A dedicated network keeps a job's traffic - /// off the bridge every other container on the box shares. - /// * **DNS keeps working.** On a user-defined network a container resolves through docker's - /// embedded resolver at `127.0.0.11` inside its own netns, so no packet crosses to a host or LAN - /// resolver. On the shared default bridge docker copies the host's `resolv.conf` instead, and if - /// that names a LAN or host resolver then denying the LAN also denies DNS — which presents as - /// "the internet is broken" rather than as a firewall rule. + /// A *named* network rather than the default bridge, for a reason that predates all of that: + /// **the seller's own services are not on it.** The rules deny by destination, and the seat's LAN + /// and host addresses fall inside those denies, so a job must not share the bridge every other + /// container on the box uses. /// /// See [`crate::sandbox_net`] for what the rules are and [`crate::sandbox_netns`] for how they are /// put in force. diff --git a/crates/maxplayer-core/src/sandbox_net.rs b/crates/maxplayer-core/src/sandbox_net.rs index 42f023b2..8e089b29 100644 --- a/crates/maxplayer-core/src/sandbox_net.rs +++ b/crates/maxplayer-core/src/sandbox_net.rs @@ -950,6 +950,131 @@ impl NetPolicy { } } +/// Docker's documented hook in the root namespace's FORWARD path. Docker jumps to it before its +/// own rules, and it survives docker rewriting the rest of the chain — which a bare `-I FORWARD` +/// does not. +pub const DOCKER_USER_CHAIN: &str = "DOCKER-USER"; + +/// The root namespace's INPUT chain: where packets addressed to the host itself land. +pub const INPUT_CHAIN: &str = "INPUT"; + +/// The containment a gVisor job cannot step around, installed on the **host** side of the veth. +/// +/// # Why this exists at all +/// +/// [`NetPolicy`] renders rules for the job's own namespace, and for a runc job that is the whole +/// story. For a **gVisor** job it is not. Measured in `docs/gvisor-dns-delivery` (aarch64, runsc +/// release-20260817.0): with the full 26-rule plan installed and read back in the namespace, a +/// runsc job REACHED a live listener inside `-d 172.16.0.0/12 -j DROP`, while a runc job in that +/// same namespace got `timeout`. gVisor terminates the network inside the sandbox and writes +/// frames to the veth itself, so the host kernel's OUTPUT chain in that namespace — which only +/// ever sees packets from host sockets — never sees the job's. +/// +/// So the netns plan stays (it is what binds a runc job, and it costs nothing as defence in depth) +/// and this is added beside it, where the host kernel handles the packet whatever produced it. +/// +/// # Why two chains and not one +/// +/// Also measured, same evidence directory, against live listeners: +/// +/// | destination | `DOCKER-USER` | `INPUT` | +/// | --- | --- | --- | +/// | the host's own LAN address | `REACHED` — useless | `timeout` — binds | +/// | `169.254.169.254` (routed via the gateway) | binds | — | +/// +/// Packets addressed to the host are delivered locally and never traverse FORWARD, so DOCKER-USER +/// cannot see them. Packets routed onward do traverse it. Neither chain covers the other, which is +/// why both are rendered. +/// +/// # What this deliberately does not try to cover +/// +/// A peer on the job's **own bridge** is reached by switching, not routing, and on a host without +/// `br_netfilter` those frames enter no iptables chain at all — measured `REACHED` with the +/// DOCKER-USER rule installed. No host rule fixes that. A **per-job network** does, by leaving the +/// job no on-link peer but its gateway; see `sandbox_netns::establish`. +/// +/// IPv6 is not rendered here. `ip6tables` has a `DOCKER-USER` chain only when the daemon has IPv6 +/// enabled, and a missing chain is an install failure that would fail every job launch on a v4-only +/// host. The netns plan still carries [`DENIED_DESTINATIONS_V6`]; host-side v6 containment is +/// UNMEASURED and named as such in the runlog rather than rendered on faith. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostPolicy { + /// The job namespace's own address. Every rule is keyed to it as `-s`, so the policy denies + /// this job and nothing else on the host. + pub job_addr: String, +} + +/// One host-side rule: the chain it belongs in, what it denies, and why. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostRule { + pub chain: &'static str, + pub destination: String, + pub why: &'static str, +} + +impl HostPolicy { + /// Every rule this policy installs, in install order. + pub fn rules(&self) -> Vec { + let mut rules = vec![HostRule { + chain: DOCKER_USER_CHAIN, + destination: METADATA_ENDPOINT.to_owned(), + why: "instance credentials, reached by route through the gateway", + }]; + for denied in DENIED_DESTINATIONS { + rules.push(HostRule { + chain: DOCKER_USER_CHAIN, + destination: (*denied).to_owned(), + why: "a private destination the job reaches by route", + }); + } + for denied in DENIED_DESTINATIONS { + rules.push(HostRule { + chain: INPUT_CHAIN, + destination: (*denied).to_owned(), + why: "the same range addressed to the host itself, which never enters FORWARD", + }); + } + rules + } + + /// The `iptables` argv that installs the policy. + /// + /// `-I` rather than `-A`: DOCKER-USER is a shared chain and docker appends its own rules to it, + /// so appending would put this policy behind whatever is already there. + pub fn install_argv(&self) -> Vec> { + self.rules().iter().map(|rule| self.argv("-I", rule)).collect() + } + + /// The argv that removes it, exactly inverting [`Self::install_argv`]. + /// + /// This is not optional housekeeping. These rules are keyed to one job's address in a chain + /// that outlives the job; without the teardown the chain grows by one ruleset per job until the + /// host is a linear scan, and a recycled address inherits a dead job's policy. + pub fn teardown_argv(&self) -> Vec> { + let mut argv: Vec> = + self.rules().iter().map(|rule| self.argv("-D", rule)).collect(); + argv.reverse(); + argv + } + + fn argv(&self, op: &str, rule: &HostRule) -> Vec { + [ + Family::V4.binary(), + op, + rule.chain, + "-s", + &format!("{}/32", self.job_addr), + "-d", + &rule.destination, + "-j", + "DROP", + ] + .into_iter() + .map(String::from) + .collect() + } +} + #[cfg(test)] mod tests { use super::*; @@ -2030,4 +2155,248 @@ mod tests { let missing = policy.verify_readback(Family::V6, &elsewhere).expect_err("wrong v6 resolver"); assert!(missing.contains("2001:4860:4860::8888/128"), "{missing}"); } + + + fn host_policy() -> HostPolicy { + HostPolicy { job_addr: "172.31.19.2".into() } + } + + /// The combined-plane check §8 of the compatibility report asks for, in the only form that is + /// offline: the host plane installed for a job, and the namespace verifier this repo now + /// carries passing on that same job's readback — without either plane's rules being visible to + /// the other. + /// + /// The interaction this catches is real and neither PR could have tested it: the namespace + /// verifier counts rules and refuses a namespace holding one it did not render, while the host + /// plane installs rules for the SAME job on the SAME host. If a host rule ever landed in + /// `OUTPUT` — the one chain the verifier reads — every contained job would be refused at + /// launch, because the count would exceed what the policy rendered. + /// + /// What this does NOT do is prove the two planes behave that way in a live kernel: the readback + /// here is the recorded fixture, not a namespace this test built. The live matrix is named as + /// unrun in the PR. + #[test] + fn the_host_plane_and_the_namespace_verifier_agree_on_the_same_job() { + let namespace = measured_policy(); + let host = HostPolicy { job_addr: "172.31.19.2".to_owned() }; + + // The verifier still accepts the contained namespace with the host plane rendered beside it. + assert_eq!(namespace.verify_readback(Family::V4, MEASURED_V4), Ok(())); + assert_eq!(namespace.verify_readback(Family::V6, MEASURED_V6), Ok(())); + + // And it does so because the two planes are disjoint by CHAIN, not by luck: no host rule is + // rendered into the chain the verifier reads. + for rule in host.rules() { + assert_ne!( + rule.chain, OUTPUT_CHAIN, + "a host rule in {OUTPUT_CHAIN} would be counted by the namespace verifier and every \ + contained job would be refused at launch: {rule:?}" + ); + } + + // The negative control, so the assertion above is not vacuous: a namespace carrying ONE + // extra rule of the shape the host plane installs IS refused by the verifier. + let intruder = format!("{MEASURED_V4}\n-A OUTPUT -s 172.31.19.2/32 -d 10.0.0.0/8 -j DROP"); + assert!( + namespace.verify_readback(Family::V4, &intruder).is_err(), + "the verifier must refuse a namespace holding a rule this policy did not render" + ); + } + + /// The rule that makes this policy safe to install on a shared host. Every rule is keyed to one + /// job's address; a rule that lost its `-s` would deny the range to the WHOLE host, including + /// the seller's own traffic and every other job. + #[test] + fn every_host_rule_is_keyed_to_this_job_and_nothing_else() { + let policy = host_policy(); + for argv in policy.install_argv().iter().chain(policy.teardown_argv().iter()) { + let source = argv.windows(2).find(|pair| pair[0] == "-s").map(|pair| &pair[1]); + assert_eq!( + source.map(String::as_str), + Some("172.31.19.2/32"), + "a host-side rule without this job's source key denies the range host-wide: {argv:?}" + ); + } + } + + /// Two chains, because neither covers the other: DOCKER-USER never sees a packet addressed to + /// the host, and INPUT never sees one routed onward. Both results are measured. + #[test] + fn the_host_policy_covers_the_routed_path_and_the_host_itself() { + let rules = host_policy().rules(); + for denied in DENIED_DESTINATIONS { + for chain in [DOCKER_USER_CHAIN, INPUT_CHAIN] { + assert!( + rules + .iter() + .any(|rule| rule.chain == chain && rule.destination == *denied), + "{denied} has no DROP in {chain}" + ); + } + } + assert!( + rules + .iter() + .any(|rule| rule.chain == DOCKER_USER_CHAIN + && rule.destination == METADATA_ENDPOINT), + "the metadata endpoint has no host-side DROP on the routed path" + ); + } + + /// The metadata drop goes in FIRST, so no rule this policy adds can precede it. + #[test] + fn the_metadata_drop_is_the_first_rule_rendered() { + assert_eq!(host_policy().rules()[0].destination, METADATA_ENDPOINT); + } + + /// Install renders `-I`, never `-A`: DOCKER-USER is shared and docker appends to it, so an + /// appended policy sits behind whatever is already there. + #[test] + fn the_host_policy_inserts_rather_than_appends() { + for argv in host_policy().install_argv() { + assert_eq!(argv[0], "iptables"); + assert_eq!(argv[1], "-I", "appending puts this policy behind docker's own rules"); + } + } + + /// Teardown is the exact inverse of install, reversed. These rules outlive the job's container + /// in a chain nothing else cleans up: a teardown that misses one leaks a rule per job, and a + /// recycled address inherits a dead job's policy. + #[test] + fn teardown_is_the_exact_inverse_of_install_in_reverse_order() { + let policy = host_policy(); + let install = policy.install_argv(); + let teardown = policy.teardown_argv(); + assert_eq!(install.len(), teardown.len()); + for (installed, removed) in install.iter().rev().zip(teardown.iter()) { + let mut expected = installed.clone(); + expected[1] = "-D".into(); + assert_eq!(&expected, removed); + } + } + + /// The exact argv, once, so a silent change to the shape of these rules has to be deliberate. + #[test] + fn the_first_host_rule_renders_exactly() { + assert_eq!( + host_policy().install_argv()[0], + vec![ + "iptables", + "-I", + "DOCKER-USER", + "-s", + "172.31.19.2/32", + "-d", + "169.254.169.254/32", + "-j", + "DROP" + ] + ); + } + + /// The host policy reuses [`DENIED_DESTINATIONS`] rather than carrying its own copy. A second + /// list is a second thing to forget: the range added to one and not the other is reachable. + #[test] + fn the_host_policy_denies_every_range_the_namespace_plan_denies() { + let host = host_policy().rules(); + let covered: Vec<&str> = DENIED_DESTINATIONS + .iter() + .copied() + .filter(|denied| host.iter().any(|rule| rule.destination == *denied)) + .collect(); + assert_eq!( + covered.len(), + DENIED_DESTINATIONS.len(), + "the host policy and the namespace plan disagree about what is denied" + ); + } + + /// Teardown must remove exactly what install added, in reverse. + /// + /// Gate 5h proved this on a live host: a recycled address inherited zero stale rules. That + /// was one measurement on one machine. This is the invariant, checked on every build — if + /// the two plans ever drift apart, teardown leaks rules into a shared chain and the next job + /// to be handed this address inherits a dead job's firewall. + #[test] + fn the_host_teardown_exactly_inverts_the_install() { + let policy = HostPolicy { job_addr: "172.18.0.2".to_owned() }; + let install = policy.install_argv(); + let teardown = policy.teardown_argv(); + assert_eq!( + install.len(), + teardown.len(), + "install and teardown must be the same length or teardown leaves rules behind" + ); + for (i, up) in install.iter().enumerate() { + let down = &teardown[teardown.len() - 1 - i]; + assert_eq!(up[1], "-I", "install must insert"); + assert_eq!(down[1], "-D", "teardown must delete"); + assert_eq!( + up[2..], + down[2..], + "teardown rule {i} does not match the install rule it is meant to remove" + ); + } + } + + /// Every rule, both directions, must carry this job's `/32` source key. + /// + /// A host rule without `-s` is not this job's policy — it is a deny for the whole range on a + /// chain shared with every container on the daemon. + #[test] + fn every_host_rule_is_keyed_to_the_job_address() { + let policy = HostPolicy { job_addr: "172.18.0.2".to_owned() }; + for argv in policy.install_argv().iter().chain(policy.teardown_argv().iter()) { + let at = argv + .iter() + .position(|arg| arg == "-s") + .unwrap_or_else(|| panic!("a host rule with no source key: {argv:?}")); + assert_eq!( + argv[at + 1], + "172.18.0.2/32", + "a host rule keyed to something other than this job: {argv:?}" + ); + } + } + + /// The count `establish()` cross-checks must equal the number of rules actually rendered. + /// + /// The applier reports a number and the caller compares it against this one; if the rendered + /// count and the plan's line count could disagree, a truncated plan would pass the check. + #[test] + fn the_rendered_host_plan_counts_exactly_what_it_renders() { + let policy = HostPolicy { job_addr: "172.18.0.2".to_owned() }; + let (plan, count) = crate::sandbox_netns::host_install_stdin(&policy); + assert_eq!(count, policy.install_argv().len(), "the install count is not the rule count"); + assert_eq!( + plan.lines().filter(|line| !line.trim().is_empty()).count(), + count, + "the install plan has a different number of lines than it claims rules" + ); + let (teardown, teardown_count) = crate::sandbox_netns::host_teardown_stdin(&policy); + assert_eq!(teardown_count, count, "teardown claims a different rule count than install"); + assert_eq!( + teardown.lines().filter(|line| !line.trim().is_empty()).count(), + teardown_count, + "the teardown plan has a different number of lines than it claims rules" + ); + } + + /// Why `establish()` refuses an empty address rather than rendering with it. + /// + /// This test asserts the hazard, not the fix: with no address the source key renders as bare + /// `/32`, which is not a host. Such a rule does not scope the deny to this job, so the guard + /// in `establish()` is load-bearing and must not be relaxed into a warning. + #[test] + fn an_empty_job_address_renders_a_source_key_that_is_not_a_host() { + let policy = HostPolicy { job_addr: String::new() }; + let argv = policy.install_argv(); + let first = &argv[0]; + let at = first.iter().position(|arg| arg == "-s").expect("a source key"); + assert_eq!( + first[at + 1], + "/32", + "an empty address must render an obviously-invalid key, which establish() then refuses" + ); + } } diff --git a/crates/maxplayer-core/src/sandbox_netns.rs b/crates/maxplayer-core/src/sandbox_netns.rs index d971d88f..539944fa 100644 --- a/crates/maxplayer-core/src/sandbox_netns.rs +++ b/crates/maxplayer-core/src/sandbox_netns.rs @@ -135,7 +135,13 @@ impl Drop for NetnsHolder { /// cannot disagree. #[derive(Debug)] pub struct Containment { + /// Declared first so it is DROPPED first: docker refuses to remove a network that still has an + /// endpoint attached, so the holder must be gone before [`Self::network`] is. pub holder: NetnsHolder, + /// The host-side rules keyed to this job's address. Dropping them removes the rules. + pub host_rules: HostRules, + /// This job's own network, dropped after the holder that sits on it. + pub network: JobNetwork, pub proxy_host: String, } @@ -209,13 +215,24 @@ pub fn holder_argv( /// `--rm` is safe here specifically because the caller captures stdout and stderr before the container /// is removed; the evidence is in hand before the container is gone. pub fn sidecar_argv(holder: &NetnsHolder, image: &str) -> Vec { + sidecar_argv_for(holder.name(), image) +} + +/// [`sidecar_argv`] for a namespace addressed by NAME. +/// +/// Exists for the doctor's delivery-route preflight, which builds and tears down its own throwaway +/// namespace and so holds a name rather than a [`NetnsHolder`] guard. Taking the guard there would +/// hand out a `Drop` that destroys a namespace the caller did not create. The argv is the same one +/// the awarded-job path uses because it IS that argv — a preflight rendering its own would test a +/// sidecar production never runs. +pub fn sidecar_argv_for(holder: &str, image: &str) -> Vec { [ "docker", "run", "--rm", "--interactive", "--network", - &holder.network_mode(), + &NetnsHolder::network_mode_for(holder), "--cap-drop", "ALL", "--cap-add", @@ -283,6 +300,256 @@ pub fn plan_stdin(policy: &NetPolicy) -> (String, usize) { (out, plan.len()) } +/// The per-job network's name, under the operator's configured network name as a prefix. +/// +/// The configured `[sandbox] network` used to name one shared bridge every job sat on. It keeps +/// its naming role — that is how a seat's networks are told from another daemon's on a shared host +/// — and loses its sharing role. The job id is appended for the same reason [`holder_name`] uses +/// it: a leaked network can be attributed to the job that leaked it. +pub fn job_network_name(configured: &str, job_id: &str) -> String { + format!("{configured}-job-{job_id}") +} + +/// `docker network create` argv for one job's own network. +/// +/// # Why a job gets a network to itself +/// +/// Measured in `docs/gvisor-dns-delivery` (aarch64, runsc release-20260817.0). With every job on +/// one shared bridge, a gVisor job REACHED a live listener in another job's namespace, and no host +/// rule stopped it: two containers on one bridge are **switched**, not routed, and on a host +/// without `br_netfilter` those frames enter no iptables chain at all — `DOCKER-USER` with the +/// right source key still read `REACHED`. Moving the neighbour to its own network changed the +/// result to `timeout`. +/// +/// So the per-job network is not tidiness. It is what leaves the job no on-link peer but its own +/// gateway, which makes every other destination **routed** — and routed packets from a gVisor +/// sandbox do traverse the host's chains, where [`crate::sandbox_net::HostPolicy`] can bind them. +pub fn network_create_argv(name: &str) -> Vec { + ["docker", "network", "create", "--driver", "bridge", name] + .into_iter() + .map(String::from) + .collect() +} + +/// `docker network rm` argv. Only succeeds once no container is attached, which is why the holder +/// is dropped before the network is. +pub fn network_rm_argv(name: &str) -> Vec { + ["docker", "network", "rm", name].into_iter().map(String::from).collect() +} + +/// `docker inspect` argv that reads the holder's address on `network`. +/// +/// The host-side policy is keyed to this address, so it is read from docker rather than computed +/// from the subnet: a policy keyed to a guess denies some other container and leaves this job open. +pub fn holder_address_argv(holder_name: &str, network: &str) -> Vec { + [ + "docker", + "inspect", + "--format", + &format!("{{{{(index .NetworkSettings.Networks \"{network}\").IPAddress}}}}"), + holder_name, + ] + .into_iter() + .map(String::from) + .collect() +} + +/// `docker run` argv for the container that applies the **host-side** policy. +/// +/// `--network host` is the whole point and the whole risk: these rules must land in the ROOT +/// namespace's chains, because that is the only place a gVisor job's packets can be seen. It runs +/// the same applier image as the sidecar, reading the same ` ` plan on stdin, so +/// there is one applier in this design and not two. +/// +/// It is safe to hand this container the host's network only because nothing untrusted is ever in +/// it: it runs our own image, for milliseconds, on a plan rendered in Rust from +/// [`crate::sandbox_net::HostPolicy`], and it is gone before the job starts. The job itself never +/// comes near `--network host`. +pub fn host_rules_argv(image: &str) -> Vec { + [ + "docker", + "run", + "--rm", + "--interactive", + "--network", + "host", + "--cap-drop", + "ALL", + "--cap-add", + "NET_ADMIN", + "--security-opt", + "no-new-privileges", + image, + ] + .into_iter() + .map(String::from) + .collect() +} + +/// One host-side plan as the applier reads it, plus its rule count for the same truncation +/// cross-check [`plan_stdin`] exists for. +fn host_stdin(argv: &[Vec]) -> (String, usize) { + let mut out = String::new(); + for rule in argv { + out.push_str(&rule.join(" ")); + out.push('\n'); + } + (out, argv.len()) +} + +/// The host-side plan that installs [`crate::sandbox_net::HostPolicy`]. +pub fn host_install_stdin(policy: &crate::sandbox_net::HostPolicy) -> (String, usize) { + host_stdin(&policy.install_argv()) +} + +/// The host-side plan that removes it again. +pub fn host_teardown_stdin(policy: &crate::sandbox_net::HostPolicy) -> (String, usize) { + host_stdin(&policy.teardown_argv()) +} + +/// A created per-job network, and the guarantee that it goes away. +/// +/// Same bargain as [`NetnsHolder`]: constructed the moment the network exists, so every later `?` +/// removes it on the way out. Dropped AFTER the holder — docker refuses to remove a network that +/// still has an endpoint attached — which the field order of [`Containment`] is what enforces. +#[derive(Debug)] +pub struct JobNetwork { + name: String, +} + +impl JobNetwork { + /// Gated to the feature that contains the one caller, so a build without `acp` does not carry + /// a constructor nothing can reach. + #[cfg(feature = "acp")] + fn adopt(name: String) -> Self { + Self { name } + } + + pub fn name(&self) -> &str { + &self.name + } +} + +impl Drop for JobNetwork { + fn drop(&mut self) { + let outcome = std::process::Command::new("docker") + .args(["network", "rm", &self.name]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .output(); + match outcome { + Ok(done) if done.status.success() => {} + Ok(done) => eprintln!( + "sandbox: could not remove job network {}: {}", + self.name, + String::from_utf8_lossy(&done.stderr).trim() + ), + Err(error) => { + eprintln!("sandbox: could not run docker network rm for {}: {error}", self.name) + } + } + } +} + +/// Installed host-side rules, and the guarantee that they are removed. +/// +/// The most important guard of the three, because its resource is invisible. A leaked holder is a +/// container someone will notice; a leaked host rule is a line in a shared chain that outlives the +/// job silently. Without this, `DOCKER-USER` grows by one ruleset per job forever, and a recycled +/// address inherits a dead job's policy. +#[derive(Debug)] +pub struct HostRules { + policy: crate::sandbox_net::HostPolicy, + image: String, + /// Whether the install was **verified complete** — the applier exited 0 and its count matched + /// what was rendered. Set after that cross-check, never at construction. + /// + /// It decides how teardown is run, and gate 5i is why it exists. `apply-policy` aborts on the + /// first rule that fails, and the teardown plan is the exact inverse in reverse order. After a + /// PARTIAL install that inverse begins with a rule which was never created, so the applier + /// aborts on its first line and removes **nothing** — measured: 9 rules installed, 9 rules + /// still in `DOCKER-USER` afterwards. + /// + /// The applier's exit-3 contract tells the caller to destroy the holder, and for the namespace + /// plan that is a complete remedy because those rules die with the netns. These do not: they + /// are in the root netns, in chains shared with every container on the daemon. + complete: bool, +} + +impl HostRules { + /// Adopt rules that may be only partly installed. Always the first thing done with them. + fn adopt(policy: crate::sandbox_net::HostPolicy, image: String) -> Self { + Self { policy, image, complete: false } + } + + /// Record that every rendered rule is in the kernel, which licenses the one-shot teardown. + fn mark_complete(&mut self) { + self.complete = true; + } +} + +/// Feeds one plan to the host-rule applier and reports whether it applied cleanly. +fn run_host_plan(image: &str, plan: &str) -> Result<(), String> { + use std::io::Write; + let argv = host_rules_argv(image); + let (program, args) = argv.split_first().expect("a docker argv is never empty"); + let mut child = std::process::Command::new(program) + .args(args) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn() + .map_err(|error| error.to_string())?; + if let Some(stdin) = child.stdin.as_mut() { + let _ = stdin.write_all(plan.as_bytes()); + } + drop(child.stdin.take()); + match child.wait_with_output() { + Ok(done) if done.status.success() => Ok(()), + Ok(done) => Err(String::from_utf8_lossy(&done.stderr).trim().to_owned()), + Err(error) => Err(error.to_string()), + } +} + +impl Drop for HostRules { + /// Synchronous, like [`NetnsHolder::drop`] and for the same reason: a task spawned here is + /// discarded on runtime shutdown, which is the path an aborted job takes. + fn drop(&mut self) { + if self.complete { + // Every rule is present, so the inverse plan matches it rule for rule and one + // invocation is both correct and cheapest. + let (plan, _) = host_teardown_stdin(&self.policy); + if let Err(error) = run_host_plan(&self.image, &plan) { + eprintln!( + "sandbox: host-rule teardown for {} failed: {error} — \ + DOCKER-USER and INPUT still carry this job's rules", + self.policy.job_addr + ); + } + return; + } + + // The install did not complete, so an unknown prefix of the plan is in the kernel and the + // rest never existed. One invocation would abort on the first absent rule and strand the + // present ones. Each rule gets its own invocation: a failure then means only "that rule was + // not there", which on this path is the expected case and not an error. + let rules = self.policy.teardown_argv(); + let total = rules.len(); + let mut removed = 0usize; + for rule in &rules { + let (plan, _) = host_stdin(std::slice::from_ref(rule)); + if run_host_plan(&self.image, &plan).is_ok() { + removed += 1; + } + } + eprintln!( + "sandbox: host-rule install for {} did not complete; removed {removed} of {total} \ + rules one at a time", + self.policy.job_addr + ); + } +} + /// `docker run` argv that asks **docker** what `host-gateway` means on this platform, by resolving /// `alias` inside a throwaway container that is allowed to carry `--add-host`. /// @@ -638,14 +905,58 @@ pub async fn establish( format!("resolving {proxy_alias} produced no IPv4 address (got {probe_stdout:?})") })?; + // This job's own network, created before the holder that will sit on it. Adopted into a guard + // immediately, for the same reason the holder is: every `?` below must remove it. + let net_name = job_network_name(network, job_id); + run_docker(network_create_argv(&net_name), None) + .await + .map_err(|error| format!("could not create the job network {net_name} — {error}"))?; + let job_network = JobNetwork::adopt(net_name.clone()); + let name = holder_name(job_id); - run_docker(holder_argv(&name, network, holder_image, uid, gid, job_id, seat), None) + run_docker(holder_argv(&name, &net_name, holder_image, uid, gid, job_id, seat), None) .await .map_err(|error| format!("could not start the netns holder {name} — {error}"))?; // From here on the container exists, so every early return must tear it down. Adopting it into // the guard immediately is what makes that automatic rather than remembered. let holder = NetnsHolder::adopt(name); + // The host-side policy, keyed to the address docker actually gave the holder. Read, never + // computed: a policy keyed to a guessed address denies some other container and leaves this + // job wide open. + let (job_addr, _) = run_docker(holder_address_argv(holder.name(), &net_name), None) + .await + .map_err(|error| format!("could not read the job namespace's address — {error}"))?; + let job_addr = job_addr.trim().to_owned(); + if job_addr.is_empty() { + return Err(format!( + "docker reported no address for {} on {net_name}, so the host-side policy would have \ + no source key and would deny the range host-wide", + holder.name() + )); + } + let host_policy = crate::sandbox_net::HostPolicy { job_addr }; + let (host_plan, host_expected) = host_install_stdin(&host_policy); + let host_applied = run_docker(host_rules_argv(sidecar_image), Some(host_plan)).await; + // Adopted before the result is examined: a plan that failed part-way has already installed + // rules, and those rules must come out whichever way this returns. + let mut host_rules = HostRules::adopt(host_policy, sidecar_image.to_owned()); + let (host_applied, _) = host_applied + .map_err(|error| format!("host-side containment was not installed — {error}"))?; + let host_applied: usize = host_applied + .parse() + .map_err(|_| format!("the host-rule applier reported {host_applied:?}, not a number"))?; + if host_applied != host_expected { + return Err(format!( + "host-side containment is incomplete: {host_applied} of {host_expected} rules applied \ + (the plan was truncated in transit)" + )); + } + // Only now is the one-shot inverse teardown known to match what is in the kernel. Before this + // line every early return unwinds rule-by-rule instead, which is the only way a partial + // install comes back out (gate 5i). + host_rules.mark_complete(); + // The resolvers arrive from the caller rather than being discovered here, and that is the one // property that keeps the job's `/etc/resolv.conf` and this policy in agreement: the caller // resolves once, writes that file from the result, and hands the same addresses here. Two @@ -690,7 +1001,7 @@ pub async fn establish( })?; } - Ok(Containment { holder, proxy_host }) + Ok(Containment { holder, host_rules, network: job_network, proxy_host }) } #[cfg(test)] @@ -1006,4 +1317,71 @@ mod tests { assert_eq!(accepts.len(), 1, "exactly one pinhole: {accepts:?}"); assert!(accepts[0].contains(&measured), "the pinhole must name the measured host: {accepts:?}"); } + + /// The containment plane runs on the daemon's own runtime, never the job's. + /// + /// Measured in the gVisor repro (aarch64, runsc release-20260817.0, evidence + /// `docs/gvisor-dns-delivery/evidence/gate2a-runsc-holder-FAIL-*.txt`): a runsc container + /// joining a **runsc** holder's namespace sees `lo` only — no eth0, no route, every lookup + /// `EAI_AGAIN` — because a gVisor sandbox's netstack lives inside that sandbox and a second one + /// cannot enter it. Joining a **runc** holder, the same runsc job gets the holder's interface + /// and address, so the host kernel's rules govern its traffic. `iptables-nft` also refuses to + /// initialise inside gVisor, so a sandboxed sidecar could not install the plan even if the + /// namespace were shared. + /// + /// So this is not a default anyone may "improve" by threading the seat's runtime through: doing + /// that returns a job with no network at all, and a policy nothing enforces. + #[test] + fn the_containment_plane_never_carries_the_jobs_runtime() { + let holder = holder_argv("h", "net", "img", 1000, 1000, "abc", &seat_b()); + assert!( + !holder.iter().any(|a| a == "--runtime"), + "the holder must run on the daemon runtime, or the job cannot join its namespace: \ + {holder:?}" + ); + let sidecar = sidecar_argv(&NetnsHolder::adopt("h".into()), "netfilter"); + assert!( + !sidecar.iter().any(|a| a == "--runtime"), + "the sidecar must run on the daemon runtime, or iptables cannot initialise: {sidecar:?}" + ); + } + + #[test] + + + /// Adoption must assume the install is partial; the one-shot teardown has to be earned. + /// + /// Gate 5i measured what happens when it is not: a 9-of-17 install torn down by the inverse + /// plan removed **nothing**, because the applier aborts on the first rule that was never + /// created. So `complete` starts false, and only the count cross-check sets it. + #[test] + fn adopted_host_rules_are_not_complete_until_the_count_check_passes() { + let policy = crate::sandbox_net::HostPolicy { job_addr: "172.18.0.2".to_owned() }; + let mut rules = HostRules::adopt(policy, "image:tag".to_owned()); + assert!(!rules.complete, "adoption must assume a partial install"); + rules.mark_complete(); + assert!(rules.complete, "the count cross-check is what licenses the one-shot teardown"); + // Drop shells out to docker; this test is about the flag, not the teardown. + std::mem::forget(rules); + } + + /// Each rule of the rule-by-rule teardown must stand alone as one valid delete. + /// + /// This is the path a partial install unwinds through, and the applier refuses an empty plan + /// (exit 4) and anything that is not iptables (exit 5), so every single-rule plan must be one + /// line, a delete, and still keyed to this job. + #[test] + fn the_per_rule_teardown_renders_one_valid_delete_per_rule() { + let policy = crate::sandbox_net::HostPolicy { job_addr: "172.18.0.2".to_owned() }; + let rules = policy.teardown_argv(); + assert!(!rules.is_empty(), "there is nothing to tear down"); + for rule in &rules { + let (plan, count) = host_stdin(std::slice::from_ref(rule)); + assert_eq!(count, 1, "a per-rule plan must carry exactly one rule"); + assert_eq!(plan.lines().count(), 1, "a per-rule plan must be one line: {plan:?}"); + let line = plan.lines().next().expect("one line"); + assert!(line.starts_with("iptables -D "), "must be a delete: {line}"); + assert!(line.contains("172.18.0.2/32"), "must stay keyed to this job: {line}"); + } + } } diff --git a/crates/maxplayer/src/doctor.rs b/crates/maxplayer/src/doctor.rs index 0c3dc746..1073888f 100644 --- a/crates/maxplayer/src/doctor.rs +++ b/crates/maxplayer/src/doctor.rs @@ -1117,6 +1117,430 @@ mod checks { } } + const DELIVERY_ROUTE_CHECK: &str = "sandbox delivery route"; + + /// The directory the route preflight runs its probe in, under the seat's real job tree — same + /// argument [`crate::sandbox_probe`] makes for its own paths: a launcher is configured for where + /// jobs run, so a probe somewhere else measures a route no job takes. + const ROUTE_WORKDIR_NAME: &str = ".route-preflight"; + + /// What exercising the real job route produced. + /// + /// Deliberately four outcomes rather than a bool: "the name never resolved", "it resolved but + /// TLS did not complete", and "the route could not be built at all" send an operator to three + /// different places, and collapsing them is how a doctor becomes something people skip. + #[derive(Debug, Clone, PartialEq, Eq)] + pub(super) enum RouteProbe { + /// Resolved a name AND completed a certificate-VERIFIED TLS handshake, from inside the + /// namespace an awarded job gets. + Delivered { resolver: String, address: String, subject: String }, + /// The lookup failed inside the job. The gVisor case this check was built for. + NoDns(String), + /// The name resolved, but TLS did not complete or its chain was not verified. + NoTls(String), + /// The route could not be built or the probe never reported. NOT a pass: a route that cannot + /// be measured has not been shown to work. + Unbuildable(String), + /// The instrument is absent — no docker on PATH — so there is nothing to measure and no + /// finding to report. Distinct from [`RouteProbe::Unbuildable`], which means the route WAS + /// asked and did not answer; the launcher check owns the missing-docker verdict and this one + /// must not double-report it. + Unmeasurable(String), + } + + /// #? gVisor DNS delivery: does a job on THIS seat actually reach the network it is supposed to? + /// + /// Every other network check here asks the question from the HOST. That is precisely the hole: + /// the measured failure is a seat whose host resolves and fetches perfectly while every job it + /// runs dies on `EAI_AGAIN`, because docker's embedded resolver at `127.0.0.11` is unreachable + /// from inside a gVisor sandbox. A host-side probe reports that seat READY and it then fails + /// every job it wins. So this one runs the real thing: the seat's own image, in the namespace + /// [`maxplayer_core::sandbox_netns`] builds, under the seat's own runtime, uid, dropped + /// capabilities and `no-new-privileges`, with the resolver file a job is handed — and it must + /// both resolve a name and complete a VERIFIED TLS handshake. + /// + /// It BLOCKS. A `Fail` here keeps [`readiness_ok`] false, because advertising a seat whose jobs + /// cannot deliver is the exact outcome the gate exists to prevent. It is `transient`, so the + /// boot gate's bounded retry gives a daemon or a resolver a moment to come back before the seat + /// is refused — and if it is still broken after that, refused is correct. + /// + /// The advisory `check_sandbox_egress` above is untouched (petar, 2026-08-18): that one reports + /// whether the network EXISTS and never blocks. This one asks whether the route WORKS. + pub(super) fn check_sandbox_delivery_route( + sandbox: Option, + home_root: std::path::PathBuf, + ) -> Check { + check_sandbox_delivery_route_in(sandbox, discover_resolvers, |policy, resolvers| { + // Ordering rule shared with check_sandbox_image and the Engine floor: ask only once + // docker itself resolves, or the spawn ENOENTs and a missing daemon is misreported as a + // broken route. A missing docker is the launcher check's verdict, not this one's. It + // lives in the REAL probe rather than in the injectable core so the unit tests below + // measure the verdicts and not the machine they run on. + if !argv0_resolvable("docker") { + return RouteProbe::Unmeasurable( + "docker not resolvable; the job route was not measured (see sandbox launcher)" + .to_owned(), + ); + } + run_delivery_route(policy, resolvers, &home_root) + }) + } + + /// The resolvers a job would be handed, by the product's own selection order. + fn discover_resolvers(configured: &[String]) -> Result, String> { + maxplayer_core::sandbox_dns::resolve( + configured, + maxplayer_core::sandbox_dns::host_resolv_conf, + maxplayer_core::sandbox_dns::host_resolvectl, + ) + .map(|resolvers| resolvers.addresses().to_vec()) + .map_err(|error| error.to_string()) + } + + /// [`check_sandbox_delivery_route`] over injected resolver discovery and an injected route + /// probe, so every verdict is testable on a host with no docker daemon at all — including the + /// one that matters most: host connectivity fine, job route broken, result still `Fail`. + pub(super) fn check_sandbox_delivery_route_in( + sandbox: Option, + resolvers: impl Fn(&[String]) -> Result, String>, + probe: impl Fn(&SandboxPolicy, &[String]) -> RouteProbe, + ) -> Check { + let policy = match SandboxPolicy::from_config(sandbox.as_ref()) { + // Already FAILed by the launcher check; do not double-report. + Err(_) => return Check::pass(DELIVERY_ROUTE_CHECK, "no resolvable docker executor"), + Ok(policy) => policy, + }; + if policy.docker_image().is_none() { + return Check::pass( + DELIVERY_ROUTE_CHECK, + "not a docker executor; jobs use this host's own network", + ); + } + let resolvers = match resolvers(policy.dns_servers()) { + Ok(resolvers) => resolvers, + // NOT transient: no retry discovers a resolver a misconfigured box does not have. + Err(error) => { + return Check::fail( + DELIVERY_ROUTE_CHECK, + format!("no resolver can be given to a job: {error}"), + "set `[sandbox] dns_servers` to one or more resolver ADDRESSES the seat can \ + reach (for example `dns_servers = [\"1.1.1.1\"]`)", + ) + } + }; + match probe(&policy, &resolvers) { + RouteProbe::Delivered { resolver, address, subject } => Check::pass( + DELIVERY_ROUTE_CHECK, + format!( + "a job resolved via {resolver} to {address} and completed a verified TLS \ + handshake with {subject}" + ), + ), + RouteProbe::NoDns(detail) => Check::fail_transient( + DELIVERY_ROUTE_CHECK, + format!("a job in the sandbox could not resolve names: {detail}"), + "the host resolving is not enough — the JOB must. Check `[sandbox] dns_servers` \ + and that port 53 to those addresses survives the job's egress policy", + ), + RouteProbe::NoTls(detail) => Check::fail_transient( + DELIVERY_ROUTE_CHECK, + format!("a job resolved, but could not complete a verified TLS handshake: {detail}"), + "check the job's egress policy allows 443 to the public internet and that the \ + sandbox image carries current CA certificates", + ), + RouteProbe::Unbuildable(detail) => Check::fail_transient( + DELIVERY_ROUTE_CHECK, + format!("the job's network route could not be measured: {detail}"), + "a route that cannot be measured has not been shown to work; check the docker \ + daemon, `[sandbox] network`, and that the sandbox image can run the probe", + ), + RouteProbe::Unmeasurable(detail) => Check::pass(DELIVERY_ROUTE_CHECK, detail), + } + } + + /// The host the route probe resolves and shakes hands with: the relay, because that is where a + /// job's answer is delivered, so a route that cannot reach it cannot earn anything. Kept in step + /// with [`maxplayer_core::home::DEFAULT_RELAY_URL`] by the test below. + pub(super) const ROUTE_PROBE_HOST: &str = "relay.maxplayer.ai"; + + /// The markers the in-container payload prints. Parsed rather than trusted to an exit code, + /// because "the payload never ran" and "the payload ran and failed" must not be one outcome. + const ROUTE_DNS_OK: &str = "route-dns-ok"; + const ROUTE_DNS_FAIL: &str = "route-dns-fail"; + const ROUTE_TLS_OK: &str = "route-tls-ok"; + const ROUTE_TLS_FAIL: &str = "route-tls-fail"; + + /// Build the real namespace, run the probe inside it, tear it down. + /// + /// Every container here comes from the PRODUCT's argv builders — `holder_argv`, + /// `sidecar_argv_for`, `plan_stdin`, and `SandboxPolicy::launch` — so this measures the route an + /// awarded job takes. A preflight that rendered its own argv would be a test of itself. + fn run_delivery_route( + policy: &SandboxPolicy, + resolvers: &[String], + home_root: &std::path::Path, + ) -> RouteProbe { + let workdir = home_root.join("seller-jobs").join(ROUTE_WORKDIR_NAME); + if let Err(error) = std::fs::create_dir_all(&workdir) { + return RouteProbe::Unbuildable(format!( + "cannot create the probe workdir {} ({error})", + workdir.display() + )); + } + let Some((uid, gid)) = owner_uid_gid(&workdir) else { + return RouteProbe::Unbuildable( + "cannot read the probe workdir's owner, so the probe could not run as the uid an \ + awarded job would get" + .to_owned(), + ); + }; + let resolv_path = workdir.join("resolv.conf"); + // Re-validated rather than trusted: these addresses came from discovery, and the file a job + // reads must be built by the same code that builds a job's real one. + let rendered = match maxplayer_core::sandbox_dns::from_config(resolvers) { + Ok(Some(resolvers)) => resolvers.render_resolv_conf(), + Ok(None) => { + return RouteProbe::Unbuildable( + "resolver discovery returned no addresses at all".to_owned(), + ) + } + Err(error) => return RouteProbe::Unbuildable(format!("resolvers rejected: {error}")), + }; + if let Err(error) = std::fs::write(&resolv_path, rendered) { + return RouteProbe::Unbuildable(format!( + "cannot write the probe resolver file {} ({error})", + resolv_path.display() + )); + } + + // No `[sandbox] network` ⇒ no namespace is established for a job either, so the honest route + // to measure is the daemon default the seat actually uses — with no resolver file, because + // nothing opened port 53 for one. + let Some(network) = policy.sandbox_network() else { + return job_leg(policy, &workdir, uid, gid, None, None, "the daemon's default network"); + }; + + let gateway = match network_gateway(network) { + Ok(gateway) => gateway, + Err(error) => { + return RouteProbe::Unbuildable(format!( + "cannot read the gateway of `[sandbox] network` '{network}': {error}" + )) + } + }; + let image = policy.docker_image().unwrap_or_default().to_owned(); + let holder_name = format!("maxplayer-route-preflight-{}", std::process::id()); + // Torn down on EVERY exit below, including the early returns — a preflight that leaks a + // holder container leaves the seat holding a namespace nothing will ever reap. + let _ = docker_rm_f(&holder_name); + let holder = maxplayer_core::sandbox_netns::holder_argv( + &holder_name, + network, + &image, + uid, + gid, + "route-preflight", + "route-preflight", + ); + if let Err(error) = run_docker_argv(&holder, None) { + let _ = docker_rm_f(&holder_name); + return RouteProbe::Unbuildable(format!("the namespace holder would not start: {error}")); + } + + let net_policy = maxplayer_core::sandbox_net::NetPolicy { + gateway, + proxy_ports: None, + log_connections: false, + dns_resolvers: resolvers.to_vec(), + }; + let (plan, _rules) = maxplayer_core::sandbox_netns::plan_stdin(&net_policy); + let sidecar = maxplayer_core::sandbox_netns::sidecar_argv_for(&holder_name, &image); + if let Err(error) = run_docker_argv(&sidecar, Some(plan)) { + let _ = docker_rm_f(&holder_name); + return RouteProbe::Unbuildable(format!("the egress policy would not install: {error}")); + } + + let outcome = job_leg( + policy, + &workdir, + uid, + gid, + Some(holder_name.as_str()), + Some(resolv_path.as_path()), + "the job namespace", + ); + let _ = docker_rm_f(&holder_name); + outcome + } + + /// The job leg: the seat's own image, launched by [`SandboxPolicy::launch`] exactly as an + /// awarded job is, reporting the two legs that matter. + fn job_leg( + policy: &SandboxPolicy, + workdir: &std::path::Path, + uid: u32, + gid: u32, + netns: Option<&str>, + resolv_conf: Option<&std::path::Path>, + where_: &str, + ) -> RouteProbe { + let job = maxplayer_core::seller_exec::JobLaunch { + workdir, + env: &[], + uid, + gid, + netns, + resolv_conf, + }; + let launch = match policy.launch(&route_payload(), &job) { + Ok(launch) => launch, + Err(error) => { + return RouteProbe::Unbuildable(format!("cannot build the probe launch: {error}")) + } + }; + let mut argv = Vec::with_capacity(launch.args.len() + 1); + argv.push(launch.program); + argv.extend(launch.args); + let output = match run_docker_argv_capturing(&argv) { + Ok(output) => output, + Err(error) => { + return RouteProbe::Unbuildable(format!("the probe container did not run: {error}")) + } + }; + read_route_markers(&output, where_) + } + + /// Judge the payload's own words. Kept separate from the spawning so the verdicts are unit- + /// testable without a daemon, and so "neither marker appeared" stays a distinct outcome from + /// "the DNS marker said it failed". + pub(super) fn read_route_markers(output: &str, where_: &str) -> RouteProbe { + let marker = |name: &str| { + output.lines().find_map(|line| line.trim().strip_prefix(name).map(str::trim)) + }; + if let Some(detail) = marker(ROUTE_DNS_FAIL) { + return RouteProbe::NoDns(format!("{detail} (from inside {where_})")); + } + let Some(address) = marker(ROUTE_DNS_OK) else { + return RouteProbe::Unbuildable(format!( + "the probe in {where_} reported neither success nor failure; it likely never ran" + )); + }; + if let Some(detail) = marker(ROUTE_TLS_FAIL) { + return RouteProbe::NoTls(format!("{detail} (from inside {where_})")); + } + match marker(ROUTE_TLS_OK) { + Some(subject) => RouteProbe::Delivered { + resolver: address.split_whitespace().nth(1).unwrap_or("?").to_owned(), + address: address.split_whitespace().next().unwrap_or("?").to_owned(), + subject: subject.to_owned(), + }, + None => RouteProbe::NoTls(format!( + "the lookup succeeded in {where_} but the handshake reported nothing" + )), + } + } + + /// The payload, in the image's own node: resolve, then complete a TLS request whose certificate + /// chain is VERIFIED. + /// + /// `rejectUnauthorized` is left at its default and `socket.authorized` is REPORTED, so a pass + /// cannot be a handshake that skipped verification — which is the failure mode a preflight for + /// delivery would be worst at catching. + fn route_payload() -> Vec { + let script = format!( + "const dns=require('dns'),https=require('https');const h='{ROUTE_PROBE_HOST}';\ + dns.lookup(h,(e,a)=>{{if(e){{console.log('{ROUTE_DNS_FAIL} '+e.code);process.exit(0);}}\ + console.log('{ROUTE_DNS_OK} '+a);\ + const r=https.request({{host:h,port:443,path:'/',method:'HEAD',timeout:15000}},(res)=>{{\ + const c=res.socket.getPeerCertificate();\ + if(res.socket.authorized){{console.log('{ROUTE_TLS_OK} '+((c&&c.subject&&c.subject.CN)||h));}}\ + else{{console.log('{ROUTE_TLS_FAIL} certificate chain not verified');}}process.exit(0);}});\ + r.on('timeout',()=>{{console.log('{ROUTE_TLS_FAIL} timeout');process.exit(0);}});\ + r.on('error',(err)=>{{console.log('{ROUTE_TLS_FAIL} '+err.code);process.exit(0);}});r.end();}});" + ); + vec!["node".to_owned(), "-e".to_owned(), script] + } + + /// The uid/gid owning `path` — the uid an awarded job's container runs as, read from the + /// filesystem rather than through a `libc` dependency this crate does not otherwise carry. + fn owner_uid_gid(path: &std::path::Path) -> Option<(u32, u32)> { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + std::fs::metadata(path).ok().map(|m| (m.uid(), m.gid())) + } + #[cfg(not(unix))] + { + let _ = path; + None + } + } + + /// The gateway of a docker network, asked of the daemon rather than computed — the same trap + /// [`maxplayer_core::sandbox_netns`] documents: a computed gateway puts the pinhole on an + /// address nothing listens on while every rendering test stays green. + fn network_gateway(network: &str) -> Result { + let output = std::process::Command::new("docker") + .args([ + "network", + "inspect", + network, + "--format", + "{{(index .IPAM.Config 0).Gateway}}", + ]) + .output() + .map_err(|error| error.to_string())?; + if !output.status.success() { + return Err(String::from_utf8_lossy(&output.stderr).trim().to_owned()); + } + let gateway = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + if gateway.is_empty() { + return Err("the daemon reported no gateway for it".to_owned()); + } + Ok(gateway) + } + + fn docker_rm_f(name: &str) -> std::io::Result { + std::process::Command::new("docker").args(["rm", "-f", name]).output() + } + + /// Run a `docker ...` argv (element 0 is the program), optionally feeding it stdin. + fn run_docker_argv(argv: &[String], stdin: Option) -> Result { + let (program, args) = argv.split_first().ok_or("empty argv")?; + let mut command = std::process::Command::new(program); + command.args(args); + command.stdin(match stdin { + Some(_) => std::process::Stdio::piped(), + None => std::process::Stdio::null(), + }); + command.stdout(std::process::Stdio::piped()); + command.stderr(std::process::Stdio::piped()); + let mut child = command.spawn().map_err(|error| error.to_string())?; + if let Some(stdin) = stdin { + use std::io::Write as _; + let mut pipe = child.stdin.take().ok_or("no stdin pipe")?; + pipe.write_all(stdin.as_bytes()).map_err(|error| error.to_string())?; + } + let output = child.wait_with_output().map_err(|error| error.to_string())?; + if !output.status.success() { + return Err(String::from_utf8_lossy(&output.stderr).trim().to_owned()); + } + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) + } + + /// Like [`run_docker_argv`], but the payload's OUTPUT is the evidence, so a non-zero exit still + /// yields what it said. + fn run_docker_argv_capturing(argv: &[String]) -> Result { + let (program, args) = argv.split_first().ok_or("empty argv")?; + let output = std::process::Command::new(program) + .args(args) + .output() + .map_err(|error| error.to_string())?; + let mut combined = String::from_utf8_lossy(&output.stdout).into_owned(); + combined.push_str(&String::from_utf8_lossy(&output.stderr)); + Ok(combined) + } + const CREDENTIAL_CONTAINMENT_CHECK: &str = "sandbox credential containment"; /// The #647 credential proxy keeps out of a docker container every model-credential variable named @@ -2324,8 +2748,10 @@ fn build_checks( let sandbox_for_engine = sandbox.clone(); let sandbox_for_egress = sandbox.clone(); let sandbox_for_token_policy = sandbox.clone(); + let sandbox_for_route = sandbox.clone(); // The probe runs in the seat's OWN home, because that is where a launcher's config points. let home_root = home.root.clone(); + let route_home_root = home.root.clone(); // Reaching this box with stranger-written code is what the containment gate is about, and there // are TWO surfaces that do it. Both are off by default, so an unconfigured seat is reachable only // by the buyers its operator named and every one of these findings stays advisory. @@ -2399,6 +2825,14 @@ fn build_checks( // one who can be told, so this WARNs. Advisory — never blocks boot, because turning a working // docker seat red on upgrade is a behaviour change, not a doctor's call. checks.push(Box::new(move || checks::check_sandbox_egress(sandbox_for_egress))); + // The route a JOB takes, measured from inside a job's own namespace. Unlike every other network + // check here it does not ask the host anything: a seat whose host resolves perfectly while its + // gVisor jobs die on EAI_AGAIN is exactly the state that produced this check, and a host-side + // probe calls that seat ready. BLOCKING, and transient so the boot gate's bounded retry gives a + // daemon a moment to come back before the seat is refused. + checks.push(Box::new(move || { + checks::check_sandbox_delivery_route(sandbox_for_route, route_home_root) + })); // #792 phase 3: under mode=docker, the image the seat runs jobs in must be present or pullable, // or the first awarded job stalls. On absence this prints the exact `docker pull` command. A // non-docker policy is a no-op Pass. Placed after the launcher (docker-resolves) check. @@ -5698,4 +6132,156 @@ mod tests { run.err ); } + /// RED-PROVE for the gVisor delivery failure: a seat whose HOST resolves and fetches perfectly + /// while its jobs cannot must be refused, and refused with words that send the operator to the + /// job's route rather than to their own network. + /// + /// This is the whole point of the check. Every other network row here is answered by the host, + /// and the measured failure — docker's embedded resolver at `127.0.0.11` being unreachable from + /// inside a gVisor sandbox — is invisible from there: the seat looks healthy, advertises, wins a + /// job, and fails it. So the probe result is INJECTED here and the host's own connectivity is + /// never consulted: these assertions hold on a laptop with perfect internet. + #[test] + fn doctor_delivery_route_fails_when_the_job_route_is_broken_however_healthy_the_host_is() { + use checks::RouteProbe; + use maxplayer_core::home::{SandboxConfig, SandboxMode}; + let docker = || { + Some(SandboxConfig { + mode: SandboxMode::Docker, + image: Some("maxplayer-sandbox:latest".into()), + network: Some("sbx".into()), + ..Default::default() + }) + }; + let resolvers_ok = |_: &[String]| Ok(vec!["1.1.1.1".to_owned()]); + + // The gVisor case itself: resolvers were found, the namespace was built, and the JOB still + // could not resolve. Blocking, and retryable. + let no_dns = checks::check_sandbox_delivery_route_in(docker(), resolvers_ok, |_, _| { + RouteProbe::NoDns("EAI_AGAIN (from inside the job namespace)".to_owned()) + }); + assert_eq!(no_dns.status, Status::Fail, "{}", no_dns.render()); + assert!(no_dns.transient, "a resolver blip deserves the bounded retry: {}", no_dns.render()); + assert!( + !readiness_ok(&[no_dns.clone()]), + "a seat whose jobs cannot resolve must not be advertised as ready: {}", + no_dns.render() + ); + assert!( + no_dns.render().contains("JOB") && no_dns.render().contains("dns_servers"), + "the remedy must point at the job's route and the key that fixes it, not at the host: \ + {}", + no_dns.render() + ); + + // Resolved, but the handshake never completed or its chain was not verified. Also blocking: + // a job that cannot complete verified TLS cannot deliver an answer. + let no_tls = checks::check_sandbox_delivery_route_in(docker(), resolvers_ok, |_, _| { + RouteProbe::NoTls("certificate chain not verified".to_owned()) + }); + assert_eq!(no_tls.status, Status::Fail, "{}", no_tls.render()); + assert!(!readiness_ok(&[no_tls.clone()]), "{}", no_tls.render()); + + // Could not be measured at all. NOT a pass — "I could not ask" is not "it works", and this + // is the arm a future edit is most likely to soften into a Warn. + let unbuildable = checks::check_sandbox_delivery_route_in(docker(), resolvers_ok, |_, _| { + RouteProbe::Unbuildable("the namespace holder would not start".to_owned()) + }); + assert_eq!(unbuildable.status, Status::Fail, "{}", unbuildable.render()); + assert!(!readiness_ok(&[unbuildable])); + + // No resolver can be given to a job at all. Blocking and NOT transient: no retry discovers + // a resolver the box does not have, so burning the backoff budget only delays the same + // refusal. + let no_resolver = checks::check_sandbox_delivery_route_in( + docker(), + |_| Err("the host names only the systemd stub and resolvectl reported none".to_owned()), + |_, _| panic!("the probe must not run when no resolver can be handed to a job"), + ); + assert_eq!(no_resolver.status, Status::Fail, "{}", no_resolver.render()); + assert!( + !no_resolver.transient, + "retrying cannot conjure a resolver; refuse immediately: {}", + no_resolver.render() + ); + + // The healthy route passes, and says what it actually proved — a resolver, an address, and + // a VERIFIED peer — so a green row cannot be read as "docker looked fine". + let ok = checks::check_sandbox_delivery_route_in(docker(), resolvers_ok, |_, _| { + RouteProbe::Delivered { + resolver: "1.1.1.1".to_owned(), + address: "34.225.223.145".to_owned(), + subject: "relay.maxplayer.ai".to_owned(), + } + }); + assert_eq!(ok.status, Status::Pass, "{}", ok.render()); + assert!( + ok.detail.contains("verified") && ok.detail.contains("1.1.1.1"), + "a pass must name the resolver it used and that the handshake was verified: {}", + ok.detail + ); + assert!(readiness_ok(&[ok])); + + // A host executor has no container route to measure ⇒ never a spurious failure. + assert_eq!( + checks::check_sandbox_delivery_route_in(None, resolvers_ok, |_, _| panic!( + "a host executor has no job container to probe" + )) + .status, + Status::Pass, + ); + + // No docker on PATH is the launcher check's verdict, not this one's: reporting it here too + // would refuse a box twice for one fault and bury the row that names the real fix. + let unmeasurable = + checks::check_sandbox_delivery_route_in(docker(), resolvers_ok, |_, _| { + RouteProbe::Unmeasurable("docker not resolvable".to_owned()) + }); + assert_eq!(unmeasurable.status, Status::Pass, "{}", unmeasurable.render()); + } + + /// The payload's own words are judged, and silence is not consent: a container that printed + /// nothing (no node in the image, an entrypoint that swallowed the payload) must never read as a + /// working route. + #[test] + fn a_silent_route_probe_is_not_a_passing_route() { + use checks::RouteProbe; + assert!(matches!( + checks::read_route_markers("", "the job namespace"), + RouteProbe::Unbuildable(_) + )); + assert!(matches!( + checks::read_route_markers("sh: node: not found", "the job namespace"), + RouteProbe::Unbuildable(_) + )); + // Resolved, then nothing about the handshake: not a pass either. + assert!(matches!( + checks::read_route_markers("route-dns-ok 34.225.223.145", "the job namespace"), + RouteProbe::NoTls(_) + )); + assert!(matches!( + checks::read_route_markers("route-dns-fail EAI_AGAIN", "the job namespace"), + RouteProbe::NoDns(_) + )); + assert!(matches!( + checks::read_route_markers( + "route-dns-ok 34.225.223.145\nroute-tls-ok relay.maxplayer.ai", + "the job namespace" + ), + RouteProbe::Delivered { .. } + )); + } + + /// The host the probe shakes hands with is the relay the seat actually delivers to. Pinned so a + /// relay move cannot leave the preflight proving reachability to an address nothing uses. + #[test] + fn the_route_probe_targets_the_configured_relay_host() { + assert!( + maxplayer_core::home::DEFAULT_RELAY_URL.contains(checks::ROUTE_PROBE_HOST), + "probe host {} is not the relay in DEFAULT_RELAY_URL {}", + checks::ROUTE_PROBE_HOST, + maxplayer_core::home::DEFAULT_RELAY_URL + ); + } + } diff --git a/docs/gvisor-dns-delivery/DESIGN.md b/docs/gvisor-dns-delivery/DESIGN.md new file mode 100644 index 00000000..4189935e --- /dev/null +++ b/docs/gvisor-dns-delivery/DESIGN.md @@ -0,0 +1,94 @@ +# Fix design — gVisor named-network DNS, and a delivery preflight that can fail + +Status: **drafted, not yet executed as code.** The measurements it rests on are +real (`RUNLOG.md`, `evidence/gate1-runsc-vs-runc-20260910T0046Z.txt`); the code +below is not written until a disposable Linux host is ruled, because an +unexecuted fix is not a fix. + +## What has to change, and why each piece exists + +### 1. The job sandbox gets a resolver it can actually reach + +Measured: under runsc, `127.0.0.11:53` answers nothing at all, and `--dns` does +not move it — docker writes `nameserver 127.0.0.11` on any user-defined network +regardless. So the only lever that works from outside the daemon is the file +itself: mount the job's `/etc/resolv.conf` read-only, naming real upstream +resolvers. + +Lands in `seller_exec.rs::run_argv` (the same argv that already carries +`--runtime`, `--cap-drop ALL`, `--user`, `--security-opt no-new-privileges`) as +one more `-v :/etc/resolv.conf:ro`. The generated file is per-seat, +not per-job — it holds no job data — and is written under the seller home with +mode 0444. + +Resolver selection, in order, with **no silent fallback**: + +1. explicit config (`[sandbox] dns_servers`), if set — an operator on a VPS with + a mandated resolver needs this and it is the only way to express it; +2. otherwise the host's real upstream resolvers, discovered by reading + `/etc/resolv.conf` and, when that names only a local stub (`127.0.0.53`, + systemd-resolved — exactly what Bob's VPS shows), `resolvectl status` for the + actual upstreams; +3. otherwise **fail loudly at boot**. Guessing `8.8.8.8` here would be a silent + host-side fallback and is precisely what the brief forbids. + +A stub address (`127.0.0.0/8`) is never written into the sandbox file: it is +unreachable from inside the sandbox by construction, and writing it would +reproduce the bug with a different address. + +### 2. The egress policy opens port 53 to exactly those resolvers + +`NetPolicy` currently carries `gateway`, `proxy_ports`, `log_connections`, and +denies the private ranges wholesale (`DENIED_DESTINATIONS`) while never denying +loopback. DNS to an upstream resolver is new traffic that the deny ranges may +shadow, so the policy grows one field: the resolver addresses. + +Rules added in `NetPolicy::rules()`, before the range denies for the same reason +the proxy pinhole is: + +- `-p udp -d /32 --dport 53 -j ACCEPT` and the tcp counterpart, one + pair per resolver, **/32 (or /128) only** — never a subnet, never "port 53 + anywhere in the private range". If an operator's resolver is itself a private + address, this opens that single host and nothing else, and the check in §3 + proves what it opened. +- `verify_readback` learns the same rules, so a namespace missing them is + reported rather than assumed. + +That keeps gate 5 intact: every other private, loopback, link-local and metadata +destination stays denied, and a DNS name resolving to a denied address still +dies at the deny rules because resolution and reachability are separate rules. + +### 3. Doctor/readiness runs the real route, and a failure blocks ready + +Today `check_sandbox_egress` answers "can a namespace be built" (docker network +exists) and is deliberately `Warn`-only (petar, 2026-08-18: automate first, then +require). That check is not touched — it answers a different question, and its +advisory status was a ruling, not an oversight. + +The new check is a different thing: **it launches the actual contained sandbox +and makes it resolve and complete a certificate-validated TLS handshake to the +delivery host**, under the production runtime, user, cap-drop and namespace. +Host connectivity is never consulted, so a host that can reach the internet +while the sandbox cannot produces a FAIL, which is the whole point. + +- Status `Fail`, so `readiness_ok` (any `Fail` ⇒ not ready) refuses the seat. +- `transient: true`, so a genuine network blip is retried on the existing + bounded schedule (5 attempts, 20s/40s/60s/80s) and a still-broken route is + then refused. Transient-retry is not transient-forgiveness. +- The failure message names the resolver it used, the runtime, and the exact + docker command to reproduce, because "DNS failed" sends nobody anywhere. + +### 4. Delivery is proven from inside the sandbox, not beside it + +Gate 4 requires a real container-side git push whose remote hash matches. The +preflight in §3 proves DNS+TLS; the delivery gate proves the actual `git push` +path from the same contained sandbox to a disposable remote, with the remote's +hash read back afterwards. A host-side upload passing while the sandbox path is +broken is the exact false-ready this work exists to kill. + +## What is deliberately NOT done + +- No `--network=host`, no runsc network passthrough, no runc fallback: the + measured fix needs none of them. +- No broad private-range allowance; resolver pinholes are single addresses. +- No relaxation of `check_sandbox_egress`'s existing ruling in either direction. diff --git a/docs/gvisor-dns-delivery/RUNLOG.md b/docs/gvisor-dns-delivery/RUNLOG.md new file mode 100644 index 00000000..11d06972 --- /dev/null +++ b/docs/gvisor-dns-delivery/RUNLOG.md @@ -0,0 +1,814 @@ +# gVisor named-network DNS + container delivery — run log + +Lane: `w-gvisor-dns-delivery-r2` · ordering seat: maxie (requested by Bob) +Brief: `/Users/forge/forge/v2/maxie/runs/gvisor-dns-delivery-brief-20260909.md` +(3,851 B, sha256 `3b0a20feab6ae107e9ddc28cb70b30b1a73656c15fa95c72abc62edd25430812`, verified 2026-09-09) + +Six acceptance gates. Nothing below is claimed as passing unless the command +output is recorded here or in `evidence/`. + +## Environment reality (2026-09-09, recorded before any test) + +| Fact | Value | +| --- | --- | +| Forge host | macOS 26.4.1, arm64 (Apple silicon) | +| Reported failing host (Bob) | Ubuntu 24.04, kernel 6.8.0-124, x86_64, Docker 29.8.0, runsc release-20260831.0 | +| Shared container runtime on this host | colima `default`, aarch64, Docker 29.7.2 client / 29.5.2 server — **off limits**, other lanes use it, no runsc install there | +| Disposable Linux for gates | lima VM `gvisor-repro`, Ubuntu 24.04 (noble release-20260705), aarch64, vz driver, 2 CPU / 4 GiB / 20 GiB | + +**Architecture divergence is named, not papered over.** The reproduction host +available to this lane is aarch64; the reported evidence is x86_64. Any gate +result carries the arch it was executed on. If a failure mode proves x86-only, +that is reported as a limitation, not as a pass. + +## Source map (read before edits) + +- `crates/maxplayer-core/src/sandbox_net.rs` (1126 lines) — renders the egress + policy. Already documents that loopback must never be denied because docker's + embedded DNS answers at `127.0.0.11` inside the namespace, and carries a + load-bearing test for it. +- `crates/maxplayer-core/src/sandbox_netns.rs` (939 lines) — puts the policy in + force via holder → sidecar → job, all sharing one network namespace. Header + asserts "name resolution is unaffected: a container joining the namespace + still gets its own `/etc/resolv.conf` pointing at docker's embedded resolver + on `127.0.0.11`". That assertion is exactly what the reported failure + contradicts under runsc, so it is the first thing to test rather than trust. +- `crates/maxplayer-core/tests/sandbox_netns_live.rs` (840 lines) — existing + live tests. + +## Timeline + +- 17:39 PDT — brief read, hash verified, worktree + branch created off + immutable commit `b45f8651dc9cab5b71c962eadd7b84840f5791061dc9cab5b71c962eadd7b84840f579106`. +- 17:41 PDT — disposable lima VM `gvisor-repro` creation started (Ubuntu 24.04 + cloud image, arm64). Shared colima VM deliberately untouched. +- 17:46 PDT — **failure reproduced** (`evidence/gate1-runsc-vs-runc-20260910T0046Z.txt`). +- 17:52 PDT — root cause isolated and a containment-preserving fix validated by + hand before any code was written. + +## Gate 1 result — reproduced (aarch64) + +Exact run recorded in `evidence/gate1-runsc-vs-runc-20260910T0046Z.txt`. +Identical image, identical named bridge, identical container security settings; +only `--runtime` differs: + +| Probe | Result | +| --- | --- | +| `dns.lookup(relay.maxplayer.ai)` under `--runtime runsc` | `ERR EAI_AGAIN`, exit 1 | +| same under `--runtime runc` (control) | `OK 34.225.223.145`, exit 0 | +| raw UDP datagram to `127.0.0.11:53` under runsc | **TIMEOUT — no answer at all** | + +Both containers were handed the *same* `/etc/resolv.conf` +(`nameserver 127.0.0.11`, docker's embedded resolver, `ExtServers: +[host(127.0.0.53)]`). Digests: image +`sha256:1c50e46a35dfe91fcdbbba11876bff312a95567bda98d6dcb7f675c884777412` +(arm64/linux), docker 29.1.3, runsc release-20260817.0, kernel 6.8.0-134, +network subnet 172.18.0.0/16. + +**Causal source.** The failure is not resolver policy, not the allowlist, and +not name-specific: a bare UDP packet to `127.0.0.11:53` gets no reply inside the +sandbox. Docker's embedded DNS on a *user-defined* network is a socket bound by +the daemon inside the container's network namespace on `127.0.0.11:`, +reached through NAT rules installed in that namespace. Under runsc the sandbox +runs its own network stack and terminates loopback inside the sentry, so those +packets never reach the namespace-side rules or the daemon's socket. Under runc +the container shares the host kernel's stack, so they do. That is the whole +delta, and it explains why the shared job namespace fails identically — the +holder's namespace has exactly the same embedded resolver. + +**`--dns` does not fix it** (measured): with `--dns 1.1.1.1` on a user-defined +network docker *still* writes `nameserver 127.0.0.11` and merely forwards +upstream from the daemon side, so the container still fails `EAI_AGAIN`. Any +fix that only sets docker DNS flags is theatre. + +**Validated fix direction** (measured, same runsc runtime, same named network, +same `--user 65534:65534 --cap-drop ALL --security-opt no-new-privileges`): +supply the sandbox its own `/etc/resolv.conf` naming real upstream resolvers, +read-only, instead of the unreachable embedded one. Result: `lookup: OK +34.225.223.145`, `tls: 200 cert-verified`. Containment is untouched — still the +named bridge, no host networking, no runc, no added capability. The egress +policy must then explicitly permit port 53 to exactly those resolver addresses +and nothing wider. + +## Gate status + +| Gate | State | +| --- | --- | +| 1 repro + runc control, digests, causal evidence | **done (aarch64), evidence committed** | +| 2 DNS + TLS from real shared job namespace, fresh + recreated | not started | +| 3 doctor/readiness on the real sandbox route + regression tests | not started | +| 4 real container-side Git delivery, remote hash match | not started | +| 5 private/metadata denial + concurrent public success | not started | +| 6 bounded gate script, executed test counts, PR | not started | + +## 2026-09-10 — gate 5 turned up a hole bigger than the one I was sent for + +Gate 5's cross-job leg failed: from a runsc job in a namespace carrying the full +26-rule plan, a container at `172.31.12.3:8080` — inside `-d 172.16.0.0/12 -j DROP` — +was **REACHED**. + +`gate5b-does-the-plan-bind-a-runsc-job.sh` isolates it. Same namespace, same plan +(read back from the netns: the DROP is rule 11/12 and it is there), same probe, +one variable — the job's runtime: + +| job runtime | result to a listener inside a DROPped range | +| --- | --- | +| runc | `timeout` — the DROP is enforced | +| runsc | `REACHED` — the DROP is not | + +**The per-job egress plan does not bind the runtime it was written for.** gVisor's +netstack terminates the network inside the sandbox and writes frames to the veth +itself; the host kernel's OUTPUT chain in that netns only sees packets from host +sockets, so it never sees the job's. The chain is installed, correct, verified by +readback — and irrelevant to a gVisor job. + +This is not a regression from the DNS work; it predates this branch. The DNS +change opens port 53 to a `/32` in a chain that was already not constraining the +job. + +### It also devalues part of my own gate-2 evidence +Gate 2 recorded `metadata: denied (ENETUNREACH)`. I read that as policy. It is +not evidence of policy: nothing listens on `169.254.169.254` in this VM, and +**absence is indistinguishable from enforcement** unless the denied destination +has a live listener. Every denial leg in gate 5 that "passed" against a dead +address proves nothing. Only the neighbour leg — a real listener inside a real +DROP range — was a valid test, and it failed. + +Rule for the remaining gates: a denial is only proven against a destination that +answers when it is allowed to. + +### Where containment has to live instead +Not in the netns OUTPUT chain. The candidate that gVisor cannot bypass is the +host side of the veth: FORWARD-chain rules in the root netns keyed to the job +namespace's source address, and/or a per-job network rather than one shared +`maxplayer-sbx` bridge (all jobs currently share it, which is why job A could see +job B at all). Both need measuring before either goes in. + +## 2026-09-10 — the shared job namespace is SINGLE-USE for gVisor + +`gate5c` tried to measure the two candidate enforcement sites and returned +`ENETUNREACH` for everything late in the run — including the **runc control**, +which had worked minutes earlier in that same namespace, and including DNS to +`1.1.1.1`, which no rule under test touched. A control that dies is not a +control, so gate5c's verdicts are **void**. + +`gate5d` settles why. One namespace, read with `os.networkInterfaces()`: + +| moment | interfaces | dns | +| --- | --- | --- | +| before any gVisor container | `lo=127.0.0.1 eth0=172.31.16.2` | `34.225.223.145` | +| during the runsc job | `lo=127.0.0.1 eth0=172.31.16.2` | `34.225.223.145` | +| after it exits, via runc | `lo=127.0.0.1` | `EAI_AGAIN` | +| after it exits, via a second runsc job | `lo=127.0.0.1` | `EAI_AGAIN` | + +**A gVisor container takes the namespace's addresses into its netstack and does +not give them back when it exits.** The namespace is usable exactly once. The +second container to enter it — whatever runtime — finds a namespace with nothing +but loopback. + +### What this voids, and what survives +- **VOID**: gate5c, both candidates. Measured against a dead namespace. +- **VOID**: gate 5's git leg (`Could not resolve host: github.com`). It was the + second runsc container in that namespace, not a DNS bug. +- **STANDS**: gate5b. Its runc control ran FIRST, while the namespace was + healthy, and was correctly dropped; the runsc probe was the first gVisor + container in that namespace. The finding holds: the plan binds runc and not runsc. +- **STANDS**: gate 4 and gate 2 — one runsc container per namespace in each. +- **STANDS**: the gate-2 retraction, for the separate dead-address reason. + +### Rule for every remaining measurement +One gVisor container per namespace, and a health check of the namespace +immediately before any leg whose result is meant to be evidence. Re-run gate5c +under that rule before either enforcement site is chosen. + +## 2026-09-10 — gate5c, re-run soundly: DOCKER-USER does not bind a bridged gVisor job; a per-job network does + +Rewritten under the one-gVisor-container-per-namespace rule, with a health check +printed beside every leg. Every leg below ran in a namespace verified HEALTHY +(address present, DNS resolving) immediately before the probe, against a LIVE +listener. + +| leg | runc | runsc | +| --- | --- | --- | +| live neighbour, same bridge, no host rule | `timeout` | **`REACHED`** | +| live neighbour, same bridge, `DOCKER-USER -s /32 -d 172.16.0.0/12 -j DROP` | `timeout` | **`REACHED`** | +| live neighbour, **other bridge** | — | `timeout` | + +With that DOCKER-USER rule in place the public route the job must keep is +untouched: `PUBLIC-PASS dns=34.225.223.145 tls=200 verified=true` and +`PUBLIC-PASS git 7fd1a60b…`. + +**(a) DOCKER-USER does not bind a gVisor job talking to a peer on its own bridge.** +The mechanism is in the environment line: `br_netfilter=absent`. Without +`bridge-nf-call-iptables`, frames switched between two containers on the same +bridge never enter iptables' FORWARD path at all, so DOCKER-USER cannot see them. +The runc leg's `timeout` in that row is its own netns OUTPUT plan doing the work, +not DOCKER-USER — which is exactly why a control that only ever shows "denied" +proves nothing about the site under test. + +**(b) A per-job network does bind it.** Cross-bridge traffic is routed rather than +switched, so it meets DOCKER-ISOLATION in the root netns and is dropped, gVisor +or not. + +### The shape of the fix this implies +A per-job network is not merely nicer isolation, it is what makes host-side +enforcement possible at all: once the job's only on-link peer is its gateway, +every other destination is ROUTED, and routed packets from a gVisor sandbox do +traverse the host's FORWARD path where a source-keyed policy can bind them. +Shared bridge + gVisor is the combination with no enforcement point. + +Still to measure before any of this is written into product code: with a per-job +network, does a source-keyed DOCKER-USER policy actually deny a runsc job a +ROUTED private destination that docker isolation does not already block, and +does it deny the metadata address? Host-directed traffic (the VM's own +`192.168.5.15`) lands in INPUT, not FORWARD, and needs its own answer. + +## 2026-09-10 — gate5e: the enforcement sites, named + +Same discipline: fresh holder+plan namespace per probe, one gVisor container in +each, health printed beside every leg, live listeners. + +**1. The host itself (`192.168.5.15:49252`, a real listener, reached by route)** + +| leg | result | +| --- | --- | +| runc, bare | `timeout` — its netns plan drops 192.168/16 | +| runsc, bare | **`REACHED`** | +| runsc, `DOCKER-USER -s /32 -d 192.168.0.0/16 -j DROP` | **`REACHED`** | +| runsc, `INPUT -s /32 -d 192.168.0.0/16 -j DROP` | `timeout` | + +A gVisor job can reach the host's own LAN address today, and DOCKER-USER cannot +stop it: host-directed packets are delivered locally, so they land in **INPUT** +and never traverse FORWARD. INPUT binds it. + +**2. The metadata address.** Nothing listens on it here, so only the difference +between runs is evidence — and there is one: bare `ECONNREFUSED`, with +`DOCKER-USER -s /32 -d 169.254.169.254/32 -j DROP` `timeout`. The rule +changes the outcome, so DOCKER-USER does bind the metadata address for a runsc +job. It is reached by route through the gateway, which is why FORWARD sees it +and the same chain was useless for the host and for a same-bridge peer. + +**3. Cross-job, each job on its own network:** runsc → job 2's live listener, +`timeout`. Per-job networks are the cross-job answer. + +**4. The public route survives all of it:** with both rules installed, +`PUBLIC-PASS dns=34.225.223.145 tls=200 verified=true` and +`PUBLIC-PASS git 7fd1a60b…`. + +### The fix, now fully specified by measurement +Three parts, none of which replaces the others: +1. **A per-job network**, not the one shared `maxplayer-sbx` bridge. It contains + job-to-job traffic and it converts everything else into routed traffic that + host chains can see at all. +2. **A host-side, source-keyed policy** applied daemon-side for the job + namespace's address: DOCKER-USER for routed destinations (metadata, private + ranges off-link) and INPUT for host-directed ones. +3. **The existing netns OUTPUT plan, kept** — it is what binds a runc job, and + it costs nothing to leave in place as defence in depth. + +## Gate 5f — the product's own host policy, measured against the hole it was built for + +`scripts/gate5f-product-host-rules-bind-runsc.sh`, evidence in +`evidence/gate5f-evidence.txt`. aarch64, kernel 6.8.0-134, docker 29.1.3, +runsc release-20260817.0, `br_netfilter=absent`, daemon `default-runtime=runc`. + +The rules under test are **rendered by the product**, not written by the script: +`cargo run -p maxplayer-core --example render_host_plan -- 172.31.21.10`, from +`HostPolicy`, 17 rules (`evidence/gate5f-host-install-rendered.txt`), plus the +exact inverse for teardown. The probe namespace's address is pinned with +`--ip` and the script **refuses to run** unless the rendered plan carries +`-s 172.31.21.10/32`; a plan keyed to the wrong address would deny some other +container while leaving this job open, and that mistake would otherwise be +invisible. They are installed by piping the plan into the **same applier image +the sidecar uses**, in a `--network host` container — the product's path, so +the image's ability to write the root namespace's chains is itself under test. + +### What changed, and what did not +| leg | before | after | what actually binds it | +|---|---|---|---| +| runsc → host `192.168.5.15:49253` (live) | REACHED | timeout | **the new INPUT rules** | +| runc → host `192.168.5.15:49253` (live) | timeout | timeout | the netns OUTPUT plan, already | +| runsc → own-network neighbour (live) | timeout | timeout | the per-job network + DOCKER-ISOLATION | +| runsc → same-bridge neighbour (live) | REACHED | REACHED | **nothing — and that is the point** | + +Only the first row is a delta produced by the host policy, and it is the exact +hole gate5e found: a gVisor job reaching the host itself, which DOCKER-USER +cannot stop because host-directed packets never traverse FORWARD. Gate5e +supplies the matching delta for DOCKER-USER (metadata: bare `ECONNREFUSED` vs +ruled `timeout`). Together the two chains of the rendered plan are each caught +working, on evidence, against a runsc job. + +The rows that did not change are recorded deliberately. The own-network row was +already denied before the policy was installed, so **this gate does not prove +the host rules deny cross-job traffic** — the per-job network does that, and +claiming otherwise would repeat the gate-2 error of reading an absence as +enforcement. The same-bridge row stays REACHED because switched frames enter no +chain on a host without `br_netfilter`. That is not a defect in the policy; it +is why the product gives every job its own network instead of trying to rule +its way out of a shared one, and it is measured here so that reason stays +evidenced rather than asserted. + +### The route survives +With the full policy installed: `PUBLIC-PASS dns=34.225.223.145 tls=200 +verified=true` and `PUBLIC-PASS git 7fd1a60b01f91b314f59955a4e4d4e80d8edf11d`. + +### Teardown leaves no trace +The rendered teardown plan removed all 17 rules; host readback for the job +address went 9/8 → 0/0, and both chains returned to depth 1, the depth they +had before the run. This is the failure mode `HostRules`' drop guard exists +to prevent: a leaked container gets noticed, a leaked rule in a shared chain +does not, and a recycled address would inherit a dead job's policy. + +## Gate 5 — denial holds, and three concurrent jobs still deliver + +`scripts/gate5-denial-and-concurrent-success.sh` (rewritten), +evidence `evidence/gate5-denial-and-concurrent-success-PASS-20260910T0311Z.txt`. +The first version's FAIL is kept beside it as +`…-20260910T0245Z.txt`. aarch64, kernel 6.8.0-134, docker 29.1.3, runsc +release-20260817.0, `br_netfilter=absent`, daemon `default-runtime=runc`. +Five namespaces, each on its own network, each with rules rendered by the +product and guarded against a stale plan by source key. **Verdict: PASS, +0 denial legs failing.** + +### Denial, before and after the host policy +| destination | before | after | attributable to | +|---|---|---|---| +| host `192.168.5.15:49254`, **live** | REACHED | timeout | **the host policy (INPUT)** | +| `denied-lan.maxplayer.test:49254` → same live listener, **by name** | REACHED | timeout | **the host policy (INPUT)** | +| neighbour job `172.31.34.10:8080`, **live** | timeout | timeout | the per-job network, not the policy | +| `169.254.169.254:80`, nothing listens | ECONNREFUSED | timeout | **the host policy (DOCKER-USER)** — a real difference | + +Two legs are new evidence and two are careful non-claims. The by-name leg is +coverage no earlier gate had: a job that reaches a denied address through a +NAME is denied exactly as one that dials the address, which matters because +every real exfiltration attempt is a hostname. It uses a mounted hosts file +rather than a third-party wildcard DNS service so the leg cannot pass or fail +for an unrelated reason. + +The metadata leg is evidence only because bare and ruled runs DIFFER +(`ECONNREFUSED` → `timeout`). With nothing listening there, an identical +result in both sections would prove nothing, and this is the same reasoning +that made me retract gate 2's "metadata denied" line rather than defend it. + +The neighbour leg was already denied before the policy went in, so the host +policy is **not** credited with it. The per-job network plus docker's own +isolation does that, and gate5c is where it was isolated. + +### IPv6 — measured, and left unproven on purpose +The job namespace has no global IPv6 address at all (`NO-V6` under both +runtimes), so there was nothing here to deny and nothing was proved. The +host-side plan deliberately renders no ip6tables rules, because `DOCKER-USER` +is not guaranteed to exist in the v6 table. **Host-side IPv6 denial for a +gVisor job is UNPROVEN and must not be claimed.** The netns plan does cover +v6, and its readback is verified per family at install time, but gate5b showed +the netns plan does not bind a runsc job — so on a host where the job DOES get +a global v6 address, this is an open hole and is listed as such in the +limitations. + +### Concurrent delivery +Three jobs at once, each with its own network, holder, netns plan and +host policy installed simultaneously (17 rules each, all reading back on the +host kernel). Every one delivered: + +``` +c1: OK dns=34.225.223.145 tls=200 verified=true OK git 7fd1a60b… +c2: OK dns=34.225.223.145 tls=200 verified=true OK git 7fd1a60b… +c3: OK dns=34.225.223.145 tls=200 verified=true OK git 7fd1a60b… +``` + +Denial and delivery are therefore not in tension: the policy that blocks the +host, the metadata address and a denied name by name is the same policy under +which three jobs resolved, verified a certificate and cloned a repository at +the same time. + +### No leaks +All four policies torn down by their rendered inverse; the host kernel went +from 17 rules each to 0, and `DOCKER-USER`/`INPUT` returned to depth 1, where +they started. + +## Gate 6 — reproducibility, and what this branch does NOT prove + +`scripts/run-all-gates.sh` runs the set bounded, writes each gate's output to +its own evidence file, and prints a verdict per gate. Proof run (aarch64, +gate1 + gate5f): `ALL GATES: PASS` in 61s — +`evidence/run-all-gates-proof-summary-20260910T0316Z.txt`. I checked the +per-gate logs rather than the summary line, because 61s looked too fast for a +suite that had taken minutes before; both are complete runs, the speed being +warm images. A full five-gate run through the runner was not executed in one +sitting; gates 1, 2, 4, 5 and 5f each have their own full-run evidence file +from a direct run, and the runner is proved on two of them. **Saying which is +the point of this section.** + +### Limitations — each one a thing a reader should not assume +1. **x86_64 is OUTSTANDING.** Every result here is aarch64, kernel 6.8.0-134, + docker 29.1.3, runsc **release-20260817.0**. The host is arm64 and runsc + release-20260831.0 ships no aarch64 artifact, so neither a newer runsc nor + a different architecture was tested. gVisor's netstack behaviour is the + whole subject of this branch, and it is exactly the kind of thing that can + differ per platform. +2. **`br_netfilter` is ABSENT on this host**, and that shaped the measurements. + It is why a same-bridge peer is reachable and unbindable here. Where it is + enabled, switched frames do enter the chains and that leg may read + differently. The per-job network makes the product correct either way — with + no on-link peer, the case does not arise — but the MEASUREMENT is + host-specific and should not be quoted as universal. +3. **Host-side IPv6 is not rendered, and v6 denial for a gVisor job is + UNPROVEN.** `HostPolicy` deliberately emits no ip6tables rules because + `DOCKER-USER` is not guaranteed to exist in the v6 table. The netns plan does + cover v6 and its readback is verified per family — but gate5b showed the + netns plan does not bind a runsc job. In this VM the job namespace has no + global v6 address, so nothing was denied and nothing was proved. **On a host + whose jobs do get one, this is an open hole.** +4. **The container-side git push uses an unauthenticated disposable remote, by + design.** A credentialed push would mean putting a secret inside a + stranger's sandbox. Gate 4 therefore proves the network path for a write, + not an authenticated push. +5. **The runtime boundary is baseline, not new — but it deserves review.** + Holder, sidecar and the host-rule applier carry no `--runtime` and inherit + the daemon default; only the JOB carries the configured runtime. That is + true of that baseline commit too: `sandbox_netns.rs` there has no `--runtime` in + `holder_argv`/`sidecar_argv`, and `seller_exec.rs` `run_argv` (lines + 681–687) emits it for the job alone. This branch adds the test that pins it + (`the_containment_plane_never_carries_the_jobs_runtime`). An operator who + sets `default-runtime=runsc` gets a runsc holder; measured, that fails + CLOSED (the job sees `lo` only). None of the three helpers executes any + seller- or task-controlled input: the holder is `--entrypoint sleep … infinity`, + the other two read a plan rendered in Rust. +6. **The `--network host` rule applier is the one privileged surface this + branch adds.** It must be `--network host` because the rules have to land in + the root namespace's chains, which is the only place a gVisor job's packets + can be seen. It runs our own image, on a Rust-rendered plan, for + milliseconds, and it is gone before the job starts; the job never touches + it. That is my judgement and it should not be only mine — **advisor review + is requested on this specifically.** + +### Retractions kept in the record +Gate 2's "metadata denied (ENETUNREACH)" line is **withdrawn**: nothing listens +there, so absence was read as enforcement. Gate5c's first run is **VOID** +(namespace reuse under gVisor) and its file is kept marked VOID. The first +gate 5 script FAILED and that evidence is kept beside the passing rewrite. + +## Gate 5g — the baseline vulnerability, reproduced on that baseline commit + +Maxie's ruling: *"Prior-baseline vulnerability claim needs a baseline +reproduction/source citation."* Correct demand — the branch had been asserting +"this predates the fix" from source reading alone. Here is the reproduction. +`scripts/gate5g-baseline-vulnerability-repro.sh`, evidence +`evidence/gate5g-baseline-vulnerability-CONFIRMED-20260910T0330Z.txt`. + +Built to be unfair to my own claim: +* **Baseline rules.** Rendered from immutable commit `b45f8651dc9cab5b71c962eadd7b84840f5791061dc9cab5b71c962eadd7b84840f579106` by a throwaway + example compiled against BASELINE's `NetPolicy` (three fields, no + `dns_resolvers`) — 24 rules, kept as + `evidence/gate5g-baseline-plan-from-origin-main.txt`. A renderer built + against the fix would reproduce the fix, not the bug. The script refuses to + run if the staged plan carries resolver pinholes, which only the fixed plan has. +* **Baseline arrangement.** ONE shared network, as a single `[sandbox] network` + produced, and **no host-side rules at all** — that baseline commit has no + `HostPolicy` (grep: 0 occurrences). The script installs none. +* **Positive controls.** `runc` runs the identical probe. Had the plan failed to + bind runc too, the finding would be "the plan was never installed". +* **Readback.** Every attacker namespace prints `applier=24/24` and a kernel + readback of the `172.16.0.0/12` rules, so a REACHED cannot mean "no rules". + +### Result on `b45f8651dc9cab5b71c962eadd7b84840f579106` +| leg | runc (control) | runsc | | +|---|---|---|---| +| cross-job → live victim `172.31.40.20:8080` | timeout | **REACHED** | containment failure | +| private egress → live host `192.168.5.15:49255` | timeout | **REACHED** | containment failure | + +**BASELINE-VULNERABLE: CONFIRMED, 2 findings.** The netns plan binds a runc job +and not a gVisor one, and baseline has no second line of defence. The branch +does not introduce this; it repairs it. + +### A confounded first run, kept +The first attempt read `timeout` on the cross-job leg and looked like baseline +containment working. It was nothing of the kind. The victim carries the +baseline plan too, and its listener was running under **runc** — so the plan +bound the *victim*, and its own OUTPUT rules dropped its replies to a +172.16/12 peer. The probe measured a victim that could not answer, not an +attacker that could not reach. + +Fixed by running the victim's listener under **runsc**, which is the real +arrangement on a baseline seat (every job is a gVisor job), and by printing a +`VICTIM-SERVING` liveness check from inside the victim's own namespace before +any conclusion is drawn from a timeout. The confounded run is kept as +`evidence/gate5g-baseline-repro-CONFOUNDED-runc-victim-20260910T0325Z.txt`. + +It is the same failure this branch has now hit three times in different +clothes — gate 2's metadata line, gate5c's voided run, and this — and the +lesson is identical each time: **a timeout is only evidence when something was +proved able to answer.** + +## Gate 5h — lifecycle: teardown, recycled address, recreation + +Maxie: *"Prove selected enforcement path handles runsc, lifecycle +cleanup/recreation and fail-closed setup/readiness."* Gate 5 proved the rules +**deny**. It never proved they **go away**, and that gap is dangerous in both +directions: a rule keyed to job A's address does not stop existing when A does, +and docker hands addresses back — so the next job to get `172.31.55.2` would +inherit a firewall written for a stranger (traffic denied that should be +allowed, or an ACCEPT pinhole that was A's proxy and is now someone else's open +door). A network that fails to delete wedges the next job with the same id. + +`scripts/gate5h-lifecycle-and-recycled-address.sh`, evidence +`evidence/gate5h-lifecycle-recycled-address-PASS-20260910T0332Z.txt`, plans +`evidence/gate5h-plans/` (17 install / 17 teardown, rendered by +`render_host_plan`, never transcribed). + +**GATE 5h: PASS — 0 failing checks** (aarch64, runsc release-20260817.0): + +| check | result | +|---|---| +| A establishes → host rules appear | 0 → **17** rules keyed to `172.31.55.2` | +| A contained while installed | runsc → live host `timeout` | +| A tears down → rules gone | **0** rules survive | +| A tears down → network gone | removed | +| B **recycles A's address** `172.31.55.2` | genuinely recycled | +| B inherits stale firewall? | **0** stale rules | +| B bare (control) → live host | **REACHED** | +| B after its own rules → live host | **ENETUNREACH** | +| same job id twice | both came up, no "already exists" wedge | +| chains returned to start depth | 2 → 2, nothing leaked | + +### What each leg is worth +The recycled-address leg is the strong one, and it is the reason the gate +exists: a **bare-vs-ruled difference measured against a live listener** +(`REACHED` → `ENETUNREACH`), on an address that a previous job had owned. It +rules out both "the listener was never reachable" and "the old rules were doing +the work". + +Leg 1 proves less and should be read that way: it measures job A only with its +rules installed, with no bare control, so on its own it is consistent with the +host simply being unreachable from that namespace. It is corroboration, not +proof; the bare control in leg 3 and gate5f's `REACHED → timeout` carry the +weight. + +Note the two denials read differently — `timeout` for A, `ENETUNREACH` for B. +Both are denials, and the difference is not yet explained; it is most likely +DROP versus an unreachable route at the moment of probe. Recorded as an +observation, not a claim. + +## Fail-closed setup — what the product does, and what now holds it + +Maxie: *"…and fail-closed setup/readiness."* Read back from +`sandbox_netns.rs::establish()`, the host-side path refuses the job at **four** +distinct points rather than warning and continuing: + +1. the holder's address cannot be read → `could not read the job namespace's address`; +2. the address comes back **empty** → refused, because a policy with no source + key `would deny the range host-wide`; +3. the host-rule applier fails → `host-side containment was not installed`; +4. the applier's count ≠ the rendered count → `host-side containment is + incomplete … the plan was truncated in transit`. + +`HostRules` is adopted **before** the applier's result is examined, so a plan +that failed part-way still has its rules removed on the way out. A namespace +readback (#797 R1) then asks the kernel directly, because everything above it +is the installer's own account of its work. + +### The gap that was there +All four guards were held by **reading the source**, not by tests. Gate 5h +measured the teardown once, on one host. Four rendering tests now lock the +invariants on every build (`sandbox_net.rs`): + +* `the_host_teardown_exactly_inverts_the_install` — same length, `-I`↔`-D`, + reverse order, every field otherwise identical. If these two plans drift, + teardown leaks rules into a shared chain and a recycled address inherits a + dead job's firewall. +* `every_host_rule_is_keyed_to_the_job_address` — both directions; a rule + without `-s` is a deny for the whole range on a chain shared with every + container on the daemon. +* `the_rendered_host_plan_counts_exactly_what_it_renders` — the count guard 4 + relies on; if rendered count and line count could disagree, a truncated plan + would pass the cross-check. +* `an_empty_job_address_renders_a_source_key_that_is_not_a_host` — asserts the + *hazard* (bare `/32`), documenting why guard 2 must never be relaxed. + +### Test counts, corrected and verified +Earlier notes said "60 sandbox + 55 doctor". The doctor tests are **not** in +`maxplayer-core` — that crate has a `doctor` module with zero `#[test]`. They +live in the `maxplayer` **binary** target. Verified this run: + +| suite | command | result | +|---|---|---| +| core lib (all) | `cargo test -p maxplayer-core --lib` | **387 passed, 0 failed** | +| sandbox filter | `cargo test -p maxplayer-core --lib sandbox` | **65 passed, 0 failed** | +| doctor | `cargo test -p maxplayer --bins doctor` | **55 passed, 0 failed** (73 filtered) | + +The four new tests were confirmed **by name** in the output, not inferred from +the total moving. + +## Gate 5i — fail-closed, and a defect this branch introduced + +`scripts/gate5i-fail-closed-live.sh`, evidence +`evidence/gate5i-fail-closed-FAIL-teardown-leak-20260910T0339Z.txt`. +**GATE 5i: FAIL, 2 failing checks.** Kept as a FAIL, unfixed in this commit, +because it found something real. + +### What passed +* **A — applier cannot start** (bad image tag): 0 rules installed. `establish()` + maps this to a hard error. +* **B — applier without `NET_ADMIN`**: `apply-policy: namespace is PARTIALLY + configured — destroy the holder, do not retry`, **0** rules installed. +* **C — truncated plan**: the code comment claimed a truncated plan "applies + cleanly and exits 0, so no exit code reveals it". **Confirmed, measured**: + exit `0`, applier reported `9`, kernel held **9 of 17**. The count + cross-check is genuinely the only thing that catches it. And the question + that matters — with 9 of 17 rules, `runsc` → live host `192.168.5.15:49257` + came back **REACHED**. A partially-installed policy leaves the job + **uncontained**, so refusing the job is the only safe response. + +### What failed — and it is mine +* **D — teardown of a partial install left all 9 rules in place.** Chain depth + went **2 → 11**. The gate leaked rules into a chain shared with every + container on the daemon. + +Mechanism, confirmed directly against the image rather than inferred: + +``` +apply-policy: rule 1 failed: iptables -D DOCKER-USER -s 10.99.99.99/32 ... +apply-policy: namespace is PARTIALLY configured — destroy the holder, do not retry +``` +…and the *next* rule in that probe plan never ran. The applier aborts on the +first failure, by design. + +Teardown is the exact inverse — 17 `-D` in reverse order — so it begins with +rule 17, which a 9-rule partial install never created. That first delete fails, +the applier aborts, and **none of the 9 real rules come out**. + +### Why this is a real defect and not a script artefact +`apply-policy`'s exit-3 contract says *destroy the holder*. For the **namespace** +plan that is a complete remedy: the rules live in the holder's netns and die +with it. My **host-side** plan puts rules in the **root netns**, in `DOCKER-USER` +and `INPUT`. Destroying the holder removes none of them. The host path reuses an +applier whose failure contract assumes namespace-scoped rules. + +This defeats the intent documented at the adoption site — *"a plan that failed +part-way has already installed rules, and those rules must come out whichever +way this returns."* `HostRules` is adopted correctly; the teardown it runs is +what cannot do the job. + +Consequence if shipped: any partial host-rule install strands rules keyed to a +job address in a shared chain, and gate 5h showed those addresses get recycled. +Gate 5h passed only because its install was **complete**, so its teardown +matched rule-for-rule. + +**Not fixed in this commit.** The failing gate and its evidence land first. + +## Gate 5i, after the fix — PASS + +`evidence/gate5i-fail-closed-PASS-after-fix-20260910T0347Z.txt`. The FAIL run +is kept alongside it, not overwritten. + +**The fix** (`sandbox_netns.rs`): `HostRules` now carries `complete`, set only +after the applier's count cross-check passes — never at construction. Teardown +branches on it: + +* **complete** → the one-shot inverse plan, exactly as before. Every rule is + present, so the inverse matches rule-for-rule and one invocation is correct + and cheapest. +* **not complete** → **one applier invocation per rule**. A failure then means + only "that rule was not there", which on this path is expected rather than an + error, so a rule that was never created can no longer strand the ones that were. + +Adoption still happens **before** the result is examined; that was always right. +What changed is that the teardown it runs can now cope with the state adoption +exists to clean up. + +Held by two unit tests: `adopted_host_rules_are_not_complete_until_the_count_check_passes` +(the default must be the rule-by-rule path; the fast path is earned) and +`the_per_rule_teardown_renders_one_valid_delete_per_rule` (each single-rule plan +is one line, a delete, still keyed to this job's `/32` — the applier refuses an +empty plan with exit 4 and a non-iptables binary with exit 5). + +**Both strategies now run back to back on the same partial install**, because +the difference between them *is* the fix: + +| leg | result | +|---|---| +| C — truncated plan | applier exit `0`, reported `9`, kernel held **9 of 17** | +| C — runsc → live host with 9 of 17 rules | **REACHED** — a partial policy is no policy | +| D1 — one-shot inverse (pre-fix) | **9 of 9 remain**, applier aborts on the first absent rule | +| D2 — per-rule (post-fix) | **removed 9, 0 remain** | +| chain depth | **2 → 2** | + +**GATE 5i: PASS, 0 failing checks.** + +D1 is kept deliberately as a live regression witness rather than deleted: if +the applier's abort-on-first-failure behaviour ever changes, this gate will say +so instead of quietly preserving a workaround nobody needs any more. + +### What this leg does and does not prove +The Rust branch in `HostRules::drop` is covered by the two unit tests above. +What the shell gate proves is narrower and worth stating plainly: that the +per-rule **strategy** clears real rules from real chains where the one-shot +inverse plan cannot. The gate does not drive `establish()` end to end. + +## Gate 5j — IPv6: measured, and the code's stated reason does not hold here + +Gate 5 left IPv6 as `NO-V6`/UNPROVEN. Maxie asked for "applicable IPv6". +Evidence: `evidence/gate5j-ipv6-finding-20260910T0347Z.txt`. + +### What is true by construction +`HostPolicy::argv` uses `Family::V4.binary()` unconditionally, so **no +host-side IPv6 rule is rendered at all**. The netns plan still carries +`DENIED_DESTINATIONS_V6`. This is deliberate and documented at +`sandbox_net.rs`. + +### What was measured (aarch64, docker 29.1.3, gvisor-repro) +* `/etc/docker/daemon.json` declares only the `runsc` runtime — **no IPv6**. +* `docker info` reports no ipv6 key. +* Inside a job-shaped holder on a per-job network: `ip -6 addr` shows only + `::1/128 scope host`. **Zero global IPv6 addresses.** + +So on this host a job has no IPv6 egress path, and host-side v6 containment is +not exercised. **IPv6 is not-applicable here** — and that is a statement about +this host, not a safety property of the branch. + +### The part that contradicts the code comment +The doc comment justifies omitting v6 rules like this: *"`ip6tables` has a +`DOCKER-USER` chain only when the daemon has IPv6 enabled, and a missing chain +is an install failure that would fail every job launch on a v4-only host."* + +Measured on this v4-only host: + +``` +$ sudo ip6tables -S DOCKER-USER +-N DOCKER-USER +``` + +**The chain exists**, with IPv6 disabled on the daemon. The stated premise does +not hold here, so it is not the reason v6 rules are safe to omit. The honest +reason is the one gate 5 already gave: host-side v6 containment is +**UNMEASURED**. The comment should be corrected rather than relied on, and I +have not rewritten it in this branch because changing v6 behaviour is outside +the minimum repair maxie scoped — it is named here and in the PR instead. + +### The hole, stated plainly +A host **with IPv6 enabled** is **not covered** by this branch. If jobs there +receive global v6 addresses, they have an egress path with no host-side +containment in front of it, and the per-job network does not help for +destinations reached by routing. That is an open hole in an unshipped +configuration — not a proof of safety, and not something this branch fixes. + +## Baseline pinning (maxie's ruling, 9 Sep 2026) + +Maxie: *"Pin baseline source evidence to a full immutable hash, not moving +`origin/main`."* Every baseline citation in this RUNLOG and in the gate5g +script now names the full commit + + b45f8651dc9cab5b71c962eadd7b84840f579106 + +and no longer identifies the baseline by a branch ref, which can move under a +reader and would make the reproduction unverifiable later. + +The evidence FILES under `evidence/` still print the abbreviated `b45f865`, +and that is deliberate: they are verbatim records of runs that printed it. +Editing a recorded run's output to look tidier would falsify the record. The +abbreviation resolves to the full commit above, which a reviewer can confirm +with `git rev-parse b45f865`. + +## Gate 5k — an unsupported daemon default runtime fails CLOSED + +Maxie: *"Missing `--runtime` means daemon default, not guaranteed runc; test +unsupported defaults fail closed."* The correction lands on the product. +`holder_argv()` deliberately passes **no `--runtime`**, and a test locks it +there for a sound reason — a runsc holder's namespace is unusable, a job +joining it sees `lo` only (gate2a). So the design **assumes the daemon default +is runc**, and nothing in the product verifies that assumption. + +Evidence: `evidence/gate5k-v3-PASS-daemon-default-20260910T0345Z.txt`. + +| daemon default | listener | holder runtime | job | +|---|---|---|---| +| `runc` (supported) | SERVING | runc | `HEALTHY 172.18.0.3` → **REACHED** | +| `runsc` (unsupported) | SERVING | runsc | `SICK no-address` → **ENETUNREACH** | + +`failing_checks=0`: both legs had a *serving* listener, so leg 2's denial is a +measurement and not a silence. Leg 1 is the positive control — it proves the +harness can observe reachability at all. `/etc/docker/daemon.json` was restored +and the default verified back to `runc` by the EXIT trap. + +**Verdict: fails CLOSED.** With an unsupported default the job has no address +and no route, so there is no egress path to contain. That is an availability +failure, not a containment bypass — the product breaks loudly rather than +running jobs uncontained. + +**The gap that remains, named not fixed:** nothing detects this at startup. A +seat whose daemon default is not runc will fail every job with no explanatory +signal. The honest fix is a doctor/readiness row asserting +`docker info --format '{{.DefaultRuntime}}' == runc`, which is a product change +outside the minimum containment repair maxie scoped, so it is recorded here for +the follow-up rather than smuggled into this branch. + +### Three attempts, two failures kept + +v1 (`gate5k-CONFOUNDED-harness-defect-*.txt`) started the listener with +`docker exec` into a runsc holder, which runsc refuses; the target never served +in either leg, including the control. v2 +(`gate5k-v2-daemon-default-runtime-*.txt`) found the root cause: the sandbox +image has **no `nc` and no `wget`**, so every probe was doomed before it ran. +Both are committed. They are also the reason the earlier gates were re-checked: +gate5f/5g/5h/5i never use those tools — they probe with `--entrypoint node` and +inline JS — so their SERVING and REACHED readings stand. diff --git a/docs/gvisor-dns-delivery/evidence/gate1-runsc-vs-runc-20260910T0046Z.txt b/docs/gvisor-dns-delivery/evidence/gate1-runsc-vs-runc-20260910T0046Z.txt new file mode 100644 index 00000000..f502d6c6 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate1-runsc-vs-runc-20260910T0046Z.txt @@ -0,0 +1,26 @@ +=== gate1: environment === +utc=2026-09-10T00:46:01Z +kernel=6.8.0-134-generic arch=aarch64 +os=Ubuntu 24.04.4 LTS +docker=29.1.3 +runsc=runsc version release-20260817.0 +image=ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8 +image_digest=ghcr.io/makeprisms/maxplayer-sandbox@sha256:1c50e46a35dfe91fcdbbba11876bff312a95567bda98d6dcb7f675c884777412 +image_arch=arm64/linux +network=maxplayer-dns-repro subnet=172.18.0.0/16 + +=== gate1: dns lookup under --runtime runsc === +resolv.conf: # Generated by Docker Engine.|# This file can be edited; Docker Engine will not make further changes once it|# has been modified.||nameserver 127.0.0.11|search lan|options edns0 trust-ad ndots:0||# Based on host file: '/etc/resolv.conf' (internal resolver)|# ExtServers: [host(127.0.0.53)]|# Overrides: []|# Option ndots from: internal +lookup: ERR EAI_AGAIN +exit=1 + +=== gate1: dns lookup under --runtime runc === +resolv.conf: # Generated by Docker Engine.|# This file can be edited; Docker Engine will not make further changes once it|# has been modified.||nameserver 127.0.0.11|search lan|options edns0 trust-ad ndots:0||# Based on host file: '/etc/resolv.conf' (internal resolver)|# ExtServers: [host(127.0.0.53)]|# Overrides: []|# Option ndots from: internal +lookup: OK 34.225.223.145 +exit=0 + +=== gate1: raw udp/53 to the embedded resolver under runsc === +udp53: TIMEOUT (no answer from 127.0.0.11) +exit=1 + +=== gate1: done === diff --git a/docs/gvisor-dns-delivery/evidence/gate2-plan-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate2-plan-rendered.txt new file mode 100644 index 00000000..aa16b5b1 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate2-plan-rendered.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.7.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate2a-runsc-holder-FAIL-20260910T0145Z.txt b/docs/gvisor-dns-delivery/evidence/gate2a-runsc-holder-FAIL-20260910T0145Z.txt new file mode 100644 index 00000000..d53a1af4 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate2a-runsc-holder-FAIL-20260910T0145Z.txt @@ -0,0 +1,66 @@ +=== gate2: environment === +utc=2026-09-10T01:45:37Z +kernel=6.8.0-134-generic arch=aarch64 +os=Ubuntu 24.04.4 LTS +docker=29.1.3 +runsc=runsc version release-20260817.0 +image=ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8 +image_digest=ghcr.io/makeprisms/maxplayer-sandbox@sha256:1c50e46a35dfe91fcdbbba11876bff312a95567bda98d6dcb7f675c884777412 +netfilter_digest=ghcr.io/makeprisms/maxplayer-netfilter@sha256:913985269261c7169b4f7b14a5db69843b9a3421aa483145a26d6ac38db3da3a +resolver=1.1.1.1 +resolv_conf=/home/forge.guest/gate2-resolv.conf sha256=24c2b7ed40eef08c2661229c666dce74d955b8e365b7ec598456cbf042bece91 +plan=/home/forge.guest/gate2-plan.txt rules=26 sha256=eb0fc310b18db6104445fd2f08553d1f0d068f3747eb58d97c434f7670360f43 +network=maxplayer-dns-gate2 subnet=172.31.7.0/24 gateway=172.31.7.1 + +=== gate2/fresh: establish the shared job namespace === +holder=gate2-holder started=0 +--- sidecar applies the rendered plan --- +iptables: Failed to initialize nft: Protocol not supported +apply-policy: rule 1 failed: iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +apply-policy: namespace is PARTIALLY configured — destroy the holder, do not retry +sidecar_exit=3 +--- readback (iptables -S) --- +iptables: Failed to initialize nft: Protocol not supported +readback_exit=1 + +=== gate2/fresh: job in the shared namespace — dns + verified tls === +resolv.conf: nameserver 1.1.1.1|options timeout:2 attempts:2 +lookup: ERR EAI_AGAIN +job_exit=1 +--- containment still holds: metadata address --- +metadata: denied (ENETUNREACH) +metadata_exit=0 + +=== gate2: destroy the namespace and rebuild it === + +=== gate2/recreated: establish the shared job namespace === +holder=gate2-holder started=0 +--- sidecar applies the rendered plan --- +iptables: Failed to initialize nft: Protocol not supported +apply-policy: rule 1 failed: iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +apply-policy: namespace is PARTIALLY configured — destroy the holder, do not retry +sidecar_exit=3 +--- readback (iptables -S) --- +iptables: Failed to initialize nft: Protocol not supported +readback_exit=1 + +=== gate2/recreated: job in the shared namespace — dns + verified tls === +resolv.conf: nameserver 1.1.1.1|options timeout:2 attempts:2 +lookup: ERR EAI_AGAIN +job_exit=1 +--- containment still holds: metadata address --- +metadata: denied (ENETUNREACH) +metadata_exit=0 + +=== gate2: verdict === +GATE2: FAIL + +=== gate2 diagnostic: interfaces a container sees, by runtime and network mode === +(probe: node os.networkInterfaces(); the sandbox image ships no iproute2) +A runsc job joining a RUNSC holder netns: lo +B runsc job joining a RUNC holder netns: lo,eth0 +C runsc job on the bridge DIRECTLY: lo,eth0 +D runc job joining the RUNC holder: lo +=== iptables backend available to the sidecar, by runtime === +runc sidecar on runc holder: -P OUTPUT ACCEPT +runsc sidecar on runsc holder: iptables: Failed to initialize nft: Protocol not supported diff --git a/docs/gvisor-dns-delivery/evidence/gate2b-runc-plane-runsc-job-PASS-20260910T0210Z.txt b/docs/gvisor-dns-delivery/evidence/gate2b-runc-plane-runsc-job-PASS-20260910T0210Z.txt new file mode 100644 index 00000000..161b5291 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate2b-runc-plane-runsc-job-PASS-20260910T0210Z.txt @@ -0,0 +1,102 @@ +=== gate2: environment === +utc=2026-09-10T01:48:54Z +kernel=6.8.0-134-generic arch=aarch64 +os=Ubuntu 24.04.4 LTS +docker=29.1.3 +runsc=runsc version release-20260817.0 +holder_runtime=runc job_runtime=runsc +image=ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8 +image_digest=ghcr.io/makeprisms/maxplayer-sandbox@sha256:1c50e46a35dfe91fcdbbba11876bff312a95567bda98d6dcb7f675c884777412 +netfilter_digest=ghcr.io/makeprisms/maxplayer-netfilter@sha256:913985269261c7169b4f7b14a5db69843b9a3421aa483145a26d6ac38db3da3a +resolver=1.1.1.1 +/Users/forge/forge/v2/wt/w-gvisor-dns-delivery-r2/docs/gvisor-dns-delivery/scripts/gate2-namespace-dns-tls.sh: line 55: /home/forge.guest/gate2-resolv.conf: Permission denied +resolv_conf=/home/forge.guest/gate2-resolv.conf sha256=24c2b7ed40eef08c2661229c666dce74d955b8e365b7ec598456cbf042bece91 +plan=/home/forge.guest/gate2-plan.txt rules=26 sha256=eb0fc310b18db6104445fd2f08553d1f0d068f3747eb58d97c434f7670360f43 +network=maxplayer-dns-gate2 subnet=172.31.7.0/24 gateway=172.31.7.1 + +=== gate2/fresh: establish the shared job namespace === +holder=gate2-holder started=0 +--- sidecar applies the rendered plan --- +26 +sidecar_exit=0 +--- readback (iptables -S) --- +-P OUTPUT ACCEPT +-A OUTPUT -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-conn:" +-A OUTPUT -p udp -m udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-dns:" +-A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny-metadata:" +-A OUTPUT -d 169.254.169.254/32 -j DROP +-A OUTPUT -d 172.31.7.1/32 -p tcp -m tcp --dport 49200:49299 -j ACCEPT +-A OUTPUT -d 1.1.1.1/32 -p udp -m udp --dport 53 -j ACCEPT +-A OUTPUT -d 1.1.1.1/32 -p tcp -m tcp --dport 53 -j ACCEPT +-A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 10.0.0.0/8 -j DROP +-A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 172.16.0.0/12 -j DROP +-A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 192.168.0.0/16 -j DROP +-A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 169.254.0.0/16 -j DROP +-A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 100.64.0.0/10 -j DROP +-A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 198.18.0.0/15 -j DROP +-A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 224.0.0.0/4 -j DROP +-A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 240.0.0.0/4 -j DROP +readback_exit=0 + +=== gate2/fresh: job in the shared namespace — dns + verified tls === +resolv.conf: nameserver 1.1.1.1|options timeout:2 attempts:2 +lookup: OK 34.225.223.145 +tls: 200 cert-verified subject=relay.maxplayer.ai issuer=YE1 authorized=true +job_exit=0 +--- containment still holds: metadata address --- +metadata: denied (ENETUNREACH) +metadata_exit=0 + +=== gate2: destroy the namespace and rebuild it === + +=== gate2/recreated: establish the shared job namespace === +holder=gate2-holder started=0 +--- sidecar applies the rendered plan --- +26 +sidecar_exit=0 +--- readback (iptables -S) --- +-P OUTPUT ACCEPT +-A OUTPUT -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-conn:" +-A OUTPUT -p udp -m udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-dns:" +-A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny-metadata:" +-A OUTPUT -d 169.254.169.254/32 -j DROP +-A OUTPUT -d 172.31.7.1/32 -p tcp -m tcp --dport 49200:49299 -j ACCEPT +-A OUTPUT -d 1.1.1.1/32 -p udp -m udp --dport 53 -j ACCEPT +-A OUTPUT -d 1.1.1.1/32 -p tcp -m tcp --dport 53 -j ACCEPT +-A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 10.0.0.0/8 -j DROP +-A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 172.16.0.0/12 -j DROP +-A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 192.168.0.0/16 -j DROP +-A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 169.254.0.0/16 -j DROP +-A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 100.64.0.0/10 -j DROP +-A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 198.18.0.0/15 -j DROP +-A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 224.0.0.0/4 -j DROP +-A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +-A OUTPUT -d 240.0.0.0/4 -j DROP +readback_exit=0 + +=== gate2/recreated: job in the shared namespace — dns + verified tls === +resolv.conf: nameserver 1.1.1.1|options timeout:2 attempts:2 +lookup: OK 34.225.223.145 +tls: 200 cert-verified subject=relay.maxplayer.ai issuer=YE1 authorized=true +job_exit=0 +--- containment still holds: metadata address --- +metadata: denied (ENETUNREACH) +metadata_exit=0 + +=== gate2: verdict === +GATE2: PASS (fresh and recreated namespaces both resolved and completed verified TLS) diff --git a/docs/gvisor-dns-delivery/evidence/gate4-container-git-delivery-PASS-20260910T0330Z.txt b/docs/gvisor-dns-delivery/evidence/gate4-container-git-delivery-PASS-20260910T0330Z.txt new file mode 100644 index 00000000..8c045cad --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate4-container-git-delivery-PASS-20260910T0330Z.txt @@ -0,0 +1,42 @@ +=== gate4: environment === +utc=2026-09-10T02:21:28Z +kernel=6.8.0-134-generic arch=aarch64 +os=Ubuntu 24.04.4 LTS +docker=29.1.3 +runsc=runsc version release-20260817.0 +holder_runtime=runc job_runtime=runsc +image_digest=ghcr.io/makeprisms/maxplayer-sandbox@sha256:1c50e46a35dfe91fcdbbba11876bff312a95567bda98d6dcb7f675c884777412 +container_git=git version 2.39.5 +public_repo=https://github.com/octocat/Hello-World.git +/Users/forge/forge/v2/wt/w-gvisor-dns-delivery-r2/docs/gvisor-dns-delivery/scripts/gate4-container-git-delivery.sh: line 65: /home/forge.guest/gate4-resolv.conf: Permission denied +plan=/home/forge.guest/gate4-plan.txt rules=26 sha256=70ab9b4e65dfe6a1a48911ea734e0fecf6ca84f9d0879a890d460e84830e3d78 +network=maxplayer-dns-gate4 subnet=172.31.8.0/24 gateway=172.31.8.1 + +=== gate4: control — the remote's real HEAD, read from the VM (outside any container) === +control_head=7fd1a60b01f91b314f59955a4e4d4e80d8edf11d + +=== gate4: disposable write remote === +write_remote=git://172.31.8.1:49250/answer.git (disposable, unauthenticated, torn down on exit) +remote_refs_before=0 + +=== gate4: establish the shared job namespace === +holder_started=0 +26 +sidecar_exit=0 + +=== gate4: job container (runsc, non-root, cap-drop ALL) === +--- read leg: clone over https from inside the sandbox --- +delivered_head=7fd1a60b01f91b314f59955a4e4d4e80d8edf11d +--- write leg: commit here, push to the remote outside this container --- +answer_commit=a4a883b087273c86480ea1269f7e50ff965296b2 +push_exit=0 +job_exit=0 + +=== gate4: verdict === +control_head=7fd1a60b01f91b314f59955a4e4d4e80d8edf11d +delivered_head=7fd1a60b01f91b314f59955a4e4d4e80d8edf11d +READ: PASS — the sandbox delivered the remote's real HEAD +answer_commit=a4a883b087273c86480ea1269f7e50ff965296b2 +remote_hash=a4a883b087273c86480ea1269f7e50ff965296b2 +WRITE: PASS — the commit made inside the sandbox reached the remote, hash matches +GATE4: PASS diff --git a/docs/gvisor-dns-delivery/evidence/gate4-plan-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate4-plan-rendered.txt new file mode 100644 index 00000000..1c9eb3de --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate4-plan-rendered.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.8.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-denial-and-concurrent-success-20260910T0245Z.txt b/docs/gvisor-dns-delivery/evidence/gate5-denial-and-concurrent-success-20260910T0245Z.txt new file mode 100644 index 00000000..b62d905d --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-denial-and-concurrent-success-20260910T0245Z.txt @@ -0,0 +1,46 @@ +=== gate5: environment === +utc=2026-09-10T02:29:10Z +kernel=6.8.0-134-generic arch=aarch64 +os=Ubuntu 24.04.4 LTS +docker=29.1.3 +runsc=runsc version release-20260817.0 +holder_runtime=runc job_runtime=runsc +image_digest=ghcr.io/makeprisms/maxplayer-sandbox@sha256:1c50e46a35dfe91fcdbbba11876bff312a95567bda98d6dcb7f675c884777412 +/Users/forge/forge/v2/wt/w-gvisor-dns-delivery-r2/docs/gvisor-dns-delivery/scripts/gate5-denial-and-concurrent-success.sh: line 58: /home/forge.guest/gate5-resolv.conf: Permission denied +plan=/home/forge.guest/gate5-plan.txt rules=26 sha256=2d312d2e5c59f974a4aa857fecb943b03d081e5c9ef2ff2de23dbb539e66d59d +network=maxplayer-dns-gate5 subnet=172.31.12.0/24 gateway=172.31.12.1 +host_loopback_listener=127.0.0.1:49251 + +=== gate5: two job namespaces, both contained === +established=gate5-holder-a sidecar_exit=0 +established=gate5-holder-b sidecar_exit=0 +job_b_address=172.31.12.3 +neighbour_listening=true + +=== gate5: job A — denial legs and public legs, one namespace, one run === +DENY-PASS ipv6 link-local (fe80::1:80) -> ENETUNREACH +DENY-PASS ipv6 unique-local (fc00::1:80) -> ENETUNREACH +DENY-PASS host loopback via 127.0.0.1 (127.0.0.1:49251) -> ECONNREFUSED +DENY-PASS ipv6 loopback (::1:80) -> ECONNREFUSED +DENY-PASS metadata (direct ip) (169.254.169.254:80) -> ECONNREFUSED +DENY-PASS link-local (direct ip) (169.254.1.1:80) -> ECONNREFUSED +DENY-FAIL another job namespace (172.31.12.3:8080) -> REACHED +DENY-PASS metadata BY NAME (169.254.169.254.nip.io:80) -> ECONNREFUSED +PUBLIC-PASS dns relay.maxplayer.ai -> 34.225.223.145 +PUBLIC-PASS tls 200 verified relay.maxplayer.ai +DENY-PASS rfc1918 10/8 (10.0.0.1:80) -> timeout +DENY-PASS rfc1918 192.168/16 (192.168.1.1:80) -> timeout +DENY-PASS rfc1918 BY NAME (10.0.0.1.nip.io:80) -> timeout +denial_failures=1 +PUBLIC-FAIL tls timeout + +=== gate5: job A — git clone over https, same namespace === +fatal: unable to access 'https://github.com/octocat/Hello-World.git/': Could not resolve host: github.com +git_exit=128 + +=== gate5: verdict === +denial_failures=1 +denials_proven=10 +DENIAL: FAIL — something a contained job must not reach was reachable +PUBLIC: FAIL — the sandbox did not demonstrate the public route it is supposed to keep (dns_ok=1 tls_ok=1 git_exit=128) +GATE5: FAIL diff --git a/docs/gvisor-dns-delivery/evidence/gate5-denial-and-concurrent-success-PASS-20260910T0311Z.txt b/docs/gvisor-dns-delivery/evidence/gate5-denial-and-concurrent-success-PASS-20260910T0311Z.txt new file mode 100644 index 00000000..18abb1cf --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-denial-and-concurrent-success-PASS-20260910T0311Z.txt @@ -0,0 +1,51 @@ +=== gate5: environment === +utc=2026-09-10T03:11:45Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 runsc=runsc version release-20260817.0 +docker default-runtime=runc · br_netfilter=absent +chain depth before anything: DOCKER-USER=1 INPUT=1 +live listeners: host 192.168.5.15:49254 · neighbour job 172.31.34.10:8080 · name denied-lan.maxplayer.test -> 192.168.5.15 + +=== gate5: 1. denial, BEFORE the host policy (so each leg has a baseline) === + runsc -> host 192.168.5.15:49254 (live): REACHED + runsc -> denied-lan.maxplayer.test:49254 (live, by name): REACHED + runsc -> neighbour job 172.31.34.10:8080 (live): timeout + runsc -> 169.254.169.254:80 (nothing listens): ECONNREFUSED + +=== gate5: 2. the product's host policy, installed for the probe namespace === + host policy install for 172.31.30.10: applier=17/17, host kernel now carries 17 + +=== gate5: 3. denial, AFTER — every one of these MUST be denied === + runsc -> host 192.168.5.15:49254 (live): timeout (denied) + runsc -> denied-lan.maxplayer.test:49254 (live, by name): timeout (denied) + runsc -> neighbour job 172.31.34.10:8080 (live): timeout (denied) + runsc -> 169.254.169.254:80 (nothing listens): timeout (denied) + ^ read the metadata leg ONLY as the difference against section 1; with no listener, + an identical result in both sections is NO EVIDENCE either way. + +=== gate5: 4. IPv6, measured rather than assumed === + namespace v6: NO-V6 + runsc v6: NO-V6 + The host-side plan renders NO ip6tables rules (DOCKER-USER may not exist there). + Where the job namespace has no global IPv6 there is nothing to deny; where it has, + host-side v6 denial for a runsc job is UNPROVEN and must not be claimed. + +=== gate5: 5. concurrent delivery, three jobs at once, all policies installed === + host policy install for 172.31.31.10: applier=17/17, host kernel now carries 17 + host policy install for 172.31.32.10: applier=17/17, host kernel now carries 17 + host policy install for 172.31.33.10: applier=17/17, host kernel now carries 17 + c1: OK dns=34.225.223.145 tls=200 verified=true OK git 7fd1a60b01f91b314f59955a4e4d4e80d8edf11d + c3: OK dns=34.225.223.145 tls=200 verified=true OK git 7fd1a60b01f91b314f59955a4e4d4e80d8edf11d + c2: OK dns=34.225.223.145 tls=200 verified=true OK git 7fd1a60b01f91b314f59955a4e4d4e80d8edf11d + host policy teardown for 172.31.31.10: applier=17/17, host kernel now carries 0 + host policy teardown for 172.31.32.10: applier=17/17, host kernel now carries 0 + host policy teardown for 172.31.33.10: applier=17/17, host kernel now carries 0 + +=== gate5: 6. teardown leaves the shared chains as it found them === + host policy teardown for 172.31.30.10: applier=17/17, host kernel now carries 0 + DOCKER-USER=1 (was 1) · INPUT=1 (was 1) + CLEAN + +=== gate5: verdict === +denial legs failing: 0 +A concurrent line counts as delivery only if it reads OK dns=… tls=200 verified=true AND OK git . +GATE5-DENIAL: PASS diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plan-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate5-plan-rendered.txt new file mode 100644 index 00000000..fa81a019 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plan-rendered.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.12.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c1-teardown.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c1-teardown.txt new file mode 100644 index 00000000..c5c077bb --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c1-teardown.txt @@ -0,0 +1,17 @@ +iptables -D INPUT -s 172.31.31.10/32 -d 240.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.31.10/32 -d 224.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.31.10/32 -d 198.18.0.0/15 -j DROP +iptables -D INPUT -s 172.31.31.10/32 -d 100.64.0.0/10 -j DROP +iptables -D INPUT -s 172.31.31.10/32 -d 169.254.0.0/16 -j DROP +iptables -D INPUT -s 172.31.31.10/32 -d 192.168.0.0/16 -j DROP +iptables -D INPUT -s 172.31.31.10/32 -d 172.16.0.0/12 -j DROP +iptables -D INPUT -s 172.31.31.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 240.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 224.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 198.18.0.0/15 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 100.64.0.0/10 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 169.254.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 192.168.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 172.16.0.0/12 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.31.10/32 -d 169.254.169.254/32 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c1.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c1.txt new file mode 100644 index 00000000..684f9222 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c1.txt @@ -0,0 +1,17 @@ +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 169.254.169.254/32 -j DROP +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 10.0.0.0/8 -j DROP +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 172.16.0.0/12 -j DROP +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 192.168.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 169.254.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 100.64.0.0/10 -j DROP +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 198.18.0.0/15 -j DROP +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 224.0.0.0/4 -j DROP +iptables -I DOCKER-USER -s 172.31.31.10/32 -d 240.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.31.10/32 -d 10.0.0.0/8 -j DROP +iptables -I INPUT -s 172.31.31.10/32 -d 172.16.0.0/12 -j DROP +iptables -I INPUT -s 172.31.31.10/32 -d 192.168.0.0/16 -j DROP +iptables -I INPUT -s 172.31.31.10/32 -d 169.254.0.0/16 -j DROP +iptables -I INPUT -s 172.31.31.10/32 -d 100.64.0.0/10 -j DROP +iptables -I INPUT -s 172.31.31.10/32 -d 198.18.0.0/15 -j DROP +iptables -I INPUT -s 172.31.31.10/32 -d 224.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.31.10/32 -d 240.0.0.0/4 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c2-teardown.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c2-teardown.txt new file mode 100644 index 00000000..0044ef78 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c2-teardown.txt @@ -0,0 +1,17 @@ +iptables -D INPUT -s 172.31.32.10/32 -d 240.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.32.10/32 -d 224.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.32.10/32 -d 198.18.0.0/15 -j DROP +iptables -D INPUT -s 172.31.32.10/32 -d 100.64.0.0/10 -j DROP +iptables -D INPUT -s 172.31.32.10/32 -d 169.254.0.0/16 -j DROP +iptables -D INPUT -s 172.31.32.10/32 -d 192.168.0.0/16 -j DROP +iptables -D INPUT -s 172.31.32.10/32 -d 172.16.0.0/12 -j DROP +iptables -D INPUT -s 172.31.32.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 240.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 224.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 198.18.0.0/15 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 100.64.0.0/10 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 169.254.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 192.168.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 172.16.0.0/12 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.32.10/32 -d 169.254.169.254/32 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c2.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c2.txt new file mode 100644 index 00000000..d5d279f2 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c2.txt @@ -0,0 +1,17 @@ +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 169.254.169.254/32 -j DROP +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 10.0.0.0/8 -j DROP +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 172.16.0.0/12 -j DROP +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 192.168.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 169.254.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 100.64.0.0/10 -j DROP +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 198.18.0.0/15 -j DROP +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 224.0.0.0/4 -j DROP +iptables -I DOCKER-USER -s 172.31.32.10/32 -d 240.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.32.10/32 -d 10.0.0.0/8 -j DROP +iptables -I INPUT -s 172.31.32.10/32 -d 172.16.0.0/12 -j DROP +iptables -I INPUT -s 172.31.32.10/32 -d 192.168.0.0/16 -j DROP +iptables -I INPUT -s 172.31.32.10/32 -d 169.254.0.0/16 -j DROP +iptables -I INPUT -s 172.31.32.10/32 -d 100.64.0.0/10 -j DROP +iptables -I INPUT -s 172.31.32.10/32 -d 198.18.0.0/15 -j DROP +iptables -I INPUT -s 172.31.32.10/32 -d 224.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.32.10/32 -d 240.0.0.0/4 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c3-teardown.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c3-teardown.txt new file mode 100644 index 00000000..4989b3f8 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c3-teardown.txt @@ -0,0 +1,17 @@ +iptables -D INPUT -s 172.31.33.10/32 -d 240.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.33.10/32 -d 224.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.33.10/32 -d 198.18.0.0/15 -j DROP +iptables -D INPUT -s 172.31.33.10/32 -d 100.64.0.0/10 -j DROP +iptables -D INPUT -s 172.31.33.10/32 -d 169.254.0.0/16 -j DROP +iptables -D INPUT -s 172.31.33.10/32 -d 192.168.0.0/16 -j DROP +iptables -D INPUT -s 172.31.33.10/32 -d 172.16.0.0/12 -j DROP +iptables -D INPUT -s 172.31.33.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 240.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 224.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 198.18.0.0/15 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 100.64.0.0/10 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 169.254.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 192.168.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 172.16.0.0/12 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.33.10/32 -d 169.254.169.254/32 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c3.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c3.txt new file mode 100644 index 00000000..2672b181 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-c3.txt @@ -0,0 +1,17 @@ +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 169.254.169.254/32 -j DROP +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 10.0.0.0/8 -j DROP +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 172.16.0.0/12 -j DROP +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 192.168.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 169.254.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 100.64.0.0/10 -j DROP +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 198.18.0.0/15 -j DROP +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 224.0.0.0/4 -j DROP +iptables -I DOCKER-USER -s 172.31.33.10/32 -d 240.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.33.10/32 -d 10.0.0.0/8 -j DROP +iptables -I INPUT -s 172.31.33.10/32 -d 172.16.0.0/12 -j DROP +iptables -I INPUT -s 172.31.33.10/32 -d 192.168.0.0/16 -j DROP +iptables -I INPUT -s 172.31.33.10/32 -d 169.254.0.0/16 -j DROP +iptables -I INPUT -s 172.31.33.10/32 -d 100.64.0.0/10 -j DROP +iptables -I INPUT -s 172.31.33.10/32 -d 198.18.0.0/15 -j DROP +iptables -I INPUT -s 172.31.33.10/32 -d 224.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.33.10/32 -d 240.0.0.0/4 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-d-teardown.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-d-teardown.txt new file mode 100644 index 00000000..f33c795f --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-d-teardown.txt @@ -0,0 +1,17 @@ +iptables -D INPUT -s 172.31.30.10/32 -d 240.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.30.10/32 -d 224.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.30.10/32 -d 198.18.0.0/15 -j DROP +iptables -D INPUT -s 172.31.30.10/32 -d 100.64.0.0/10 -j DROP +iptables -D INPUT -s 172.31.30.10/32 -d 169.254.0.0/16 -j DROP +iptables -D INPUT -s 172.31.30.10/32 -d 192.168.0.0/16 -j DROP +iptables -D INPUT -s 172.31.30.10/32 -d 172.16.0.0/12 -j DROP +iptables -D INPUT -s 172.31.30.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 240.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 224.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 198.18.0.0/15 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 100.64.0.0/10 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 169.254.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 192.168.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 172.16.0.0/12 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.30.10/32 -d 169.254.169.254/32 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-d.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-d.txt new file mode 100644 index 00000000..c1aeca8f --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-d.txt @@ -0,0 +1,17 @@ +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 169.254.169.254/32 -j DROP +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 10.0.0.0/8 -j DROP +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 172.16.0.0/12 -j DROP +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 192.168.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 169.254.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 100.64.0.0/10 -j DROP +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 198.18.0.0/15 -j DROP +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 224.0.0.0/4 -j DROP +iptables -I DOCKER-USER -s 172.31.30.10/32 -d 240.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.30.10/32 -d 10.0.0.0/8 -j DROP +iptables -I INPUT -s 172.31.30.10/32 -d 172.16.0.0/12 -j DROP +iptables -I INPUT -s 172.31.30.10/32 -d 192.168.0.0/16 -j DROP +iptables -I INPUT -s 172.31.30.10/32 -d 169.254.0.0/16 -j DROP +iptables -I INPUT -s 172.31.30.10/32 -d 100.64.0.0/10 -j DROP +iptables -I INPUT -s 172.31.30.10/32 -d 198.18.0.0/15 -j DROP +iptables -I INPUT -s 172.31.30.10/32 -d 224.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.30.10/32 -d 240.0.0.0/4 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-n-teardown.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-n-teardown.txt new file mode 100644 index 00000000..a4f1ef81 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-n-teardown.txt @@ -0,0 +1,17 @@ +iptables -D INPUT -s 172.31.34.10/32 -d 240.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.34.10/32 -d 224.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.34.10/32 -d 198.18.0.0/15 -j DROP +iptables -D INPUT -s 172.31.34.10/32 -d 100.64.0.0/10 -j DROP +iptables -D INPUT -s 172.31.34.10/32 -d 169.254.0.0/16 -j DROP +iptables -D INPUT -s 172.31.34.10/32 -d 192.168.0.0/16 -j DROP +iptables -D INPUT -s 172.31.34.10/32 -d 172.16.0.0/12 -j DROP +iptables -D INPUT -s 172.31.34.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 240.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 224.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 198.18.0.0/15 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 100.64.0.0/10 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 169.254.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 192.168.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 172.16.0.0/12 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.34.10/32 -d 169.254.169.254/32 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/host-n.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-n.txt new file mode 100644 index 00000000..8f81a1b9 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/host-n.txt @@ -0,0 +1,17 @@ +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 169.254.169.254/32 -j DROP +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 10.0.0.0/8 -j DROP +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 172.16.0.0/12 -j DROP +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 192.168.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 169.254.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 100.64.0.0/10 -j DROP +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 198.18.0.0/15 -j DROP +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 224.0.0.0/4 -j DROP +iptables -I DOCKER-USER -s 172.31.34.10/32 -d 240.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.34.10/32 -d 10.0.0.0/8 -j DROP +iptables -I INPUT -s 172.31.34.10/32 -d 172.16.0.0/12 -j DROP +iptables -I INPUT -s 172.31.34.10/32 -d 192.168.0.0/16 -j DROP +iptables -I INPUT -s 172.31.34.10/32 -d 169.254.0.0/16 -j DROP +iptables -I INPUT -s 172.31.34.10/32 -d 100.64.0.0/10 -j DROP +iptables -I INPUT -s 172.31.34.10/32 -d 198.18.0.0/15 -j DROP +iptables -I INPUT -s 172.31.34.10/32 -d 224.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.34.10/32 -d 240.0.0.0/4 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c1.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c1.txt new file mode 100644 index 00000000..74158933 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c1.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.31.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c2.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c2.txt new file mode 100644 index 00000000..82827ad8 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c2.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.32.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c3.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c3.txt new file mode 100644 index 00000000..5995217f --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-c3.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.33.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-d.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-d.txt new file mode 100644 index 00000000..d7477903 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-d.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.30.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-n.txt b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-n.txt new file mode 100644 index 00000000..03ededd9 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5-plans/plan-n.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.34.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5b-plan-does-not-bind-runsc-FINDING-20260910T0250Z.txt b/docs/gvisor-dns-delivery/evidence/gate5b-plan-does-not-bind-runsc-FINDING-20260910T0250Z.txt new file mode 100644 index 00000000..7524ba48 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5b-plan-does-not-bind-runsc-FINDING-20260910T0250Z.txt @@ -0,0 +1,25 @@ +=== gate5b: environment === +utc=2026-09-10T02:33:28Z +kernel=6.8.0-134-generic arch=aarch64 +docker=29.1.3 runsc=runsc version release-20260817.0 +network=maxplayer-dns-gate5b subnet=172.31.13.0/24 plan_rules=26 +26 +plan_applied_rules_reported=0 +neighbour=172.31.13.3:8080 running=true +neighbour_is_inside_a_dropped_range=172.16.0.0/12 + +=== gate5b: the same probe, the same namespace, two runtimes === +job_runtime=runc RESULT=timeout +job_runtime=runsc RESULT=REACHED + +=== gate5b: the rule as installed, read back from the namespace === +11:-A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix "sbx-net-deny:" +12:-A OUTPUT -d 172.16.0.0/12 -j DROP + +=== gate5b: verdict === +runc_result=RESULT=timeout +runsc_result=RESULT=REACHED +FINDING: the plan binds a runc job and DOES NOT BIND a runsc job. +gVisor's netstack emits packets to the veth itself; the host kernel's OUTPUT chain +in that netns never sees them, so per-job egress policy is not enforced for the job +it is written for. Containment for gVisor jobs cannot live in the netns OUTPUT chain. diff --git a/docs/gvisor-dns-delivery/evidence/gate5b-plan-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate5b-plan-rendered.txt new file mode 100644 index 00000000..b443c287 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5b-plan-rendered.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.13.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5c-plan-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate5c-plan-rendered.txt new file mode 100644 index 00000000..cb5f616a --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5c-plan-rendered.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.17.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5c-where-does-containment-bind-20260910T0244Z.txt b/docs/gvisor-dns-delivery/evidence/gate5c-where-does-containment-bind-20260910T0244Z.txt new file mode 100644 index 00000000..351991af --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5c-where-does-containment-bind-20260910T0244Z.txt @@ -0,0 +1,28 @@ +=== gate5c: environment === +utc=2026-09-10T02:44:06Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 runsc=runsc version release-20260817.0 +br_netfilter=absent +rule: one gVisor container per namespace; health check before every evidential leg +/Users/forge/forge/v2/wt/w-gvisor-dns-delivery-r2/docs/gvisor-dns-delivery/scripts/gate5c-where-does-containment-bind.sh: line 68: /home/forge.guest/gate5c-resolv.conf: Permission denied +live neighbours: same_bridge=172.31.17.2:8080 other_bridge=172.31.18.2:8080 + +=== gate5c: baseline — today's behaviour, no host-side rule === + runc -> live neighbour, same bridge: timeout [ns=172.31.17.3 rule=none health=HEALTHY 172.31.17.3 dns=34.225.223.145] + runsc -> live neighbour, same bridge: REACHED [ns=172.31.17.3 rule=none health=HEALTHY 172.31.17.3 dns=34.225.223.145] + +=== gate5c: (b) per-job network — the neighbour is on a DIFFERENT bridge === + runsc -> live neighbour, other bridge: timeout [ns=172.31.17.3 rule=none health=HEALTHY 172.31.17.3 dns=34.225.223.145] + +=== gate5c: (a) DOCKER-USER keyed to the namespace's own address === + runc -> live neighbour, same bridge: timeout [ns=172.31.17.3 rule=-s 172.31.17.3/32 -d 172.16.0.0/12 -j DROP health=HEALTHY 172.31.17.3 dns=34.225.223.145] + runsc -> live neighbour, same bridge: REACHED [ns=172.31.17.3 rule=-s 172.31.17.3/32 -d 172.16.0.0/12 -j DROP health=HEALTHY 172.31.17.3 dns=34.225.223.145] + +=== gate5c: does (a) cost the public route the job must keep? === + health before the public leg: HEALTHY 172.31.17.3 dns=34.225.223.145 + PUBLIC-PASS dns=34.225.223.145 tls=200 verified=true + health before the git leg: HEALTHY 172.31.17.3 dns=34.225.223.145 + PUBLIC-PASS git 7fd1a60b01f91b314f59955a4e4d4e80d8edf11d + +=== gate5c: how to read this === +(a) binds runsc only if BOTH rule-legs are denied AND both public legs still pass. +(b) binds runsc only if the other-bridge leg is denied from a HEALTHY namespace. diff --git a/docs/gvisor-dns-delivery/evidence/gate5c-where-does-containment-bind-VOID-20260910T0310Z.txt b/docs/gvisor-dns-delivery/evidence/gate5c-where-does-containment-bind-VOID-20260910T0310Z.txt new file mode 100644 index 00000000..3549f480 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5c-where-does-containment-bind-VOID-20260910T0310Z.txt @@ -0,0 +1,27 @@ +=== gate5c: environment === +utc=2026-09-10T02:37:36Z +kernel=6.8.0-134-generic arch=aarch64 +docker=29.1.3 runsc=runsc version release-20260817.0 +br_netfilter=absent +net_a=maxplayer-dns-gate5c-a 172.31.14.0/24 | net_b=maxplayer-dns-gate5c-b 172.31.15.0/24 +job_a=172.31.14.2 | live_neighbour_same_network=172.31.14.3:8080 | live_neighbour_other_network=172.31.15.2:8080 + +=== gate5c: baseline — no host-side rule (this is today's behaviour) === +runc -> live neighbour, same bridge: timeout +runsc -> live neighbour, same bridge: REACHED + +=== gate5c: (b) per-job network — the neighbour is on a DIFFERENT bridge === +runsc -> live neighbour, other bridge: ENETUNREACH + +=== gate5c: (a) DOCKER-USER, keyed to the job namespace's source address === +installed: iptables -I DOCKER-USER -s 172.31.14.2/32 -d 172.16.0.0/12 -j DROP +runc -> live neighbour, same bridge: ENETUNREACH +runsc -> live neighbour, same bridge: ENETUNREACH + +=== gate5c: does that rule cost the public route the job must keep? === +PUBLIC-FAIL dns EAI_AGAIN +fatal: unable to access 'https://github.com/octocat/Hello-World.git/': Could not resolve host: github.com + +=== gate5c: read the finding off the table above === +(a) binds runsc if the DOCKER-USER runs show 'timeout' for BOTH runtimes +(b) binds runsc if the other-bridge probe is not REACHED diff --git a/docs/gvisor-dns-delivery/evidence/gate5d-namespace-is-single-use-for-gvisor-FINDING-20260910T0310Z.txt b/docs/gvisor-dns-delivery/evidence/gate5d-namespace-is-single-use-for-gvisor-FINDING-20260910T0310Z.txt new file mode 100644 index 00000000..ab3b4627 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5d-namespace-is-single-use-for-gvisor-FINDING-20260910T0310Z.txt @@ -0,0 +1,18 @@ +=== gate5d: environment === +utc=2026-09-10T02:40:40Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 runsc=runsc version release-20260817.0 +namespace established, plan applied (26 rules) + +=== gate5d: the namespace before any gVisor container has touched it === + [before via runc] interfaces: lo=127.0.0.1 eth0=172.31.16.2 dns: 34.225.223.145 + +=== gate5d: one runsc container runs and exits === + [the gVisor job itself via runsc] interfaces: lo=127.0.0.1 eth0=172.31.16.2 dns: 34.225.223.145 + +=== gate5d: the same namespace afterwards === + [after, host runtime via runc] interfaces: lo=127.0.0.1 dns: EAI_AGAIN + [after, a second gVisor job via runsc] interfaces: lo=127.0.0.1 dns: EAI_AGAIN + +=== gate5d: read it off the two 'after' lines === +If 'before' has an eth0 address and 'after' says NONE, the namespace is single-use +for gVisor, and gate5c's ENETUNREACH verdicts are void — a dead control, not a policy. diff --git a/docs/gvisor-dns-delivery/evidence/gate5d-plan-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate5d-plan-rendered.txt new file mode 100644 index 00000000..9b57d9ff --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5d-plan-rendered.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.16.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5e-plan-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate5e-plan-rendered.txt new file mode 100644 index 00000000..f7729072 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5e-plan-rendered.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.19.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5e-routed-enforcement-under-runsc-20260910T0305Z.txt b/docs/gvisor-dns-delivery/evidence/gate5e-routed-enforcement-under-runsc-20260910T0305Z.txt new file mode 100644 index 00000000..2b7de26a --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5e-routed-enforcement-under-runsc-20260910T0305Z.txt @@ -0,0 +1,33 @@ +=== gate5e: environment === +utc=2026-09-10T02:48:54Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 runsc=runsc version release-20260817.0 +br_netfilter=absent +host_lan=192.168.5.15 (the VM's own eth0 — host-directed traffic lands in INPUT, not FORWARD) +per-job networks: job1=172.31.19.0/24 job2=172.31.20.0/24 +live host listener: 192.168.5.15:49252 (bare tcp, answers every connection) +live neighbour in job 2's own namespace: 172.31.20.2:8080 + +=== gate5e: 1. the host itself, a routed private destination with a LIVE listener === + runc -> 192.168.5.15:49252, bare: timeout [ns=172.31.19.2 rules=none] + runsc -> 192.168.5.15:49252, bare: REACHED [ns=172.31.19.2 rules=none] + runsc -> 192.168.5.15:49252, DOCKER-USER: REACHED [ns=172.31.19.2 rules=DOCKER-USER|-s 172.31.19.2/32 -d 192.168.0.0/16 -j DROP] + runsc -> 192.168.5.15:49252, INPUT: timeout [ns=172.31.19.2 rules=INPUT|-s 172.31.19.2/32 -d 192.168.0.0/16 -j DROP] + +=== gate5e: 2. the metadata address (no listener anywhere — read only the DIFFERENCE) === + runsc -> 169.254.169.254:80, bare: ECONNREFUSED [ns=172.31.19.2 rules=none] + runsc -> 169.254.169.254:80, DOCKER-USER: timeout [ns=172.31.19.2 rules=DOCKER-USER|-s 172.31.19.2/32 -d 169.254.169.254/32 -j DROP] + +=== gate5e: 3. cross-job, each job on its OWN network (the regression proof) === + runsc -> job 2's live listener: timeout [ns=172.31.19.2 rules=none] + +=== gate5e: 4. the public route, with the candidate rules installed === + health: HEALTHY 172.31.19.2 rules=DOCKER-USER|-s 172.31.19.2/32 -d 169.254.169.254/32 -j DROP INPUT|-s 172.31.19.2/32 -d 192.168.0.0/16 -j DROP + PUBLIC-PASS dns=34.225.223.145 tls=200 verified=true + health: HEALTHY 172.31.19.2 + PUBLIC-PASS git 7fd1a60b01f91b314f59955a4e4d4e80d8edf11d + +=== gate5e: how to read this === +leg 1 names the chain that binds a routed, host-directed destination for runsc. +leg 2 is evidence ONLY if bare and ruled differ; identical timeouts prove nothing. +leg 3 must be denied for per-job networks to be the cross-job answer. +leg 4 must pass, or the candidate costs the job the route it exists to have. diff --git a/docs/gvisor-dns-delivery/evidence/gate5f-host-install-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate5f-host-install-rendered.txt new file mode 100644 index 00000000..9444d3d3 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5f-host-install-rendered.txt @@ -0,0 +1,17 @@ +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 169.254.169.254/32 -j DROP +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 10.0.0.0/8 -j DROP +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 172.16.0.0/12 -j DROP +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 192.168.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 169.254.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 100.64.0.0/10 -j DROP +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 198.18.0.0/15 -j DROP +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 224.0.0.0/4 -j DROP +iptables -I DOCKER-USER -s 172.31.21.10/32 -d 240.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.21.10/32 -d 10.0.0.0/8 -j DROP +iptables -I INPUT -s 172.31.21.10/32 -d 172.16.0.0/12 -j DROP +iptables -I INPUT -s 172.31.21.10/32 -d 192.168.0.0/16 -j DROP +iptables -I INPUT -s 172.31.21.10/32 -d 169.254.0.0/16 -j DROP +iptables -I INPUT -s 172.31.21.10/32 -d 100.64.0.0/10 -j DROP +iptables -I INPUT -s 172.31.21.10/32 -d 198.18.0.0/15 -j DROP +iptables -I INPUT -s 172.31.21.10/32 -d 224.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.21.10/32 -d 240.0.0.0/4 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5f-host-teardown-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate5f-host-teardown-rendered.txt new file mode 100644 index 00000000..3b9a79a9 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5f-host-teardown-rendered.txt @@ -0,0 +1,17 @@ +iptables -D INPUT -s 172.31.21.10/32 -d 240.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.21.10/32 -d 224.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.21.10/32 -d 198.18.0.0/15 -j DROP +iptables -D INPUT -s 172.31.21.10/32 -d 100.64.0.0/10 -j DROP +iptables -D INPUT -s 172.31.21.10/32 -d 169.254.0.0/16 -j DROP +iptables -D INPUT -s 172.31.21.10/32 -d 192.168.0.0/16 -j DROP +iptables -D INPUT -s 172.31.21.10/32 -d 172.16.0.0/12 -j DROP +iptables -D INPUT -s 172.31.21.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 240.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 224.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 198.18.0.0/15 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 100.64.0.0/10 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 169.254.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 192.168.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 172.16.0.0/12 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.21.10/32 -d 169.254.169.254/32 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5f-plan-rendered.txt b/docs/gvisor-dns-delivery/evidence/gate5f-plan-rendered.txt new file mode 100644 index 00000000..66bca772 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5f-plan-rendered.txt @@ -0,0 +1,26 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.21.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -p udp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -p tcp -d 1.1.1.1/32 --dport 53 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5f-product-host-rules-bind-runsc-20260910T0305Z.txt b/docs/gvisor-dns-delivery/evidence/gate5f-product-host-rules-bind-runsc-20260910T0305Z.txt new file mode 100644 index 00000000..2d36601b --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5f-product-host-rules-bind-runsc-20260910T0305Z.txt @@ -0,0 +1,43 @@ +=== gate5f: environment === +utc=2026-09-10T03:05:46Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 runsc=runsc version release-20260817.0 +docker default-runtime=runc (helpers inherit this; the JOB always carries --runtime runsc) +br_netfilter=absent +host plan rendered by the product for job_addr=172.31.21.10: 17 rules +chain depth before anything: DOCKER-USER=1 INPUT=1 +live listeners: same-bridge 172.31.21.20:8080 · own-network 172.31.22.10:8080 · host 192.168.5.15:49253 + +=== gate5f: 1. BEFORE — the hole, with no host rules === + runsc -> same-bridge neighbour 172.31.21.20:8080: REACHED + runsc -> own-network neighbour 172.31.22.10:8080: timeout + runsc -> host 192.168.5.15:49253: REACHED + runc -> host 192.168.5.15:49253: timeout + +=== gate5f: 2. installing the product's rendered host policy === + host-rule install: applier reported 17, plan had 17 rules + readback from the host kernel: DOCKER-USER carries 9, INPUT carries 8 rules for 172.31.21.10 + +=== gate5f: 3. AFTER — the same probes, unchanged === + runsc -> own-network neighbour 172.31.22.10:8080 (MUST be denied): timeout + runsc -> host 192.168.5.15:49253 (MUST be denied): timeout + runc -> host 192.168.5.15:49253 (MUST be denied): timeout + runsc -> same-bridge neighbour 172.31.21.20:8080 (known unbindable): REACHED + +=== gate5f: 4. the public route must survive the policy === + health: HEALTHY 172.31.21.10 + PUBLIC-PASS dns=34.225.223.145 tls=200 verified=true + health: HEALTHY 172.31.21.10 + PUBLIC-PASS git 7fd1a60b01f91b314f59955a4e4d4e80d8edf11d + +=== gate5f: 5. teardown must leave the shared chains exactly as it found them === + host-rule teardown: applier reported 17, plan had 17 rules + readback from the host kernel: DOCKER-USER carries 0, INPUT carries 0 rules for 172.31.21.10 + chain depth after teardown: DOCKER-USER=1 (was 1) INPUT=1 (was 1) + CLEAN — no rule leaked (this is the failure mode the HostRules drop guard exists to prevent) + +=== gate5f: how to read this === +The fix is proved by section 3 differing from section 1 on the own-network and host legs. +The same-bridge leg is expected to stay REACHED: switched frames enter no chain on a host +without br_netfilter, which is WHY the product gives every job its own network rather than +trying to rule its way out of a shared one. It is measured here so that reason stays evidenced. +Section 4 must pass, or the policy costs the job the delivery route it exists to have. diff --git a/docs/gvisor-dns-delivery/evidence/gate5g-baseline-plan-from-origin-main.txt b/docs/gvisor-dns-delivery/evidence/gate5g-baseline-plan-from-origin-main.txt new file mode 100644 index 00000000..51b0d9de --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5g-baseline-plan-from-origin-main.txt @@ -0,0 +1,24 @@ +iptables -A OUTPUT -p tcp --syn -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-conn: +iptables -A OUTPUT -p udp --dport 53 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-dns: +iptables -A OUTPUT -d 169.254.169.254/32 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny-metadata: +iptables -A OUTPUT -d 169.254.169.254/32 -j DROP +iptables -A OUTPUT -p tcp -d 172.31.40.1 --dport 49200:49299 -j ACCEPT +iptables -A OUTPUT -d 10.0.0.0/8 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 10.0.0.0/8 -j DROP +iptables -A OUTPUT -d 172.16.0.0/12 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 172.16.0.0/12 -j DROP +iptables -A OUTPUT -d 192.168.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 192.168.0.0/16 -j DROP +iptables -A OUTPUT -d 169.254.0.0/16 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 169.254.0.0/16 -j DROP +iptables -A OUTPUT -d 100.64.0.0/10 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 100.64.0.0/10 -j DROP +iptables -A OUTPUT -d 198.18.0.0/15 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 198.18.0.0/15 -j DROP +iptables -A OUTPUT -d 224.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 224.0.0.0/4 -j DROP +iptables -A OUTPUT -d 240.0.0.0/4 -m limit --limit 6/min --limit-burst 12 -j LOG --log-prefix sbx-net-deny: +iptables -A OUTPUT -d 240.0.0.0/4 -j DROP +ip6tables -A OUTPUT -d fc00::/7 -j DROP +ip6tables -A OUTPUT -d fe80::/10 -j DROP +ip6tables -A OUTPUT -d ff00::/8 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5g-baseline-repro-CONFOUNDED-runc-victim-20260910T0325Z.txt b/docs/gvisor-dns-delivery/evidence/gate5g-baseline-repro-CONFOUNDED-runc-victim-20260910T0325Z.txt new file mode 100644 index 00000000..ebc50383 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5g-baseline-repro-CONFOUNDED-runc-victim-20260910T0325Z.txt @@ -0,0 +1,25 @@ +=== gate5g: environment === +utc=2026-09-10T03:25:09Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 runsc=runsc version release-20260817.0 +baseline commit=b45f865 (origin/main) · plan rules=24 +host rules installed by this script: NONE (origin/main has no HostPolicy — grep says 0) +arrangement: ONE shared network 172.31.40.0/24, as a single [sandbox] network produced +chain depth (untouched by this gate): DOCKER-USER=1 INPUT=1 +live listeners: victim job 172.31.40.20:8080 (inside 172.16.0.0/12, which the baseline plan DROPs) · host 192.168.5.15:49255 + +=== gate5g: 1. cross-job on the baseline shared network, live victim === + [ns gate5g-attacker-1: applier=24/24 rules, kernel readback shows 2 rule(s) for 172.16.0.0/12] + runc -> victim 172.31.40.20:8080 (positive control: the plan MUST bind runc): timeout + [ns gate5g-attacker-2: applier=24/24 rules, kernel readback shows 2 rule(s) for 172.16.0.0/12] + runsc -> victim 172.31.40.20:8080 (the vulnerability): timeout + +=== gate5g: 2. private egress to the host itself, live listener === + [ns gate5g-attacker-3: applier=24/24 rules, kernel readback shows 2 rule(s) for 172.16.0.0/12] + runc -> host 192.168.5.15:49255 (positive control): timeout + [ns gate5g-attacker-4: applier=24/24 rules, kernel readback shows 2 rule(s) for 172.16.0.0/12] + runsc -> host 192.168.5.15:49255 (the vulnerability): REACHED + +=== gate5g: verdict === +baseline REACHED count (each is a containment failure on origin/main): 1 +BASELINE-VULNERABLE: CONFIRMED on b45f865 — the netns plan binds runc and not runsc, +with no host-side enforcement present to catch it. The branch does not introduce this. diff --git a/docs/gvisor-dns-delivery/evidence/gate5g-baseline-vulnerability-CONFIRMED-20260910T0330Z.txt b/docs/gvisor-dns-delivery/evidence/gate5g-baseline-vulnerability-CONFIRMED-20260910T0330Z.txt new file mode 100644 index 00000000..f432fc0b --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5g-baseline-vulnerability-CONFIRMED-20260910T0330Z.txt @@ -0,0 +1,27 @@ +=== gate5g: environment === +utc=2026-09-10T03:27:15Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 runsc=runsc version release-20260817.0 +baseline commit=b45f865 (origin/main) · plan rules=24 +host rules installed by this script: NONE (origin/main has no HostPolicy — grep says 0) +arrangement: ONE shared network 172.31.40.0/24, as a single [sandbox] network produced +chain depth (untouched by this gate): DOCKER-USER=1 INPUT=1 +victim liveness (from inside its own namespace): VICTIM-DEAD ECONNREFUSED +live listeners: victim job 172.31.40.20:8080 (inside 172.16.0.0/12, which the baseline plan DROPs) · host 192.168.5.15:49255 + +=== gate5g: 1. cross-job on the baseline shared network, live victim === + (valid only if the victim is SERVING above; a dead victim makes every timeout meaningless) + [ns gate5g-attacker-1: applier=24/24 rules, kernel readback shows 2 rule(s) for 172.16.0.0/12] + runc -> victim 172.31.40.20:8080 (positive control: the plan MUST bind runc): timeout + [ns gate5g-attacker-2: applier=24/24 rules, kernel readback shows 2 rule(s) for 172.16.0.0/12] + runsc -> victim 172.31.40.20:8080 (the vulnerability): REACHED + +=== gate5g: 2. private egress to the host itself, live listener === + [ns gate5g-attacker-3: applier=24/24 rules, kernel readback shows 2 rule(s) for 172.16.0.0/12] + runc -> host 192.168.5.15:49255 (positive control): timeout + [ns gate5g-attacker-4: applier=24/24 rules, kernel readback shows 2 rule(s) for 172.16.0.0/12] + runsc -> host 192.168.5.15:49255 (the vulnerability): REACHED + +=== gate5g: verdict === +baseline REACHED count (each is a containment failure on origin/main): 2 +BASELINE-VULNERABLE: CONFIRMED on b45f865 — the netns plan binds runc and not runsc, +with no host-side enforcement present to catch it. The branch does not introduce this. diff --git a/docs/gvisor-dns-delivery/evidence/gate5h-lifecycle-recycled-address-PASS-20260910T0332Z.txt b/docs/gvisor-dns-delivery/evidence/gate5h-lifecycle-recycled-address-PASS-20260910T0332Z.txt new file mode 100644 index 00000000..1fd69a6c --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5h-lifecycle-recycled-address-PASS-20260910T0332Z.txt @@ -0,0 +1,38 @@ +=== gate5h: environment === +utc=2026-09-10T03:32:21Z +kernel=6.8.0-134-generic arch=aarch64 runsc=runsc version release-20260817.0 +chain depth at start (DOCKER-USER + INPUT): 2 +live host listener: 192.168.5.15:49256 + +=== gate5h: 1. job A establishes, rules appear, containment holds === + job A address (read from docker inspect): 172.31.55.2 + rules keyed to 172.31.55.2: before=0 after-install=17 + ok: install added 17 host rules + namespace health: HEALTHY 172.31.55.2 + runsc -> live host 192.168.5.15:49256: timeout + ok: A contained while its rules are installed + +=== gate5h: 2. job A tears down — rules must be GONE, network must be GONE === + rules keyed to 172.31.55.2 after teardown: 0 + ok: no leftover host rules + ok: network removed + +=== gate5h: 3. RECYCLED address — job B takes A's old address === + job B address: 172.31.55.2 (A's was 172.31.55.2) + ok: address genuinely recycled — this is the case that matters + rules keyed to 172.31.55.2 BEFORE B installs its own: 0 + ok: B inherits no stale firewall + runsc -> live host BEFORE B's rules (bare control, expect REACHED): REACHED + runsc -> live host AFTER B's rules: ENETUNREACH + ok: B contained by its own rules + +=== gate5h: 4. RECREATION — the same job id twice must not wedge === + first incarnation address: 172.31.55.2 + second incarnation address: 172.31.55.2 + ok: same job id came up twice, no 'network already exists' wedge + +=== gate5h: verdict === +chain depth: start=2 end=2 + ok: chains returned to starting depth — the gate left nothing behind +failing checks: 0 +GATE 5h: PASS diff --git a/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.55.2-install.txt b/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.55.2-install.txt new file mode 100644 index 00000000..60855519 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.55.2-install.txt @@ -0,0 +1,17 @@ +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 169.254.169.254/32 -j DROP +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 10.0.0.0/8 -j DROP +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 172.16.0.0/12 -j DROP +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 192.168.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 169.254.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 100.64.0.0/10 -j DROP +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 198.18.0.0/15 -j DROP +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 224.0.0.0/4 -j DROP +iptables -I DOCKER-USER -s 172.31.55.2/32 -d 240.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.55.2/32 -d 10.0.0.0/8 -j DROP +iptables -I INPUT -s 172.31.55.2/32 -d 172.16.0.0/12 -j DROP +iptables -I INPUT -s 172.31.55.2/32 -d 192.168.0.0/16 -j DROP +iptables -I INPUT -s 172.31.55.2/32 -d 169.254.0.0/16 -j DROP +iptables -I INPUT -s 172.31.55.2/32 -d 100.64.0.0/10 -j DROP +iptables -I INPUT -s 172.31.55.2/32 -d 198.18.0.0/15 -j DROP +iptables -I INPUT -s 172.31.55.2/32 -d 224.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.55.2/32 -d 240.0.0.0/4 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.55.2-teardown.txt b/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.55.2-teardown.txt new file mode 100644 index 00000000..32a3befd --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.55.2-teardown.txt @@ -0,0 +1,17 @@ +iptables -D INPUT -s 172.31.55.2/32 -d 240.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.55.2/32 -d 224.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.55.2/32 -d 198.18.0.0/15 -j DROP +iptables -D INPUT -s 172.31.55.2/32 -d 100.64.0.0/10 -j DROP +iptables -D INPUT -s 172.31.55.2/32 -d 169.254.0.0/16 -j DROP +iptables -D INPUT -s 172.31.55.2/32 -d 192.168.0.0/16 -j DROP +iptables -D INPUT -s 172.31.55.2/32 -d 172.16.0.0/12 -j DROP +iptables -D INPUT -s 172.31.55.2/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 240.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 224.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 198.18.0.0/15 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 100.64.0.0/10 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 169.254.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 192.168.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 172.16.0.0/12 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.55.2/32 -d 169.254.169.254/32 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.56.2-install.txt b/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.56.2-install.txt new file mode 100644 index 00000000..3c9e6431 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.56.2-install.txt @@ -0,0 +1,17 @@ +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 169.254.169.254/32 -j DROP +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 10.0.0.0/8 -j DROP +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 172.16.0.0/12 -j DROP +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 192.168.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 169.254.0.0/16 -j DROP +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 100.64.0.0/10 -j DROP +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 198.18.0.0/15 -j DROP +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 224.0.0.0/4 -j DROP +iptables -I DOCKER-USER -s 172.31.56.2/32 -d 240.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.56.2/32 -d 10.0.0.0/8 -j DROP +iptables -I INPUT -s 172.31.56.2/32 -d 172.16.0.0/12 -j DROP +iptables -I INPUT -s 172.31.56.2/32 -d 192.168.0.0/16 -j DROP +iptables -I INPUT -s 172.31.56.2/32 -d 169.254.0.0/16 -j DROP +iptables -I INPUT -s 172.31.56.2/32 -d 100.64.0.0/10 -j DROP +iptables -I INPUT -s 172.31.56.2/32 -d 198.18.0.0/15 -j DROP +iptables -I INPUT -s 172.31.56.2/32 -d 224.0.0.0/4 -j DROP +iptables -I INPUT -s 172.31.56.2/32 -d 240.0.0.0/4 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.56.2-teardown.txt b/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.56.2-teardown.txt new file mode 100644 index 00000000..330586ec --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5h-plans/172.31.56.2-teardown.txt @@ -0,0 +1,17 @@ +iptables -D INPUT -s 172.31.56.2/32 -d 240.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.56.2/32 -d 224.0.0.0/4 -j DROP +iptables -D INPUT -s 172.31.56.2/32 -d 198.18.0.0/15 -j DROP +iptables -D INPUT -s 172.31.56.2/32 -d 100.64.0.0/10 -j DROP +iptables -D INPUT -s 172.31.56.2/32 -d 169.254.0.0/16 -j DROP +iptables -D INPUT -s 172.31.56.2/32 -d 192.168.0.0/16 -j DROP +iptables -D INPUT -s 172.31.56.2/32 -d 172.16.0.0/12 -j DROP +iptables -D INPUT -s 172.31.56.2/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 240.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 224.0.0.0/4 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 198.18.0.0/15 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 100.64.0.0/10 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 169.254.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 192.168.0.0/16 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 172.16.0.0/12 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 10.0.0.0/8 -j DROP +iptables -D DOCKER-USER -s 172.31.56.2/32 -d 169.254.169.254/32 -j DROP diff --git a/docs/gvisor-dns-delivery/evidence/gate5i-fail-closed-FAIL-teardown-leak-20260910T0339Z.txt b/docs/gvisor-dns-delivery/evidence/gate5i-fail-closed-FAIL-teardown-leak-20260910T0339Z.txt new file mode 100644 index 00000000..2d17ff65 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5i-fail-closed-FAIL-teardown-leak-20260910T0339Z.txt @@ -0,0 +1,36 @@ +=== gate5i: environment === +utc=2026-09-10T03:39:18Z +kernel=6.8.0-134-generic arch=aarch64 runsc=runsc version release-20260817.0 +chain depth at start: 2 +live host listener: 192.168.5.15:49257 +job address: 172.31.56.2 +rendered plan: 17 rules + +=== gate5i: A. applier cannot start (bad image) — nothing may be installed === + applier said: Run 'docker run --help' for more information + rules keyed to 172.31.56.2: 0 + ok: a non-starting applier installs nothing (establish() maps this to a hard error) + +=== gate5i: B. applier without NET_ADMIN — must not silently succeed === + applier said: apply-policy: namespace is PARTIALLY configured — destroy the holder, do not retry + rules keyed to 172.31.56.2: 0 + ok: no capability, no rules + +=== gate5i: C. TRUNCATED plan — does it apply cleanly and lie? === + feeding 17 rendered rules as a 9-line plan + applier exit=0 said=9 + rules actually in the kernel: 9 of 17 rendered + ok: CONFIRMED: a truncated plan exits 0 while under-installing — only the count reveals it + ok: count cross-check would refuse this job (9 != 17) + runsc -> live host 192.168.5.15:49257 with a PARTIAL firewall: REACHED + ok: a partially-installed policy leaves the job UNCONTAINED — refusing the job is the only safe move + +=== gate5i: D. teardown unwinds the partial install === + rules keyed to 172.31.56.2 after teardown: 9 + FAIL: 9 rule(s) survived teardown of a partial install + +=== gate5i: verdict === +chain depth: start=2 end=11 + FAIL: chains changed 2 -> 11 +failing checks: 2 +GATE 5i: FAIL diff --git a/docs/gvisor-dns-delivery/evidence/gate5i-fail-closed-PASS-after-fix-20260910T0347Z.txt b/docs/gvisor-dns-delivery/evidence/gate5i-fail-closed-PASS-after-fix-20260910T0347Z.txt new file mode 100644 index 00000000..e9030588 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5i-fail-closed-PASS-after-fix-20260910T0347Z.txt @@ -0,0 +1,41 @@ +=== gate5i: environment === +utc=2026-09-10T03:44:36Z +kernel=6.8.0-134-generic arch=aarch64 runsc=runsc version release-20260817.0 +chain depth at start: 2 +./gate5i.sh: line 59: /home/forge.guest/gate5i-resolv.conf: Permission denied +live host listener: 192.168.5.15:49257 +job address: 172.31.56.2 +rendered plan: 17 rules + +=== gate5i: A. applier cannot start (bad image) — nothing may be installed === + applier said: Run 'docker run --help' for more information + rules keyed to 172.31.56.2: 0 + ok: a non-starting applier installs nothing (establish() maps this to a hard error) + +=== gate5i: B. applier without NET_ADMIN — must not silently succeed === + applier said: apply-policy: namespace is PARTIALLY configured — destroy the holder, do not retry + rules keyed to 172.31.56.2: 0 + ok: no capability, no rules + +=== gate5i: C. TRUNCATED plan — does it apply cleanly and lie? === + feeding 17 rendered rules as a 9-line plan + applier exit=0 said=9 + rules actually in the kernel: 9 of 17 rendered + ok: CONFIRMED: a truncated plan exits 0 while under-installing — only the count reveals it + ok: count cross-check would refuse this job (9 != 17) + runsc -> live host 192.168.5.15:49257 with a PARTIAL firewall: REACHED + ok: a partially-installed policy leaves the job UNCONTAINED — refusing the job is the only safe move + +=== gate5i: D. teardown of a PARTIAL install === + D1: one-shot inverse plan (the pre-fix behaviour) + rules remaining: 9 (of 9 installed) + as expected: the applier aborts on the first never-created rule and removes nothing + D2: per-rule teardown (what HostRules::drop now does on the partial path) + removed 9 rule(s) one at a time; 0 remain + ok: the partial install came out — a missing rule no longer strands the present ones + +=== gate5i: verdict === +chain depth: start=2 end=2 + ok: chains returned to starting depth +failing checks: 0 +GATE 5i: PASS diff --git a/docs/gvisor-dns-delivery/evidence/gate5j-ipv6-finding-20260910T0347Z.txt b/docs/gvisor-dns-delivery/evidence/gate5j-ipv6-finding-20260910T0347Z.txt new file mode 100644 index 00000000..cdc68e00 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5j-ipv6-finding-20260910T0347Z.txt @@ -0,0 +1,22 @@ +utc=2026-09-10T03:47:21Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 +=== daemon.json === +{ + "runtimes": { + "runsc": { + "path": "/usr/local/bin/runsc" + } + } +}=== daemon ipv6 setting === +(docker info reports no ipv6 key) +=== a job-shaped holder on a per-job network === +--- ip -6 addr inside the holder namespace --- +1: lo: mtu 65536 state UNKNOWN qlen 1000 + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever +--- global (non-link-local) v6 addresses seen --- +0 +0 +--- does ip6tables even have a DOCKER-USER chain here? --- +-N DOCKER-USER +=== end === diff --git a/docs/gvisor-dns-delivery/evidence/gate5k-CONFOUNDED-harness-defect-20260910T0310Z.txt b/docs/gvisor-dns-delivery/evidence/gate5k-CONFOUNDED-harness-defect-20260910T0310Z.txt new file mode 100644 index 00000000..6decd5bb --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5k-CONFOUNDED-harness-defect-20260910T0310Z.txt @@ -0,0 +1,32 @@ +gate5k — unsupported daemon default runtime +kernel: Linux 6.8.0-134-generic aarch64 +docker: 29.1.3 +runsc: runsc version release-20260817.0 +ORIGINAL default runtime: runc + +listener address: 172.18.0.2 + +---- LEG 1 — control: daemon default is runc (the supported configuration) +default runtime now: runc +listener liveness: NOT-SERVING + holder runtime: runc + holder ifaces: exec failed + job: + job: <= no-answer + +---- LEG 2 — daemon default switched to runsc (the UNSUPPORTED configuration) +default runtime now: runsc +listener liveness: NOT-SERVING +listener down after the daemon restart — leg 2 is NO EVIDENCE + holder runtime: runsc + holder ifaces: executing processes for container + job: + job: <= no-answer + +---- VERDICT +Read leg 2's job line: a job that shows only 'lo' and cannot reach the live +listener FAILED CLOSED (no egress, no containment bypass). A job that prints +REACHED under the unsupported default FAILED OPEN and is a security defect. +failing_checks=2 +---- restoring /etc/docker/daemon.json +restored default runtime = runc diff --git a/docs/gvisor-dns-delivery/evidence/gate5k-v2-daemon-default-runtime-20260910T0325Z.txt b/docs/gvisor-dns-delivery/evidence/gate5k-v2-daemon-default-runtime-20260910T0325Z.txt new file mode 100644 index 00000000..177269f7 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5k-v2-daemon-default-runtime-20260910T0325Z.txt @@ -0,0 +1,27 @@ +gate5k v2 — unsupported daemon default runtime +kernel: Linux 6.8.0-134-generic aarch64 +docker: 29.1.3 +runsc: runsc version release-20260817.0 +ORIGINAL default runtime: runc + +---- LEG 1 — control: daemon default is runc (the supported configuration) +default runtime now: runc +listener 172.18.0.2 liveness: NOT-SERVING +leg 1 target is dead — NO EVIDENCE + holder runtime: runc + job: ifaces= lo eth0 | routes=eth0 eth0 | target=no-answer + +---- LEG 2 — daemon default switched to runsc (the UNSUPPORTED configuration) +default runtime now: runsc +listener 172.18.0.2 liveness: NOT-SERVING +leg 2 target is dead — NO EVIDENCE, not a fail-closed result + holder runtime: runsc + job: ifaces= lo | routes=| target=no-answer + +---- VERDICT +Leg 2 with a SERVING listener: a job showing only 'lo', or refusing to run, +FAILED CLOSED — no egress and no containment bypass. A job printing REACHED +under the unsupported default FAILED OPEN and is a security defect. +failing_checks=2 (a non-zero count means legs above proved nothing) +---- restoring /etc/docker/daemon.json +restored default runtime = runc diff --git a/docs/gvisor-dns-delivery/evidence/gate5k-v3-PASS-daemon-default-20260910T0345Z.txt b/docs/gvisor-dns-delivery/evidence/gate5k-v3-PASS-daemon-default-20260910T0345Z.txt new file mode 100644 index 00000000..5e18161f --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/gate5k-v3-PASS-daemon-default-20260910T0345Z.txt @@ -0,0 +1,29 @@ +gate5k v3 — unsupported daemon default runtime +kernel: Linux 6.8.0-134-generic aarch64 +docker: 29.1.3 +runsc: runsc version release-20260817.0 +ORIGINAL default runtime: runc + +---- LEG 1 — control: daemon default is runc (the supported configuration) +default runtime now: runc +listener 172.18.0.2 liveness: SERVING + holder runtime: runc + job: HEALTHY 172.18.0.3 | target=REACHED + ^ expected here: HEALTHY and target=REACHED. This leg is the + positive control: it proves the harness CAN observe reachability, so + leg 2's silence means something. + +---- LEG 2 — daemon default switched to runsc (the UNSUPPORTED configuration) +default runtime now: runsc +listener 172.18.0.2 liveness: SERVING + holder runtime: runsc + job: SICK no-address | target=ENETUNREACH + +---- VERDICT +With a SERVING listener in both legs: leg 1 REACHED and leg 2 SICK/denied +means the unsupported default FAILS CLOSED — no egress path to contain, an +availability failure rather than a containment bypass. Leg 2 printing +target=REACHED would mean it FAILS OPEN and is a security defect. +failing_checks=0 (non-zero means a leg above proved nothing) +---- restoring /etc/docker/daemon.json +restored default runtime = runc diff --git a/docs/gvisor-dns-delivery/evidence/run-all-gates-proof-run-20260910T0316Z.txt b/docs/gvisor-dns-delivery/evidence/run-all-gates-proof-run-20260910T0316Z.txt new file mode 100644 index 00000000..acffc125 --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/run-all-gates-proof-run-20260910T0316Z.txt @@ -0,0 +1,15 @@ +=== gVisor DNS/delivery gates === +utc=2026-09-10T03:15:13Z +kernel=6.8.0-134-generic arch=aarch64 docker=29.1.3 runsc=runsc version release-20260817.0 +per-gate timeout=420s total budget=800s evidence=/home/forge.guest/gate-evidence +NOTE: results are labelled by architecture on purpose — x86_64 is NOT covered by this run. + +--- gate1: running (timeout 420s) -> /home/forge.guest/gate-evidence/gate1-20260910T031513Z.log +--- gate5f: running (timeout 420s) -> /home/forge.guest/gate-evidence/gate5f-20260910T031522Z.log +=== summary === +utc=2026-09-10T03:16:14Z +arch=aarch64 runsc=runsc version release-20260817.0 +gate1 PASS exit=0 log=gate1-20260910T031513Z.log +gate5f PASS exit=0 log=gate5f-20260910T031522Z.log +total=61s gates=2 not-passing=0 +ALL GATES: PASS diff --git a/docs/gvisor-dns-delivery/evidence/run-all-gates-proof-summary-20260910T0316Z.txt b/docs/gvisor-dns-delivery/evidence/run-all-gates-proof-summary-20260910T0316Z.txt new file mode 100644 index 00000000..54cf7aca --- /dev/null +++ b/docs/gvisor-dns-delivery/evidence/run-all-gates-proof-summary-20260910T0316Z.txt @@ -0,0 +1,7 @@ +=== summary === +utc=2026-09-10T03:16:14Z +arch=aarch64 runsc=runsc version release-20260817.0 +gate1 PASS exit=0 log=gate1-20260910T031513Z.log +gate5f PASS exit=0 log=gate5f-20260910T031522Z.log +total=61s gates=2 not-passing=0 +ALL GATES: PASS diff --git a/docs/gvisor-dns-delivery/scripts/gate1-repro.sh b/docs/gvisor-dns-delivery/scripts/gate1-repro.sh new file mode 100755 index 00000000..8c0fa452 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate1-repro.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Gate 1: reproduce the named-bridge DNS failure under runsc, with runc as a +# DIAGNOSTIC CONTROL only. Runs inside the disposable VM. Bounded: every docker +# run carries a timeout and the network is removed on exit. +# +# runc appears here to isolate the variable. It is never a fallback path for +# production jobs. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NET="${NET:-maxplayer-dns-repro}" +HOST_TARGET="${HOST_TARGET:-relay.maxplayer.ai}" +OUT="${OUT:-$HOME/gate1-evidence.txt}" + +exec > >(tee "${OUT}") 2>&1 + +echo "=== gate1: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m)" +. /etc/os-release && echo "os=${PRETTY_NAME}" +echo "docker=$(sudo docker version --format '{{.Server.Version}}')" +echo "runsc=$(runsc --version | head -1)" +echo "image=${IMAGE}" +echo "image_digest=$(sudo docker image inspect "${IMAGE}" --format '{{index .RepoDigests 0}}')" +echo "image_arch=$(sudo docker image inspect "${IMAGE}" --format '{{.Architecture}}/{{.Os}}')" + +cleanup() { sudo docker network rm "${NET}" >/dev/null 2>&1 || true; } +trap cleanup EXIT +sudo docker network rm "${NET}" >/dev/null 2>&1 || true +sudo docker network create "${NET}" >/dev/null +echo "network=${NET} subnet=$(sudo docker network inspect "${NET}" --format '{{(index .IPAM.Config 0).Subnet}}')" + +# One probe body, run identically under both runtimes: resolve, then report the +# resolver the container was actually handed. +PROBE='const dns=require("dns");const fs=require("fs"); +console.log("resolv.conf:", fs.readFileSync("/etc/resolv.conf","utf8").trim().replace(/\n/g,"|")); +dns.lookup(process.argv[1],(e,a)=>{console.log("lookup:", e?("ERR "+e.code):("OK "+a));process.exitCode=e?1:0});' + +run_probe() { + local runtime="$1" + echo + echo "=== gate1: dns lookup under --runtime ${runtime} ===" + sudo timeout 90 docker run --rm --runtime "${runtime}" --network "${NET}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + --entrypoint node "${IMAGE}" -e "${PROBE}" "${HOST_TARGET}" + echo "exit=$?" +} + +run_probe runsc +run_probe runc + +echo +echo "=== gate1: raw udp/53 to the embedded resolver under runsc ===" +sudo timeout 90 docker run --rm --runtime runsc --network "${NET}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + --entrypoint node "${IMAGE}" -e ' +const dgram=require("dgram");const s=dgram.createSocket("udp4"); +const q=Buffer.from("abcd01000001000000000000057265" + + "6c6179096d6178706c61796572026169000001" + "0001","hex"); +const t=setTimeout(()=>{console.log("udp53: TIMEOUT (no answer from 127.0.0.11)");s.close();process.exitCode=1;},8000); +s.on("message",(m)=>{clearTimeout(t);console.log("udp53: ANSWER "+m.length+" bytes");s.close();}); +s.on("error",(e)=>{clearTimeout(t);console.log("udp53: ERR "+e.code);s.close();process.exitCode=1;}); +s.send(q,53,"127.0.0.11");' +echo "exit=$?" + +echo +echo "=== gate1: done ===" diff --git a/docs/gvisor-dns-delivery/scripts/gate2-namespace-dns-tls.sh b/docs/gvisor-dns-delivery/scripts/gate2-namespace-dns-tls.sh new file mode 100755 index 00000000..fbddccb4 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate2-namespace-dns-tls.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Gate 2: DNS **and** certificate-validated TLS from inside the REAL shared job +# namespace — holder + sidecar-applied policy + job — under runsc, non-root, +# cap-drop ALL, no-new-privileges. Proven on a fresh namespace and again after +# the namespace is destroyed and recreated. +# +# Runs inside the disposable gvisor-repro VM. Bounded: every docker run carries a +# timeout, and every container and the network are removed on exit. +# +# The iptables plan is NOT transcribed here. It is rendered by the product's own +# `NetPolicy` (cargo run -p maxplayer-core --example render_net_plan) and passed +# in via PLAN_FILE, so this gate cannot pass against a firewall the product no +# longer builds. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +NET="${NET:-maxplayer-dns-gate2}" +HOST_TARGET="${HOST_TARGET:-relay.maxplayer.ai}" +RESOLVER="${RESOLVER:-1.1.1.1}" +SUBNET="${SUBNET:-172.31.7.0/24}" +GATEWAY="${GATEWAY:-172.31.7.1}" +PLAN_FILE="${PLAN_FILE:-$HOME/gate2-plan.txt}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate2-resolv.conf}" +OUT="${OUT:-$HOME/gate2-evidence.txt}" +HOLDER="gate2-holder" +# The containment plane (holder + sidecar) runs on the HOST runtime; only the job runs +# under gVisor. Measured, not preference: a runsc container joining a runsc holder's +# network namespace sees `lo` ONLY — no eth0, no route, every lookup EAI_AGAIN — because +# a gVisor sandbox's netstack lives inside that sandbox and cannot be entered by a second +# one. Joining a runc holder, a runsc job gets the holder's own interface and address +# (172.31.11.2 in both, measured), so the host kernel's rules govern its traffic. The +# sidecar needs the host runtime for a second reason: iptables-nft inside gVisor fails +# `Failed to initialize nft: Protocol not supported`. +HOLDER_RUNTIME="${HOLDER_RUNTIME:-runc}" +JOB_RUNTIME="${JOB_RUNTIME:-runsc}" + +exec > >(tee "${OUT}") 2>&1 + +fail=0 + +echo "=== gate2: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m)" +. /etc/os-release && echo "os=${PRETTY_NAME}" +echo "docker=$(sudo docker version --format '{{.Server.Version}}')" +echo "runsc=$(runsc --version | head -1)" +echo "holder_runtime=${HOLDER_RUNTIME} job_runtime=${JOB_RUNTIME}" +echo "image=${IMAGE}" +echo "image_digest=$(sudo docker image inspect "${IMAGE}" --format '{{index .RepoDigests 0}}')" +echo "netfilter_digest=$(sudo docker image inspect "${NETFILTER_IMAGE}" --format '{{index .RepoDigests 0}}')" +echo "resolver=${RESOLVER}" + +# The resolver file the product would write, in the product's format. +cat > "${RESOLV_FILE}" </dev/null 2>&1 || true +} +cleanup() { + teardown_ns + sudo docker network rm "${NET}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +# A FIXED subnet, so the gateway in the rendered plan and the gateway of the network +# the job actually joins are the same address. Letting docker pick would render the +# proxy pinhole for one address while the job reaches the host at another — rules +# that look right in every log and route nothing. +sudo docker network rm "${NET}" >/dev/null 2>&1 || true +sudo docker network create --subnet "${SUBNET}" --gateway "${GATEWAY}" "${NET}" >/dev/null +echo "network=${NET} subnet=$(sudo docker network inspect "${NET}" --format '{{(index .IPAM.Config 0).Subnet}}') gateway=${GATEWAY}" + +# The job probe: resolve, then complete a TLS handshake whose certificate chain is +# VERIFIED against the image's own trust store. `rejectUnauthorized` stays default +# (true) and the peer certificate is printed, so a passing gate cannot be a +# handshake that skipped verification. +PROBE='const dns=require("dns"),https=require("https"),fs=require("fs"); +const host=process.argv[1]; +console.log("resolv.conf:", fs.readFileSync("/etc/resolv.conf","utf8").trim().split("\n").filter(l=>!l.startsWith("#")).join("|")); +dns.lookup(host,(e,a)=>{ + if(e){console.log("lookup: ERR "+e.code);process.exit(1);} + console.log("lookup: OK "+a); + const req=https.request({host,port:443,path:"/",method:"HEAD",timeout:15000},(res)=>{ + const c=res.socket.getPeerCertificate(); + console.log("tls: "+res.statusCode+" cert-verified subject="+(c&&c.subject&&c.subject.CN)+" issuer="+(c&&c.issuer&&c.issuer.CN)+" authorized="+res.socket.authorized); + process.exit(res.socket.authorized?0:1); + }); + req.on("timeout",()=>{console.log("tls: TIMEOUT");process.exit(1);}); + req.on("error",(err)=>{console.log("tls: ERR "+err.code+" "+err.message);process.exit(1);}); + req.end(); +});' + +# A second probe proving the containment the DNS pinhole must not have widened: +# the cloud metadata address stays denied while public egress works. +DENY_PROBE='const net=require("net"); +const s=net.connect({host:"169.254.169.254",port:80,timeout:6000}); +s.on("connect",()=>{console.log("metadata: REACHED (containment broken)");process.exit(1);}); +s.on("timeout",()=>{console.log("metadata: denied (timeout)");process.exit(0);}); +s.on("error",(e)=>{console.log("metadata: denied ("+e.code+")");process.exit(0);});' + +establish_namespace() { + local label="$1" + echo + echo "=== gate2/${label}: establish the shared job namespace ===" + # Holder: owns the namespace, holds no capability, runs as nobody, read-only. + sudo timeout 120 docker run --detach --name "${HOLDER}" --runtime "${HOLDER_RUNTIME}" \ + --network "${NET}" --read-only --cap-drop ALL --security-opt no-new-privileges \ + --user 65534:65534 --entrypoint sleep "${IMAGE}" infinity >/dev/null + echo "holder=${HOLDER} started=$?" + + # Sidecar: the ONLY container handed NET_ADMIN, scoped to the holder's namespace, + # gone before the job starts. Same runtime as the holder, so it writes into the netns + # the job will actually join. + echo "--- sidecar applies the rendered plan ---" + sudo timeout 120 docker run --rm --interactive --runtime "${HOLDER_RUNTIME}" \ + --network "container:${HOLDER}" --cap-drop ALL --cap-add NET_ADMIN \ + --security-opt no-new-privileges "${NETFILTER_IMAGE}" < "${PLAN_FILE}" + echo "sidecar_exit=$?" + + # Read the rules back out of the namespace with a DIFFERENT container running a + # DIFFERENT verb, because the question is what the netstack holds and not whether + # the installer believes it succeeded. + echo "--- readback (iptables -S) ---" + sudo timeout 60 docker run --rm --runtime "${HOLDER_RUNTIME}" --network "container:${HOLDER}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + --entrypoint iptables "${NETFILTER_IMAGE}" -S OUTPUT + echo "readback_exit=$?" +} + +run_job() { + local label="$1" + echo + echo "=== gate2/${label}: job in the shared namespace — dns + verified tls ===" + sudo timeout 120 docker run --rm --runtime "${JOB_RUNTIME}" --network "container:${HOLDER}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" \ + --entrypoint node "${IMAGE}" -e "${PROBE}" "${HOST_TARGET}" + local rc=$? + echo "job_exit=${rc}" + [ "${rc}" -eq 0 ] || fail=1 + + echo "--- containment still holds: metadata address ---" + sudo timeout 60 docker run --rm --runtime "${JOB_RUNTIME}" --network "container:${HOLDER}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" \ + --entrypoint node "${IMAGE}" -e "${DENY_PROBE}" + local drc=$? + echo "metadata_exit=${drc}" + [ "${drc}" -eq 0 ] || fail=1 +} + +establish_namespace fresh +run_job fresh + +echo +echo "=== gate2: destroy the namespace and rebuild it ===" +teardown_ns +sleep 2 +establish_namespace recreated +run_job recreated + +echo +echo "=== gate2: verdict ===" +if [ "${fail}" -eq 0 ]; then + echo "GATE2: PASS (fresh and recreated namespaces both resolved and completed verified TLS)" +else + echo "GATE2: FAIL" +fi +exit "${fail}" diff --git a/docs/gvisor-dns-delivery/scripts/gate4-container-git-delivery.sh b/docs/gvisor-dns-delivery/scripts/gate4-container-git-delivery.sh new file mode 100755 index 00000000..ba5f971f --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate4-container-git-delivery.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# Gate 4: REAL git delivery originating INSIDE the job sandbox — not a mock, not a +# host-side upload. Two legs, both from the job container under runsc, non-root, +# cap-drop ALL, no-new-privileges, inside the shared job namespace: +# +# READ — clone a real public repository over HTTPS, and prove the delivered +# commit is the remote's real one by comparing against `git ls-remote` +# taken independently OUTSIDE the sandbox. +# WRITE — commit in the container and PUSH to a disposable bare remote that +# lives outside the container, then prove the remote's ref now holds +# exactly the hash the container produced. +# +# The write remote is reached at the namespace gateway on a port inside the +# policy's proxy pinhole — the one host-facing hole the design already opens — +# so the push crosses the job's egress policy rather than sidestepping it. +# +# ⛔ No credential is used, needed, or logged. A container-side push to a +# credentialed remote would require putting a secret inside a stranger's +# sandbox, which is the one thing the whole containment design exists to +# prevent; the write leg therefore uses an unauthenticated disposable remote. +# Named as a limitation in the runlog rather than papered over. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +NET="${NET:-maxplayer-dns-gate4}" +SUBNET="${SUBNET:-172.31.8.0/24}" +GATEWAY="${GATEWAY:-172.31.8.1}" +RESOLVER="${RESOLVER:-1.1.1.1}" +# Inside the proxy pinhole range the rendered plan opens (49200-49299). +REMOTE_PORT="${REMOTE_PORT:-49250}" +PUBLIC_REPO="${PUBLIC_REPO:-https://github.com/octocat/Hello-World.git}" +PLAN_FILE="${PLAN_FILE:-$HOME/gate4-plan.txt}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate4-resolv.conf}" +REMOTE_ROOT="${REMOTE_ROOT:-$HOME/gate4-remote}" +OUT="${OUT:-$HOME/gate4-evidence.txt}" +HOLDER="gate4-holder" +HOLDER_RUNTIME="${HOLDER_RUNTIME:-runc}" +JOB_RUNTIME="${JOB_RUNTIME:-runsc}" + +exec > >(tee "${OUT}") 2>&1 +fail=0 + +echo "=== gate4: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m)" +. /etc/os-release && echo "os=${PRETTY_NAME}" +echo "docker=$(sudo docker version --format '{{.Server.Version}}')" +echo "runsc=$(runsc --version | head -1)" +echo "holder_runtime=${HOLDER_RUNTIME} job_runtime=${JOB_RUNTIME}" +echo "image_digest=$(sudo docker image inspect "${IMAGE}" --format '{{index .RepoDigests 0}}')" +echo "container_git=$(sudo docker run --rm --entrypoint git "${IMAGE}" --version)" +echo "public_repo=${PUBLIC_REPO}" + +cleanup() { + sudo docker rm -f "${HOLDER}" >/dev/null 2>&1 || true + sudo docker network rm "${NET}" >/dev/null 2>&1 || true + pkill -f "git-daemon.*${REMOTE_PORT}" >/dev/null 2>&1 || true + pkill -f "git daemon.*${REMOTE_PORT}" >/dev/null 2>&1 || true + rm -rf "${REMOTE_ROOT}" +} +trap cleanup EXIT +cleanup + +cat > "${RESOLV_FILE}" </dev/null +echo "network=${NET} subnet=${SUBNET} gateway=${GATEWAY}" + +# The CONTROL for the read leg, taken outside the sandbox: what the remote really holds. +echo +echo "=== gate4: control — the remote's real HEAD, read from the VM (outside any container) ===" +CONTROL_HEAD="$(git ls-remote "${PUBLIC_REPO}" HEAD | awk '{print $1}')" +echo "control_head=${CONTROL_HEAD}" +[ -n "${CONTROL_HEAD}" ] || { echo "GATE4: FAIL (no control hash)"; exit 1; } + +# The disposable write remote: a bare repo outside every container, served on the +# gateway address at a port inside the policy's proxy pinhole. +echo +echo "=== gate4: disposable write remote ===" +mkdir -p "${REMOTE_ROOT}" +git init --bare --quiet "${REMOTE_ROOT}/answer.git" +git --git-dir="${REMOTE_ROOT}/answer.git" config http.receivepack true +setsid git daemon --reuseaddr --listen="${GATEWAY}" --port="${REMOTE_PORT}" \ + --base-path="${REMOTE_ROOT}" --export-all --enable=receive-pack \ + >/dev/null 2>&1 /dev/null +echo "holder_started=$?" +sudo timeout 120 docker run --rm --interactive --runtime "${HOLDER_RUNTIME}" \ + --network "container:${HOLDER}" --cap-drop ALL --cap-add NET_ADMIN \ + --security-opt no-new-privileges "${NETFILTER_IMAGE}" < "${PLAN_FILE}" +echo "sidecar_exit=$?" + +# One payload, both legs, run as the job: clone over https, commit, push to the +# remote outside the container, and print the hashes for comparison. +PAYLOAD=' +set -e +export HOME=/tmp GIT_TERMINAL_PROMPT=0 +cd /tmp +echo "--- read leg: clone over https from inside the sandbox ---" +# A FULL clone, not --depth 1: a shallow history cannot be pushed on ("shallow update +# not allowed"), and the write leg is half the gate. The repository is deliberately tiny. +git clone --quiet "$1" work +cd work +echo "delivered_head=$(git rev-parse HEAD)" +echo "--- write leg: commit here, push to the remote outside this container ---" +git config user.email job@sandbox.invalid +git config user.name "sandbox job" +date -u +%s > answer.txt +git add answer.txt +git commit --quiet -m "answer from the sandboxed job" +echo "answer_commit=$(git rev-parse HEAD)" +git push --quiet "$2" HEAD:refs/heads/answer +echo "push_exit=$?" +' + +echo +echo "=== gate4: job container (runsc, non-root, cap-drop ALL) ===" +JOB_OUT="$(sudo timeout 180 docker run --rm --runtime "${JOB_RUNTIME}" \ + --network "container:${HOLDER}" --user 65534:65534 --cap-drop ALL \ + --security-opt no-new-privileges -v "${RESOLV_FILE}:/etc/resolv.conf:ro" \ + --entrypoint sh "${IMAGE}" -c "${PAYLOAD}" gate4 \ + "${PUBLIC_REPO}" "git://${GATEWAY}:${REMOTE_PORT}/answer.git" 2>&1)" +echo "${JOB_OUT}" +echo "job_exit=$?" + +DELIVERED_HEAD="$(printf '%s\n' "${JOB_OUT}" | sed -n 's/^delivered_head=//p')" +ANSWER_COMMIT="$(printf '%s\n' "${JOB_OUT}" | sed -n 's/^answer_commit=//p')" + +echo +echo "=== gate4: verdict ===" +echo "control_head=${CONTROL_HEAD}" +echo "delivered_head=${DELIVERED_HEAD}" +if [ -n "${DELIVERED_HEAD}" ] && [ "${DELIVERED_HEAD}" = "${CONTROL_HEAD}" ]; then + echo "READ: PASS — the sandbox delivered the remote's real HEAD" +else + echo "READ: FAIL — delivered hash does not match the remote's real HEAD" + fail=1 +fi + +# --verify, so a MISSING ref is empty rather than the string "refs/heads/answer" — which +# would otherwise be compared against a hash and merely look like a mismatch. +REMOTE_HASH="$(git --git-dir="${REMOTE_ROOT}/answer.git" rev-parse --verify -q refs/heads/answer 2>/dev/null)" +echo "answer_commit=${ANSWER_COMMIT}" +echo "remote_hash=${REMOTE_HASH}" +if [ -n "${ANSWER_COMMIT}" ] && [ "${ANSWER_COMMIT}" = "${REMOTE_HASH}" ]; then + echo "WRITE: PASS — the commit made inside the sandbox reached the remote, hash matches" +else + echo "WRITE: FAIL — the remote does not hold the container's commit" + fail=1 +fi + +if [ "${fail}" -eq 0 ]; then + echo "GATE4: PASS" +else + echo "GATE4: FAIL" +fi +exit "${fail}" diff --git a/docs/gvisor-dns-delivery/scripts/gate5-denial-and-concurrent-success.sh b/docs/gvisor-dns-delivery/scripts/gate5-denial-and-concurrent-success.sh new file mode 100755 index 00000000..ee045caa --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5-denial-and-concurrent-success.sh @@ -0,0 +1,269 @@ +#!/usr/bin/env bash +# Gate 5: denial holds, and concurrent jobs still deliver. +# +# This is a REWRITE. The first version of this script produced a FAIL that is kept in +# evidence/, and it was unsound in three ways that the sub-experiments (gate5b–gate5f) +# then exposed. All three are fixed here, and naming them is part of the gate: +# +# 1. It probed addresses where NOTHING LISTENED and read the resulting timeouts as +# denial. Absence and enforcement are indistinguishable that way — the same error +# cost gate 2 its "metadata denied" line. Every denial leg below is either aimed at +# a LIVE listener, or reported as NO EVIDENCE unless bare and ruled runs DIFFER. +# 2. It reused one namespace across gVisor probes. The namespace is single-use for +# gVisor: runsc takes the addresses into its netstack and never gives them back, so +# every later leg ran in a namespace with `lo` only and "passed" by being broken. +# One gVisor container per namespace here, with a health check beside every leg. +# 3. It double-counted a request timeout and scored a SUCCESS as PUBLIC-FAIL. +# +# The rules under test are rendered by the PRODUCT (`--example render_net_plan` and +# `--example render_host_plan`), never transcribed here. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +PLAN_DIR="${PLAN_DIR:-$HOME/gate5-plans}" +RESOLVER="${RESOLVER:-1.1.1.1}" +HOST_LAN="${HOST_LAN:-192.168.5.15}" +HOST_PORT="${HOST_PORT:-49254}" +PUBLIC_REPO="${PUBLIC_REPO:-https://github.com/octocat/Hello-World.git}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate5-resolv.conf}" +HOSTS_FILE="${HOSTS_FILE:-$HOME/gate5-hosts}" +OUT="${OUT:-$HOME/gate5-evidence.txt}" +DENIED_NAME="denied-lan.maxplayer.test" + +exec > >(tee "${OUT}") 2>&1 +CUR_NS="" +NS_SEQ=0 +declare -a RULES_UP=() +FAILURES=0 +note_fail() { FAILURES=$((FAILURES + 1)); } + +for tag in d n c1 c2 c3; do + for f in "plan-${tag}.txt" "host-${tag}.txt" "host-${tag}-teardown.txt"; do + [ -s "${PLAN_DIR}/${f}" ] || { echo "MISSING rendered plan ${PLAN_DIR}/${f}"; exit 2; } + done +done + +# tag -> network, subnet, gateway, pinned address +net_of() { echo "maxplayer-dns-gate5-$1"; } +subnet_of() { case "$1" in d) echo 172.31.30.0/24;; n) echo 172.31.34.0/24;; c1) echo 172.31.31.0/24;; c2) echo 172.31.32.0/24;; c3) echo 172.31.33.0/24;; esac; } +gw_of() { case "$1" in d) echo 172.31.30.1;; n) echo 172.31.34.1;; c1) echo 172.31.31.1;; c2) echo 172.31.32.1;; c3) echo 172.31.33.1;; esac; } +addr_of() { case "$1" in d) echo 172.31.30.10;; n) echo 172.31.34.10;; c1) echo 172.31.31.10;; c2) echo 172.31.32.10;; c3) echo 172.31.33.10;; esac; } + +# The stale-plan guard from gate5f: a host plan keyed to the wrong address denies some +# other container and leaves this job open, and nothing in the run would show it. +for tag in d n c1 c2 c3; do + grep -q -- "-s $(addr_of "${tag}")/32" "${PLAN_DIR}/host-${tag}.txt" || + { echo "REFUSING TO RUN: host-${tag}.txt is not keyed to $(addr_of "${tag}")"; exit 2; } +done + +echo "=== gate5: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) docker=$(sudo docker version --format '{{.Server.Version}}') runsc=$(runsc --version | head -1)" +echo "docker default-runtime=$(sudo docker info --format '{{.DefaultRuntime}}') · br_netfilter=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables 2>/dev/null || echo absent)" +BASE_DU="$(sudo iptables -S DOCKER-USER | wc -l)"; BASE_IN="$(sudo iptables -S INPUT | wc -l)" +echo "chain depth before anything: DOCKER-USER=${BASE_DU} INPUT=${BASE_IN}" + +apply_host() { # tag install|teardown + local tag="$1" verb="$2" file="${PLAN_DIR}/host-$1.txt" + [ "${verb}" = teardown ] && file="${PLAN_DIR}/host-$1-teardown.txt" + local applied expected + applied="$(sudo timeout 120 docker run --rm --interactive --network host \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${file}" 2>&1 | tr -d '\r' | tail -1)" + expected="$(wc -l < "${file}" | tr -d ' ')" + local landed; landed="$(( $(sudo iptables -S DOCKER-USER | grep -c -- "-s $(addr_of "${tag}")/32") + $(sudo iptables -S INPUT | grep -c -- "-s $(addr_of "${tag}")/32") ))" + echo " host policy ${verb} for $(addr_of "${tag}"): applier=${applied}/${expected}, host kernel now carries ${landed}" + if [ "${verb}" = install ]; then RULES_UP+=("${tag}"); else + local keep=(); local t; for t in "${RULES_UP[@]:-}"; do [ -n "${t}" ] && [ "${t}" != "${tag}" ] && keep+=("${t}"); done + RULES_UP=("${keep[@]:-}") + fi +} +drop_ns() { [ -n "${CUR_NS}" ] && { sudo docker rm -f "${CUR_NS}" >/dev/null 2>&1; CUR_NS=""; }; return 0; } +cleanup() { + drop_ns + local t; for t in "${RULES_UP[@]:-}"; do [ -n "${t}" ] && apply_host "${t}" teardown >/dev/null 2>&1; done + sudo docker rm -f gate5-neigh-holder gate5-neigh $(sudo docker ps -aq --filter "name=gate5-") >/dev/null 2>&1 || true + for t in d n c1 c2 c3; do sudo docker network rm "$(net_of "${t}")" >/dev/null 2>&1; done + pkill -f "gate5-host-listener" >/dev/null 2>&1 || true +} +trap cleanup EXIT +cleanup + +rm -f "${RESOLV_FILE}" "${HOSTS_FILE}" +printf 'nameserver %s\noptions timeout:2 attempts:2\n' "${RESOLVER}" > "${RESOLV_FILE}" +# A name that resolves INTO a denied range, without depending on a third-party wildcard +# DNS service being up. The point of the leg is that reaching a denied address BY NAME is +# denied too, which no earlier gate measured. +printf '127.0.0.1 localhost\n%s %s\n' "${HOST_LAN}" "${DENIED_NAME}" > "${HOSTS_FILE}" +chmod 0444 "${RESOLV_FILE}" "${HOSTS_FILE}" + +for t in d n c1 c2 c3; do + sudo docker network create --subnet "$(subnet_of "${t}")" --gateway "$(gw_of "${t}")" "$(net_of "${t}")" >/dev/null +done + +setsid python3 -c " +import socket +s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) +s.bind(('0.0.0.0',${HOST_PORT})); s.listen(16) # gate5-host-listener +while True: + c,_=s.accept(); c.sendall(b'REACHED'); c.close() +" >/dev/null 2>&1 /dev/null +sudo timeout 60 docker run --detach --name gate5-neigh --network "container:gate5-neigh-holder" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges --entrypoint node "${IMAGE}" \ + -e "require('http').createServer((q,r)=>r.end('REACHED')).listen(8080,'0.0.0.0')" >/dev/null +sleep 3 +echo "live listeners: host ${HOST_LAN}:${HOST_PORT} · neighbour job $(addr_of n):8080 · name ${DENIED_NAME} -> ${HOST_LAN}" + +in_ns() { # runtime script args... + local rt="$1"; shift; local script="$1"; shift + sudo timeout 180 docker run --rm --runtime "${rt}" --network "container:${CUR_NS}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" -v "${HOSTS_FILE}:/etc/hosts:ro" \ + --entrypoint node "${IMAGE}" -e "${script}" "$@" 2>&1 | tr -d '\r' | tail -1 +} +HEALTH=' +const os=require("os"),dns=require("dns"); +const v4=Object.values(os.networkInterfaces()).flat().filter(x=>x&&x.family==="IPv4"&&!x.internal); +if(!v4.length){console.log("SICK no-address");process.exit(0);} +dns.lookup("relay.maxplayer.ai",(e,a)=>console.log(e?("SICK dns-"+e.code):("HEALTHY "+v4[0].address))); +' +PROBE=' +const net=require("net"); +const s=net.connect({host:process.argv[1],port:Number(process.argv[2]),timeout:8000}); +let said=false; const say=(w)=>{ if(!said){said=true;console.log(w);} s.destroy(); }; +s.on("connect",()=>say("REACHED")); +s.on("timeout",()=>say("timeout")); +s.on("error",(e)=>say(e.code)); +' +V6=' +const os=require("os"); +const v6=Object.values(os.networkInterfaces()).flat().filter(x=>x&&x.family==="IPv6"&&!x.internal); +console.log(v6.length?("HAS-V6 "+v6.map(x=>x.address).join(",")):"NO-V6"); +' +fresh_ns() { # tag + drop_ns + NS_SEQ=$((NS_SEQ + 1)); CUR_NS="gate5-ns-$1-${NS_SEQ}" + sudo timeout 120 docker run --detach --name "${CUR_NS}" --network "$(net_of "$1")" \ + --ip "$(addr_of "$1")" --read-only --cap-drop ALL --security-opt no-new-privileges \ + --user 65534:65534 --entrypoint sleep "${IMAGE}" infinity >/dev/null + sudo timeout 120 docker run --rm --interactive --network "container:${CUR_NS}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${PLAN_DIR}/plan-$1.txt" >/dev/null +} +probe_leg() { # label expect host port + local label="$1" expect="$2" host="$3" port="$4" + fresh_ns d + local h; h="$(in_ns runc "${HEALTH}")" + case "${h}" in + HEALTHY*) ;; + *) echo " ${label}: UNSOUND — ${h}"; note_fail; return ;; + esac + local got; got="$(in_ns runsc "${PROBE}" "${host}" "${port}")" + if [ "${expect}" = denied ]; then + case "${got}" in + REACHED) echo " ${label}: ${got} *** FAIL, expected denial ***"; note_fail ;; + *) echo " ${label}: ${got} (denied)" ;; + esac + else + echo " ${label}: ${got}" + fi +} + +echo +echo "=== gate5: 1. denial, BEFORE the host policy (so each leg has a baseline) ===" +probe_leg "runsc -> host ${HOST_LAN}:${HOST_PORT} (live)" measure "${HOST_LAN}" "${HOST_PORT}" +probe_leg "runsc -> ${DENIED_NAME}:${HOST_PORT} (live, by name)" measure "${DENIED_NAME}" "${HOST_PORT}" +probe_leg "runsc -> neighbour job $(addr_of n):8080 (live)" measure "$(addr_of n)" 8080 +probe_leg "runsc -> 169.254.169.254:80 (nothing listens)" measure 169.254.169.254 80 + +echo +echo "=== gate5: 2. the product's host policy, installed for the probe namespace ===" +apply_host d install + +echo +echo "=== gate5: 3. denial, AFTER — every one of these MUST be denied ===" +probe_leg "runsc -> host ${HOST_LAN}:${HOST_PORT} (live)" denied "${HOST_LAN}" "${HOST_PORT}" +probe_leg "runsc -> ${DENIED_NAME}:${HOST_PORT} (live, by name)" denied "${DENIED_NAME}" "${HOST_PORT}" +probe_leg "runsc -> neighbour job $(addr_of n):8080 (live)" denied "$(addr_of n)" 8080 +probe_leg "runsc -> 169.254.169.254:80 (nothing listens)" denied 169.254.169.254 80 +echo " ^ read the metadata leg ONLY as the difference against section 1; with no listener," +echo " an identical result in both sections is NO EVIDENCE either way." + +echo +echo "=== gate5: 4. IPv6, measured rather than assumed ===" +fresh_ns d +echo " namespace v6: $(in_ns runc "${V6}")" +echo " runsc v6: $(in_ns runsc "${V6}")" +echo " The host-side plan renders NO ip6tables rules (DOCKER-USER may not exist there)." +echo " Where the job namespace has no global IPv6 there is nothing to deny; where it has," +echo " host-side v6 denial for a runsc job is UNPROVEN and must not be claimed." + +echo +echo "=== gate5: 5. concurrent delivery, three jobs at once, all policies installed ===" +for t in c1 c2 c3; do apply_host "${t}" install; done +declare -a PIDS=() +for t in c1 c2 c3; do + ( + ns="gate5-ns-${t}" + sudo timeout 120 docker run --detach --name "${ns}" --network "$(net_of "${t}")" \ + --ip "$(addr_of "${t}")" --read-only --cap-drop ALL --security-opt no-new-privileges \ + --user 65534:65534 --entrypoint sleep "${IMAGE}" infinity >/dev/null + sudo timeout 120 docker run --rm --interactive --network "container:${ns}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${PLAN_DIR}/plan-${t}.txt" >/dev/null + # ONE gVisor container in this namespace, doing all three things, because a second one + # would find the namespace already emptied into the first one's netstack. + out="$(sudo timeout 240 docker run --rm --runtime runsc --network "container:${ns}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint sh "${IMAGE}" -c ' + export HOME=/tmp GIT_TERMINAL_PROMPT=0 + node -e '"'"' + const dns=require("dns"),https=require("https"); + dns.lookup("relay.maxplayer.ai",(e,a)=>{ + if(e){console.log("FAIL dns "+e.code);process.exit(0);} + let done=false; const say=(w)=>{if(!done){done=true;console.log(w);}}; + const r=https.request({host:"relay.maxplayer.ai",port:443,path:"/",method:"HEAD",timeout:20000},(res)=>{ + say((res.socket.authorized?"OK":"FAIL")+" dns="+a+" tls="+res.statusCode+" verified="+res.socket.authorized); + res.resume(); r.destroy(); + }); + r.on("timeout",()=>{say("FAIL tls timeout");r.destroy();}); + r.on("error",(x)=>say("FAIL tls "+x.code)); + r.end(); + }); + '"'"' + cd /tmp && git clone --quiet "$1" repo >/dev/null 2>&1 && + echo "OK git $(git -C repo rev-parse HEAD)" || echo "FAIL git" + ' gate5 "${PUBLIC_REPO}" 2>&1 | tr -d '\r')" + echo " ${t}: $(echo "${out}" | paste -sd' | ' -)" + sudo docker rm -f "${ns}" >/dev/null 2>&1 + ) & + PIDS+=($!) +done +for p in "${PIDS[@]}"; do wait "${p}"; done +for t in c1 c2 c3; do apply_host "${t}" teardown; done + +echo +echo "=== gate5: 6. teardown leaves the shared chains as it found them ===" +drop_ns +apply_host d teardown +AFTER_DU="$(sudo iptables -S DOCKER-USER | wc -l)"; AFTER_IN="$(sudo iptables -S INPUT | wc -l)" +echo " DOCKER-USER=${AFTER_DU} (was ${BASE_DU}) · INPUT=${AFTER_IN} (was ${BASE_IN})" +if [ "${AFTER_DU}" = "${BASE_DU}" ] && [ "${AFTER_IN}" = "${BASE_IN}" ]; then + echo " CLEAN" +else + echo " LEAKED"; note_fail +fi + +echo +echo "=== gate5: verdict ===" +echo "denial legs failing: ${FAILURES}" +echo "A concurrent line counts as delivery only if it reads OK dns=… tls=200 verified=true AND OK git ." +[ "${FAILURES}" -eq 0 ] && echo "GATE5-DENIAL: PASS" || echo "GATE5-DENIAL: FAIL" diff --git a/docs/gvisor-dns-delivery/scripts/gate5b-does-the-plan-bind-a-runsc-job.sh b/docs/gvisor-dns-delivery/scripts/gate5b-does-the-plan-bind-a-runsc-job.sh new file mode 100755 index 00000000..960f1a61 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5b-does-the-plan-bind-a-runsc-job.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# Gate 5b: does the egress plan actually BIND a gVisor job? +# +# Gate 5 turned up something that cannot be waved away: from a runsc job inside a +# namespace carrying the full 26-rule plan, a container at 172.31.x.x — squarely +# inside the `172.16.0.0/12 -j DROP` rule — was REACHED. +# +# Every other "denial" in gate 5 was a timeout or a refusal to an address where +# NOTHING IS LISTENING. Absence looks exactly like enforcement. So the only honest +# test is a destination that is (a) inside a DROPped range and (b) has a real +# listener, approached from the same namespace by two runtimes: +# +# runsc job -> neighbour vs runc job -> neighbour +# +# If runc is dropped and runsc gets through, the plan is not binding the gVisor job +# and the containment story for this branch is wrong as written. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +NET="${NET:-maxplayer-dns-gate5b}" +SUBNET="${SUBNET:-172.31.13.0/24}" +GATEWAY="${GATEWAY:-172.31.13.1}" +RESOLVER="${RESOLVER:-1.1.1.1}" +PLAN_FILE="${PLAN_FILE:-$HOME/gate5b-plan.txt}" +OUT="${OUT:-$HOME/gate5b-evidence.txt}" +HOLDER_A="gate5b-holder-a" +HOLDER_B="gate5b-holder-b" +NEIGHBOUR="gate5b-neighbour" + +exec > >(tee "${OUT}") 2>&1 + +echo "=== gate5b: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m)" +echo "docker=$(sudo docker version --format '{{.Server.Version}}') runsc=$(runsc --version | head -1)" + +cleanup() { + sudo docker rm -f "${HOLDER_A}" "${HOLDER_B}" "${NEIGHBOUR}" >/dev/null 2>&1 || true + sudo docker network rm "${NET}" >/dev/null 2>&1 || true +} +trap cleanup EXIT +cleanup + +sudo docker network create --subnet "${SUBNET}" --gateway "${GATEWAY}" "${NET}" >/dev/null +echo "network=${NET} subnet=${SUBNET} plan_rules=$(grep -c . "${PLAN_FILE}")" + +for holder in "${HOLDER_A}" "${HOLDER_B}"; do + sudo timeout 120 docker run --detach --name "${holder}" --runtime runc --network "${NET}" \ + --read-only --cap-drop ALL --security-opt no-new-privileges --user 65534:65534 \ + --entrypoint sleep "${IMAGE}" infinity >/dev/null +done +# Only namespace A carries the plan; B is just where the neighbour lives. +sudo timeout 120 docker run --rm --interactive --runtime runc \ + --network "container:${HOLDER_A}" --cap-drop ALL --cap-add NET_ADMIN \ + --security-opt no-new-privileges "${NETFILTER_IMAGE}" < "${PLAN_FILE}" +echo "plan_applied_rules_reported=$?" + +B_ADDR="$(sudo docker inspect "${HOLDER_B}" --format "{{(index .NetworkSettings.Networks \"${NET}\").IPAddress}}")" +sudo timeout 60 docker run --detach --name "${NEIGHBOUR}" --runtime runc \ + --network "container:${HOLDER_B}" --user 65534:65534 --cap-drop ALL \ + --security-opt no-new-privileges --entrypoint node "${IMAGE}" \ + -e "require('http').createServer((q,r)=>r.end('REACHED')).listen(8080,'0.0.0.0')" >/dev/null +sleep 3 +echo "neighbour=${B_ADDR}:8080 running=$(sudo docker inspect -f '{{.State.Running}}' "${NEIGHBOUR}")" +echo "neighbour_is_inside_a_dropped_range=172.16.0.0/12" + +PROBE=' +const net=require("net"); +const s=net.connect({host:process.argv[1],port:8080,timeout:8000}); +s.on("connect",()=>{console.log("RESULT=REACHED");s.destroy();}); +s.on("timeout",()=>{console.log("RESULT=timeout");s.destroy();}); +s.on("error",(e)=>{console.log("RESULT="+e.code);s.destroy();}); +' + +echo +echo "=== gate5b: the same probe, the same namespace, two runtimes ===" +for rt in runc runsc; do + r="$(sudo timeout 60 docker run --rm --runtime "${rt}" --network "container:${HOLDER_A}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + --entrypoint node "${IMAGE}" -e "${PROBE}" "${B_ADDR}" 2>&1 | tr -d '\r')" + echo "job_runtime=${rt} ${r}" + eval "res_${rt}=\"${r}\"" +done + +# Is the rule even visible from inside the namespace? Ask the host runtime, which can see it. +echo +echo "=== gate5b: the rule as installed, read back from the namespace ===" +sudo timeout 60 docker run --rm --runtime runc --network "container:${HOLDER_A}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + --entrypoint iptables "${NETFILTER_IMAGE}" -S OUTPUT 2>&1 | grep -n "172.16.0.0/12" || true + +echo +echo "=== gate5b: verdict ===" +echo "runc_result=${res_runc}" +echo "runsc_result=${res_runsc}" +if [ "${res_runc}" = "RESULT=REACHED" ]; then + echo "INCONCLUSIVE: the plan did not stop the host runtime either — the harness, not the runtime, is wrong" + exit 2 +fi +if [ "${res_runsc}" = "RESULT=REACHED" ]; then + echo "FINDING: the plan binds a runc job and DOES NOT BIND a runsc job." + echo "gVisor's netstack emits packets to the veth itself; the host kernel's OUTPUT chain" + echo "in that netns never sees them, so per-job egress policy is not enforced for the job" + echo "it is written for. Containment for gVisor jobs cannot live in the netns OUTPUT chain." + exit 1 +fi +echo "NO FINDING: both runtimes were denied; gate 5's REACHED was a harness artifact" +exit 0 diff --git a/docs/gvisor-dns-delivery/scripts/gate5c-where-does-containment-bind.sh b/docs/gvisor-dns-delivery/scripts/gate5c-where-does-containment-bind.sh new file mode 100755 index 00000000..f5cdc69a --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5c-where-does-containment-bind.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +# Gate 5c (rewritten): if the netns OUTPUT chain does not bind a gVisor job, what does? +# +# The first version of this script measured a dead namespace and had to be thrown +# away: gate5d showed a runsc container takes the namespace's addresses into its +# netstack and never returns them, so the namespace is usable exactly ONCE and +# every leg after the first gVisor container was reading a corpse. +# +# This version obeys the rule that finding forced: +# * ONE gVisor container per namespace — every probe gets a FRESH holder+plan; +# * a health check immediately before any leg meant to be evidence, and a leg +# whose namespace was already sick is reported UNSOUND, never as a denial; +# * every denial targets a LIVE listener, because a timeout to an address where +# nothing listens is indistinguishable from enforcement. +# +# Candidates, both on the far side of the veth where the host kernel handles the +# packet whatever produced it: +# (a) DOCKER-USER (root-netns FORWARD path) keyed to the namespace's address; +# (b) a per-job network instead of the one shared bridge all jobs share today. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +NET_A="${NET_A:-maxplayer-dns-gate5c-a}" +NET_B="${NET_B:-maxplayer-dns-gate5c-b}" +SUBNET_A="${SUBNET_A:-172.31.17.0/24}" +GATEWAY_A="${GATEWAY_A:-172.31.17.1}" +SUBNET_B="${SUBNET_B:-172.31.18.0/24}" +GATEWAY_B="${GATEWAY_B:-172.31.18.1}" +RESOLVER="${RESOLVER:-1.1.1.1}" +PUBLIC_REPO="${PUBLIC_REPO:-https://github.com/octocat/Hello-World.git}" +PLAN_FILE="${PLAN_FILE:-$HOME/gate5c-plan.txt}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate5c-resolv.conf}" +OUT="${OUT:-$HOME/gate5c-evidence.txt}" +HOLDER_SAME="gate5c-holder-same" +HOLDER_FAR="gate5c-holder-far" +NEIGH_SAME="gate5c-neighbour-same" +NEIGH_FAR="gate5c-neighbour-far" + +exec > >(tee "${OUT}") 2>&1 +NS_SEQ=0 +CUR_NS="" +CUR_RULE="" + +echo "=== gate5c: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) docker=$(sudo docker version --format '{{.Server.Version}}') runsc=$(runsc --version | head -1)" +echo "br_netfilter=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables 2>/dev/null || echo absent)" +echo "rule: one gVisor container per namespace; health check before every evidential leg" + +drop_ns() { + [ -n "${CUR_RULE}" ] && { sudo iptables -D DOCKER-USER ${CUR_RULE} >/dev/null 2>&1; CUR_RULE=""; } + [ -n "${CUR_NS}" ] && { sudo docker rm -f "${CUR_NS}" >/dev/null 2>&1; CUR_NS=""; } + return 0 +} +cleanup() { + drop_ns + sudo docker rm -f "${HOLDER_SAME}" "${HOLDER_FAR}" "${NEIGH_SAME}" "${NEIGH_FAR}" >/dev/null 2>&1 || true + sudo docker network rm "${NET_A}" "${NET_B}" >/dev/null 2>&1 || true + # Belt and braces: any stray rule this script could have left in DOCKER-USER. + while sudo iptables -S DOCKER-USER 2>/dev/null | grep -q -- "-d 172.16.0.0/12 -j DROP"; do + sudo iptables -D DOCKER-USER $(sudo iptables -S DOCKER-USER | grep -m1 -- "-d 172.16.0.0/12 -j DROP" | sed 's/^-A DOCKER-USER //') || break + done +} +trap cleanup EXIT +cleanup + +printf 'nameserver %s\noptions timeout:2 attempts:2\n' "${RESOLVER}" > "${RESOLV_FILE}" +chmod 0444 "${RESOLV_FILE}" +sudo docker network create --subnet "${SUBNET_A}" --gateway "${GATEWAY_A}" "${NET_A}" >/dev/null +sudo docker network create --subnet "${SUBNET_B}" --gateway "${GATEWAY_B}" "${NET_B}" >/dev/null + +holder_on() { # name network + sudo timeout 120 docker run --detach --name "$1" --runtime runc --network "$2" \ + --read-only --cap-drop ALL --security-opt no-new-privileges --user 65534:65534 \ + --entrypoint sleep "${IMAGE}" infinity >/dev/null +} +addr_of() { sudo docker inspect "$1" --format "{{(index .NetworkSettings.Networks \"$2\").IPAddress}}"; } + +# The live neighbours. Both are runc-only, so no gVisor container ever enters their +# namespaces and they stay alive for the whole run. +holder_on "${HOLDER_SAME}" "${NET_A}" +holder_on "${HOLDER_FAR}" "${NET_B}" +for pair in "${NEIGH_SAME} ${HOLDER_SAME}" "${NEIGH_FAR} ${HOLDER_FAR}"; do + set -- ${pair} + sudo timeout 60 docker run --detach --name "$1" --runtime runc --network "container:$2" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + --entrypoint node "${IMAGE}" \ + -e "require('http').createServer((q,r)=>r.end('REACHED')).listen(8080,'0.0.0.0')" >/dev/null +done +sleep 3 +SAME_ADDR="$(addr_of "${HOLDER_SAME}" "${NET_A}")" +FAR_ADDR="$(addr_of "${HOLDER_FAR}" "${NET_B}")" +echo "live neighbours: same_bridge=${SAME_ADDR}:8080 other_bridge=${FAR_ADDR}:8080" + +in_ns() { # runtime script args... + local rt="$1"; shift + local script="$1"; shift + sudo timeout 180 docker run --rm --runtime "${rt}" --network "container:${CUR_NS}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint node "${IMAGE}" \ + -e "${script}" "$@" 2>&1 | tr -d '\r' | tail -1 +} + +HEALTH=' +const os=require("os"),dns=require("dns"); +const v4=Object.values(os.networkInterfaces()).flat().filter(x=>x&&x.family==="IPv4"&&!x.internal); +if(!v4.length){console.log("SICK no-address");process.exit(0);} +dns.lookup("relay.maxplayer.ai",(e,a)=>console.log(e?("SICK dns-"+e.code):("HEALTHY "+v4[0].address+" dns="+a))); +' +PROBE=' +const net=require("net"); +const s=net.connect({host:process.argv[1],port:Number(process.argv[2]),timeout:8000}); +let said=false; const say=(w)=>{ if(!said){said=true;console.log(w);} s.destroy(); }; +s.on("connect",()=>say("REACHED")); +s.on("timeout",()=>say("timeout")); +s.on("error",(e)=>say(e.code)); +' + +fresh_ns() { # with_rule("rule"|"") + drop_ns + NS_SEQ=$((NS_SEQ + 1)) + CUR_NS="gate5c-ns-${NS_SEQ}" + holder_on "${CUR_NS}" "${NET_A}" + sudo timeout 120 docker run --rm --interactive --runtime runc \ + --network "container:${CUR_NS}" --cap-drop ALL --cap-add NET_ADMIN \ + --security-opt no-new-privileges "${NETFILTER_IMAGE}" < "${PLAN_FILE}" >/dev/null + NS_ADDR="$(addr_of "${CUR_NS}" "${NET_A}")" + if [ "$1" = "rule" ]; then + CUR_RULE="-s ${NS_ADDR}/32 -d 172.16.0.0/12 -j DROP" + sudo iptables -I DOCKER-USER ${CUR_RULE} + fi +} + +leg() { # label with_rule runtime host port + fresh_ns "$2" + local h; h="$(in_ns runc "${HEALTH}")" + case "${h}" in + HEALTHY*) echo " ${1}: $(in_ns "$3" "${PROBE}" "$4" "$5") [ns=${NS_ADDR} rule=${CUR_RULE:-none} health=${h}]" ;; + *) echo " ${1}: UNSOUND — namespace was already sick before the probe (${h})" ;; + esac +} + +echo +echo "=== gate5c: baseline — today's behaviour, no host-side rule ===" +leg "runc -> live neighbour, same bridge" "" runc "${SAME_ADDR}" 8080 +leg "runsc -> live neighbour, same bridge" "" runsc "${SAME_ADDR}" 8080 + +echo +echo "=== gate5c: (b) per-job network — the neighbour is on a DIFFERENT bridge ===" +leg "runsc -> live neighbour, other bridge" "" runsc "${FAR_ADDR}" 8080 + +echo +echo "=== gate5c: (a) DOCKER-USER keyed to the namespace's own address ===" +leg "runc -> live neighbour, same bridge" rule runc "${SAME_ADDR}" 8080 +leg "runsc -> live neighbour, same bridge" rule runsc "${SAME_ADDR}" 8080 + +echo +echo "=== gate5c: does (a) cost the public route the job must keep? ===" +fresh_ns rule +H="$(in_ns runc "${HEALTH}")" +echo " health before the public leg: ${H}" +in_ns runsc ' +const dns=require("dns"),https=require("https"); +dns.lookup("relay.maxplayer.ai",(e,a)=>{ + if(e) return console.log("PUBLIC-FAIL dns "+e.code); + const r=https.request({host:"relay.maxplayer.ai",port:443,path:"/",method:"HEAD",timeout:15000},(res)=>{ + console.log((res.socket.authorized?"PUBLIC-PASS":"PUBLIC-FAIL")+" dns="+a+" tls="+res.statusCode+" verified="+res.socket.authorized); + res.resume(); r.destroy(); + }); + r.on("timeout",()=>{console.log("PUBLIC-FAIL tls timeout");r.destroy();}); + r.on("error",(x)=>console.log("PUBLIC-FAIL tls "+x.code)); + r.end(); +}); +' | sed 's/^/ /' + +fresh_ns rule +echo " health before the git leg: $(in_ns runc "${HEALTH}")" +sudo timeout 180 docker run --rm --runtime runsc --network "container:${CUR_NS}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint sh "${IMAGE}" -c ' + export HOME=/tmp GIT_TERMINAL_PROMPT=0 + cd /tmp && git clone --quiet "$1" repo 2>&1 && echo "PUBLIC-PASS git $(git -C repo rev-parse HEAD)" + ' gate5c "${PUBLIC_REPO}" 2>&1 | tr -d '\r' | sed 's/^/ /' + +echo +echo "=== gate5c: how to read this ===" +echo "(a) binds runsc only if BOTH rule-legs are denied AND both public legs still pass." +echo "(b) binds runsc only if the other-bridge leg is denied from a HEALTHY namespace." diff --git a/docs/gvisor-dns-delivery/scripts/gate5d-is-the-namespace-single-use.sh b/docs/gvisor-dns-delivery/scripts/gate5d-is-the-namespace-single-use.sh new file mode 100755 index 00000000..ddfce701 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5d-is-the-namespace-single-use.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Gate 5d: does a runsc container leave the shared namespace usable behind it? +# +# gate5c's later legs all returned ENETUNREACH — including the runc CONTROL, which +# had worked minutes earlier in the same namespace, and including DNS to 1.1.1.1 +# which no rule under test touched. A control that dies is not a control, so +# gate5c's verdicts on both candidate enforcement sites are void until this is +# settled. The same signature appeared in gate 5: the first runsc container +# resolved fine, the second could not resolve at all. +# +# The suspicion: runsc's netstack takes the veth's addresses and routes INTO the +# sandbox, and does not put them back when it exits — leaving the namespace +# stripped for whatever runs next. If so, the shared job namespace is SINGLE-USE +# for gVisor, and every multi-container measurement in this branch has to be read +# again with that in mind. +# +# The image has no iproute2, so the namespace is read with node's +# os.networkInterfaces() — the same way the earlier probes read it. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +NET="${NET:-maxplayer-dns-gate5d}" +SUBNET="${SUBNET:-172.31.16.0/24}" +GATEWAY="${GATEWAY:-172.31.16.1}" +RESOLVER="${RESOLVER:-1.1.1.1}" +PLAN_FILE="${PLAN_FILE:-$HOME/gate5d-plan.txt}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate5d-resolv.conf}" +OUT="${OUT:-$HOME/gate5d-evidence.txt}" +HOLDER="gate5d-holder" + +exec > >(tee "${OUT}") 2>&1 + +echo "=== gate5d: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) docker=$(sudo docker version --format '{{.Server.Version}}') runsc=$(runsc --version | head -1)" + +cleanup() { + sudo docker rm -f "${HOLDER}" >/dev/null 2>&1 || true + sudo docker network rm "${NET}" >/dev/null 2>&1 || true +} +trap cleanup EXIT +cleanup + +printf 'nameserver %s\noptions timeout:2 attempts:2\n' "${RESOLVER}" > "${RESOLV_FILE}" +chmod 0444 "${RESOLV_FILE}" +sudo docker network create --subnet "${SUBNET}" --gateway "${GATEWAY}" "${NET}" >/dev/null +sudo timeout 120 docker run --detach --name "${HOLDER}" --runtime runc --network "${NET}" \ + --read-only --cap-drop ALL --security-opt no-new-privileges --user 65534:65534 \ + --entrypoint sleep "${IMAGE}" infinity >/dev/null +sudo timeout 120 docker run --rm --interactive --runtime runc --network "container:${HOLDER}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${PLAN_FILE}" >/dev/null +echo "namespace established, plan applied ($(grep -c . "${PLAN_FILE}") rules)" + +READ_NS=' +const os=require("os"),dns=require("dns"); +const ifs=os.networkInterfaces(); +const v4=Object.entries(ifs).flatMap(([n,a])=>(a||[]).filter(x=>x.family==="IPv4").map(x=>n+"="+x.address)); +console.log("interfaces: "+(v4.join(" ")||"NONE")); +dns.lookup("relay.maxplayer.ai",(e,a)=>console.log("dns: "+(e?e.code:a))); +' +look() { # runtime label + echo " [$2 via $1] $(sudo timeout 60 docker run --rm --runtime "$1" --network "container:${HOLDER}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint node "${IMAGE}" -e "${READ_NS}" 2>&1 | tr -d '\r' | tr '\n' ' ')" +} + +echo +echo "=== gate5d: the namespace before any gVisor container has touched it ===" +look runc "before" + +echo +echo "=== gate5d: one runsc container runs and exits ===" +look runsc "the gVisor job itself" + +echo +echo "=== gate5d: the same namespace afterwards ===" +look runc "after, host runtime" +look runsc "after, a second gVisor job" + +echo +echo "=== gate5d: read it off the two 'after' lines ===" +echo "If 'before' has an eth0 address and 'after' says NONE, the namespace is single-use" +echo "for gVisor, and gate5c's ENETUNREACH verdicts are void — a dead control, not a policy." diff --git a/docs/gvisor-dns-delivery/scripts/gate5e-routed-enforcement-under-runsc.sh b/docs/gvisor-dns-delivery/scripts/gate5e-routed-enforcement-under-runsc.sh new file mode 100755 index 00000000..76c99331 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5e-routed-enforcement-under-runsc.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# Gate 5e: with a per-job network, which host-side chain actually binds a gVisor job? +# +# gate5c settled two things: a same-bridge peer is unreachable to iptables (this host +# has no br_netfilter, so switched frames never enter FORWARD), and a per-job network +# turns cross-job traffic into ROUTED traffic that DOCKER-ISOLATION already drops. +# What is still unmeasured is everything a job reaches by ROUTE rather than by switch: +# the host itself, and the metadata address. +# +# Discipline carried over, because it is what made the last two results trustworthy: +# * a FRESH holder+plan namespace for every probe, ONE gVisor container in each; +# * a health check printed beside every leg, and UNSOUND rather than a verdict if +# the namespace was already sick; +# * live listeners. Where a live listener is impossible (the metadata address), the +# leg is reported as NO EVIDENCE unless bare and ruled runs DIFFER. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +NET_J1="${NET_J1:-maxplayer-dns-gate5e-job1}" +NET_J2="${NET_J2:-maxplayer-dns-gate5e-job2}" +SUBNET_J1="${SUBNET_J1:-172.31.19.0/24}" +GATEWAY_J1="${GATEWAY_J1:-172.31.19.1}" +SUBNET_J2="${SUBNET_J2:-172.31.20.0/24}" +GATEWAY_J2="${GATEWAY_J2:-172.31.20.1}" +RESOLVER="${RESOLVER:-1.1.1.1}" +HOST_LAN="${HOST_LAN:-192.168.5.15}" +HOST_PORT="${HOST_PORT:-49252}" +PUBLIC_REPO="${PUBLIC_REPO:-https://github.com/octocat/Hello-World.git}" +PLAN_FILE="${PLAN_FILE:-$HOME/gate5e-plan.txt}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate5e-resolv.conf}" +OUT="${OUT:-$HOME/gate5e-evidence.txt}" +HOLDER_J2="gate5e-holder-job2" +NEIGH_J2="gate5e-neighbour-job2" + +exec > >(tee "${OUT}") 2>&1 +NS_SEQ=0 +CUR_NS="" +NS_ADDR="" +declare -a ADDED_RULES=() + +echo "=== gate5e: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) docker=$(sudo docker version --format '{{.Server.Version}}') runsc=$(runsc --version | head -1)" +echo "br_netfilter=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables 2>/dev/null || echo absent)" +echo "host_lan=${HOST_LAN} (the VM's own eth0 — host-directed traffic lands in INPUT, not FORWARD)" + +undo_rules() { + local i + for ((i = ${#ADDED_RULES[@]} - 1; i >= 0; i--)); do + local chain="${ADDED_RULES[i]%%|*}" spec="${ADDED_RULES[i]#*|}" + sudo iptables -D "${chain}" ${spec} >/dev/null 2>&1 + done + ADDED_RULES=() +} +drop_ns() { + undo_rules + [ -n "${CUR_NS}" ] && { sudo docker rm -f "${CUR_NS}" >/dev/null 2>&1; CUR_NS=""; } + return 0 +} +cleanup() { + drop_ns + sudo docker rm -f "${HOLDER_J2}" "${NEIGH_J2}" >/dev/null 2>&1 || true + sudo docker network rm "${NET_J1}" "${NET_J2}" >/dev/null 2>&1 || true + pkill -f "gate5e-host-listener" >/dev/null 2>&1 || true +} +trap cleanup EXIT +cleanup + +# chmod 0444 below makes the file unwritable, so a second run of this script fails +# to rewrite it unless it is removed first. That cost a run's worth of noise once. +rm -f "${RESOLV_FILE}" +printf 'nameserver %s\noptions timeout:2 attempts:2\n' "${RESOLVER}" > "${RESOLV_FILE}" +chmod 0444 "${RESOLV_FILE}" + +sudo docker network create --subnet "${SUBNET_J1}" --gateway "${GATEWAY_J1}" "${NET_J1}" >/dev/null +sudo docker network create --subnet "${SUBNET_J2}" --gateway "${GATEWAY_J2}" "${NET_J2}" >/dev/null +echo "per-job networks: job1=${SUBNET_J1} job2=${SUBNET_J2}" + +# A live listener on the VM's own LAN address: a private destination reached by ROUTE, +# which docker isolation does not block, and which no rule blocks yet. +setsid python3 -c " +import socket,sys +s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) +s.bind(('0.0.0.0',${HOST_PORT})); s.listen(8) # gate5e-host-listener +while True: + c,_=s.accept(); c.sendall(b'REACHED'); c.close() +" >/dev/null 2>&1 /dev/null +sudo timeout 60 docker run --detach --name "${NEIGH_J2}" --runtime runc \ + --network "container:${HOLDER_J2}" --user 65534:65534 --cap-drop ALL \ + --security-opt no-new-privileges --entrypoint node "${IMAGE}" \ + -e "require('http').createServer((q,r)=>r.end('REACHED')).listen(8080,'0.0.0.0')" >/dev/null +sleep 3 +J2_ADDR="$(sudo docker inspect "${HOLDER_J2}" --format "{{(index .NetworkSettings.Networks \"${NET_J2}\").IPAddress}}")" +echo "live neighbour in job 2's own namespace: ${J2_ADDR}:8080" + +in_ns() { # runtime script args... + local rt="$1"; shift + local script="$1"; shift + sudo timeout 180 docker run --rm --runtime "${rt}" --network "container:${CUR_NS}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint node "${IMAGE}" \ + -e "${script}" "$@" 2>&1 | tr -d '\r' | tail -1 +} +HEALTH=' +const os=require("os"),dns=require("dns"); +const v4=Object.values(os.networkInterfaces()).flat().filter(x=>x&&x.family==="IPv4"&&!x.internal); +if(!v4.length){console.log("SICK no-address");process.exit(0);} +dns.lookup("relay.maxplayer.ai",(e,a)=>console.log(e?("SICK dns-"+e.code):("HEALTHY "+v4[0].address))); +' +PROBE=' +const net=require("net"); +const s=net.connect({host:process.argv[1],port:Number(process.argv[2]),timeout:8000}); +let said=false; const say=(w)=>{ if(!said){said=true;console.log(w);} s.destroy(); }; +s.on("connect",()=>say("REACHED")); +s.on("timeout",()=>say("timeout")); +s.on("error",(e)=>say(e.code)); +' + +fresh_ns() { # rules... each "CHAIN|spec with %NS% for this namespace's address" + drop_ns + NS_SEQ=$((NS_SEQ + 1)) + CUR_NS="gate5e-ns-${NS_SEQ}" + sudo timeout 120 docker run --detach --name "${CUR_NS}" --runtime runc --network "${NET_J1}" \ + --read-only --cap-drop ALL --security-opt no-new-privileges --user 65534:65534 \ + --entrypoint sleep "${IMAGE}" infinity >/dev/null + sudo timeout 120 docker run --rm --interactive --runtime runc --network "container:${CUR_NS}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${PLAN_FILE}" >/dev/null + NS_ADDR="$(sudo docker inspect "${CUR_NS}" --format "{{(index .NetworkSettings.Networks \"${NET_J1}\").IPAddress}}")" + local r + for r in "$@"; do + [ -z "${r}" ] && continue + local chain="${r%%|*}" spec="${r#*|}" + spec="${spec//%NS%/${NS_ADDR}}" + sudo iptables -I "${chain}" ${spec} + ADDED_RULES+=("${chain}|${spec}") + done +} +leg() { # label runtime host port rules... + local label="$1" rt="$2" host="$3" port="$4"; shift 4 + fresh_ns "$@" + local h; h="$(in_ns runc "${HEALTH}")" + local shown="none"; [ ${#ADDED_RULES[@]} -gt 0 ] && shown="${ADDED_RULES[*]}" + case "${h}" in + HEALTHY*) echo " ${label}: $(in_ns "${rt}" "${PROBE}" "${host}" "${port}") [ns=${NS_ADDR} rules=${shown}]" ;; + *) echo " ${label}: UNSOUND — namespace sick before the probe (${h})" ;; + esac +} + +echo +echo "=== gate5e: 1. the host itself, a routed private destination with a LIVE listener ===" +leg "runc -> ${HOST_LAN}:${HOST_PORT}, bare" runc "${HOST_LAN}" "${HOST_PORT}" +leg "runsc -> ${HOST_LAN}:${HOST_PORT}, bare" runsc "${HOST_LAN}" "${HOST_PORT}" +leg "runsc -> ${HOST_LAN}:${HOST_PORT}, DOCKER-USER" runsc "${HOST_LAN}" "${HOST_PORT}" \ + "DOCKER-USER|-s %NS%/32 -d 192.168.0.0/16 -j DROP" +leg "runsc -> ${HOST_LAN}:${HOST_PORT}, INPUT" runsc "${HOST_LAN}" "${HOST_PORT}" \ + "INPUT|-s %NS%/32 -d 192.168.0.0/16 -j DROP" + +echo +echo "=== gate5e: 2. the metadata address (no listener anywhere — read only the DIFFERENCE) ===" +leg "runsc -> 169.254.169.254:80, bare" runsc "169.254.169.254" 80 +leg "runsc -> 169.254.169.254:80, DOCKER-USER" runsc "169.254.169.254" 80 \ + "DOCKER-USER|-s %NS%/32 -d 169.254.169.254/32 -j DROP" + +echo +echo "=== gate5e: 3. cross-job, each job on its OWN network (the regression proof) ===" +leg "runsc -> job 2's live listener" runsc "${J2_ADDR}" 8080 + +echo +echo "=== gate5e: 4. the public route, with the candidate rules installed ===" +fresh_ns "DOCKER-USER|-s %NS%/32 -d 169.254.169.254/32 -j DROP" \ + "INPUT|-s %NS%/32 -d 192.168.0.0/16 -j DROP" +echo " health: $(in_ns runc "${HEALTH}") rules=${ADDED_RULES[*]}" +in_ns runsc ' +const dns=require("dns"),https=require("https"); +dns.lookup("relay.maxplayer.ai",(e,a)=>{ + if(e) return console.log("PUBLIC-FAIL dns "+e.code); + const r=https.request({host:"relay.maxplayer.ai",port:443,path:"/",method:"HEAD",timeout:15000},(res)=>{ + console.log((res.socket.authorized?"PUBLIC-PASS":"PUBLIC-FAIL")+" dns="+a+" tls="+res.statusCode+" verified="+res.socket.authorized); + res.resume(); r.destroy(); + }); + r.on("timeout",()=>{console.log("PUBLIC-FAIL tls timeout");r.destroy();}); + r.on("error",(x)=>console.log("PUBLIC-FAIL tls "+x.code)); + r.end(); +}); +' | sed 's/^/ /' + +fresh_ns "DOCKER-USER|-s %NS%/32 -d 169.254.169.254/32 -j DROP" \ + "INPUT|-s %NS%/32 -d 192.168.0.0/16 -j DROP" +echo " health: $(in_ns runc "${HEALTH}")" +sudo timeout 180 docker run --rm --runtime runsc --network "container:${CUR_NS}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint sh "${IMAGE}" -c ' + export HOME=/tmp GIT_TERMINAL_PROMPT=0 + cd /tmp && git clone --quiet "$1" repo 2>&1 && echo "PUBLIC-PASS git $(git -C repo rev-parse HEAD)" + ' gate5e "${PUBLIC_REPO}" 2>&1 | tr -d '\r' | sed 's/^/ /' + +echo +echo "=== gate5e: how to read this ===" +echo "leg 1 names the chain that binds a routed, host-directed destination for runsc." +echo "leg 2 is evidence ONLY if bare and ruled differ; identical timeouts prove nothing." +echo "leg 3 must be denied for per-job networks to be the cross-job answer." +echo "leg 4 must pass, or the candidate costs the job the route it exists to have." diff --git a/docs/gvisor-dns-delivery/scripts/gate5f-product-host-rules-bind-runsc.sh b/docs/gvisor-dns-delivery/scripts/gate5f-product-host-rules-bind-runsc.sh new file mode 100755 index 00000000..ed189fc4 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5f-product-host-rules-bind-runsc.sh @@ -0,0 +1,258 @@ +#!/usr/bin/env bash +# Gate 5f: does the PRODUCT's own host-side policy bind a gVisor job? +# +# gate5b found the hole: the namespace OUTPUT plan does not bind a runsc job, because +# gVisor's netstack writes frames straight to the veth and the host OUTPUT chain in that +# namespace only ever sees host sockets. gate5c and gate5e then found where a runsc job +# CAN be bound: routed destinations, in DOCKER-USER, and host-directed ones in INPUT — +# while a same-bridge peer is bound by neither, this host having no br_netfilter. +# +# This is the regression proof for the fix built on those findings, and the rules under +# test are rendered by the product (`--example render_host_plan`, from HostPolicy), never +# transcribed here. A transcription would drift from the policy and this gate would keep +# passing against a firewall the product no longer builds. +# +# They are also installed the way the product installs them: piped into the SAME applier +# image the sidecar uses, in a --network host container, which is itself under test — if +# that image cannot write the root namespace's chains, this gate must fail, not paper over it. +# +# Discipline from gate5c/5d, which is what made those results trustworthy: +# * a FRESH holder+plan namespace per probe, ONE gVisor container in each (the namespace +# is single-use for gVisor: runsc takes the addresses into its netstack and never +# gives them back, which voided an earlier run of gate5c); +# * a health check printed beside every leg, UNSOUND instead of a verdict if it is sick; +# * LIVE listeners. A timeout against an address where nothing listens is NO EVIDENCE: +# absence and enforcement are indistinguishable. That mistake is why part of gate 2's +# evidence was retracted. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +NET_J1="${NET_J1:-maxplayer-dns-gate5f-job1}" +NET_J2="${NET_J2:-maxplayer-dns-gate5f-job2}" +SUBNET_J1="${SUBNET_J1:-172.31.21.0/24}" +GATEWAY_J1="${GATEWAY_J1:-172.31.21.1}" +SUBNET_J2="${SUBNET_J2:-172.31.22.0/24}" +GATEWAY_J2="${GATEWAY_J2:-172.31.22.1}" +# The probe namespace's address is PINNED, because the host plan is rendered ahead of time +# for exactly this source key and a plan keyed to the wrong address would deny some other +# container while leaving this one open. The script refuses to run if they disagree. +JOB_ADDR="${JOB_ADDR:-172.31.21.10}" +SAME_BRIDGE_ADDR="${SAME_BRIDGE_ADDR:-172.31.21.20}" +J2_ADDR="${J2_ADDR:-172.31.22.10}" +RESOLVER="${RESOLVER:-1.1.1.1}" +HOST_LAN="${HOST_LAN:-192.168.5.15}" +HOST_PORT="${HOST_PORT:-49253}" +PUBLIC_REPO="${PUBLIC_REPO:-https://github.com/octocat/Hello-World.git}" +PLAN_FILE="${PLAN_FILE:-$HOME/gate5f-plan.txt}" +HOST_INSTALL="${HOST_INSTALL:-$HOME/gate5f-host-install.txt}" +HOST_TEARDOWN="${HOST_TEARDOWN:-$HOME/gate5f-host-teardown.txt}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate5f-resolv.conf}" +OUT="${OUT:-$HOME/gate5f-evidence.txt}" +HOLDER_SAME="gate5f-holder-same-bridge" +NEIGH_SAME="gate5f-neighbour-same-bridge" +HOLDER_J2="gate5f-holder-job2" +NEIGH_J2="gate5f-neighbour-job2" + +exec > >(tee "${OUT}") 2>&1 +NS_SEQ=0 +CUR_NS="" +HOST_RULES_UP=0 + +for f in "${PLAN_FILE}" "${HOST_INSTALL}" "${HOST_TEARDOWN}"; do + [ -s "${f}" ] || { echo "MISSING rendered plan ${f} — render it on the host first"; exit 2; } +done +# The stale-plan guard. Cheap, and it is the difference between measuring this job's policy +# and measuring a rule that belongs to nothing. +grep -q -- "-s ${JOB_ADDR}/32" "${HOST_INSTALL}" || { + echo "REFUSING TO RUN: ${HOST_INSTALL} carries no -s ${JOB_ADDR}/32 source key"; exit 2; } + +echo "=== gate5f: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) docker=$(sudo docker version --format '{{.Server.Version}}') runsc=$(runsc --version | head -1)" +echo "docker default-runtime=$(sudo docker info --format '{{.DefaultRuntime}}') (helpers inherit this; the JOB always carries --runtime runsc)" +echo "br_netfilter=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables 2>/dev/null || echo absent)" +echo "host plan rendered by the product for job_addr=${JOB_ADDR}: $(wc -l < "${HOST_INSTALL}") rules" + +BASE_DOCKER_USER="$(sudo iptables -S DOCKER-USER | wc -l)" +BASE_INPUT="$(sudo iptables -S INPUT | wc -l)" +echo "chain depth before anything: DOCKER-USER=${BASE_DOCKER_USER} INPUT=${BASE_INPUT}" + +host_rules() { # install|teardown + local file="${HOST_INSTALL}" verb="install" + [ "$1" = teardown ] && { file="${HOST_TEARDOWN}"; verb="teardown"; } + # The product's applier, on the product's plan, in the root namespace. Same image as the + # sidecar: one applier in this design, not two. + local applied + applied="$(sudo timeout 120 docker run --rm --interactive --network host \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${file}" 2>&1 | tr -d '\r' | tail -1)" + local expected; expected="$(wc -l < "${file}" | tr -d ' ')" + echo " host-rule ${verb}: applier reported ${applied}, plan had ${expected} rules" + [ "${verb}" = install ] && HOST_RULES_UP=1 || HOST_RULES_UP=0 + # The applier's own account is not proof it reached the ROOT namespace's chains, so the + # host kernel is asked directly. This readback is the whole reason the leg is trustworthy. + local landed; landed="$(sudo iptables -S DOCKER-USER | grep -c -- "-s ${JOB_ADDR}/32")" + local landed_in; landed_in="$(sudo iptables -S INPUT | grep -c -- "-s ${JOB_ADDR}/32")" + echo " readback from the host kernel: DOCKER-USER carries ${landed}, INPUT carries ${landed_in} rules for ${JOB_ADDR}" +} + +drop_ns() { + [ -n "${CUR_NS}" ] && { sudo docker rm -f "${CUR_NS}" >/dev/null 2>&1; CUR_NS=""; } + return 0 +} +cleanup() { + drop_ns + [ "${HOST_RULES_UP}" = 1 ] && host_rules teardown >/dev/null 2>&1 + sudo docker rm -f "${HOLDER_SAME}" "${NEIGH_SAME}" "${HOLDER_J2}" "${NEIGH_J2}" >/dev/null 2>&1 || true + sudo docker network rm "${NET_J1}" "${NET_J2}" >/dev/null 2>&1 || true + pkill -f "gate5f-host-listener" >/dev/null 2>&1 || true +} +trap cleanup EXIT +cleanup + +rm -f "${RESOLV_FILE}" +printf 'nameserver %s\noptions timeout:2 attempts:2\n' "${RESOLVER}" > "${RESOLV_FILE}" +chmod 0444 "${RESOLV_FILE}" + +sudo docker network create --subnet "${SUBNET_J1}" --gateway "${GATEWAY_J1}" "${NET_J1}" >/dev/null +sudo docker network create --subnet "${SUBNET_J2}" --gateway "${GATEWAY_J2}" "${NET_J2}" >/dev/null + +# A live listener on the VM's own LAN address: private, reached by ROUTE, and the one +# destination in this gate where the INPUT half of the plan can be caught working. +setsid python3 -c " +import socket +s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) +s.bind(('0.0.0.0',${HOST_PORT})); s.listen(8) # gate5f-host-listener +while True: + c,_=s.accept(); c.sendall(b'REACHED'); c.close() +" >/dev/null 2>&1 /dev/null +sudo timeout 60 docker run --detach --name "${NEIGH_SAME}" --network "container:${HOLDER_SAME}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges --entrypoint node "${IMAGE}" \ + -e "require('http').createServer((q,r)=>r.end('REACHED')).listen(8080,'0.0.0.0')" >/dev/null +sudo timeout 120 docker run --detach --name "${HOLDER_J2}" --network "${NET_J2}" \ + --ip "${J2_ADDR}" --read-only --cap-drop ALL --security-opt no-new-privileges \ + --user 65534:65534 --entrypoint sleep "${IMAGE}" infinity >/dev/null +sudo timeout 60 docker run --detach --name "${NEIGH_J2}" --network "container:${HOLDER_J2}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges --entrypoint node "${IMAGE}" \ + -e "require('http').createServer((q,r)=>r.end('REACHED')).listen(8080,'0.0.0.0')" >/dev/null +sleep 3 +echo "live listeners: same-bridge ${SAME_BRIDGE_ADDR}:8080 · own-network ${J2_ADDR}:8080 · host ${HOST_LAN}:${HOST_PORT}" + +in_ns() { # runtime script args... + local rt="$1"; shift + local script="$1"; shift + sudo timeout 180 docker run --rm --runtime "${rt}" --network "container:${CUR_NS}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint node "${IMAGE}" \ + -e "${script}" "$@" 2>&1 | tr -d '\r' | tail -1 +} +HEALTH=' +const os=require("os"),dns=require("dns"); +const v4=Object.values(os.networkInterfaces()).flat().filter(x=>x&&x.family==="IPv4"&&!x.internal); +if(!v4.length){console.log("SICK no-address");process.exit(0);} +dns.lookup("relay.maxplayer.ai",(e,a)=>console.log(e?("SICK dns-"+e.code):("HEALTHY "+v4[0].address))); +' +PROBE=' +const net=require("net"); +const s=net.connect({host:process.argv[1],port:Number(process.argv[2]),timeout:8000}); +let said=false; const say=(w)=>{ if(!said){said=true;console.log(w);} s.destroy(); }; +s.on("connect",()=>say("REACHED")); +s.on("timeout",()=>say("timeout")); +s.on("error",(e)=>say(e.code)); +' + +fresh_ns() { + drop_ns + NS_SEQ=$((NS_SEQ + 1)) + CUR_NS="gate5f-ns-${NS_SEQ}" + sudo timeout 120 docker run --detach --name "${CUR_NS}" --network "${NET_J1}" \ + --ip "${JOB_ADDR}" --read-only --cap-drop ALL --security-opt no-new-privileges \ + --user 65534:65534 --entrypoint sleep "${IMAGE}" infinity >/dev/null + sudo timeout 120 docker run --rm --interactive --network "container:${CUR_NS}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${PLAN_FILE}" >/dev/null + local got; got="$(sudo docker inspect "${CUR_NS}" --format "{{(index .NetworkSettings.Networks \"${NET_J1}\").IPAddress}}")" + [ "${got}" = "${JOB_ADDR}" ] || echo " WARNING: namespace came up as ${got}, not the ${JOB_ADDR} the host plan is keyed to" +} +leg() { # label runtime host port + local label="$1" rt="$2" host="$3" port="$4" + fresh_ns + local h; h="$(in_ns runc "${HEALTH}")" + case "${h}" in + HEALTHY*) echo " ${label}: $(in_ns "${rt}" "${PROBE}" "${host}" "${port}")" ;; + *) echo " ${label}: UNSOUND — namespace sick before the probe (${h})" ;; + esac +} + +echo +echo "=== gate5f: 1. BEFORE — the hole, with no host rules ===" +leg "runsc -> same-bridge neighbour ${SAME_BRIDGE_ADDR}:8080" runsc "${SAME_BRIDGE_ADDR}" 8080 +leg "runsc -> own-network neighbour ${J2_ADDR}:8080" runsc "${J2_ADDR}" 8080 +leg "runsc -> host ${HOST_LAN}:${HOST_PORT}" runsc "${HOST_LAN}" "${HOST_PORT}" +leg "runc -> host ${HOST_LAN}:${HOST_PORT}" runc "${HOST_LAN}" "${HOST_PORT}" + +echo +echo "=== gate5f: 2. installing the product's rendered host policy ===" +host_rules install + +echo +echo "=== gate5f: 3. AFTER — the same probes, unchanged ===" +leg "runsc -> own-network neighbour ${J2_ADDR}:8080 (MUST be denied)" runsc "${J2_ADDR}" 8080 +leg "runsc -> host ${HOST_LAN}:${HOST_PORT} (MUST be denied)" runsc "${HOST_LAN}" "${HOST_PORT}" +leg "runc -> host ${HOST_LAN}:${HOST_PORT} (MUST be denied)" runc "${HOST_LAN}" "${HOST_PORT}" +leg "runsc -> same-bridge neighbour ${SAME_BRIDGE_ADDR}:8080 (known unbindable)" runsc "${SAME_BRIDGE_ADDR}" 8080 + +echo +echo "=== gate5f: 4. the public route must survive the policy ===" +fresh_ns +echo " health: $(in_ns runc "${HEALTH}")" +in_ns runsc ' +const dns=require("dns"),https=require("https"); +dns.lookup("relay.maxplayer.ai",(e,a)=>{ + if(e) return console.log("PUBLIC-FAIL dns "+e.code); + const r=https.request({host:"relay.maxplayer.ai",port:443,path:"/",method:"HEAD",timeout:15000},(res)=>{ + console.log((res.socket.authorized?"PUBLIC-PASS":"PUBLIC-FAIL")+" dns="+a+" tls="+res.statusCode+" verified="+res.socket.authorized); + res.resume(); r.destroy(); + }); + r.on("timeout",()=>{console.log("PUBLIC-FAIL tls timeout");r.destroy();}); + r.on("error",(x)=>console.log("PUBLIC-FAIL tls "+x.code)); + r.end(); +}); +' | sed 's/^/ /' +fresh_ns +echo " health: $(in_ns runc "${HEALTH}")" +sudo timeout 180 docker run --rm --runtime runsc --network "container:${CUR_NS}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint sh "${IMAGE}" -c ' + export HOME=/tmp GIT_TERMINAL_PROMPT=0 + cd /tmp && git clone --quiet "$1" repo 2>&1 && echo "PUBLIC-PASS git $(git -C repo rev-parse HEAD)" + ' gate5f "${PUBLIC_REPO}" 2>&1 | tr -d '\r' | sed 's/^/ /' + +echo +echo "=== gate5f: 5. teardown must leave the shared chains exactly as it found them ===" +drop_ns +host_rules teardown +AFTER_DOCKER_USER="$(sudo iptables -S DOCKER-USER | wc -l)" +AFTER_INPUT="$(sudo iptables -S INPUT | wc -l)" +echo " chain depth after teardown: DOCKER-USER=${AFTER_DOCKER_USER} (was ${BASE_DOCKER_USER}) INPUT=${AFTER_INPUT} (was ${BASE_INPUT})" +if [ "${AFTER_DOCKER_USER}" = "${BASE_DOCKER_USER}" ] && [ "${AFTER_INPUT}" = "${BASE_INPUT}" ]; then + echo " CLEAN — no rule leaked (this is the failure mode the HostRules drop guard exists to prevent)" +else + echo " LEAKED — the shared chains kept rules after teardown" +fi + +echo +echo "=== gate5f: how to read this ===" +echo "The fix is proved by section 3 differing from section 1 on the own-network and host legs." +echo "The same-bridge leg is expected to stay REACHED: switched frames enter no chain on a host" +echo "without br_netfilter, which is WHY the product gives every job its own network rather than" +echo "trying to rule its way out of a shared one. It is measured here so that reason stays evidenced." +echo "Section 4 must pass, or the policy costs the job the delivery route it exists to have." diff --git a/docs/gvisor-dns-delivery/scripts/gate5g-baseline-vulnerability-repro.sh b/docs/gvisor-dns-delivery/scripts/gate5g-baseline-vulnerability-repro.sh new file mode 100755 index 00000000..24607ef8 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5g-baseline-vulnerability-repro.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# Gate 5g: reproduce the containment failure ON THE UNFIXED BASELINE. +# +# The branch claims the netns plan does not bind a gVisor job and that this PREDATES the +# fix. A claim like that is not settled by reading source — a reviewer is entitled to a +# reproduction on the code as it ships today. This is that reproduction, and it is built +# to be unfair to my own claim: +# +# * The rules come from that baseline commit (b45f8651dc9cab5b71c962eadd7b84840f579106), rendered by a throwaway example that +# compiles against BASELINE's NetPolicy — three fields, no dns_resolvers. A renderer +# built against the fix would be reproducing the fix, not the bug. 24 rules. +# * The ARRANGEMENT is baseline's too: ONE shared network for every job, which is what a +# single `[sandbox] network` setting produced, and NO host-side rules, because +# that baseline commit has no HostPolicy at all (grep: 0 occurrences). +# * Every denial leg targets a LIVE listener. A timeout against a dead address would +# prove nothing here, and reading absence as enforcement is the exact error that cost +# gate 2 its metadata line. +# * runc runs the identical probe as the positive control. If the plan failed to bind +# runc too, the finding would be "the plan was never installed", not "runsc escapes". +# +# Runs only in the disposable gvisor-repro VM. It installs no host rules whatsoever, so it +# cannot touch shared infrastructure even by accident. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +NET="${NET:-maxplayer-dns-gate5g-shared}" +SUBNET="${SUBNET:-172.31.40.0/24}" +GATEWAY="${GATEWAY:-172.31.40.1}" +VICTIM_ADDR="${VICTIM_ADDR:-172.31.40.20}" +RESOLVER="${RESOLVER:-1.1.1.1}" +HOST_LAN="${HOST_LAN:-192.168.5.15}" +HOST_PORT="${HOST_PORT:-49255}" +PLAN_FILE="${PLAN_FILE:-$HOME/gate5g-baseline-plan.txt}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate5g-resolv.conf}" +OUT="${OUT:-$HOME/gate5g-evidence.txt}" + +exec > >(tee "${OUT}") 2>&1 +CUR_NS=""; NS_SEQ=0; FINDINGS=0 + +[ -s "${PLAN_FILE}" ] || { echo "MISSING ${PLAN_FILE} — render it from that baseline commit first"; exit 2; } +# Guard: this must be the BASELINE plan. The fix's plan carries port-53 resolver pinholes; +# baseline's does not. If this file has them, the wrong plan was staged and the whole +# reproduction would be meaningless. +if grep -q -- "--dport 53 -d" "${PLAN_FILE}"; then + echo "REFUSING TO RUN: ${PLAN_FILE} carries resolver pinholes — that is the FIXED plan, not baseline"; exit 2 +fi + +echo "=== gate5g: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) docker=$(sudo docker version --format '{{.Server.Version}}') runsc=$(runsc --version | head -1)" +echo "baseline commit=b45f8651dc9cab5b71c962eadd7b84840f579106 (immutable commit, not a moving ref) · plan rules=$(wc -l < "${PLAN_FILE}")" +echo "host rules installed by this script: NONE (that baseline commit has no HostPolicy — grep says 0)" +echo "arrangement: ONE shared network ${SUBNET}, as a single [sandbox] network produced" +echo "chain depth (untouched by this gate): DOCKER-USER=$(sudo iptables -S DOCKER-USER | wc -l) INPUT=$(sudo iptables -S INPUT | wc -l)" + +drop_ns() { [ -n "${CUR_NS}" ] && { sudo docker rm -f "${CUR_NS}" >/dev/null 2>&1; CUR_NS=""; }; return 0; } +cleanup() { + drop_ns + sudo docker rm -f gate5g-victim-holder gate5g-victim $(sudo docker ps -aq --filter "name=gate5g-") >/dev/null 2>&1 || true + sudo docker network rm "${NET}" >/dev/null 2>&1 || true + pkill -f "gate5g-host-listener" >/dev/null 2>&1 || true +} +trap cleanup EXIT +cleanup + +rm -f "${RESOLV_FILE}" +printf 'nameserver %s\noptions timeout:2 attempts:2\n' "${RESOLVER}" > "${RESOLV_FILE}" +chmod 0444 "${RESOLV_FILE}" +sudo docker network create --subnet "${SUBNET}" --gateway "${GATEWAY}" "${NET}" >/dev/null + +setsid python3 -c " +import socket +s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) +s.bind(('0.0.0.0',${HOST_PORT})); s.listen(8) # gate5g-host-listener +while True: + c,_=s.accept(); c.sendall(b'REACHED'); c.close() +" >/dev/null 2>&1 /dev/null +sudo timeout 120 docker run --rm --interactive --network "container:gate5g-victim-holder" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${PLAN_FILE}" >/dev/null +# The victim's listener runs under **runsc**, and that detail decides whether this leg is +# evidence at all. A first run used a runc listener and read `timeout` — which looked like +# containment and was nothing of the kind: the victim carries the baseline plan too, the +# plan DOES bind a runc process, so the victim's own OUTPUT rules dropped its replies to a +# 172.16/12 peer. The probe measured a victim that could not answer, not an attacker that +# could not reach. Under runsc the victim is unbound exactly as the attacker is, which is +# the real arrangement on a baseline seat: every job is a gVisor job. +sudo timeout 60 docker run --detach --name gate5g-victim --runtime runsc \ + --network "container:gate5g-victim-holder" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges --entrypoint node "${IMAGE}" \ + -e "require('http').createServer((q,r)=>r.end('REACHED')).listen(8080,'0.0.0.0')" >/dev/null +sleep 4 +# Proof the victim is actually serving before any conclusion is drawn from a timeout. +VICTIM_UP="$(sudo timeout 60 docker run --rm --network "container:gate5g-victim-holder" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges --entrypoint node "${IMAGE}" \ + -e 'const n=require("net");const s=n.connect({host:"127.0.0.1",port:8080,timeout:5000}); + s.on("connect",()=>{console.log("VICTIM-SERVING");s.destroy();}); + s.on("timeout",()=>{console.log("VICTIM-DEAD timeout");s.destroy();}); + s.on("error",e=>console.log("VICTIM-DEAD "+e.code));' 2>&1 | tr -d '\r' | tail -1)" +echo "victim liveness (from inside its own namespace): ${VICTIM_UP}" +echo "live listeners: victim job ${VICTIM_ADDR}:8080 (inside 172.16.0.0/12, which the baseline plan DROPs) · host ${HOST_LAN}:${HOST_PORT}" + +in_ns() { local rt="$1"; shift; local script="$1"; shift + sudo timeout 180 docker run --rm --runtime "${rt}" --network "container:${CUR_NS}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint node "${IMAGE}" -e "${script}" "$@" 2>&1 | tr -d '\r' | tail -1 +} +HEALTH=' +const os=require("os"); +const v4=Object.values(os.networkInterfaces()).flat().filter(x=>x&&x.family==="IPv4"&&!x.internal); +console.log(v4.length?("HEALTHY "+v4[0].address):"SICK no-address"); +' +PROBE=' +const net=require("net"); +const s=net.connect({host:process.argv[1],port:Number(process.argv[2]),timeout:8000}); +let said=false; const say=(w)=>{ if(!said){said=true;console.log(w);} s.destroy(); }; +s.on("connect",()=>say("REACHED")); +s.on("timeout",()=>say("timeout")); +s.on("error",(e)=>say(e.code)); +' +# A fresh attacker namespace carrying the baseline plan, verified installed before use. +fresh_ns() { + drop_ns + NS_SEQ=$((NS_SEQ+1)); CUR_NS="gate5g-attacker-${NS_SEQ}" + sudo timeout 120 docker run --detach --name "${CUR_NS}" --network "${NET}" \ + --read-only --cap-drop ALL --security-opt no-new-privileges --user 65534:65534 \ + --entrypoint sleep "${IMAGE}" infinity >/dev/null + local applied + applied="$(sudo timeout 120 docker run --rm --interactive --network "container:${CUR_NS}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "${PLAN_FILE}" 2>&1 | tr -d '\r' | tail -1)" + # Readback: the plan must be IN FORCE, or a REACHED below would only mean "no rules". + local readback + readback="$(sudo timeout 60 docker run --rm --network "container:${CUR_NS}" \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + --entrypoint iptables "${NETFILTER_IMAGE}" -S 2>&1 | grep -c "172.16.0.0/12")" + echo " [ns ${CUR_NS}: applier=${applied}/$(wc -l < "${PLAN_FILE}") rules, kernel readback shows ${readback} rule(s) for 172.16.0.0/12]" +} +leg() { # label runtime host port + local label="$1" rt="$2" host="$3" port="$4" + fresh_ns + local h; h="$(in_ns runc "${HEALTH}")" + case "${h}" in HEALTHY*) ;; *) echo " ${label}: UNSOUND — ${h}"; return ;; esac + local got; got="$(in_ns "${rt}" "${PROBE}" "${host}" "${port}")" + echo " ${label}: ${got}" + [ "${got}" = REACHED ] && FINDINGS=$((FINDINGS+1)) + return 0 +} + +echo +echo "=== gate5g: 1. cross-job on the baseline shared network, live victim ===" +echo " (valid only if the victim is SERVING above; a dead victim makes every timeout meaningless)" +leg "runc -> victim ${VICTIM_ADDR}:8080 (positive control: the plan MUST bind runc)" runc "${VICTIM_ADDR}" 8080 +leg "runsc -> victim ${VICTIM_ADDR}:8080 (the vulnerability)" runsc "${VICTIM_ADDR}" 8080 + +echo +echo "=== gate5g: 2. private egress to the host itself, live listener ===" +leg "runc -> host ${HOST_LAN}:${HOST_PORT} (positive control)" runc "${HOST_LAN}" "${HOST_PORT}" +leg "runsc -> host ${HOST_LAN}:${HOST_PORT} (the vulnerability)" runsc "${HOST_LAN}" "${HOST_PORT}" + +echo +echo "=== gate5g: verdict ===" +echo "baseline REACHED count (each is a containment failure at that baseline commit): ${FINDINGS}" +if [ "${FINDINGS}" -gt 0 ]; then + echo "BASELINE-VULNERABLE: CONFIRMED on b45f8651dc9cab5b71c962eadd7b84840f579106 — the netns plan binds runc and not runsc," + echo "with no host-side enforcement present to catch it. The branch does not introduce this." +else + echo "BASELINE-VULNERABLE: NOT REPRODUCED — the 'predates the branch' claim must be WITHDRAWN." +fi diff --git a/docs/gvisor-dns-delivery/scripts/gate5h-lifecycle-and-recycled-address.sh b/docs/gvisor-dns-delivery/scripts/gate5h-lifecycle-and-recycled-address.sh new file mode 100644 index 00000000..8901fd60 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5h-lifecycle-and-recycled-address.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# Gate 5h: lifecycle — teardown leaves nothing behind, and a RECYCLED address is safe. +# +# Maxie: "Prove selected enforcement path handles runsc, lifecycle cleanup/recreation and +# fail-closed setup/readiness." Gate 5 proved the rules DENY. It never proved they GO AWAY. +# +# That gap is the dangerous one, and in both directions: +# * A leftover rule keyed to job A's address does not stop existing when A does. Docker +# hands addresses back. The next job to get 172.x.0.2 inherits a firewall written for +# someone else — denied traffic it should be allowed, or worse, an ACCEPT pinhole that +# was A's proxy and is now a stranger's open door. +# * A network that fails to delete wedges the next job with the same id at +# "network already exists", which is a fail-OPEN if anything in the caller shrugs at it. +# +# So this gate refuses to accept "teardown ran" as teardown. It counts rules keyed to the +# address in the real chains before and after, recycles the address deliberately, and makes +# the recycled job prove containment on its OWN rules against a LIVE listener with runc as +# the positive control. +# +# Every rule installed here is rendered by the PRODUCT (`render_host_plan`), never +# transcribed. Runs only in the disposable gvisor-repro VM; the trap removes every rule and +# network it created, and the final check asserts the chains are back to their start depth. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +WT="${WT:-/Users/forge/forge/v2/wt/w-gvisor-dns-delivery-r2}" +PLANDIR="${PLANDIR:-$HOME/gate5h-plans}" +SUBNET="${SUBNET:-172.31.55.0/24}" +HOST_LAN="${HOST_LAN:-192.168.5.15}" +HOST_PORT="${HOST_PORT:-49256}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate5h-resolv.conf}" +OUT="${OUT:-$HOME/gate5h-evidence.txt}" + +exec > >(tee "${OUT}") 2>&1 +FAIL=0 +CREATED_NETS=() +CUR_NS="" +NS_SEQ=0 + +fail() { echo " FAIL: $*"; FAIL=$((FAIL+1)); } +ok() { echo " ok: $*"; } + +# --- rule accounting ------------------------------------------------------------------- +# Counts rules keyed to an address in the two chains the product writes to. This is the +# measurement the whole gate turns on, so it reads the KERNEL, never a script variable. +rules_for() { local addr="$1" + local d i + d=$(sudo iptables -S DOCKER-USER 2>/dev/null | grep -c -- "${addr}") + i=$(sudo iptables -S INPUT 2>/dev/null | grep -c -- "${addr}") + echo $((d + i)) +} +chain_depth() { echo "$(( $(sudo iptables -S DOCKER-USER | wc -l) + $(sudo iptables -S INPUT | wc -l) ))"; } + +apply_plan() { # plan-file -> applies on the HOST netns, exactly as the product's applier does + sudo timeout 120 docker run --rm --interactive --network host \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" < "$1" >/dev/null 2>&1 +} + +drop_ns() { [ -n "${CUR_NS}" ] && { sudo docker rm -f "${CUR_NS}" >/dev/null 2>&1; CUR_NS=""; }; return 0; } + +cleanup() { + drop_ns + sudo docker rm -f $(sudo docker ps -aq --filter "name=gate5h-") >/dev/null 2>&1 || true + for f in "${PLANDIR}"/*-teardown.txt; do [ -f "$f" ] && apply_plan "$f"; done + for n in "${CREATED_NETS[@]:-}"; do [ -n "$n" ] && sudo docker network rm "$n" >/dev/null 2>&1; done + pkill -f "gate5h-host-listener" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +echo "=== gate5h: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) runsc=$(runsc --version | head -1)" +[ -d "${PLANDIR}" ] || { echo "MISSING ${PLANDIR} — render product plans first"; exit 2; } +START_DEPTH="$(chain_depth)" +echo "chain depth at start (DOCKER-USER + INPUT): ${START_DEPTH}" + +printf 'nameserver 1.1.1.1\noptions timeout:2 attempts:2\n' > "${RESOLV_FILE}"; chmod 0444 "${RESOLV_FILE}" + +setsid python3 -c " +import socket +s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) +s.bind(('0.0.0.0',${HOST_PORT})); s.listen(8) # gate5h-host-listener +while True: + c,_=s.accept(); c.sendall(b'REACHED'); c.close() +" >/dev/null 2>&1 prints addr + local net="$1" holder="$2" + sudo docker network create --driver bridge --subnet "${SUBNET}" "${net}" >/dev/null 2>&1 + CREATED_NETS+=("${net}") + sudo timeout 120 docker run --detach --name "${holder}" --network "${net}" \ + --read-only --cap-drop ALL --security-opt no-new-privileges --user 65534:65534 \ + --entrypoint sleep "${IMAGE}" infinity >/dev/null 2>&1 + sudo docker inspect --format "{{(index .NetworkSettings.Networks \"${net}\").IPAddress}}" "${holder}" 2>/dev/null +} + +probe_in() { # runtime holder script args... -> last line + local rt="$1" holder="$2" script="$3"; shift 3 + sudo timeout 180 docker run --rm --runtime "${rt}" --network "container:${holder}" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint node "${IMAGE}" -e "${script}" "$@" 2>&1 | tr -d '\r' | tail -1 +} + +# --------------------------------------------------------------------------------------- +echo +echo "=== gate5h: 1. job A establishes, rules appear, containment holds ===" +NET_A="maxplayer-dns-gate5h-job-alpha" +ADDR_A="$(establish_holder "${NET_A}" gate5h-holder-a)" +[ -n "${ADDR_A}" ] || { echo "FATAL: empty address for A"; exit 2; } +echo " job A address (read from docker inspect): ${ADDR_A}" +BEFORE_A="$(rules_for "${ADDR_A}")" +apply_plan "${PLANDIR}/${ADDR_A}-install.txt" +AFTER_A="$(rules_for "${ADDR_A}")" +echo " rules keyed to ${ADDR_A}: before=${BEFORE_A} after-install=${AFTER_A}" +[ "${AFTER_A}" -gt "${BEFORE_A}" ] && ok "install added ${AFTER_A} host rules" || fail "install added no rules" +H="$(probe_in runc gate5h-holder-a "${HEALTH}")"; echo " namespace health: ${H}" +GOT="$(probe_in runsc gate5h-holder-a "${PROBE}" "${HOST_LAN}" "${HOST_PORT}")" +echo " runsc -> live host ${HOST_LAN}:${HOST_PORT}: ${GOT}" +[ "${GOT}" = REACHED ] && fail "A not contained" || ok "A contained while its rules are installed" + +# --------------------------------------------------------------------------------------- +echo +echo "=== gate5h: 2. job A tears down — rules must be GONE, network must be GONE ===" +sudo docker rm -f gate5h-holder-a >/dev/null 2>&1 +apply_plan "${PLANDIR}/${ADDR_A}-teardown.txt" +LEFT_A="$(rules_for "${ADDR_A}")" +echo " rules keyed to ${ADDR_A} after teardown: ${LEFT_A}" +[ "${LEFT_A}" -eq 0 ] && ok "no leftover host rules" || fail "${LEFT_A} rule(s) survived teardown" +sudo docker network rm "${NET_A}" >/dev/null 2>&1 +sudo docker network inspect "${NET_A}" >/dev/null 2>&1 && fail "network ${NET_A} survived" || ok "network removed" + +# --------------------------------------------------------------------------------------- +echo +echo "=== gate5h: 3. RECYCLED address — job B takes A's old address ===" +NET_B="maxplayer-dns-gate5h-job-bravo" +ADDR_B="$(establish_holder "${NET_B}" gate5h-holder-b)" +echo " job B address: ${ADDR_B} (A's was ${ADDR_A})" +if [ "${ADDR_B}" = "${ADDR_A}" ]; then + ok "address genuinely recycled — this is the case that matters" +else + echo " NOTE: docker handed a different address; recycling not exercised this run" +fi +STALE_B="$(rules_for "${ADDR_B}")" +echo " rules keyed to ${ADDR_B} BEFORE B installs its own: ${STALE_B}" +[ "${STALE_B}" -eq 0 ] && ok "B inherits no stale firewall" || fail "B inherited ${STALE_B} stale rule(s)" +# B must be contained by B's OWN rules, proven against a live listener with a runc control. +GOT_BARE="$(probe_in runsc gate5h-holder-b "${PROBE}" "${HOST_LAN}" "${HOST_PORT}")" +echo " runsc -> live host BEFORE B's rules (bare control, expect REACHED): ${GOT_BARE}" +[ "${GOT_BARE}" = REACHED ] || echo " NOTE: bare leg did not reach; the denial below proves less than intended" +apply_plan "${PLANDIR}/${ADDR_B}-install.txt" +GOT_B="$(probe_in runsc gate5h-holder-b "${PROBE}" "${HOST_LAN}" "${HOST_PORT}")" +echo " runsc -> live host AFTER B's rules: ${GOT_B}" +[ "${GOT_B}" = REACHED ] && fail "recycled job B not contained" || ok "B contained by its own rules" +apply_plan "${PLANDIR}/${ADDR_B}-teardown.txt" +sudo docker rm -f gate5h-holder-b >/dev/null 2>&1 +sudo docker network rm "${NET_B}" >/dev/null 2>&1 + +# --------------------------------------------------------------------------------------- +echo +echo "=== gate5h: 4. RECREATION — the same job id twice must not wedge ===" +NET_R="maxplayer-dns-gate5h-job-repeat" +A1="$(establish_holder "${NET_R}" gate5h-holder-r1)" +echo " first incarnation address: ${A1:-}" +sudo docker rm -f gate5h-holder-r1 >/dev/null 2>&1 +sudo docker network rm "${NET_R}" >/dev/null 2>&1 +A2="$(establish_holder "${NET_R}" gate5h-holder-r2)" +echo " second incarnation address: ${A2:-}" +if [ -n "${A1}" ] && [ -n "${A2}" ]; then ok "same job id came up twice, no 'network already exists' wedge" +else fail "recreation wedged (empty address on one incarnation)"; fi +sudo docker rm -f gate5h-holder-r2 >/dev/null 2>&1 +sudo docker network rm "${NET_R}" >/dev/null 2>&1 + +# --------------------------------------------------------------------------------------- +echo +echo "=== gate5h: verdict ===" +END_DEPTH="$(chain_depth)" +echo "chain depth: start=${START_DEPTH} end=${END_DEPTH}" +[ "${END_DEPTH}" -eq "${START_DEPTH}" ] && ok "chains returned to starting depth — the gate left nothing behind" \ + || fail "chains changed depth ${START_DEPTH} -> ${END_DEPTH}: this gate leaked rules" +echo "failing checks: ${FAIL}" +[ "${FAIL}" -eq 0 ] && echo "GATE 5h: PASS" || echo "GATE 5h: FAIL" diff --git a/docs/gvisor-dns-delivery/scripts/gate5i-fail-closed-live.sh b/docs/gvisor-dns-delivery/scripts/gate5i-fail-closed-live.sh new file mode 100644 index 00000000..1b301d32 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5i-fail-closed-live.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +# Gate 5i: fail-closed — a job must not run on a network the product could not contain. +# +# Maxie: "Prove selected enforcement path handles runsc, lifecycle cleanup/recreation and +# fail-closed setup/readiness." +# +# `establish()` refuses at four points and the rendering tests now lock the plan invariants. +# What neither covers is the claim the code makes about the WORLD, in this comment: +# +# "A truncated stdin applies cleanly and exits 0, so no exit code reveals it; only +# comparing the sidecar's own total against what was rendered does." +# +# If that is wrong — if a truncated plan failed loudly — the count cross-check would be +# belt-and-braces. If it is right, the cross-check is the ONLY thing standing between a +# half-installed firewall and a job that believes it is contained. That is worth measuring +# rather than asserting, so leg C measures it, and then asks the question that actually +# matters: with a partial firewall, is the job still contained? Against a LIVE listener. +# +# Product-rendered plans only. Runs in the disposable gvisor-repro VM. The trap removes +# every rule it created and the verdict asserts the chains came back to their start depth. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +NETFILTER_IMAGE="${NETFILTER_IMAGE:-ghcr.io/makeprisms/maxplayer-netfilter:v0.5.8}" +PLANDIR="${PLANDIR:-$HOME/gate5h-plans}" +SUBNET="${SUBNET:-172.31.56.0/24}" +NET="${NET:-maxplayer-dns-gate5i-job}" +HOST_LAN="${HOST_LAN:-192.168.5.15}" +HOST_PORT="${HOST_PORT:-49257}" +RESOLV_FILE="${RESOLV_FILE:-$HOME/gate5i-resolv.conf}" +OUT="${OUT:-$HOME/gate5i-evidence.txt}" + +exec > >(tee "${OUT}") 2>&1 +FAIL=0 +fail() { echo " FAIL: $*"; FAIL=$((FAIL+1)); } +ok() { echo " ok: $*"; } + +rules_for() { local a="$1" + echo $(( $(sudo iptables -S DOCKER-USER 2>/dev/null | grep -c -- "${a}") \ + + $(sudo iptables -S INPUT 2>/dev/null | grep -c -- "${a}") )); } +chain_depth() { echo "$(( $(sudo iptables -S DOCKER-USER | wc -l) + $(sudo iptables -S INPUT | wc -l) ))"; } + +cleanup() { + sudo docker rm -f $(sudo docker ps -aq --filter "name=gate5i-") >/dev/null 2>&1 || true + [ -n "${ADDR:-}" ] && [ -f "${PLANDIR}/${ADDR}-teardown.txt" ] && \ + sudo timeout 120 docker run --rm --interactive --network host --cap-drop ALL \ + --cap-add NET_ADMIN --security-opt no-new-privileges "${NETFILTER_IMAGE}" \ + < "${PLANDIR}/${ADDR}-teardown.txt" >/dev/null 2>&1 + sudo docker network rm "${NET}" >/dev/null 2>&1 || true + pkill -f "gate5i-host-listener" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +echo "=== gate5i: environment ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) runsc=$(runsc --version | head -1)" +START_DEPTH="$(chain_depth)"; echo "chain depth at start: ${START_DEPTH}" + +printf 'nameserver 1.1.1.1\noptions timeout:2 attempts:2\n' > "${RESOLV_FILE}"; chmod 0444 "${RESOLV_FILE}" +setsid python3 -c " +import socket +s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) +s.bind(('0.0.0.0',${HOST_PORT})); s.listen(8) # gate5i-host-listener +while True: + c,_=s.accept(); c.sendall(b'REACHED'); c.close() +" >/dev/null 2>&1 /dev/null 2>&1 +sudo timeout 120 docker run --detach --name gate5i-holder --network "${NET}" \ + --read-only --cap-drop ALL --security-opt no-new-privileges --user 65534:65534 \ + --entrypoint sleep "${IMAGE}" infinity >/dev/null 2>&1 +ADDR="$(sudo docker inspect --format "{{(index .NetworkSettings.Networks \"${NET}\").IPAddress}}" gate5i-holder)" +echo "job address: ${ADDR}" +PLAN="${PLANDIR}/${ADDR}-install.txt" +[ -s "${PLAN}" ] || { echo "MISSING ${PLAN} — render it with render_host_plan"; exit 2; } +EXPECTED="$(grep -c . "${PLAN}")" +echo "rendered plan: ${EXPECTED} rules" + +PROBE=' +const net=require("net"); +const s=net.connect({host:process.argv[1],port:Number(process.argv[2]),timeout:8000}); +let said=false; const say=(w)=>{ if(!said){said=true;console.log(w);} s.destroy(); }; +s.on("connect",()=>say("REACHED")); s.on("timeout",()=>say("timeout")); s.on("error",e=>say(e.code)); +' +probe() { sudo timeout 180 docker run --rm --runtime "$1" --network "container:gate5i-holder" \ + --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges \ + -v "${RESOLV_FILE}:/etc/resolv.conf:ro" --entrypoint node "${IMAGE}" -e "${PROBE}" \ + "${HOST_LAN}" "${HOST_PORT}" 2>&1 | tr -d '\r' | tail -1; } + +# --------------------------------------------------------------------------------------- +echo +echo "=== gate5i: A. applier cannot start (bad image) — nothing may be installed ===" +OUT_A="$(sudo timeout 120 docker run --rm --interactive --network host --cap-drop ALL \ + --cap-add NET_ADMIN --security-opt no-new-privileges \ + "ghcr.io/makeprisms/maxplayer-netfilter:definitely-not-a-tag" < "${PLAN}" 2>&1 | tail -1)" +RC_A=$? +echo " applier said: $(echo "${OUT_A}" | cut -c1-90)" +LEFT_A="$(rules_for "${ADDR}")" +echo " rules keyed to ${ADDR}: ${LEFT_A}" +[ "${LEFT_A}" -eq 0 ] && ok "a non-starting applier installs nothing (establish() maps this to a hard error)" \ + || fail "${LEFT_A} rules present after the applier failed to start" + +echo +echo "=== gate5i: B. applier without NET_ADMIN — must not silently succeed ===" +OUT_B="$(sudo timeout 120 docker run --rm --interactive --network host --cap-drop ALL \ + --security-opt no-new-privileges "${NETFILTER_IMAGE}" < "${PLAN}" 2>&1 | tail -1)" +echo " applier said: $(echo "${OUT_B}" | cut -c1-90)" +LEFT_B="$(rules_for "${ADDR}")" +echo " rules keyed to ${ADDR}: ${LEFT_B}" +if [ "${LEFT_B}" -eq "${EXPECTED}" ]; then + fail "the full policy installed without NET_ADMIN — the capability is not what gates this" +elif [ "${LEFT_B}" -eq 0 ]; then + ok "no capability, no rules" +else + echo " PARTIAL: ${LEFT_B} of ${EXPECTED} — exactly the half-installed state the count check exists for" +fi + +echo +echo "=== gate5i: C. TRUNCATED plan — does it apply cleanly and lie? ===" +TRUNC="$HOME/gate5i-truncated.txt" +head -9 "${PLAN}" > "${TRUNC}" +echo " feeding ${EXPECTED} rendered rules as a $(grep -c . "${TRUNC}")-line plan" +OUT_C="$(sudo timeout 120 docker run --rm --interactive --network host --cap-drop ALL \ + --cap-add NET_ADMIN --security-opt no-new-privileges "${NETFILTER_IMAGE}" < "${TRUNC}" 2>&1 | tail -1)" +RC_C=$? +echo " applier exit=${RC_C} said=$(echo "${OUT_C}" | cut -c1-60)" +INSTALLED_C="$(rules_for "${ADDR}")" +echo " rules actually in the kernel: ${INSTALLED_C} of ${EXPECTED} rendered" +if [ "${RC_C}" -eq 0 ] && [ "${INSTALLED_C}" -lt "${EXPECTED}" ]; then + ok "CONFIRMED: a truncated plan exits 0 while under-installing — only the count reveals it" +else + echo " NOTE: truncation did not present as a clean exit; the code comment overstates the risk" +fi +[ "${INSTALLED_C}" -ne "${EXPECTED}" ] && ok "count cross-check would refuse this job (${INSTALLED_C} != ${EXPECTED})" \ + || fail "count check could not distinguish the truncated plan" + +# The question that matters: a partial firewall is not a firewall. Measure it. +GOT_C="$(probe runsc)" +echo " runsc -> live host ${HOST_LAN}:${HOST_PORT} with a PARTIAL firewall: ${GOT_C}" +if [ "${GOT_C}" = REACHED ]; then + ok "a partially-installed policy leaves the job UNCONTAINED — refusing the job is the only safe move" +else + echo " NOTE: this particular truncation still denied the probed destination; another would not" +fi + +echo +echo "=== gate5i: D. teardown of a PARTIAL install ===" +# Two strategies, run back to back on the same partial install, because the difference between +# them IS the fix. D1 is what HostRules::drop did before gate 5i; it is kept as a live regression +# witness rather than deleted, so that if the applier's abort-on-first-failure behaviour ever +# changes, this gate says so instead of silently keeping a workaround nobody needs. +# +# Note honestly what this leg is: the Rust branch in HostRules::drop is covered by unit tests +# (`adopted_host_rules_are_not_complete_until_the_count_check_passes`, +# `the_per_rule_teardown_renders_one_valid_delete_per_rule`). What a shell gate can prove, and +# what these two legs do prove, is that the per-rule STRATEGY actually clears real rules from +# real chains where the one-shot inverse plan cannot. +echo " D1: one-shot inverse plan (the pre-fix behaviour)" +sudo timeout 120 docker run --rm --interactive --network host --cap-drop ALL \ + --cap-add NET_ADMIN --security-opt no-new-privileges "${NETFILTER_IMAGE}" \ + < "${PLANDIR}/${ADDR}-teardown.txt" >/dev/null 2>&1 +LEFT_D1="$(rules_for "${ADDR}")" +echo " rules remaining: ${LEFT_D1} (of ${INSTALLED_C} installed)" +if [ "${LEFT_D1}" -gt 0 ]; then + echo " as expected: the applier aborts on the first never-created rule and removes nothing" +else + echo " NOTE: the one-shot plan cleared it — the applier no longer aborts; revisit the fix" +fi + +echo " D2: per-rule teardown (what HostRules::drop now does on the partial path)" +REMOVED=0 +while IFS= read -r line; do + [ -n "${line}" ] || continue + if printf '%s\n' "${line}" | sudo timeout 60 docker run --rm --interactive --network host \ + --cap-drop ALL --cap-add NET_ADMIN --security-opt no-new-privileges \ + "${NETFILTER_IMAGE}" >/dev/null 2>&1; then + REMOVED=$((REMOVED+1)) + fi +done < "${PLANDIR}/${ADDR}-teardown.txt" +LEFT_D="$(rules_for "${ADDR}")" +echo " removed ${REMOVED} rule(s) one at a time; ${LEFT_D} remain" +[ "${LEFT_D}" -eq 0 ] && ok "the partial install came out — a missing rule no longer strands the present ones" \ + || fail "${LEFT_D} rule(s) survived even the per-rule teardown" + +echo +echo "=== gate5i: verdict ===" +END_DEPTH="$(chain_depth)" +echo "chain depth: start=${START_DEPTH} end=${END_DEPTH}" +[ "${END_DEPTH}" -eq "${START_DEPTH}" ] && ok "chains returned to starting depth" \ + || fail "chains changed ${START_DEPTH} -> ${END_DEPTH}" +echo "failing checks: ${FAIL}" +[ "${FAIL}" -eq 0 ] && echo "GATE 5i: PASS" || echo "GATE 5i: FAIL" diff --git a/docs/gvisor-dns-delivery/scripts/gate5k-unsupported-daemon-default.sh b/docs/gvisor-dns-delivery/scripts/gate5k-unsupported-daemon-default.sh new file mode 100644 index 00000000..87e779ca --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/gate5k-unsupported-daemon-default.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# gate5k — does an UNSUPPORTED daemon default runtime fail CLOSED? +# +# Maxie's ruling (9 Sep 2026): "Missing --runtime means daemon default, not +# guaranteed runc; test unsupported defaults fail closed." +# +# The correction lands on the product. `holder_argv()` deliberately passes NO +# `--runtime`, and `the_containment_plane_never_carries_the_jobs_runtime` locks +# it there, because a runsc holder's namespace is unusable: a job joining it +# sees `lo` only (gate2a). The design therefore ASSUMES the daemon default is +# runc, and nothing in the product verifies that assumption. +# +# Fail CLOSED = the job gets no usable egress, or never starts. +# Fail OPEN = the job runs and REACHES a live listener with no containment. +# Only the second is a security defect; the first is an availability failure. +# +# v3. Two earlier drafts are committed as failures and kept: +# v1 gate5k-CONFOUNDED-harness-defect-*.txt — listener started with `docker +# exec` into a runsc holder, which runsc refuses; target never served. +# v2 gate5k-v2-daemon-default-runtime-*.txt — root cause found: the sandbox +# image has NO nc and NO wget, so every probe was doomed before it ran. +# This version uses the primitives the other gates already proved work in this +# image: `--entrypoint node` with inline JS. gate5f/5g/5h/5i never used nc or +# wget, which is why their SERVING/REACHED readings were real. +# +# Discipline: ONE gVisor container per namespace (gate5d: namespaces are +# single-use for gVisor), a fresh namespace per probe, a LIVE listener, and a +# liveness reading printed beside every leg so a silence is never mistaken for +# containment. +# +# Runs inside the disposable gvisor-repro VM ONLY: it edits +# /etc/docker/daemon.json and restarts docker. +set -uo pipefail + +IMAGE="${IMAGE:-ghcr.io/makeprisms/maxplayer-sandbox:v0.5.8}" +DAEMON_JSON=/etc/docker/daemon.json +BACKUP="/home/forge.guest/daemon.json.gate5k.bak" +NET="gate5k-net" +FAIL=0 +LISTEN_ADDR="" + +say() { printf '%s\n' "$*"; } +hr() { printf -- '---- %s\n' "$*"; } + +[ -f "${DAEMON_JSON}" ] || { say "ABORT: no ${DAEMON_JSON}"; exit 2; } +if command -v limactl >/dev/null 2>&1; then + say "ABORT: limactl present — this looks like the HOST, not the disposable VM"; exit 2 +fi + +SERVER_JS='require("http").createServer((q,r)=>r.end("REACHED")).listen(8080,"0.0.0.0");' + +LIVENESS_JS='const n=require("net");const s=n.connect({host:process.argv[1],port:8080,timeout:5000}); +s.on("connect",()=>{console.log("SERVING");s.destroy();}); +s.on("timeout",()=>{console.log("DEAD timeout");s.destroy();}); +s.on("error",e=>console.log("DEAD "+e.code));' + +PROBE_JS='const os=require("os"),http=require("http"); +const v4=Object.values(os.networkInterfaces()).flat().filter(x=>x&&x.family==="IPv4"&&!x.internal); +process.stdout.write(v4.length?("HEALTHY "+v4[0].address):"SICK no-address"); +const r=http.get({host:process.argv[1],port:8080,timeout:8000},res=>{let d=""; + res.on("data",c=>d+=c);res.on("end",()=>console.log(" | target="+d.trim()));}); +r.on("timeout",()=>{console.log(" | target=TIMEOUT");r.destroy();}); +r.on("error",e=>console.log(" | target="+e.code));' + +default_runtime() { sudo docker info --format '{{.DefaultRuntime}}' 2>/dev/null; } + +restore() { + hr "restoring ${DAEMON_JSON}" + if [ -f "${BACKUP}" ]; then + sudo cp "${BACKUP}" "${DAEMON_JSON}" + sudo systemctl restart docker || true + sleep 5 + say "restored default runtime = $(default_runtime)" + fi + sudo docker rm -f gate5k-listener gate5k-ns1 gate5k-ns2 >/dev/null 2>&1 + sudo docker network rm "${NET}" >/dev/null 2>&1 +} +trap restore EXIT + +sudo cp "${DAEMON_JSON}" "${BACKUP}" +say "gate5k v3 — unsupported daemon default runtime" +say "kernel: $(uname -srm)" +say "docker: $(sudo docker version --format '{{.Server.Version}}')" +say "runsc: $(runsc --version 2>/dev/null | head -1)" +say "ORIGINAL default runtime: $(default_runtime)" +say "" + +sudo docker network create --driver bridge "${NET}" >/dev/null 2>&1 + +# Test infrastructure, pinned to runc so it keeps working when the default flips +# underneath it. gate5g's confound (a listener's own OUTPUT rules dropping its +# replies) cannot arise here: gate5k installs no netns plan at all. +sudo docker rm -f gate5k-listener >/dev/null 2>&1 +sudo timeout 120 docker run --detach --name gate5k-listener --runtime runc \ + --network "${NET}" --entrypoint node "${IMAGE}" -e "${SERVER_JS}" >/dev/null 2>&1 + +serve() { # a daemon restart stops containers; revive and re-read the address + sudo docker start gate5k-listener >/dev/null 2>&1 + sleep 4 + LISTEN_ADDR="$(sudo docker inspect -f "{{(index .NetworkSettings.Networks \"${NET}\").IPAddress}}" gate5k-listener 2>/dev/null)" +} + +liveness() { + sudo timeout 60 docker run --rm --runtime runc --network "${NET}" \ + --entrypoint node "${IMAGE}" -e "${LIVENESS_JS}" "${LISTEN_ADDR}" 2>&1 | tr -d '\r' | tail -1 +} + +probe() { # ns_name — product-SHAPED holder carrying NO --runtime, then ONE gVisor job + local ns="$1" + sudo docker rm -f "${ns}" >/dev/null 2>&1 + sudo timeout 120 docker run --detach --name "${ns}" --network "${NET}" \ + --entrypoint sleep "${IMAGE}" infinity >/dev/null 2>&1 + local started=$? + sleep 4 + if ! sudo docker ps --format '{{.Names}}' | grep -qx "${ns}"; then + say " holder: DID NOT START (docker run exit=${started}) -> no job can run" + return 3 + fi + say " holder runtime: $(sudo docker inspect -f '{{.HostConfig.Runtime}}' "${ns}" 2>/dev/null)" + local out + out="$(sudo timeout 120 docker run --rm --runtime runsc --network "container:${ns}" \ + --entrypoint node "${IMAGE}" -e "${PROBE_JS}" "${LISTEN_ADDR}" 2>&1 | tr -d '\r' | tail -2 | tr '\n' ' ')" + if [ -z "${out}" ]; then + say " job: PRODUCED NO OUTPUT -> could not run in this namespace (fail closed by refusal)" + else + say " job: ${out}" + fi +} + +hr "LEG 1 — control: daemon default is runc (the supported configuration)" +say "default runtime now: $(default_runtime)" +serve +L1="$(liveness)" +say "listener ${LISTEN_ADDR} liveness: ${L1}" +if [ "${L1}" != "SERVING" ]; then + say "leg 1 target is dead — NO EVIDENCE"; FAIL=$((FAIL + 1)) +fi +probe gate5k-ns1 +say " ^ expected here: HEALTHY and target=REACHED. This leg is the" +say " positive control: it proves the harness CAN observe reachability, so" +say " leg 2's silence means something." +say "" + +hr "LEG 2 — daemon default switched to runsc (the UNSUPPORTED configuration)" +printf '{\n "default-runtime": "runsc",\n "runtimes": { "runsc": { "path": "/usr/local/bin/runsc" } }\n}\n' \ + | sudo tee "${DAEMON_JSON}" >/dev/null +sudo systemctl restart docker +sleep 7 +NOW="$(default_runtime)" +say "default runtime now: ${NOW}" +if [ "${NOW}" != "runsc" ]; then + say "could not switch the default runtime — leg 2 is NO EVIDENCE" + FAIL=$((FAIL + 1)) +else + serve + L2="$(liveness)" + say "listener ${LISTEN_ADDR} liveness: ${L2}" + if [ "${L2}" != "SERVING" ]; then + say "leg 2 target is dead — NO EVIDENCE, not a fail-closed result" + FAIL=$((FAIL + 1)) + fi + probe gate5k-ns2 +fi +say "" + +hr "VERDICT" +say "With a SERVING listener in both legs: leg 1 REACHED and leg 2 SICK/denied" +say "means the unsupported default FAILS CLOSED — no egress path to contain, an" +say "availability failure rather than a containment bypass. Leg 2 printing" +say "target=REACHED would mean it FAILS OPEN and is a security defect." +say "failing_checks=${FAIL} (non-zero means a leg above proved nothing)" +exit 0 diff --git a/docs/gvisor-dns-delivery/scripts/provision-repro-vm.sh b/docs/gvisor-dns-delivery/scripts/provision-repro-vm.sh new file mode 100755 index 00000000..09405e4e --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/provision-repro-vm.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Provision a DISPOSABLE Linux VM for the gVisor DNS reproduction. +# Runs INSIDE the throwaway lima VM `gvisor-repro`. Never run on the forge host, +# and never against the shared colima VM: other lanes depend on that daemon. +# +# Installs: docker engine, gVisor runsc, registered as the `runsc` docker runtime. +# Everything it writes lives inside the VM and dies with `limactl delete gvisor-repro`. +set -euo pipefail + +RUNSC_RELEASE="${RUNSC_RELEASE:-20260831.0}" +ARCH="$(uname -m)" +LOG=/var/log/gvisor-provision.log + +log() { echo "[provision] $*"; } + +log "arch=${ARCH} kernel=$(uname -r) os=$(. /etc/os-release && echo "${PRETTY_NAME}")" + +export DEBIAN_FRONTEND=noninteractive +sudo -E apt-get update -qq +sudo -E apt-get install -y -qq docker.io curl ca-certificates git iproute2 dnsutils >/dev/null +sudo systemctl enable --now docker + +log "docker: $(docker --version)" + +# gVisor. Try the release named in the brief first; fall back to the current +# release only if that exact one has no artifact for this arch, and say so loudly +# so no report can silently claim the briefed version. +install_runsc() { + local rel="$1" base url tmp + base="https://storage.googleapis.com/gvisor/releases/release/${rel}/${ARCH}" + tmp="$(mktemp -d)" + for f in runsc containerd-shim-runsc-v1; do + url="${base}/${f}" + if ! curl -fsSL "${url}" -o "${tmp}/${f}"; then + echo "MISS ${url}" >&2 + rm -rf "${tmp}" + return 1 + fi + curl -fsSL "${url}.sha512" -o "${tmp}/${f}.sha512" || true + done + ( cd "${tmp}" && sha512sum -c ./*.sha512 ) || { echo "checksum failed for ${rel}" >&2; rm -rf "${tmp}"; return 1; } + sudo install -m 0755 -t /usr/local/bin "${tmp}/runsc" "${tmp}/containerd-shim-runsc-v1" + rm -rf "${tmp}" + echo "${rel}" | sudo tee /etc/gvisor-installed-release >/dev/null + return 0 +} + +if install_runsc "${RUNSC_RELEASE}"; then + log "runsc installed from briefed release ${RUNSC_RELEASE}" +else + log "WARNING: briefed release ${RUNSC_RELEASE} has no ${ARCH} artifact; falling back to 'latest'" + install_runsc latest + log "runsc installed from 'latest' — every result must record this substitution" +fi + +log "runsc: $(runsc --version | tr '\n' ' ')" + +sudo runsc install +sudo systemctl restart docker +sleep 3 +docker info --format 'runtimes={{.Runtimes}}' | tee -a "${LOG}" 2>/dev/null || docker info | grep -i runtime + +log "provision complete" diff --git a/docs/gvisor-dns-delivery/scripts/run-all-gates.sh b/docs/gvisor-dns-delivery/scripts/run-all-gates.sh new file mode 100755 index 00000000..52691435 --- /dev/null +++ b/docs/gvisor-dns-delivery/scripts/run-all-gates.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Runs the gVisor DNS/delivery gates in order, bounded, and reports a verdict per gate. +# +# The gates are the deliverable, but a gate nobody can re-run is a claim, not evidence. +# This is how someone else reproduces the whole set on a fresh VM without knowing which +# script needs which rendered plan. +# +# Two rules it will not bend: +# +# * **BOUNDED.** Every gate gets a `timeout`, and the run gets a total budget. A gate +# that hangs is a FAIL with a reason, never a run that sits there until a watchdog +# kills the session and leaves no evidence at all. +# * **A GATE THAT DID NOT RUN IS NOT A PASS.** Missing prerequisites print SKIPPED with +# the reason and count against the run. The failure mode this exists to prevent is a +# green summary produced by a suite that quietly executed nothing — which is exactly +# the class of bug gates 2 and 5 were caught making about their own probes. +# +# Prerequisites: the repro VM from provision-repro-vm.sh, and the rendered plans copied to +# $HOME (see PLANS below). Plans are rendered on the REPO host, because rendering them here +# would mean this script decides what the rules are — and then the gates would be testing +# the script instead of the product. +set -uo pipefail + +EVIDENCE_DIR="${EVIDENCE_DIR:-$HOME/gate-evidence}" +PER_GATE_TIMEOUT="${PER_GATE_TIMEOUT:-900}" +TOTAL_BUDGET="${TOTAL_BUDGET:-3600}" +SUMMARY="${SUMMARY:-$EVIDENCE_DIR/summary.txt}" +STARTED=$(date +%s) + +mkdir -p "${EVIDENCE_DIR}" +declare -a NAMES=() VERDICTS=() REASONS=() + +record() { NAMES+=("$1"); VERDICTS+=("$2"); REASONS+=("$3"); } + +# gate -> the files it needs in $HOME before it can say anything true +plans_for() { + case "$1" in + gate1) echo "" ;; + gate2) echo "gate2-plan.txt" ;; + gate4) echo "gate4-plan.txt" ;; + gate5) echo "gate5-plans/plan-d.txt gate5-plans/host-d.txt gate5-plans/host-d-teardown.txt" ;; + gate5f) echo "gate5f-plan.txt gate5f-host-install.txt gate5f-host-teardown.txt" ;; + esac +} +script_for() { + case "$1" in + gate1) echo "gate1-repro.sh" ;; + gate2) echo "gate2-namespace-dns-tls.sh" ;; + gate4) echo "gate4-container-git-delivery.sh" ;; + gate5) echo "gate5-denial-and-concurrent-success.sh" ;; + gate5f) echo "gate5f-product-host-rules-bind-runsc.sh" ;; + esac +} +# The line in a gate's own output that decides it. Grepping the gate's verdict rather than +# trusting its exit code is deliberate: several of these scripts run under `set -uo pipefail` +# without `-e` and exit 0 while reporting a failure inside. +verdict_of() { # gate logfile + local gate="$1" log="$2" + case "${gate}" in + gate1) + grep -q "EAI_AGAIN" "${log}" && grep -qi "runc" "${log}" && echo PASS || echo FAIL ;; + gate2) + grep -q "verified=true\|cert-verified" "${log}" && echo PASS || echo FAIL ;; + gate4) + grep -q "GATE4: PASS" "${log}" && echo PASS || echo FAIL ;; + gate5) + grep -q "GATE5-DENIAL: PASS" "${log}" && echo PASS || echo FAIL ;; + gate5f) + # No self-verdict line: it passes when the host-directed leg changed under the policy + # and the public route survived, and when nothing leaked. + grep -q "CLEAN" "${log}" && grep -q "PUBLIC-PASS git" "${log}" && echo PASS || echo FAIL ;; + esac +} + +run_gate() { # gate + local gate="$1" + local script; script="$(script_for "${gate}")" + local log="${EVIDENCE_DIR}/${gate}-$(date -u +%Y%m%dT%H%M%SZ).log" + + local elapsed=$(( $(date +%s) - STARTED )) + if [ "${elapsed}" -ge "${TOTAL_BUDGET}" ]; then + record "${gate}" SKIPPED "total budget ${TOTAL_BUDGET}s exhausted" + return + fi + if [ ! -x "${HOME}/${script}" ]; then + record "${gate}" SKIPPED "missing ${HOME}/${script}" + return + fi + local missing="" p + for p in $(plans_for "${gate}"); do + [ -s "${HOME}/${p}" ] || missing="${missing} ${p}" + done + if [ -n "${missing}" ]; then + record "${gate}" SKIPPED "missing rendered plan(s):${missing}" + return + fi + + echo "--- ${gate}: running (timeout ${PER_GATE_TIMEOUT}s) -> ${log}" + local budget_left=$(( TOTAL_BUDGET - elapsed )) + local limit=$(( PER_GATE_TIMEOUT < budget_left ? PER_GATE_TIMEOUT : budget_left )) + timeout "${limit}" "${HOME}/${script}" > "${log}" 2>&1 + local rc=$? + if [ "${rc}" -eq 124 ]; then + record "${gate}" FAIL "timed out after ${limit}s (partial log kept)" + return + fi + record "${gate}" "$(verdict_of "${gate}" "${log}")" "exit=${rc} log=$(basename "${log}")" +} + +echo "=== gVisor DNS/delivery gates ===" +date -u +"utc=%Y-%m-%dT%H:%M:%SZ" +echo "kernel=$(uname -r) arch=$(uname -m) docker=$(sudo docker version --format '{{.Server.Version}}' 2>/dev/null) runsc=$(runsc --version 2>/dev/null | head -1)" +echo "per-gate timeout=${PER_GATE_TIMEOUT}s total budget=${TOTAL_BUDGET}s evidence=${EVIDENCE_DIR}" +echo "NOTE: results are labelled by architecture on purpose — x86_64 is NOT covered by this run." +echo + +GATES="${GATES:-gate1 gate2 gate4 gate5 gate5f}" +for g in ${GATES}; do run_gate "${g}"; done + +{ + echo "=== summary ===" + date -u +"utc=%Y-%m-%dT%H:%M:%SZ" + echo "arch=$(uname -m) runsc=$(runsc --version 2>/dev/null | head -1)" + fails=0 + for i in "${!NAMES[@]}"; do + printf '%-8s %-8s %s\n' "${NAMES[i]}" "${VERDICTS[i]}" "${REASONS[i]}" + [ "${VERDICTS[i]}" = PASS ] || fails=$((fails + 1)) + done + echo "total=$(( $(date +%s) - STARTED ))s gates=${#NAMES[@]} not-passing=${fails}" + # SKIPPED counts against the run. A suite that ran nothing must never read green. + [ "${fails}" -eq 0 ] && echo "ALL GATES: PASS" || echo "ALL GATES: FAIL (${fails} not passing)" +} | tee "${SUMMARY}"