From d525765c48177c907f6c0bc9e7a59ca3ab206e36 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Tue, 21 Jul 2026 04:32:49 +0800 Subject: [PATCH 1/8] =?UTF-8?q?test(e2e):=20Machine=20networking=20?= =?UTF-8?q?=E2=80=94=20first=20exercise=20of=20the=20network=20plane?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit machine.rs only asserts an agent-reported IP over vsock; it never drives a packet. This runs real traffic from inside a Machine over the machines.exec vsock channel (the Machine's docker exec): - M1 egress TCP: wget a host-local origin at the gateway (10.0.2.1), byte-exact — first proof a Machine reaches the network at all. - M2 DNS: host.docker.internal / gateway.docker.internal resolve to the gateway via the in-VMM DnsForwarder. - M3 egress volume: 16 MiB download, byte-exact and bounded. - M4 metadata: inspect reports gateway 10.0.2.1 + it as a DNS server, IP a valid routable IPv4. - M5 SSH contract: ssh_info is still Unimplemented — pins the gap. Datapath source-verified: a Machine's primary NIC is the same socketpair netstack + TcpBridge as the System VM (gateway 10.0.2.1), egress is pure in-process host-socket proxying with no helper/route dependency, so it runs in the isolated e2e daemon. Machine<->Machine / Machine->container / host->Machine are isolated-by-architecture (documented, no active test). Finding: on this Surge/fake-IP host inspect() reported the machine IP as 198.18.11.51 while the datapath IP is 10.0.2.2 — select_routable_ip picked a non-datapath address (the desktop UI shows this field). M4 WARN-logs it rather than gating (host-tangled); flagged for a clean-host repro. Verified: 5/5 green on a real VZ VM (86 s incl. machine boot + CDN pull). --- internal-docs/plans/machine-network-e2e.md | 72 ++++ tests/e2e/tests/machine_network.rs | 380 +++++++++++++++++++++ 2 files changed, 452 insertions(+) create mode 100644 internal-docs/plans/machine-network-e2e.md create mode 100644 tests/e2e/tests/machine_network.rs diff --git a/internal-docs/plans/machine-network-e2e.md b/internal-docs/plans/machine-network-e2e.md new file mode 100644 index 000000000..60486dd99 --- /dev/null +++ b/internal-docs/plans/machine-network-e2e.md @@ -0,0 +1,72 @@ +# Machine networking E2E + +`tests/e2e/tests/machine_network.rs` — the first exercise of a **Machine's +actual network plane**. The lifecycle test (`machine.rs`) only asserts an +agent-reported IP over vsock; it never drives a packet. This runs real +traffic from inside a Machine over the `machines.exec` vsock channel (the +Machine's `docker exec`). + +``` +cargo test -p arcbox-e2e --test machine_network -- --ignored --nocapture +``` + +Needs internet (create pulls alpine from the live `image.arcboxcdn.com` +mirror) and a guest `arcbox-agent` (musl cross-build, or the installed +app's agent staged by newest-mtime). + +## Datapath (source-verified 2026-07-21) + +A Machine's **primary NIC is the same socketpair userspace netstack + +TcpBridge as the System VM's** (`virt/arcbox-vmm/src/vmm/darwin.rs` — +`gateway=10.0.2.1, guest=10.0.2.2`, DHCP + `DnsForwarder` at the gateway). +Egress is pure in-process host-socket proxying +(`common/arcbox-proxy/src/egress/mod.rs`) — **no privileged helper, no host +route, no `/etc/resolver`**. The System VM's helper-installed `172.16/12` +route (`route_reconciler`) is System-VM-only and was never wired to +Machines (`app/arcbox-core/src/machine.rs` never calls it). So Machine +networking runs fully in the isolated e2e daemon. + +The agent channel is vsock, independent of the network plane +(`MachineManager::connect_agent`), which is why `exec` drives in-Machine +commands even while testing the network. + +## Scenarios (one Machine, one boot) + +| # | What | Assertion | +|---|---|---| +| M1 | egress TCP | `wget` a host-local origin at `10.0.2.1:`; `wc -c` byte-exact — first proof a Machine reaches the network *(implemented)* | +| M2 | DNS | `nslookup host.docker.internal` / `gateway.docker.internal` resolve to `10.0.2.1` via the in-VMM `DnsForwarder` *(implemented)* | +| M3 | egress volume | 16 MiB download, byte-exact and bounded *(implemented)* | +| M4 | metadata | `inspect` reports gateway `10.0.2.1` and it as a DNS server; IP is a valid routable IPv4 *(implemented)* | +| M5 | SSH contract | `ssh_info` is still `Unimplemented` — pins the gap so a future SSH feature trips this test *(implemented)* | + +## Not covered — by architecture, not omission + +Documented here because the architecture, not the harness, is the reason; +no active test (would be flaky/meaningless today): + +- **Machine ↔ Machine, Machine → container, Machine → System VM**: each VM + gets its own private per-process socketpair netstack; the second (vmnet + bridge) NIC is never brought up guest-side for Machines + (`guest/arcbox-agent/src/init.rs` `machine_init()` does DHCP on the + primary NIC only). Two Machines can even both be `10.0.2.2` — there is no + shared segment and no cross-VM route. When cross-machine networking is + added, M5's pattern (assert-the-gap-then-grow) is the template. +- **host → Machine inbound / SSH**: `ssh_info` is unimplemented and + `InboundListenerManager` is only ever wired to the System VM + (`app/arcbox-docker/src/handlers/container/mod.rs`), never to a Machine. + M5 pins this. + +## Finding (2026-07-21): reported IP ≠ datapath IP on a fake-IP host + +The datapath logs `gateway=10.0.2.1, guest=10.0.2.2`, and egress/DNS work +through it (M1–M3 pass), but `inspect().network.ip_address` came back as +`198.18.11.51` — the Surge/Clash fake-IP range this host runs, not the +datapath's `10.0.2.2`. `select_routable_ip` +(`app/arcbox-core/src/machine.rs`) picks from the agent's enumerated +addresses and chose a non-datapath address here. The desktop UI shows this +field as "the machine's IP", so a user on such a host sees a bogus address. +M4 asserts the robust facts (gateway, DNS, valid IPv4) and WARN-logs the +mismatch rather than gating on it (host-environment-tangled). Worth +reproducing on a clean, non-fake-IP host to decide whether +`select_routable_ip` should prefer the datapath subnet. diff --git a/tests/e2e/tests/machine_network.rs b/tests/e2e/tests/machine_network.rs new file mode 100644 index 000000000..da386ca7d --- /dev/null +++ b/tests/e2e/tests/machine_network.rs @@ -0,0 +1,380 @@ +//! Machine networking e2e — the first exercise of a Machine's actual +//! network plane (not just readiness/metadata over the vsock agent). +//! +//! A Machine's primary NIC is the same socketpair userspace netstack + +//! TcpBridge as the System VM's: gateway/DNS `10.0.2.1`, guest `10.0.2.x`, +//! egress via in-process host-socket proxying — no privileged helper, no +//! host route, so it runs in the isolated e2e daemon +//! (`virt/arcbox-vmm/src/vmm/darwin.rs`, `app/arcbox-core/src/machine.rs`). +//! The existing `machine.rs` test only asserts an agent-reported IP; it +//! never drives a packet. This drives real traffic from inside the Machine +//! over the `machines.exec` vsock channel (the Machine's `docker exec`). +//! +//! Scenarios (one Machine, one boot): +//! - **M1 egress TCP**: `wget` a host-local origin at the Machine's gateway +//! — first proof a Machine can reach the network at all. +//! - **M2 DNS**: `nslookup host.docker.internal` / `gateway.docker.internal` +//! resolve to the gateway via the in-VMM `DnsForwarder`. +//! - **M3 egress volume**: a larger download, byte-exact and bounded. +//! - **M4 metadata**: `inspect` reports gateway `10.0.2.1` and that gateway +//! as a DNS server. +//! - **M5 SSH contract**: `ssh_info` is still `unimplemented` — pins the +//! documented gap so a future SSH feature flags this test to grow. +//! +//! Not covered, by architecture (documented in the plan, no active test): +//! Machine↔Machine, Machine→container, Machine→System VM are isolated +//! per-VM netstacks with no cross path today; host→Machine inbound/SSH does +//! not exist. +//! +//! Requires internet (create pulls alpine from the live CDN mirror) and a +//! musl-cross `arcbox-agent`, like `machine.rs`. + +use std::sync::Once; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use arcbox_e2e::boot_assets::{resolve_boot_version, stage_dev_boot_assets}; +use arcbox_e2e::daemon::{DaemonConfig, DaemonHandle, connect_unix}; +use arcbox_e2e::metrics::RunMetrics; +use arcbox_e2e::net_fixtures::spawn_blob_server; +use arcbox_grpc::v1::machine_service_client::MachineServiceClient; +use arcbox_protocol::v1::{ + CreateMachineRequest, InspectMachineRequest, MachineExecRequest, RemoveMachineRequest, + SshInfoRequest, StartMachineRequest, StopMachineRequest, +}; +use tonic::transport::Channel; + +static TRACING: Once = Once::new(); + +const READY_TIMEOUT: Duration = Duration::from_secs(180); +const CREATE_BUDGET: Duration = Duration::from_secs(120); +const START_BUDGET: Duration = Duration::from_secs(120); +const RPC_BUDGET: Duration = Duration::from_secs(30); +/// Budget for an in-Machine network command (download etc.). +const NET_BUDGET: Duration = Duration::from_secs(60); + +const MACHINE: &str = "e2e-net-alpine"; +/// The gateway/DNS IP a Machine's primary NIC always routes through +/// (`darwin.rs` hardcodes it; `app/arcbox-api/src/grpc/machine.rs` documents +/// it as `NAT_GATEWAY`). +const GATEWAY: &str = "10.0.2.1"; +/// M3 download size — enough to span many segments, small enough to stay +/// quick over the loopback-backed egress path. +const VOLUME_BYTES: usize = 16 * 1024 * 1024; + +fn init_tracing() { + TRACING.call_once(|| { + let _ = tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .try_init(); + }); +} + +#[test] +#[ignore = "boots a real VZ daemon, pulls a distro image from the live CDN, drives Machine traffic"] +fn machine_network_end_to_end() -> Result<()> { + init_tracing(); + + let root = arcbox_e2e::repo_root(); + if !arcbox_e2e::env_flag("SKIP_BUILD") { + let shell = xshell::Shell::new()?; + shell.change_dir(&root); + xshell::cmd!(shell, "cargo build --release -p arcbox-daemon").run()?; + xshell::cmd!( + shell, + "cargo build --release -p arcbox-agent --target aarch64-unknown-linux-musl" + ) + .run()?; + } + + let version = resolve_boot_version(&root)?; + let data_dir = tempfile::Builder::new() + .prefix("arcbox-machine-net-") + .tempdir()?; + stage_dev_boot_assets(&root, data_dir.path(), &version)?; + + let mut daemon = DaemonHandle::spawn(DaemonConfig { + binary: root.join("target/release/arcbox-daemon"), + data_dir: data_dir.path().to_owned(), + args: vec![], + env: vec![ + ("ARCBOX_BOOT_ASSET_VERSION".to_owned(), version), + ("ARCBOX_VM_BACKEND".to_owned(), "vz".to_owned()), + ("ARCBOX_DNS_PORT".to_owned(), "0".to_owned()), + ], + })?; + + let mut metrics = RunMetrics::new("machine_network", Some("vz")); + let result = scenario(&mut daemon, &mut metrics); + metrics.passed = result.is_ok(); + if let Err(error) = metrics.write(Some(data_dir.path())) { + tracing::warn!("writing run metrics failed: {error:#}"); + } + if result.is_err() || arcbox_e2e::env_flag("KEEP_TEST_DIR") { + let kept = data_dir.keep(); + tracing::warn!(path = %kept.display(), "preserving test directory"); + } + result +} + +fn scenario(daemon: &mut DaemonHandle, metrics: &mut RunMetrics) -> Result<()> { + metrics.time("daemon_ready", || daemon.wait_ready_blocking(READY_TIMEOUT))?; + let socket = daemon.grpc_socket(); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .context("building tokio runtime")?; + + runtime.block_on(async { + let channel = connect_unix(&socket).await?; + let mut machines = MachineServiceClient::new(channel); + + create_and_start(&mut machines, metrics).await?; + + // Run all network sub-scenarios, aggregating failures so one boot + // reports the full picture rather than stopping at the first gap. + let mut failures = Vec::new(); + for (name, result) in [ + ("m1_egress_tcp", m1_egress_tcp(&mut machines).await), + ("m2_dns", m2_dns(&mut machines).await), + ("m3_egress_volume", m3_egress_volume(&mut machines).await), + ( + "m4_network_metadata", + m4_network_metadata(&mut machines).await, + ), + ( + "m5_ssh_unimplemented", + m5_ssh_unimplemented(&mut machines).await, + ), + ] { + match result { + Ok(()) => tracing::info!(scenario = name, "passed"), + Err(error) => { + tracing::warn!(scenario = name, "failed: {error:#}"); + failures.push(format!("{name}: {error:#}")); + } + } + } + + // Teardown regardless of scenario outcome. + let _ = machines + .stop(StopMachineRequest { + id: MACHINE.to_owned(), + force: true, + }) + .await; + let _ = machines + .remove(RemoveMachineRequest { + id: MACHINE.to_owned(), + force: true, + volumes: true, + }) + .await; + + if failures.is_empty() { + Ok(()) + } else { + bail!( + "{} of 5 machine-network scenarios failed:\n{}", + failures.len(), + failures.join("\n") + ) + } + }) +} + +async fn create_and_start( + machines: &mut MachineServiceClient, + metrics: &mut RunMetrics, +) -> Result<()> { + tokio::time::timeout( + CREATE_BUDGET, + machines.create(CreateMachineRequest { + name: MACHINE.to_owned(), + cpus: 1, + memory: 1024 * 1024 * 1024, + disk_size: 2 * 1024 * 1024 * 1024, + distro: "alpine".to_owned(), + version: "3.24".to_owned(), + ..Default::default() + }), + ) + .await + .context("create timed out (CDN pull)")? + .context("create failed")?; + + tokio::time::timeout( + START_BUDGET, + machines.start(StartMachineRequest { + id: MACHINE.to_owned(), + }), + ) + .await + .context("start timed out")? + .context("start failed")?; + + metrics.record("machine_started", 1.0); + Ok(()) +} + +/// Runs `cmd` inside the Machine over the vsock agent exec channel and +/// returns (stdout, exit_code). The exec channel is independent of the +/// network plane under test — the out-of-band `docker exec` equivalent. +async fn exec_capture( + machines: &mut MachineServiceClient, + cmd: &[&str], + budget: Duration, +) -> Result<(String, i32)> { + let mut stream = tokio::time::timeout( + budget, + machines.exec(MachineExecRequest { + id: MACHINE.to_owned(), + cmd: cmd.iter().map(|s| (*s).to_owned()).collect(), + ..Default::default() + }), + ) + .await + .context("exec timed out")? + .context("exec failed")? + .into_inner(); + + let mut stdout = Vec::new(); + let mut exit = None; + while let Some(out) = tokio::time::timeout(budget, stream.message()) + .await + .context("exec output timed out")? + .context("exec stream error")? + { + if out.stream == "stdout" { + stdout.extend_from_slice(&out.data); + } + if out.done { + exit = Some(out.exit_code); + } + } + Ok(( + String::from_utf8_lossy(&stdout).into_owned(), + exit.context("exec produced no completion frame")?, + )) +} + +/// M1: the Machine reaches a host-local origin through its gateway — the +/// first proof its network plane carries traffic. `wget … | wc -c` asserts +/// both reachability and a complete (un-truncated) transfer. +async fn m1_egress_tcp(machines: &mut MachineServiceClient) -> Result<()> { + let blob = 256 * 1024; + let server = spawn_blob_server(blob)?; + let url = format!("http://{GATEWAY}:{}/blob", server.port()); + let cmd = format!("wget -q -O - '{url}' | wc -c"); + let (out, exit) = exec_capture(machines, &["/bin/sh", "-c", &cmd], NET_BUDGET).await?; + if exit != 0 { + bail!("wget exited {exit} (out: {out:?})"); + } + let got: usize = out.trim().parse().context("parsing wc -c output")?; + if got != blob { + bail!("machine received {got} of {blob} bytes from the gateway origin"); + } + Ok(()) +} + +/// M2: the Machine's resolver (`10.0.2.1`, from DHCP) answers the +/// gateway-internal names via the in-VMM `DnsForwarder`. +async fn m2_dns(machines: &mut MachineServiceClient) -> Result<()> { + for name in ["host.docker.internal", "gateway.docker.internal"] { + let (out, exit) = exec_capture( + machines, + &["/bin/sh", "-c", &format!("nslookup {name}")], + RPC_BUDGET, + ) + .await?; + // busybox nslookup exits 0 on NXDOMAIN too, so assert on the answer. + if !out.contains(GATEWAY) { + bail!("nslookup {name} did not resolve to {GATEWAY} (exit {exit}): {out:?}"); + } + } + Ok(()) +} + +/// M3: a larger download completes, byte-exact and within budget — the +/// Machine egress path sustains volume, not just a token request. +async fn m3_egress_volume(machines: &mut MachineServiceClient) -> Result<()> { + let server = spawn_blob_server(VOLUME_BYTES)?; + let url = format!("http://{GATEWAY}:{}/blob", server.port()); + let cmd = format!("wget -q -O - '{url}' | wc -c"); + let (out, exit) = exec_capture(machines, &["/bin/sh", "-c", &cmd], NET_BUDGET).await?; + if exit != 0 { + bail!("volume wget exited {exit} (out: {out:?})"); + } + let got: usize = out.trim().parse().context("parsing wc -c output")?; + if got != VOLUME_BYTES { + bail!("machine received {got} of {VOLUME_BYTES} bytes"); + } + Ok(()) +} + +/// M4: `inspect` reports the gateway and DNS the Machine actually uses — +/// metadata the lifecycle test never checks. +async fn m4_network_metadata(machines: &mut MachineServiceClient) -> Result<()> { + let info = machines + .inspect(InspectMachineRequest { + id: MACHINE.to_owned(), + }) + .await + .context("inspect failed")? + .into_inner(); + let net = info.network.context("no network in inspect")?; + if net.gateway != GATEWAY { + bail!("inspect gateway is {:?}, expected {GATEWAY}", net.gateway); + } + if !net.dns_servers.iter().any(|d| d == GATEWAY) { + bail!( + "inspect dns_servers {:?} does not include the gateway {GATEWAY}", + net.dns_servers + ); + } + // The reported IP must be a real, usable IPv4 — but NOT hardcoded to the + // datapath's 10.0.2.0/24: `select_routable_ip` picks from the agent's + // enumerated addresses, and on this host it returned 198.18.11.51 (the + // Surge/Clash fake-IP range) while the datapath IP is 10.0.2.2. That + // disagreement is worth a look (see the plan's findings) but is + // host-environment-tangled, so flag it rather than gate on it. + let addr: std::net::Ipv4Addr = net + .ip_address + .parse() + .with_context(|| format!("machine IP {:?} is not a valid IPv4", net.ip_address))?; + if addr.is_loopback() || addr.is_unspecified() || addr.is_link_local() { + bail!("machine reported a non-routable IP {addr}"); + } + if !net.ip_address.starts_with("10.0.2.") { + tracing::warn!( + ip = %net.ip_address, + "machine's reported IP is outside the datapath 10.0.2.0/24 (gateway 10.0.2.1, \ + guest 10.0.2.2) — select_routable_ip picked a non-datapath address; \ + investigate on a non-fake-IP host" + ); + } + Ok(()) +} + +/// M5: host→Machine SSH is not implemented; pin that contract so a future +/// SSH feature trips this test and prompts real SSH-networking coverage. +async fn m5_ssh_unimplemented(machines: &mut MachineServiceClient) -> Result<()> { + let status = machines + .ssh_info(SshInfoRequest { + id: MACHINE.to_owned(), + }) + .await + .err() + .context("ssh_info unexpectedly succeeded — SSH may now exist; add real SSH coverage")?; + if status.code() != tonic::Code::Unimplemented { + bail!( + "ssh_info failed with {:?}, expected Unimplemented", + status.code() + ); + } + Ok(()) +} From ed779e7931e7bb0386f7b7b454766cc98ceb03ed Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Sat, 1 Aug 2026 06:00:39 +0800 Subject: [PATCH 2/8] fix(e2e): assert the nslookup answer block, not the resolver banner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M2 matched the gateway anywhere in busybox nslookup's stdout, but busybox echoes its configured resolver ("Server: 10.0.2.1") unconditionally and exits 0 on NXDOMAIN. Since the resolver is the expected answer here, the check could never fail — a broken DnsForwarder mapping still passed. Parse the block after the `Name:` line and require the gateway among the answer addresses. Three unit tests cover the preamble-only (NXDOMAIN), answer, and wrong-answer shapes; they need no VM, so the parser stays honest even when the #[ignore]d scenario is not run. Also correct M4's IP claim to what it enforces. Tracing the reported-IP finding turned up its root cause: the agent fills SystemInfo.ip_addresses from `hostname -I` falling back to `hostname -i`, and busybox's `-i` resolves the guest's own hostname through DNS rather than enumerating interfaces — so the field is a resolver artifact (198.18.11.51 here), not an address the Machine holds. Gate gateway/dns_servers; characterize ip_address and say so. --- internal-docs/plans/machine-network-e2e.md | 35 +++++-- tests/e2e/tests/machine_network.rs | 113 ++++++++++++++++++--- 2 files changed, 124 insertions(+), 24 deletions(-) diff --git a/internal-docs/plans/machine-network-e2e.md b/internal-docs/plans/machine-network-e2e.md index 60486dd99..c234063e6 100644 --- a/internal-docs/plans/machine-network-e2e.md +++ b/internal-docs/plans/machine-network-e2e.md @@ -35,9 +35,9 @@ commands even while testing the network. | # | What | Assertion | |---|---|---| | M1 | egress TCP | `wget` a host-local origin at `10.0.2.1:`; `wc -c` byte-exact — first proof a Machine reaches the network *(implemented)* | -| M2 | DNS | `nslookup host.docker.internal` / `gateway.docker.internal` resolve to `10.0.2.1` via the in-VMM `DnsForwarder` *(implemented)* | +| M2 | DNS | `nslookup host.docker.internal` / `gateway.docker.internal` resolve to `10.0.2.1` via the in-VMM `DnsForwarder`. Asserted on the parsed **answer block only** — busybox echoes its resolver (`Server: 10.0.2.1`) and exits 0 on NXDOMAIN, so matching whole output is a tautology when the resolver is the expected answer *(implemented)* | | M3 | egress volume | 16 MiB download, byte-exact and bounded *(implemented)* | -| M4 | metadata | `inspect` reports gateway `10.0.2.1` and it as a DNS server; IP is a valid routable IPv4 *(implemented)* | +| M4 | metadata | `inspect` reports gateway `10.0.2.1` and it as a DNS server (gated); `ip_address` is characterized only — see the finding below *(implemented)* | | M5 | SSH contract | `ssh_info` is still `Unimplemented` — pins the gap so a future SSH feature trips this test *(implemented)* | ## Not covered — by architecture, not omission @@ -57,16 +57,29 @@ no active test (would be flaky/meaningless today): (`app/arcbox-docker/src/handlers/container/mod.rs`), never to a Machine. M5 pins this. -## Finding (2026-07-21): reported IP ≠ datapath IP on a fake-IP host +## Finding (2026-07-21): `inspect` reports a DNS answer, not the Machine's IP The datapath logs `gateway=10.0.2.1, guest=10.0.2.2`, and egress/DNS work through it (M1–M3 pass), but `inspect().network.ip_address` came back as `198.18.11.51` — the Surge/Clash fake-IP range this host runs, not the -datapath's `10.0.2.2`. `select_routable_ip` -(`app/arcbox-core/src/machine.rs`) picks from the agent's enumerated -addresses and chose a non-datapath address here. The desktop UI shows this -field as "the machine's IP", so a user on such a host sees a bogus address. -M4 asserts the robust facts (gateway, DNS, valid IPv4) and WARN-logs the -mismatch rather than gating on it (host-environment-tangled). Worth -reproducing on a clean, non-fake-IP host to decide whether -`select_routable_ip` should prefer the datapath subnet. +datapath's `10.0.2.2`. + +Root cause (traced 2026-08-01, was filed as "host-environment-tangled"): +the field is not an interface address at all. `select_routable_ip` +(`app/arcbox-core/src/machine.rs`) picks the first usable entry from +`SystemInfo.ip_addresses`, which the guest agent fills by running +`hostname -I`, falling back to `hostname -i` +(`guest/arcbox-agent/src/agent/linux/system_info.rs`). Alpine ships busybox +`hostname`, which has no `-I`, and its `-i` **resolves the guest's own +hostname through DNS** instead of enumerating interfaces. So the reported +address is whatever the resolver answers for the Machine's hostname — +a fake-IP here, plausibly NXDOMAIN (hence an empty field) on a clean host. +The desktop UI shows this as "the machine's IP", so users see a bogus value. + +Consequence for M4: gating on the datapath subnet — or on the guest's real +interfaces — would fail for a producer reason, not a datapath reason. M4 +therefore gates the robust facts (gateway, `dns_servers`) and only +characterizes `ip_address` (valid non-special IPv4 + WARN on a non-datapath +value). The fix belongs in the guest agent (enumerate interfaces — +`/proc/net` or `getifaddrs` — rather than shelling out to `hostname`); +tighten M4 to the datapath subnet once that lands. diff --git a/tests/e2e/tests/machine_network.rs b/tests/e2e/tests/machine_network.rs index da386ca7d..c89f3df3f 100644 --- a/tests/e2e/tests/machine_network.rs +++ b/tests/e2e/tests/machine_network.rs @@ -29,6 +29,7 @@ //! Requires internet (create pulls alpine from the live CDN mirror) and a //! musl-cross `arcbox-agent`, like `machine.rs`. +use std::net::Ipv4Addr; use std::sync::Once; use std::time::Duration; @@ -281,9 +282,42 @@ async fn m1_egress_tcp(machines: &mut MachineServiceClient) -> Result<( Ok(()) } +/// Extracts the *answer* addresses from busybox `nslookup` stdout. +/// +/// The output is two blocks — a resolver preamble, then the answer: +/// +/// ```text +/// Server: 10.0.2.1 +/// Address: 10.0.2.1:53 +/// +/// Name: host.docker.internal +/// Address 1: 10.0.2.1 +/// ``` +/// +/// busybox echoes the configured resolver in that preamble unconditionally +/// and exits 0 even on NXDOMAIN, so a whole-output substring match for the +/// gateway is a tautology here: the resolver *is* the expected answer. Only +/// the block after the `Name:` line is an answer, so parse from there. A +/// successful reverse lookup appends a hostname to the address +/// (`Address 1: 10.0.2.1 host.docker.internal`), hence the first-token split. +fn nslookup_answer_addrs(out: &str) -> Vec { + out.lines() + .skip_while(|line| !line.trim_start().starts_with("Name:")) + .skip(1) + .filter_map(|line| line.trim_start().strip_prefix("Address")) + .filter_map(|rest| rest.split_once(':')) + .filter_map(|(_, value)| value.split_whitespace().next()) + .filter_map(|token| token.parse::().ok()) + .collect() +} + /// M2: the Machine's resolver (`10.0.2.1`, from DHCP) answers the -/// gateway-internal names via the in-VMM `DnsForwarder`. +/// gateway-internal names via the in-VMM `DnsForwarder`. Both names are +/// registered into the `LocalHostsTable` the daemon shares with every VM +/// (`register_host_dns`, `app/arcbox-daemon/src/services.rs`), so a Machine +/// resolves them exactly like a container does. async fn m2_dns(machines: &mut MachineServiceClient) -> Result<()> { + let gateway: Ipv4Addr = GATEWAY.parse().expect("GATEWAY is an IPv4 literal"); for name in ["host.docker.internal", "gateway.docker.internal"] { let (out, exit) = exec_capture( machines, @@ -291,9 +325,12 @@ async fn m2_dns(machines: &mut MachineServiceClient) -> Result<()> { RPC_BUDGET, ) .await?; - // busybox nslookup exits 0 on NXDOMAIN too, so assert on the answer. - if !out.contains(GATEWAY) { - bail!("nslookup {name} did not resolve to {GATEWAY} (exit {exit}): {out:?}"); + let answers = nslookup_answer_addrs(&out); + if !answers.contains(&gateway) { + bail!( + "nslookup {name} answered {answers:?}, expected {GATEWAY} \ + (exit {exit}); full output: {out:?}" + ); } } Ok(()) @@ -318,6 +355,11 @@ async fn m3_egress_volume(machines: &mut MachineServiceClient) -> Resul /// M4: `inspect` reports the gateway and DNS the Machine actually uses — /// metadata the lifecycle test never checks. +/// +/// Scope note: gateway and `dns_servers` are gated; `ip_address` is only +/// characterized (valid, non-special IPv4 + a WARN on a non-datapath value), +/// because that field is fed by a known-broken producer — see the comment at +/// the check. async fn m4_network_metadata(machines: &mut MachineServiceClient) -> Result<()> { let info = machines .inspect(InspectMachineRequest { @@ -336,13 +378,23 @@ async fn m4_network_metadata(machines: &mut MachineServiceClient) -> Re net.dns_servers ); } - // The reported IP must be a real, usable IPv4 — but NOT hardcoded to the - // datapath's 10.0.2.0/24: `select_routable_ip` picks from the agent's - // enumerated addresses, and on this host it returned 198.18.11.51 (the - // Surge/Clash fake-IP range) while the datapath IP is 10.0.2.2. That - // disagreement is worth a look (see the plan's findings) but is - // host-environment-tangled, so flag it rather than gate on it. - let addr: std::net::Ipv4Addr = net + // `ip_address` is characterized, NOT gated, and the weak check below is + // deliberate: the field's producer is broken, so gating would pin the bug. + // `select_routable_ip` (`app/arcbox-core/src/machine.rs`) picks the first + // usable address out of `SystemInfo.ip_addresses`, which the guest agent + // fills from `hostname -I` falling back to `hostname -i` + // (`guest/arcbox-agent/src/agent/linux/system_info.rs`). Alpine ships + // busybox `hostname`, which has no `-I`; its `-i` does not enumerate + // interfaces at all — it *resolves the guest's own hostname through DNS*. + // So on this host the field came back 198.18.11.51 (a Surge/Clash fake-IP + // answer) while the datapath address is 10.0.2.2: a resolver artifact, not + // an address the Machine holds. Asserting datapath membership would fail + // for that reason rather than a datapath reason, and asserting the address + // against the guest's real interfaces would fail too — so assert only what + // holds regardless (syntactically valid, not a special-use address) and + // WARN the mismatch. Fixing the producer is a guest-agent change, tracked + // separately; when it lands, tighten this to the datapath subnet. + let addr: Ipv4Addr = net .ip_address .parse() .with_context(|| format!("machine IP {:?} is not a valid IPv4", net.ip_address))?; @@ -353,13 +405,48 @@ async fn m4_network_metadata(machines: &mut MachineServiceClient) -> Re tracing::warn!( ip = %net.ip_address, "machine's reported IP is outside the datapath 10.0.2.0/24 (gateway 10.0.2.1, \ - guest 10.0.2.2) — select_routable_ip picked a non-datapath address; \ - investigate on a non-fake-IP host" + guest 10.0.2.2) — busybox `hostname -i` resolved the guest hostname via DNS \ + instead of enumerating interfaces; see the plan's finding" ); } Ok(()) } +/// The regression M2 shipped with: the resolver preamble echoes `10.0.2.1` +/// on a *failed* lookup too, so a whole-output match could never fail. These +/// run without a VM, so the parser stays honest even when nobody runs the +/// `#[ignore]`d scenario. +#[test] +fn nslookup_preamble_is_not_an_answer() { + // busybox on NXDOMAIN: preamble on stdout, the error goes to stderr + // (which `exec_capture` drops), and it still exits 0. + let out = "Server:\t10.0.2.1\nAddress:\t10.0.2.1:53\n\n"; + assert!(nslookup_answer_addrs(out).is_empty()); +} + +#[test] +fn nslookup_answer_block_is_parsed() { + let out = "Server:\t10.0.2.1\nAddress:\t10.0.2.1:53\n\n\ + Name: host.docker.internal\nAddress 1: 10.0.2.1\n"; + assert_eq!( + nslookup_answer_addrs(out), + vec![Ipv4Addr::new(10, 0, 2, 1)], + "the answer address must be read from the block after `Name:`" + ); +} + +#[test] +fn nslookup_wrong_answer_is_distinguishable() { + // The failure M2 must catch: resolver is the gateway, answer is not. + let out = "Server:\t10.0.2.1\nAddress:\t10.0.2.1:53\n\n\ + Name: host.docker.internal\nAddress 1: 203.0.113.9 wrong.example\n"; + assert_eq!( + nslookup_answer_addrs(out), + vec![Ipv4Addr::new(203, 0, 113, 9)], + "a reverse-resolved hostname must not swallow the address" + ); +} + /// M5: host→Machine SSH is not implemented; pin that contract so a future /// SSH feature trips this test and prompts real SSH-networking coverage. async fn m5_ssh_unimplemented(machines: &mut MachineServiceClient) -> Result<()> { From 01d1d7805cf8bd196d4f9148a5f366e7eb32e68e Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Sat, 1 Aug 2026 06:50:03 +0800 Subject: [PATCH 3/8] fix(e2e): make the nslookup preamble fixture faithful to busybox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NXDOMAIN fixture used `Address:\t10.0.2.1:53`, which fails parse::() on its own — so the empty-answer assertion held even without the `Name:` gate, and the regression it exists to pin was not actually pinned. busybox 1.37.0 networking/nslookup.c has two output formats and only one of them is dangerous. The FEATURE_NSLOOKUP_BIG build (Alpine's) formats the preamble address through xmalloc_sockaddr2dotted, which keeps the port; the legacy build routes it through print_host, which prints a bare `Address 1: 10.0.2.1` — parseable, and indistinguishable from an answer without the gate. Fixtures now cover both, and the bare-preamble one carries the regression: deleting the gate fails it (verified A/B). Answer shapes differ too (`Address: x` vs `Address 1: x [revhost]`); both are covered. --- tests/e2e/tests/machine_network.rs | 89 +++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 21 deletions(-) diff --git a/tests/e2e/tests/machine_network.rs b/tests/e2e/tests/machine_network.rs index c89f3df3f..0e46a7414 100644 --- a/tests/e2e/tests/machine_network.rs +++ b/tests/e2e/tests/machine_network.rs @@ -284,22 +284,37 @@ async fn m1_egress_tcp(machines: &mut MachineServiceClient) -> Result<( /// Extracts the *answer* addresses from busybox `nslookup` stdout. /// -/// The output is two blocks — a resolver preamble, then the answer: +/// The output is always a resolver preamble, a blank line, then the answer. +/// busybox emits the preamble unconditionally and exits 0 even on NXDOMAIN, +/// so a whole-output substring match for the gateway is a tautology here — +/// the resolver *is* the expected answer. Only the block after the `Name:` +/// line is an answer, so parse from there. +/// +/// busybox 1.37.0 (`networking/nslookup.c`) ships two formats and both must +/// parse. With `FEATURE_NSLOOKUP_BIG` (Alpine's build) the preamble address +/// carries the port, because it formats through `xmalloc_sockaddr2dotted` +/// rather than the `_noport` variant the answers use: +/// +/// ```text +/// Server: 10.0.2.1 +/// Address: 10.0.2.1:53 +/// +/// Name: host.docker.internal +/// Address: 10.0.2.1 +/// ``` +/// +/// The legacy build routes both through `print_host`, so the preamble +/// address is bare — parseable, and thus indistinguishable from an answer +/// without the `Name:` gate. Answers there are numbered, and a successful +/// reverse lookup appends the hostname, hence the first-token split: /// /// ```text /// Server: 10.0.2.1 -/// Address: 10.0.2.1:53 +/// Address 1: 10.0.2.1 /// /// Name: host.docker.internal -/// Address 1: 10.0.2.1 +/// Address 1: 10.0.2.1 host.docker.internal /// ``` -/// -/// busybox echoes the configured resolver in that preamble unconditionally -/// and exits 0 even on NXDOMAIN, so a whole-output substring match for the -/// gateway is a tautology here: the resolver *is* the expected answer. Only -/// the block after the `Name:` line is an answer, so parse from there. A -/// successful reverse lookup appends a hostname to the address -/// (`Address 1: 10.0.2.1 host.docker.internal`), hence the first-token split. fn nslookup_answer_addrs(out: &str) -> Vec { out.lines() .skip_while(|line| !line.trim_start().starts_with("Name:")) @@ -415,19 +430,39 @@ async fn m4_network_metadata(machines: &mut MachineServiceClient) -> Re /// The regression M2 shipped with: the resolver preamble echoes `10.0.2.1` /// on a *failed* lookup too, so a whole-output match could never fail. These /// run without a VM, so the parser stays honest even when nobody runs the -/// `#[ignore]`d scenario. +/// `#[ignore]`d scenario. Fixtures are transcribed from busybox 1.37.0 +/// `networking/nslookup.c`; both builds are covered because the preamble +/// address is bare in one and ported in the other. +/// +/// This is the fixture that makes the `Name:` gate load-bearing: the legacy +/// build's preamble address parses cleanly as an IPv4, so a parser that +/// scanned the whole output would report it as an answer and M2 would go +/// back to passing on a dead resolver. #[test] -fn nslookup_preamble_is_not_an_answer() { - // busybox on NXDOMAIN: preamble on stdout, the error goes to stderr - // (which `exec_capture` drops), and it still exits 0. - let out = "Server:\t10.0.2.1\nAddress:\t10.0.2.1:53\n\n"; +fn nslookup_bare_preamble_address_is_not_an_answer() { + // NXDOMAIN: preamble on stdout, the error goes to stderr (which + // `exec_capture` drops), and busybox still exits 0. + let out = "Server: 10.0.2.1\nAddress 1: 10.0.2.1\n\n"; + assert!( + nslookup_answer_addrs(out).is_empty(), + "the preamble address must not count as an answer" + ); +} + +/// Alpine's `FEATURE_NSLOOKUP_BIG` build, which is what M2 actually runs +/// against. Its preamble address carries `:53`, so it fails to parse even +/// without the gate — hence the bare-address fixture above carries the +/// regression, and this one pins the shape M2 really sees. +#[test] +fn nslookup_ported_preamble_address_is_not_an_answer() { + let out = "Server:\t\t10.0.2.1\nAddress:\t10.0.2.1:53\n\n"; assert!(nslookup_answer_addrs(out).is_empty()); } #[test] -fn nslookup_answer_block_is_parsed() { - let out = "Server:\t10.0.2.1\nAddress:\t10.0.2.1:53\n\n\ - Name: host.docker.internal\nAddress 1: 10.0.2.1\n"; +fn nslookup_big_answer_is_parsed() { + let out = "Server:\t\t10.0.2.1\nAddress:\t10.0.2.1:53\n\n\ + Name:\thost.docker.internal\nAddress: 10.0.2.1\n"; assert_eq!( nslookup_answer_addrs(out), vec![Ipv4Addr::new(10, 0, 2, 1)], @@ -435,15 +470,27 @@ fn nslookup_answer_block_is_parsed() { ); } +#[test] +fn nslookup_legacy_answer_is_parsed() { + let out = "Server: 10.0.2.1\nAddress 1: 10.0.2.1\n\n\ + Name: host.docker.internal\n\ + Address 1: 10.0.2.1 host.docker.internal\n"; + assert_eq!( + nslookup_answer_addrs(out), + vec![Ipv4Addr::new(10, 0, 2, 1)], + "a reverse-resolved hostname must not swallow the address" + ); +} + #[test] fn nslookup_wrong_answer_is_distinguishable() { // The failure M2 must catch: resolver is the gateway, answer is not. - let out = "Server:\t10.0.2.1\nAddress:\t10.0.2.1:53\n\n\ - Name: host.docker.internal\nAddress 1: 203.0.113.9 wrong.example\n"; + let out = "Server:\t\t10.0.2.1\nAddress:\t10.0.2.1:53\n\n\ + Name:\thost.docker.internal\nAddress: 203.0.113.9\n"; assert_eq!( nslookup_answer_addrs(out), vec![Ipv4Addr::new(203, 0, 113, 9)], - "a reverse-resolved hostname must not swallow the address" + "a wrong answer must not be masked by the gateway in the preamble" ); } From 7e226386db255108c4a36ff1639dd85faf6348a3 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Sat, 1 Aug 2026 06:59:23 +0800 Subject: [PATCH 4/8] test(e2e): make M3 a content check, not a length check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M3 downloaded from spawn_blob_server, which repeats one 64 KiB all-zero chunk, and asserted `wc -c` — so any stream arriving 16 MiB long passed, however reordered, duplicated, or corrupted. The merge landed spawn_pattern_server + PatternServer::sha256 (seeded xorshift fill) and network_workload's W15 as the worked example; M3 now uses the same construction and hashes in-guest against the origin's digest. M1 keeps wc -c on purpose: it is the reachability probe, not an integrity one. --- internal-docs/plans/machine-network-e2e.md | 2 +- tests/e2e/tests/machine_network.rs | 33 ++++++++++++++++------ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/internal-docs/plans/machine-network-e2e.md b/internal-docs/plans/machine-network-e2e.md index c234063e6..82b526a96 100644 --- a/internal-docs/plans/machine-network-e2e.md +++ b/internal-docs/plans/machine-network-e2e.md @@ -36,7 +36,7 @@ commands even while testing the network. |---|---|---| | M1 | egress TCP | `wget` a host-local origin at `10.0.2.1:`; `wc -c` byte-exact — first proof a Machine reaches the network *(implemented)* | | M2 | DNS | `nslookup host.docker.internal` / `gateway.docker.internal` resolve to `10.0.2.1` via the in-VMM `DnsForwarder`. Asserted on the parsed **answer block only** — busybox echoes its resolver (`Server: 10.0.2.1`) and exits 0 on NXDOMAIN, so matching whole output is a tautology when the resolver is the expected answer *(implemented)* | -| M3 | egress volume | 16 MiB download, byte-exact and bounded *(implemented)* | +| M3 | egress volume | 16 MiB download from a seeded-pattern origin, hashed in-guest (`sha256sum`) against the origin's digest and bounded — `wc -c` against `spawn_blob_server`'s repeated all-zero chunk would pass on any stream of the right length, however reordered or corrupted *(implemented)* | | M4 | metadata | `inspect` reports gateway `10.0.2.1` and it as a DNS server (gated); `ip_address` is characterized only — see the finding below *(implemented)* | | M5 | SSH contract | `ssh_info` is still `Unimplemented` — pins the gap so a future SSH feature trips this test *(implemented)* | diff --git a/tests/e2e/tests/machine_network.rs b/tests/e2e/tests/machine_network.rs index 0e46a7414..8f52a597d 100644 --- a/tests/e2e/tests/machine_network.rs +++ b/tests/e2e/tests/machine_network.rs @@ -15,7 +15,8 @@ //! — first proof a Machine can reach the network at all. //! - **M2 DNS**: `nslookup host.docker.internal` / `gateway.docker.internal` //! resolve to the gateway via the in-VMM `DnsForwarder`. -//! - **M3 egress volume**: a larger download, byte-exact and bounded. +//! - **M3 egress volume**: a larger download, hashed in-guest against the +//! origin's SHA-256 and bounded. //! - **M4 metadata**: `inspect` reports gateway `10.0.2.1` and that gateway //! as a DNS server. //! - **M5 SSH contract**: `ssh_info` is still `unimplemented` — pins the @@ -37,7 +38,7 @@ use anyhow::{Context, Result, bail}; use arcbox_e2e::boot_assets::{resolve_boot_version, stage_dev_boot_assets}; use arcbox_e2e::daemon::{DaemonConfig, DaemonHandle, connect_unix}; use arcbox_e2e::metrics::RunMetrics; -use arcbox_e2e::net_fixtures::spawn_blob_server; +use arcbox_e2e::net_fixtures::{spawn_blob_server, spawn_pattern_server}; use arcbox_grpc::v1::machine_service_client::MachineServiceClient; use arcbox_protocol::v1::{ CreateMachineRequest, InspectMachineRequest, MachineExecRequest, RemoveMachineRequest, @@ -62,6 +63,9 @@ const GATEWAY: &str = "10.0.2.1"; /// M3 download size — enough to span many segments, small enough to stay /// quick over the loopback-backed egress path. const VOLUME_BYTES: usize = 16 * 1024 * 1024; +/// Pattern seed for M3's origin. Any fixed value works; distinct from +/// `network_workload`'s so a crossed-wires fixture cannot hash-match. +const VOLUME_SEED: u64 = 0x004D_3E2E_5345_4544; fn init_tracing() { TRACING.call_once(|| { @@ -351,19 +355,30 @@ async fn m2_dns(machines: &mut MachineServiceClient) -> Result<()> { Ok(()) } -/// M3: a larger download completes, byte-exact and within budget — the -/// Machine egress path sustains volume, not just a token request. +/// M3: a larger download arrives byte-for-byte correct and within budget — +/// the Machine egress path sustains volume, not just a token request. +/// +/// Hashed in-guest rather than counted: `spawn_blob_server` repeats one +/// 64 KiB all-zero chunk, so `wc -c` passes on any stream that happens to +/// arrive 16 MiB long, however reordered, duplicated, or corrupted. The +/// pattern origin serves a seeded xorshift fill whose SHA-256 the host +/// knows, which is what makes this a content check (same construction as +/// `network_workload`'s W15). async fn m3_egress_volume(machines: &mut MachineServiceClient) -> Result<()> { - let server = spawn_blob_server(VOLUME_BYTES)?; + let server = spawn_pattern_server(VOLUME_BYTES, VOLUME_SEED)?; let url = format!("http://{GATEWAY}:{}/blob", server.port()); - let cmd = format!("wget -q -O - '{url}' | wc -c"); + let cmd = format!("wget -q -O - '{url}' | sha256sum | cut -d' ' -f1"); let (out, exit) = exec_capture(machines, &["/bin/sh", "-c", &cmd], NET_BUDGET).await?; if exit != 0 { bail!("volume wget exited {exit} (out: {out:?})"); } - let got: usize = out.trim().parse().context("parsing wc -c output")?; - if got != VOLUME_BYTES { - bail!("machine received {got} of {VOLUME_BYTES} bytes"); + let got = out.trim(); + if got != server.sha256() { + bail!( + "machine received a corrupt {VOLUME_BYTES}-byte blob: sha256 {got}, \ + expected {}", + server.sha256() + ); } Ok(()) } From a96c8b125e8329ed268929e2b2398525c1f6b49a Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Sat, 1 Aug 2026 07:18:01 +0800 Subject: [PATCH 5/8] test(e2e): let the pattern origin linger before closing, like the blob origin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit spawn_pattern_server closed its socket the instant the last write was queued; spawn_blob_server waits for the client's close first. Make the two origins behave the same — a real HTTP server does not slam the socket shut the moment the body is queued, and two fixtures that model the same thing should not differ in when they close. This is a consistency fix, NOT a fix for the M3 flake. An earlier version of this commit claimed it was, on the strength of 3 green runs after the change; at the ~1-in-4 failure rate since measured, 3 green runs carry almost no signal. The flake is a Machine networking defect (empty guest route table mid-session, ENETUNREACH), filed separately — see the M3 comment. --- tests/e2e/src/net_fixtures.rs | 11 ++++++++++- tests/e2e/tests/machine_network.rs | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/e2e/src/net_fixtures.rs b/tests/e2e/src/net_fixtures.rs index f28fd8196..311189e6b 100644 --- a/tests/e2e/src/net_fixtures.rs +++ b/tests/e2e/src/net_fixtures.rs @@ -221,7 +221,16 @@ pub fn spawn_pattern_server(len: usize, seed: u64) -> Result { if stream.write_all(header.as_bytes()).is_err() { return; } - let _ = stream.write_all(&body); + if stream.write_all(&body).is_err() { + return; + } + // Wait for the client to close before dropping the socket, as + // `spawn_blob_server` does: a real HTTP origin does not slam + // the socket shut the moment the body is queued, and two + // fixtures modelling the same thing should not differ in when + // they close. + let mut sink = [0u8; 1024]; + while matches!(stream.read(&mut sink), Ok(n) if n > 0) {} }); } }); diff --git a/tests/e2e/tests/machine_network.rs b/tests/e2e/tests/machine_network.rs index 8f52a597d..fcad9f498 100644 --- a/tests/e2e/tests/machine_network.rs +++ b/tests/e2e/tests/machine_network.rs @@ -66,6 +66,9 @@ const VOLUME_BYTES: usize = 16 * 1024 * 1024; /// Pattern seed for M3's origin. Any fixed value works; distinct from /// `network_workload`'s so a crossed-wires fixture cannot hash-match. const VOLUME_SEED: u64 = 0x004D_3E2E_5345_4544; +/// SHA-256 of the empty input — what `sha256sum` reports when the download +/// delivered nothing at all. See `m3_egress_volume`. +const EMPTY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; fn init_tracing() { TRACING.call_once(|| { @@ -373,6 +376,21 @@ async fn m3_egress_volume(machines: &mut MachineServiceClient) -> Resul bail!("volume wget exited {exit} (out: {out:?})"); } let got = out.trim(); + // A pipeline's status is its last stage's, so a wget that delivered + // nothing still exits 0 here and hands sha256sum an empty stream — + // whose digest is a perfectly valid hash to compare against. Name that + // case instead of reporting it as generic corruption. + if got == EMPTY_SHA256 { + bail!( + "machine received no data at all: wget delivered nothing and the \ + pipeline masked its failure (hash is the empty-input digest). \ + Known cause: CORE-66 — the Machine's guest route table goes \ + empty mid-session, so the connect fails with ENETUNREACH before \ + any datapath code runs. Confirm with `cat /proc/net/route` in \ + the Machine (`ip` is unusable there); a header-only table is \ + CORE-66, anything else is a new failure" + ); + } if got != server.sha256() { bail!( "machine received a corrupt {VOLUME_BYTES}-byte blob: sha256 {got}, \ From 9469f28ce5d1d9cda4c0f71bab08888c2e31f9d4 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Sat, 1 Aug 2026 11:16:41 +0800 Subject: [PATCH 6/8] docs(e2e): correct the nslookup exit-code and error-stream claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both busybox builds exit 1 on NXDOMAIN, not 0: the legacy path returns (rc != 0) at nslookup.c:139, and the BIG path sets G.exitcode = EXIT_FAILURE at :1015, returned at :1430. The comments claimed otherwise. The error stream differs by build, which the comments also flattened. The legacy path uses bb_error_msg (:132) -> stderr, which exec_capture drops. The BIG build Alpine ships printfs '** server can't find ...' to *stdout* (:1013), so on the shape M2 actually parses the error line is part of the input. The BIG NXDOMAIN fixture now carries that line and asserts it is not read as an answer. Not gating on the exit code, and saying why at the site: the BIG build queries A *and* AAAA (:1345-1347) and sets G.exitcode on either failing. This datapath is IPv4-only by design, so an absent AAAA record would turn a good A answer into exit 1 — the parsed answer stays the honest gate. --- tests/e2e/tests/machine_network.rs | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/e2e/tests/machine_network.rs b/tests/e2e/tests/machine_network.rs index fcad9f498..e0dd29cef 100644 --- a/tests/e2e/tests/machine_network.rs +++ b/tests/e2e/tests/machine_network.rs @@ -292,10 +292,11 @@ async fn m1_egress_tcp(machines: &mut MachineServiceClient) -> Result<( /// Extracts the *answer* addresses from busybox `nslookup` stdout. /// /// The output is always a resolver preamble, a blank line, then the answer. -/// busybox emits the preamble unconditionally and exits 0 even on NXDOMAIN, -/// so a whole-output substring match for the gateway is a tautology here — -/// the resolver *is* the expected answer. Only the block after the `Name:` -/// line is an answer, so parse from there. +/// busybox emits that preamble unconditionally — including on NXDOMAIN, +/// where it echoes the resolver and prints no answer block. Since the +/// resolver here *is* the expected answer, a whole-output substring match +/// for the gateway is a tautology. Only the block after the `Name:` line is +/// an answer, so parse from there. /// /// busybox 1.37.0 (`networking/nslookup.c`) ships two formats and both must /// parse. With `FEATURE_NSLOOKUP_BIG` (Alpine's build) the preamble address @@ -347,6 +348,13 @@ async fn m2_dns(machines: &mut MachineServiceClient) -> Result<()> { RPC_BUDGET, ) .await?; + // `exit` is reported but deliberately not gated on. busybox does + // exit 1 on NXDOMAIN, so it looks like a free extra assertion — but + // the `FEATURE_NSLOOKUP_BIG` build Alpine ships queries A *and* + // AAAA (`nslookup.c:1345-1347`) and sets `G.exitcode` on either + // one's failure. This datapath is IPv4-only by design, so the AAAA + // half can legitimately come back NXDOMAIN and turn a perfectly + // good A answer into exit 1. The parsed answer is the honest gate. let answers = nslookup_answer_addrs(&out); if !answers.contains(&gateway) { bail!( @@ -473,8 +481,9 @@ async fn m4_network_metadata(machines: &mut MachineServiceClient) -> Re /// back to passing on a dead resolver. #[test] fn nslookup_bare_preamble_address_is_not_an_answer() { - // NXDOMAIN: preamble on stdout, the error goes to stderr (which - // `exec_capture` drops), and busybox still exits 0. + // NXDOMAIN on the legacy build: the preamble goes to stdout and the + // error to stderr via `bb_error_msg` (`nslookup.c:132`), which + // `exec_capture` drops. Exit is 1 — `return (rc != 0)` at `:139`. let out = "Server: 10.0.2.1\nAddress 1: 10.0.2.1\n\n"; assert!( nslookup_answer_addrs(out).is_empty(), @@ -488,7 +497,12 @@ fn nslookup_bare_preamble_address_is_not_an_answer() { /// regression, and this one pins the shape M2 really sees. #[test] fn nslookup_ported_preamble_address_is_not_an_answer() { - let out = "Server:\t\t10.0.2.1\nAddress:\t10.0.2.1:53\n\n"; + // This build `printf`s its failure to *stdout* (`nslookup.c:1013`), so + // unlike the legacy shape the error line is part of what M2 parses — + // it must not be mistaken for an answer. Exit is 1 here too + // (`G.exitcode = EXIT_FAILURE` at `:1015`, returned at `:1430`). + let out = "Server:\t\t10.0.2.1\nAddress:\t10.0.2.1:53\n\n\ + ** server can't find host.docker.internal: NXDOMAIN\n"; assert!(nslookup_answer_addrs(out).is_empty()); } From 68e05013d2b9e9baa4e8f141ccc446738a1c1003 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Fri, 14 Aug 2026 08:43:34 +0800 Subject: [PATCH 7/8] =?UTF-8?q?docs(e2e):=20drop=20the=20plan=20doc=20?= =?UTF-8?q?=E2=80=94=20internal-docs=20moved=20to=20the=20company=20repo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `648b6f62` moved `internal-docs/` out of this repo. This branch predates that and would re-create the directory with a planning document that is no longer meant to live here; the plan belongs alongside its siblings in the company repo. The test file carries what a reader of this repo needs. --- internal-docs/plans/machine-network-e2e.md | 85 ---------------------- 1 file changed, 85 deletions(-) delete mode 100644 internal-docs/plans/machine-network-e2e.md diff --git a/internal-docs/plans/machine-network-e2e.md b/internal-docs/plans/machine-network-e2e.md deleted file mode 100644 index 82b526a96..000000000 --- a/internal-docs/plans/machine-network-e2e.md +++ /dev/null @@ -1,85 +0,0 @@ -# Machine networking E2E - -`tests/e2e/tests/machine_network.rs` — the first exercise of a **Machine's -actual network plane**. The lifecycle test (`machine.rs`) only asserts an -agent-reported IP over vsock; it never drives a packet. This runs real -traffic from inside a Machine over the `machines.exec` vsock channel (the -Machine's `docker exec`). - -``` -cargo test -p arcbox-e2e --test machine_network -- --ignored --nocapture -``` - -Needs internet (create pulls alpine from the live `image.arcboxcdn.com` -mirror) and a guest `arcbox-agent` (musl cross-build, or the installed -app's agent staged by newest-mtime). - -## Datapath (source-verified 2026-07-21) - -A Machine's **primary NIC is the same socketpair userspace netstack + -TcpBridge as the System VM's** (`virt/arcbox-vmm/src/vmm/darwin.rs` — -`gateway=10.0.2.1, guest=10.0.2.2`, DHCP + `DnsForwarder` at the gateway). -Egress is pure in-process host-socket proxying -(`common/arcbox-proxy/src/egress/mod.rs`) — **no privileged helper, no host -route, no `/etc/resolver`**. The System VM's helper-installed `172.16/12` -route (`route_reconciler`) is System-VM-only and was never wired to -Machines (`app/arcbox-core/src/machine.rs` never calls it). So Machine -networking runs fully in the isolated e2e daemon. - -The agent channel is vsock, independent of the network plane -(`MachineManager::connect_agent`), which is why `exec` drives in-Machine -commands even while testing the network. - -## Scenarios (one Machine, one boot) - -| # | What | Assertion | -|---|---|---| -| M1 | egress TCP | `wget` a host-local origin at `10.0.2.1:`; `wc -c` byte-exact — first proof a Machine reaches the network *(implemented)* | -| M2 | DNS | `nslookup host.docker.internal` / `gateway.docker.internal` resolve to `10.0.2.1` via the in-VMM `DnsForwarder`. Asserted on the parsed **answer block only** — busybox echoes its resolver (`Server: 10.0.2.1`) and exits 0 on NXDOMAIN, so matching whole output is a tautology when the resolver is the expected answer *(implemented)* | -| M3 | egress volume | 16 MiB download from a seeded-pattern origin, hashed in-guest (`sha256sum`) against the origin's digest and bounded — `wc -c` against `spawn_blob_server`'s repeated all-zero chunk would pass on any stream of the right length, however reordered or corrupted *(implemented)* | -| M4 | metadata | `inspect` reports gateway `10.0.2.1` and it as a DNS server (gated); `ip_address` is characterized only — see the finding below *(implemented)* | -| M5 | SSH contract | `ssh_info` is still `Unimplemented` — pins the gap so a future SSH feature trips this test *(implemented)* | - -## Not covered — by architecture, not omission - -Documented here because the architecture, not the harness, is the reason; -no active test (would be flaky/meaningless today): - -- **Machine ↔ Machine, Machine → container, Machine → System VM**: each VM - gets its own private per-process socketpair netstack; the second (vmnet - bridge) NIC is never brought up guest-side for Machines - (`guest/arcbox-agent/src/init.rs` `machine_init()` does DHCP on the - primary NIC only). Two Machines can even both be `10.0.2.2` — there is no - shared segment and no cross-VM route. When cross-machine networking is - added, M5's pattern (assert-the-gap-then-grow) is the template. -- **host → Machine inbound / SSH**: `ssh_info` is unimplemented and - `InboundListenerManager` is only ever wired to the System VM - (`app/arcbox-docker/src/handlers/container/mod.rs`), never to a Machine. - M5 pins this. - -## Finding (2026-07-21): `inspect` reports a DNS answer, not the Machine's IP - -The datapath logs `gateway=10.0.2.1, guest=10.0.2.2`, and egress/DNS work -through it (M1–M3 pass), but `inspect().network.ip_address` came back as -`198.18.11.51` — the Surge/Clash fake-IP range this host runs, not the -datapath's `10.0.2.2`. - -Root cause (traced 2026-08-01, was filed as "host-environment-tangled"): -the field is not an interface address at all. `select_routable_ip` -(`app/arcbox-core/src/machine.rs`) picks the first usable entry from -`SystemInfo.ip_addresses`, which the guest agent fills by running -`hostname -I`, falling back to `hostname -i` -(`guest/arcbox-agent/src/agent/linux/system_info.rs`). Alpine ships busybox -`hostname`, which has no `-I`, and its `-i` **resolves the guest's own -hostname through DNS** instead of enumerating interfaces. So the reported -address is whatever the resolver answers for the Machine's hostname — -a fake-IP here, plausibly NXDOMAIN (hence an empty field) on a clean host. -The desktop UI shows this as "the machine's IP", so users see a bogus value. - -Consequence for M4: gating on the datapath subnet — or on the guest's real -interfaces — would fail for a producer reason, not a datapath reason. M4 -therefore gates the robust facts (gateway, `dns_servers`) and only -characterizes `ip_address` (valid non-special IPv4 + WARN on a non-datapath -value). The fix belongs in the guest agent (enumerate interfaces — -`/proc/net` or `getifaddrs` — rather than shelling out to `hostname`); -tighten M4 to the datapath subnet once that lands. From 694c039c527139b7bf6a70d44afb5fd513734ad1 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Fri, 14 Aug 2026 10:04:28 +0800 Subject: [PATCH 8/8] docs(e2e): repoint the module doc after the restructure and the plan move --- tests/e2e/tests/machine_network.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/e2e/tests/machine_network.rs b/tests/e2e/tests/machine_network.rs index e0dd29cef..fd291a5a1 100644 --- a/tests/e2e/tests/machine_network.rs +++ b/tests/e2e/tests/machine_network.rs @@ -5,7 +5,7 @@ //! TcpBridge as the System VM's: gateway/DNS `10.0.2.1`, guest `10.0.2.x`, //! egress via in-process host-socket proxying — no privileged helper, no //! host route, so it runs in the isolated e2e daemon -//! (`virt/arcbox-vmm/src/vmm/darwin.rs`, `app/arcbox-core/src/machine.rs`). +//! (`virt/arcbox-vmm/src/vmm/darwin.rs`, `engine/arcbox-engine/src/machine.rs`). //! The existing `machine.rs` test only asserts an agent-reported IP; it //! never drives a packet. This drives real traffic from inside the Machine //! over the `machines.exec` vsock channel (the Machine's `docker exec`). @@ -22,7 +22,8 @@ //! - **M5 SSH contract**: `ssh_info` is still `unimplemented` — pins the //! documented gap so a future SSH feature flags this test to grow. //! -//! Not covered, by architecture (documented in the plan, no active test): +//! Not covered, by architecture (no active test; rationale in +//! `../company/engineering/arcbox/plans/machine-network-e2e.md`): //! Machine↔Machine, Machine→container, Machine→System VM are isolated //! per-VM netstacks with no cross path today; host→Machine inbound/SSH does //! not exist. @@ -57,7 +58,7 @@ const NET_BUDGET: Duration = Duration::from_secs(60); const MACHINE: &str = "e2e-net-alpine"; /// The gateway/DNS IP a Machine's primary NIC always routes through -/// (`darwin.rs` hardcodes it; `app/arcbox-api/src/grpc/machine.rs` documents +/// (`darwin.rs` hardcodes it; `app/arcbox-api/src/connect/machine.rs` documents /// it as `NAT_GATEWAY`). const GATEWAY: &str = "10.0.2.1"; /// M3 download size — enough to span many segments, small enough to stay @@ -436,7 +437,7 @@ async fn m4_network_metadata(machines: &mut MachineServiceClient) -> Re } // `ip_address` is characterized, NOT gated, and the weak check below is // deliberate: the field's producer is broken, so gating would pin the bug. - // `select_routable_ip` (`app/arcbox-core/src/machine.rs`) picks the first + // `select_routable_ip` (`engine/arcbox-engine/src/machine.rs`) picks the first // usable address out of `SystemInfo.ip_addresses`, which the guest agent // fills from `hostname -I` falling back to `hostname -i` // (`guest/arcbox-agent/src/agent/linux/system_info.rs`). Alpine ships @@ -462,7 +463,7 @@ async fn m4_network_metadata(machines: &mut MachineServiceClient) -> Re ip = %net.ip_address, "machine's reported IP is outside the datapath 10.0.2.0/24 (gateway 10.0.2.1, \ guest 10.0.2.2) — busybox `hostname -i` resolved the guest hostname via DNS \ - instead of enumerating interfaces; see the plan's finding" + instead of enumerating interfaces" ); } Ok(())