From edbcb9b1a2500524b77287fd198b2fb5ad5c294c Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Sat, 24 Oct 2026 21:41:20 -0700 Subject: [PATCH 1/2] fix(daemon): keep the daemon port out of OS ephemeral ranges and bind loopback (#1700) On Linux the default daemon window (49152-65535) overlapped the kernel's ephemeral range (32768-60999), so an unrelated outgoing connection could hold the daemon's port; the 0.0.0.0 bind then failed with EADDRINUSE while nothing listened, and startup burned its spawn budget retrying. Non-Windows daemons now take 10000-32767, below the Linux and macOS ephemeral ranges, and bind 127.0.0.1 instead of 0.0.0.0 (every client connects to loopback). Adds a Linux repro of the wildcard-vs-loopback collision and a test against the host's live ip_local_port_range. --- CLAUDE.md | 2 +- crates/CLAUDE.md | 2 +- crates/fbuild-core/src/platform/linux/ipc.rs | 32 +++++++++++++++ crates/fbuild-daemon/src/handlers/health.rs | 2 +- crates/fbuild-daemon/src/main.rs | 5 ++- crates/fbuild-paths/README.md | 2 +- crates/fbuild-paths/src/lib.rs | 43 ++++++++++++++++---- 7 files changed, 74 insertions(+), 14 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 46041dcab..3d318e4df 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -176,7 +176,7 @@ The current inventory is auto-published to a stable tracking issue every Monday ## Key Constraints - **No file-based locks, with one sanctioned exception** — almost all synchronization is through the daemon's in-memory managers. The narrow exception is fbuild-daemon startup/lifetime root-ownership and spawn-herd election (soldr-style, FastLED/fbuild#1159): a version-blind `root-owner.lock` held for the daemon's whole lifetime, plus a `spawn.lock` single-flight election for concurrent CLI spawns. zccache compile/object access keeps its own zccache-internal in-memory synchronization — these locks never gate cache reads/writes. Locks are OS-released on process death and must never be manually broken or deleted. -- **Dev mode isolation** — `FBUILD_DEV_MODE=1` → `~/.fbuild/dev/`. The daemon endpoint is no longer a fixed port: it's derived per (backend version + cache identity) in 10000–49151 on Windows (below commonly excluded dynamic ports; FastLED/fbuild#1322) and the IANA dynamic range 49152–65535 elsewhere (`fbuild_paths::default_daemon_port` / `daemon_endpoint_key`), so different-version checkouts get isolated daemons and can't serve each other wrong-version builds (FastLED/fbuild#1009). Override with `FBUILD_DAEMON_PORT`. +- **Dev mode isolation** — `FBUILD_DEV_MODE=1` → `~/.fbuild/dev/`. The daemon endpoint is no longer a fixed port: it's derived per (backend version + cache identity) in 10000–49151 on Windows (below commonly excluded dynamic ports; FastLED/fbuild#1322) and 10000–32767 elsewhere, below the Linux/macOS ephemeral ranges so outgoing connections cannot occupy it (FastLED/fbuild#1700) (`fbuild_paths::default_daemon_port` / `daemon_endpoint_key`), so different-version checkouts get isolated daemons and can't serve each other wrong-version builds (FastLED/fbuild#1009). Override with `FBUILD_DAEMON_PORT`. - **HTTP API compatibility** — same endpoints and JSON schemas as the Python daemon - **Windows USB-CDC** — 30 retries, aggressive buffer drain, DTR/RTS toggling after flash - **Emulator CLI convention** — prefer `fbuild test-emu` for CI; `fbuild deploy --to emu [--emulator ]` for interactive use; keep `--target` and `--qemu` only as compatibility aliases diff --git a/crates/CLAUDE.md b/crates/CLAUDE.md index 19132cd8b..918f76b46 100644 --- a/crates/CLAUDE.md +++ b/crates/CLAUDE.md @@ -59,7 +59,7 @@ fbuild-test-support (test utilities) ────────────── - **fbuild-core** — `FbuildError`/`Result`, `BuildProfile`, `Platform`, `SizeInfo`, `DaemonState`. USB identity catalogues are fetched from FastLED/boards and must never be generated or embedded here outside test fixtures. - **fbuild-config** — `PlatformIOConfig` (INI parser with `extends` inheritance), `BoardConfig`, `McuSpec` -- **fbuild-paths** — Dev/prod path isolation (`~/.fbuild/{dev|prod}/`), version+identity-keyed daemon endpoint (`daemon_endpoint_key`/`default_daemon_port`, Windows 10000–49151 to avoid excluded dynamic ports, 49152–65535 elsewhere; FastLED/fbuild#1009), cache dirs +- **fbuild-paths** — Dev/prod path isolation (`~/.fbuild/{dev|prod}/`), version+identity-keyed daemon endpoint (`daemon_endpoint_key`/`default_daemon_port`, Windows 10000–49151 to avoid excluded dynamic ports, 10000–32767 elsewhere, below the OS ephemeral ranges (FastLED/fbuild#1700); FastLED/fbuild#1009), cache dirs - **fbuild-packages** — URL-based package downloads, toolchain resolution, library manager, parallel pipeline - **fbuild-serial** — `SharedSerialManager` (centralized serial I/O), deploy preemption protocol, WebSocket messages, USB-CDC retry logic - **fbuild-build** — `BuildOrchestrator` trait, per-platform orchestrators (AVR, ESP32, ESP8266, RP2040, STM32, Teensy, WASM) diff --git a/crates/fbuild-core/src/platform/linux/ipc.rs b/crates/fbuild-core/src/platform/linux/ipc.rs index 339b83c6c..27df85a5f 100644 --- a/crates/fbuild-core/src/platform/linux/ipc.rs +++ b/crates/fbuild-core/src/platform/linux/ipc.rs @@ -75,4 +75,36 @@ mod tests { assert_eq!(mode & 0o077, 0); drop(listener); } + + /// FastLED/fbuild#1700: an outgoing connection whose source is + /// `127.0.0.2:P` makes a wildcard listener on `P` fail with EADDRINUSE + /// although nothing listens there, which the daemon used to report as + /// "stale state" and retry until its spawn budget ran out. A loopback + /// listener on the same port binds. + #[tokio::test] + async fn outgoing_socket_blocks_wildcard_bind_but_not_loopback() { + use socket2::{Domain, Protocol, Socket, Type}; + + let server = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let client = Socket::new(Domain::IPV4, Type::STREAM, Some(Protocol::TCP)).unwrap(); + client + .bind(&"127.0.0.2:0".parse::().unwrap().into()) + .unwrap(); + client + .connect(&server.local_addr().unwrap().into()) + .unwrap(); + let port = client.local_addr().unwrap().as_socket().unwrap().port(); + + let wildcard = crate::platform::ipc::bind_tcp_listener( + format!("0.0.0.0:{port}").parse().unwrap(), + ); + assert!( + wildcard.is_err(), + "wildcard bind on {port} unexpectedly succeeded" + ); + let loopback = crate::platform::ipc::bind_tcp_listener( + format!("127.0.0.1:{port}").parse().unwrap(), + ); + assert!(loopback.is_ok(), "loopback bind on {port}: {loopback:?}"); + } } diff --git a/crates/fbuild-daemon/src/handlers/health.rs b/crates/fbuild-daemon/src/handlers/health.rs index 54c868310..24b780a82 100644 --- a/crates/fbuild-daemon/src/handlers/health.rs +++ b/crates/fbuild-daemon/src/handlers/health.rs @@ -168,7 +168,7 @@ pub async fn daemon_info(State(ctx): State>) -> Json, ) -> (StatusCode, Json) { - // Loopback only. The daemon binds 0.0.0.0 (see `main.rs`), and this + // Loopback only. The daemon binds 127.0.0.1 (see `main.rs`), but this // endpoint is not a read: it can switch process-wide profiling on and make // the daemon serialize its whole heap on demand. Reachable from off-box // that is both a denial-of-service lever and a way to extract allocation diff --git a/crates/fbuild-daemon/src/main.rs b/crates/fbuild-daemon/src/main.rs index 7713214ed..e744f3a66 100644 --- a/crates/fbuild-daemon/src/main.rs +++ b/crates/fbuild-daemon/src/main.rs @@ -159,7 +159,10 @@ async fn main() { // `another_daemon_is_listening`) instead of paying for full initialization // only to fail the bind at the very end. The winner keeps `listener` and // serves everyone. - let addr = format!("0.0.0.0:{}", port); + // Loopback only: every client (CLI, PyO3 bindings) connects to 127.0.0.1, + // and a wildcard bind also collides with outgoing sockets bound to other + // local addresses on the same port (FastLED/fbuild#1700). + let addr = format!("127.0.0.1:{}", port); // Clean up a stale PID file from a previous unclean kill so the bind-retry // path (kernel TIME_WAIT) and `fbuild daemon list` reflect reality. if let Some(stale_pid) = read_stale_daemon_pid().await { diff --git a/crates/fbuild-paths/README.md b/crates/fbuild-paths/README.md index 66488784b..29d13e3f8 100644 --- a/crates/fbuild-paths/README.md +++ b/crates/fbuild-paths/README.md @@ -9,7 +9,7 @@ Single source of truth for all `.fbuild` directory paths, with dev/prod isolatio - `get_other_fbuild_root()` -- Returns the opposite mode's root (for cross-mode daemon discovery) - `get_daemon_dir()` / `get_daemon_pid_file()` / `get_daemon_port_file()` / `get_daemon_log_file()` / `get_daemon_status_file()` -- Daemon file paths - `daemon_ownership` module -- `RootOwnershipGuard` (version-blind, per-cache-root exclusive lock at `root-owner.lock`, held by the daemon for its whole lifetime; `fbuild clean cache` takes it exclusively before deleting the zccache store), `SpawnLockGuard` (`spawn.lock` single-flight election so concurrent CLI spawns don't race), and `OwnerClaim`/`write_owner_claim()`/`read_owner_claim()`/`remove_owner_claim()` (a `root-owner.json` claim recording pid/exe/version/mode/cache_root_key/port, written after the daemon acquires ownership and knows its port; never authoritative on its own — always verified against pid liveness + exe identity before acting on it) -- `get_daemon_port()` -- Port resolution priority: explicit environment override, live endpoint-keyed port file, then a deterministic version/cache-identity default (10000–49151 on Windows to avoid dynamic exclusion blocks; 49152–65535 elsewhere) +- `get_daemon_port()` -- Port resolution priority: explicit environment override, live endpoint-keyed port file, then a deterministic version/cache-identity default (10000–49151 on Windows to avoid dynamic exclusion blocks; 10000–32767 elsewhere, below the OS ephemeral ranges) - `get_daemon_url()` -- Daemon HTTP URL (`http://127.0.0.1:{port}`) - `get_cache_root()` -- Global cache dir (`FBUILD_CACHE_DIR` override or `~/.fbuild/{mode}/cache`) - `long_paths::initialize()` -- Captures the Windows long-path policy and the requested/effective paths once, before cache or daemon work. When the policy is disabled or cannot be read, selects a writable short cache/scratch root. Automatic roots are direct local-drive children such as `C:\fb-\{dev|prod}`, protected for the current user; foreign-owned directories and links/reparse points are rejected. A suitably short `FBUILD_CACHE_DIR` wins and uses `t`/`b` sidecars within that intentionally configured cache, without requiring access to the drive root or changing the configured root's ACL. A deep override is reported and redirected. The automatic selection is persisted under the normal fbuild root and its private hierarchy verified on reuse, so enabling long paths later reuses downloaded packages. Environment changes take effect in the next process. diff --git a/crates/fbuild-paths/src/lib.rs b/crates/fbuild-paths/src/lib.rs index 54155b48a..4992ae19a 100644 --- a/crates/fbuild-paths/src/lib.rs +++ b/crates/fbuild-paths/src/lib.rs @@ -154,8 +154,11 @@ fn endpoint_key_from_material(material: &str) -> String { /// Deterministic default daemon port derived from [`daemon_endpoint_key`]. /// /// On Windows this lands in 10000–49151, below the dynamic range that -/// Hyper-V/HNS commonly reserves in excluded blocks. Other platforms use the -/// IANA dynamic range (49152–65535). The result is stable for a given +/// Hyper-V/HNS commonly reserves in excluded blocks. Other platforms use +/// 10000–32767, below both the Linux (32768–60999) and macOS (49152–65535) +/// ephemeral ranges: a daemon port inside them can be taken by any outgoing +/// connection's source port, which fails the wildcard/loopback bind with +/// EADDRINUSE while nothing is listening (FastLED/fbuild#1700). The result is stable for a given /// version+identity; distinct versions (and dev vs prod, since mode is part of /// the identity) get distinct ports. `FBUILD_DAEMON_PORT` still overrides this /// (see [`get_daemon_port`]). @@ -163,13 +166,17 @@ pub fn default_daemon_port() -> u16 { port_from_endpoint_key(&daemon_endpoint_key()) } +/// First port of the default Linux ephemeral range (`ip_local_port_range`). +/// The non-Windows daemon window stays below it (FastLED/fbuild#1700). +const LINUX_EPHEMERAL_LOW: u32 = 32768; + /// Map a hex endpoint key into the platform's default daemon-port window. /// Pure and deterministic (unit-tested). fn port_from_endpoint_key(key: &str) -> u16 { let (low, span): (u32, u32) = if fbuild_core::platform::host::is_windows() { (10000, 49152 - 10000) } else { - (49152, 65536 - 49152) + (10000, LINUX_EPHEMERAL_LOW - 10000) }; let n = u64::from_str_radix(key, 16).unwrap_or(0); @@ -634,12 +641,33 @@ mod tests { ); } else { assert!( - (49152..=65535).contains(&p), - "port {p} outside dynamic range" + (10000..32768).contains(&p), + "port {p} is inside an OS ephemeral range" ); } } + #[test] + fn default_daemon_port_is_outside_this_hosts_ephemeral_range() { + // FastLED/fbuild#1700: read the live range rather than trusting the + // default, since distros and CI images can widen it. + let Ok(range) = std::fs::read_to_string("/proc/sys/net/ipv4/ip_local_port_range") else { + return; + }; + let bounds: Vec = range + .split_whitespace() + .filter_map(|n| n.parse().ok()) + .collect(); + let [low, high] = bounds[..] else { + panic!("unexpected ip_local_port_range: {range:?}"); + }; + let p = default_daemon_port(); + assert!( + !(low..=high).contains(&p), + "daemon port {p} is inside the ephemeral range {low}-{high}" + ); + } + #[test] fn port_from_key_is_deterministic_and_ranged() { // Same key → same port; keys differing (e.g. by version or checkout) @@ -653,10 +681,7 @@ mod tests { if fbuild_core::platform::host::is_windows() { assert!((10000..49152).contains(&p), "key {key} -> {p} out of range"); } else { - assert!( - (49152..=65535).contains(&p), - "key {key} -> {p} out of range" - ); + assert!((10000..32768).contains(&p), "key {key} -> {p} out of range"); } } // Two distinct version/identity keys should not collapse to one port From 21f3b1336b5ec438fad06150fc2607d9d48f8f74 Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Sat, 24 Oct 2026 21:45:14 -0700 Subject: [PATCH 2/2] ci: refresh platform-boundary inventory for the #1700 socket2 test --- ci/platform_boundary_research.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/platform_boundary_research.tsv b/ci/platform_boundary_research.tsv index b2113ac42..5d77da2ce 100644 --- a/ci/platform_boundary_research.tsv +++ b/ci/platform_boundary_research.tsv @@ -26,6 +26,7 @@ crates/fbuild-core/src/platform/linux/ipc.rs 2 native_path interprocess::local_s crates/fbuild-core/src/platform/linux/ipc.rs 3 native_path interprocess::os::unix ipc host_mechanic crates/fbuild-core/src/platform/linux/ipc.rs 4 native_path socket2:: ipc host_mechanic crates/fbuild-core/src/platform/linux/ipc.rs 66 native_path std::os::unix::fs::PermissionsExt ipc host_mechanic +crates/fbuild-core/src/platform/linux/ipc.rs 86 native_path socket2:: ipc host_mechanic crates/fbuild-core/src/platform/linux/mod.rs 14 compile_host_fact std::env::consts::ARCH host host_mechanic crates/fbuild-core/src/platform/linux/process.rs 1 native_path std::os::unix::process::ExitStatusExt process host_mechanic crates/fbuild-core/src/platform/macos/fs.rs 2 native_path std::os::unix::fs::PermissionsExt fs host_mechanic