fix(daemon): controlled exit on SIGTERM — gate new work, drain 5 s, flush

ci/platform_boundary_ledger.tsv

@@ -32,8 +32,6 @@ crates/fbuild-paths/src/executable_hash.rs	attr_cfg	#[cfg(unix)]	1	host	host_mec
 crates/fbuild-paths/src/executable_hash.rs	attr_cfg	#[cfg(unix)]	2	host	host_mechanic
 crates/fbuild-paths/src/executable_hash.rs	attr_cfg	#[cfg(unix)]	3	host	host_mechanic
 crates/fbuild-paths/src/executable_hash.rs	attr_cfg	#[cfg(unix)]	4	host	host_mechanic
-crates/fbuild-paths/src/executable_hash.rs	attr_cfg	#[cfg(unix)]	5	host	host_mechanic
-crates/fbuild-paths/src/executable_hash.rs	native_path	std::os::unix::fs::MetadataExt	0	fs	host_mechanic
 crates/fbuild-python/tests/python_facades.rs	native_path	std::env::current_exe	0	host_executable	host_mechanic
 crates/fbuild-toolchain/src/toolchain/esp_qemu.rs	attr_cfg	#[cfg(not(windows))]	0	host_executable	host_mechanic
 crates/fbuild-toolchain/src/toolchain/esp_qemu.rs	attr_cfg	#[cfg(not(windows))]	1	host_executable	host_mechanic

ci/platform_boundary_research.tsv

@@ -10,11 +10,12 @@ crates/fbuild-core/src/platform/linux/device.rs	33	native_path	std::os::unix::fs
 crates/fbuild-core/src/platform/linux/device.rs	41	native_path	libc::	process	host_mechanic
 crates/fbuild-core/src/platform/linux/device.rs	41	native_path	libc::	process	host_mechanic
 crates/fbuild-core/src/platform/linux/fs.rs	2	native_path	std::os::unix::fs::PermissionsExt	fs	host_mechanic
-crates/fbuild-core/src/platform/linux/fs.rs	40	native_path	std::os::unix::fs::symlink	fs	host_mechanic
-crates/fbuild-core/src/platform/linux/fs.rs	84	native_path	std::os::unix::ffi::OsStrExt	process	host_mechanic
-crates/fbuild-core/src/platform/linux/fs.rs	89	native_path	libc::	process	host_mechanic
-crates/fbuild-core/src/platform/linux/fs.rs	91	native_path	libc::	process	host_mechanic
-crates/fbuild-core/src/platform/linux/fs.rs	101	native_path	libc::	process	host_mechanic
+crates/fbuild-core/src/platform/linux/fs.rs	13	native_path	std::os::unix::fs::MetadataExt	fs	host_mechanic
+crates/fbuild-core/src/platform/linux/fs.rs	57	native_path	std::os::unix::fs::symlink	fs	host_mechanic
+crates/fbuild-core/src/platform/linux/fs.rs	101	native_path	std::os::unix::ffi::OsStrExt	process	host_mechanic
+crates/fbuild-core/src/platform/linux/fs.rs	106	native_path	libc::	process	host_mechanic
+crates/fbuild-core/src/platform/linux/fs.rs	108	native_path	libc::	process	host_mechanic
+crates/fbuild-core/src/platform/linux/fs.rs	118	native_path	libc::	process	host_mechanic
 crates/fbuild-core/src/platform/linux/ipc.rs	1	native_path	interprocess::local_socket	ipc	host_mechanic
 crates/fbuild-core/src/platform/linux/ipc.rs	2	native_path	interprocess::local_socket	ipc	host_mechanic
 crates/fbuild-core/src/platform/linux/ipc.rs	3	native_path	interprocess::os::unix	ipc	host_mechanic
@@ -23,11 +24,12 @@ crates/fbuild-core/src/platform/linux/ipc.rs	66	native_path	std::os::unix::fs::P
 crates/fbuild-core/src/platform/linux/mod.rs	13	compile_host_fact	std::env::consts::ARCH	host	host_mechanic
 crates/fbuild-core/src/platform/linux/process.rs	1	native_path	std::os::unix::process::ExitStatusExt	process	host_mechanic
 crates/fbuild-core/src/platform/macos/fs.rs	2	native_path	std::os::unix::fs::PermissionsExt	fs	host_mechanic
-crates/fbuild-core/src/platform/macos/fs.rs	40	native_path	std::os::unix::fs::symlink	fs	host_mechanic
-crates/fbuild-core/src/platform/macos/fs.rs	84	native_path	std::os::unix::ffi::OsStrExt	process	host_mechanic
-crates/fbuild-core/src/platform/macos/fs.rs	89	native_path	libc::	process	host_mechanic
-crates/fbuild-core/src/platform/macos/fs.rs	91	native_path	libc::	process	host_mechanic
-crates/fbuild-core/src/platform/macos/fs.rs	101	native_path	libc::	process	host_mechanic
+crates/fbuild-core/src/platform/macos/fs.rs	13	native_path	std::os::unix::fs::MetadataExt	fs	host_mechanic
+crates/fbuild-core/src/platform/macos/fs.rs	57	native_path	std::os::unix::fs::symlink	fs	host_mechanic
+crates/fbuild-core/src/platform/macos/fs.rs	101	native_path	std::os::unix::ffi::OsStrExt	process	host_mechanic
+crates/fbuild-core/src/platform/macos/fs.rs	106	native_path	libc::	process	host_mechanic
+crates/fbuild-core/src/platform/macos/fs.rs	108	native_path	libc::	process	host_mechanic
+crates/fbuild-core/src/platform/macos/fs.rs	118	native_path	libc::	process	host_mechanic
 crates/fbuild-core/src/platform/macos/ipc.rs	1	native_path	interprocess::local_socket	ipc	host_mechanic
 crates/fbuild-core/src/platform/macos/ipc.rs	2	native_path	interprocess::local_socket	ipc	host_mechanic
 crates/fbuild-core/src/platform/macos/ipc.rs	3	native_path	socket2::	ipc	host_mechanic
@@ -43,22 +45,22 @@ crates/fbuild-core/src/platform/windows/device.rs	35	native_path	windows_sys::	p
 crates/fbuild-core/src/platform/windows/fs.rs	2	native_path	std::os::windows::ffi::OsStrExt	process	host_mechanic
 crates/fbuild-core/src/platform/windows/fs.rs	3	native_path	std::os::windows::fs	fs	host_mechanic
 crates/fbuild-core/src/platform/windows/fs.rs	4	native_path	std::os::windows::io::AsRawHandle	process	host_mechanic
-crates/fbuild-core/src/platform/windows/fs.rs	50	native_path	std::os::windows::fs::symlink_dir	fs	host_mechanic
-crates/fbuild-core/src/platform/windows/fs.rs	78	native_path	windows_sys::	process	host_mechanic
-crates/fbuild-core/src/platform/windows/fs.rs	133	native_path	windows_sys::	process	host_mechanic
-crates/fbuild-core/src/platform/windows/fs.rs	177	native_path	windows_sys::	process	host_mechanic
+crates/fbuild-core/src/platform/windows/fs.rs	60	native_path	std::os::windows::fs::symlink_dir	fs	host_mechanic
+crates/fbuild-core/src/platform/windows/fs.rs	88	native_path	windows_sys::	process	host_mechanic
+crates/fbuild-core/src/platform/windows/fs.rs	143	native_path	windows_sys::	process	host_mechanic
+crates/fbuild-core/src/platform/windows/fs.rs	187	native_path	windows_sys::	process	host_mechanic
 crates/fbuild-core/src/platform/windows/ipc.rs	1	native_path	interprocess::local_socket	ipc	host_mechanic
 crates/fbuild-core/src/platform/windows/ipc.rs	2	native_path	interprocess::local_socket	ipc	host_mechanic
 crates/fbuild-core/src/platform/windows/ipc.rs	3	native_path	interprocess::os::windows	ipc	host_mechanic
 crates/fbuild-core/src/platform/windows/ipc.rs	4	native_path	interprocess::os::windows	ipc	host_mechanic
 crates/fbuild-core/src/platform/windows/ipc.rs	5	native_path	socket2::	ipc	host_mechanic
 crates/fbuild-core/src/platform/windows/ipc.rs	6	native_path	std::os::windows::io::AsRawSocket	ipc	host_mechanic
 crates/fbuild-core/src/platform/windows/mod.rs	13	compile_host_fact	std::env::consts::ARCH	host	host_mechanic
-crates/fbuild-core/src/platform/windows/process.rs	77	native_path	std::os::windows::ffi::OsStrExt	process	host_mechanic
-crates/fbuild-core/src/platform/windows/process.rs	79	native_path	windows_sys::	process	host_mechanic
-crates/fbuild-core/src/platform/windows/process.rs	82	native_path	windows_sys::	process	host_mechanic
-crates/fbuild-core/src/platform/windows/process.rs	85	native_path	windows_sys::	process	host_mechanic
-crates/fbuild-core/src/platform/windows/process.rs	88	native_path	windows_sys::	process	host_mechanic
+crates/fbuild-core/src/platform/windows/process.rs	87	native_path	std::os::windows::ffi::OsStrExt	process	host_mechanic
+crates/fbuild-core/src/platform/windows/process.rs	89	native_path	windows_sys::	process	host_mechanic
+crates/fbuild-core/src/platform/windows/process.rs	92	native_path	windows_sys::	process	host_mechanic
+crates/fbuild-core/src/platform/windows/process.rs	95	native_path	windows_sys::	process	host_mechanic
+crates/fbuild-core/src/platform/windows/process.rs	98	native_path	windows_sys::	process	host_mechanic
 crates/fbuild-core/src/platform/windows/usb_pnp.rs	13	native_path	windows_sys::	process	host_mechanic
 crates/fbuild-core/src/platform/windows/usb_pnp.rs	24	native_path	windows_sys::	process	host_mechanic
 crates/fbuild-core/src/platform/windows/usb_pnp.rs	28	native_path	windows_sys::	process	host_mechanic
@@ -98,9 +100,7 @@ crates/fbuild-paths/src/executable_hash.rs	23	attr_cfg	#[cfg(unix)]	host	host_me
 crates/fbuild-paths/src/executable_hash.rs	66	attr_cfg	#[cfg(not(unix))]	host	host_mechanic
 crates/fbuild-paths/src/executable_hash.rs	71	attr_cfg	#[cfg(unix)]	host	host_mechanic
 crates/fbuild-paths/src/executable_hash.rs	76	attr_cfg	#[cfg(unix)]	host	host_mechanic
-crates/fbuild-paths/src/executable_hash.rs	78	native_path	std::os::unix::fs::MetadataExt	fs	host_mechanic
-crates/fbuild-paths/src/executable_hash.rs	94	attr_cfg	#[cfg(unix)]	host	host_mechanic
-crates/fbuild-paths/src/executable_hash.rs	103	attr_cfg	#[cfg(all(test,unix))]	host	host_mechanic
+crates/fbuild-paths/src/executable_hash.rs	85	attr_cfg	#[cfg(all(test,unix))]	host	host_mechanic
 crates/fbuild-python/tests/python_facades.rs	317	native_path	std::env::current_exe	host_executable	host_mechanic
 crates/fbuild-toolchain/src/toolchain/esp_qemu.rs	528	attr_cfg	#[cfg(windows)]	host_executable	host_mechanic
 crates/fbuild-toolchain/src/toolchain/esp_qemu.rs	534	attr_cfg	#[cfg(windows)]	host_executable	host_mechanic

ci/test_enforce_platform_boundary.py

@@ -15,10 +15,9 @@ def setUpClass(cls) -> None:
 
     def test_committed_exact_occurrence_ledger_matches_whole_tree(self) -> None:
         # Keep the row count explicit so additions to host mechanics require
-        # a deliberate inventory update. Serial PTY tests and the merged
-        # main-branch daemon/executable changes and the Windows file-URL
-        # resolver fixture and its Windows-only import bring the total to 46.
-        self.assertEqual(len(self.expected), 46)
+        # a deliberate inventory update. The executable identity now lives
+        # behind the platform boundary, leaving 44 occurrences.
+        self.assertEqual(len(self.expected), 44)
         self.assertFalse(boundary.validate_ledger(self.expected))
         self.assertFalse(boundary.compare(self.expected, self.observed))
 
@@ -102,17 +101,10 @@ def test_no_raw_host_fact_reads_remain_outside_the_boundary(self) -> None:
         )
 
     def test_no_filesystem_mechanics_remain_outside_the_boundary(self) -> None:
-        # The executable-hash implementation added on main uses Unix file
-        # metadata; keep that one exception exact and reject any new site.
+        # Filesystem mechanics belong behind the platform boundary.
         self.assertEqual(
             [(row.path, row.kind, row.normalized) for row in self.expected if row.capability == "fs"],
-            [
-                (
-                    "crates/fbuild-paths/src/executable_hash.rs",
-                    "native_path",
-                    "std::os::unix::fs::MetadataExt",
-                )
-            ],
+            [],
         )
 
     def test_rp2040_filesystem_mechanics_use_the_neutral_facade(self) -> None:

crates/fbuild-cli/src/cli/daemon_stop.rs

@@ -75,7 +75,7 @@ const DAEMON_PROCESS_STEM: &str = "fbuild-daemon";
 /// shutting down, it is stuck.
 const GRACEFUL_STOP_BUDGET: std::time::Duration = std::time::Duration::from_secs(5);
 
-/// How long a signalled process gets to actually disappear. Termination is
+/// How long a force-killed process gets to actually disappear. Termination is
 /// asynchronous on both OS families, but a process that has not gone in 5 s
 /// is not going.
 const TERMINATION_BUDGET: std::time::Duration = std::time::Duration::from_secs(5);
@@ -186,10 +186,16 @@ pub async fn run_daemon_stop(client: &DaemonClient) -> fbuild_core::Result<()> {
 /// anomaly — which is why the graceful attempt's exit status is ignored and
 /// only the liveness check decides.
 async fn terminate_and_confirm(pid: u32) -> fbuild_core::Result<()> {
-    if let Err(error) = kill_process(pid, false).await {
-        tracing::debug!(pid, %error, "graceful terminate refused; escalating to a forced kill");
-    }
-    if wait_for_process_exit(pid, TERMINATION_BUDGET).await {
+    // A delivered terminate gets the daemon's full controlled-exit budget; a
+    // refused one keeps the plain liveness wait.
+    let graceful_budget = match kill_process(pid, false).await {
+        Ok(()) => fbuild_core::platform::process::daemon_graceful_termination_budget(),
+        Err(error) => {
+            tracing::debug!(pid, %error, "graceful terminate refused; escalating to a forced kill");
+            TERMINATION_BUDGET
+        }
+    };
+    if wait_for_process_exit(pid, graceful_budget).await {
         return Ok(());
     }
 

crates/fbuild-core/src/daemon_health.rs

@@ -30,6 +30,22 @@ pub const STARTING_BUDGET: Duration = Duration::from_secs(120);
 /// Interval between `/health` polls.
 pub const POLL_INTERVAL: Duration = Duration::from_millis(100);
 
+/// Longest a daemon told to terminate (SIGTERM) waits for in-flight
+/// operations before it exits anyway. New operations are refused from the
+/// moment the signal arrives.
+pub const SHUTDOWN_DRAIN_BUDGET: Duration = Duration::from_secs(5);
+
+/// Cap on the daemon's final zccache flush. A normal flush takes well under
+/// 100 ms; the cap only matters when zccache is stuck behind a slow disk or a
+/// startup load (zackees/zccache#1652).
+pub const EXIT_FLUSH_BUDGET: Duration = Duration::from_secs(4);
+
+/// Longest a terminated daemon can take to exit: drain, then flush. Clients
+/// that send SIGTERM must wait at least this long before escalating to a
+/// forced kill, or the kill lands mid-flush.
+pub const TERMINATE_EXIT_BUDGET: Duration =
+    Duration::from_secs(SHUTDOWN_DRAIN_BUDGET.as_secs() + EXIT_FLUSH_BUDGET.as_secs());
+
 /// What one `/health` probe says about the daemon.
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum DaemonHealth {

crates/fbuild-core/src/platform/fs.rs

@@ -46,6 +46,11 @@ pub fn same_file(left: &Path, right: &Path) -> std::io::Result<bool> {
     Ok(file_identity(left)? == file_identity(right)?)
 }
 
+/// Return a change-sensitive identity for a short-lived executable hash memo.
+pub fn executable_memo_identity(metadata: &std::fs::Metadata) -> std::io::Result<String> {
+    super::selected::fs::executable_memo_identity(metadata)
+}
+
 /// Normalize a lexical path into the host's comparison-key representation.
 #[must_use]
 pub fn comparison_key(path: &Path) -> String {

crates/fbuild-core/src/platform/linux/fs.rs

@@ -9,6 +9,23 @@ pub(crate) fn file_identity(path: &Path) -> std::io::Result<same_file::Handle> {
     same_file::Handle::from_path(path)
 }
 
+pub(crate) fn executable_memo_identity(metadata: &std::fs::Metadata) -> std::io::Result<String> {
+    use std::os::unix::fs::MetadataExt;
+    let modified = metadata
+        .modified()?
+        .duration_since(std::time::UNIX_EPOCH)
+        .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
+    Ok(format!(
+        "{} {} {} {} {}.{}",
+        metadata.len(),
+        modified.as_nanos(),
+        metadata.dev(),
+        metadata.ino(),
+        metadata.ctime(),
+        metadata.ctime_nsec()
+    ))
+}
+
 pub(crate) fn comparison_key(path: &Path) -> String {
     path.as_os_str().to_string_lossy().into_owned()
 }

crates/fbuild-core/src/platform/linux/process.rs

@@ -9,6 +9,25 @@ pub(crate) fn register_daemon_shutdown_handler(
     Ok(())
 }
 
+pub(crate) async fn daemon_terminate_signal() {
+    use tokio::signal::unix::{SignalKind, signal};
+    match signal(SignalKind::terminate()) {
+        Ok(mut sigterm) => {
+            sigterm.recv().await;
+        }
+        Err(error) => {
+            tracing::warn!(
+                "cannot install SIGTERM handler ({error}); SIGTERM will kill the daemon without a flush"
+            );
+            std::future::pending::<()>().await
+        }
+    }
+}
+
+pub(crate) fn daemon_graceful_termination_budget() -> std::time::Duration {
+    std::time::Duration::from_secs(crate::daemon_health::TERMINATE_EXIT_BUDGET.as_secs() + 1)
+}
+
 pub(crate) fn configure_tokio_owner_death(
     command: &mut tokio::process::Command,
 ) -> std::io::Result<()> {

crates/fbuild-core/src/platform/macos/fs.rs

@@ -9,6 +9,23 @@ pub(crate) fn file_identity(path: &Path) -> std::io::Result<same_file::Handle> {
     same_file::Handle::from_path(path)
 }
 
+pub(crate) fn executable_memo_identity(metadata: &std::fs::Metadata) -> std::io::Result<String> {
+    use std::os::unix::fs::MetadataExt;
+    let modified = metadata
+        .modified()?
+        .duration_since(std::time::UNIX_EPOCH)
+        .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
+    Ok(format!(
+        "{} {} {} {} {}.{}",
+        metadata.len(),
+        modified.as_nanos(),
+        metadata.dev(),
+        metadata.ino(),
+        metadata.ctime(),
+        metadata.ctime_nsec()
+    ))
+}
+
 pub(crate) fn comparison_key(path: &Path) -> String {
     path.as_os_str().to_string_lossy().to_lowercase()
 }

crates/fbuild-core/src/platform/macos/process.rs

@@ -9,6 +9,25 @@ pub(crate) fn register_daemon_shutdown_handler(
     Ok(())
 }
 
+pub(crate) async fn daemon_terminate_signal() {
+    use tokio::signal::unix::{SignalKind, signal};
+    match signal(SignalKind::terminate()) {
+        Ok(mut sigterm) => {
+            sigterm.recv().await;
+        }
+        Err(error) => {
+            tracing::warn!(
+                "cannot install SIGTERM handler ({error}); SIGTERM will kill the daemon without a flush"
+            );
+            std::future::pending::<()>().await
+        }
+    }
+}
+
+pub(crate) fn daemon_graceful_termination_budget() -> std::time::Duration {
+    std::time::Duration::from_secs(crate::daemon_health::TERMINATE_EXIT_BUDGET.as_secs() + 1)
+}
+
 pub(crate) fn configure_tokio_owner_death(
     command: &mut tokio::process::Command,
 ) -> std::io::Result<()> {

crates/fbuild-core/src/platform/process.rs

@@ -254,6 +254,19 @@ pub fn register_daemon_shutdown_handler(
     super::selected::process::register_daemon_shutdown_handler(shutdown_tx)
 }
 
+/// Resolve when the host asks the daemon process to terminate (SIGTERM on
+/// Unix). Never resolves on hosts whose termination requests arrive through
+/// [`register_daemon_shutdown_handler`] instead (Windows close/logoff/shutdown).
+pub async fn daemon_terminate_signal() {
+    super::selected::process::daemon_terminate_signal().await
+}
+
+/// How long a caller should wait after a successfully delivered graceful
+/// daemon termination request before escalating to a forced kill.
+pub fn daemon_graceful_termination_budget() -> Duration {
+    super::selected::process::daemon_graceful_termination_budget()
+}
+
 /// Build the host-correct child environment while preserving caller overlays.
 pub(crate) fn command_environment(
     program: &str,

crates/fbuild-core/src/platform/windows/fs.rs

@@ -11,6 +11,16 @@ pub(crate) fn file_identity(path: &Path) -> std::io::Result<same_file::Handle> {
     same_file::Handle::from_path(path)
 }
 
+pub(crate) fn executable_memo_identity(metadata: &std::fs::Metadata) -> std::io::Result<String> {
+    Ok(format!(
+        "{} {} {} {}",
+        metadata.file_size(),
+        metadata.last_write_time(),
+        metadata.creation_time(),
+        metadata.file_attributes()
+    ))
+}
+
 pub(crate) fn comparison_key(path: &Path) -> String {
     let mut value = display_slash(path);
     value.make_ascii_lowercase();

crates/fbuild-core/src/platform/windows/process.rs

@@ -17,6 +17,16 @@ unsafe impl Sync for JobHandle {}
 static TOKIO_JOB: OnceLock<JobHandle> = OnceLock::new();
 static SHUTDOWN_TX: OnceLock<tokio::sync::watch::Sender<bool>> = OnceLock::new();
 
+/// Windows delivers termination requests as console control events, which
+/// `register_daemon_shutdown_handler` already routes to graceful shutdown.
+pub(crate) async fn daemon_terminate_signal() {
+    std::future::pending::<()>().await
+}
+
+pub(crate) fn daemon_graceful_termination_budget() -> std::time::Duration {
+    std::time::Duration::from_secs(5)
+}
+
 pub(crate) fn register_daemon_shutdown_handler(
     shutdown_tx: tokio::sync::watch::Sender<bool>,
 ) -> std::io::Result<()> {

crates/fbuild-daemon/src/README.md

@@ -7,6 +7,7 @@
 - **`context.rs`** -- `DaemonContext` (shared state), `BroadcastHub`, self-eviction/idle timeout constants
 - **`device_manager.rs`** -- `DeviceManager` with exclusive/monitor leases, preemption, and stale device cleanup
 - **`models.rs`** -- Request/response serde types for all API endpoints (build, deploy, monitor, devices, locks, reset)
+- **`shutdown.rs`** -- Exit paths: `refuse_new_operations_when_shutting_down` middleware (503 once shutdown starts), SIGTERM controlled exit (`exit_on_terminate`: drain in-flight operations up to 5 s, then flush), and `persist_and_clean_up` (pid/port/status cleanup + bounded zccache flush) shared by every clean exit
 - **`startup.rs`** -- `StartupGate`: answers every request with `503 {"status":"starting","phase":...}` on a duplicate of the bound listener while `main` initializes, then hands the endpoint to the full router (FastLED/fbuild#1480)
 - **`status_manager.rs`** -- `StatusManager` for atomic read-modify-write of `daemon_status.json`
 - **`handlers/`** -- HTTP and WebSocket route handler modules