From dd119dfa13a7faee74b9ecf7da212499e70c0b0c Mon Sep 17 00:00:00 2001 From: Bill Sommers Date: Tue, 10 Nov 2026 01:07:39 -0500 Subject: [PATCH 1/6] feat(secure_fs): durable state writes with a last-good backup write_private_file_durable keeps the previous file as .bak (hard link), writes temp + fsync + rename and fsyncs the directory. read_with_backup falls back to the backup when the primary is missing, truncated or invalid, puts the good copy back and keeps the damaged one as .corrupt. Co-Authored-By: Claude Fable 5.2 --- src/secure_fs.rs | 75 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/src/secure_fs.rs b/src/secure_fs.rs index bb96a7f2..52a63214 100644 --- a/src/secure_fs.rs +++ b/src/secure_fs.rs @@ -219,6 +219,81 @@ pub fn write_private_file(path: &Path, bytes: &[u8]) -> io::Result<()> { Ok(()) } +/// Flushes directory metadata so a completed `rename` survives power loss (best effort). +fn sync_dir(dir: &Path) { + #[cfg(unix)] + if let Ok(d) = File::open(dir) { + let _ = d.sync_all(); + } + #[cfg(not(unix))] + let _ = dir; +} + +/// Path of the last-good backup kept next to a durable state file (`.bak`). +pub fn backup_path(path: &Path) -> PathBuf { + let mut name = path.file_name().map(|n| n.to_os_string()).unwrap_or_default(); + name.push(".bak"); + path.with_file_name(name) +} + +/// Durable replacement of a state file: the current file becomes `.bak` (hard link, so the +/// old content is never copied or truncated), then the new content is written to a sibling temp +/// file, fsynced, renamed into place, and the directory is fsynced. +pub fn write_private_file_durable(path: &Path, bytes: &[u8]) -> io::Result<()> { + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + if fs::symlink_metadata(path).is_ok_and(|md| md.is_file() && md.len() > 0) { + let bak = backup_path(path); + let staging = parent.join(format!(".bak.tmp.{}", unique_suffix())); + if fs::hard_link(path, &staging).is_ok() && fs::rename(&staging, &bak).is_err() { + let _ = fs::remove_file(&staging); + } + } + write_private_file(path, bytes)?; + sync_dir(parent); + Ok(()) +} + +/// Where the content returned by [`read_with_backup`] came from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DurableSource { + Primary, + /// The primary file was missing, truncated or invalid; `.bak` was used and written + /// back as the primary. A corrupt primary is kept as `.corrupt`. + Backup, +} + +/// Reads a durable state file, falling back to `.bak` when the primary is unreadable or +/// rejected by `valid`. Errors only when neither copy is usable; never returns invalid content. +pub fn read_with_backup(path: &Path, valid: &dyn Fn(&str) -> bool) -> io::Result<(String, DurableSource)> { + let primary = fs::read_to_string(path); + if let Ok(text) = &primary + && valid(text) + { + return Ok((text.clone(), DurableSource::Primary)); + } + let bak = backup_path(path); + match fs::read_to_string(&bak) { + Ok(text) if valid(&text) => { + if primary.is_ok() { + let mut name = path.file_name().map(|n| n.to_os_string()).unwrap_or_default(); + name.push(".corrupt"); + let _ = fs::rename(path, path.with_file_name(name)); + } + // Put the good copy back without rotating the corrupt file into the backup slot. + write_private_file(path, text.as_bytes())?; + sync_dir(path.parent().unwrap_or_else(|| Path::new("."))); + Ok((text, DurableSource::Backup)) + } + _ => match primary { + Ok(_) => Err(io::Error::new( + ErrorKind::InvalidData, + format!("{} is corrupt and no usable backup exists", path.display()), + )), + Err(e) => Err(e), + }, + } +} + /// Reads an authentication token from `path`. /// /// Verifies the file is a regular file owned by the effective UID with mode 0600 (no group/world permissions). From 6c5136fc83c4dba219d986e1ce281b4ed11c861d Mon Sep 17 00:00:00 2001 From: Bill Sommers Date: Tue, 10 Nov 2026 01:07:39 -0500 Subject: [PATCH 2/6] fix(daemon): flock is the only liveness truth; reap children; show crashed daemons - liveness.rs: daemon state comes from the lockfile flock (held = running, record without a holder = crashed, empty = stopped), so zombies and reused PIDs are never mistaken for a live daemon; kill(pid,0) also rejects zombies. - The lockfile is never unlinked (release/stop truncate the record under the lock), which removes the race where a concurrent starter lost its file. - acquire_lock no longer deletes sockets from a lock it does not hold; stale sockets are only removed after a refused connect. - spawn_daemon_in / spawn_background_daemon_for_session reap the child. - stop_daemon refuses to act on its own process instead of unlinking its own lock and socket while still running. - daemon.status / the daemon window report 'crashed' instead of 'locked'; daemon.* RPC resolves the runtime dir like the colon command does. - phase55_daemon_parity no longer locks and stops 'default' in the user's real runtime dir. Co-Authored-By: Claude Fable 5.2 --- src/api/rpc_daemon.rs | 13 +- src/daemon/liveness.rs | 145 ++++++++++++++ src/daemon/mod.rs | 2 + src/daemon/process.rs | 7 +- src/daemon/registry.rs | 47 ++++- src/daemon/supervisor.rs | 233 +++++++++------------- src/ui/daemon_window_ui.rs | 2 + src/wm/daemon_window.rs | 56 ++++-- tests/daemon_resilience_lifecycle.rs | 108 ++++++++++ tests/phase55_daemon_lifecycle.rs | 5 +- tests/phase55_daemon_lock_and_registry.rs | 6 +- tests/phase55_daemon_parity.rs | 25 ++- 12 files changed, 472 insertions(+), 177 deletions(-) create mode 100644 src/daemon/liveness.rs create mode 100644 tests/daemon_resilience_lifecycle.rs diff --git a/src/api/rpc_daemon.rs b/src/api/rpc_daemon.rs index 84811581..d03dc143 100644 --- a/src/api/rpc_daemon.rs +++ b/src/api/rpc_daemon.rs @@ -3,13 +3,12 @@ use crate::api::json::JsonValue; use crate::daemon::registry::DaemonRegistry; use crate::daemon::supervisor::{ - daemon_lock_path_in, ensure_started, stop_daemon, DaemonLockRecord, DaemonStartResult, + daemon_lock_path_in, ensure_started, resolve_runtime_dir, stop_daemon, DaemonLockRecord, DaemonStartResult, }; -use crate::secure_fs::private_runtime_dir; /// Handles `daemon.list`. pub fn handle_daemon_list() -> Result { - let runtime_dir = private_runtime_dir().map_err(|e| format!("runtime directory error: {e}"))?; + let runtime_dir = resolve_runtime_dir().map_err(|e| format!("runtime directory error: {e}"))?; let active = DaemonRegistry::list_active(&runtime_dir); let mut daemons = Vec::new(); @@ -36,7 +35,7 @@ pub fn handle_daemon_list() -> Result { /// Handles `daemon.status`. pub fn handle_daemon_status(params: &JsonValue) -> Result { - let runtime_dir = private_runtime_dir().map_err(|e| format!("runtime directory error: {e}"))?; + let runtime_dir = resolve_runtime_dir().map_err(|e| format!("runtime directory error: {e}"))?; let name = params .get("name") .and_then(JsonValue::as_str) @@ -76,7 +75,7 @@ pub fn handle_daemon_status(params: &JsonValue) -> Result { "socket_path", JsonValue::String(rec.socket_path.to_string_lossy().into_owned()), ), - ("status", JsonValue::String("locked".to_string())), + ("status", JsonValue::String("crashed".to_string())), ])) } else { Ok(JsonValue::obj([ @@ -88,7 +87,7 @@ pub fn handle_daemon_status(params: &JsonValue) -> Result { /// Handles `daemon.start`. pub fn handle_daemon_start(params: &JsonValue) -> Result { - let runtime_dir = private_runtime_dir().map_err(|e| format!("runtime directory error: {e}"))?; + let runtime_dir = resolve_runtime_dir().map_err(|e| format!("runtime directory error: {e}"))?; let name = params .get("name") .and_then(JsonValue::as_str) @@ -119,7 +118,7 @@ pub fn handle_daemon_start(params: &JsonValue) -> Result { /// Handles `daemon.stop`. pub fn handle_daemon_stop(params: &JsonValue) -> Result { - let runtime_dir = private_runtime_dir().map_err(|e| format!("runtime directory error: {e}"))?; + let runtime_dir = resolve_runtime_dir().map_err(|e| format!("runtime directory error: {e}"))?; let name = params .get("name") .and_then(JsonValue::as_str) diff --git a/src/daemon/liveness.rs b/src/daemon/liveness.rs new file mode 100644 index 00000000..e8bf8ec8 --- /dev/null +++ b/src/daemon/liveness.rs @@ -0,0 +1,145 @@ +//! Daemon liveness derived from the lockfile `flock` (`DAEM-05`). +//! +//! The kernel drops a `flock` when its holder dies, so "is the lock held?" cannot be fooled by +//! a zombie or by a PID that was reused by an unrelated process. The lockfile is never unlinked: +//! a clean exit truncates the record, so a record nobody holds means the daemon crashed. + +use std::fs::File; +use std::io::{self, Read}; +use std::path::Path; +use std::time::{Duration, Instant}; + +use crate::daemon::supervisor::{daemon_lock_path_in, DaemonLockRecord}; + +/// What the lockfile says about daemon `name`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DaemonLiveness { + /// A live process holds the lock. + Running(DaemonLockRecord), + /// A record is present but nobody holds the lock: the daemon died without cleaning up. + Crashed(DaemonLockRecord), + /// No lockfile, or an empty one left by a clean exit. + Stopped, +} + +impl DaemonLiveness { + /// Status word used by `:daemon status`, `daemon.status` and the daemon window. + pub fn label(&self) -> &'static str { + match self { + Self::Running(_) => "running", + Self::Crashed(_) => "crashed", + Self::Stopped => "stopped", + } + } +} + +/// `true` when `pid` is a zombie (exited, not yet reaped). Linux only; `false` elsewhere. +pub fn pid_is_zombie(pid: u32) -> bool { + #[cfg(target_os = "linux")] + { + // `/proc//stat` is `pid (comm) state ...`; `comm` may contain spaces and parens. + std::fs::read_to_string(format!("/proc/{pid}/stat")) + .ok() + .and_then(|s| s.rsplit_once(')').map(|(_, rest)| rest.trim_start().starts_with(['Z', 'X']))) + .unwrap_or(false) + } + #[cfg(not(target_os = "linux"))] + { + let _ = pid; + false + } +} + +#[cfg(unix)] +fn try_flock(file: &File, op: libc::c_int) -> io::Result { + use std::os::unix::io::AsRawFd; + if unsafe { libc::flock(file.as_raw_fd(), op | libc::LOCK_NB) } == 0 { + return Ok(true); + } + let err = io::Error::last_os_error(); + if err.raw_os_error() == Some(libc::EWOULDBLOCK) { Ok(false) } else { Err(err) } +} + +#[cfg(unix)] +fn unlock(file: &File) { + use std::os::unix::io::AsRawFd; + unsafe { + libc::flock(file.as_raw_fd(), libc::LOCK_UN); + } +} + +fn read_record(lock_path: &Path) -> Option { + let mut s = String::new(); + File::open(lock_path).ok()?.read_to_string(&mut s).ok()?; + DaemonLockRecord::from_json(&s).ok() +} + +/// Reads the liveness of daemon `name` from its lockfile. Never signals, never unlinks. +pub fn daemon_liveness(runtime_dir: &Path, name: &str) -> DaemonLiveness { + let Ok(lock_path) = daemon_lock_path_in(runtime_dir, name) else { + return DaemonLiveness::Stopped; + }; + lock_liveness(&lock_path) +} + +/// [`daemon_liveness`] for an explicit lockfile path. +pub fn lock_liveness(lock_path: &Path) -> DaemonLiveness { + #[cfg(unix)] + { + // A starting daemon holds the lock a moment before its record is written: retry briefly. + for _ in 0..10 { + let Ok(file) = crate::secure_fs::private_open_options().read(true).open(lock_path) else { + return DaemonLiveness::Stopped; + }; + match try_flock(&file, libc::LOCK_SH) { + Ok(true) => { + let rec = read_record(lock_path); + unlock(&file); + return rec.map_or(DaemonLiveness::Stopped, DaemonLiveness::Crashed); + } + Ok(false) => { + if let Some(rec) = read_record(lock_path) { + return DaemonLiveness::Running(rec); + } + std::thread::sleep(Duration::from_millis(5)); + } + Err(_) => return DaemonLiveness::Stopped, + } + } + DaemonLiveness::Stopped + } + #[cfg(not(unix))] + { + let _ = lock_path; + DaemonLiveness::Stopped + } +} + +/// Truncates the record of a lockfile nobody holds, waiting up to `wait` for the holder to die. +/// Returns `true` when the record was cleared (or there was none). The file itself stays. +pub fn clear_stale_record(lock_path: &Path, wait: Duration) -> bool { + #[cfg(unix)] + { + let deadline = Instant::now() + wait; + loop { + let file = match crate::secure_fs::private_open_options().read(true).write(true).open(lock_path) { + Ok(f) => f, + Err(e) => return e.kind() == io::ErrorKind::NotFound, + }; + if try_flock(&file, libc::LOCK_EX).unwrap_or(false) { + let ok = file.set_len(0).and_then(|()| file.sync_all()).is_ok(); + unlock(&file); + return ok; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(10)); + } + } + #[cfg(not(unix))] + { + let _ = (lock_path, wait); + false + } +} diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index dca57ad8..ff93b537 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -20,6 +20,8 @@ use serde::{Deserialize, Serialize}; pub mod types; pub use types::*; +pub mod liveness; + pub mod process; pub use process::*; diff --git a/src/daemon/process.rs b/src/daemon/process.rs index 5af456dd..adf5c5aa 100644 --- a/src/daemon/process.rs +++ b/src/daemon/process.rs @@ -96,7 +96,12 @@ pub fn spawn_background_daemon_for_session(session: &str) -> io::Result cmd.process_group(0); } - let _child = cmd.spawn()?; + let mut child = cmd.spawn()?; + std::thread::Builder::new() + .name("daemon-reaper".to_string()) + .spawn(move || { + let _ = child.wait(); + })?; Ok(sock_path) } diff --git a/src/daemon/registry.rs b/src/daemon/registry.rs index f65e5b35..5f88b198 100644 --- a/src/daemon/registry.rs +++ b/src/daemon/registry.rs @@ -11,6 +11,7 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; use crate::api::json::JsonValue; +use crate::daemon::liveness::DaemonLiveness; use crate::daemon::supervisor::{check_pid_alive, now_ms}; use crate::secure_fs::{ensure_private_dir, validate_safe_name, write_private_file}; @@ -209,14 +210,12 @@ impl DaemonRegistry { continue; }; - if check_pid_alive(item.pid) { + if Self::entry_is_live(base_dir, &item) { active.push(item); } else { - // Prune dead peer + // Prune dead peer (its lock record, if any, stays as crash evidence). let _ = fs::remove_file(&path); - if fs::symlink_metadata(&item.socket_path).is_ok() { - let _ = fs::remove_file(&item.socket_path); - } + let _ = crate::daemon::process::probe_or_cleanup_stale_socket(&item.socket_path); } } @@ -225,6 +224,34 @@ impl DaemonRegistry { active } + /// The lock decides when it has a record (held = live, unheld = crashed); a registry entry + /// with no lock record falls back to the PID probe, which treats zombies as dead. + fn entry_is_live(base_dir: &Path, item: &DaemonRegistryEntry) -> bool { + match crate::daemon::liveness::daemon_liveness(base_dir, &item.name) { + DaemonLiveness::Running(_) => true, + DaemonLiveness::Crashed(_) => false, + DaemonLiveness::Stopped => check_pid_alive(item.pid), + } + } + + /// Lock records left behind by daemons that died without cleaning up, sorted by name. + pub fn list_crashed(base_dir: &Path) -> Vec { + let Ok(read_dir) = fs::read_dir(base_dir.join("daemons")) else { + return Vec::new(); + }; + let mut out: Vec<_> = read_dir + .flatten() + .map(|e| e.path()) + .filter(|p| p.extension().and_then(|s| s.to_str()) == Some("lock")) + .filter_map(|p| match crate::daemon::liveness::lock_liveness(&p) { + DaemonLiveness::Crashed(rec) => Some(rec), + _ => None, + }) + .collect(); + out.sort_by(|a, b| a.name.cmp(&b.name)); + out + } + /// Adds entries for live `daemon-*.lock` / `daemon-singleton.lock` records and live /// `daemon-*.sock` sockets that have no registry json yet (dedupes by name). fn synthesize_from_locks_and_sockets(base_dir: &Path, active: &mut Vec) { @@ -247,7 +274,9 @@ impl DaemonRegistry { let Ok(rec) = crate::daemon::supervisor::DaemonLockRecord::from_json(&content) else { continue; }; - if !check_pid_alive(rec.pid) || active.iter().any(|e| e.name == rec.name) { + if !matches!(crate::daemon::liveness::lock_liveness(path), DaemonLiveness::Running(_)) + || active.iter().any(|e| e.name == rec.name) + { continue; } active.push(DaemonRegistryEntry { @@ -324,11 +353,9 @@ impl DaemonRegistry { continue; }; - if !check_pid_alive(item.pid) { + if !Self::entry_is_live(base_dir, &item) { let _ = fs::remove_file(&path); - if fs::symlink_metadata(&item.socket_path).is_ok() { - let _ = fs::remove_file(&item.socket_path); - } + let _ = crate::daemon::process::probe_or_cleanup_stale_socket(&item.socket_path); count += 1; } } diff --git a/src/daemon/supervisor.rs b/src/daemon/supervisor.rs index b1e2c783..1ceadd86 100644 --- a/src/daemon/supervisor.rs +++ b/src/daemon/supervisor.rs @@ -4,7 +4,7 @@ //! graceful stop, and auto-spawn capabilities for Bilterm background daemons. use std::collections::BTreeMap; -use std::fs::{self, File}; +use std::fs::File; use std::io::{self, ErrorKind, Read, Write}; #[cfg(unix)] use std::io::{Seek, SeekFrom}; @@ -21,6 +21,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use serde::{Deserialize, Serialize}; use crate::api::json::JsonValue; +pub use crate::daemon::liveness::{clear_stale_record, daemon_liveness, lock_liveness, DaemonLiveness}; pub use crate::daemon::types::{DaemonError, SignalablePid}; use crate::secure_fs::{ensure_private_dir, validate_safe_name}; #[cfg(unix)] @@ -128,9 +129,18 @@ impl DaemonLock { &self.lock_path } - /// Explicitly releases the lock, unlinking the lockfile and releasing `flock`. + /// Clears the record while the lock is still held (a clean exit leaves an empty file). + /// The lockfile is never unlinked: another process may already have it open. + fn clear_record(&mut self) { + if let Some(f) = self.file.as_mut() { + let _ = f.set_len(0); + let _ = f.sync_all(); + } + } + + /// Explicitly releases the lock: clears the record, then drops the `flock`. pub fn release(mut self) -> io::Result<()> { - let _ = fs::remove_file(&self.lock_path); + self.clear_record(); #[cfg(unix)] { if let Some(ref f) = self.file { @@ -148,6 +158,7 @@ impl DaemonLock { impl Drop for DaemonLock { fn drop(&mut self) { + self.clear_record(); #[cfg(unix)] { if let Some(ref f) = self.file { @@ -209,7 +220,7 @@ pub fn check_signalable_pid_alive(pid: SignalablePid) -> bool { { let ret = unsafe { libc::kill(pid.as_raw() as libc::pid_t, 0) }; if ret == 0 { - true + !crate::daemon::liveness::pid_is_zombie(pid.as_raw()) } else { let err = io::Error::last_os_error(); err.raw_os_error() != Some(libc::ESRCH) @@ -250,19 +261,24 @@ pub fn acquire_lock( let mut file = open_opts.open(&lock_path)?; let fd = file.as_raw_fd(); - let flock_res = unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) }; + // A liveness probe holds a shared lock for microseconds: retry before giving up. + let mut flock_res = unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) }; + for _ in 0..5 { + if flock_res == 0 { + break; + } + std::thread::sleep(Duration::from_millis(10)); + flock_res = unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) }; + } if flock_res == 0 { // Flock acquired. Check if prior dead daemon left a stale socket. let mut old_content = String::new(); let _ = file.read_to_string(&mut old_content); - if let Ok(old_rec) = DaemonLockRecord::from_json(&old_content) - && fs::symlink_metadata(&old_rec.socket_path).is_ok() - { - let _ = fs::remove_file(&old_rec.socket_path); - } - if fs::symlink_metadata(&socket_path).is_ok() { - let _ = fs::remove_file(&socket_path); + // We own the lock now; only sockets nobody answers on are removed. + if let Ok(old_rec) = DaemonLockRecord::from_json(&old_content) { + let _ = crate::daemon::process::probe_or_cleanup_stale_socket(&old_rec.socket_path); } + let _ = crate::daemon::process::probe_or_cleanup_stale_socket(&socket_path); // Truncate and write fresh record. let record = DaemonLockRecord { @@ -283,39 +299,11 @@ pub fn acquire_lock( })); } - // Flock blocked. Inspect lockfile contents. + // The lock is held, and a held lock is a live daemon (PIDs are not consulted). let mut content = String::new(); let _ = file.read_to_string(&mut content); - if let Ok(rec) = DaemonLockRecord::from_json(&content) { - if check_pid_alive(rec.pid) { - return Ok(DaemonLockResult::AlreadyHeld(rec)); - } - // Stale lock detected: PID is dead! - if fs::symlink_metadata(&rec.socket_path).is_ok() { - let _ = fs::remove_file(&rec.socket_path); - } - // Retry flock after brief backoff - std::thread::sleep(Duration::from_millis(20)); - let retry_flock = unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) }; - if retry_flock == 0 { - let record = DaemonLockRecord { - pid: std::process::id(), - name: name.to_string(), - version: env!("CARGO_PKG_VERSION").to_string(), - started_at_ms: now_ms(), - socket_path, - }; - file.set_len(0)?; - file.seek(SeekFrom::Start(0))?; - file.write_all(record.to_json().as_bytes())?; - file.sync_all()?; - return Ok(DaemonLockResult::Acquired(DaemonLock { - file: Some(file), - record, - lock_path, - })); - } + return Ok(DaemonLockResult::AlreadyHeld(rec)); } } @@ -376,14 +364,15 @@ pub fn spawn_daemon_in(runtime_dir: &Path, name: &str) -> io::Result<(u32, PathB cmd.process_group(0); } - let child = cmd.spawn()?; - Ok((child.id(), sock_path)) -} - -fn read_lock_record(lock_path: &Path) -> Option { - let mut s = String::new(); - File::open(lock_path).ok()?.read_to_string(&mut s).ok()?; - DaemonLockRecord::from_json(&s).ok() + let mut child = cmd.spawn()?; + let pid = child.id(); + // Reap the daemon when it exits so it never lingers as a zombie of this process. + std::thread::Builder::new() + .name("daemon-reaper".to_string()) + .spawn(move || { + let _ = child.wait(); + })?; + Ok((pid, sock_path)) } /// Ensures the daemon for `name` is running. Returns `AlreadyRunning` or `Started`. @@ -396,8 +385,7 @@ pub fn ensure_started(runtime_dir: &Path, name: &str) -> io::Result io::Result Some(r.pid), + _ => None, + } .or_else(|| { crate::daemon::registry::DaemonRegistry::list_active(runtime_dir) .into_iter() @@ -542,99 +531,72 @@ pub fn kill_daemon_process(pid: SignalablePid, force: bool) -> io::Result<()> { } } -/// Tries a non-blocking exclusive `flock` on `lock_path`. -/// -/// Returns `Ok(Some(file))` (lock now held by the caller) when no daemon holds it, i.e. the -/// lockfile is stale, and `Ok(None)` when a live daemon holds it. -#[cfg(unix)] -fn claim_unheld_lock(lock_path: &Path) -> io::Result> { - let file = private_open_options().read(true).open(lock_path)?; - if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0 { - return Ok(Some(file)); - } - let err = io::Error::last_os_error(); - if err.raw_os_error() == Some(libc::EWOULDBLOCK) { - Ok(None) - } else { - Err(err) - } -} - -/// Gracefully stops daemon `name`. +/// Stops daemon `name`. Returns `Ok(true)` when a live daemon was stopped, `Ok(false)` when +/// none was running (a crashed daemon's leftovers are cleared). The lockfile is never unlinked. pub fn stop_daemon(runtime_dir: &Path, name: &str, force: bool) -> io::Result { - #[cfg(not(unix))] - let _ = force; validate_safe_name(name) .map_err(|e| io::Error::new(ErrorKind::InvalidInput, format!("invalid daemon name: {e}")))?; - let lock_path = daemon_lock_path_in(runtime_dir, name)?; - let record = if let Ok(mut f) = File::open(&lock_path) { - let mut s = String::new(); - if f.read_to_string(&mut s).is_ok() { - DaemonLockRecord::from_json(&s).ok() - } else { - None - } - } else { - None + let unregister = || { + let _ = crate::daemon::registry::DaemonRegistry::unregister(runtime_dir, name); }; - let Some(rec) = record else { - // No lock record: fall back to the registry (json / synthesized) for a live daemon. - let mut stopped = false; - let sock = crate::daemon::process::session_daemon_socket_path_in(runtime_dir, name)?; - if let Some(e) = crate::daemon::registry::DaemonRegistry::list_active(runtime_dir) - .into_iter() - .find(|e| e.name == name) - { - if let Ok(sig_pid) = SignalablePid::try_from(i64::from(e.pid)) - && sig_pid.as_raw() != std::process::id() - && check_signalable_pid_alive(sig_pid) - && crate::daemon::process::probe_or_cleanup_stale_socket(&e.socket_path) - { + match lock_liveness(&lock_path) { + DaemonLiveness::Running(rec) => { + if rec.pid == std::process::id() { + return Err(io::Error::new( + ErrorKind::InvalidInput, + format!("daemon '{name}' is this process (PID {}); stop it from another instance", rec.pid), + )); + } + // The lock is held, so the recorded PID is the daemon and not a reused PID. + if let Ok(sig_pid) = SignalablePid::try_from(i64::from(rec.pid)) { kill_daemon_process(sig_pid, force)?; - stopped = true; } - let _ = fs::remove_file(&e.socket_path); + if clear_stale_record(&lock_path, Duration::from_millis(1000)) { + let _ = crate::daemon::process::probe_or_cleanup_stale_socket(&rec.socket_path); + unregister(); + } + Ok(true) } - let _ = fs::remove_file(&lock_path); - let _ = fs::remove_file(sock); - let _ = crate::daemon::registry::DaemonRegistry::unregister(runtime_dir, name); - return Ok(stopped); - }; - - // Only signal a PID whose daemon still holds the flock: if we can claim the lock - // ourselves it is stale, and the recorded PID may now belong to an unrelated process. - #[cfg(unix)] - match claim_unheld_lock(&lock_path) { - Ok(None) => {} - Ok(Some(_stale_guard)) => { - let _ = fs::remove_file(&lock_path); - let _ = fs::remove_file(&rec.socket_path); - let _ = crate::daemon::registry::DaemonRegistry::unregister(runtime_dir, name); - return Ok(false); + DaemonLiveness::Crashed(rec) => { + clear_stale_record(&lock_path, Duration::ZERO); + let _ = crate::daemon::process::probe_or_cleanup_stale_socket(&rec.socket_path); + unregister(); + Ok(false) + } + DaemonLiveness::Stopped => { + // No lock record: fall back to the registry (json / synthesized) for a live daemon. + let mut stopped = false; + let sock = crate::daemon::process::session_daemon_socket_path_in(runtime_dir, name)?; + if let Some(e) = crate::daemon::registry::DaemonRegistry::list_active(runtime_dir) + .into_iter() + .find(|e| e.name == name) + { + if let Ok(sig_pid) = SignalablePid::try_from(i64::from(e.pid)) + && sig_pid.as_raw() != std::process::id() + && check_signalable_pid_alive(sig_pid) + && crate::daemon::process::probe_or_cleanup_stale_socket(&e.socket_path) + { + kill_daemon_process(sig_pid, force)?; + stopped = true; + // The process we just stopped was the one answering on this socket. + let _ = std::fs::remove_file(&e.socket_path); + } + let _ = crate::daemon::process::probe_or_cleanup_stale_socket(&e.socket_path); + } + clear_stale_record(&lock_path, Duration::ZERO); + let _ = crate::daemon::process::probe_or_cleanup_stale_socket(&sock); + unregister(); + Ok(stopped) } - Err(e) if e.kind() == ErrorKind::NotFound => return Ok(false), - Err(e) => return Err(e), - } - - if let Ok(sig_pid) = SignalablePid::try_from(i64::from(rec.pid)) - && sig_pid.as_raw() != std::process::id() - && check_signalable_pid_alive(sig_pid) - { - kill_daemon_process(sig_pid, force)?; } - - let _ = fs::remove_file(&lock_path); - let _ = fs::remove_file(&rec.socket_path); - let _ = crate::daemon::registry::DaemonRegistry::unregister(runtime_dir, name); - - Ok(true) } #[cfg(test)] mod tests { use super::*; + use std::fs; fn record_json(pid: &str) -> String { format!(r#"{{"pid":{pid},"name":"t","version":"0","started_at_ms":0,"socket_path":"/x.sock"}}"#) @@ -683,7 +645,7 @@ mod tests { let _ = victim.wait(); assert!(!stopped, "stale lock must report not_running"); assert!(survived, "unrelated process must not be signaled"); - assert!(!lock_path.exists(), "stale lockfile is cleaned up"); + assert_eq!(fs::read_to_string(&lock_path).unwrap(), "", "stale record is cleared, file kept"); } #[cfg(unix)] @@ -695,7 +657,8 @@ mod tests { let DaemonLockResult::Acquired(_lock) = acquire_lock(runtime_dir, "live", sock).unwrap() else { panic!("expected to acquire fresh lock"); }; - // Held by this process: recognized as live (not stale), and never self-signaled. - assert!(stop_daemon(runtime_dir, "live", false).unwrap()); + // Held by this process: recognized as live, never self-signaled, and left intact. + assert!(stop_daemon(runtime_dir, "live", false).is_err()); + assert!(matches!(daemon_liveness(runtime_dir, "live"), DaemonLiveness::Running(_))); } } diff --git a/src/ui/daemon_window_ui.rs b/src/ui/daemon_window_ui.rs index 29f7bb0c..f16f4cd1 100644 --- a/src/ui/daemon_window_ui.rs +++ b/src/ui/daemon_window_ui.rs @@ -149,6 +149,8 @@ pub fn render_daemon_window( let name = if e.current { format!("*{}", e.name) } else { format!(" {}", e.name) }; let (badge_txt, badge_fg) = if e.running { ("● RUNNING", tokens.success) + } else if e.is_crashed() { + ("✖ CRASHED", tokens.danger) } else { ("○ STOPPED", tokens.text_muted) }; diff --git a/src/wm/daemon_window.rs b/src/wm/daemon_window.rs index 652245bb..7ab5182b 100644 --- a/src/wm/daemon_window.rs +++ b/src/wm/daemon_window.rs @@ -26,6 +26,13 @@ pub struct DaemonMenuEntry { pub current: bool, } +impl DaemonMenuEntry { + /// A daemon that is not running but still has a recorded PID died without cleaning up. + pub fn is_crashed(&self) -> bool { + !self.running && self.pid.is_some() + } +} + /// Buttons in the action footer, in left-to-right render order. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DaemonWindowButton { @@ -160,24 +167,28 @@ impl WindowManager { } } +fn crashed_entry(r: crate::daemon::supervisor::DaemonLockRecord) -> DaemonMenuEntry { + DaemonMenuEntry { + name: r.name, + running: false, + pid: Some(r.pid), + socket_path: r.socket_path.to_string_lossy().into_owned(), + ..Default::default() + } +} + fn default_entry(runtime_dir: &Path) -> DaemonMenuEntry { - use crate::daemon::supervisor::{DaemonLockRecord, check_pid_alive, daemon_lock_path_in}; - let rec = daemon_lock_path_in(runtime_dir, "default") - .ok() - .and_then(|p| std::fs::read_to_string(p).ok()) - .and_then(|s| DaemonLockRecord::from_json(&s).ok()); - match rec { - Some(r) => { - let alive = check_pid_alive(r.pid); - DaemonMenuEntry { - name: "default".into(), - running: alive, - pid: alive.then_some(r.pid), - socket_path: r.socket_path.to_string_lossy().into_owned(), - ..Default::default() - } - } - None => DaemonMenuEntry { + use crate::daemon::supervisor::{daemon_liveness, DaemonLiveness}; + match daemon_liveness(runtime_dir, "default") { + DaemonLiveness::Running(r) => DaemonMenuEntry { + name: "default".into(), + running: true, + pid: Some(r.pid), + socket_path: r.socket_path.to_string_lossy().into_owned(), + ..Default::default() + }, + DaemonLiveness::Crashed(r) => crashed_entry(r), + DaemonLiveness::Stopped => DaemonMenuEntry { name: "default".into(), socket_path: runtime_dir.join("daemon.sock").to_string_lossy().into_owned(), ..Default::default() @@ -225,6 +236,11 @@ impl WindowManager { socket_path: d.socket_path.to_string_lossy().into_owned(), }) .collect(); + for rec in crate::daemon::registry::DaemonRegistry::list_crashed(runtime_dir) { + if !entries.iter().any(|e| e.name == rec.name) { + entries.push(crashed_entry(rec)); + } + } if !entries.iter().any(|e| e.name == "default") { entries.push(default_entry(runtime_dir)); } @@ -232,12 +248,14 @@ impl WindowManager { (a.name != "default", a.name.as_str()).cmp(&(b.name != "default", b.name.as_str())) }); let running = entries.iter().filter(|e| e.running).count(); + let crashed = entries.iter().filter(|e| e.is_crashed()).count(); let w = &mut self.daemon_window; w.status_line = format!( - "{} daemon{}, {} running", + "{} daemon{}, {} running{}", entries.len(), if entries.len() == 1 { "" } else { "s" }, - running + running, + if crashed > 0 { format!(", {crashed} crashed") } else { String::new() } ); w.entries = entries; w.selected = w.selected.min(w.entries.len().saturating_sub(1)); diff --git a/tests/daemon_resilience_lifecycle.rs b/tests/daemon_resilience_lifecycle.rs new file mode 100644 index 00000000..36f00d23 --- /dev/null +++ b/tests/daemon_resilience_lifecycle.rs @@ -0,0 +1,108 @@ +//! Daemon lifecycle hardening: reaped children, zombie/PID-reuse-proof liveness, and a lockfile +//! that is never unlinked (`flock` is the only source of truth). +#![cfg(unix)] + +use std::fs; +use std::process::Command; +use std::time::{Duration, Instant}; + +use bilterm::daemon::registry::{DaemonRegistry, DaemonRegistryEntry}; +use bilterm::daemon::supervisor::{ + acquire_lock, daemon_lock_path_in, spawn_daemon_in, stop_daemon, DaemonLockRecord, DaemonLockResult, +}; + +fn proc_state(pid: u32) -> Option { + let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + stat.rsplit_once(')')?.1.trim_start().chars().next() +} + +fn entry(name: &str, pid: u32, dir: &std::path::Path) -> DaemonRegistryEntry { + DaemonRegistryEntry { + id: format!("daemon-{name}-{pid}"), + name: name.to_string(), + pid, + version: "0".into(), + started_at_ms: 0, + socket_path: dir.join(format!("{name}.sock")), + sessions_count: 1, + panes_count: 0, + agents_count: 0, + last_heartbeat_ms: 0, + } +} + +fn record(name: &str, pid: u32, dir: &std::path::Path) -> DaemonLockRecord { + DaemonLockRecord { + pid, + name: name.to_string(), + version: "0".into(), + started_at_ms: 0, + socket_path: dir.join(format!("{name}.sock")), + } +} + +#[test] +fn release_and_stop_never_unlink_the_lockfile() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let lock_path = daemon_lock_path_in(dir, "keep").unwrap(); + let DaemonLockResult::Acquired(lock) = acquire_lock(dir, "keep", dir.join("keep.sock")).unwrap() else { + panic!("fresh lock must be acquired"); + }; + lock.release().unwrap(); + assert!(lock_path.exists(), "release must not unlink a lockfile another starter may hold open"); + + // A crashed daemon's record: stop clears it but keeps the file. + let mut victim = Command::new("sleep").arg("30").spawn().unwrap(); + fs::write(&lock_path, record("keep", victim.id(), dir).to_json()).unwrap(); + assert!(!stop_daemon(dir, "keep", true).unwrap()); + let alive = victim.try_wait().unwrap().is_none(); + let _ = victim.kill(); + let _ = victim.wait(); + assert!(alive, "a PID from an unheld lock must never be signalled"); + assert!(lock_path.exists(), "stop must not unlink the lockfile"); +} + +#[test] +fn zombie_registry_entry_is_not_listed_active() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let mut child = Command::new("true").spawn().unwrap(); + let pid = child.id(); + let deadline = Instant::now() + Duration::from_secs(5); + while proc_state(pid) != Some('Z') && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!(proc_state(pid), Some('Z'), "child should be an unreaped zombie"); + DaemonRegistry::register(dir, &entry("zed", pid, dir)).unwrap(); + let active = DaemonRegistry::list_active(dir); + let _ = child.wait(); + assert!(active.iter().all(|e| e.name != "zed"), "a zombie is not a running daemon: {active:?}"); +} + +#[test] +fn unheld_lock_record_with_a_live_reused_pid_is_not_active() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + // The recorded PID now belongs to an unrelated live process; nobody holds the flock. + let mut other = Command::new("sleep").arg("30").spawn().unwrap(); + let lock_path = daemon_lock_path_in(dir, "reused").unwrap(); + fs::write(&lock_path, record("reused", other.id(), dir).to_json()).unwrap(); + DaemonRegistry::register(dir, &entry("reused", other.id(), dir)).unwrap(); + let active = DaemonRegistry::list_active(dir); + let _ = other.kill(); + let _ = other.wait(); + assert!(active.iter().all(|e| e.name != "reused"), "crashed daemon listed as active: {active:?}"); +} + +#[test] +fn spawned_daemon_child_is_reaped() { + let tmp = tempfile::tempdir().unwrap(); + // `current_exe` is this test binary: it rejects the daemon flags and exits at once. + let (pid, _sock) = spawn_daemon_in(tmp.path(), "reap").unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while proc_state(pid).is_some() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(20)); + } + assert_eq!(proc_state(pid), None, "exited daemon child must be reaped, not left as a zombie"); +} diff --git a/tests/phase55_daemon_lifecycle.rs b/tests/phase55_daemon_lifecycle.rs index 6062d514..79e54379 100644 --- a/tests/phase55_daemon_lifecycle.rs +++ b/tests/phase55_daemon_lifecycle.rs @@ -184,9 +184,10 @@ fn test_graceful_stop_and_cleanup() { let stopped = stop_daemon(runtime_dir, "test-stop", false).expect("stop daemon"); assert!(!stopped); // dead PID, returns false for active process, but cleans resources - // Socket, lock, and registry file must be cleaned up + // Socket and registry file are removed; the lockfile is kept (never unlinked, another + // starter may hold it open) but its stale content is cleared. assert!(!entry.socket_path.exists()); - assert!(!lock_path.exists()); + assert_eq!(fs::read(&lock_path).unwrap().len(), 0); let reg_path = DaemonRegistry::entry_path_in(runtime_dir, "test-stop").unwrap(); assert!(!reg_path.exists()); } diff --git a/tests/phase55_daemon_lock_and_registry.rs b/tests/phase55_daemon_lock_and_registry.rs index fbf9ef8a..b4a2396c 100644 --- a/tests/phase55_daemon_lock_and_registry.rs +++ b/tests/phase55_daemon_lock_and_registry.rs @@ -59,7 +59,11 @@ fn test_singleton_lock_and_recovery() { // 3. Release lock cleanly lock.release().expect("clean release"); let lock_path = daemon_lock_path_in(runtime_dir, "default").unwrap(); - assert!(!lock_path.exists(), "lockfile must be unlinked on clean release"); + assert_eq!( + fs::read(&lock_path).unwrap().len(), + 0, + "clean release clears the record and keeps the lockfile (flock is the only truth)" + ); // 4. Stale lock recovery: write a fake lockfile with a dead PID (e.g. 9999999) let fake_record = DaemonLockRecord { diff --git a/tests/phase55_daemon_parity.rs b/tests/phase55_daemon_parity.rs index 4b567057..056c0e38 100644 --- a/tests/phase55_daemon_parity.rs +++ b/tests/phase55_daemon_parity.rs @@ -30,11 +30,27 @@ const AREA: Rect = Rect { height: 40, }; +/// Points the runtime dir at a throwaway directory for the duration of a test, so nothing here +/// can lock, signal or unlink anything in the user's real `$XDG_RUNTIME_DIR/bilterm`. +fn sandbox_runtime() -> (std::sync::MutexGuard<'static, ()>, tempfile::TempDir) { + static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + let guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let dir = tempfile::tempdir().expect("sandbox runtime dir"); + // SAFETY: serialized by ENV_LOCK; every test in this file goes through this function. + unsafe { + std::env::set_var("XDG_RUNTIME_DIR", dir.path()); + std::env::remove_var("BILTERM_RUNTIME_DIR"); + } + (guard, dir) +} + #[test] fn test_daemon_surface_parity() { - let runtime_dir = bilterm::secure_fs::private_runtime_dir().expect("runtime dir"); + // This test takes the "default" daemon lock and sends `daemon.stop`. + let (_env, _sandbox) = sandbox_runtime(); + let runtime_dir = bilterm::daemon::supervisor::resolve_runtime_dir().expect("runtime dir"); let sock_path = bilterm::daemon::process::session_daemon_socket_path_in(&runtime_dir, "default").unwrap(); - let _lock = bilterm::daemon::supervisor::acquire_lock(&runtime_dir, "default", sock_path.clone()).unwrap(); + let mut lock = Some(bilterm::daemon::supervisor::acquire_lock(&runtime_dir, "default", sock_path.clone()).unwrap()); #[cfg(unix)] let _listener = std::os::unix::net::UnixListener::bind(&sock_path).ok(); @@ -94,6 +110,10 @@ fn test_daemon_surface_parity() { let mut cp = AutomationControlPlane::new_headless(24, 80); for (name, colon, chord_str, subchord_code, rpc_method, mcp_action, canonical_action) in matrix { + // A daemon refuses to stop its own process, so release the in-process lock first. + if name == "stop" { + lock.take(); + } // 1. Colon command parsing parity let parsed = parse_colon_command(colon) .unwrap_or_else(|| panic!("failed to recognize {colon}")) @@ -156,6 +176,7 @@ fn test_daemon_surface_parity() { #[test] fn test_cross_daemon_find_indexing() { + let (_env, _sandbox) = sandbox_runtime(); let runtime_dir = bilterm::secure_fs::private_runtime_dir().expect("runtime dir"); // Register 2 live peer daemons From 9389ae221fb65cb76636fcad6dcd1ca82fa83292 Mon Sep 17 00:00:00 2001 From: Bill Sommers Date: Tue, 10 Nov 2026 01:07:39 -0500 Subject: [PATCH 3/6] feat(wm): checkpoints, crash recovery, durable sessions, honest detach - wm/resilience.rs: debounced (750 ms) and periodic (30 s) checkpoints of the active session on layout, pane or cwd change; the UI thread only builds the snapshot, a worker serializes and writes .ckpt. A .running marker detects an unclean shutdown; the next start restores the last checkpoint and reports it. - :daemon start/stop run on a worker when a UI is attached and report through the tick, so the UI thread never waits on a daemon. - Session files carry format_version, are written durably with a .bak, and a corrupt or truncated file falls back to the backup with a message; a failed restore at startup is shown instead of silently starting fresh. - detach_client cancels and says why when the session cannot be saved; wording no longer implies programs keep running. - Journal writes/loads are durable and versioned and use one directory. - The session tree (reparenting) persists across restarts; the MCP reparent path reports rejected grafts (self, cycle, missing) instead of success. - session.save / session.restore over RPC share the colon command's function and default (the active session). - bilterm ctl auto-start retries on the socket the started daemon listens on. Co-Authored-By: Claude Fable 5.2 --- src/api/rpc_handlers.rs | 21 +- src/colon_command/execute_daemon.rs | 29 ++- src/colon_command/execute_wm.rs | 3 +- src/main.rs | 31 ++- src/ui/help_catalog.rs | 2 +- src/ui/help_catalog_data.rs | 2 +- src/wm/journal.rs | 19 +- src/wm/mod.rs | 4 + src/wm/resilience.rs | 381 ++++++++++++++++++++++++++++ src/wm/session.rs | 24 +- src/wm/v2_runtime.rs | 54 ++-- src/wm/window_manage_bulk.rs | 1 + tests/daemon_resilience_state.rs | 190 ++++++++++++++ 13 files changed, 714 insertions(+), 47 deletions(-) create mode 100644 src/wm/resilience.rs create mode 100644 tests/daemon_resilience_state.rs diff --git a/src/api/rpc_handlers.rs b/src/api/rpc_handlers.rs index 30e89c57..835003dc 100644 --- a/src/api/rpc_handlers.rs +++ b/src/api/rpc_handlers.rs @@ -157,14 +157,12 @@ impl AutomationControlPlane { &mut self, params: &JsonValue, ) -> std::result::Result { - let name = params - .get("name") - .and_then(JsonValue::as_str) - .unwrap_or("default"); - let path = self.wm.save_session(name).map_err(|e| e.to_string())?; + // Same function and default (the active session) as `:session save [name]`. + let name = params.get("name").and_then(JsonValue::as_str); + let (name, path) = self.wm.save_and_activate_session(name).map_err(|e| e.to_string())?; Ok(JsonValue::obj([ ("saved", JsonValue::Bool(true)), - ("session", JsonValue::String(name.to_string())), + ("session", JsonValue::String(name)), ("path", JsonValue::String(path.display().to_string())), ("panes", JsonValue::Number(self.wm.pane_count() as i64)), ])) @@ -175,14 +173,19 @@ impl AutomationControlPlane { &mut self, params: &JsonValue, ) -> std::result::Result { + // Same default (the active session) as `:session restore [name]`. let name = params .get("name") .and_then(JsonValue::as_str) - .unwrap_or("default"); - let count = self.wm.restore_session(name).map_err(|e| e.to_string())?; + .map(str::trim) + .filter(|s| !s.is_empty()) + .map_or_else(|| self.wm.config().history_session.clone(), str::to_string); + let count = self.wm.restore_session(&name).map_err(|e| e.to_string())?; + let note = self.wm.take_restore_note().map_or(JsonValue::Null, JsonValue::String); Ok(JsonValue::obj([ ("restored", JsonValue::Bool(true)), - ("session", JsonValue::String(name.to_string())), + ("note", note), + ("session", JsonValue::String(name)), ("panes", JsonValue::Number(count as i64)), ])) } diff --git a/src/colon_command/execute_daemon.rs b/src/colon_command/execute_daemon.rs index ba16e054..ff98019e 100644 --- a/src/colon_command/execute_daemon.rs +++ b/src/colon_command/execute_daemon.rs @@ -6,6 +6,7 @@ use crate::daemon::registry::DaemonRegistry; use crate::daemon::supervisor::{ daemon_lock_path_in, ensure_started, resolve_runtime_dir, stop_daemon, DaemonLockRecord, DaemonStartResult, }; +use crate::wm::resilience::DaemonJob; use crate::wm::WindowManager; /// Executes a daemon supervisor operation against the local runtime directory. @@ -19,6 +20,28 @@ pub fn execute_daemon_command(wm: &mut WindowManager, op: &DaemonSupervisorOp) - } }; + // With a UI attached, start/stop wait on a worker; the result arrives as a status message. + if !wm.is_headless() { + let job = match op { + DaemonSupervisorOp::Start { name } => { + Some(DaemonJob::Start { name: name.clone().unwrap_or_else(|| "default".to_string()) }) + } + DaemonSupervisorOp::Stop { name, force } => Some(DaemonJob::Stop { + name: name.clone().unwrap_or_else(|| "default".to_string()), + force: *force, + }), + _ => None, + }; + if let Some(job) = job { + let verb = if matches!(job, DaemonJob::Start { .. }) { "starting" } else { "stopping" }; + let (DaemonJob::Start { name } | DaemonJob::Stop { name, .. }) = &job; + wm.active_pane_mut().set_status_message(Some(format!("Daemon '{name}': {verb}…"))); + wm.spawn_daemon_job(&runtime_dir, job); + wm.open_daemon_window_from(&runtime_dir); + return false; + } + } + match op { DaemonSupervisorOp::List => { let active = DaemonRegistry::list_active(&runtime_dir); @@ -99,12 +122,12 @@ pub fn execute_daemon_command(wm: &mut WindowManager, op: &DaemonSupervisorOp) - let res = JsonValue::obj([ ("name", JsonValue::String(rec.name.clone())), ("pid", JsonValue::Number(rec.pid as i64)), - ("status", JsonValue::String("locked".to_string())), + ("status", JsonValue::String("crashed".to_string())), ]); wm.set_command_data(res.to_json_string()); wm.active_pane_mut().set_status_message(Some(format!( - "Daemon '{}': lock held by PID {}", - rec.name, rec.pid + "Daemon '{}': crashed (PID {} exited without cleanup) — ':daemon start {}' restarts it", + rec.name, rec.pid, rec.name ))); } else { let res = JsonValue::obj([ diff --git a/src/colon_command/execute_wm.rs b/src/colon_command/execute_wm.rs index de108f52..d11d438f 100644 --- a/src/colon_command/execute_wm.rs +++ b/src/colon_command/execute_wm.rs @@ -325,8 +325,9 @@ pub fn execute_wm_colon_command(wm: &mut WindowManager, cmd: ColonCommand) -> bo }; match wm.restore_session(&target) { Ok(count) => { + let note = wm.take_restore_note().map(|n| format!(" — {n}")).unwrap_or_default(); wm.active_pane_mut().set_status_message( - Some(format!("Restored session '{target}' ({count} pane(s))")), + Some(format!("Restored session '{target}' ({count} pane(s)){note}")), ); } Err(err) => { diff --git a/src/main.rs b/src/main.rs index 49495c45..a3d4a84c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -188,10 +188,18 @@ async fn main() -> color_eyre::Result<()> { let resp = match send_rpc_request_with_token(&endpoint, &method, params.clone(), ctl_token.as_deref()).await { Ok(r) => r, Err(_) if endpoint.starts_with("unix:") => { - if let Ok(runtime_dir) = bilterm::secure_fs::private_runtime_dir() { - let _ = bilterm::daemon::supervisor::ensure_started(&runtime_dir, "default"); + use bilterm::daemon::supervisor::{ensure_started, DaemonStartResult}; + // Nothing answered: start the default daemon and, when no endpoint was given, + // talk to the socket it actually listens on. + let mut retry_ep = endpoint.clone(); + if let Ok(runtime_dir) = bilterm::secure_fs::private_runtime_dir() + && let Ok(DaemonStartResult::Started { socket_path, .. } | DaemonStartResult::AlreadyRunning { socket_path, .. }) = + ensure_started(&runtime_dir, "default") + && endpoint == default_ep + { + retry_ep = format!("unix:{}", socket_path.display()); } - send_rpc_request_with_token(&endpoint, &method, params, ctl_token.as_deref()).await? + send_rpc_request_with_token(&retry_ep, &method, params, ctl_token.as_deref()).await? } Err(e) => return Err(e.into()), }; @@ -538,8 +546,9 @@ async fn main() -> color_eyre::Result<()> { } else if attach_subcmd || attach_flag || app.wm.has_detached_session(&session_to_restore) { match app.wm.restore_session(&session_to_restore) { Ok(count) => { + let note = app.wm.take_restore_note().map(|n| format!(" — {n}")).unwrap_or_default(); app.wm.active_pane_mut().set_status_message(Some(format!( - "Re-attached session '{session_to_restore}' ({count} pane(s) window layout restored)" + "Re-attached session '{session_to_restore}' ({count} pane(s) window layout restored; programs restarted){note}" ))); } Err(err) if attach_subcmd || attach_flag => { @@ -547,7 +556,11 @@ async fn main() -> color_eyre::Result<()> { "Attach '{session_to_restore}': {err} (started fresh session)" ))); } - Err(_) => {} + Err(err) => { + app.wm.active_pane_mut().set_status_message(Some(format!( + "Session '{session_to_restore}' could not be restored: {err} (started fresh)" + ))); + } } } else if (explicit_profile.is_some() || app.wm.config().layout_autosave) && let Some((prof, count)) = app.wm.restore_default_layout_profile_on_startup() @@ -567,6 +580,11 @@ async fn main() -> color_eyre::Result<()> { if cfg_error.is_some() { app.wm.active_pane_mut().set_status_message(cfg_error); } + // Crash recovery: a leftover running marker means the last run of this session died. + app.wm.load_persisted_session_tree(); + if let Some(note) = app.wm.begin_run(!(blank_session_flag || new_session_flag)) { + app.wm.active_pane_mut().set_status_message(Some(note)); + } if gui_mode { bilterm::ui::gui_window::run_native_gui_window(app)?; @@ -580,11 +598,12 @@ async fn main() -> color_eyre::Result<()> { let run_res = app.run(&mut terminal).await; host_guard.restore(); run_res?; + app.wm.end_run(); if app.wm.is_client_detached() { let sess = &app.wm.config().history_session; let _ = bilterm::daemon::spawn_background_daemon_for_session(sess); println!( - "[bilterm: detached session '{}' ({} pane(s) window layout intact) — run 'bilterm' or 'bilterm attach {}' to resume]", + "[bilterm: detached session '{}' ({} pane(s): layout saved; running programs were ended and restart on attach) — run 'bilterm' or 'bilterm attach {}' to resume]", sess, app.wm.pane_count(), sess diff --git a/src/ui/help_catalog.rs b/src/ui/help_catalog.rs index aee57fd1..6c7095a1 100644 --- a/src/ui/help_catalog.rs +++ b/src/ui/help_catalog.rs @@ -43,7 +43,7 @@ pub const PALETTE_CATALOG: &[(&str, &str, &str)] = &[ (":keys check", "—", "Check the active key profile for conflicting bindings"), (":keys explain ", "—", "Explain what a chord does in the active key profile"), (":keys profile ", "—", "Show or switch the key profile (auto|mac|windows|linux|portable|legacy)"), - (":detach", "Ctrl+X D", "Detach client & save full multi-pane window layout intact"), + (":detach", "Ctrl+X D", "Detach client & save the multi-pane window layout (programs restart on attach)"), (":attach", "bilterm -a", "Re-attach & restore saved/detached multi-pane window layout"), (":exit-guard guard", "—", "Enable Exit Guard modal on final/root shell exit"), (":exit-guard respawn", "—", "Auto-respawn shell in-place on final/root shell exit"), diff --git a/src/ui/help_catalog_data.rs b/src/ui/help_catalog_data.rs index 23d61148..95bf4d47 100644 --- a/src/ui/help_catalog_data.rs +++ b/src/ui/help_catalog_data.rs @@ -5,7 +5,7 @@ pub const HELP_LINES: &[(&str, &str)] = &[ ("═══ LEADER PREFIX (Ctrl+X), DETACH & GLOBAL HOTKEYS ═══", ""), ("Ctrl+X ", "Universal Leader Prefix (works in Command Box & Raw PTY)"), ("Ctrl+X ? / F1 / :help", "Toggle this interactive Help & :colon Command reference"), - ("Ctrl+X D / :detach [name]", "Detach client & preserve full multi-pane window layout intact"), + ("Ctrl+X D / :detach [name]", "Detach client & save the multi-pane window layout (programs restart on attach)"), (":attach [name] / bilterm -a", "Re-attach & restore saved/detached multi-pane window layout"), ("bilterm ls / bilterm sessions", "List live background daemon sockets & saved .toml session layouts"), (":daemon [list|status|start|stop|switch|prune]", "Supervise background daemons: PIDs, sockets, peer discovery, graceful stop"), diff --git a/src/wm/journal.rs b/src/wm/journal.rs index fef188a1..043c8053 100644 --- a/src/wm/journal.rs +++ b/src/wm/journal.rs @@ -7,7 +7,6 @@ //! Enforces strict layout-mode geometry separation so restoring a `"tiled"` workspace never //! contaminates `ZStack` custom floating window coordinates. -use std::fs; use std::io::{self, ErrorKind}; use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -15,8 +14,8 @@ use std::time::{SystemTime, UNIX_EPOCH}; use serde::{Deserialize, Serialize}; use crate::secure_fs::{ - ensure_private_dir, private_dir_in, private_runtime_dir, validate_plain_file_name, - write_private_file, + ensure_private_dir, private_dir_in, private_runtime_dir, read_with_backup, validate_plain_file_name, + write_private_file_durable, }; use crate::wm::session_tree::SessionTree; @@ -141,11 +140,11 @@ impl WorkspaceJournal { format!("{slot_name}.toml") }; let slot_path = journal_dir.join(file_name); - write_private_file(&slot_path, payload.as_bytes())?; + write_private_file_durable(&slot_path, payload.as_bytes())?; let latest_path = journal_dir.join("latest.toml"); if latest_path != slot_path { - write_private_file(&latest_path, payload.as_bytes())?; + write_private_file_durable(&latest_path, payload.as_bytes())?; } Ok(slot_path) } @@ -161,9 +160,16 @@ impl WorkspaceJournal { format!("{slot_name}.toml") }; let slot_path = journal_dir.join(file_name); - let raw = fs::read_to_string(&slot_path)?; + // Falls back to `.toml.bak` when the newest checkpoint is truncated or corrupt. + let (raw, _source) = read_with_backup(&slot_path, &|t| toml::from_str::(t).is_ok())?; let mut journal: Self = toml::from_str(&raw) .map_err(|e| io::Error::new(ErrorKind::InvalidData, e.to_string()))?; + if journal.version > JOURNAL_SCHEMA_VERSION { + return Err(io::Error::new( + ErrorKind::InvalidData, + format!("journal schema v{} is newer than supported v{JOURNAL_SCHEMA_VERSION}", journal.version), + )); + } for session in &mut journal.sessions { session.z_stack_rects = Self::sanitized_z_stack_rects_for_mode(session); @@ -371,6 +377,7 @@ impl super::WindowManager { #[cfg(test)] mod tests { use super::*; + use std::fs; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; diff --git a/src/wm/mod.rs b/src/wm/mod.rs index 21d048f9..2420b047 100644 --- a/src/wm/mod.rs +++ b/src/wm/mod.rs @@ -1,4 +1,5 @@ pub mod agent_mail; +pub mod resilience; pub mod swarm_cmd; mod agent_mail_apply; pub mod agent_state; @@ -272,6 +273,7 @@ pub struct WindowManager { pub(crate) mcp_console: crate::mcp_console::McpConsoleState, /// Whether this client detached cleanly without terminating the session (`SAFE-02`). pub(crate) client_detached: bool, + pub(crate) resilience: resilience::ResilienceState, /// In-memory cache of saved/autosaved named layout profiles for instant dynamic switching. pub(crate) layout_profiles_mem: BTreeMap, /// Timestamp of the most recent software cursor blink toggle (`UI-05`). @@ -387,6 +389,7 @@ impl WindowManager { web_surface: crate::api::web_surface::WebSurfaceState::new(), mcp_console: crate::mcp_console::McpConsoleState::new(), client_detached: false, + resilience: Default::default(), layout_profiles_mem: BTreeMap::new(), last_blink_toggle: Instant::now(), pane_chrome: BTreeMap::new(), @@ -968,6 +971,7 @@ impl WindowManager { registry::capture::tick(self, Instant::now()); self.sync_clipboard_records(); self.maybe_rebalance_scrollback(Instant::now()); + changed |= self.resilience_tick(Instant::now()); changed } diff --git a/src/wm/resilience.rs b/src/wm/resilience.rs new file mode 100644 index 00000000..db89c974 --- /dev/null +++ b/src/wm/resilience.rs @@ -0,0 +1,381 @@ +//! Workspace resilience: debounced background checkpoints, unclean-shutdown recovery, a +//! persisted session tree, and daemon start/stop jobs that never block the UI thread. +//! +//! The UI thread only builds an in-memory snapshot; serialization and all file I/O happen on a +//! worker thread. A `.running` marker is written at startup and removed on a clean +//! exit, so finding one whose process is gone means the previous run ended uncleanly. + +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc::{self, Receiver, Sender}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use super::session::SavedWorkspaceSession; +use super::session_tree::SessionTree; +use super::WindowManager; +use crate::secure_fs::{read_with_backup, write_private_file_durable, DurableSource}; + +/// How often the cheap change probe runs. +const PROBE_EVERY: Duration = Duration::from_millis(500); +/// Quiet time after a layout/pane/cwd change before a checkpoint is written. +const DEBOUNCE: Duration = Duration::from_millis(750); +/// Upper bound between checkpoints while the workspace is otherwise unchanged. +const PERIODIC: Duration = Duration::from_secs(30); + +/// A daemon lifecycle operation run off the UI thread. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DaemonJob { + Start { name: String }, + Stop { name: String, force: bool }, +} + +enum CheckpointJob { + Session { path: PathBuf, snap: Box }, + Tree { path: PathBuf, tree: SessionTree }, +} + +/// Per-workspace resilience state (one field on [`WindowManager`]). +#[derive(Default)] +pub struct ResilienceState { + daemon_jobs: Vec>, + restore_note: Option, + worker: Option>, + worker_errors: Option>, + last_probe: Option, + last_sig: Option, + last_tree_sig: Option, + dirty_since: Option, + last_write: Option, + run_marker: Option, + last_ui_cost: Duration, + written: Arc, +} + +impl std::fmt::Debug for ResilienceState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ResilienceState") + .field("run_marker", &self.run_marker) + .field("checkpoints_written", &self.checkpoints_written()) + .finish_non_exhaustive() + } +} + +impl ResilienceState { + /// Checkpoints completed by the worker so far. + pub fn checkpoints_written(&self) -> u64 { + self.written.load(Ordering::Relaxed) + } + + /// Time the last checkpoint cost on the calling (UI) thread. + pub fn last_ui_cost(&self) -> Duration { + self.last_ui_cost + } + + fn sender(&mut self) -> Option> { + if self.worker.is_none() { + let (tx, rx) = mpsc::channel::(); + let (err_tx, err_rx) = mpsc::channel::(); + let written = Arc::clone(&self.written); + std::thread::Builder::new() + .name("checkpoint-writer".to_string()) + .spawn(move || { + for job in rx { + let res = match &job { + CheckpointJob::Session { path, snap } => toml::to_string_pretty(snap.as_ref()) + .map_err(|e| e.to_string()) + .and_then(|t| write_private_file_durable(path, t.as_bytes()).map_err(|e| e.to_string())), + CheckpointJob::Tree { path, tree } => toml::to_string_pretty(tree) + .map_err(|e| e.to_string()) + .and_then(|t| write_private_file_durable(path, t.as_bytes()).map_err(|e| e.to_string())), + }; + match res { + Ok(()) => { + if matches!(job, CheckpointJob::Session { .. }) { + written.fetch_add(1, Ordering::Relaxed); + } + } + Err(e) => { + let _ = err_tx.send(e); + } + } + } + }) + .ok()?; + self.worker = Some(tx); + self.worker_errors = Some(err_rx); + } + self.worker.clone() + } +} + +impl Drop for ResilienceState { + fn drop(&mut self) { + // A clean exit clears the marker; a panic or kill leaves it for the next start. + if !std::thread::panicking() + && let Some(marker) = self.run_marker.take() + { + let _ = std::fs::remove_file(marker); + } + } +} + +fn sanitize(name: &str) -> String { + let s: String = name.trim().chars().filter(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_')).collect(); + if s.is_empty() { "default".to_string() } else { s } +} + +fn marker_owner_alive(pid: u32) -> bool { + if pid == std::process::id() || !crate::daemon::supervisor::check_pid_alive(pid) { + return false; + } + // Guard against PID reuse where the platform lets us look. + match std::fs::read_to_string(format!("/proc/{pid}/comm")) { + Ok(comm) => comm.trim().contains("bilterm"), + Err(_) => !cfg!(target_os = "linux"), + } +} + +fn parse_session(text: &str) -> Option { + toml::from_str::(text).ok().filter(|s| !s.panes.is_empty()) +} + +impl WindowManager { + /// Resilience state (checkpoint counters, timings). + pub fn resilience(&self) -> &ResilienceState { + &self.resilience + } + + /// Note left by the last restore (backup fallback, crash recovery), consumed by the caller. + pub fn take_restore_note(&mut self) -> Option { + self.resilience.restore_note.take() + } + + pub(crate) fn set_restore_note(&mut self, note: String) { + self.resilience.restore_note = Some(note); + } + + /// `/.ckpt`: the crash-recovery checkpoint of the active session. + pub fn checkpoint_path(&self) -> Option { + let dir = self.resolve_session_dir().ok()?; + Some(dir.join(format!("{}.ckpt", sanitize(&self.config.history_session)))) + } + + fn session_tree_path(&self) -> Option { + Some(self.resolve_session_dir().ok()?.join("session_tree.state")) + } + + fn workspace_signature(&self) -> u64 { + let mut h = DefaultHasher::new(); + self.config.history_session.hash(&mut h); + format!("{:?}{:?}{:?}{:?}", self.layout_mode, self.split_tree, self.z_stack.order(), self.zoomed_pane).hash(&mut h); + for p in self.panes.values() { + (p.id(), p.title(), p.resolve_cwd()).hash(&mut h); + } + h.finish() + } + + fn tree_signature(&self) -> u64 { + let mut h = DefaultHasher::new(); + for n in &self.daemon_state.session_tree.nodes { + (&n.name, &n.parent, &n.children, n.collapsed, n.frozen).hash(&mut h); + format!("{:?}", n.kind).hash(&mut h); + } + h.finish() + } + + /// Queues a checkpoint of the active session now. The UI thread only clones the snapshot; + /// the worker serializes and writes it. Returns `false` when no session dir is available. + pub fn checkpoint_now(&mut self) -> bool { + let started = Instant::now(); + let Some(path) = self.checkpoint_path() else { return false }; + if let Some(dir) = path.parent() { + let _ = crate::secure_fs::create_private_dir_all(dir); + } + let snap = Box::new(self.snapshot_session(&sanitize(&self.config.history_session))); + let Some(tx) = self.resilience.sender() else { return false }; + let sent = tx.send(CheckpointJob::Session { path, snap }).is_ok(); + self.resilience.last_ui_cost = started.elapsed(); + self.resilience.dirty_since = None; + self.resilience.last_write = Some(Instant::now()); + sent + } + + fn persist_session_tree(&mut self) { + let Some(path) = self.session_tree_path() else { return }; + if let Some(dir) = path.parent() { + let _ = crate::secure_fs::create_private_dir_all(dir); + } + let tree = self.daemon_state.session_tree.clone(); + if let Some(tx) = self.resilience.sender() { + let _ = tx.send(CheckpointJob::Tree { path, tree }); + } + } + + /// Loads the persisted session tree (parents, order, fold/freeze) saved by an earlier run. + pub fn load_persisted_session_tree(&mut self) -> bool { + let Some(path) = self.session_tree_path() else { return false }; + let valid = |t: &str| toml::from_str::(t).is_ok_and(|tr| !tr.nodes.is_empty()); + let Ok((text, source)) = read_with_backup(&path, &valid) else { return false }; + let Ok(mut tree) = toml::from_str::(&text) else { return false }; + // Drop links to sessions that no longer exist so a damaged file cannot create cycles. + let names: Vec = tree.nodes.iter().map(|n| n.name.clone()).collect(); + for n in &mut tree.nodes { + if n.parent.as_ref().is_some_and(|p| !names.contains(p) || *p == n.name) { + n.parent = None; + } + n.children.retain(|c| names.contains(c)); + } + self.daemon_state.session_tree = tree; + self.resilience.last_tree_sig = Some(self.tree_signature()); + if source == DurableSource::Backup { + self.set_restore_note("session tree file was corrupt; restored from backup".to_string()); + } + true + } + + /// Per-tick resilience work: finishes daemon jobs, reports checkpoint write failures, and + /// checkpoints after a debounced change (layout, pane open/close, cwd) or periodically. + /// Returns `true` when something user-visible changed. + pub fn resilience_tick(&mut self, now: Instant) -> bool { + let mut changed = self.poll_daemon_jobs(); + if let Some(err) = self.resilience.worker_errors.as_ref().and_then(|rx| rx.try_iter().last()) { + self.active_pane_mut().set_status_message(Some(format!("Checkpoint write failed: {err}"))); + changed = true; + } + if self.resilience.run_marker.is_none() { + return changed; + } + if self.resilience.last_probe.is_some_and(|t| now.duration_since(t) < PROBE_EVERY) { + return changed; + } + self.resilience.last_probe = Some(now); + let sig = self.workspace_signature(); + if self.resilience.last_sig != Some(sig) { + self.resilience.last_sig = Some(sig); + self.resilience.dirty_since = Some(now); + } + let tree_sig = self.tree_signature(); + if self.resilience.last_tree_sig != Some(tree_sig) { + self.resilience.last_tree_sig = Some(tree_sig); + self.persist_session_tree(); + } + let debounced = self.resilience.dirty_since.is_some_and(|t| now.duration_since(t) >= DEBOUNCE); + let periodic = self.resilience.last_write.is_none_or(|t| now.duration_since(t) >= PERIODIC); + if debounced || (periodic && self.resilience.dirty_since.is_none()) { + self.checkpoint_now(); + } + changed + } + + /// Marks this process as running the active session and, when `recover` is set and the + /// previous run ended uncleanly, restores its last checkpoint. Returns a status message + /// when an unclean shutdown was detected. + pub fn begin_run(&mut self, recover: bool) -> Option { + let dir = self.resolve_session_dir().ok()?; + crate::secure_fs::create_private_dir_all(&dir).ok()?; + let sess = sanitize(&self.config.history_session); + let marker = dir.join(format!("{sess}.running")); + let previous = std::fs::read_to_string(&marker).ok(); + if let Some(pid) = previous.as_deref().and_then(|s| s.trim().parse::().ok()) + && marker_owner_alive(pid) + { + return None; // another live instance owns this session; leave its marker alone + } + let note = previous.filter(|_| recover).map(|_| self.recover_from_checkpoint(&sess)); + if crate::secure_fs::write_private_file(&marker, format!("{}\n", std::process::id()).as_bytes()).is_ok() { + self.resilience.run_marker = Some(marker); + } + self.resilience.last_sig = Some(self.workspace_signature()); + note + } + + fn recover_from_checkpoint(&mut self, sess: &str) -> String { + let Some(path) = self.checkpoint_path() else { + return format!("Session '{sess}' ended uncleanly; no checkpoint location available"); + }; + let age = std::fs::metadata(&path) + .and_then(|m| m.modified()) + .ok() + .and_then(|t| t.elapsed().ok()) + .map(|d| format!(" from {}s ago", d.as_secs())) + .unwrap_or_default(); + match read_with_backup(&path, &|t| parse_session(t).is_some()) { + Ok((text, source)) => match parse_session(&text) { + Some(snap) => { + let panes = self.apply_saved_session_snapshot(&snap); + let from = if source == DurableSource::Backup { " (backup copy; newest was corrupt)" } else { "" }; + format!( + "Recovered session '{sess}' after an unclean shutdown: {panes} pane(s) restored from the checkpoint{age}{from}; programs were restarted, not kept running" + ) + } + None => format!("Session '{sess}' ended uncleanly; its checkpoint could not be read"), + }, + Err(e) => format!("Session '{sess}' ended uncleanly; no usable checkpoint ({e})"), + } + } + + /// Clears the running marker (clean exit). Also happens when the workspace is dropped. + pub fn end_run(&mut self) { + if let Some(marker) = self.resilience.run_marker.take() { + let _ = std::fs::remove_file(marker); + } + } + + /// Runs a daemon start/stop on a worker thread; the outcome is reported by the next + /// [`Self::resilience_tick`] as a status message. Never blocks the caller. + pub fn spawn_daemon_job(&mut self, runtime_dir: &Path, job: DaemonJob) { + let (tx, rx) = mpsc::channel(); + let dir = runtime_dir.to_path_buf(); + let spawned = std::thread::Builder::new().name("daemon-job".to_string()).spawn(move || { + let _ = tx.send(run_daemon_job(&dir, &job)); + }); + if spawned.is_ok() { + self.resilience.daemon_jobs.push(rx); + } + } + + /// `true` while a daemon start/stop is still running in the background. + pub fn daemon_job_pending(&self) -> bool { + !self.resilience.daemon_jobs.is_empty() + } + + fn poll_daemon_jobs(&mut self) -> bool { + let mut done = Vec::new(); + self.resilience.daemon_jobs.retain(|rx| match rx.try_recv() { + Ok(msg) => { + done.push(msg); + false + } + Err(mpsc::TryRecvError::Empty) => true, + Err(mpsc::TryRecvError::Disconnected) => false, + }); + let Some(msg) = done.pop() else { return false }; + self.active_pane_mut().set_status_message(Some(msg)); + if self.daemon_window().active { + self.refresh_daemon_window(); + } + true + } +} + +/// Executes `job` synchronously and returns the status message for it. +pub fn run_daemon_job(runtime_dir: &Path, job: &DaemonJob) -> String { + use crate::daemon::supervisor::{ensure_started, stop_daemon, DaemonStartResult}; + match job { + DaemonJob::Start { name } => match ensure_started(runtime_dir, name) { + Ok(DaemonStartResult::Started { pid, .. }) => format!("Daemon '{name}' started (PID {pid})"), + Ok(DaemonStartResult::AlreadyRunning { pid, .. }) => { + format!("Daemon '{name}' is already running (PID {pid})") + } + Err(e) => format!("Failed to start daemon '{name}': {e}"), + }, + DaemonJob::Stop { name, force } => match stop_daemon(runtime_dir, name, *force) { + Ok(true) => format!("Daemon '{name}' stopped"), + Ok(false) => format!("Daemon '{name}' was not running"), + Err(e) => format!("Failed to stop daemon '{name}': {e}"), + }, + } +} diff --git a/src/wm/session.rs b/src/wm/session.rs index 35b1383c..d5269ef8 100644 --- a/src/wm/session.rs +++ b/src/wm/session.rs @@ -73,10 +73,16 @@ pub struct SavedPaneState { pub colors: BTreeMap, } +/// Newest session file format this build writes and understands. +pub const SESSION_FORMAT_VERSION: u32 = 1; + /// Declarative TOML snapshot of a multi-pane Bilterm workspace (`ECO-02`). #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct SavedWorkspaceSession { pub name: String, + /// On-disk format version; files written before versioning read as `0`. + #[serde(default)] + pub format_version: u32, pub layout_mode: WorkspaceLayoutMode, pub split_tree: SplitTree, pub z_order: Vec, @@ -429,6 +435,7 @@ impl WindowManager { .collect(); SavedWorkspaceSession { name: name.to_string(), + format_version: SESSION_FORMAT_VERSION, layout_mode: self.layout_mode, split_tree: self.split_tree.clone(), z_order: self.z_stack.order().to_vec(), @@ -528,7 +535,7 @@ impl WindowManager { let snap = self.snapshot_session(file_stem); let toml_str = toml::to_string_pretty(&snap) .map_err(|e| crate::error::BiltermError::Config(e.to_string()))?; - crate::secure_fs::write_private_file(&path, toml_str.as_bytes())?; + crate::secure_fs::write_private_file_durable(&path, toml_str.as_bytes())?; Ok(path) } @@ -550,9 +557,22 @@ impl WindowManager { safe_name }; let path = self.resolve_session_dir()?.join(format!("{file_stem}.toml")); - let raw = std::fs::read_to_string(&path)?; + // A truncated or corrupt file falls back to the last good `.bak`; never start empty silently. + let parses = |t: &str| toml::from_str::(t).is_ok(); + let (raw, source) = crate::secure_fs::read_with_backup(&path, &parses)?; let mut saved: SavedWorkspaceSession = toml::from_str(&raw) .map_err(|e| crate::error::BiltermError::Config(e.to_string()))?; + if saved.format_version > SESSION_FORMAT_VERSION { + return Err(crate::error::BiltermError::Config(format!( + "session '{file_stem}' uses file format v{} but this bilterm reads up to v{SESSION_FORMAT_VERSION}", + saved.format_version + ))); + } + if source == crate::secure_fs::DurableSource::Backup { + self.set_restore_note(format!( + "session file '{file_stem}.toml' was corrupt: restored the last good backup (damaged copy kept as {file_stem}.toml.corrupt)" + )); + } let was_detached = saved.detached; self.config.history_session = file_stem.clone(); self.config.active_layout_profile = file_stem.clone(); diff --git a/src/wm/v2_runtime.rs b/src/wm/v2_runtime.rs index 00034659..854535df 100644 --- a/src/wm/v2_runtime.rs +++ b/src/wm/v2_runtime.rs @@ -207,16 +207,26 @@ impl WindowManager { /// Detaches the current UI client while preserving the complete multi-pane window layout & session (`SAFE-02`, `DAEM-02`). pub fn detach_client(&mut self) -> bool { - self.client_detached = true; let sess = self.config.history_session.clone(); let pane_count = self.panes.len(); - let _ = self.save_session(&sess); - let _ = self.save_journal_checkpoint(None, &sess); + self.client_detached = true; // recorded in the snapshot + if let Err(e) = self.save_session(&sess) { + // Without a saved session there is nothing to attach to: stay attached and say why. + self.client_detached = false; + self.active_pane_mut().set_status_message(Some(format!( + "Detach cancelled: session '{sess}' could not be saved ({e}); nothing was closed" + ))); + return false; + } + let journal = match self.save_journal_checkpoint(None, &sess) { + Ok(_) => String::new(), + Err(e) => format!("; journal checkpoint failed: {e}"), + }; self.daemon_state .session_panes .insert(sess.clone(), pane_count); self.active_pane_mut().set_status_message(Some(format!( - "Detached session '{sess}' ({pane_count} pane(s) window layout preserved — run 'bilterm' or ':attach {sess}' to resume)" + "Detached session '{sess}' ({pane_count} pane(s): layout, cwd and screen text saved{journal}; running programs end with this client and are restarted on attach — run 'bilterm attach {sess}')" ))); true } @@ -337,8 +347,7 @@ impl WindowManager { ) -> std::io::Result { let dir = match journal_dir.or(self.daemon_state.journal_dir.as_deref()) { Some(d) => d.to_path_buf(), - None => crate::secure_fs::private_runtime_dir() - .and_then(|base| crate::secure_fs::private_dir_in(&base, "journal"))?, + None => WorkspaceJournal::default_journal_dir()?, }; let journal = self.build_workspace_journal(); journal.save_atomic(&dir, slot_name) @@ -664,18 +673,27 @@ impl WindowManager { } else { sess_nodes.get(pos + 1).cloned() }; - if let Some(parent) = parent_cand - && parent != target - { - let _ = self - .mcp_console - .execute(&format!("graft {target} {parent}"), &mut self.daemon_state); - self.active_pane_mut().set_status_message(Some(format!( - "MCP: Grafted '{target}' under '{parent}'" - ))); - true - } else { - false + match parent_cand.filter(|p| p != &target) { + // Self, cycle and missing-session targets are rejected by `graft` with a reason. + Some(parent) => match self.daemon_state.session_tree.graft(&target, &parent) { + Ok(()) => { + self.active_pane_mut().set_status_message(Some(format!( + "MCP: Grafted '{target}' under '{parent}'" + ))); + true + } + Err(e) => { + self.active_pane_mut() + .set_status_message(Some(format!("MCP: cannot reparent '{target}': {e}"))); + false + } + }, + None => { + self.active_pane_mut().set_status_message(Some(format!( + "MCP: no other session to reparent '{target}' under" + ))); + false + } } } else { let _ = self diff --git a/src/wm/window_manage_bulk.rs b/src/wm/window_manage_bulk.rs index 2303c1cb..6c5f8dbd 100644 --- a/src/wm/window_manage_bulk.rs +++ b/src/wm/window_manage_bulk.rs @@ -348,6 +348,7 @@ impl WindowManager { .collect(); let snap = SavedWorkspaceSession { name: s.clone(), + format_version: crate::wm::session::SESSION_FORMAT_VERSION, layout_mode: self.layout_mode, split_tree: bg.split_tree.clone(), z_order: bg.z_stack.order().to_vec(), diff --git a/tests/daemon_resilience_state.rs b/tests/daemon_resilience_state.rs new file mode 100644 index 00000000..4262baaa --- /dev/null +++ b/tests/daemon_resilience_state.rs @@ -0,0 +1,190 @@ +//! Durable session state, checkpoints, crash recovery, reparent persistence and the shared +//! RPC/colon session save path. + +use std::fs; +use std::path::Path; +use std::time::{Duration, Instant}; + +use bilterm::action::Action; +use bilterm::api::json::JsonValue; +use bilterm::api::AutomationControlPlane; +use bilterm::config::BiltermConfig; +use bilterm::secure_fs::{backup_path, read_with_backup, write_private_file_durable, DurableSource}; +use bilterm::wm::resilience::DaemonJob; +use bilterm::wm::WindowManager; + +fn wm_in(dir: &Path) -> WindowManager { + let mut wm = WindowManager::new_headless(24, 80); + let cfg = BiltermConfig { session_dir: Some(dir.to_string_lossy().into_owned()), ..BiltermConfig::default() }; + wm.apply_config(cfg); + wm +} + +fn wait_for(mut done: impl FnMut() -> bool) -> bool { + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline { + if done() { + return true; + } + std::thread::sleep(Duration::from_millis(10)); + } + false +} + +#[test] +fn durable_write_keeps_last_good_backup_and_read_falls_back() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("state.toml"); + write_private_file_durable(&path, b"v = 1\n").unwrap(); + write_private_file_durable(&path, b"v = 2\n").unwrap(); + assert_eq!(fs::read_to_string(backup_path(&path)).unwrap(), "v = 1\n"); + let valid = |t: &str| t.starts_with("v = "); + assert_eq!(read_with_backup(&path, &valid).unwrap(), ("v = 2\n".to_string(), DurableSource::Primary)); + + fs::write(&path, "v").unwrap(); // truncated + assert_eq!(read_with_backup(&path, &valid).unwrap(), ("v = 1\n".to_string(), DurableSource::Backup)); + assert_eq!(fs::read_to_string(&path).unwrap(), "v = 1\n", "good copy is put back"); + assert!(tmp.path().join("state.toml.corrupt").exists(), "damaged copy is kept for inspection"); + + fs::write(&path, "x").unwrap(); + fs::write(backup_path(&path), "y").unwrap(); + assert!(read_with_backup(&path, &valid).is_err(), "two bad copies are an error, not empty state"); +} + +#[test] +fn truncated_session_file_restores_from_backup_and_says_so() { + let tmp = tempfile::tempdir().unwrap(); + let mut wm = wm_in(tmp.path()); + wm.apply(Action::SplitVertical).unwrap(); + wm.save_session("work").unwrap(); + let path = wm.save_session("work").unwrap(); // second save rotates the first into .bak + let full = fs::read_to_string(&path).unwrap(); + assert!(full.contains("format_version = 1")); + fs::write(&path, &full[..full.len() / 2]).unwrap(); + + let mut fresh = wm_in(tmp.path()); + assert_eq!(fresh.restore_session("work").expect("falls back to the backup"), 2); + let note = fresh.take_restore_note().expect("the user is told about the fallback"); + assert!(note.contains("corrupt") && note.contains("backup"), "{note}"); + + // Both copies damaged: an error (never a panic, never a silent empty session). + fs::write(&path, "not toml [").unwrap(); + fs::write(backup_path(&path), "also bad [").unwrap(); + assert!(wm_in(tmp.path()).restore_session("work").is_err()); +} + +#[test] +fn newer_session_format_is_refused_with_a_reason() { + let tmp = tempfile::tempdir().unwrap(); + let wm = wm_in(tmp.path()); + let path = wm.save_session("future").unwrap(); + let text = fs::read_to_string(&path).unwrap().replace("format_version = 1", "format_version = 99"); + fs::write(&path, text).unwrap(); + let err = wm_in(tmp.path()).restore_session("future").unwrap_err().to_string(); + assert!(err.contains("v99"), "{err}"); +} + +#[test] +fn detach_does_not_claim_success_when_the_save_fails() { + let tmp = tempfile::tempdir().unwrap(); + let blocker = tmp.path().join("not-a-dir"); + fs::write(&blocker, "x").unwrap(); + let mut wm = wm_in(&blocker.join("sessions")); + assert!(!wm.detach_client(), "detach must not proceed without a saved session"); + assert!(!wm.is_client_detached()); + let msg = wm.active_pane().status_message().unwrap_or_default().to_string(); + assert!(msg.contains("could not be saved"), "{msg}"); +} + +#[test] +fn unclean_shutdown_is_detected_and_last_checkpoint_restored() { + let tmp = tempfile::tempdir().unwrap(); + let mut wm = wm_in(tmp.path()); + assert_eq!(wm.begin_run(true), None, "first run has nothing to recover"); + wm.apply(Action::SplitVertical).unwrap(); + wm.apply(Action::SplitVertical).unwrap(); + assert!(wm.checkpoint_now()); + let ui_cost = wm.resilience().last_ui_cost(); + assert!(ui_cost < Duration::from_millis(5), "checkpoint cost {ui_cost:?} on the UI thread"); + assert!(wait_for(|| wm.resilience().checkpoints_written() >= 1), "worker writes the checkpoint"); + std::mem::forget(wm); // crash: no clean-exit path runs, the running marker stays + + let mut next = wm_in(tmp.path()); + let note = next.begin_run(true).expect("unclean shutdown is reported"); + assert!(note.contains("Recovered session 'default'") && note.contains("3 pane(s)"), "{note}"); + assert_eq!(next.pane_count(), 3); + drop(next); // clean exit clears the marker + + assert_eq!(wm_in(tmp.path()).begin_run(true), None, "a clean exit is not a crash"); +} + +#[test] +fn layout_change_is_checkpointed_after_the_debounce() { + let tmp = tempfile::tempdir().unwrap(); + let mut wm = wm_in(tmp.path()); + wm.begin_run(false); + let t0 = Instant::now(); + wm.resilience_tick(t0); // periodic baseline checkpoint + assert!(wait_for(|| wm.resilience().checkpoints_written() >= 1)); + wm.apply(Action::SplitVertical).unwrap(); + wm.resilience_tick(t0 + Duration::from_millis(600)); // change seen, debounce starts + assert_eq!(wm.resilience().checkpoints_written(), 1, "no write inside the debounce window"); + wm.resilience_tick(t0 + Duration::from_millis(1500)); + assert!(wait_for(|| wm.resilience().checkpoints_written() >= 2), "debounced checkpoint written"); + let saved = fs::read_to_string(wm.checkpoint_path().unwrap()).unwrap(); + assert_eq!(saved.matches("[[panes]]").count(), 2); +} + +#[test] +fn reparent_rejects_cycles_and_persists_across_restart() { + let tmp = tempfile::tempdir().unwrap(); + let mut wm = wm_in(tmp.path()); + wm.begin_run(false); + wm.session_tree_mut().branch("main", "a").unwrap(); + wm.session_tree_mut().branch("a", "b").unwrap(); + assert!(wm.session_tree_mut().graft("a", "b").unwrap_err().contains("cycle")); + assert!(wm.session_tree_mut().graft("a", "a").is_err()); + assert!(wm.session_tree_mut().graft("a", "ghost").unwrap_err().contains("does not exist")); + wm.session_tree_mut().graft("b", "main").unwrap(); + wm.resilience_tick(Instant::now()); + assert!(wait_for(|| tmp.path().join("session_tree.state").exists())); + + let mut next = wm_in(tmp.path()); + assert!(next.load_persisted_session_tree()); + assert_eq!(next.session_tree().node("b").unwrap().parent.as_deref(), Some("main")); + assert_eq!(next.session_tree().node("a").unwrap().parent.as_deref(), Some("main")); +} + +#[test] +fn rpc_session_save_uses_the_same_default_and_activation_as_the_colon_command() { + let tmp = tempfile::tempdir().unwrap(); + let mut plane = AutomationControlPlane::new_headless(24, 80); + let cfg = BiltermConfig { + session_dir: Some(tmp.path().to_string_lossy().into_owned()), + history_session: "alpha".to_string(), + ..BiltermConfig::default() + }; + plane.wm.apply_config(cfg); + let none = JsonValue::obj([] as [(&str, JsonValue); 0]); + let saved = plane.handle_rpc_session_save(&none).unwrap(); + assert_eq!(saved.get("session").and_then(JsonValue::as_str), Some("alpha"), "defaults to the active session"); + let named = JsonValue::obj([("name", JsonValue::String("beta".to_string()))]); + plane.handle_rpc_session_save(&named).unwrap(); + assert_eq!(plane.wm.config().history_session, "beta", "naming a session activates it, as `:session save beta` does"); + let restored = plane.handle_rpc_session_restore(&none).unwrap(); + assert_eq!(restored.get("session").and_then(JsonValue::as_str), Some("beta")); +} + +#[test] +fn daemon_stop_job_reports_through_the_tick_instead_of_blocking() { + let tmp = tempfile::tempdir().unwrap(); + let mut wm = WindowManager::new_headless(24, 80); + let started = Instant::now(); + wm.spawn_daemon_job(tmp.path(), DaemonJob::Stop { name: "nobody".to_string(), force: false }); + assert!(started.elapsed() < Duration::from_millis(50), "the caller returns at once"); + assert!(wait_for(|| { + wm.resilience_tick(Instant::now()); + !wm.daemon_job_pending() + })); + assert_eq!(wm.active_pane().status_message(), Some("Daemon 'nobody' was not running")); +} From 837e3d5be621f03b8186e00ca62e8f2c1ec36927 Mon Sep 17 00:00:00 2001 From: Bill Sommers Date: Tue, 10 Nov 2026 01:07:39 -0500 Subject: [PATCH 4/6] docs: detach saves the layout, it does not keep programs running Co-Authored-By: Claude Fable 5.2 --- README.md | 4 ++-- docs/README.md | 2 +- docs/user-guide/06-keyboard-mouse-and-command-reference.md | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 49473fe3..3d40a8f0 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ - **Composite Terminal Panes & Mainframe Paging**: Pin the command prompt to **Top** (`Alt+T`), **Bottom** (`Alt+B`), **Free-Float** (`Alt+Up`/`Down`), or **Fixed Split** (`Alt+S`) over a 60 FPS flood-immune PTY canvas with `DECSET 1049` auto-hide (`vim`, `htop`, `nmon`), 3-signal `!ECHO LOCK` password protection, and Mainframe Block/Page-Hold (`MORE...` / `HOLDING`) or Smooth Kinetic scrolling (`:scroll`). → *[User Guide Ch. 2](docs/user-guide/02-floating-command-prompt.md)* - **2D Rectangular Block Selection & Copy**: Drag-select 2D rectangular columns with `Alt+LeftDrag` or persistent `:rect mode` across terminal output, log columns, or tables. Pastes cleanly to system and OSC 52 clipboards (`Alt+R` / `r` in Copy Mode). → *[User Guide Ch. 6](docs/user-guide/06-keyboard-mouse-and-command-reference.md)* -- **Full Session Persistence & Exit Guard**: Detach with `Ctrl+X D` or `:detach` leaving split tree ratios, Z-order stack, prompt anchors, active shells, running commands, and per-pane CWD intact; resume via `bilterm attach [session]`. Protects root shell exits via interactive Borland Exit Guard modals. → *[User Guide Ch. 3](docs/user-guide/03-window-manager-and-z-stack.md)* +- **Full Session Persistence & Exit Guard**: Detach with `Ctrl+X D` or `:detach` to save split tree ratios, Z-order stack, prompt anchors, per-pane shell, CWD and visible screen text; `bilterm attach [session]` rebuilds the layout and restarts each pane's shell and recorded command. Programs do **not** keep running while detached (run them inside a tmux link if they must survive). The workspace is also checkpointed in the background, and after a crash or `kill -9` the next start restores the last checkpoint and says so. Protects root shell exits via interactive Borland Exit Guard modals. → *[User Guide Ch. 3](docs/user-guide/03-window-manager-and-z-stack.md)* - **Native Desktop GUI (`--gui`) & Per-Pane Font Scaling**: Run standalone without a host terminal on Linux (X11), macOS (Cocoa), or Windows (GDI). Scale each pane's font independently (`50%..300%` via `Ctrl++`/`Ctrl+-`/`Ctrl+Wheel`), drag all borders to resize, and plot high-res Kitty/Sixel/Braille graphics (`bilterm plot`, `:canvas`). → *[User Guide Ch. 8](docs/user-guide/08-extended-cli-and-gui-examples.md)* - **Zero-Trust Confinement**: All runtime sockets, session TOMLs, command history exports, and `.cast` recordings reside in `0700` directories with `0600` atomic file permissions. Unix sockets enforce `SO_PEERCRED` UID checks; HTTP transports require local loopback and bearer token authentication. → *[Security & Best Practices (Ch. 10)](docs/user-guide/10-security-and-best-practices.md)* @@ -103,7 +103,7 @@ bilterm run starter-packs/01-multi-agent-swarm/playbook.toml | **Global Fuzzy Finder** | `Ctrl+Alt+F` | `:find [agent \| session \| history] ` | Unified registry search with secret masking | | **Agent Messaging & Swarms** | `Alt+]` (next blocked) | `:agent [name \| send \| inbox \| list \| snoop]` | Inter-agent messaging, mailboxes & packet telemetry | | **Appearance & Display Modes** | `F11` / `F4` (` e`) | `:appearance [obsidian \| legacy \| ...]` / `:mode [standard \| zen]` | 8 visual profiles & distraction-free Zen mode | -| **Detach & Re-Attach Workspace** | `Ctrl+X D` / Exit Guard `[d]` | `:detach [name]` / `:attach [name]` | Leaves shells running in background daemon | +| **Detach & Re-Attach Workspace** | `Ctrl+X D` / Exit Guard `[d]` | `:detach [name]` / `:attach [name]` | Saves the layout; shells and commands are restarted on attach (not kept running) | | **Split Horizontal / Split Vertical / Zoom** | `F2` (` -`) / `Alt+V` (` \|`) / `F5` (` z`) | `:split h` / `:split v` / `:zoom` | Binary space partitioning tiling | | **Tiled Splits vs. Z-Stack / Shuffle Deck** | `Cmd+L` / `Ctrl+Shift+L` (` M`, Window mode `s`) / `F6` (` o`) / `Alt+1..9` | `:layout tile\|stack` / `:deck next\|prev` | Overlapping Borland window cascade & switching | | **Utility Windows (Fonts / Studio)** | `*` (pin) / `[` / `]` (scale) | `:fonts` / `:commands` / `:studio` | Workbench windows, live recorder & step debugger | diff --git a/docs/README.md b/docs/README.md index 09058252..106102ca 100644 --- a/docs/README.md +++ b/docs/README.md @@ -39,7 +39,7 @@ Welcome to the comprehensive documentation suite for **Bilterm (`v3.2.0`)** — | Command | Description | |---------|-------------| | `bilterm` | Launch interactive IBM 3270 × Borland Turbo Vision TUI multiplexer (auto-resumes detached session) | -| `bilterm attach [session]` (`-a` / `--attach`) | Re-attach & restore saved/detached multi-pane window layout intact (via live daemon RPC or saved session) | +| `bilterm attach [session]` (`-a` / `--attach`) | Re-attach: rebuild the saved/detached multi-pane window layout from the session file and restart its shells and recorded commands | | `bilterm ls` (`bilterm sessions` / `--list-sessions`) | List live background daemon sockets (`LIVE DAEMON`) & saved `.toml` layouts (`SAVED LAYOUT`) | | `bilterm -G` / `--gui` | Launch standalone native desktop GUI window (X11 / Win32 GDI / Cocoa) | | `bilterm --gui-font ` (`-F` / `--font`) | Set font family or preset (`"JetBrains Mono"`, `"IBM 3270"`, `"Fira Code"`, `"vga"`, etc.) | diff --git a/docs/user-guide/06-keyboard-mouse-and-command-reference.md b/docs/user-guide/06-keyboard-mouse-and-command-reference.md index 88764586..926170e6 100644 --- a/docs/user-guide/06-keyboard-mouse-and-command-reference.md +++ b/docs/user-guide/06-keyboard-mouse-and-command-reference.md @@ -97,7 +97,7 @@ | `Alt+O` | ` L` | `:spotlight` | Cycle Focal Spotlight shader around active cursor (`Circle` -> `Ellipse` -> `Pixel` -> `Off`) | | `Alt+Shift+O` | — | `:focalzoom` | Cycle Focal Zoom Magnifying Lens (`1.25x` -> `1.5x` -> `2.0x` -> `Off`) | | `Alt+}` / `Alt+{` | ` }` / ` {` | `:spotlight +\|-` | Expand or shrink Focal Spotlight / Focal Zoom lens radius | -| — | `Ctrl+X D` | `:detach [name]` | Detach client & spawn background daemon keeping full multi-pane layout intact | +| — | `Ctrl+X D` | `:detach [name]` | Detach client & save the full multi-pane layout (programs are restarted on attach, not kept running) | | — | — | `:attach [name]` | Re-attach & restore saved/detached multi-pane window layout in-place | | — | — | `:mcp [status\|tree\|...]` | Open interactive Master Control Program Console (`↑`/`↓`/`j`/`k`/`Enter`/Click, `r`, `f`, `n`, `m`, `d`) | | **`Ctrl+Q`** | ` Ctrl+Q` | `:quit` / `:q` / `:quit!` | Exit Bilterm (`:quit` honors Exit Guard; `:quit!` forces immediate exit) | @@ -442,7 +442,7 @@ The previous defaults stay available on the `legacy` profile and, for one releas | `:command` | Aliases | Arguments | Description | |------------|---------|-----------|-------------| -| `:detach` | — | `[name]` | Save full multi-pane layout (`SplitTree` + `ZStack` + prompts + screen lines), spawn background daemon (`daemon-.sock`), and detach | +| `:detach` | — | `[name]` | Save full multi-pane layout (`SplitTree` + `ZStack` + prompts + screen lines), spawn a headless daemon (`daemon-.sock`) that re-launches the saved session, and detach. Running programs end with the client; a failed save cancels the detach | | `:attach` | — | `[name]` | Re-attach to live background daemon via JSON-RPC or restore saved `.toml` workspace layout | | `:mcp` | — | `[status \| tree \| float \| push [edge] \| attach \| spawn \| graft \| detach-subtree \| broadcast \| freeze \| thaw \| derez --confirm \| checkpoint \| shutdown --confirm]` | Master Control Program Root Console (`↑`/`↓`/`j`/`k`/`Enter`/Click to activate row, `r`=Refresh, `f`=Freeze/Thaw, `n`=Branch, `m`=Float↔Push, `d`=Edge, `Esc`=Close) | | `:exit-guard` | `:exitguard` | `[guard \| respawn \| detach \| off]` | Configure final/root shell exit protection (`[r]` Respawn, `[d]` Detach, `[q]` Quit) | From c3233bc41a04bb8e03c2f0ea7903a3f92b97610e Mon Sep 17 00:00:00 2001 From: Bill Sommers Date: Tue, 10 Nov 2026 01:11:12 -0500 Subject: [PATCH 5/6] fix(wm): an older backup checkpoint never replaces a newer saved layout Co-Authored-By: Claude Fable 5.2 --- src/wm/resilience.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/wm/resilience.rs b/src/wm/resilience.rs index db89c974..6df7f65c 100644 --- a/src/wm/resilience.rs +++ b/src/wm/resilience.rs @@ -305,6 +305,14 @@ impl WindowManager { match read_with_backup(&path, &|t| parse_session(t).is_some()) { Ok((text, source)) => match parse_session(&text) { Some(snap) => { + // The backup is one generation older: never let it replace a richer layout + // that the normal startup restore already brought back. + if source == DurableSource::Backup && snap.panes.len() < self.panes.len() { + return format!( + "Session '{sess}' ended uncleanly; its newest checkpoint was corrupt, kept the last saved layout ({} pane(s))", + self.panes.len() + ); + } let panes = self.apply_saved_session_snapshot(&snap); let from = if source == DurableSource::Backup { " (backup copy; newest was corrupt)" } else { "" }; format!( From 22e5d602597f2e27beeb66cc222992bec2f70e94 Mon Sep 17 00:00:00 2001 From: Bill Sommers Date: Tue, 10 Nov 2026 01:15:34 -0500 Subject: [PATCH 6/6] test: MCP reparent of a session under its own child reports the cycle Co-Authored-By: Claude Fable 5.2 --- tests/input_modal_wm_coverage.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/input_modal_wm_coverage.rs b/tests/input_modal_wm_coverage.rs index b5ac96fc..d0a14fc1 100644 --- a/tests/input_modal_wm_coverage.rs +++ b/tests/input_modal_wm_coverage.rs @@ -647,7 +647,8 @@ fn mcp_reparent_keys_graft_and_promote_nodes() { let mut w = mcp_tree(); press(&mut w, ch('n')); // give 'main' a child to reparent against press(&mut w, key); - assert!(status(&w).starts_with("MCP: Grafted 'main' under"), "{key:?}: {}", status(&w)); + // 'main' cannot go under its own child: the cycle is rejected and reported, not claimed. + assert!(status(&w).starts_with("MCP: cannot reparent 'main': cycle detected"), "{key:?}: {}", status(&w)); } for key in [ch('<'), ch('u'), ch('U'), km(KeyCode::Left, KeyModifiers::ALT)] { let mut w = mcp_tree();