From b20e9f7016d1dcf95471bf01345f2487ae00d443 Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 13:18:21 -0700 Subject: [PATCH 01/79] fix(monitor): look for the plugin where horus install puts it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #184. `horus install horus-monitor` symlinks into `PluginRegistry::global_bin_dir()`, which is `paths::horus_dir()/bin` — the platform CONFIG dir, so `~/.config/horus/bin` on Linux. `which_monitor_binary` checked `~/.horus/bin` and nothing else, so the install succeeded and `horus monitor` then reported the plugin as not installed. The two halves sit about 800 lines apart in different files, and the comment on the path that was checked still called it "the plugin install location". The lookup now asks the registry for that directory rather than making a second hardcoded guess at it, and keeps `~/.horus/bin` behind it so an existing install goes on working. The candidate list is split out as `plugin_bin_dirs()` so the relationship can be asserted rather than assumed — the same shape as `run_python::python_cache_roots`, which exists because the package cache had this exact defect. Two tests: that the lookup searches the directory the installer writes, and that the legacy path is still searched. Verified non-vacuous: removing the registry lookup fails the first with exit 101. Three documentation pages carried this as a working recipe and now carry an `export PATH=...` workaround (softmata/horus-docs#16). Those lines should come out once this ships. --- horus_manager/src/main.rs | 76 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/horus_manager/src/main.rs b/horus_manager/src/main.rs index 4cbeabb24..f5ed5e0e6 100644 --- a/horus_manager/src/main.rs +++ b/horus_manager/src/main.rs @@ -3689,13 +3689,41 @@ fn run_command(command: Commands) -> HorusResult<()> { } } +/// The directories a globally installed plugin binary can live in, in the order +/// they are searched. +/// +/// Split out from `which_monitor_binary` so it can be asserted against the +/// directory the installer actually writes, rather than being a pair of +/// hardcoded paths that drift apart silently — which is what happened: +/// `horus install horus-monitor` symlinks into +/// `PluginRegistry::global_bin_dir()`, the platform CONFIG dir +/// (`~/.config/horus/bin` on Linux, `$XDG_CONFIG_HOME/horus/bin` when set), +/// while this lookup checked `~/.horus/bin` and nothing else. Install +/// succeeded, `horus monitor` reported the plugin as not installed, and the +/// comment on the path it did check still called it "the plugin install +/// location". +fn plugin_bin_dirs() -> Vec { + let mut dirs = Vec::new(); + if let Ok(d) = horus_manager::plugins::PluginRegistry::global_bin_dir() { + dirs.push(d); + } + // The legacy location, kept so an existing install keeps working. + if let Some(home) = dirs::home_dir() { + let legacy = home.join(".horus/bin"); + if !dirs.contains(&legacy) { + dirs.push(legacy); + } + } + dirs +} + /// Find the horus-monitor binary on known paths. fn which_monitor_binary() -> Option { - // 1. Check ~/.horus/bin/ (plugin install location) - if let Some(home) = dirs::home_dir() { - let plugin_path = home.join(".horus/bin/horus-monitor"); - if plugin_path.exists() { - return Some(plugin_path); + // 1. Global plugin install locations. + for dir in plugin_bin_dirs() { + let candidate = dir.join("horus-monitor"); + if candidate.exists() { + return Some(candidate); } } @@ -3742,3 +3770,41 @@ fn which_monitor_binary() -> Option { None } + +#[cfg(test)] +mod monitor_lookup_tests { + use super::plugin_bin_dirs; + + /// `horus monitor` must look where `horus install` writes. + /// + /// These were two hardcoded paths in two files about 800 lines apart, and + /// they disagreed: install symlinked into the platform config dir and the + /// lookup checked `~/.horus/bin`, so the documented two-step workflow + /// could not work and said only "Monitor plugin is not installed." + /// Asserting the relationship rather than the literal path means a change + /// to either side fails here instead of at a user's terminal. + #[test] + fn the_lookup_searches_the_directory_the_installer_writes() { + let install_dir = horus_manager::plugins::PluginRegistry::global_bin_dir() + .expect("global_bin_dir resolves"); + let searched = plugin_bin_dirs(); + assert!( + searched.contains(&install_dir), + "horus install writes {install_dir:?}, but horus monitor searches only \ + {searched:?} — the install succeeds and the launcher then reports the \ + plugin as missing" + ); + } + + /// The legacy path stays searched, so an existing install keeps working. + #[test] + fn the_legacy_location_is_still_searched() { + if let Some(home) = dirs::home_dir() { + assert!( + plugin_bin_dirs().contains(&home.join(".horus/bin")), + "dropping ~/.horus/bin silently breaks anyone who installed before the \ + lookup was corrected" + ); + } + } +} From 6640fdac8196e4c6cd04c352d91fb83a6ef5da25 Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 13:25:36 -0700 Subject: [PATCH 02/79] fix(control): make horus node restart actually restart the node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `restart_node` sent PauseNode immediately followed by ResumeNode and printed "The scheduler will re-initialize this node on next tick". It did not. Both arms only call `NodeControls::set_paused` on a shared atomic; the code that re-runs `init()` is `reinit_pending_nodes`, which keys off `RegisteredNode::initialized`, and outside registration nothing in production ever set that back to false — the only two such assignments in the workspace are in `#[cfg(test)]` code. Four places in the runtime asserted the behaviour anyway: the clap help ("re-initialize without killing scheduler"), `restart_node`'s doc comment, an inline comment reading "Pause then resume triggers re-init in the scheduler", and `reinit_pending_nodes`'s own comment, "set by control commands". No control command set it. On a robot the operator restarts a wedged node, is told it worked, and moves on. Adds `ControlCommand::RestartNode { name }`, appended to the enum so a command serialised by an older CLI still decodes. The scheduler collects restart requests while draining the control topic — the loop holds a mutable borrow of `control_topic` and clearing the flag needs `self.nodes` — then clears `initialized` and lifts any operator pause, since a wedged node is often paused before being restarted and would otherwise be skipped by the very check that would have re-inited it. An unknown name logs rather than silently doing nothing. Tests drive the real control topic through `run_for`, not `tick_once`: the control topic is created in `run_with_filter`, so under `tick_once` it is None and no control command is polled at all. Worth knowing separately — operator control reaches a scheduler only while it is in `run()`. Verified non-vacuous: removing the RestartNode arm fails the test with exit 101 and `left: 1, right: 2`, which is precisely the pre-fix behaviour. Closes #184 is separate; this addresses the restart defect found by the bidirectional audit. --- horus_core/src/scheduling/control.rs | 16 +++ horus_core/src/scheduling/scheduler/mod.rs | 39 ++++++ horus_core/tests/control_restart.rs | 141 +++++++++++++++++++++ horus_manager/src/commands/node.rs | 15 +-- 4 files changed, 202 insertions(+), 9 deletions(-) create mode 100644 horus_core/tests/control_restart.rs diff --git a/horus_core/src/scheduling/control.rs b/horus_core/src/scheduling/control.rs index 42e41ce37..2dce507dd 100644 --- a/horus_core/src/scheduling/control.rs +++ b/horus_core/src/scheduling/control.rs @@ -34,6 +34,21 @@ pub enum ControlCommand { GracefulShutdown, /// Force a metrics flush to the SHM registry (useful for debugging). RequestMetricsFlush, + /// Re-run a node's `init()` on the next tick without stopping the scheduler. + /// + /// `horus node restart` used to send `PauseNode` followed immediately by + /// `ResumeNode` and claim that re-initialised the node. It did not: both + /// arms only toggle a shared atomic, and `reinit_pending_nodes` — the code + /// that actually re-runs `init()` — keys off `registered.initialized`, + /// which nothing outside registration ever set back to false. The command + /// printed "The scheduler will re-initialize this node on next tick" and + /// nothing happened. + /// + /// Added last so a command serialised by an older CLI still decodes. + RestartNode { + /// Name of the node to re-initialise. + name: String, + }, } impl ControlCommand { @@ -47,6 +62,7 @@ impl ControlCommand { } Self::GracefulShutdown => "Graceful shutdown".to_string(), Self::RequestMetricsFlush => "Flush metrics".to_string(), + Self::RestartNode { name } => format!("Restart node '{}'", name), } } } diff --git a/horus_core/src/scheduling/scheduler/mod.rs b/horus_core/src/scheduling/scheduler/mod.rs index 3337594b1..f1e5ccd6b 100644 --- a/horus_core/src/scheduling/scheduler/mod.rs +++ b/horus_core/src/scheduling/scheduler/mod.rs @@ -4557,6 +4557,11 @@ impl Scheduler { /// Process control commands — delegates to execution.rs topic-based implementation. fn process_control_commands(&mut self) { + // Names asked to restart. Collected rather than applied in place because + // the loop below holds a mutable borrow of `self.control_topic`, and + // clearing `initialized` needs `self.nodes`. + let mut restart_requests: Vec = Vec::new(); + // Poll the control topic for commands from CLI (horus node kill/pause/resume) if let Some(ref mut ctl) = self.control_topic { while let Some(cmd) = ctl.recv() { @@ -4583,10 +4588,44 @@ impl Scheduler { self.running .store(false, std::sync::atomic::Ordering::SeqCst); } + ControlCommand::RestartNode { ref name } => { + restart_requests.push(name.clone()); + } _ => {} // Other commands (SetNodeRate, etc.) — not yet implemented } } } + + // Clearing `initialized` is the whole mechanism: `reinit_pending_nodes` + // runs on the tick loop and re-runs `init()` for any node that is not + // stopped, not paused, and not initialised. Also lift an operator pause, + // since a wedged node is often paused before it is restarted and would + // otherwise be skipped by the very check that would have re-inited it. + for name in restart_requests { + if let Some(ref controls) = self.node_controls { + controls.set_paused(&name, false); + } + match self + .nodes + .iter_mut() + .find(|n| n.name.as_ref() == name.as_str()) + { + Some(registered) => { + registered.initialized = false; + registered.is_paused = false; + print_line(&format!( + "[CONTROL] Node '{}' will re-initialize on the next tick", + name + )); + } + None => { + print_line(&format!( + "[CONTROL] Restart requested for unknown node '{}'", + name + )); + } + } + } } /// Clean up session directory (no-op with flat namespace) diff --git a/horus_core/tests/control_restart.rs b/horus_core/tests/control_restart.rs new file mode 100644 index 000000000..5f6e09d5a --- /dev/null +++ b/horus_core/tests/control_restart.rs @@ -0,0 +1,141 @@ +//! `horus node restart` must actually re-initialise the node. +//! +//! The CLI sent `PauseNode` immediately followed by `ResumeNode` and printed +//! "The scheduler will re-initialize this node on next tick". It did not. +//! Both arms only toggle a shared atomic; the code that re-runs `init()` — +//! `reinit_pending_nodes` — keys off `RegisteredNode::initialized`, and outside +//! registration nothing in production ever set that back to false. A grep of +//! the workspace found the only two `initialized = false` assignments in +//! `#[cfg(test)]` code. +//! +//! Four places in the runtime asserted the behaviour anyway: the clap help +//! ("re-initialize without killing scheduler"), `restart_node`'s doc comment, +//! an inline comment reading "Pause then resume triggers re-init in the +//! scheduler", and `reinit_pending_nodes`'s own comment, "set by control +//! commands". On a robot the operator restarts a wedged node, is told it +//! worked, and moves on. +//! +//! Run: `cargo test -p horus_core --test control_restart` + +use horus_core::core::duration_ext::DurationExt; +use horus_core::core::Node; +use horus_core::scheduling::control::ControlCommand; +use horus_core::scheduling::Scheduler; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::Arc; + +struct CountsInit { + inits: Arc, +} + +impl Node for CountsInit { + fn name(&self) -> &str { + "restartable" + } + fn init(&mut self) -> horus_core::error::Result<()> { + self.inits.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + fn tick(&mut self) {} +} + +/// A `RestartNode` command re-runs `init()` while the scheduler is running. +/// +/// Driven through `run_for` rather than `tick_once`: the control topic is +/// created in `run_with_filter`, so under `tick_once` `self.control_topic` is +/// None and no control command is ever polled. That is worth knowing on its +/// own — `horus node pause/resume/restart` reach a scheduler only while it is +/// in `run()`. +#[test] +fn restart_command_reinitialises_the_node() { + let inits = Arc::new(AtomicU32::new(0)); + let sched_name = format!("restart_test_{}", std::process::id()); + let ctl_name = format!("horus.ctl.{}", sched_name); + + // Open the control topic before the run so the send cannot race setup. + let ctl: horus_core::communication::Topic = + horus_core::communication::Topic::new_with_kind( + &ctl_name, + horus_core::communication::TopicKind::System as u8, + ) + .expect("control topic opens"); + + let inits_for_thread = inits.clone(); + let name_for_thread = sched_name.clone(); + let handle = std::thread::spawn(move || { + let mut scheduler = Scheduler::new() + .name(&name_for_thread) + .tick_rate(200_u64.hz()); + scheduler + .add(CountsInit { + inits: inits_for_thread, + }) + .build() + .expect("node registers"); + scheduler.run_for(1200_u64.ms()).expect("run_for"); + }); + + // Let it start and perform the initial init(). + std::thread::sleep(std::time::Duration::from_millis(400)); + assert_eq!( + inits.load(Ordering::SeqCst), + 1, + "the node should have been initialised once by startup" + ); + + ctl.send(ControlCommand::RestartNode { + name: "restartable".to_string(), + }); + + handle.join().expect("scheduler thread joins"); + assert_eq!( + inits.load(Ordering::SeqCst), + 2, + "RestartNode must clear `initialized` so reinit_pending_nodes re-runs init(); \ + Pause+Resume left it at 1 while the CLI reported success" + ); +} + +/// A restart naming an unregistered node must not disturb a registered one. +#[test] +fn restart_for_an_unknown_node_leaves_the_others_alone() { + let inits = Arc::new(AtomicU32::new(0)); + let sched_name = format!("restart_unknown_{}", std::process::id()); + let ctl_name = format!("horus.ctl.{}", sched_name); + + let ctl: horus_core::communication::Topic = + horus_core::communication::Topic::new_with_kind( + &ctl_name, + horus_core::communication::TopicKind::System as u8, + ) + .expect("control topic opens"); + + let inits_for_thread = inits.clone(); + let name_for_thread = sched_name.clone(); + let handle = std::thread::spawn(move || { + let mut scheduler = Scheduler::new() + .name(&name_for_thread) + .tick_rate(200_u64.hz()); + scheduler + .add(CountsInit { + inits: inits_for_thread, + }) + .build() + .expect("node registers"); + scheduler.run_for(1200_u64.ms()).expect("run_for"); + }); + + std::thread::sleep(std::time::Duration::from_millis(400)); + assert_eq!(inits.load(Ordering::SeqCst), 1); + + ctl.send(ControlCommand::RestartNode { + name: "no_such_node".to_string(), + }); + + handle.join().expect("scheduler thread joins"); + assert_eq!( + inits.load(Ordering::SeqCst), + 1, + "a restart naming an unregistered node must not re-init a different one" + ); +} diff --git a/horus_manager/src/commands/node.rs b/horus_manager/src/commands/node.rs index 64a50ad09..b02890722 100644 --- a/horus_manager/src/commands/node.rs +++ b/horus_manager/src/commands/node.rs @@ -580,7 +580,7 @@ pub fn kill_node(name: &str, force: bool) -> HorusResult<()> { Ok(()) } -/// Restart a node (pause + resume via control topic to trigger re-init) +/// Restart a node — asks the scheduler to re-run its `init()` on the next tick. pub fn restart_node(name: &str) -> HorusResult<()> { let nodes = discover_nodes()?; let node = find_node(&nodes, name)?; @@ -591,16 +591,13 @@ pub fn restart_node(name: &str) -> HorusResult<()> { node.name.white().bold() ); - // Pause then resume triggers re-init in the scheduler + // Pause+Resume was sent here and claimed to trigger re-init. It did not: + // both arms only toggle a shared atomic, and nothing cleared the flag that + // `reinit_pending_nodes` keys off, so the node kept running untouched while + // this printed success. send_control_command( node, - ControlCommand::PauseNode { - name: node.name.clone(), - }, - )?; - send_control_command( - node, - ControlCommand::ResumeNode { + ControlCommand::RestartNode { name: node.name.clone(), }, )?; From 3028b021a3a1fcc3d45f336f42c0dd6999aab6ad Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 15:56:03 -0700 Subject: [PATCH 03/79] fix(safety): count consecutive deadline misses, not budget overruns `consecutive_misses` is the number the `max_deadline_misses` ceiling and the whole degradation ladder compare against, and nothing reset it from the deadline check. The reset lived in `NodeTimingState::record_tick`, reachable only through `check_budget`, which gave three wrong answers: - RT executor: `tick_node` calls `check_tick_budget` unconditionally and the deadline check afterwards, so a node woken 5 ms late that executes in microseconds cleared the run its own miss had just started. The counter was pinned at 1 forever. Neither `warn_after: 3` nor the ceiling could fire for the failure mode `check_deadline_from_release` was written to catch -- the one `primitives.rs` calls dominant. - Main loop: the call sits inside the budget-violation branch, so a clean tick never reaches it and the count is a lifetime total. - A node with `.deadline()` and no `.tick_budget()` skips the budget block entirely, so nothing cleared it on either path. `node_builder` derives a deadline from a budget and never the reverse, so this is an ordinary shape, and for it the per-node consecutive ceiling was a lifetime total wearing a safety threshold's name -- exactly what the per-node ceiling was introduced to end. A budget overrun and a deadline miss are different failures, so a tick that stayed inside its budget is no evidence its deadline was met. The reset moves to `SafetyMonitor::record_deadline_met`, called wherever the deadline check comes back clean on both dispatch paths. Also: a miss now clears `recovery_counter`. `record_successful_tick` means "did not panic", and both callers reach it on a tick already recorded as a miss in the same pass, so a node at RateReduced that missed EVERY deadline still accumulated `recovery_ticks` of "success" and was restored to the rate it had just proved it could not hold. The field's own comment already called itself a consecutive run. Two existing tests asserted the old contract by calling `check_budget` with an in-budget duration to mean "recovered"; they now say `record_deadline_met`, which is the thing they were reaching for. Each fix was reverted in turn to confirm the tests catch it: the in-budget run reads 1 instead of 10, the no-budget scheduler node reads 10 instead of 0, the RT node stays at 5 after meeting its deadline, and the never-recovering node is declared recovered at tick 8. --- horus_core/src/scheduling/rt_executor.rs | 56 ++++++ horus_core/src/scheduling/safety_monitor.rs | 195 +++++++++++++++++-- horus_core/src/scheduling/scheduler/mod.rs | 9 + horus_core/src/scheduling/scheduler/tests.rs | 81 ++++++++ 4 files changed, 326 insertions(+), 15 deletions(-) diff --git a/horus_core/src/scheduling/rt_executor.rs b/horus_core/src/scheduling/rt_executor.rs index 24cee717a..8f95a4c6e 100644 --- a/horus_core/src/scheduling/rt_executor.rs +++ b/horus_core/src/scheduling/rt_executor.rs @@ -1359,6 +1359,15 @@ impl RtExecutor { deadline, node.miss_policy, ); + if miss.is_none() { + // The tick met its deadline: end the consecutive-miss run. + // `check_tick_budget` above must not do this — it is measuring + // a different thing, and a node released late misses its + // deadline while sitting comfortably inside its budget. + if let Some(ref monitor) = monitors.safety { + monitor.record_deadline_met(&node.name); + } + } if miss.is_none() && node.in_safe_mode { // Met the deadline again: clear the latch so a node that // recovers can be safed once more if it degrades later. @@ -2221,6 +2230,53 @@ mod tests { ); } + /// The RT executor's half of the consecutive-miss contract, on the failure + /// mode `check_deadline_from_release` exists to catch: a node woken late + /// that then executes in microseconds. Its deadline is blown from release; + /// its budget is untouched. + /// + /// `tick_node` calls `check_tick_budget` unconditionally, before the + /// deadline check. While the budget check owned the reset, that in-budget + /// tick cleared the run the deadline miss had just started, so the counter + /// was pinned at 1 no matter how long the node had been failing — neither + /// `warn_after: 3` nor the `max_deadline_misses` ceiling could ever fire + /// for it. A met deadline, and only that, ends the run. + #[test] + fn late_release_misses_accumulate_and_a_met_deadline_clears_them() { + use std::sync::atomic::AtomicU64; + + let count = Arc::new(AtomicU64::new(0)); + let mut node = make_rt_registered("arm_controller", count.clone()); + node.deadline = Some(Duration::from_millis(1)); + // A budget it comfortably meets — the tick itself is a counter bump. + node.tick_budget = Some(Duration::from_millis(1)); + + let monitor = Arc::new(SafetyMonitor::new(1000)); + let mut monitors = test_monitors(); + monitors.safety = Some(monitor.clone()); + let running = Arc::new(AtomicBool::new(true)); + + // Released 5 ms late, five times over. Every tick is inside its budget + // and every tick has missed its 1 ms deadline before it even began. + for _ in 0..5 { + RtExecutor::tick_node(&mut node, &monitors, &running, false, 5_000_000); + } + assert_eq!( + monitor.consecutive_misses("arm_controller"), + 5, + "five late releases in a row are five consecutive deadline misses; \ + the node staying inside its BUDGET says nothing about its DEADLINE" + ); + + // Released on time: the deadline is met, and the run ends. + RtExecutor::tick_node(&mut node, &monitors, &running, false, 0); + assert_eq!( + monitor.consecutive_misses("arm_controller"), + 0, + "a tick that met its deadline must clear the run" + ); + } + struct CounterNode { name: String, count: Arc, diff --git a/horus_core/src/scheduling/safety_monitor.rs b/horus_core/src/scheduling/safety_monitor.rs index b0066d7c4..5120e4c10 100644 --- a/horus_core/src/scheduling/safety_monitor.rs +++ b/horus_core/src/scheduling/safety_monitor.rs @@ -535,16 +535,12 @@ impl NodeTimingState { } } - // Reset the consecutive-miss run only on a tick that did NOT violate. - // - // This used to run unconditionally at the TOP of the function, before - // the budget check. For a node that overruns its budget and misses its - // deadline on the same tick — the common case, since an overrunning - // tick is exactly what blows the deadline — the sequence was: clear to - // 0 here, then `record_miss` increments to 1. Pinned at 1 forever, so - // `evaluate_degradation`, whose ladder starts at 3 consecutive misses, - // could never fire for the nodes in the worst trouble. - self.consecutive_misses = 0; + // The consecutive-miss run is NOT cleared here. It counts deadline + // misses, and this function knows only about the budget; clearing it + // from the budget check pinned the counter at 1 for a node that made + // its budget and missed its deadline every single tick, which is the + // dominant failure mode (see `check_deadline_from_release`). + // `SafetyMonitor::record_deadline_met` owns the reset. None } @@ -734,6 +730,17 @@ impl BudgetEnforcer { self.node_timing.insert(node.to_string(), state); } + /// Record that a node MET its deadline, clearing its consecutive-miss run. + /// + /// No allocation and no insert on the miss of the map: a node with no entry + /// has no run to clear, and this runs on the RT thread on every healthy + /// tick — by far the hottest of these paths. + pub(crate) fn record_deadline_met(&mut self, node: &str) { + if let Some(state) = self.node_timing.get_mut(node) { + state.consecutive_misses = 0; + } + } + /// Get timing stats for a specific node. #[cfg(test)] pub(crate) fn node_stats(&self, node: &str) -> Option { @@ -1121,6 +1128,26 @@ impl SafetyMonitor { result } + /// Record that a node met its deadline, clearing its consecutive-miss run. + /// + /// The counterpart to `record_deadline_miss`, and the ONLY thing that + /// clears the run. Call it on every tick whose deadline check came back + /// clean, from both dispatch paths. + /// + /// This used to be a side effect of `check_budget`, which is the wrong + /// contract twice over. A budget overrun and a deadline miss are different + /// failures — a node woken late can blow its deadline having executed in a + /// tenth of its budget — so an in-budget tick is no evidence the deadline + /// was met. And neither production caller ran it on a healthy tick anyway: + /// the main loop calls `check_tick_budget` only from inside the + /// budget-violation branch, and a node with a deadline and no budget skips + /// that block entirely, so on those two paths nothing ever cleared the + /// count and `max_deadline_misses` was a lifetime total wearing the name of + /// a consecutive one. + pub(crate) fn record_deadline_met(&self, node_name: &str) { + self.budget_enforcer.lock().record_deadline_met(node_name); + } + /// Record a deadline miss with severity tracking. /// /// Tracks per-node: total misses, consecutive misses, worst miss duration. @@ -1158,7 +1185,8 @@ impl SafetyMonitor { // is not keeping up right now. A run of misses is a node in trouble; a // scattered few that each recover are jitter, and jitter on a stock // kernel is not a reason to stop a robot. `consecutive_misses` is reset - // by `check_budget` on any tick that did not violate, so recovery + // by `record_deadline_met` on any tick that MET its deadline, on both + // dispatch paths and whether or not the node has a budget, so recovery // genuinely clears the count. let consecutive = { let mut enforcer = self.budget_enforcer.lock(); @@ -1214,6 +1242,15 @@ impl SafetyMonitor { .get_mut(node_name) .expect("inserted immediately above"); + // Every caller reaches this only after recording a deadline miss, so + // the run of recovery ticks is over. `record_successful_tick` means + // "the tick did not panic" and both callers reach it on a tick already + // recorded as a miss in the same pass; without this, a node that missed + // EVERY deadline still accumulated `recovery_ticks` of "success" and + // was restored to the rate it had just proved it could not hold. The + // field's own comment already claimed to be a consecutive run. + state.recovery_counter = 0; + if consecutive_misses >= policy.kill_after && state.stage != DegradationStage::Killed { state.stage = DegradationStage::Killed; state.recovery_counter = 0; @@ -2096,9 +2133,15 @@ mod tests { assert_eq!(state.consecutive_misses, 3); assert!(state.is_chronic()); - // Successful tick resets consecutive counter + // A tick inside budget is NOT a reset: this state knows only about the + // budget, and a node released late misses its deadline having executed + // well inside it. Clearing here pinned the run at 1 for exactly that + // node. Only a met DEADLINE ends the run, via `record_deadline_met`. state.record_tick(10_u64.us()); - assert_eq!(state.consecutive_misses, 0); + assert_eq!(state.consecutive_misses, 3); + assert!(state.is_chronic()); + + state.consecutive_misses = 0; // what `record_deadline_met` does assert!(!state.is_chronic()); // But total misses persist assert_eq!(state.total_deadline_misses, 3); @@ -2658,8 +2701,12 @@ mod tests { for _ in 0..(ceiling * 10) { monitor.record_deadline_miss("wheel_controller"); - // A tick inside budget: `check_budget` clears the consecutive run. - let _ = monitor.check_tick_budget("wheel_controller", Duration::from_micros(100)); + // The next tick MET its deadline, which is what ends the run. This + // used to be spelled `check_tick_budget(.., in_budget_duration)`, + // asserting a contract the runtime does not hold: an in-budget tick + // is not a met deadline, and neither production caller reached that + // call on a healthy tick anyway. + monitor.record_deadline_met("wheel_controller"); } assert!( @@ -2706,6 +2753,124 @@ mod tests { assert!(monitor.is_emergency_stop()); } + /// The consecutive-miss run must count DEADLINE misses, not budget overruns. + /// + /// This is the RT executor's exact call order for the failure mode + /// `check_deadline_from_release` was written to catch: a node woken late + /// that then executes well inside its budget. `rt_executor.rs` calls + /// `check_tick_budget` unconditionally (:1273) and the deadline check + /// afterwards (:1356), so every one of these ticks is in budget and every + /// one misses its deadline. + /// + /// The reset used to live in `check_budget`, so the in-budget tick cleared + /// the run the deadline miss had just started: the counter was pinned at 1 + /// forever. Neither `warn_after: 3` nor the `max_deadline_misses` ceiling + /// could ever fire for the failure mode that, by `primitives.rs`'s own + /// account, "actually dominates". + #[test] + fn an_in_budget_tick_does_not_clear_a_deadline_miss_run() { + let monitor = SafetyMonitor::new(1000); + monitor.set_tick_budget("arm_controller".to_string(), Duration::from_millis(1)); + + for _ in 0..10 { + // Well inside the 1 ms budget — the budget check sees nothing wrong. + let _ = monitor.check_tick_budget("arm_controller", Duration::from_micros(100)); + // ...and yet the deadline, measured from the scheduled release, was missed. + monitor.record_deadline_miss("arm_controller"); + } + + assert_eq!( + monitor.consecutive_misses("arm_controller"), + 10, + "ten consecutive deadline misses must read as ten; a tick that stayed \ + inside its BUDGET says nothing about whether it met its DEADLINE" + ); + } + + /// The other half of the contract: a tick that meets its deadline clears + /// the run, for a node that has a deadline and no budget at all. + /// + /// `node_builder.rs` derives a deadline from a budget and never the + /// reverse, so this node's `tick_budget` is `None` and the main loop's + /// whole `if let Some(tick_budget)` block — the only place that used to + /// clear the counter — is skipped. Before `record_deadline_met` existed + /// there was no path that cleared it, and the count was a lifetime total: + /// a node missing once an hour reached any ceiling eventually. + #[test] + fn a_tick_that_meets_its_deadline_clears_the_run_without_any_budget() { + let monitor = SafetyMonitor::new(1000); + // Deliberately no `set_tick_budget`. + + for _ in 0..10 { + monitor.record_deadline_miss("wheel_controller"); + monitor.record_deadline_met("wheel_controller"); + } + + assert_eq!( + monitor.consecutive_misses("wheel_controller"), + 0, + "a node that misses and immediately recovers, ten times over, is \ + jitter — its consecutive run is zero, not ten" + ); + } + + /// A tick that missed its deadline is not evidence of recovery. + /// + /// `record_successful_tick` means "did not panic", and both callers reach + /// it on a tick already recorded as a deadline miss in the same pass. With + /// nothing clearing `recovery_counter` on a miss, a node at `RateReduced` + /// that missed EVERY deadline still accumulated `recovery_ticks` of + /// "success" and was restored to the rate it had just demonstrated it + /// could not hold. + #[test] + fn a_node_that_never_meets_a_deadline_is_never_declared_recovered() { + let mut monitor = SafetyMonitor::new(10_000); + // isolate_after is out of reach so the node stays at RateReduced and + // the test observes recovery, not a stage transition. + monitor.set_degradation_policy(DegradationPolicy { + warn_after: 3, + reduce_after: 5, + isolate_after: 10_000, + kill_after: 20_000, + recovery_ticks: 10, + }); + + let miss_tick = |m: &SafetyMonitor| { + m.record_deadline_miss("arm_controller"); + let consecutive = m.consecutive_misses("arm_controller"); + m.evaluate_degradation("arm_controller", consecutive, Some(100.0)); + // The RT executor reaches this on any tick whose closure returned + // `Ok` — including the one just recorded as a miss. + m.record_successful_tick("arm_controller") + }; + + for _ in 0..5 { + miss_tick(&monitor); + } + assert_eq!( + monitor.degradation_stage("arm_controller"), + DegradationStage::RateReduced, + "five consecutive misses must reduce the rate" + ); + + // Keep missing, for twice the recovery window. + for i in 0..(10 * 2) { + let action = miss_tick(&monitor); + assert!( + !matches!(action, DegradationAction::RestoreRate { .. }), + "tick {} missed its deadline like every tick before it — \ + restoring its full rate says it recovered when it never did", + i + ); + } + + assert_eq!( + monitor.degradation_stage("arm_controller"), + DegradationStage::RateReduced, + "a node that has not met a deadline yet must still be degraded" + ); + } + // ======================================================================== // Stress and multi-node tests // ======================================================================== diff --git a/horus_core/src/scheduling/scheduler/mod.rs b/horus_core/src/scheduling/scheduler/mod.rs index f1e5ccd6b..e532b8a0b 100644 --- a/horus_core/src/scheduling/scheduler/mod.rs +++ b/horus_core/src/scheduling/scheduler/mod.rs @@ -5414,6 +5414,15 @@ impl Scheduler { if let Some(deadline) = self.nodes[i].deadline { let miss = TimingEnforcer::check_deadline(tick_start, deadline, self.nodes[i].miss_policy); + if miss.is_none() { + // The tick met its deadline: end the consecutive-miss run. + // This is the only path that clears it, and it has to run + // for every RT node with a deadline — including one with no + // tick budget, which never reaches the budget block above. + if let Some(ref monitor) = self.monitor.safety { + monitor.record_deadline_met(&node_name); + } + } if miss.is_none() && self.nodes[i].in_safe_mode { // See the RT executor: a latch that never clears turns // Miss::SafeMode into a one-shot for the process lifetime. diff --git a/horus_core/src/scheduling/scheduler/tests.rs b/horus_core/src/scheduling/scheduler/tests.rs index 1186be27b..bf1657604 100644 --- a/horus_core/src/scheduling/scheduler/tests.rs +++ b/horus_core/src/scheduling/scheduler/tests.rs @@ -5709,6 +5709,87 @@ fn shared_monitors_carry_a_safety_handle_for_executors() { ); } +/// The main loop must clear a node's consecutive-miss run on a tick that MET +/// its deadline — and this node has a deadline and no tick budget, so it never +/// enters the budget block that used to nominally own the reset. +/// +/// `node_builder` derives a deadline from a budget and never the reverse, so +/// this shape is ordinary: `.deadline(..)` with no `.tick_budget(..)`. For it, +/// `check_timing_violations` skipped the whole `if let Some(tick_budget)` +/// block, nothing ever called `check_budget`, and `consecutive_misses` was a +/// lifetime total. A node missing once an hour and recovering instantly +/// reached any ceiling eventually — the exact "timer wearing a safety +/// threshold's name" the per-node ceiling was introduced to end. +struct AlternatingNode { + name: String, + ticks: Arc, +} + +impl Node for AlternatingNode { + fn name(&self) -> &str { + &self.name + } + + fn tick(&mut self) { + // Miss the deadline on every other tick, comfortably meeting it in + // between. Never two misses in a row. + if self.ticks.fetch_add(1, Ordering::SeqCst).is_multiple_of(2) { + std::thread::sleep(Duration::from_millis(5)); + } + } +} + +#[test] +fn a_met_deadline_clears_the_miss_run_for_a_node_with_no_budget() { + let _guard = lock_scheduler(); + let ticks = Arc::new(AtomicUsize::new(0)); + + let mut scheduler = Scheduler::new() + .tick_rate(100_u64.hz()) + // Keeps the RT node on the main thread, so this exercises + // `check_timing_violations` rather than the RT executor. + .deterministic(true); + scheduler + .add(AlternatingNode { + name: "flaky".to_string(), + ticks: ticks.clone(), + }) + .deadline(1_u64.ms()) // deliberately no .tick_budget() + .build(); + + let _ = scheduler.run_for(400_u64.ms()); + + let monitor = scheduler + .monitor + .safety + .as_ref() + .expect("a node with a deadline produces a safety monitor"); + + // Anti-vacuity: the node must actually have missed, repeatedly, or the + // assertion below passes for the wrong reason. + let total_misses = monitor + .all_node_timing() + .into_iter() + .find(|r| r.name == "flaky") + .map(|r| r.deadline_misses) + .unwrap_or(0); + assert!( + total_misses >= 3, + "precondition: the node must miss its deadline repeatedly for this \ + test to mean anything (saw {} misses in {} ticks)", + total_misses, + ticks.load(Ordering::SeqCst) + ); + + assert!( + monitor.consecutive_misses("flaky") <= 1, + "the node met its deadline between every miss, so its CONSECUTIVE run \ + is at most 1 — it read {} because nothing on this path ever cleared \ + it, making a per-node consecutive ceiling a lifetime total", + monitor.consecutive_misses("flaky") + ); +} + #[cfg(test)] mod watchdog_message_honesty { use super::super::watchdog_critical_message; From 88688dc6970827313b181e3bcff087c1afaf9067 Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 16:03:57 -0700 Subject: [PATCH 04/79] fix(safety): make the max_deadline_misses emergency stop reachable `max_deadline_misses` is documented as the per-node consecutive ceiling that stops the robot, defaults to 100, and could not fire at that value -- or at any value above 20 -- in any shipped build. `DegradationAction::Kill` sets `is_stopped`, and a stopped node is skipped forever on both dispatch paths. So the ladder's terminal rung, hardcoded at 20 consecutive misses, permanently retired the node before its count could climb any higher: `consecutive_misses` froze at 20 and the ceiling was unreachable. Nothing warned. `max_deadline_misses(n)` only clamps with `n.max(1)`, and the sole setter for the ladder is `#[cfg(test)]`, so no user build could raise the rung either. Two of the runtime's own defaults contradicted each other, and the unit tests only ever exercised the ceiling at 3 and 5 -- both below 20 -- which is why the interaction went unnoticed. A hardcoded constant must not silently cap a documented safety threshold, so the rung moves: `kill_after` is raised to the ceiling when the ceiling is higher. The graduated stages below it (warn at 3, half rate at 5, isolate at 10) are untouched, and an isolated node keeps ticking -- every tick on the RT executor, on a probe interval on the main loop -- so it can still reach the ceiling. Where the two now coincide the e-stop wins, because both paths record the miss, which evaluates the ceiling, before evaluating the ladder. Note this changes what a chronically missing node does under the default configuration: it is no longer killed at 20 while the robot runs on, it e-stops the robot at 100. That is what the knob, its default, the builder doc and five documentation pages all say it does. Both tests assert the ORDER of the two terminal events rather than the counter -- at this level nothing removes the node, so the count climbs regardless and the e-stop would fire on its own; what killed the ceiling is the `Kill` that lands first. Reverting to the hardcoded rung fails them at 20 for both the configured ceiling of 25 and the default of 100. --- horus_core/src/scheduling/safety_monitor.rs | 107 +++++++++++++++++++- horus_core/src/scheduling/scheduler/mod.rs | 12 ++- 2 files changed, 116 insertions(+), 3 deletions(-) diff --git a/horus_core/src/scheduling/safety_monitor.rs b/horus_core/src/scheduling/safety_monitor.rs index 5120e4c10..e0d2a0a9d 100644 --- a/horus_core/src/scheduling/safety_monitor.rs +++ b/horus_core/src/scheduling/safety_monitor.rs @@ -571,7 +571,9 @@ pub(crate) struct DegradationPolicy { pub(crate) reduce_after: u64, /// Consecutive misses before isolating node (default: 10) pub(crate) isolate_after: u64, - /// Consecutive misses before killing node — permanently removing from execution (default: 20) + /// Consecutive misses before killing node — permanently removing from + /// execution (default: 20, raised to `max_deadline_misses` when that + /// ceiling is higher, so the ceiling stays reachable — see `reaching`) pub(crate) kill_after: u64, /// Successful ticks at reduced rate before restoring original (default: 100) pub(crate) recovery_ticks: u64, @@ -589,6 +591,32 @@ impl Default for DegradationPolicy { } } +impl DegradationPolicy { + /// The default ladder, with its terminal rung moved out of the way of a + /// configured `max_deadline_misses` ceiling. + /// + /// `Kill` is permanent — it sets `is_stopped`, and a stopped node is + /// skipped forever — so a node killed at 20 consecutive misses can never + /// tick again, and its consecutive count can never climb any higher. With + /// `kill_after` fixed at 20 and `max_deadline_misses` defaulting to 100, + /// the documented emergency stop was unreachable in every shipped + /// configuration: the ladder always retired the node first. Nothing warned, + /// because the only setter for the ladder is `#[cfg(test)]`, so no user + /// build could raise the rung either. + /// + /// A hardcoded constant must not silently cap a documented safety + /// threshold, so the rung moves instead. Where the two now coincide the + /// e-stop wins, because both dispatch paths record the miss — which + /// evaluates the ceiling — before evaluating the ladder. + fn reaching(max_deadline_misses: u64) -> Self { + let base = Self::default(); + Self { + kill_after: base.kill_after.max(max_deadline_misses), + ..base + } + } +} + /// Current degradation stage for a node. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum DegradationStage { @@ -876,7 +904,7 @@ impl SafetyMonitor { deadline_misses: AtomicU64::new(0), max_deadline_misses, degrade_activations: AtomicU64::new(0), - degradation_policy: DegradationPolicy::default(), + degradation_policy: DegradationPolicy::reaching(max_deadline_misses), degradation_states: Mutex::new(HashMap::new()), external_safe_state_pending: AtomicBool::new(false), } @@ -2690,6 +2718,81 @@ mod tests { /// there eventually whatever its health. That is a timer wearing a safety /// threshold's name. /// + /// The configured `max_deadline_misses` ceiling must be reachable. + /// + /// `DegradationAction::Kill` sets `is_stopped`, and a stopped node is + /// skipped forever on both dispatch paths — so a node the ladder kills at + /// 20 consecutive misses can never tick again, and its consecutive count + /// can never climb any higher. With `kill_after` hardcoded at 20 and the + /// ceiling defaulting to 100, the emergency stop this knob names could not + /// fire in ANY shipped configuration, and the ladder's only setter is + /// `#[cfg(test)]`, so no user build could raise the rung either. + /// + /// The assertion is on the ORDER of the two terminal events, not on the + /// counter: at this level nothing removes the node, so the count keeps + /// climbing and the e-stop would fire on its own. What made the ceiling + /// dead is the `Kill` that lands first. + #[test] + fn the_ladder_does_not_retire_a_node_before_its_configured_ceiling() { + let _estop_serial = super::estop_queue_guard(); + // Above the ladder's default terminal rung of 20 — as the shipped + // default of 100 also is. + let ceiling = 25u64; + let monitor = SafetyMonitor::new(ceiling); + + for i in 1..=ceiling { + monitor.record_deadline_miss("arm_controller"); + let consecutive = monitor.consecutive_misses("arm_controller"); + let action = monitor.evaluate_degradation("arm_controller", consecutive, Some(100.0)); + + if !monitor.is_emergency_stop() { + assert!( + !matches!(action, DegradationAction::Kill(_)), + "the ladder killed the node at {} consecutive misses. It can \ + never tick again, so it can never reach the configured \ + ceiling of {} and the emergency stop that number names is \ + unreachable", + i, + ceiling + ); + } + } + + assert!( + monitor.is_emergency_stop(), + "{} consecutive misses is the configured ceiling — the emergency \ + stop must fire", + ceiling + ); + } + + /// The same thing at the value users actually ship: the documented default. + #[test] + fn the_default_ceiling_is_reachable() { + let _estop_serial = super::estop_queue_guard(); + let default_ceiling = super::super::config::SchedulerConfig::default() + .realtime + .max_deadline_misses; + let monitor = SafetyMonitor::new(default_ceiling); + + for i in 1..=default_ceiling { + monitor.record_deadline_miss("arm_controller"); + let consecutive = monitor.consecutive_misses("arm_controller"); + let action = monitor.evaluate_degradation("arm_controller", consecutive, Some(100.0)); + if !monitor.is_emergency_stop() { + assert!( + !matches!(action, DegradationAction::Kill(_)), + "the default ladder retired the node at {} of the default \ + ceiling's {} consecutive misses", + i, + default_ceiling + ); + } + } + + assert!(monitor.is_emergency_stop()); + } + /// Here one node misses ten times the ceiling, recovering between each. It /// must not stop the robot. #[test] diff --git a/horus_core/src/scheduling/scheduler/mod.rs b/horus_core/src/scheduling/scheduler/mod.rs index e532b8a0b..5280381fd 100644 --- a/horus_core/src/scheduling/scheduler/mod.rs +++ b/horus_core/src/scheduling/scheduler/mod.rs @@ -825,7 +825,17 @@ impl Scheduler { /// Set the maximum number of deadline misses before emergency stop. /// - /// Default: 100. Only meaningful when nodes have `.rate()` set. + /// Default: 100. Per node and consecutive: a tick that meets its deadline + /// clears the run, so a node that misses intermittently and recovers never + /// reaches the ceiling. + /// + /// The graduated degradation ladder (warn at 3 consecutive misses, half + /// rate at 5, isolate at 10) runs below this. Its terminal rung — killing + /// the node outright, normally at 20 — is moved above `n` when `n` is + /// higher, so this ceiling is always reachable; killing the node is + /// permanent, and a killed node cannot go on to miss anything. + /// + /// Only meaningful when nodes have `.rate()` set. /// /// # Example /// ```rust,ignore From 51ade0312dd676e784f37d2f3d32eeb9804c8dd9 Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 16:57:37 -0700 Subject: [PATCH 05/79] fix(run): apply [network] on every path that spawns a run child `safety.on_link_lost`, `import` and `deny_export` are safety controls. The 2026-07-30 audit found the whole `[network]` table parsed and then read by nothing, and `apply_network_config` was written to fix that -- then wired into ONE of the five paths that spawn a run child. That one is `execute_multiple_files`, reached only when `horus run` is handed more than one file. A bare `horus run`, a `horus.toml` run, a directory run and a `[workspace]` run all resolve to a single executable. On those the child environment came from `build_child_env()`, which sets PATH, LD_LIBRARY_PATH and PYTHONPATH and nothing else, so `NetConfig::default()` read an environment nobody had populated: `import` stayed `Auto`, `deny_export` stayed empty, and the documented comms-loss safe state never armed. The gap the two existing comments describe was still open on every invocation a real single-binary robot makes. `apply_network_config` becomes `network_env() -> Vec<(String, String)>`, which `build_child_env` now extends with -- so every single-executable path picks it up from the function they all already call, and no future path can quietly miss it. `execute_workspace` builds its `Command` by hand rather than through `build_child_env`, so it gets an explicit `apply_network_config` call. The operator still wins: `network_env` skips any key already present in the environment, so an explicit `HORUS_NET_*=...` beats the manifest on every path. That is asserted too. Reverting the `build_child_env` line gives `left: None` for HORUS_NET_ON_LINK_LOST -- the safety control simply absent. `horus launch` still performs no translation; that is a separate gap and the documentation says so. --- horus_manager/src/commands/run/mod.rs | 150 ++++++++++++++++++++++++-- 1 file changed, 144 insertions(+), 6 deletions(-) diff --git a/horus_manager/src/commands/run/mod.rs b/horus_manager/src/commands/run/mod.rs index 74d8b9357..7ed2f64ec 100644 --- a/horus_manager/src/commands/run/mod.rs +++ b/horus_manager/src/commands/run/mod.rs @@ -73,24 +73,34 @@ enum ExecutionTarget { /// `AllowList` were unreachable in shipped code (the import guard was /// permanently `Auto`). The 2026-07-30 audit flagged that as a live safety gap. /// +/// That remedy was then wired into ONE of the five paths that spawn a run +/// child — `execute_multiple_files`, reached only when `horus run` is handed +/// more than one file. A bare `horus run`, a `horus.toml` run, a directory run +/// and a `[workspace]` run all resolve to a single executable, so on every +/// invocation a real single-binary robot makes, the safety gap those comments +/// describe was still open. Hence pairs rather than a `Command`: the same +/// translation now feeds `build_child_env`, which every single-executable path +/// already uses, so there is one answer and no path can quietly miss it. +/// /// An environment variable already set by the operator wins: an explicit /// `HORUS_NET_*=...` on the command line should not be silently overridden by /// a value in a project file. -fn apply_network_config(cmd: &mut std::process::Command) { +fn network_env() -> Vec<(String, String)> { + let mut out: Vec<(String, String)> = Vec::new(); let manifest_path = Path::new(HORUS_TOML); if !manifest_path.exists() { - return; + return out; } let Ok(manifest) = HorusManifest::load_from(manifest_path) else { - return; + return out; }; let Some(net) = manifest.network.as_ref() else { - return; + return out; }; let mut set = |key: &str, value: String| { if std::env::var_os(key).is_none() { - cmd.env(key, value); + out.push((key.to_string(), value)); } }; @@ -139,6 +149,17 @@ fn apply_network_config(cmd: &mut std::process::Command) { set("HORUS_NET_ON_LINK_LOST", action.clone()); } } + out +} + +/// Apply the `[network]` translation to a child `Command`. +/// +/// For the paths that build a `Command` directly rather than going through +/// `build_child_env`. +fn apply_network_config(cmd: &mut std::process::Command) { + for (key, value) in network_env() { + cmd.env(key, value); + } } pub fn execute_run( @@ -303,7 +324,13 @@ fn execute_workspace( let binary = crate::build_dirs::binary_path(&project_dir, profile, member); if binary.exists() { cli_output::success(&format!("Running {}", member)); - let status = std::process::Command::new(&binary).args(&args).status()?; + let mut cmd = std::process::Command::new(&binary); + cmd.args(&args); + // This path builds its child `Command` by hand instead of going + // through `build_child_env`, so it needs the `[network]` + // translation applied explicitly. + apply_network_config(&mut cmd); + let status = cmd.status()?; if !status.success() { let code = status.code().unwrap_or(1); return Err(anyhow!("Process exited with code {}", code)); @@ -983,6 +1010,15 @@ pub(crate) fn build_child_env() -> Result> { // that does not work. One builder, one answer. env_vars.push(("PYTHONPATH".to_string(), run_python::build_python_path()?)); + // `[network]` from horus.toml. This belongs here, not at one spawn site: + // `safety.on_link_lost`, `import` and `deny_export` are safety controls, + // and they used to reach the runtime only when `horus run` was handed + // several files. Every path that runs a single executable — a bare + // `horus run`, a `horus.toml` run, a directory run — comes through this + // function, so wiring it here is what makes the table apply to the + // invocation a real robot actually uses. + env_vars.extend(network_env()); + Ok(env_vars) } @@ -1235,6 +1271,108 @@ mod tests { assert!(names.contains(&"PYTHONPATH"), "Must set PYTHONPATH"); } + /// `[network]` must reach the child on the path a single-binary robot runs. + /// + /// `safety.on_link_lost`, `import` and `deny_export` are safety controls. + /// The translation that arms them was wired into ONE of the five spawn + /// paths — `execute_multiple_files`, reached only when `horus run` is + /// handed more than one file. A bare `horus run`, a `horus.toml` run and a + /// directory run all resolve to a single executable and went through + /// `build_child_env`, which set `PATH`, `LD_LIBRARY_PATH` and `PYTHONPATH` + /// and nothing else. On those paths `NetConfig::default()` read an + /// environment nobody had populated: `import` stayed `Auto`, `deny_export` + /// stayed empty, and the documented comms-loss safe state never armed. + #[test] + fn build_child_env_carries_the_network_table_to_the_child() { + let tmp = tempfile::TempDir::new().unwrap(); + fs::create_dir_all(tmp.path().join(".horus/bin")).unwrap(); + fs::create_dir_all(tmp.path().join(".horus/lib")).unwrap(); + fs::write( + tmp.path().join(HORUS_TOML), + r#" +[project] +name = "link_loss_robot" +version = "0.1.0" + +[network] +import = "deny" +deny_export = ["/motor/cmd", "/estop"] + +[network.safety] +on_link_lost = "safe_state" +missed_threshold = 3 +"#, + ) + .unwrap(); + + let _guard = crate::CWD_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let original = std::env::current_dir().unwrap_or_else(|_| std::env::temp_dir()); + std::env::set_current_dir(tmp.path()).unwrap(); + // The operator's environment wins over the manifest, so a stray + // HORUS_NET_* here would mask the thing under test. + let preset: Vec<&str> = [ + "HORUS_NET_IMPORT", + "HORUS_NET_DENY_EXPORT", + "HORUS_NET_ON_LINK_LOST", + ] + .into_iter() + .filter(|k| std::env::var_os(k).is_some()) + .collect(); + let result = build_child_env(); + std::env::set_current_dir(original).unwrap(); + drop(_guard); + + assert!( + preset.is_empty(), + "precondition: {:?} set in this environment would override the manifest", + preset + ); + + let env_vars = result.unwrap(); + let get = |key: &str| { + env_vars + .iter() + .find(|(k, _)| k == key) + .map(|(_, v)| v.as_str()) + }; + + assert_eq!( + get("HORUS_NET_ON_LINK_LOST"), + Some("safe_state"), + "the comms-loss safe state must arm on the path a single-binary \ + robot actually runs, not only on a multi-file `horus run`" + ); + assert_eq!(get("HORUS_NET_IMPORT"), Some("deny")); + assert_eq!(get("HORUS_NET_DENY_EXPORT"), Some("/motor/cmd,/estop")); + assert_eq!(get("HORUS_NET_MISSED_THRESHOLD"), Some("3")); + } + + /// An explicit `HORUS_NET_*` in the operator's environment still wins. + #[test] + fn an_explicit_network_variable_beats_the_manifest() { + let tmp = tempfile::TempDir::new().unwrap(); + fs::write( + tmp.path().join(HORUS_TOML), + "[project]\nname = \"p\"\nversion = \"0.1.0\"\n\n[network]\nimport = \"deny\"\n", + ) + .unwrap(); + + let _guard = crate::CWD_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let original = std::env::current_dir().unwrap_or_else(|_| std::env::temp_dir()); + std::env::set_current_dir(tmp.path()).unwrap(); + // SAFETY: the CWD lock serialises the tests that touch process globals. + unsafe { std::env::set_var("HORUS_NET_IMPORT", "auto") }; + let pairs = network_env(); + unsafe { std::env::remove_var("HORUS_NET_IMPORT") }; + std::env::set_current_dir(original).unwrap(); + drop(_guard); + + assert!( + !pairs.iter().any(|(k, _)| k == "HORUS_NET_IMPORT"), + "the manifest must not override an explicit HORUS_NET_IMPORT" + ); + } + #[test] fn build_child_env_path_includes_horus_bin() { let tmp = tempfile::TempDir::new().unwrap(); From 01d59391096236c1c7047955da64ad369b000a4d Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 16:57:51 -0700 Subject: [PATCH 06/79] fix(log): HORUS_LOG_FILE actually starts the file drain `start_log_file_drain` was called by no shipped code path. A tree-wide grep for the symbol returned four hits: the definition, its re-export, and two lines of one integration test that invoked it by hand. So `HORUS_LOG_FILE=1 horus run` wrote no `.horus/logs/horus.log`, and `HORUS_LOG_DIR`, `HORUS_LOG_MAX_SIZE` and `HORUS_LOG_MAX_FILES` were dead with it -- while the test that proves the drain works kept it green. The env guard lives inside the function precisely so a call site can be unconditional, and the function's own doc-comment calls the variable an opt-in, which is only true if something always calls it. `Scheduler::new` already does this kind of env-driven wiring twice, for HORUS_NET_ENABLED and HORUS_RECORD_SESSION, so the drain goes beside them. A process that does not set the variable pays one env lookup and spawns nothing. Once per process, via a `OnceLock`: a program may build several schedulers, and each drain thread tracks its own index into the same global ring, so N of them would write every entry N times into one file. This also keeps the promise for C++ and Python, which the doc-comment's "or calling this function directly" escape hatch could not: the symbol has no FFI export anywhere in the tree. Both bindings construct a core `Scheduler` (horus_cpp/src/scheduler_ffi.rs:26, horus_py/src/scheduler.rs:882), so both now get the drain. The test does not call the drain. It sets the variable, builds a `Scheduler` the way any program does, and asserts a log file appears; it is alone in its binary because it writes process-global environment. Reverting the wiring leaves the file absent and it fails. An operator who sets a documented variable to get a post-incident log and finds an empty disk is the exact failure that variable exists to prevent. --- horus_core/src/core/log_buffer.rs | 20 ++++++ horus_core/src/core/mod.rs | 4 +- horus_core/src/scheduling/scheduler/mod.rs | 16 +++++ horus_core/tests/log_file_drain_wiring.rs | 77 ++++++++++++++++++++++ 4 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 horus_core/tests/log_file_drain_wiring.rs diff --git a/horus_core/src/core/log_buffer.rs b/horus_core/src/core/log_buffer.rs index 67f9b64aa..a866e171b 100644 --- a/horus_core/src/core/log_buffer.rs +++ b/horus_core/src/core/log_buffer.rs @@ -639,6 +639,26 @@ pub fn start_log_file_drain() -> Option> { .ok() } +/// The process-wide log-file drain, started at most once. +static LOG_FILE_DRAIN: std::sync::OnceLock>> = + std::sync::OnceLock::new(); + +/// Start the log-file drain once per process, if `HORUS_LOG_FILE` asks for it. +/// +/// `Scheduler::new` calls this unconditionally: `start_log_file_drain` reads +/// the variable itself and returns `None` when it is unset, so a process that +/// has not opted in pays one env lookup and spawns nothing. +/// +/// Once, because a process may build several `Scheduler`s (tests do, and so +/// does any program that runs one scheduler after another). Each drain thread +/// tracks its own `last_idx` into the same global ring, so N of them would +/// write every entry N times into the same file. +/// +/// Returns whether a drain thread is running. +pub fn start_log_file_drain_once() -> bool { + LOG_FILE_DRAIN.get_or_init(start_log_file_drain).is_some() +} + fn log_drain_loop(log_dir: &str, max_size: u64, max_files: usize) { use std::io::Write; diff --git a/horus_core/src/core/mod.rs b/horus_core/src/core/mod.rs index 954fb76dc..a09b6f408 100644 --- a/horus_core/src/core/mod.rs +++ b/horus_core/src/core/mod.rs @@ -38,8 +38,8 @@ pub mod timer; // Log buffer types — pub for hlog! macro expansion and CLI log viewing (horus log) #[doc(hidden)] pub use log_buffer::{ - start_log_file_drain, LogEntry, LogType, SharedLogBuffer, GLOBAL_ERROR_BUFFER, - GLOBAL_LOG_BUFFER, GLOBAL_REMOTE_LOG_BUFFER, + start_log_file_drain, start_log_file_drain_once, LogEntry, LogType, SharedLogBuffer, + GLOBAL_ERROR_BUFFER, GLOBAL_LOG_BUFFER, GLOBAL_REMOTE_LOG_BUFFER, }; pub use duration_ext::{DurationExt, Frequency}; diff --git a/horus_core/src/scheduling/scheduler/mod.rs b/horus_core/src/scheduling/scheduler/mod.rs index 5280381fd..5380b506b 100644 --- a/horus_core/src/scheduling/scheduler/mod.rs +++ b/horus_core/src/scheduling/scheduler/mod.rs @@ -1,4 +1,5 @@ use crate::core::hlog::{clear_node_context, set_node_context}; +use crate::core::log_buffer::start_log_file_drain_once; use crate::core::tick_context::{clear_tick_context, set_tick_context}; use crate::core::{DurationExt, NodeInfo, NodePresence}; use crate::error::{HorusContext, HorusResult}; @@ -621,6 +622,21 @@ impl Scheduler { s = s.with_recording(); } + // Drain the log ring buffer to disk if HORUS_LOG_FILE is set. + // + // `start_log_file_drain` reads the variable itself and returns `None` + // when it is unset, so this call is unconditional and users who do not + // opt in pay nothing. That guard lives INSIDE the function precisely so + // a call site could be unconditional — but there was no call site. The + // symbol appeared four times in the tree: the definition, its + // re-export, and two lines of one integration test that invoked it by + // hand. So `HORUS_LOG_FILE=1 horus run` wrote no `.horus/logs/horus.log` + // and `HORUS_LOG_DIR`/`_MAX_SIZE`/`_MAX_FILES` were dead with it, while + // the test that proves the drain works kept it green. An operator who + // sets a documented variable to get a post-incident log and finds an + // empty disk is the exact failure that variable exists to prevent. + start_log_file_drain_once(); + s } diff --git a/horus_core/tests/log_file_drain_wiring.rs b/horus_core/tests/log_file_drain_wiring.rs new file mode 100644 index 000000000..43f2a2a1a --- /dev/null +++ b/horus_core/tests/log_file_drain_wiring.rs @@ -0,0 +1,77 @@ +//! `HORUS_LOG_FILE=1` has to actually start something. +//! +//! `start_log_file_drain` was called by no shipped code path. A tree-wide grep +//! for the symbol returned four hits: the definition, its re-export, and two +//! lines of one integration test that invoked it by hand. So the drain worked, +//! its test passed, and `HORUS_LOG_FILE=1 horus run` still wrote no +//! `.horus/logs/horus.log` — with `HORUS_LOG_DIR`, `_MAX_SIZE` and +//! `_MAX_FILES` dead alongside it. An operator who sets a documented variable +//! to get a post-incident log and finds an empty disk is the exact failure the +//! variable exists to prevent. +//! +//! This test therefore does NOT call the drain. It sets the variable, builds a +//! `Scheduler` the way any program does, and asserts a log file appears. It is +//! alone in its binary because it writes process-global environment and the +//! drain starts once per process. + +use horus_core::core::log_buffer::{LogEntry, LogType, GLOBAL_LOG_BUFFER}; +use horus_core::scheduling::Scheduler; +use std::time::Duration; + +#[test] +fn setting_horus_log_file_makes_a_scheduler_write_logs_to_disk() { + let dir = std::env::temp_dir().join(format!("horus-log-drain-wiring-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("temp log dir"); + + // SAFETY: this test is alone in its binary, so nothing else is reading or + // writing the environment concurrently. + unsafe { + std::env::set_var("HORUS_LOG_FILE", "1"); + std::env::set_var("HORUS_LOG_DIR", dir.to_str().unwrap()); + } + + // The whole point: building a scheduler is all a program does. Nothing + // here calls `start_log_file_drain`. + let _scheduler = Scheduler::new(); + + for i in 0..20 { + GLOBAL_LOG_BUFFER.push(LogEntry { + timestamp: format!("2026-09-04T00:00:{:02}.000Z", i), + tick_number: GLOBAL_LOG_BUFFER.write_idx() + 1, + node_name: "wiring_probe".to_string(), + log_type: LogType::Info, + topic: None, + message: format!("entry {}", i), + tick_us: 0, + ipc_ns: 0, + }); + } + + // The drain polls every 500 ms. + let log_file = dir.join("horus.log"); + let mut content = String::new(); + for _ in 0..40 { + std::thread::sleep(Duration::from_millis(250)); + if let Ok(c) = std::fs::read_to_string(&log_file) { + if c.contains("wiring_probe") { + content = c; + break; + } + } + } + + let found = content.contains("wiring_probe"); + let _ = std::fs::remove_dir_all(&dir); + unsafe { + std::env::remove_var("HORUS_LOG_FILE"); + std::env::remove_var("HORUS_LOG_DIR"); + } + + assert!( + found, + "HORUS_LOG_FILE=1 was set and a Scheduler was built, so {} should \ + contain the log entries. Nothing started the drain.", + log_file.display() + ); +} From ff8239f1f4a85887990aced266c8ab2b9bf6114c Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 17:33:33 -0700 Subject: [PATCH 07/79] fix(topic): try_send and send_blocking move the staleness counter `messages_total` is not a metric. `SubscriptionFreshness::refresh` watches it as the "new data exists" signal behind `.subscribe_with_timeout()`, and its answer drives `StalePolicy::SafeState` and `StalePolicy::Stop`. Only `RingTopic::send` bumped it. `Topic::try_send` and `Topic::send_blocking` go straight to the ring, so a topic published entirely through either read as 0 messages and 0 Hz while carrying every message -- and could safe-state or halt a subscriber node whose data was arriving normally. That is exactly the failure the counter was introduced to fix, reintroduced for backpressure-aware publishers, and these are the two APIs the documentation points users to on critical topics. The runtime states the invariant in both places that consume the counter: "every publisher increments `messages_total` in the topic's SHM header on every send, so watching that counter is a true 'new data exists' signal" (scheduling/types.rs) and "incremented on every `send()` regardless of backend" (header.rs). Neither is true. Bumped once per public publish call, not inside `RingTopic::try_send`, which `send_lossy_retry` calls up to ~70 times for one message. `try_send` counts before the call, as `send()` does: an attempt count, and a publisher hammering a full ring is alive. `send_blocking` does NOT count a `NoBackpressure` refusal -- that is returned before the send is even attempted, so nothing reached the ring, and counting it would report a healthy flow on a topic delivering nothing. A `Timeout` did engage the ring, so it counts like `send()`'s dropped message does. Reverting both bumps reads 0 for a topic that carried ten messages. --- horus_core/src/communication/topic/mod.rs | 121 +++++++++++++++++++++- 1 file changed, 120 insertions(+), 1 deletion(-) diff --git a/horus_core/src/communication/topic/mod.rs b/horus_core/src/communication/topic/mod.rs index d769a743a..df7167171 100644 --- a/horus_core/src/communication/topic/mod.rs +++ b/horus_core/src/communication/topic/mod.rs @@ -1453,6 +1453,17 @@ impl RingTopic< unsafe { &*(self.storage.as_ptr() as *const TopicHeader) } } + /// Bump the "new data exists" counter the staleness watchdog reads. + /// + /// `RingTopic::send` does this inline. It lives here as well so the public + /// `Topic::try_send` / `Topic::send_blocking` can do it exactly once per + /// call: it must NOT move into `RingTopic::try_send`, which + /// `send_lossy_retry` calls up to ~70 times for a single message. + #[inline(always)] + fn bump_messages_total(&self) { + self.header().messages_total.fetch_add(1, Ordering::Relaxed); + } + /// Check if verbose content logging is enabled via the SHM header flag. /// Uses the stable `header_ptr` (not `LocalState::cached_header_ptr`, which /// is repurposed by the role=Both same-instance fast path). @@ -4458,6 +4469,19 @@ where // A backpressure-aware publisher is still a publisher; this had no // registration block, so `try_send`-only nodes were unattributed. self.register_pub_lazy(); + // `messages_total` is not a metric — `SubscriptionFreshness` watches it + // as the "new data exists" signal behind `.subscribe_with_timeout()`, + // and that drives `StalePolicy::SafeState` and `Stop`. Only `send()` + // used to bump it, so a topic driven entirely by `try_send` read as 0 + // messages and 0 Hz while carrying full traffic, and could safe-state + // or halt a subscriber node whose data was arriving normally. That is + // the exact failure the counter was introduced to prevent, and these + // are the APIs the docs send users to on critical topics. + // + // Counted before the call, like `send()`: this is an attempt count, and + // a publisher hammering a full ring is alive — a ring nobody drains is + // a different fault, and one the subscriber's own staleness sees. + self.ring.bump_messages_total(); self.ring.try_send(msg) } @@ -4520,7 +4544,17 @@ where timeout: std::time::Duration, ) -> Result<(), SendBlockingError> { self.register_pub_lazy(); - self.ring.send_blocking(msg, timeout) + let result = self.ring.send_blocking(msg, timeout); + // Same "new data exists" signal as `try_send` above — but NOT for + // `NoBackpressure`. That refusal is returned before the send is even + // attempted, so nothing reached the ring; counting it would report a + // healthy flow on a topic delivering nothing, which is worse than the + // silence it replaced. A `Timeout` did engage the ring, so it counts as + // an attempt like `send()`'s dropped message does. + if !matches!(result, Err(SendBlockingError::NoBackpressure)) { + self.ring.bump_messages_total(); + } + result } /// Low-level receive without logging/recording hooks. @@ -5086,3 +5120,88 @@ mod untrusted_ring_geometry_tests { cleanup(&name); } } + +#[cfg(test)] +mod staleness_signal_tests { + use super::{SendBlockingError, Topic}; + use std::time::Duration; + + fn cleanup(name: &str) { + if let Some(path) = horus_sys::shm::topic_shm_path_checked(name) { + let _ = std::fs::remove_file(path); + } + } + + /// `messages_total` is the staleness watchdog's "new data exists" signal. + /// + /// `SubscriptionFreshness::refresh` watches it behind + /// `.subscribe_with_timeout()`, and its answer drives + /// `StalePolicy::SafeState` and `StalePolicy::Stop`. Only `RingTopic::send` + /// bumped it, so a topic published entirely through `try_send` read as 0 + /// messages and 0 Hz while carrying every message — and could safe-state or + /// halt a subscriber whose data was arriving normally. That is precisely + /// the failure the counter exists to prevent, and `try_send` / + /// `send_blocking` are the two APIs the docs point users to on critical + /// topics. + #[test] + fn try_send_moves_the_staleness_counter() { + let name = format!("staleness_try_send_{}", std::process::id()); + cleanup(&name); + let Ok(topic) = Topic::::new(&name) else { + eprintln!("skipping: no shared memory available"); + return; + }; + + assert_eq!(topic.ring.header().messages_total(), 0); + for i in 0..10u64 { + let _ = topic.try_send(i); + } + + assert_eq!( + topic.ring.header().messages_total(), + 10, + "ten try_send calls delivered ten messages; a watchdog reading this \ + counter must not conclude the topic is dead" + ); + + drop(topic); + cleanup(&name); + } + + /// The same for `send_blocking`, on a topic that has backpressure. + #[test] + fn send_blocking_moves_the_staleness_counter() { + let name = format!("staleness_send_blocking_{}", std::process::id()); + cleanup(&name); + let Ok(topic) = Topic::::new(&name) else { + eprintln!("skipping: no shared memory available"); + return; + }; + + let mut delivered = 0u64; + for i in 0..10u64 { + match topic.send_blocking(i, Duration::from_millis(50)) { + Ok(()) => delivered += 1, + // A broadcast backend refuses before it ever touches the ring. + Err(SendBlockingError::NoBackpressure) => {} + Err(SendBlockingError::Timeout) => delivered += 1, + } + } + + assert_eq!( + topic.ring.header().messages_total(), + delivered, + "every send_blocking that engaged the ring must move the counter, \ + and a NoBackpressure refusal — which never reaches the ring — must \ + not, or a topic delivering nothing would read as healthy" + ); + assert!( + delivered > 0, + "precondition: a single-subscriber topic has backpressure, so these \ + sends must have engaged the ring" + ); + + drop(topic); + cleanup(&name); + } +} From 8c87bf31dcd68423c47fa762b17dc7d47b70e90d Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 17:33:47 -0700 Subject: [PATCH 08/79] fix(topic): tell the 17th participant why it has no comms A topic holds at most 16 participants. `register_role` builds a precise error for the one that does not fit -- it names the limit, the count, and what to do about it -- and all three call sites threw it away: `recv_uninitialized` and `read_latest` were `if ....is_err() { return None }`, `send_uninitialized` the same with `Err(msg)`. Those are the only `ensure_consumer`/`ensure_producer` call sites in the tree, so that message reached nothing, ever. Because the role stays `Unregistered`, every later `recv()` re-enters the same cold path and fails identically: `None` forever, from a subscriber indistinguishable from an idle one. Nothing anywhere defends the silence, and the identical 16-slot wall one layer down is handled the opposite way on purpose -- rate-limited `tracing::warn!` under "COMM-H1: never let 'all 16 slots live' become a SILENT no-comms endpoint (worse than the pre-fix loud panic)". The rate limiter for it sits in the same file, unused by this path. Meanwhile header.rs defends a separate design decision by claiming the residual cost "surfaces as the existing 'No available participant slots' error. Loud, and it says what to do" -- a guarantee the code did not provide. All three sites now report through that same limiter, so a hot loop warns once a minute per topic rather than once ever. The limiter is also keyed by reason now, not just topic. A topic can hit two different walls -- its FanoutShm endpoint bitmap and the participant table -- with different causes and different fixes, and one bucket let whichever fired first silence the other for a minute. That is the masking the per-topic keying was introduced to prevent, one level down. The test fills the table from 20 threads (the participant-count leak is what lets one process do that), asserts it really is full before asserting anything about diagnosis, and captures `tracing` events through a hand-rolled `Subscriber` so it needs no new dependency. Reverting the fix prints nothing at all: "Warnings seen: []". --- .../src/communication/topic/dispatch.rs | 61 ++++++++-- horus_core/src/communication/topic/mod.rs | 18 ++- .../participant_exhaustion_is_reported.rs | 113 ++++++++++++++++++ 3 files changed, 182 insertions(+), 10 deletions(-) create mode 100644 horus_core/tests/participant_exhaustion_is_reported.rs diff --git a/horus_core/src/communication/topic/dispatch.rs b/horus_core/src/communication/topic/dispatch.rs index 3ac37e8b0..84f8e0674 100644 --- a/horus_core/src/communication/topic/dispatch.rs +++ b/horus_core/src/communication/topic/dispatch.rs @@ -1199,7 +1199,7 @@ pub(super) fn send_fanout_shm { // COMM-H1: same as the send path — loud, rate-limited, never silent. - if should_report_endpoint_exhaustion(topic.name()) { + if should_report_endpoint_exhaustion(topic.name(), Exhausted::FanoutEndpoints) { tracing::warn!( "FanoutShm topic '{}': all {} subscriber endpoint slots are \ live — this subscriber has NO comms until a slot frees \ @@ -1887,7 +1887,16 @@ pub(super) fn send_uninitialized< topic: &RingTopic, msg: T, ) -> Result<(), T> { - if topic.ensure_producer().is_err() { + if let Err(e) = topic.ensure_producer() { + // As `recv_uninitialized`: the caller gets its message back with no + // indication that the cause is permanent for this handle. + if should_report_endpoint_exhaustion(topic.name(), Exhausted::ParticipantTable) { + tracing::warn!( + "topic '{}': this publisher is NOT registered and will deliver nothing — {}", + topic.name(), + e + ); + } return Err(msg); } // SAFETY: ensure_producer() → initialize_backend() → set_dispatch_fn_ptrs() @@ -2612,7 +2621,24 @@ pub(super) fn recv_uninitialized< >( topic: &RingTopic, ) -> Option { - if topic.ensure_consumer().is_err() { + if let Err(e) = topic.ensure_consumer() { + // Registration failed — almost always the 16-slot participant table + // being full. The role stays `Unregistered`, so every later `recv()` + // re-enters this cold path and fails identically: `None` forever, from + // a subscriber that looks merely idle. Discarding the error here made + // the message `register_role` builds — which names the limit and says + // what to do about it — reach nothing at all. + // + // The same 16-slot wall one layer down is reported, deliberately, by + // this same rate limiter: "never let 'all 16 slots live' become a + // SILENT no-comms endpoint". + if should_report_endpoint_exhaustion(topic.name(), Exhausted::ParticipantTable) { + tracing::warn!( + "topic '{}': this subscriber is NOT registered and will receive nothing — {}", + topic.name(), + e + ); + } return None; } // SAFETY: ensure_consumer() → initialize_backend() → set_dispatch_fn_ptrs() @@ -2632,13 +2658,20 @@ pub(super) fn recv_uninitialized< /// Keyed by topic and rate-limited per topic instead, so a hot loop still emits /// once a minute rather than once ever, and a second topic hitting the same /// wall is never masked by the first. -fn should_report_endpoint_exhaustion(topic: &str) -> bool { +/// +/// Keyed by `reason` as well as topic, for the same argument one level down: a +/// topic can hit two different walls — its FanoutShm endpoint bitmap and the +/// 16-slot participant table — and they have different causes and different +/// fixes. Sharing one bucket would let whichever fired first silence the other +/// for a minute, which is the masking this function exists to prevent. +pub(super) fn should_report_endpoint_exhaustion(topic: &str, reason: Exhausted) -> bool { use std::collections::HashMap; use std::sync::Mutex; use std::time::{Duration, Instant}; const QUIET: Duration = Duration::from_secs(60); - static LAST: Mutex>> = Mutex::new(None); + #[allow(clippy::type_complexity)] + static LAST: Mutex>> = Mutex::new(None); let mut guard = match LAST.lock() { Ok(g) => g, @@ -2646,15 +2679,27 @@ fn should_report_endpoint_exhaustion(topic: &str) -> bool { }; let seen = guard.get_or_insert_with(HashMap::new); let now = Instant::now(); - match seen.get(topic) { + let key = (topic.to_string(), reason); + match seen.get(&key) { Some(t) if now.duration_since(*t) < QUIET => false, _ => { - seen.insert(topic.to_string(), now); + seen.insert(key, now); true } } } +/// Which wall a topic ran into. Rate-limited separately; see +/// `should_report_endpoint_exhaustion`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(super) enum Exhausted { + /// The FanoutShm publisher or subscriber endpoint bitmap is full. + FanoutEndpoints, + /// The topic's 16-slot participant table is full, so this handle holds no + /// registration and will never send or receive anything. + ParticipantTable, +} + #[cfg(test)] mod tests { use super::*; diff --git a/horus_core/src/communication/topic/mod.rs b/horus_core/src/communication/topic/mod.rs index df7167171..7727f0d4d 100644 --- a/horus_core/src/communication/topic/mod.rs +++ b/horus_core/src/communication/topic/mod.rs @@ -3440,8 +3440,22 @@ impl RingTopic< T: Copy, { // Ensure we're registered as a consumer so the header is initialized - if self.local().role == TopicRole::Unregistered && self.ensure_consumer().is_err() { - return None; + if self.local().role == TopicRole::Unregistered { + if let Err(e) = self.ensure_consumer() { + // See `dispatch::recv_uninitialized`: silence here is a + // subscriber that reads `None` forever and looks idle. + if dispatch::should_report_endpoint_exhaustion( + self.name(), + dispatch::Exhausted::ParticipantTable, + ) { + tracing::warn!( + "topic '{}': this reader is NOT registered and will see nothing — {}", + self.name(), + e + ); + } + return None; + } } // Role==Both AND POD uses the local fast path (LocalState head/tail diff --git a/horus_core/tests/participant_exhaustion_is_reported.rs b/horus_core/tests/participant_exhaustion_is_reported.rs new file mode 100644 index 000000000..b33ec65f7 --- /dev/null +++ b/horus_core/tests/participant_exhaustion_is_reported.rs @@ -0,0 +1,113 @@ +//! The 17th participant must be told why it has no comms. +//! +//! A topic holds at most 16 participants. `register_role` builds a precise +//! error for the one that does not fit — it names the limit and says what to do +//! about it — and all three call sites threw it away: +//! `recv_uninitialized` and `read_latest` were `if ....is_err() { return None }` +//! and `send_uninitialized` the same with `Err(msg)`. Those are the only +//! `ensure_consumer`/`ensure_producer` call sites in the tree, so the message +//! reached nothing. +//! +//! Because the role stays `Unregistered`, every later `recv()` re-enters the +//! same cold path and fails identically: `None` forever, from a subscriber that +//! is indistinguishable from an idle one. The identical 16-slot wall one layer +//! down is reported deliberately and loudly, under a comment reading "never let +//! 'all 16 slots live' become a SILENT no-comms endpoint" — with the rate +//! limiter for it sitting in the same file, unused by this path. + +use horus_core::communication::Topic; +use std::sync::{Arc, Mutex}; + +/// A `tracing::Subscriber` that keeps the formatted fields of every event. +/// +/// Hand-rolled against `tracing` core so this test needs no new dependency. +struct Capture(Arc>>); + +impl tracing::Subscriber for Capture { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::Id { + tracing::Id::from_u64(1) + } + fn record(&self, _: &tracing::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::Id, _: &tracing::Id) {} + fn event(&self, event: &tracing::Event<'_>) { + struct Fields(String); + impl tracing::field::Visit for Fields { + fn record_debug(&mut self, _: &tracing::field::Field, value: &dyn std::fmt::Debug) { + self.0.push_str(&format!("{:?} ", value)); + } + } + let mut fields = Fields(String::new()); + event.record(&mut fields); + self.0 + .lock() + .unwrap_or_else(|p| p.into_inner()) + .push(fields.0); + } + fn enter(&self, _: &tracing::Id) {} + fn exit(&self, _: &tracing::Id) {} +} + +#[test] +fn a_participant_that_does_not_fit_is_told_so() { + let name = format!("participant_exhaustion_{}", std::process::id()); + + // Process-wide, not `set_default`: the registration that gets refused + // happens on a spawned thread, and `set_default` is thread-local. A thread + // is required — the main thread already holds a participant entry for its + // publisher, and `ensure_consumer` upgrades that entry's role in place + // rather than claiming a second slot, so it is never refused. + // This test is alone in its binary, so the global default is free. + let captured = Arc::new(Mutex::new(Vec::new())); + tracing::subscriber::set_global_default(Capture(Arc::clone(&captured))) + .expect("no other subscriber in this test binary"); + + let Ok(producer) = Topic::::new(&name) else { + eprintln!("skipping: no shared memory available"); + return; + }; + producer.send(1u64); + + // Fill the participant table. Each thread claims its own slot, and — this + // is the open leak documented in participant_count_leak.rs — the slot is + // not returned when the thread ends, which is what lets one process fill + // the table at all. The producer holds one, so 15 subscribers saturate it + // and every thread after that is refused. + for _ in 0..20 { + let n = name.clone(); + std::thread::spawn(move || { + if let Ok(sub) = Topic::::new(&n) { + let _ = sub.try_recv(); + } + }) + .join() + .expect("subscriber thread"); + } + + let messages = captured.lock().unwrap_or_else(|p| p.into_inner()).clone(); + + // Anti-vacuity: assert the table really did fill BEFORE asking whether the + // overflow was reported. Without this the test passes on a platform where a + // slot was always available and nothing was ever refused. + let occupied = producer.sub_count() + producer.pub_count(); + if occupied < 16 { + eprintln!("skipping: the participant table did not fill ({occupied}/16)"); + return; + } + + assert!( + messages.iter().any(|m| m.contains("participant slots")), + "five subscribers were refused a registration and will receive nothing \ + for the life of their handles. Not one of them was told. Warnings \ + seen: {:?}", + messages + ); + assert!( + messages.iter().any(|m| m.contains(&name)), + "the warning must name the topic — an operator needs to know WHICH \ + subsystem went quiet. Warnings seen: {:?}", + messages + ); +} From dbd4d942abd241ec82cdeaa589ae2f10e54cfc63 Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 17:46:06 -0700 Subject: [PATCH 09/79] fix(python): a refusal for want of backpressure is not a timeout `send_blocking_pod` ended in `.is_ok()`, discarding which `SendBlockingError` came back, and both arms then reported `HorusTimeoutError`. So the one error horus_core writes specifically to be un-retryable arrived in Python as the one error callers retry. The core is explicit about the distinction, twice: "Refusing is deliberate: reporting success for a delivery nobody guaranteed is worse on an e-stop path than failing loudly. It does not clear on retry -- it lasts as long as the topology that caused it." The variant carries a four-line message naming the fix. The binding replaced it with the exception whose own doc-comment means the opposite -- "the buffer stayed full for the whole timeout" -- and its private helper asserted, falsely, that a `false` return meant the ring stayed full. This is easy to reach and permanent: `detect_optimal_backend` resolves any POD topic with a second subscriber to `PodShm`, so the moment a `horus topic echo` or a monitor attaches to a POD e-stop topic, every `send_blocking` starts raising forever -- as a "timeout", on the path the method is documented for. The variant is now threaded out of both arms and mapped to a new `horus.HorusBackpressureError` carrying the core's own remediation text. `HorusTimeoutError` keeps `Timeout` alone. Also exposes `Topic.provides_backpressure`, which the core's doc-comment instructs callers to assert on after the first send and which Python had no way to ask. It is answered through a new `horus_core::communication::topic::backend_provides_backpressure`, keyed by backend NAME because a name is all a binding can see -- and `provides_backpressure()` is now defined in terms of it too, so the two cannot disagree. A test walks every `BackendMode` through an exhaustive match, so a new backend stops compiling rather than silently defaulting to "no backpressure". Reverting the mapping fails the test at "must raise HorusBackpressureError". --- horus_core/src/communication/topic/mod.rs | 89 ++++++++++++--- horus_py/horus/__init__.py | 10 ++ horus_py/horus/_horus.pyi | 3 + horus_py/src/errors.rs | 9 ++ horus_py/src/lib.rs | 4 + horus_py/src/topic.rs | 131 +++++++++++++++++++--- 6 files changed, 216 insertions(+), 30 deletions(-) diff --git a/horus_core/src/communication/topic/mod.rs b/horus_core/src/communication/topic/mod.rs index 7727f0d4d..7ac83a5ee 100644 --- a/horus_core/src/communication/topic/mod.rs +++ b/horus_core/src/communication/topic/mod.rs @@ -590,6 +590,31 @@ pub enum SendBlockingError { NoBackpressure, } +/// Whether the backend named by [`Topic::backend_name`] refuses a send when its +/// ring is full. +/// +/// Keyed by NAME rather than by the private `BackendMode`, because the bindings +/// are where this question actually gets asked and a name is all they can see: +/// `backend_type` is a string over both the Python and C++ FFI. Answering it +/// there by re-deriving the mapping is how a binding drifts from the runtime, so +/// `provides_backpressure()` is defined in terms of this too and the two cannot +/// disagree. +/// +/// - `SpscShm`, `MpscShm`, `SpmcShm` — ring-full is a real `Err`: the send paths +/// compare the claimed sequence against `header.tail` and refuse. +/// - `PodShm`, `FanoutShm` — broadcast, overwrite-oldest by construction, so +/// there is no full condition to report. +/// - Anything else, including `Unknown`, fails closed. `Unknown` is the state +/// EVERY topic is in until its first send or recv — including at the "assert +/// this at startup" moment the doc comment above recommends. Answering `true` +/// there made the assertion pass on a topic that was about to resolve to +/// `PodShm` and silently drop, so the check a caller writes to catch a lossy +/// e-stop channel could not catch one. An unresolved backend is not a promise +/// of backpressure. +pub fn backend_provides_backpressure(backend_name: &str) -> bool { + matches!(backend_name, "SpscShm" | "MpscShm" | "SpmcShm") +} + impl From for crate::error::HorusError { fn from(err: SendBlockingError) -> Self { crate::error::HorusError::Communication(crate::error::CommunicationError::TopicFull { @@ -3610,22 +3635,7 @@ impl RingTopic< /// ); /// ``` pub fn provides_backpressure(&self) -> bool { - match self.mode() { - // Ring-full is a real Err on these: the mp/sp send paths compare the - // claimed sequence against `header.tail` and refuse. - BackendMode::SpscShm | BackendMode::MpscShm | BackendMode::SpmcShm => true, - // Broadcast: overwrite-oldest by construction, no full condition. - BackendMode::PodShm | BackendMode::FanoutShm => false, - // Fail closed. `Unknown` means the backend has not been resolved - // yet, which is the state EVERY topic is in until its first - // send/recv — including at the "assert this at startup" moment the - // doc comment above recommends. Answering `true` there made the - // assertion pass on a topic that was about to resolve to PodShm and - // silently drop, so the check a caller writes to catch a lossy - // e-stop channel could not catch one. An unresolved backend is not - // a promise of backpressure. - BackendMode::Unknown => false, - } + backend_provides_backpressure(self.backend_name()) } #[cfg(test)] @@ -5140,6 +5150,53 @@ mod staleness_signal_tests { use super::{SendBlockingError, Topic}; use std::time::Duration; + /// `backend_provides_backpressure` must answer for every backend there is, + /// and answer `false` for anything it does not recognise. + /// + /// The Python and C++ bindings only ever see the backend NAME, so this + /// string table is what they ask. A backend added without a row here would + /// silently report "no backpressure" — which fails closed, but would make + /// `send_blocking` refuse on a backend that actually supports it. The match + /// below is exhaustive over `BackendMode`, so adding a variant stops + /// compiling here rather than going unnoticed. + #[test] + fn every_backend_name_has_a_backpressure_answer() { + use super::backend_provides_backpressure; + use super::types::BackendMode; + + let expected = [ + (BackendMode::Unknown, false), + (BackendMode::PodShm, false), + (BackendMode::FanoutShm, false), + (BackendMode::SpscShm, true), + (BackendMode::SpmcShm, true), + (BackendMode::MpscShm, true), + ]; + + for (mode, provides) in expected { + // Mirrors `RingTopic::backend_name`; exhaustive on purpose. + let name = match mode { + BackendMode::Unknown => "Unknown", + BackendMode::PodShm => "PodShm", + BackendMode::SpscShm => "SpscShm", + BackendMode::SpmcShm => "SpmcShm", + BackendMode::MpscShm => "MpscShm", + BackendMode::FanoutShm => "FanoutShm", + }; + assert_eq!( + backend_provides_backpressure(name), + provides, + "{name} is on the wrong side of the backpressure table" + ); + } + + assert!( + !backend_provides_backpressure("SomethingNewAndUnknown"), + "an unrecognised backend must fail closed — an unresolved backend \ + is not a promise of backpressure" + ); + } + fn cleanup(name: &str) { if let Some(path) = horus_sys::shm::topic_shm_path_checked(name) { let _ = std::fs::remove_file(path); diff --git a/horus_py/horus/__init__.py b/horus_py/horus/__init__.py index 42c816f9b..3fcc25588 100644 --- a/horus_py/horus/__init__.py +++ b/horus_py/horus/__init__.py @@ -213,6 +213,7 @@ def __init__(self, topic: str, msg_type=None, capacity: int = 1024): HorusNotFoundError, HorusTransformError, HorusTimeoutError, + HorusBackpressureError, # Coordinate transforms TransformFrame, Transform, @@ -271,6 +272,14 @@ class HorusTransformError(Exception): class HorusTimeoutError(Exception): """Raised when a blocking operation times out.""" + class HorusBackpressureError(Exception): + """Raised when send_blocking() is refused for want of backpressure. + + The topic's backend overwrites unread messages and has no full ring to + wait on, so the call was refused before any write. Unlike a timeout this + is not transient: it lasts as long as the topology that caused it. + """ + # ── Python enums wrapping Rust constants ───────────────────────────────────── @@ -1988,6 +1997,7 @@ def run(*nodes: Node, "HorusNotFoundError", "HorusTransformError", "HorusTimeoutError", + "HorusBackpressureError", # Submodules "msggen", "perception", diff --git a/horus_py/horus/_horus.pyi b/horus_py/horus/_horus.pyi index 58db654db..583326fef 100644 --- a/horus_py/horus/_horus.pyi +++ b/horus_py/horus/_horus.pyi @@ -111,6 +111,7 @@ def time_rng_float() -> float: ... class HorusNotFoundError(Exception): ... class HorusTransformError(Exception): ... class HorusTimeoutError(Exception): ... +class HorusBackpressureError(Exception): ... # ── NodeState ──────────────────────────────────────────────────────────────── @@ -171,6 +172,8 @@ class Topic: def endpoint(self) -> Optional[str]: ... @property def backend_type(self) -> str: ... + @property + def provides_backpressure(self) -> bool: ... def send(self, message: Any, node: Optional[Any] = None) -> bool: ... def try_send(self, message: Any, node: Optional[Any] = None) -> bool: ... def send_blocking(self, message: Any, timeout_s: float, node: Optional[Any] = None) -> None: ... diff --git a/horus_py/src/errors.rs b/horus_py/src/errors.rs index c838e438e..3aff2ca4c 100644 --- a/horus_py/src/errors.rs +++ b/horus_py/src/errors.rs @@ -4,6 +4,7 @@ //! - `HorusNotFoundError` — missing topics, frames, nodes //! - `HorusTransformError` — transform extrapolation, stale data //! - `HorusTimeoutError` — blocking operation timeouts +//! - `HorusBackpressureError` — send_blocking() refused: no backpressure to wait on use horus_core::error::HorusError; use pyo3::create_exception; @@ -30,6 +31,14 @@ create_exception!( PyException, "Raised when a blocking operation times out." ); +create_exception!( + _horus, + HorusBackpressureError, + PyException, + "Raised when send_blocking() is refused because the topic's backend has no \ + backpressure. Unlike a timeout this is not transient: it lasts as long as \ + the topology that caused it, so retrying cannot succeed." +); /// Build the error message, appending actionable `.help()` text when available. /// diff --git a/horus_py/src/lib.rs b/horus_py/src/lib.rs index f6a1646e9..10aae6fee 100644 --- a/horus_py/src/lib.rs +++ b/horus_py/src/lib.rs @@ -111,6 +111,10 @@ fn _horus(m: &Bound<'_, PyModule>) -> PyResult<()> { "HorusTimeoutError", m.py().get_type::(), )?; + m.add( + "HorusBackpressureError", + m.py().get_type::(), + )?; m.add_function(wrap_pyfunction!(get_version, m)?)?; diff --git a/horus_py/src/topic.rs b/horus_py/src/topic.rs index c75827878..2c05fbb44 100644 --- a/horus_py/src/topic.rs +++ b/horus_py/src/topic.rs @@ -148,6 +148,24 @@ fn send_timeout_err( crate::errors::to_py_err(send_timeout_error(topic, elapsed, deadline)) } +/// `SendBlockingError::NoBackpressure` as a Python exception. +/// +/// NOT `HorusTimeoutError`. The core writes this variant specifically to be +/// un-retryable — "it does not clear on retry — it lasts as long as the +/// topology that caused it" — and a timeout is the one error a caller retries. +/// Collapsing the two handed Python the opposite instruction, on the path the +/// method is documented for: emergency stop and motor setpoints. It is also +/// permanent and easy to reach, since any POD topic with a second subscriber +/// (a `horus topic echo`, a monitor) resolves to `PodShm`. +/// +/// The core's own message names the fix, so it is passed through verbatim. +fn no_backpressure_err(topic: &str) -> PyErr { + crate::errors::HorusBackpressureError::new_err(format!( + "send_blocking on topic {topic}: {}", + horus_core::communication::topic::SendBlockingError::NoBackpressure + )) +} + /// The `HorusError` behind [`send_timeout_err`], split out so its wording can be /// asserted on without a Python interpreter to unwrap a `PyErr` with. fn send_timeout_error( @@ -357,12 +375,17 @@ macro_rules! pod_topic_types { /// Send a POD message, waiting up to `timeout` for ring space. /// - /// `Ok(Some(true))` sent, `Ok(Some(false))` the ring stayed full for - /// the whole timeout, `Ok(None)` not a POD type (caller handles it). + /// `Ok(Some(Ok(())))` sent, `Ok(Some(Err(e)))` the core refused and + /// said why, `Ok(None)` not a POD type (caller handles it). + /// + /// This used to be `.is_ok()`, which threw away WHICH + /// `SendBlockingError` came back, so the caller had nothing left to + /// distinguish "the ring stayed full" from "this backend has no + /// full ring at all" and reported both as a timeout. fn send_blocking_pod( &self, py: Python, message: &Py, node: &Option>, timeout: std::time::Duration, start: std::time::Instant, - ) -> PyResult> { + ) -> PyResult>> { match &self.topic_type { $( TopicType::$rust_ty(topic) => { @@ -378,9 +401,9 @@ macro_rules! pod_topic_types { // a 10 ms wait here would stop every other Python thread // in the process for 10 ms. let sent = py.detach(|| { - topic_ref.lock().expect("lock").send_blocking(val, timeout).is_ok() + topic_ref.lock().expect("lock").send_blocking(val, timeout) }); - if sent { + if sent.is_ok() { if let Some(s) = summary { log_ipc_event(py, node, &self.name, s, start.elapsed().as_nanos() as u64, "log_pub"); @@ -976,7 +999,12 @@ impl PyTopic { /// /// Raises: /// HorusTimeoutError: the buffer stayed full for the whole timeout; - /// the message was NOT published. + /// the message was NOT published. Transient — retrying can work. + /// HorusBackpressureError: this topic's backend overwrites unread + /// messages and has no full ring to wait on, so the call was + /// refused before any write. NOT transient: it lasts as long as + /// the topology that caused it, so retrying cannot help. Check + /// `provides_backpressure()` after the first send. /// ValueError: timeout_s is negative, NaN, or infinite. /// NotImplementedError: pool-backed topics (Image, PointCloud, /// DepthImage, Tensor) have no blocking send path. @@ -985,7 +1013,10 @@ impl PyTopic { /// try: /// estop.send_blocking(EmergencyStop(True), 0.05) /// except horus.HorusTimeoutError: - /// node.log_error("e-stop not delivered") + /// node.log_error("e-stop not delivered — ring full") + /// except horus.HorusBackpressureError: + /// node.log_error("e-stop topic is broadcast-backed; delivery " + /// "was never guaranteed") #[pyo3(signature = (message, timeout_s, node=None))] fn send_blocking( &self, @@ -1000,10 +1031,13 @@ impl PyTopic { // Fast path: POD types (generated by pod_topic_types! macro) if let Some(sent) = self.send_blocking_pod(py, &message, &node, timeout, start)? { - return if sent { - Ok(()) - } else { - Err(send_timeout_err(&self.name, start.elapsed(), timeout)) + use horus_core::communication::topic::SendBlockingError; + return match sent { + Ok(()) => Ok(()), + Err(SendBlockingError::NoBackpressure) => Err(no_backpressure_err(&self.name)), + Err(SendBlockingError::Timeout) => { + Err(send_timeout_err(&self.name, start.elapsed(), timeout)) + } }; } @@ -1019,10 +1053,15 @@ impl PyTopic { .lock() .expect("topic lock poisoned") .send_blocking(msg, timeout) - .is_ok() }); - if !sent { - return Err(send_timeout_err(&self.name, start.elapsed(), timeout)); + if let Err(e) = sent { + use horus_core::communication::topic::SendBlockingError; + return Err(match e { + SendBlockingError::NoBackpressure => no_backpressure_err(&self.name), + SendBlockingError::Timeout => { + send_timeout_err(&self.name, start.elapsed(), timeout) + } + }); } if node.is_some() { log_ipc_event( @@ -1212,6 +1251,30 @@ impl PyTopic { self.endpoint.clone() } + /// Whether a full ring can make this topic refuse a send. + /// + /// This is a property of the BACKEND, and the backend is chosen at runtime + /// from how many participants are attached — one subscriber gives you + /// `SpscShm`, which refuses when full; attach a second (a logger, a + /// recorder, a `horus topic echo`) and the same topic becomes `PodShm` or + /// `FanoutShm`, which overwrite instead. On those, `send_blocking()` raises + /// `HorusBackpressureError` and `send()` silently loses data. + /// + /// Assert it on any topic carrying commands rather than samples, AFTER the + /// first send — the backend is unresolved until a handle moves a message, + /// and an unresolved backend answers False. + /// + /// estop.send(EmergencyStop(True)) # resolves the backend + /// assert estop.provides_backpressure, estop.backend_type + /// + /// Answered from `backend_type` through the runtime's own + /// `backend_provides_backpressure`, so this cannot drift from what + /// `send_blocking()` will actually do. + #[getter] + fn provides_backpressure(&self) -> bool { + horus_core::communication::topic::backend_provides_backpressure(&self.backend_type()) + } + /// Get the backend type name #[getter] fn backend_type(&self) -> String { @@ -2027,6 +2090,46 @@ mod tests { ); } + /// A refusal for want of backpressure must NOT arrive as a timeout. + /// + /// The core writes `SendBlockingError::NoBackpressure` specifically to be + /// un-retryable — "it does not clear on retry — it lasts as long as the + /// topology that caused it" — and a timeout is the one error a caller + /// retries. `send_blocking_pod` used to end in `.is_ok()`, throwing the + /// variant away, and both arms reported `HorusTimeoutError`: the opposite + /// instruction, on the path this method is documented for (emergency stop + /// and motor setpoints). It is easy to reach, too — any POD topic resolves + /// to `PodShm` the moment a second subscriber attaches, so a `horus topic + /// echo` against an e-stop topic starts it raising forever. + #[test] + fn a_backpressure_refusal_is_not_a_timeout() { + // The exception types are Python objects, so this test needs an + // interpreter. Safe to call repeatedly. + Python::initialize(); + Python::attach(|py| { + let err = no_backpressure_err("estop"); + + assert!( + err.is_instance_of::(py), + "must raise HorusBackpressureError" + ); + assert!( + !err.is_instance_of::(py), + "must NOT be catchable as a timeout — a caller who retries a \ + timeout would retry forever, because this condition is \ + topological and permanent" + ); + + let msg = err.value(py).to_string(); + assert!(msg.contains("estop"), "must name the topic: {msg}"); + // The core's message names the fix; it must survive the crossing. + assert!( + msg.contains("MpscShm") || msg.contains("backpressured backend"), + "must carry the runtime's remediation text: {msg}" + ); + }); + } + /// The pool-backed refusal must say which call and which type, and point at /// the counter that makes the loss visible instead. #[test] From f557cddd6777caaf0fb763a72291c49a08b1ab84 Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 17:58:37 -0700 Subject: [PATCH 10/79] fix(python): make subs= type auto-detection actually run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_setup_topics` documents a three-step type resolution whose middle step, "auto-detect from SHM header", was unreachable code. It did: probe = Topic(topic_name, 64) type_name = getattr(probe, 'type_name', None) Both halves were dead. `PyTopic` exposes no `type_name` — its getters are `name`, `is_network_topic`, `endpoint`, `backend_type` — so the value was always `None`. And the probe could not have worked even with that attribute: `Topic(name, capacity)` builds a `Topic`, and attaching 4364-byte GenericMessage to an existing typed ring is refused by the size check in `RingTopic::new`, so it always raised into a bare `except Exception`. Step 2 therefore never ran for any topic that has ever existed, and the ~70-entry `_TYPE_NAME_MAP` built for it had exactly one reader, inside the unreachable branch. The size check is correct and stays: its comment records that a name-based exemption once let a `Topic` join a `Topic` ring and "write megabytes past the end of the mapping". The question just cannot be answered by attaching -- opening requires choosing a type, which is what the caller is trying to find out. So: `horus_core::communication::peek_topic_type_name(name)` reads the header without attaching, exposed to Python as `_horus.peek_topic_type`. `_detect_topic_type` calls it and maps through `_TYPE_NAME_MAP`, which now has a reachable reader. Second half: when the ring exists and holds a type this build cannot name, `_setup_topics` raises with the topic, the ring's type and the concrete fix (`subs=[Sub('scan', LaserScan)]`) instead of falling through to a generic topic and letting the runtime's byte-count RuntimeError out -- which named neither `subs=`, nor the type the ring holds, nor what to do. Auto-detection was only ever scoped to subscriptions; the `pubs=` case is unchanged, since a publisher creates the ring and there is nothing to read. Tests both ends. The Rust test asserts the old probe's refusal still happens (it must) and that the peek answers anyway; reverting the peek gives `left: None`, which is what the old probe returned for every topic. The Python test drives a real `Node` with a bare `subs=` string against a typed ring; restoring the old probe fails it. --- horus_core/src/communication/mod.rs | 4 +- horus_core/src/communication/topic/header.rs | 18 +++++ horus_core/src/communication/topic/mod.rs | 6 +- horus_py/horus/__init__.py | 59 +++++++++++++--- horus_py/src/lib.rs | 1 + horus_py/src/topic.rs | 65 +++++++++++++++++ horus_py/tests/test_subs_type_detection.py | 73 ++++++++++++++++++++ 7 files changed, 211 insertions(+), 15 deletions(-) create mode 100644 horus_py/tests/test_subs_type_detection.py diff --git a/horus_core/src/communication/mod.rs b/horus_core/src/communication/mod.rs index 703dbbf22..9eed155c3 100644 --- a/horus_core/src/communication/mod.rs +++ b/horus_core/src/communication/mod.rs @@ -41,8 +41,8 @@ pub use topic::{set_topic_verbose, TOPIC_VERBOSE_OFFSET}; // Ring-buffer inspector for CLI tools (`horus topic echo`) #[doc(hidden)] pub use topic::{ - read_latest_slot_bytes, read_slots_since, read_topic_header_info, read_topic_messages_total, - read_topic_sequence, TopicHeaderInfo, TopicSlotRead, + peek_topic_type_name, read_latest_slot_bytes, read_slots_since, read_topic_header_info, + read_topic_messages_total, read_topic_sequence, TopicHeaderInfo, TopicSlotRead, }; // Topic lifecycle hook for horus_net network replication diff --git a/horus_core/src/communication/topic/header.rs b/horus_core/src/communication/topic/header.rs index 4a27c2af4..d6a004a83 100644 --- a/horus_core/src/communication/topic/header.rs +++ b/horus_core/src/communication/topic/header.rs @@ -1868,6 +1868,24 @@ pub(crate) fn type_name_as_stored(name: &str) -> &str { &name[..end] } +/// The message type an existing topic's ring already holds, by topic NAME. +/// +/// Reads only the header — it never constructs a `Topic`, which is the point. +/// Asking "what type is this ring?" by opening it is circular: opening requires +/// choosing a type, and choosing the wrong one is refused by the size check in +/// `RingTopic::new` (correctly — a name-based exemption once let a +/// `Topic` join a `Topic` ring and write megabytes past +/// the end of the mapping). So a caller that wants to pick the type from the +/// ring has to read the header without attaching, and this is that. +/// +/// `None` if the topic does not exist, the name is not a legal region name, the +/// region has no valid header, or the header records no type. +pub fn peek_topic_type_name(name: &str) -> Option { + let path = horus_sys::shm::topic_shm_path_checked(name)?; + let info = read_topic_header_info(&path)?; + (!info.type_name.is_empty()).then_some(info.type_name) +} + pub fn read_topic_header_info(path: &std::path::Path) -> Option { let mmap = map_topic_region(path)?; let base = mmap.as_ptr(); diff --git a/horus_core/src/communication/topic/mod.rs b/horus_core/src/communication/topic/mod.rs index 7ac83a5ee..6e7a0167b 100644 --- a/horus_core/src/communication/topic/mod.rs +++ b/horus_core/src/communication/topic/mod.rs @@ -540,9 +540,9 @@ pub(crate) use header::{TOPIC_MAGIC, TOPIC_VERSION}; // Public debug flag API for external tools (TUI monitor) #[doc(hidden)] pub use header::{ - read_latest_slot_bytes, read_slots_since, read_topic_header_info, read_topic_messages_total, - read_topic_sequence, set_topic_verbose, TopicHeaderInfo, TopicKind, TopicSlotRead, - TOPIC_VERBOSE_OFFSET, + peek_topic_type_name, read_latest_slot_bytes, read_slots_since, read_topic_header_info, + read_topic_messages_total, read_topic_sequence, set_topic_verbose, TopicHeaderInfo, TopicKind, + TopicSlotRead, TOPIC_VERBOSE_OFFSET, }; use local_state::LocalState; pub(crate) use metrics::MigrationMetrics; diff --git a/horus_py/horus/__init__.py b/horus_py/horus/__init__.py index 3fcc25588..e2cde130b 100644 --- a/horus_py/horus/__init__.py +++ b/horus_py/horus/__init__.py @@ -96,6 +96,9 @@ def __init__(self, topic: str, msg_type=None, capacity: int = 1024): try: from horus._horus import ( Topic, # Unified communication API + # Reads an existing ring's recorded type from its SHM header without + # attaching to it — see Node._detect_topic_type. + peek_topic_type as _peek_topic_type, # Internal: Rust bridge classes (not user-facing — use horus.Scheduler wrapper instead) Scheduler as _PyScheduler, SchedulerConfig as _SchedulerConfig, @@ -254,6 +257,8 @@ def minimal(): return _SchedulerConfig() def get_version(): return "0.1.0-mock" + def _peek_topic_type(name): return None + class Image: pass @@ -725,6 +730,21 @@ def _setup_topics(self): # try to read the type from the existing SHM header if msg_type is None and topic in self.sub_topics: msg_type = self._detect_topic_type(topic) + if msg_type is None: + # The ring exists and holds a type this build cannot name. + # Falling through would open a GenericMessage topic against + # it, and the runtime's size check would refuse with a + # message about byte counts that names neither `subs=`, nor + # the type the ring actually holds, nor the fix. + ring_type = self._peek_ring_type(topic) + if ring_type: + raise RuntimeError( + f"topic {topic!r} already holds {ring_type!r} messages, " + f"but it was declared as a bare string in subs=, which " + f"means an untyped MessagePack topic. Those cannot share " + f"a name. Declare the type: " + f"subs=[Sub({topic!r}, {ring_type})]" + ) if msg_type is not None: # Typed topic — Pod zero-copy path (~1.5μs) @@ -733,24 +753,43 @@ def _setup_topics(self): # String topic — GenericMessage path (~5-50μs) self._topics[topic] = Topic(topic, capacity) + @staticmethod + def _peek_ring_type(topic_name: str): + """The type name an existing ring records, or None. Never raises.""" + try: + name = _peek_topic_type(topic_name) + except Exception: + return None + if not name or name.lower() == 'genericmessage': + return None + return name + def _detect_topic_type(self, topic_name: str): """Try to detect the topic's message type from an existing SHM header. Returns the message type class if found, None otherwise. + + This used to open ``Topic(topic_name, 64)`` and read a ``type_name`` + attribute off it. Neither half worked: ``Topic`` exposes no + ``type_name``, so the value was always ``None``; and the probe could + not have succeeded anyway, because ``Topic(name, capacity)`` builds a + ``GenericMessage`` topic and attaching that to an existing typed ring + is refused by the runtime's size check -- correctly, since a + name-based exemption once let a generic handle write past the end of a + typed mapping. So step 2 of the documented resolution order never ran + for any topic that has ever existed, and ``_TYPE_NAME_MAP`` had no + reachable reader. + + ``peek_topic_type`` reads the header without attaching, which is the + only way to ask this question without already knowing the answer. """ try: - # Create a temporary generic topic to probe the SHM header - probe = Topic(topic_name, 64) - # Check the topic's type name from the SHM header - type_name = getattr(probe, 'type_name', None) - if type_name and callable(type_name): - type_name = type_name() - if not type_name or type_name.lower() == 'genericmessage': - return None - # Look up the type by name in our known types - return _TYPE_NAME_MAP.get(type_name.lower()) + type_name = _peek_topic_type(topic_name) except Exception: return None + if not type_name or type_name.lower() == 'genericmessage': + return None + return _TYPE_NAME_MAP.get(type_name.lower()) def _ensure_topic(self, topic: str) -> None: """Ensure a Topic object exists for the given name (auto-create if needed).""" diff --git a/horus_py/src/lib.rs b/horus_py/src/lib.rs index 10aae6fee..e44768bad 100644 --- a/horus_py/src/lib.rs +++ b/horus_py/src/lib.rs @@ -117,6 +117,7 @@ fn _horus(m: &Bound<'_, PyModule>) -> PyResult<()> { )?; m.add_function(wrap_pyfunction!(get_version, m)?)?; + m.add_function(wrap_pyfunction!(topic::peek_topic_type, m)?)?; // Time API — horus.now(), horus.dt(), etc. m.add_function(wrap_pyfunction!(time_now, m)?)?; diff --git a/horus_py/src/topic.rs b/horus_py/src/topic.rs index 2c05fbb44..cf7bf9923 100644 --- a/horus_py/src/topic.rs +++ b/horus_py/src/topic.rs @@ -148,6 +148,20 @@ fn send_timeout_err( crate::errors::to_py_err(send_timeout_error(topic, elapsed, deadline)) } +/// The message type an existing topic's ring already holds, or `None`. +/// +/// Reads the SHM header only; it never constructs a `Topic`. That is the whole +/// point. Asking "what type is this ring?" by opening it is circular — opening +/// requires choosing a type, and choosing the wrong one is refused by the size +/// check in `RingTopic::new`. The previous auto-detection did exactly that: +/// `probe = Topic(name, 64)` builds a `Topic`, which the header +/// check is guaranteed to refuse on any typed ring, so the probe always raised +/// into a bare `except Exception` and the step could never succeed. +#[pyfunction] +pub fn peek_topic_type(name: &str) -> Option { + horus_core::communication::peek_topic_type_name(name) +} + /// `SendBlockingError::NoBackpressure` as a Python exception. /// /// NOT `HorusTimeoutError`. The core writes this variant specifically to be @@ -2090,6 +2104,57 @@ mod tests { ); } + /// Auto-detecting a subscription's type must not require attaching to it. + /// + /// `_setup_topics` documents a three-step resolution whose middle step is + /// "auto-detect from SHM header". It was implemented as `probe = + /// Topic(name, 64)` followed by `getattr(probe, "type_name", None)`, and + /// both halves were dead: `Topic` exposes no `type_name`, and the probe + /// builds a `GenericMessage` topic, which the runtime refuses to attach to + /// a typed ring. So the step could never succeed for any topic that has + /// ever existed, and the ~70-entry `_TYPE_NAME_MAP` built for it had no + /// reachable reader. + /// + /// The first assertion below is that refusal — it is correct and must + /// stay, since a name-based exemption once let a generic handle write + /// megabytes past the end of a typed mapping. The question just cannot be + /// answered by attaching, which is what `peek_topic_type` is for. + #[test] + fn the_ring_type_is_readable_without_attaching_to_it() { + use horus::communication::Topic as CoreTopic; + + let name = format!("peek_probe_{}", std::process::id()); + + let Ok(typed) = CoreTopic::::new(&name) else { + eprintln!("skipping: no shared memory available"); + return; + }; + typed.send(CmdVel::new(1.0, 0.0)); + + // The old probe. It cannot open the ring, so it could never report + // anything about it. + assert!( + CoreTopic::::new(&name).is_err(), + "a generic handle must still be refused on a typed ring" + ); + + // The peek reads the header instead of attaching, and answers. + let seen = peek_topic_type(&name); + assert_eq!( + seen.as_deref(), + Some("CmdVel"), + "the header records the ring's type and it must be readable" + ); + + assert_eq!( + peek_topic_type("peek_probe_no_such_topic_anywhere"), + None, + "a topic that does not exist has no type" + ); + + drop(typed); + } + /// A refusal for want of backpressure must NOT arrive as a timeout. /// /// The core writes `SendBlockingError::NoBackpressure` specifically to be diff --git a/horus_py/tests/test_subs_type_detection.py b/horus_py/tests/test_subs_type_detection.py new file mode 100644 index 000000000..189c3a501 --- /dev/null +++ b/horus_py/tests/test_subs_type_detection.py @@ -0,0 +1,73 @@ +"""A bare-string `subs=` must resolve the ring's real type, not guess MessagePack. + +`Node._setup_topics` documents a three-step type resolution whose middle step is +"auto-detect from SHM header". It was implemented as ``probe = Topic(name, 64)`` +followed by ``getattr(probe, "type_name", None)``, and both halves were dead: +``Topic`` exposes no ``type_name``, and the probe builds a GenericMessage topic, +which the runtime refuses to attach to a typed ring. So the step never ran for +any topic that has ever existed, the ~70-entry ``_TYPE_NAME_MAP`` built for it +had no reachable reader, and a bare ``subs=["scan"]`` against a typed ring was +answered with a RuntimeError about byte counts that named neither `subs=`, nor +the type the ring holds, nor the fix. +""" +import uuid + +import pytest + +import horus +from horus import CmdVel, LaserScan, Topic + + +def _unique(name): + return f"{name}_{uuid.uuid4().hex[:8]}" + + +def test_peek_reads_the_ring_type_without_attaching(): + name = _unique("peek_typed") + typed = Topic(CmdVel, endpoint=name) + typed.send(CmdVel(1.0, 0.0)) + + assert horus._peek_topic_type(name) == "CmdVel" + assert horus._peek_topic_type(_unique("never_created")) is None + + +def test_a_bare_subs_string_resolves_to_the_rings_type(): + name = _unique("auto_typed") + # A publisher creates the ring as a typed LaserScan topic. + pub = Topic(LaserScan, endpoint=name) + pub.send(LaserScan()) + + # The subscriber says only the name — no type. Step 2 must now resolve it. + node = horus.Node(name="detector", subs=[name]) + node._setup_topics() + + resolved = node._topics[name] + assert not resolved.is_generic(), ( + "a bare subs= string on an existing typed ring must resolve to that " + "type, not open an untyped MessagePack topic that cannot attach" + ) + + +def test_an_unmappable_ring_type_says_what_to_do(): + """The residual case must name the topic, the type and the fix. + + Falling through to a GenericMessage topic gets the runtime's size-mismatch + RuntimeError, which talks about byte counts and names none of those three. + """ + name = _unique("unmappable") + pub = Topic(CmdVel, endpoint=name) + pub.send(CmdVel(0.0, 0.0)) + + node = horus.Node(name="detector", subs=[name]) + # Pretend this build has no Python class for the ring's type. + saved = horus._TYPE_NAME_MAP.pop("cmdvel") + try: + with pytest.raises(RuntimeError) as excinfo: + node._setup_topics() + finally: + horus._TYPE_NAME_MAP["cmdvel"] = saved + + msg = str(excinfo.value) + assert name in msg, f"must name the topic: {msg}" + assert "CmdVel" in msg, f"must name the type the ring holds: {msg}" + assert "subs=" in msg, f"must name the fix: {msg}" From 5770e6e7f8eb8c9b8372c9d8128c5dd73261d188 Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 18:13:30 -0700 Subject: [PATCH 11/79] fix(cli): three flags that did not do what they said **`post_test` skipped teardown exactly when teardown matters.** It sat after an early `?` in the `Commands::Test` arm, so a failing `horus test` never ran it, while `post_run` and `post_build` go through `run_teardown_hooks` and run on both paths deliberately: "Skipping it when the run failed would skip it in exactly the case where the hardware was most likely left live." The hooks contract file calls `test` "the only phase with teardown" and then pinned only the symmetry of the KEY NAMES, so `post_test` kept success-only semantics while the other two were given real teardown semantics. A `post_test = ["release-bench"]` was skipped precisely when the tests aborted mid-run. The new guard reads the dispatch in main.rs rather than the table: every `post_` phase must go through `run_teardown_hooks` and none through the plain `run_hooks`. Reverting the arm fails it on both halves. **`horus doctor` recommended a variable that does nothing.** It printed "use --net or HORUS_NET=1 to enable" and reported networking from `HORUS_NET`, which nothing else in the tree reads. Following the advice changed nothing except that `doctor` then printed "horus_net: enabled", confirming a state that did not exist -- worse than inert. main.rs already recorded this exact confusion when it fixed `--net`; the diagnostic handing users the broken half of the same remedy was not fixed with it. `doctor` now asks the question the BUILD asks, through a shared `cargo_gen::net_capability_requested`, and names only routes that work. `HORUS_NET` is retired: no reader remains. **`horus install --plugin -t ` threw the name away.** The call site collapsed it to `let local = target.is_some()`, and what was left called `detect_or_select_workspace` -- which resolves the CURRENT DIRECTORY's workspace, or prints a menu and blocks on stdin. Both are things `horus install` documents it never does, and with no tty the read returns empty and the command dies with "Invalid selection". The package half of the same flag already resolved the name correctly; the bool was a vestige of a `--local` flag that no longer exists, still named in the doc-comment. The ablation shows how completely the name vanished: with the bug the install proceeds past the target entirely and fails with an unrelated "no package registry is configured". --- horus_manager/src/cargo_gen.rs | 81 ++++++++++++++++++++++---- horus_manager/src/commands/doctor.rs | 20 +++++-- horus_manager/src/commands/plugin.rs | 83 +++++++++++++++++++++------ horus_manager/src/main.rs | 40 +++++++------ horus_manager/tests/hooks_contract.rs | 59 +++++++++++++++++++ 5 files changed, 232 insertions(+), 51 deletions(-) diff --git a/horus_manager/src/cargo_gen.rs b/horus_manager/src/cargo_gen.rs index 1e5110325..f88f32047 100644 --- a/horus_manager/src/cargo_gen.rs +++ b/horus_manager/src/cargo_gen.rs @@ -987,6 +987,28 @@ fn write_implicit_deps(cargo: &mut String, manifest: &HorusManifest) { } } +/// Whether the `net` capability was asked for, by either route that works. +/// +/// This is the condition that becomes `cargo --features net`, and therefore the +/// only thing that decides whether the built binary contains horus_net at all. +/// +/// Extracted so `horus doctor` can ask the same question the build asks. +/// `doctor` used to answer it from `HORUS_NET`, which nothing else in the tree +/// reads — so it told operators to set a variable that changed nothing except +/// what `doctor` then said about it. +pub fn net_capability_requested(manifest: &HorusManifest) -> bool { + std::env::var("HORUS_ENABLE") + .map(|v| { + v.split(',') + .any(|c| matches!(c.trim().to_lowercase().as_str(), "net" | "network")) + }) + .unwrap_or(false) + || manifest + .enable + .iter() + .any(|c| matches!(c.to_lowercase().as_str(), "net" | "network")) +} + /// Features to enable on the `horus` dependency. /// /// `horus run --net` used to set `HORUS_NET=1` and stop there. That variable is @@ -1022,16 +1044,7 @@ fn horus_dep_features(manifest: &HorusManifest) -> Vec { // driver needs a feature on `horus`. let mut features: Vec = Vec::new(); - let requested_net = std::env::var("HORUS_ENABLE") - .map(|v| { - v.split(',') - .any(|c| matches!(c.trim().to_lowercase().as_str(), "net" | "network")) - }) - .unwrap_or(false) - || manifest - .enable - .iter() - .any(|c| matches!(c.to_lowercase().as_str(), "net" | "network")); + let requested_net = net_capability_requested(manifest); if requested_net && !features.iter().any(|f| f == "net") { features.push("net".to_string()); @@ -2275,6 +2288,54 @@ mod horus_dep_feature_tests { assert!(with_enable(Some("NET"), || horus_dep_features(&m)).contains(&"net".to_string())); } + /// `horus doctor` must answer from the same condition the build uses. + /// + /// It used to answer from `HORUS_NET`, which nothing in the tree reads, and + /// print "use --net or HORUS_NET=1 to enable" — advice whose second half + /// changed nothing except that `doctor` then reported networking as + /// enabled, confirming a state that did not exist. Worse than inert. + #[test] + fn horus_net_does_not_make_a_build_networked() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + let m = HorusManifest::default(); + + let prev = std::env::var("HORUS_NET").ok(); + // SAFETY: ENV_LOCK serialises the tests that touch process-global env. + unsafe { std::env::set_var("HORUS_NET", "1") }; + let answered = net_capability_requested(&m); + match prev { + Some(v) => unsafe { std::env::set_var("HORUS_NET", v) }, + None => unsafe { std::env::remove_var("HORUS_NET") }, + } + + assert!( + !answered, + "HORUS_NET=1 must not report a networked build — it reaches no \ + build step, so reporting it as enabled tells the operator a \ + falsehood they cannot act on" + ); + } + + /// ...and the routes that DO work must all answer yes. + #[test] + fn the_documented_routes_all_request_the_net_capability() { + let m = HorusManifest::default(); + // `horus run --net` appends "net" to the capability list, which becomes + // HORUS_ENABLE. + assert!(with_enable(Some("net"), || net_capability_requested(&m))); + assert!(with_enable(Some("network"), || net_capability_requested( + &m + ))); + + // `enable = ["net"]` in horus.toml. + let mut manifest = HorusManifest::default(); + manifest.enable = vec!["net".to_string()]; + assert!(with_enable(None, || net_capability_requested(&manifest))); + + // And nothing at all means no. + assert!(!with_enable(None, || net_capability_requested(&m))); + } + /// It must be opt-in: an ordinary build must not link the network stack. #[test] fn net_is_absent_by_default() { diff --git a/horus_manager/src/commands/doctor.rs b/horus_manager/src/commands/doctor.rs index f80990676..907f4f3e4 100644 --- a/horus_manager/src/commands/doctor.rs +++ b/horus_manager/src/commands/doctor.rs @@ -1803,9 +1803,18 @@ fn check_network() -> CheckResult { details.push(format!("Namespace: {}", namespace)); // Check 3: horus_net enabled check - let net_enabled = std::env::var("HORUS_NET") - .map(|v| v == "1" || v == "true") - .unwrap_or(false); + // + // Asked the way the BUILD asks it. This used to read `HORUS_NET`, which + // nothing else in the tree reads — so the advice it printed + // ("use --net or HORUS_NET=1 to enable") handed the operator a remedy that + // changed nothing except that `doctor` then reported networking as enabled, + // confirming a state that did not exist. What actually puts horus_net in + // the binary is the `net` capability reaching cargo_gen, via `enable` in + // horus.toml or `HORUS_ENABLE`; `horus run --net` works because it appends + // to that list. + let manifest = crate::manifest::HorusManifest::load_from(std::path::Path::new("horus.toml")) + .unwrap_or_default(); + let net_enabled = crate::cargo_gen::net_capability_requested(&manifest); let no_net = std::env::var("HORUS_NO_NETWORK") .map(|v| v == "1" || v == "true") .unwrap_or(false); @@ -1815,7 +1824,10 @@ fn check_network() -> CheckResult { } else if net_enabled { details.push("horus_net: enabled".to_string()); } else { - details.push("horus_net: available (use --net or HORUS_NET=1 to enable)".to_string()); + details.push( + "horus_net: available (enable it with `horus run --net`, enable = [\"net\"] in horus.toml, or HORUS_ENABLE=net)" + .to_string(), + ); } // Check 4: multicast reachability (basic socket test) diff --git a/horus_manager/src/commands/plugin.rs b/horus_manager/src/commands/plugin.rs index d1bb921f9..6ece15c1e 100644 --- a/horus_manager/src/commands/plugin.rs +++ b/horus_manager/src/commands/plugin.rs @@ -443,11 +443,17 @@ fn resolve_package_dir(name: &str, version: &str, global: bool) -> Option, local: bool) -> HorusResult<()> { - // Plugins default to global installation (CLI-wide) - let global = !local; - +/// Pass `-t ` to install into a registered workspace instead. +/// +/// `target` used to be collapsed to a `bool` at the call site — `let local = +/// target.is_some()` — and the NAME was thrown away. What was left then called +/// `detect_or_select_workspace`, which resolves the CURRENT DIRECTORY's +/// workspace or, outside any workspace, prints a menu and blocks on stdin. Both +/// of those are things `horus install` documents that it never does, and with +/// no tty the stdin read returns empty and the command fails with "Invalid +/// selection" rather than installing where it was told. The package half of the +/// same flag already resolved the name properly; this now does the same thing. +pub fn run_install(plugin: String, ver: Option, target: Option) -> HorusResult<()> { println!( "{} Installing plugin: {}{}", cli_output::ICON_INFO.magenta().bold(), @@ -456,21 +462,26 @@ pub fn run_install(plugin: String, ver: Option, local: bool) -> HorusRes ); // Determine install target - plugins default to global - let install_target = if global { - println!(" Scope: {}", "global (user cache)".dimmed()); - workspace::InstallTarget::Global - } else { - let target = workspace::detect_or_select_workspace(true) - .map_err(|e| HorusError::Config(ConfigError::Other(e.to_string())))?; - match &target { - workspace::InstallTarget::Local(p) => { - println!(" Scope: {}", format!("local ({})", p.display()).dimmed()); - } - workspace::InstallTarget::Global => { - println!(" Scope: {}", "global (user cache)".dimmed()); - } + let install_target = match target { + None => { + println!(" Scope: {}", "global (user cache)".dimmed()); + workspace::InstallTarget::Global + } + Some(target_name) => { + let registry = workspace::WorkspaceRegistry::load() + .map_err(|e| HorusError::Config(ConfigError::Other(e.to_string())))?; + let ws = registry.find_by_name(&target_name).ok_or_else(|| { + HorusError::Config(ConfigError::Other(format!( + "Workspace '{}' not found", + target_name + ))) + })?; + println!( + " Scope: {}", + format!("local ({})", ws.path.display()).dimmed() + ); + workspace::InstallTarget::Local(ws.path.clone()) } - target }; // Install via the existing package install flow. Without a registry there @@ -1423,4 +1434,38 @@ version = "2.0.0" ); } } + + /// `-t ` on a plugin install must name a REGISTERED WORKSPACE. + /// + /// It used to be collapsed to `let local = target.is_some()` at the call + /// site, throwing the name away, and what was left called + /// `detect_or_select_workspace`. That resolves the workspace of the + /// directory you happen to be standing in or, outside any workspace, prints + /// a menu and blocks on stdin — the two things `horus install` documents it + /// never does. With no tty the read comes back empty and the command dies + /// with "Invalid selection" instead of installing where it was asked. + /// + /// An unknown name must therefore fail by NAME, before any registry or + /// network work, exactly as the package half of the same flag does. + #[test] + fn a_plugin_target_names_a_workspace_not_the_current_directory() { + let err = super::run_install( + "some-plugin".to_string(), + None, + Some("no_such_workspace_xyzzy".to_string()), + ) + .expect_err("an unknown workspace name must be an error, not a prompt"); + + let msg = err.to_string(); + assert!( + msg.contains("no_such_workspace_xyzzy"), + "the error must name the workspace that was asked for, so the user \ + can see it was taken as a name rather than ignored: {msg}" + ); + assert!( + msg.contains("not found"), + "an unregistered workspace name is 'not found', not a selection \ + prompt or a silent fall back to the current directory: {msg}" + ); + } } diff --git a/horus_manager/src/main.rs b/horus_manager/src/main.rs index f5ed5e0e6..4dde54ccb 100644 --- a/horus_manager/src/main.rs +++ b/horus_manager/src/main.rs @@ -2522,17 +2522,18 @@ fn run_command(command: Commands) -> HorusResult<()> { horus_manager::error_wrapper::set_json_diagnostics(true); } - // `--net` used to set this variable and stop there. HORUS_NET is - // read in exactly one place — `horus doctor` — so the flag never - // reached the build: the `net` Cargo feature is not in `default`, - // the generated manifest carried no `features = ["net"]`, and the - // resulting binary contained no horus_net symbols at all. + // `--net` used to set a `HORUS_NET` variable and stop there. + // Nothing read it but `horus doctor`, so the flag never reached the + // build: the `net` Cargo feature is not in `default`, the generated + // manifest carried no `features = ["net"]`, and the resulting + // binary contained no horus_net symbols at all. // - // Push it through the capability list as well, which is what - // actually turns into `cargo --features`. + // The capability list is what actually turns into + // `cargo --features`, so that is the only thing to push it through. + // `HORUS_NET` is gone: it never affected a build, and `doctor` now + // reads the same capability the build does. let mut enable = enable; if net { - std::env::set_var("HORUS_NET", "1"); let list = enable.get_or_insert_with(Vec::new); if !list.iter().any(|c| c.eq_ignore_ascii_case("net")) { list.push("net".to_string()); @@ -2911,16 +2912,20 @@ fn run_command(command: Commands) -> HorusResult<()> { ), } } - result.map_err(HorusError::from)?; - - if let Some(ref manifest) = hooks_manifest { - if let Err(e) = commands::hooks::run_hooks("post_test", manifest) { - eprintln!("Hook failed: {}", e); - return Err(HorusError::from(e)); + // Teardown, on both paths — the same treatment `post_run` and + // `post_build` get. `post_test` used to sit after an early `?`, so a + // failing `horus test` skipped it entirely. That is backwards for + // the phase the hooks contract itself calls "the only phase with + // teardown": a `post_test = ["release-bench"]` exists to release + // the bench, and it was skipped precisely when the tests aborted + // mid-run and the hardware was most likely left live. + let result = result.map_err(HorusError::from); + match hooks_manifest { + Some(ref manifest) => { + commands::hooks::run_teardown_hooks("post_test", manifest, result) } + None => result, } - - Ok(()) } Commands::Monitor { args } => { @@ -3189,8 +3194,7 @@ fn run_command(command: Commands) -> HorusResult<()> { let (pkg_name, pkg_ver) = split_name_version(name, ver); let result = if plugin { - let local = target.is_some(); - commands::plugin::run_install(pkg_name.clone(), pkg_ver.clone(), local) + commands::plugin::run_install(pkg_name.clone(), pkg_ver.clone(), target) } else { commands::pkg::run_install_standalone(&pkg_name, pkg_ver.as_deref(), target) }; diff --git a/horus_manager/tests/hooks_contract.rs b/horus_manager/tests/hooks_contract.rs index 749322bfa..162569887 100644 --- a/horus_manager/tests/hooks_contract.rs +++ b/horus_manager/tests/hooks_contract.rs @@ -232,3 +232,62 @@ fn no_hooks_suppresses_the_post_hook_as_well() { "--no-hooks must disable post_run as well as pre_run:\n{combined}" ); } + +/// Symmetry of the key names is not the property that matters. +/// +/// `every_phase_has_both_a_pre_and_a_post_hook` pins the table, and the table +/// was symmetric while `post_test` still had success-only semantics: it sat +/// after an early `?` in the `Commands::Test` arm, so a failing `horus test` +/// skipped it entirely. That is backwards for the phase this file's own header +/// calls the one that had teardown — `post_test = ["release-bench"]` exists to +/// release the bench, and it was skipped precisely when the tests aborted +/// mid-run and the hardware was most likely left live. +/// +/// So this reads the dispatch itself. Every `post_` phase must go through +/// `run_teardown_hooks`, whose contract is "after a command finished, whatever +/// the outcome", and none may go through the plain `run_hooks`, which is the +/// setup form and only runs on the path that reached it. +#[test] +fn every_post_hook_is_dispatched_as_teardown() { + let main_rs = std::fs::read_to_string("src/main.rs").expect("horus_manager/src/main.rs"); + + let post_phases: Vec<&str> = KNOWN_HOOKS + .iter() + .copied() + .filter(|h| h.starts_with("post_")) + .collect(); + assert!( + !post_phases.is_empty(), + "precondition: there are post_ hooks to check" + ); + + let mut wrong = Vec::new(); + for phase in &post_phases { + let teardown = format!("run_teardown_hooks(\"{phase}\""); + let setup = format!("run_hooks(\"{phase}\""); + // These two patterns cannot overlap: the character before `hooks("` is + // `_` in `run_teardown_hooks(`, so `run_hooks("` is not a substring of + // it. Counting them independently is therefore exact. + let setup_calls = main_rs.matches(&setup).count(); + + if !main_rs.contains(&teardown) { + wrong.push(format!( + "`{phase}` is never dispatched through run_teardown_hooks, so it \ + does not run when the command failed" + )); + } + if setup_calls > 0 { + wrong.push(format!( + "`{phase}` is dispatched through the plain run_hooks in {setup_calls} \ + place(s); that form is skipped when the command failed" + )); + } + } + + assert!( + wrong.is_empty(), + "a post_ hook that does not run on the failure path is a hook that \ + skips teardown exactly when teardown matters:\n {}", + wrong.join("\n ") + ); +} From 8de022018d507e2efd48f260af5fe83e510461d8 Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 18:17:52 -0700 Subject: [PATCH 12/79] test(log): fix a race in the drain wiring test that CI caught `log_drain_loop` snapshots `GLOBAL_LOG_BUFFER.write_idx()` when its thread starts and only writes entries newer than that snapshot. The test pushed its entries in one burst immediately after constructing the scheduler, so on a loaded machine the drain thread could reach the snapshot after the burst -- leaving nothing newer to write, and the test waiting out its full 10-second window for a drain that was working correctly. Pushing every polling round instead guarantees entries after the snapshot whenever the thread got going. Still non-vacuous: reverting the wiring in `Scheduler::new` fails it exactly as before. --- horus_core/tests/log_file_drain_wiring.rs | 38 ++++++++++++++--------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/horus_core/tests/log_file_drain_wiring.rs b/horus_core/tests/log_file_drain_wiring.rs index 43f2a2a1a..d3e1a248e 100644 --- a/horus_core/tests/log_file_drain_wiring.rs +++ b/horus_core/tests/log_file_drain_wiring.rs @@ -35,23 +35,31 @@ fn setting_horus_log_file_makes_a_scheduler_write_logs_to_disk() { // here calls `start_log_file_drain`. let _scheduler = Scheduler::new(); - for i in 0..20 { - GLOBAL_LOG_BUFFER.push(LogEntry { - timestamp: format!("2026-09-04T00:00:{:02}.000Z", i), - tick_number: GLOBAL_LOG_BUFFER.write_idx() + 1, - node_name: "wiring_probe".to_string(), - log_type: LogType::Info, - topic: None, - message: format!("entry {}", i), - tick_us: 0, - ipc_ns: 0, - }); - } - - // The drain polls every 500 ms. + // Keep pushing while polling, rather than pushing once up front. + // + // `log_drain_loop` snapshots `GLOBAL_LOG_BUFFER.write_idx()` when its + // thread starts and only ever writes entries newer than that. On a loaded + // machine the thread can reach that snapshot after a one-shot burst has + // already been pushed, and then there is nothing newer to write and the + // test waits out its whole window for a drain that is working correctly. + // (CI caught exactly that.) Pushing every round guarantees entries after + // the snapshot whenever the thread got going. let log_file = dir.join("horus.log"); let mut content = String::new(); - for _ in 0..40 { + for round in 0..40 { + for i in 0..5 { + GLOBAL_LOG_BUFFER.push(LogEntry { + timestamp: format!("2026-09-04T00:00:{:02}.000Z", i), + tick_number: GLOBAL_LOG_BUFFER.write_idx() + 1, + node_name: "wiring_probe".to_string(), + log_type: LogType::Info, + topic: None, + message: format!("round {} entry {}", round, i), + tick_us: 0, + ipc_ns: 0, + }); + } + // The drain polls every 500 ms. std::thread::sleep(Duration::from_millis(250)); if let Ok(c) = std::fs::read_to_string(&log_file) { if c.contains("wiring_probe") { From 6e1d7a0119715c54600d58285917e06275aa8c42 Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 20:49:39 -0700 Subject: [PATCH 13/79] fix(topic): give a participant registration back when its handle drops Nothing inside a living process ever returned one. The only `decrement_to_floor` calls on `publisher_count`/`subscriber_count` are in `reap_dead_participants_now`, which wants an expired lease AND a dead pid, and in `register_role`'s last-resort pass, which only runs once all 16 slots are occupied. `Drop for RingTopic` released spill keep-alives and FanoutShm endpoint bits and left the participant table untouched. So within one process the counts only grew -- and `detect_optimal_backend` reads them. A transient subscriber on a fresh thread (a worker pool, a scoped task, a one-shot diagnostic read) permanently converted a command topic from backpressured `SpscShm` to overwrite-oldest `PodShm`, with nothing alive to justify it and no path back for the life of the segment. It silently voided `try_send`'s full-ring `Err` and `send_blocking`'s backpressure on an e-stop path. The table also filled toward its 16-slot limit with registrations belonging to nothing. `horus_core/tests/participant_count_leak.rs` has documented this as OPEN, with an `#[ignore]`d test, since the naive fix was built and rejected. # Why the naive fix was unsafe, and what makes this one safe Releasing against the recorded slot INDEX is wrong: a slot is not owned for a handle's lifetime. `register_role`'s pass 3 reclaims slots whose thread has ended, so a stale index names somebody else's registration by the time it is given back. `ParticipantEntry` now carries a `generation`, bumped on every claim in `finish_claim` and recorded by the handle beside the slot, so a release can tell its own registration from a later one. A second hazard the earlier attempt did not name: an entry is keyed on (pid, thread), NOT on the handle, so several `Topic`s on one thread deliberately share one slot and one increment of `subscriber_count`. A (slot, generation) match alone would let the first to drop clear an entry the others are still using. The entry therefore also carries per-role handle counts -- per role, because one thread publishing and subscribing to the same topic is the supported `role = Both` fast path, and a single count could not express "the last subscriber left while a publisher stays". Every check refuses rather than guesses. Under-counting is the dangerous direction: `nothing_is_draining` lets a producer retire unread slots once `sub_count()` reads 0, so a lost registration becomes silent message loss on a subscriber that is still reading. Both new fields came out of the entry's existing padding, so it is still 24 bytes and the header is still 640; the static assertions on both are unchanged. # The downgrade this exposes, and why it is not taken With the counts finally able to fall, a two-subscriber broadcast topic whose second subscriber exits would migrate BACK to `SpscShm` -- and a backwards migration mid-stream drops whatever is already in the wider ring, because the migrator's drain has nothing to observe and always succeeds. Measured directly: with the downgrade allowed, the SURVIVING subscriber of two received 198 of 200 messages and ended up back on `SpscShm`, and the two multithread non-POD broadcast tests failed the same way the earlier attempt made them fail. So `detect_optimal_backend` now never narrows a running topic. Widening on a join stays immediate; narrowing has to wait for a drain that is real, and until then the wider path is both the safe answer and the one the runtime already took when the counts could not fall at all. Both directions are pinned by tests. The counts themselves are correct now regardless, which is what the participant table, `horus topic info` and the 16-slot ceiling all depend on. Verified: the previously-`#[ignore]`d test passes and is un-ignored; `sub_count` climbs to [1, 2, 3] with the release removed. The two multithread broadcast tests pass 10/10 at base timings (0.12-0.18s), and the full lib suite is green across six runs. --- horus_core/src/communication/topic/header.rs | 400 +++++++++++++++--- .../src/communication/topic/local_state.rs | 6 + .../src/communication/topic/migration.rs | 16 +- horus_core/src/communication/topic/mod.rs | 40 +- horus_core/src/communication/topic/tests.rs | 46 +- horus_core/tests/participant_count_leak.rs | 128 ++++-- 6 files changed, 520 insertions(+), 116 deletions(-) diff --git a/horus_core/src/communication/topic/header.rs b/horus_core/src/communication/topic/header.rs index d6a004a83..16a09911b 100644 --- a/horus_core/src/communication/topic/header.rs +++ b/horus_core/src/communication/topic/header.rs @@ -8,7 +8,7 @@ use std::collections::HashMap; use std::mem; use std::mem::offset_of; -use std::sync::atomic::{AtomicU32, AtomicU64, AtomicU8, Ordering}; +use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8, Ordering}; use std::sync::{Mutex, OnceLock}; use crate::error::{HorusError, HorusResult}; @@ -104,8 +104,41 @@ pub(crate) struct ParticipantEntry { /// 0 = local process (default). Non-zero = low byte of remote peer_id_hash. /// Set by horus_net ShmRingWriter when writing network-received data. pub(crate) source_host: AtomicU8, - /// Padding for alignment - pub(crate) _pad: [u8; 5], + /// Padding to align `generation` on a 2-byte boundary. + pub(crate) _pad0: [u8; 1], + /// Bumped every time this slot is CLAIMED by a new owner. + /// + /// A handle records (slot, generation) when it registers, so `Drop` can + /// tell its own registration from a later one that reused the same slot. + /// Without it a release has only the slot index to go on, and + /// `register_role`'s pass 3 can hand that index to another thread while + /// the old handle is still alive — charging the drop to somebody else's + /// registration. Under-counting is the dangerous direction: + /// `nothing_is_draining` lets a producer retire unread slots once + /// `sub_count()` reads 0, so a lost registration becomes silent message + /// loss on a subscriber that is still reading. + /// + /// Carved out of `_pad`, so no existing field moves and the entry is still + /// 24 bytes. + pub(crate) generation: AtomicU16, + /// How many live handles on this thread hold the PRODUCER role here. + /// + /// `register_role` keys an entry on (pid, thread), not on the handle, so + /// several `Topic` handles on one thread deliberately share one slot and + /// one increment of `publisher_count`. The role may only be given back + /// when the LAST of them goes, which a (slot, generation) match alone + /// cannot tell: the first handle to drop would clear an entry two others + /// are still using. + /// + /// Counted PER ROLE because one thread publishing and subscribing to the + /// same topic is a supported configuration — it is the `role = Both` fast + /// path. A single count could not express "the last subscriber left while a + /// publisher stays", and would hold `subscriber_count` up for as long as + /// the publisher lived. + pub(crate) pub_handles: AtomicU8, + /// How many live handles on this thread hold the CONSUMER role here. + /// See [`Self::pub_handles`]. + pub(crate) sub_handles: AtomicU8, /// When this lease runs out, in [`current_time_ms`] milliseconds — /// MONOTONIC since boot, NOT since the Unix epoch. Written by the owning /// process and compared by any other process on the host; see @@ -140,6 +173,11 @@ impl ParticipantEntry { self.pid.store(0, Ordering::Release); self.thread_id_hash.store(0, Ordering::Release); self.source_host.store(0, Ordering::Release); + self.pub_handles.store(0, Ordering::Release); + self.sub_handles.store(0, Ordering::Release); + // `generation` is deliberately NOT reset. Its whole purpose is to + // outlive the registration it identifies, so that a handle holding a + // stale (slot, generation) can be told apart from the current owner. } } @@ -482,7 +520,10 @@ impl TopicHeader { role: AtomicU8::new(0), active: AtomicU8::new(0), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(0), }), } @@ -713,13 +754,13 @@ impl TopicHeader { } /// Register as a publisher (returns slot index or error) - pub fn register_producer(&self) -> HorusResult { - self.register_role(1, &self.publisher_count) + pub fn register_producer(&self, new_handle: bool) -> HorusResult<(usize, u16)> { + self.register_role(1, &self.publisher_count, new_handle) } /// Register as a subscriber (returns slot index or error) - pub fn register_consumer(&self) -> HorusResult { - self.register_role(2, &self.subscriber_count) + pub fn register_consumer(&self, new_handle: bool) -> HorusResult<(usize, u16)> { + self.register_role(2, &self.subscriber_count, new_handle) } /// Claim the first slot that is genuinely free (`active == 0`), if any. @@ -736,7 +777,7 @@ impl TopicHeader { thread_hash: u32, now_ms: u64, timeout_ms: u64, - ) -> Option { + ) -> Option<(usize, u16)> { for (i, p) in self.participants.iter().enumerate() { if p.active.load(Ordering::Acquire) != 0 { continue; @@ -748,8 +789,9 @@ impl TopicHeader { { continue; } - self.finish_claim(p, role_bit, counter, pid, thread_hash, now_ms, timeout_ms); - return Some(i); + let generation = + self.finish_claim(p, role_bit, counter, pid, thread_hash, now_ms, timeout_ms); + return Some((i, generation)); } None } @@ -766,7 +808,7 @@ impl TopicHeader { thread_hash: u32, now_ms: u64, timeout_ms: u64, - ) { + ) -> u16 { // The one place a thread takes ownership of a slot, so the one place to // record it as a live owner: the last-resort pass in `register_role` // refuses to reclaim an entry whose thread is still running. @@ -779,13 +821,39 @@ impl TopicHeader { self.total_participants.fetch_add(1, Ordering::AcqRel); self.last_topology_change_ms .store(now_ms, Ordering::Release); + // A new owner, so a new generation: any handle still holding the + // previous (slot, generation) must not be able to release this one. + // Wrapping is fine — a release compares for equality, and 65536 claims + // of one slot would have to happen while a single stale handle lived. + let generation = p.generation.fetch_add(1, Ordering::AcqRel).wrapping_add(1); + // Exactly one handle owns this role at this point; see + // `ParticipantEntry::pub_handles`. + if role_bit & 1 != 0 { + p.pub_handles.store(1, Ordering::Release); + p.sub_handles.store(0, Ordering::Release); + } else { + p.sub_handles.store(1, Ordering::Release); + p.pub_handles.store(0, Ordering::Release); + } // Finalize: transition from initializing (2) to active (1). // This makes the slot visible with all fields properly set. p.active.store(1, Ordering::Release); + generation } /// Shared registration logic for producer (role_bit=1) and consumer (role_bit=2). - fn register_role(&self, role_bit: u8, counter: &AtomicU32) -> HorusResult { + /// + /// `new_handle` says whether the CALLER is a handle that does not already + /// own this entry. An entry is keyed on (pid, thread), not on the handle, + /// so several `Topic`s on one thread deliberately share one slot; the + /// handle count decides when it may be released, and adding a second role + /// to a handle that already owns the entry must not bump it. + fn register_role( + &self, + role_bit: u8, + counter: &AtomicU32, + new_handle: bool, + ) -> HorusResult<(usize, u16)> { let now_ms = current_time_ms(); let pid = std::process::id(); let thread_hash = hash_thread_id(std::thread::current().id()) as u32; @@ -804,7 +872,17 @@ impl TopicHeader { .store(now_ms, Ordering::Release); } p.refresh_lease(now_ms, timeout_ms); - return Ok(i); + // Count this handle against the role it is registering, but + // only if it did not already hold that role here. + if old_role & role_bit == 0 || new_handle { + let counter = if role_bit & 1 != 0 { + &p.pub_handles + } else { + &p.sub_handles + }; + counter.fetch_add(1, Ordering::AcqRel); + } + return Ok((i, p.generation.load(Ordering::Acquire))); } } @@ -828,10 +906,10 @@ impl TopicHeader { // Pass 1: genuinely free slots only. No counter decrement: nothing was // registered here. - if let Some(i) = + if let Some(claim) = self.claim_free_slot(role_bit, counter, pid, thread_hash, now_ms, timeout_ms) { - return Ok(i); + return Ok(claim); } // Pass 2: nothing free — retire participants whose process is actually @@ -845,10 +923,10 @@ impl TopicHeader { // be a new failure introduced by an optimisation. Registration is a // startup-shaped event, so the probes are affordable exactly here. let _ = self.reap_dead_participants_now(now_ms); - if let Some(i) = + if let Some(claim) = self.claim_free_slot(role_bit, counter, pid, thread_hash, now_ms, timeout_ms) { - return Ok(i); + return Ok(claim); } // Pass 3, last resort: a stale entry belonging to a thread of THIS @@ -912,8 +990,9 @@ impl TopicHeader { decrement_to_floor(&self.subscriber_count); } - self.finish_claim(p, role_bit, counter, pid, thread_hash, now_ms, timeout_ms); - return Ok(i); + let generation = + self.finish_claim(p, role_bit, counter, pid, thread_hash, now_ms, timeout_ms); + return Ok((i, generation)); } // All 16 participant slots occupied by live processes. @@ -934,6 +1013,83 @@ impl TopicHeader { )) } + /// Give back a registration this handle owns. + /// + /// Called from `Drop for RingTopic`. Until it existed, nothing inside a + /// LIVING process ever returned a registration: the only decrements were in + /// `reap_dead_participants_now`, which wants an expired lease AND a dead + /// pid, and in `register_role`'s last-resort pass, which only runs once all + /// 16 slots are occupied. So the counts only ever grew — and + /// `detect_optimal_backend` reads them, so a transient subscriber on a + /// fresh thread (a worker pool, a scoped task, a one-shot diagnostic read) + /// permanently converted a command topic from backpressured `SpscShm` to + /// overwrite-oldest `PodShm`, with nothing alive to justify it and no path + /// back for the life of the segment. + /// + /// Releasing is only safe with unambiguous ownership, which is why this + /// takes a generation as well as a slot. A slot is NOT owned for a handle's + /// lifetime: `register_role`'s pass 3 reclaims slots whose thread has + /// ended, so an index alone can name somebody else's registration by the + /// time it is given back. Under-counting is the dangerous direction — + /// `nothing_is_draining` lets a producer retire unread slots once + /// `sub_count()` reads 0, so a lost registration becomes silent message loss + /// on a subscriber that is still reading, which is strictly worse than the + /// over-count being fixed. Every check below refuses rather than guesses. + pub(crate) fn release_participant( + &self, + slot: usize, + generation: u16, + held_producer: bool, + held_consumer: bool, + ) { + let Some(p) = self.participants.get(slot) else { + return; + }; + // Someone else's registration now, or already gone. + if p.active.load(Ordering::Acquire) != 1 + || p.generation.load(Ordering::Acquire) != generation + || p.pid.load(Ordering::Acquire) != std::process::id() + { + return; + } + + // Give back each role this handle actually held, and only when it was + // the last handle on this thread holding it. `checked_sub` refuses on a + // count already at zero — a double release — rather than wrapping and + // decrementing a shared counter twice. + let mut changed = false; + if held_producer + && p.pub_handles + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |n| n.checked_sub(1)) + == Ok(1) + { + p.role.fetch_and(!1, Ordering::AcqRel); + decrement_to_floor(&self.publisher_count); + changed = true; + } + if held_consumer + && p.sub_handles + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |n| n.checked_sub(1)) + == Ok(1) + { + p.role.fetch_and(!2, Ordering::AcqRel); + decrement_to_floor(&self.subscriber_count); + changed = true; + } + if !changed { + return; + } + self.last_topology_change_ms + .store(current_time_ms(), Ordering::Release); + + // The slot itself is only free once no handle holds either role on it. + if p.pub_handles.load(Ordering::Acquire) == 0 && p.sub_handles.load(Ordering::Acquire) == 0 + { + decrement_to_floor(&self.total_participants); + p.clear(); + } + } + /// Drop the registrations of participants whose process no longer exists. /// /// `subscriber_count` and `publisher_count` had exactly one decrement in @@ -1237,6 +1393,34 @@ impl TopicHeader { /// is purely a function of topology (producer/consumer counts) and whether /// the message type is POD. pub fn detect_optimal_backend(&self) -> BackendMode { + let desired = self.topology_backend(); + // Never NARROW a topic that is already running wider. + // + // Until participant registrations were given back on `Drop`, the counts + // could only grow, so this could not happen and nothing guarded it. + // Now that a departing subscriber really does decrement `sub_count`, a + // two-subscriber broadcast topic whose second subscriber exits would + // otherwise migrate back to `SpscShm` — and a backwards migration in + // the middle of a stream loses the messages already in the wider ring, + // because the migrator's drain does not actually drain: it has nothing + // to observe and always succeeds. Observed directly while building + // this: with the downgrade allowed, the SURVIVING subscriber of two + // received 198 of 200 messages and ended up back on `SpscShm` + // mid-stream. + // + // Widening on a join is safe and stays immediate. Narrowing has to wait + // for a drain that is real; until then the safe answer is the wider + // path, which is also the one the runtime took before the counts could + // fall at all. + let current = self.mode(); + if backend_width(current) > backend_width(desired) { + return current; + } + desired + } + + /// The backend this topology alone would choose, ignoring what is running. + fn topology_backend(&self) -> BackendMode { let pubs = self.pub_count(); let subs = self.sub_count(); let is_pod = self.is_pod_type(); @@ -1265,6 +1449,21 @@ impl TopicHeader { } } +/// How wide a backend is, for the "never narrow a running topic" rule in +/// [`TopicHeader::detect_optimal_backend`]. +/// +/// Only the ordering matters, not the numbers: a broadcast backend carries +/// every message to every subscriber, a multi-producer ring admits several +/// writers, and `SpscShm` is the narrowest real path. +fn backend_width(mode: BackendMode) -> u8 { + match mode { + BackendMode::Unknown => 0, + BackendMode::SpscShm => 1, + BackendMode::MpscShm | BackendMode::SpmcShm => 2, + BackendMode::PodShm | BackendMode::FanoutShm => 3, + } +} + // ============================================================================ // Public debug flag API (for external tools like the TUI monitor) // ============================================================================ @@ -2388,7 +2587,10 @@ mod tests { role: AtomicU8::new(0), active: AtomicU8::new(0), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(0), }; assert!(p.is_lease_expired(1000)); @@ -2402,7 +2604,10 @@ mod tests { role: AtomicU8::new(0), active: AtomicU8::new(0), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(5000), }; assert!(!p.is_lease_expired(4999)); @@ -2416,7 +2621,10 @@ mod tests { role: AtomicU8::new(0), active: AtomicU8::new(0), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(5000), }; assert!(p.is_lease_expired(5001)); @@ -2430,7 +2638,10 @@ mod tests { role: AtomicU8::new(0), active: AtomicU8::new(0), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(5000), }; // now_ms == expires → not expired (> check, not >=) @@ -2445,7 +2656,10 @@ mod tests { role: AtomicU8::new(0), active: AtomicU8::new(0), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(0), }; p.refresh_lease(1000, 5000); @@ -2462,7 +2676,10 @@ mod tests { role: AtomicU8::new(3), active: AtomicU8::new(1), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(99999), }; p.clear(); @@ -2475,10 +2692,70 @@ mod tests { // ── Registration ──────────────────────────────────────────────────── + /// A topic already running a wider backend must not narrow when a + /// participant leaves. + /// + /// Before registrations were given back on `Drop`, the counts could only + /// grow, so nothing could narrow and nothing guarded it. Now a departing + /// subscriber really does decrement `sub_count`, and a backwards migration + /// mid-stream loses whatever is already in the wider ring — the migrator's + /// drain has nothing to observe and always succeeds. Measured while + /// building this: with the downgrade allowed, the surviving subscriber of + /// two received 198 of 200 messages and ended up back on `SpscShm`. + #[test] + fn a_running_topic_never_narrows_when_a_participant_leaves() { + let h = make_header(8, 8, true, 16); + h.publisher_count.store(1, Ordering::Release); + h.subscriber_count.store(2, Ordering::Release); + + let wide = h.detect_optimal_backend(); + assert_eq!( + wide, + BackendMode::PodShm, + "two subscribers on a POD topic is a broadcast topology" + ); + // Publish it, the way a migration would. + h.backend_mode.store(wide as u8, Ordering::Release); + + // One subscriber leaves. The topology alone would now say SpscShm. + h.subscriber_count.store(1, Ordering::Release); + assert_eq!( + h.topology_backend(), + BackendMode::SpscShm, + "precondition: the topology really has narrowed, so this test is \ + not asserting a no-op" + ); + assert_eq!( + h.detect_optimal_backend(), + BackendMode::PodShm, + "the running topic must stay on the wider backend — narrowing \ + mid-stream drops the messages already in the wider ring for the \ + subscriber that is still reading" + ); + } + + /// Widening on a join stays immediate — the rule is one-directional. + #[test] + fn a_running_topic_still_widens_when_a_participant_joins() { + let h = make_header(8, 8, true, 16); + h.publisher_count.store(1, Ordering::Release); + h.subscriber_count.store(1, Ordering::Release); + let narrow = h.detect_optimal_backend(); + assert_eq!(narrow, BackendMode::SpscShm); + h.backend_mode.store(narrow as u8, Ordering::Release); + + h.subscriber_count.store(2, Ordering::Release); + assert_eq!( + h.detect_optimal_backend(), + BackendMode::PodShm, + "a second subscriber must widen the path immediately" + ); + } + #[test] fn register_producer_increments_pub_count() { let h = make_header(8, 8, true, 16); - let slot = h.register_producer().unwrap(); + let slot = h.register_producer(true).unwrap().0; assert!(slot < MAX_PARTICIPANTS); assert_eq!(h.pub_count(), 1); assert_eq!(h.sub_count(), 0); @@ -2488,7 +2765,7 @@ mod tests { #[test] fn register_consumer_increments_sub_count() { let h = make_header(8, 8, true, 16); - let slot = h.register_consumer().unwrap(); + let slot = h.register_consumer(true).unwrap().0; assert!(slot < MAX_PARTICIPANTS); assert_eq!(h.sub_count(), 1); assert_eq!(h.pub_count(), 0); @@ -2786,12 +3063,15 @@ mod tests { } assert_eq!(h.sub_count(), MAX_PARTICIPANTS as u32); - let slot = h.register_producer().expect( - "registration must sweep unconditionally: every slot is held by a \ + let slot = h + .register_producer(true) + .expect( + "registration must sweep unconditionally: every slot is held by a \ process that no longer exists, and refusing the node's start \ because another process swept the table moments ago would be a \ failure the throttle invented", - ); + ) + .0; assert!(slot < MAX_PARTICIPANTS); assert_eq!( h.pub_count(), @@ -2929,8 +3209,8 @@ mod tests { #[test] fn register_same_thread_reuses_slot() { let h = make_header(8, 8, true, 16); - let slot1 = h.register_producer().unwrap(); - let slot2 = h.register_consumer().unwrap(); + let slot1 = h.register_producer(true).unwrap().0; + let slot2 = h.register_consumer(true).unwrap().0; assert_eq!( slot1, slot2, "Same thread should reuse same participant slot" @@ -2945,8 +3225,8 @@ mod tests { #[test] fn register_producer_twice_same_thread_no_double_count() { let h = make_header(8, 8, true, 16); - let s1 = h.register_producer().unwrap(); - let s2 = h.register_producer().unwrap(); + let s1 = h.register_producer(true).unwrap().0; + let s2 = h.register_producer(true).unwrap().0; assert_eq!(s1, s2); assert_eq!( h.pub_count(), @@ -2958,14 +3238,14 @@ mod tests { #[test] fn register_from_different_threads_uses_different_slots() { let h = make_header(8, 8, true, 16); - let slot1 = h.register_producer().unwrap(); + let slot1 = h.register_producer(true).unwrap().0; let header_ptr = &h as *const TopicHeader as usize; let slot2 = std::thread::spawn(move || { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives this thread (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_producer().unwrap() + h.register_producer(true).unwrap().0 }) .join() .unwrap(); @@ -2987,7 +3267,7 @@ mod tests { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives all spawned threads (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_producer().unwrap() + h.register_producer(true).unwrap().0 }) }) .collect(); @@ -3015,7 +3295,7 @@ mod tests { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives all spawned threads (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_producer().unwrap() + h.register_producer(true).unwrap().0 }) }) .collect(); @@ -3028,7 +3308,7 @@ mod tests { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives this thread (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_producer() + h.register_producer(true) }) .join() .unwrap(); @@ -3067,7 +3347,10 @@ mod tests { // is no doubt it is still there. let header = &h; s.spawn(move || { - let slot = header.register_consumer().expect("subscriber registers"); + let slot = header + .register_consumer(true) + .expect("subscriber registers") + .0; let hash = hash_thread_id(std::thread::current().id()) as u32; registered_tx.send((slot, hash)).unwrap(); release_rx.recv().ok(); @@ -3090,7 +3373,7 @@ mod tests { } } - let result = h.register_producer(); + let result = h.register_producer(true); assert!( result.is_err(), @@ -3169,7 +3452,7 @@ mod tests { #[test] fn is_same_process_true_with_local_participants() { let h = make_header(8, 8, true, 16); - h.register_producer().unwrap(); + h.register_producer(true).unwrap(); assert!(h.is_same_process()); } @@ -3226,8 +3509,8 @@ mod tests { #[test] fn detect_optimal_backend_1p1c_pod_is_spsc_shm() { let h = make_inmem_header(8, 8, true, 16); - h.register_producer().unwrap(); - h.register_consumer().unwrap(); + h.register_producer(true).unwrap(); + h.register_consumer(true).unwrap(); // Every topic is SHM-backed: 1P:1C → SpscShm. assert_eq!(h.detect_optimal_backend(), BackendMode::SpscShm); } @@ -3235,15 +3518,15 @@ mod tests { #[test] fn detect_optimal_backend_1p1c_non_pod_is_spsc_shm() { let h = make_inmem_header(8, 8, false, 16); - h.register_producer().unwrap(); - h.register_consumer().unwrap(); + h.register_producer(true).unwrap(); + h.register_consumer(true).unwrap(); assert_eq!(h.detect_optimal_backend(), BackendMode::SpscShm); } #[test] fn detect_optimal_backend_single_producer_only() { let h = make_inmem_header(8, 8, true, 16); - h.register_producer().unwrap(); + h.register_producer(true).unwrap(); // 1P, 0C → SpscShm (anticipating single consumer) assert_eq!(h.detect_optimal_backend(), BackendMode::SpscShm); } @@ -3251,20 +3534,20 @@ mod tests { #[test] fn detect_optimal_backend_single_consumer_only() { let h = make_inmem_header(8, 8, true, 16); - h.register_consumer().unwrap(); + h.register_consumer(true).unwrap(); assert_eq!(h.detect_optimal_backend(), BackendMode::SpscShm); } #[test] fn detect_optimal_backend_cross_thread_spsc_shm() { let h = make_inmem_header(8, 8, true, 16); - h.register_producer().unwrap(); + h.register_producer(true).unwrap(); let header_ptr = &h as *const TopicHeader as usize; std::thread::spawn(move || { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives this thread (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_consumer().unwrap(); + h.register_consumer(true).unwrap(); }) .join() .unwrap(); @@ -3275,7 +3558,7 @@ mod tests { #[test] fn detect_optimal_backend_1p_multi_c_pod_is_pod_shm() { let h = make_inmem_header(8, 8, true, 16); - h.register_producer().unwrap(); + h.register_producer(true).unwrap(); let header_ptr = &h as *const TopicHeader as usize; // Register 2 consumers from different threads @@ -3284,7 +3567,7 @@ mod tests { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives this thread (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_consumer().unwrap(); + h.register_consumer(true).unwrap(); }) .join() .unwrap(); @@ -3299,7 +3582,7 @@ mod tests { #[test] fn detect_optimal_backend_multi_p_1c_is_mpsc_shm() { let h = make_inmem_header(8, 8, true, 16); - h.register_consumer().unwrap(); + h.register_consumer(true).unwrap(); let header_ptr = &h as *const TopicHeader as usize; // Register 2 producers from different threads @@ -3308,7 +3591,7 @@ mod tests { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives this thread (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_producer().unwrap(); + h.register_producer(true).unwrap(); }) .join() .unwrap(); @@ -3520,7 +3803,7 @@ mod tests { // subscriber. Registration now takes a genuinely FREE slot first and // leaves the expired entry alone; an expired slot is considered only // when no free one remains, and then only if its owner is really gone. - let slot = h.register_producer().unwrap(); + let slot = h.register_producer(true).unwrap().0; assert_ne!( slot, 0, "must not evict an expired slot while others are free" @@ -3561,7 +3844,10 @@ mod tests { role: AtomicU8::new(0), active: AtomicU8::new(0), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(0), }; // Realistic large timestamp (year ~2100 in ms) + large timeout diff --git a/horus_core/src/communication/topic/local_state.rs b/horus_core/src/communication/topic/local_state.rs index 29275ff98..22b1a8e9c 100644 --- a/horus_core/src/communication/topic/local_state.rs +++ b/horus_core/src/communication/topic/local_state.rs @@ -106,6 +106,11 @@ pub(crate) struct LocalState { /// Our slot index in the participant array (-1 if not registered) pub slot_index: i32, + /// Generation of the participant entry at `slot_index` when this handle + /// claimed it. Together they name a registration unambiguously, so `Drop` + /// can give back its own and never a later one that reused the slot. + pub participant_generation: u16, + /// Slot size for serialized messages (non-POD) pub slot_size: usize, @@ -254,6 +259,7 @@ impl Default for LocalState { cached_seq_ptr: std::ptr::null_mut(), cached_colo_stride: 0, slot_index: -1, + participant_generation: 0, slot_size: DEFAULT_SLOT_SIZE, cached_epoch: 0, fanout_shm_pub_id: None, diff --git a/horus_core/src/communication/topic/migration.rs b/horus_core/src/communication/topic/migration.rs index 11ce4886d..fbe14525c 100644 --- a/horus_core/src/communication/topic/migration.rs +++ b/horus_core/src/communication/topic/migration.rs @@ -468,8 +468,8 @@ mod tests { #[test] fn migrate_to_optimal_with_same_thread_pod_producer_consumer() { let h = make_header(true); - h.register_producer().unwrap(); - h.register_consumer().unwrap(); + h.register_producer(true).unwrap(); + h.register_consumer(true).unwrap(); let m = BackendMigrator::new(&h); // POD, 1P:1C → SpscShm (every topic is SHM-backed) let result = m.migrate_to_optimal(); @@ -480,8 +480,8 @@ mod tests { #[test] fn migrate_to_optimal_with_same_thread_non_pod() { let h = make_header(false); - h.register_producer().unwrap(); - h.register_consumer().unwrap(); + h.register_producer(true).unwrap(); + h.register_consumer(true).unwrap(); let m = BackendMigrator::new(&h); // Same thread, non-POD, 1P:1C → SpscShm (real topics are shm_backed // regardless of pod-ness; SHM supports serde via send_shm_sp_serde). @@ -503,8 +503,8 @@ mod tests { #[test] fn is_optimal_false_after_topology_change() { let h = make_header(true); - h.register_producer().unwrap(); - h.register_consumer().unwrap(); + h.register_producer(true).unwrap(); + h.register_consumer(true).unwrap(); // Optimal is SpscShm but current is still Unknown let m = BackendMigrator::new(&h); assert!(!m.is_optimal()); @@ -513,8 +513,8 @@ mod tests { #[test] fn is_optimal_true_after_migration() { let h = make_header(true); - h.register_producer().unwrap(); - h.register_consumer().unwrap(); + h.register_producer(true).unwrap(); + h.register_consumer(true).unwrap(); let m = BackendMigrator::new(&h); m.migrate_to_optimal(); assert!(m.is_optimal()); diff --git a/horus_core/src/communication/topic/mod.rs b/horus_core/src/communication/topic/mod.rs index 6e7a0167b..01124d6ba 100644 --- a/horus_core/src/communication/topic/mod.rs +++ b/horus_core/src/communication/topic/mod.rs @@ -1727,13 +1727,19 @@ impl RingTopic< } let header = self.header(); - let slot = if is_producer { - header.register_producer()? + // A handle that has no participant entry yet is a NEW sharer of + // whichever entry it lands on; one that already has a slot is only + // adding a second role to the entry it owns, and must not be counted + // twice — see `ParticipantEntry::handles`. + let new_handle = local.slot_index < 0; + let (slot, generation) = if is_producer { + header.register_producer(new_handle)? } else { - header.register_consumer()? + header.register_consumer(new_handle)? }; local.slot_index = slot as i32; + local.participant_generation = generation; local.cached_header_ptr = self.storage.as_ptr() as *const TopicHeader; local.cached_epoch = header.migration_epoch.load(Ordering::Acquire); // Sync BEFORE deriving the data pointer, and derive it from the synced @@ -3739,6 +3745,34 @@ impl Drop for RingTopic { local.fanout_sub_lock.take(); } + // Give the participant registration back. + // + // Nothing inside a living process used to do this, so the counts only + // ever grew — and `detect_optimal_backend` reads them, so one transient + // subscriber on a fresh thread permanently converted a backpressured + // command topic to overwrite-oldest, for the life of the segment. + // + // `release_participant` refuses unless the (slot, generation) still + // names THIS registration and this is the last handle sharing it. It + // has to: a slot is not owned for a handle's lifetime, and + // under-counting is the dangerous direction. + if local.slot_index >= 0 { + // `Drop` carries no bounds on `T`, so `header()` (which does) is + // not callable here. Same mapping, same invariant: storage was + // sized and aligned for a `TopicHeader` in the constructor. + let header = unsafe { &*(self.storage.as_ptr() as *const header::TopicHeader) }; + // Give back only the roles this handle actually registered — one + // thread can hold a publisher and a subscriber handle on the same + // topic (that is the role=Both fast path), and they share an entry. + header.release_participant( + local.slot_index as usize, + local.participant_generation, + local.role.can_send(), + local.role.can_recv(), + ); + local.slot_index = -1; + } + // Notify network layer (horus_net) that this topic instance is dropped notify_topic_lifecycle(TopicLifecycleEvent::Dropped { name: self.name.clone(), diff --git a/horus_core/src/communication/topic/tests.rs b/horus_core/src/communication/topic/tests.rs index 3c56a373e..c4554a1cc 100644 --- a/horus_core/src/communication/topic/tests.rs +++ b/horus_core/src/communication/topic/tests.rs @@ -15,7 +15,7 @@ use super::header::ParticipantEntry; use super::*; use std::mem; -use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicU8, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU16, AtomicU32, AtomicU64, AtomicU8, Ordering}; use std::sync::{Arc, Barrier}; use std::thread; use std::time::{Duration, Instant}; @@ -4176,7 +4176,7 @@ fn detect_backend_cross_process_multi_p_0c_mpsc_shm() { fn register_producer_returns_slot_index() { let mut h = TopicHeader::zeroed(); h.init(8, 4, true, 64, 8, "TestType", 0); - let slot = h.register_producer().expect("should succeed"); + let slot = h.register_producer(true).expect("should succeed").0; assert!(slot < 16, "Slot index must be < MAX_PARTICIPANTS"); assert_eq!(h.pub_count(), 1); assert_eq!(h.sub_count(), 0); @@ -4186,7 +4186,7 @@ fn register_producer_returns_slot_index() { fn register_consumer_returns_slot_index() { let mut h = TopicHeader::zeroed(); h.init(8, 4, true, 64, 8, "TestType", 0); - let slot = h.register_consumer().expect("should succeed"); + let slot = h.register_consumer(true).expect("should succeed").0; assert!(slot < 16); assert_eq!(h.sub_count(), 1); assert_eq!(h.pub_count(), 0); @@ -4196,8 +4196,8 @@ fn register_consumer_returns_slot_index() { fn register_both_roles_same_thread_reuses_slot() { let mut h = TopicHeader::zeroed(); h.init(8, 4, true, 64, 8, "TestType", 0); - let slot_pub = h.register_producer().expect("producer"); - let slot_sub = h.register_consumer().expect("consumer"); + let slot_pub = h.register_producer(true).expect("producer").0; + let slot_sub = h.register_consumer(true).expect("consumer").0; // Same thread should reuse the same participant entry assert_eq!( slot_pub, slot_sub, @@ -4229,7 +4229,9 @@ fn register_16_producers_fills_all_slots() { // Now try registering from our actual thread — should fail since all slots taken // (our thread hash won't match any of the fake ones) // But our PID matches so it may find us by PID... let's use a distinct thread - let result = test_spawn(move || h.register_producer()).join().unwrap(); + let result = test_spawn(move || h.register_producer(true)) + .join() + .unwrap(); assert!( result.is_err(), @@ -4249,7 +4251,10 @@ fn participant_entry_lease_not_expired_when_fresh() { role: AtomicU8::new(1), active: AtomicU8::new(1), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(current_time_ms() + 60_000), }; assert!(!p.is_lease_expired(current_time_ms())); @@ -4263,7 +4268,10 @@ fn participant_entry_lease_expired_after_timeout() { role: AtomicU8::new(1), active: AtomicU8::new(1), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(current_time_ms().saturating_sub(1000)), }; assert!(p.is_lease_expired(current_time_ms())); @@ -4277,7 +4285,10 @@ fn participant_entry_zero_lease_is_expired() { role: AtomicU8::new(1), active: AtomicU8::new(1), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(0), }; assert!(p.is_lease_expired(current_time_ms())); @@ -4291,7 +4302,10 @@ fn participant_entry_refresh_extends_lease() { role: AtomicU8::new(1), active: AtomicU8::new(1), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(current_time_ms().saturating_sub(1000)), }; assert!(p.is_lease_expired(current_time_ms())); @@ -4326,8 +4340,9 @@ fn expired_slot_reclaimed_by_new_registration() { // expired, and evicting it silently deregistered working publishers and // subscribers. A new registration must take a free slot while one exists. let slot = h - .register_producer() - .expect("a free slot is available, so registration must succeed"); + .register_producer(true) + .expect("a free slot is available, so registration must succeed") + .0; assert_ne!(slot, 0, "must not evict slot 0 while free slots remain"); assert_eq!( p.pid.load(Ordering::Acquire), @@ -4386,7 +4401,7 @@ fn concurrent_reclaim_exactly_one_winner() { // shared read-access from multiple threads is sound. let h = unsafe { &*(hptr as *const TopicHeader) }; b.wait(); - if h.register_producer().is_ok() { + if h.register_producer(true).is_ok() { sc.fetch_add(1, Ordering::Relaxed); } }) @@ -4415,7 +4430,10 @@ fn participant_clear_resets_all_fields() { role: AtomicU8::new(3), active: AtomicU8::new(1), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(999999), }; diff --git a/horus_core/tests/participant_count_leak.rs b/horus_core/tests/participant_count_leak.rs index becbe5eb3..e51a1ef94 100644 --- a/horus_core/tests/participant_count_leak.rs +++ b/horus_core/tests/participant_count_leak.rs @@ -1,13 +1,10 @@ -//! OPEN: participant counts only ever grow within a process, and the backend is -//! chosen from them. +//! A registration is given back when its handle is dropped. //! -//! `Drop for RingTopic` (`communication/topic/mod.rs`) releases FanoutShm -//! endpoint bits and spill keep-alives and leaves the participant table -//! untouched. The only `decrement_to_floor` calls on `publisher_count` / -//! `subscriber_count` are in `reap_dead_participants_now`, which refuses to reap -//! on an expired lease alone (an idle subscriber spends much of its life looking -//! expired) and otherwise wants a dead pid. Within one living process, nothing -//! ever gives a registration back. +//! It was not, and nothing else inside a living process did it either. The only +//! `decrement_to_floor` calls on `publisher_count`/`subscriber_count` are in +//! `reap_dead_participants_now`, which wants an expired lease AND a dead pid, +//! and in `register_role`'s last-resort pass, which only runs once all 16 slots +//! are occupied. So within one process the counts only ever grew. //! //! That is not a bookkeeping wart, because `detect_optimal_backend` //! (`topic/header.rs`) selects the backend from those counts: @@ -16,28 +13,25 @@ //! subs > 1 && is_pod -> PodShm (broadcast; overwrites the oldest) //! //! So a transient subscriber on a fresh thread -- a worker pool, a scoped task, -//! a one-shot diagnostic read -- permanently converts a command topic from +//! a one-shot diagnostic read -- permanently converted a command topic from //! backpressured to overwrite-oldest, with nothing alive to justify it and no -//! path back. The test below observes exactly that: `sub_count()` climbs 1, 2, 3 -//! across three threads that are each fully joined before the next starts, and -//! the backend flips at the second one. +//! path back for the life of the shared-memory segment. It silently voided +//! `try_send`'s full-ring `Err` and `send_blocking`'s backpressure on an e-stop +//! path. //! -//! # Why this is `#[ignore]` rather than fixed +//! # Why the obvious fix was not enough //! -//! The obvious fix -- deregister in `Drop` -- was implemented and measured, and -//! it is NOT safe as written. A handle records its participant slot index, but a -//! slot is not owned for the handle's lifetime: `register_role`'s later passes -//! reclaim lease-expired slots, and a lease looks expired whenever its owner has -//! been quiet, so a live handle's slot can be taken by another thread. Releasing +//! Deregistering in `Drop` against the recorded slot INDEX was implemented and +//! measured, and it was not safe: a slot is not owned for a handle's lifetime. +//! `register_role`'s pass 3 reclaims slots whose thread has ended, so releasing //! against a stale index charges the drop to somebody else's registration. -//! Guarding it with the entry's (pid, thread-id hash) narrows the window but does -//! not close it: thread ids are reused after a thread exits, and the hash is a -//! u32 that can collide. +//! Guarding with (pid, thread-id hash) narrows the window without closing it -- +//! thread ids are reused, and the hash is a u32 that can collide. //! -//! Measured, interleaved against unmodified main on the same machine and load, -//! three pairs of the full `horus_core` lib suite: +//! Measured then, interleaved against unmodified main on the same machine and +//! load, three pairs of the full `horus_core` lib suite: //! -//! main: 0, 0, 0 failures with the Drop fix: 0, 3, 2 failures +//! main: 0, 0, 0 failures with the naive Drop fix: 0, 3, 2 failures //! //! The failures were `multithread_nonpod_subscribers_each_get_a_contiguous_accounted_stream` //! and `multithread_nonpod_lapped_stream_stays_ordered` -- a subscriber whose @@ -49,17 +43,22 @@ //! registration becomes silent message loss on a subscriber that is still //! reading -- strictly worse than the over-count being fixed. //! -//! A correct fix needs unambiguous slot ownership: a generation counter on -//! `ParticipantEntry`, bumped on every claim and recorded by the handle, so a -//! release can tell its own registration from a later one in the same slot. The -//! entry has 3 bytes of padding left after `source_host`, which is room for a -//! `u16` generation. That is the shape; it needs building and testing properly, -//! against a machine that is not also running the rest of this suite. +//! # What makes it safe now +//! +//! `ParticipantEntry` carries a `generation`, bumped on every claim in +//! `finish_claim` and recorded by the handle beside the slot index, so a +//! release can tell its own registration from a later one in the same slot. It +//! also carries a `handles` count, because an entry is keyed on (pid, thread) +//! and several `Topic`s on one thread deliberately share one: the first to drop +//! must not clear an entry the others are still using. `release_participant` +//! refuses -- rather than guesses -- on every mismatch. +//! +//! Both fields came out of the entry's existing padding, so it is still 24 +//! bytes and the 640-byte header is unchanged. use horus_core::communication::Topic; #[test] -#[ignore = "documents an open defect: see the module comment. Run with --ignored."] fn transient_subscribers_permanently_flip_a_topic_to_a_lossy_backend() { let name = "leak_transient_subs"; let producer: Topic = Topic::new(name).expect("producer"); @@ -79,8 +78,8 @@ fn transient_subscribers_permanently_flip_a_topic_to_a_lossy_backend() { observed.push((producer.sub_count(), producer.backend_name().to_string())); } - // What SHOULD hold, and does not: no subscriber is alive at any of these - // points, so the count should be 0 or 1 and the backend should stay SpscShm. + // No subscriber is alive at any of these points, so the count is 0 or 1 + // and the backend stays SpscShm. assert!( observed.iter().all(|(c, _)| *c <= 1), "sub_count climbed to {:?} with no live subscriber at any sample point", @@ -91,3 +90,64 @@ fn transient_subscribers_permanently_flip_a_topic_to_a_lossy_backend() { "a backpressured command topic became lossy with no live subscriber: {observed:?}" ); } + +/// Several handles on ONE thread share a single registration, and the first to +/// go must not take it away from the others. +/// +/// `register_role` keys an entry on (pid, thread), not on the handle: a second +/// `Topic` on the same thread finds the existing entry, ORs in its role bit and +/// deliberately does NOT increment `subscriber_count` again. So a release that +/// matched only on (slot, generation) would clear an entry two other handles +/// were still using — and under-counting is the dangerous direction, because +/// `nothing_is_draining` lets a producer retire unread slots once `sub_count()` +/// reads 0. That is silent message loss on a subscriber that is still reading, +/// strictly worse than the over-count being fixed. +#[test] +fn one_handle_leaving_does_not_deregister_its_siblings_on_the_same_thread() { + let name = "leak_shared_entry"; + let producer: Topic = Topic::new(name).expect("producer"); + producer.send(1u64); + + let sub_a: Topic = Topic::new(name).expect("sub a"); + let sub_b: Topic = Topic::new(name).expect("sub b"); + let _ = sub_a.try_recv(); + let _ = sub_b.try_recv(); + + assert_eq!( + producer.sub_count(), + 1, + "precondition: two handles on one thread share one registration" + ); + + drop(sub_a); + assert_eq!( + producer.sub_count(), + 1, + "sub_b is still reading, so the registration must survive its \ + sibling's departure" + ); + + // And it really can still read. + producer.send(7u64); + let mut seen = None; + for _ in 0..1000 { + if let Some(v) = sub_b.try_recv() { + seen = Some(v); + if seen == Some(7) { + break; + } + } + } + assert_eq!( + seen, + Some(7), + "the surviving handle must still receive after its sibling dropped" + ); + + drop(sub_b); + assert_eq!( + producer.sub_count(), + 0, + "the last handle to go gives the registration back" + ); +} From d0fa1dfd2652246b3578e7c9a03d972da69bc069 Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 20:58:45 -0700 Subject: [PATCH 14/79] fix(topic): stop advertising a migration drain that is not performed `BackendMigrator` promised "Drain logic to ensure no in-flight message loss", and `perform_migration` said it "Drains in-flight operations before switching the backend mode. If the first drain attempt times out, retries once before failing." None of that is true. `drain_in_flight`'s own comment says why: "Nothing in `TopicHeader` counts in-flight operations -- there is no such field -- so this function has never had anything to observe. A wedged producer and an idle topic run the same number of iterations and both return `true`." It spins, yields three times, and returns `true` on every exit. So the retry and the `MigrationResult::Failed` it returned were unreachable code guarding a return value that no longer exists -- and callers wrote handlers for a case that could not occur, in mod.rs and in the test suite. The variant is gone with them. `drain_in_flight` becomes `settle_before_switch` and returns nothing, which is what it always did. The bound stays: that part was legitimate, and removing it made `send_blocking` block measurably longer under load. The guarantee itself is NOT restored here. A real one needs an in-flight counter incremented and decremented on the send and recv hot paths of a real-time framework -- a deliberate design decision with a measurable cost, which the existing comment argues should not be reintroduced by leaving a deadline around that only looks like it does the job. The doc-comments now say plainly that a message in flight across a switch can be lost; the topic's own tests already pre-warm backends "to avoid migration losses". This is also what the never-narrow rule added alongside the participant release depends on: narrowing a running topic is unsafe precisely because this drain is not real. Two tests pin what is actually true: the settle stays bounded, and an uncontended migration always completes. --- .../src/communication/topic/migration.rs | 104 ++++++++++++------ horus_core/src/communication/topic/mod.rs | 4 - horus_core/src/communication/topic/tests.rs | 4 - 3 files changed, 73 insertions(+), 39 deletions(-) diff --git a/horus_core/src/communication/topic/migration.rs b/horus_core/src/communication/topic/migration.rs index fbe14525c..89534336e 100644 --- a/horus_core/src/communication/topic/migration.rs +++ b/horus_core/src/communication/topic/migration.rs @@ -24,8 +24,6 @@ pub(crate) enum MigrationResult { AlreadyInProgress, /// Migration lock acquisition failed (contention) LockContention, - /// Migration failed due to error - Failed, } // ============================================================================ @@ -34,10 +32,21 @@ pub(crate) enum MigrationResult { /// Backend migration coordinator. /// -/// Handles safe backend transitions with: +/// Handles backend transitions with: /// - CAS-based locking to prevent concurrent migrations /// - Epoch versioning for reader/writer coordination -/// - Drain logic to ensure no in-flight message loss +/// - A bounded settle before the switch +/// +/// It does NOT guarantee that no message is lost across a switch, and this +/// used to say it did ("Drain logic to ensure no in-flight message loss"). +/// Nothing in `TopicHeader` counts in-flight operations, so there is nothing +/// to drain against — see [`Self::settle_before_switch`]. A real guarantee +/// needs an in-flight counter incremented and decremented on the send and +/// recv hot paths of a real-time framework, which is a deliberate design +/// decision with a measurable cost. Until that is taken, the honest statement +/// is that a topology change switches backends under a lock and a message in +/// flight across the switch can be lost. The topic's own test suite already +/// works around it, pre-warming backends "to avoid migration losses". pub(crate) struct BackendMigrator<'a> { header: &'a TopicHeader, /// Spin count before yielding during drain @@ -90,24 +99,15 @@ impl<'a> BackendMigrator<'a> { /// Perform the actual migration (must hold lock). /// - /// Drains in-flight operations before switching the backend mode. - /// If the first drain attempt times out, retries once before failing. + /// Settles briefly, then switches the backend mode. + /// + /// This used to say it "drains in-flight operations before switching" and + /// "retries once before failing". Both halves guarded a return value that + /// no longer exists: `settle_before_switch` cannot observe an in-flight + /// operation and so cannot report one, which made the retry and the + /// `Failed` result unreachable code describing a guarantee nothing made. fn perform_migration(&self, new_mode: BackendMode) -> MigrationResult { - if !self.drain_in_flight() { - tracing::warn!( - "migration drain timed out on first attempt, retrying (target={:?})", - new_mode - ); - // One retry: in-flight ops are <200ns, so a second attempt should succeed - // unless there is a genuine stall (e.g., thread suspended by debugger). - if !self.drain_in_flight() { - tracing::warn!( - "migration drain timed out on retry, aborting migration (target={:?})", - new_mode - ); - return MigrationResult::Failed; - } - } + self.settle_before_switch(); // IMPORTANT: Store backend_mode BEFORE incrementing epoch. // The epoch increment is the "publication fence" — observers that @@ -133,13 +133,14 @@ impl<'a> BackendMigrator<'a> { MigrationResult::Success { new_epoch } } - /// Grace period for in-flight operations to complete before migration. + /// Bounded settle before the backend mode is switched. /// - /// Since send/recv are lock-free and complete in <200ns, a short spin - /// followed by yields gives in-progress operations time to finish. - /// Returns `false` if the drain timeout is exceeded, indicating that - /// migration should be retried rather than proceeding unsafely. - fn drain_in_flight(&self) -> bool { + /// NOT a drain, despite what this was called. Send and recv are lock-free + /// and complete in under 200 ns, so a bounded spin plus a few yields gives + /// an in-progress operation a good chance to finish — but "a good chance" + /// is the whole of it. Nothing here can confirm one finished, and nothing + /// here can fail. + fn settle_before_switch(&self) { // Give any in-flight send/recv time to finish. They complete in under // 200 ns, so a bounded spin plus a few yields is the whole mechanism. // @@ -190,17 +191,16 @@ impl<'a> BackendMigrator<'a> { std::sync::atomic::fence(Ordering::SeqCst); std::hint::spin_loop(); if start.elapsed() > budget { - return true; + return; } } // Yield to let preempted producer threads complete. for _ in 0..3 { std::thread::yield_now(); if start.elapsed() > budget { - return true; + return; } } - true } /// Force a migration to the detected optimal backend. @@ -220,6 +220,49 @@ impl<'a> BackendMigrator<'a> { #[cfg(test)] mod tests { use super::*; + + /// The settle is bounded, and that bound is the only thing it guarantees. + /// + /// It cannot observe an in-flight operation — nothing in `TopicHeader` + /// counts them — so it can neither confirm a drain nor report a failure. + /// What it can do is not spin forever, and that part matters: removing the + /// budget made `send_blocking` block measurably longer under load (a test + /// asserting an immediate return went from failing 2 runs in 8 to 7 in 8). + #[test] + fn the_settle_is_bounded() { + let header = make_header(true); + let migrator = BackendMigrator::new(&header); + + let started = std::time::Instant::now(); + migrator.settle_before_switch(); + let elapsed = started.elapsed(); + + assert!( + elapsed < std::time::Duration::from_millis(250), + "the settle must stay bounded; it took {elapsed:?}" + ); + } + + /// A migration cannot report failure, so nothing may branch on one. + /// + /// `perform_migration` used to retry a drain and return + /// `MigrationResult::Failed` when it "timed out" — guarding a return value + /// `drain_in_flight` had stopped producing, so both the retry and the + /// result were unreachable code standing for a guarantee nothing made. + /// Callers wrote handlers for a case that could not occur. + #[test] + fn an_uncontended_migration_always_completes() { + let header = make_header(true); + header.publisher_count.store(1, Ordering::Release); + header.subscriber_count.store(2, Ordering::Release); + let migrator = BackendMigrator::new(&header); + + // Single-threaded, so there is nothing to contend with. + match migrator.migrate_to_optimal() { + MigrationResult::Success { .. } | MigrationResult::NotNeeded => {} + other => panic!("an uncontended migration reported {other:?}"), + } + } use std::sync::atomic::Ordering; fn make_header(is_pod: bool) -> TopicHeader { @@ -237,7 +280,6 @@ mod tests { MigrationResult::NotNeeded, MigrationResult::AlreadyInProgress, MigrationResult::LockContention, - MigrationResult::Failed, ]; for (i, a) in results.iter().enumerate() { for (j, b) in results.iter().enumerate() { diff --git a/horus_core/src/communication/topic/mod.rs b/horus_core/src/communication/topic/mod.rs index 01124d6ba..3bc080144 100644 --- a/horus_core/src/communication/topic/mod.rs +++ b/horus_core/src/communication/topic/mod.rs @@ -1934,10 +1934,6 @@ impl RingTopic< local.cached_mode = header.mode(); break; } - MigrationResult::Failed => { - local.cached_mode = header.mode(); - break; - } } } local.cached_mode = header.mode(); diff --git a/horus_core/src/communication/topic/tests.rs b/horus_core/src/communication/topic/tests.rs index c4554a1cc..78f6c9e7e 100644 --- a/horus_core/src/communication/topic/tests.rs +++ b/horus_core/src/communication/topic/tests.rs @@ -12499,10 +12499,6 @@ fn no_message_is_lost_without_being_counted() { treat this as a real failure, not a skip." ); } - MigrationResult::Failed => { - eprintln!("skipping {mode:?}: not available in this build"); - continue; - } } backends_checked += 1; trigger_shm_dispatch(&name); From c4f981bee2dbe72a27b4f742a621c76c343de367 Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 21:19:00 -0700 Subject: [PATCH 15/79] fix(rt): a safing callback that panics must not cancel the safety action Two halves of one asymmetry: the executors got the isolation half of the main loop's fault handling and not the escalation half. **A panicking `shutdown()` cancelled the emergency stop that called it.** `BudgetPolicy::EmergencyStop` ran `let _ = node.node.shutdown()` bare, and BEFORE latching. `shutdown()` is user code on a node that is already failing, so an unwind out of it escaped past `estop.trigger`, past `running.store(false)` and into the loop's outer `catch_unwind`, whose only action is to stop the node. `get_state()` kept reporting Normal, no blackbox EmergencyStop was written, `PENDING_LOCAL_ESTOP` stayed empty so horus_net broadcast nothing, and the other RT nodes kept ticking -- exactly the outcome the comment three lines below it was written to prevent. The docs promise this e-stop unconditionally. Both `shutdown()` calls (Enforce and EmergencyStop) now go through `guard_fault_callback`, which primitives.rs already says "any future `&mut dyn Node` callback reached from an executor belongs behind". The EmergencyStop arm also latches and signals the thread BEFORE calling into the node, so the safety action no longer depends on user code returning. **A critical RT node that could not reach a safe state did not halt the robot.** `note_safing_failure` states the rule as a property of the system -- "a node that cannot safe itself is a safety failure -- escalate to a system emergency stop for critical nodes" -- and `is_critical_node` had exactly one caller, on the main loop. So the identical event halted the robot on a BestEffort node and merely stopped an RT one. Every RT node under a scheduler `.watchdog()` is a critical node, so the class most likely to be critical was the one the escalation omitted. `primitives::note_safing_failure` is the executor twin, wired into the Isolate and Kill rungs of `apply_degradation_action` and into the RT executor's `Miss::SafeMode` dispatch -- which called `enter_safe_state()` entirely bare, so a panic there only ever stopped the node. Ablations: with `shutdown()` bare and ahead of the latch the panic escapes the test itself; with isolation kept but escalation removed the panic is caught and no e-stop latches, which is the shipped behaviour. --- horus_core/src/scheduling/primitives.rs | 60 ++++++-- horus_core/src/scheduling/rt_executor.rs | 150 ++++++++++++++++++-- horus_core/src/scheduling/safety_monitor.rs | 2 +- 3 files changed, 193 insertions(+), 19 deletions(-) diff --git a/horus_core/src/scheduling/primitives.rs b/horus_core/src/scheduling/primitives.rs index 17a3d851a..f83044fc8 100644 --- a/horus_core/src/scheduling/primitives.rs +++ b/horus_core/src/scheduling/primitives.rs @@ -62,6 +62,47 @@ pub(crate) fn guard_fault_callback(f: impl FnOnce()) -> bool { std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).is_err() } +/// A node's safing callback panicked on an executor, so the node did NOT reach +/// a safe state. Stop ticking it, and — because a node that cannot safe itself +/// is a safety failure — escalate to a system emergency stop for critical +/// nodes. +/// +/// The executor twin of `Scheduler::note_safing_failure`, which states the +/// intent as a property of the system rather than of one dispatch path. The +/// isolation half of that was ported to the executors (see +/// [`guard_fault_callback`]); the ESCALATION half was not, so the identical +/// event halted the robot on a BestEffort node and merely stopped an RT one — +/// and every RT node under a scheduler `.watchdog()` is a critical node, so the +/// class most likely to be critical was the one the escalation omitted. +pub(crate) fn note_safing_failure( + node: &mut RegisteredNode, + monitors: &super::types::SharedMonitors, + action: &str, +) { + node.is_stopped = true; + let critical = monitors + .safety + .as_ref() + .is_some_and(|m| m.is_critical_node(&node.name)); + if critical { + super::rt_executor::rt_diag(format_args!( + "[RT-thread] '{}' PANICKED in {} and could NOT reach a safe state — EMERGENCY STOP", + node.name, action + )); + if let Some(ref estop) = monitors.estop { + estop.trigger(format!( + "critical node '{}' panicked in {} and could not reach a safe state", + node.name, action + )); + } + } else { + super::rt_executor::rt_diag(format_args!( + "[RT-thread] '{}' panicked in {} and could NOT reach a safe state — node stopped", + node.name, action + )); + } +} + /// Honour a pending safe-state request raised by the main thread's watchdog /// ladder, if this node has one. /// @@ -168,13 +209,7 @@ pub(crate) fn apply_degradation_action( DegradationAction::Isolate(ref name) => { set_health(node, NodeHealthState::Isolated); let target = &mut node.node; - let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - target.enter_safe_state() - })) - .is_err(); - if panicked { - node.is_stopped = true; - } + let panicked = guard_fault_callback(|| target.enter_safe_state()); super::rt_executor::rt_diag(format_args!( " Degradation: '{name}' — isolated, entered safe state{}", if panicked { @@ -183,6 +218,9 @@ pub(crate) fn apply_degradation_action( "" } )); + if panicked { + note_safing_failure(node, monitors, "enter_safe_state"); + } if let Some(ref monitor) = monitors.safety { monitor.record_degrade_activation(); } @@ -190,15 +228,17 @@ pub(crate) fn apply_degradation_action( DegradationAction::Kill(ref name) => { set_health(node, NodeHealthState::Isolated); let target = &mut node.node; - let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let panicked = guard_fault_callback(|| { let _ = target.shutdown(); - })) - .is_err(); + }); node.is_stopped = true; super::rt_executor::rt_diag(format_args!( " KILL: '{name}' — permanently removed from execution after shutdown(){}", if panicked { " (shutdown panicked)" } else { "" } )); + if panicked { + note_safing_failure(node, monitors, "shutdown"); + } if let Some(ref monitor) = monitors.safety { monitor.record_degrade_activation(); } diff --git a/horus_core/src/scheduling/rt_executor.rs b/horus_core/src/scheduling/rt_executor.rs index 8f95a4c6e..b940fe0d1 100644 --- a/horus_core/src/scheduling/rt_executor.rs +++ b/horus_core/src/scheduling/rt_executor.rs @@ -1314,8 +1314,18 @@ impl RtExecutor { "[RT-thread] BUDGET ENFORCE: '{}' exceeded 2x budget ({:?} > {:?}) — node stopped", node.name, tr.duration, tick_budget * 2 )); - let _ = node.node.shutdown(); - node.is_stopped = true; + // Guarded: an unwind out of a node's own shutdown + // would otherwise escape into the loop's outer + // catch_unwind and skip everything after it. + let target = &mut node.node; + let panicked = super::primitives::guard_fault_callback(|| { + let _ = target.shutdown(); + }); + if panicked { + super::primitives::note_safing_failure(node, monitors, "shutdown"); + } else { + node.is_stopped = true; + } } } BudgetPolicy::EmergencyStop => { @@ -1324,15 +1334,21 @@ impl RtExecutor { "[RT-thread] BUDGET E-STOP: '{}' budget violation ({:?} > {:?})", node.name, tr.duration, tick_budget )); - let _ = node.node.shutdown(); - node.is_stopped = true; - // Latch the SafetyMonitor before signalling the thread to - // exit. `running.store(false)` alone is a plain shutdown - // flag: it leaves get_state() reporting Normal, writes no + // Latch the SafetyMonitor BEFORE calling into the node. + // `running.store(false)` alone is a plain shutdown flag: + // it leaves get_state() reporting Normal, writes no // blackbox EmergencyStop, and never populates // PENDING_LOCAL_ESTOP — so horus_net had nothing to // broadcast and peer robots were never told this one // emergency-stopped. + // + // Ordering matters as much as the latch. `shutdown()` is + // user code on a node that is already failing; it used to + // run first and unguarded, so a panic in it unwound past + // the latch, the running flag and everything else into + // the loop's outer catch_unwind — whose only action is to + // stop the node. The e-stop this policy exists to fire + // was silently cancelled by the callback it fired. if let Some(ref estop) = monitors.estop { estop.trigger(format!( "RT node '{}' budget violation ({:?} > {:?}) with BudgetPolicy::EmergencyStop", @@ -1341,6 +1357,11 @@ impl RtExecutor { } // Signal stop via running flag — RT thread will exit running.store(false, Ordering::SeqCst); + node.is_stopped = true; + let target = &mut node.node; + let _ = super::primitives::guard_fault_callback(|| { + let _ = target.shutdown(); + }); } } } @@ -1467,7 +1488,18 @@ impl RtExecutor { SuppressedSuffix(hidden) )); } - node.node.enter_safe_state(); + let target = &mut node.node; + if super::primitives::guard_fault_callback(|| target.enter_safe_state()) + { + // A critical node that cannot reach a safe state + // is a system safety failure, not a node-local + // one — the same escalation the main loop makes. + super::primitives::note_safing_failure( + node, + monitors, + "enter_safe_state", + ); + } } } DeadlineAction::EmergencyStop => { @@ -2277,6 +2309,108 @@ mod tests { ); } + /// A node whose safing callbacks panic — the failure both tests are about. + struct UnsafeableNode { + name: String, + } + + impl Node for UnsafeableNode { + fn name(&self) -> &str { + &self.name + } + fn tick(&mut self) {} + fn enter_safe_state(&mut self) { + panic!("cannot reach a safe state"); + } + fn shutdown(&mut self) -> crate::error::HorusResult<()> { + panic!("cannot shut down"); + } + } + + fn make_unsafeable(name: &str) -> RegisteredNode { + use crate::core::NodeInfo; + let mut node = make_rt_registered(name, Arc::new(std::sync::atomic::AtomicU64::new(0))); + node.node = super::super::types::NodeKind::new(Box::new(UnsafeableNode { + name: name.to_string(), + })); + node.context = Some(NodeInfo::new(name.to_string())); + node + } + + /// A panic in `shutdown()` must not cancel the emergency stop it was + /// called by. + /// + /// `BudgetPolicy::EmergencyStop` used to call `shutdown()` bare and BEFORE + /// latching. `shutdown()` is user code on a node that is already failing, + /// so an unwind out of it escaped past the latch, past `running.store`, and + /// into the loop's outer `catch_unwind` — whose only action is to stop the + /// node. `get_state()` kept reporting Normal, no blackbox EmergencyStop was + /// written, `PENDING_LOCAL_ESTOP` stayed empty so horus_net broadcast + /// nothing, and the other RT nodes kept ticking. The documented "e-stop on + /// any budget violation" was silently cancelled by the callback it fired. + #[test] + fn a_panicking_shutdown_does_not_cancel_the_budget_emergency_stop() { + let _estop_serial = super::super::safety_monitor::estop_queue_guard(); + let mut node = make_unsafeable("budget_estop_node"); + node.tick_budget = Some(Duration::from_nanos(1)); + node.budget_policy = super::super::safety_monitor::BudgetPolicy::EmergencyStop; + + let monitor = Arc::new(SafetyMonitor::new(1000)); + let mut monitors = test_monitors(); + monitors.estop = Some(monitor.estop_trigger()); + monitors.safety = Some(monitor.clone()); + let running = Arc::new(AtomicBool::new(true)); + + RtExecutor::tick_node(&mut node, &monitors, &running, false, 0); + + assert!( + monitor.is_emergency_stop(), + "the emergency stop must be latched even though shutdown() panicked" + ); + assert!( + !running.load(Ordering::SeqCst), + "the RT thread must still be told to exit" + ); + } + + /// A critical node that cannot reach a safe state must halt the robot. + /// + /// The docs say so without qualification, and `note_safing_failure` states + /// it as a property of the system: "a node that cannot safe itself is a + /// safety failure — escalate to a system emergency stop for critical + /// nodes". Only the main loop did it. The executors got the ISOLATION half + /// of that (`guard_fault_callback`, added "so both halves of the framework + /// isolate recovery callbacks identically") and not the escalation, so the + /// identical event halted the robot on a BestEffort node and merely stopped + /// an RT one — and every RT node under a scheduler `.watchdog()` is + /// critical, so the class most likely to be critical was the one that was + /// omitted. + #[test] + fn a_critical_rt_node_that_cannot_safe_itself_triggers_an_emergency_stop() { + let _estop_serial = super::super::safety_monitor::estop_queue_guard(); + let mut node = make_unsafeable("critical_safer"); + node.deadline = Some(Duration::from_millis(1)); + node.miss_policy = Miss::SafeMode; + + let monitor = Arc::new(SafetyMonitor::new(1000)); + monitor.add_critical_node("critical_safer".to_string(), Duration::from_millis(100)); + let mut monitors = test_monitors(); + monitors.estop = Some(monitor.estop_trigger()); + monitors.safety = Some(monitor.clone()); + let running = Arc::new(AtomicBool::new(true)); + + // Released 5 ms late against a 1 ms deadline: a miss, so SafeMode + // dispatches `enter_safe_state()` — which panics. + RtExecutor::tick_node(&mut node, &monitors, &running, false, 5_000_000); + + assert!( + monitor.is_emergency_stop(), + "a CRITICAL node that panicked trying to reach a safe state has not \ + reached one; stopping just that node leaves the robot running with \ + an unsafe actuator" + ); + } + struct CounterNode { name: String, count: Arc, diff --git a/horus_core/src/scheduling/safety_monitor.rs b/horus_core/src/scheduling/safety_monitor.rs index e0d2a0a9d..f4753272b 100644 --- a/horus_core/src/scheduling/safety_monitor.rs +++ b/horus_core/src/scheduling/safety_monitor.rs @@ -116,7 +116,7 @@ static PENDING_LOCAL_ESTOP: std::sync::Mutex> = std::sync::Mutex: static ESTOP_QUEUE_SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); #[cfg(test)] -fn estop_queue_guard() -> std::sync::MutexGuard<'static, ()> { +pub(crate) fn estop_queue_guard() -> std::sync::MutexGuard<'static, ()> { ESTOP_QUEUE_SERIAL.lock().unwrap_or_else(|e| e.into_inner()) } From d5b507cdf1456bcdcc6010d02ca9cab06109137f Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 23:02:02 -0700 Subject: [PATCH 16/79] fix: a watchdog is never tightened, and topic metrics count real traffic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **`Scheduler::watchdog()` silently tightened the window.** The config stores whole milliseconds and `as_millis()` truncates, so `.watchdog(1500_u64.us())` stored 1 ms — and the graduated ladder latches a system-wide emergency stop past 3x the window, so it fired at 3 ms instead of the 4.5 ms asked for, with no diagnostic. The builder states the rule immediately above the line that broke it: "Round up to the 1ms floor the storage can represent rather than truncating ... a slightly-looser watchdog is a safety feature, and no watchdog is not ... silently changing a safety timeout is its own problem — the operator needs to know the value actually in force." That held for the sub-1 ms case and was inverted for every other one. Tightening is the direction that halts a healthy robot. Ceiling division now, across the whole range, and the warning fires for any value that had to change — not just the sub-millisecond one. An explicit `Duration::ZERO` is still an honoured disable. **Three of the four counters on the public `TopicMetrics` were dead.** `messages_sent` and `messages_received` moved only inside `send_with_content_logging` / `recv_with_content_logging` — both `#[cold]`, both reached only while the `horus monitor` TUI has set a topic's verbose flag. So accessors documented "Messages sent through this topic" read 0 in every normal run, however much traffic the topic carried. `MockTopic` — the stand-in users write their tests against — maintained both faithfully all along, with its own tests asserting on them. A double that reports what the real transport does not is the one defect a double cannot have. Both are counted on the ordinary paths now, beside the existing `messages_total` bump. That comment prices the SHM counter at ~28 ns, but this is a process-local uncontended relaxed increment on a counter only this handle writes; the cold paths no longer double-count. `recv_failures` had no increment anywhere in the transport and no honest meaning there — a `recv()` that returns `None` is an empty ring, not a failure. The dead field is gone from the transport-side struct, the snapshot reports 0 explicitly, and the accessor now says so and points at `missed_count()` for real consumer-side loss. It stays on `TopicMetrics` because `MockTopic` genuinely populates it from its fault injection, which is the whole point of the double. Ablations: restoring `as_millis()` reads 1 ms where 2 ms was asked for; removing the ordinary-path counts reads 0 for a topic that carried five messages. --- horus_core/src/communication/topic/metrics.rs | 23 +++++++-- horus_core/src/communication/topic/mod.rs | 37 +++++++++++++- horus_core/src/communication/topic/tests.rs | 51 ++++++++++++++++--- horus_core/src/scheduling/scheduler/mod.rs | 23 ++++++--- horus_core/src/scheduling/scheduler/tests.rs | 47 +++++++++++++++++ 5 files changed, 162 insertions(+), 19 deletions(-) diff --git a/horus_core/src/communication/topic/metrics.rs b/horus_core/src/communication/topic/metrics.rs index 3c2045792..6a2b42caf 100644 --- a/horus_core/src/communication/topic/metrics.rs +++ b/horus_core/src/communication/topic/metrics.rs @@ -5,19 +5,27 @@ use std::sync::atomic::{AtomicU32, AtomicU64}; /// Migration and operational metrics for a Topic #[derive(Debug, Default)] pub(crate) struct MigrationMetrics { - /// Messages sent through this topic + /// Messages sent through this topic, counted on every `send()`. pub messages_sent: AtomicU64, - /// Messages received through this topic + /// Messages delivered by `recv()` on this topic. + /// + /// `try_recv()` is the low-level path and does not count; `recv()` is what + /// `Topic::recv` uses. pub messages_received: AtomicU64, /// Number of send failures pub send_failures: AtomicU64, - /// Number of receive failures - pub recv_failures: AtomicU64, /// Number of backend migrations performed pub migrations: AtomicU32, } /// Non-atomic snapshot of topic metrics (for external consumers) +/// +/// `messages_sent` and `messages_received` used to move only on the `#[cold]` +/// verbose-logging path, which runs while the `horus monitor` TUI has set a +/// topic's verbose flag — so in an ordinary run both read 0 no matter how much +/// traffic the topic carried, while `MockTopic` maintained them faithfully. A +/// test double that reports what the real transport does not is the one defect +/// a double cannot have. Both are counted on the ordinary paths now. #[derive(Debug, Clone, Default)] pub struct TopicMetrics { messages_sent: u64, @@ -54,6 +62,13 @@ impl TopicMetrics { self.send_failures } + /// Receive failures. + /// + /// Only [`MockTopic`](crate::testing::MockTopic) produces a non-zero value + /// here, from its `recv_fail_after` / `recv_fail_every_n` fault injection. + /// The real transport never increments it, deliberately: a `recv()` that + /// returns `None` is an empty ring, not a failure. For real consumer-side + /// loss — a publisher lapping a subscriber — use `Topic::missed_count()`. pub fn recv_failures(&self) -> u64 { self.recv_failures } diff --git a/horus_core/src/communication/topic/mod.rs b/horus_core/src/communication/topic/mod.rs index 3bc080144..a08152492 100644 --- a/horus_core/src/communication/topic/mod.rs +++ b/horus_core/src/communication/topic/mod.rs @@ -2774,7 +2774,10 @@ impl RingTopic< self.metrics.messages_sent.load(Ordering::Relaxed), self.metrics.messages_received.load(Ordering::Relaxed), self.metrics.send_failures.load(Ordering::Relaxed), - self.metrics.recv_failures.load(Ordering::Relaxed), + // The real transport has no recv-failure notion: a `recv()` that + // returns `None` is an empty ring. Only `MockTopic` reports one, + // from its fault injection. See `TopicMetrics::recv_failures`. + 0, ) } @@ -2848,6 +2851,19 @@ impl RingTopic< // ~28ns of ~95ns, and it should be done on purpose rather than by // flipping this line. self.header().messages_total.fetch_add(1, Ordering::Relaxed); + // The per-handle counter behind `TopicMetrics::messages_sent`, which was + // incremented ONLY inside `send_with_content_logging` — a `#[cold]` path + // reached only while the `horus monitor` TUI has set the topic's verbose + // flag. So the public accessor read 0 in every normal run, however much + // traffic the topic carried, while `MockTopic` (the stand-in users write + // their tests against) maintained it faithfully. A test double that + // reports what the real thing does not is the one defect a double + // cannot have. + // + // This is a process-local, uncontended relaxed increment on a counter + // only this handle writes — not the contended shared-memory RMW the + // comment above prices at ~28 ns. + self.metrics.messages_sent.fetch_add(1, Ordering::Relaxed); if unlikely(self.is_verbose()) { self.send_with_content_logging(msg); // Notify event nodes watching this topic. Gated inside @@ -2993,7 +3009,9 @@ impl RingTopic< self.send_lossy(msg); let ipc_ns = start.elapsed().as_nanos() as u64; - self.metrics.messages_sent.fetch_add(1, Ordering::Relaxed); + // NOT counted here: `send()` counts every send before dispatching to + // this cold path, so incrementing again would double-count while the + // monitor's verbose flag is set. self.state .store(ConnectionState::Connected.into_u8(), Ordering::Relaxed); use crate::core::hlog::{current_node_name, current_tick_number}; @@ -3336,6 +3354,21 @@ impl RingTopic< if unlikely(self.is_verbose()) { return self.recv_with_content_logging(); } + let received = self.recv_uncounted(); + if received.is_some() { + // Counts DELIVERED messages, the mirror of `messages_sent`. As + // there, this used to move only on the `#[cold]` verbose path, so + // the public accessor read 0 in every normal run. + self.metrics + .messages_received + .fetch_add(1, Ordering::Relaxed); + } + received + } + + /// `recv()` without the metrics bookkeeping. + #[inline(always)] + fn recv_uncounted(&self) -> Option { // Fast path: role=Both (same-instance, same-thread pub+sub) and POD. // // Gated on `is_pod` in lockstep with `send`: this reads back through diff --git a/horus_core/src/communication/topic/tests.rs b/horus_core/src/communication/topic/tests.rs index 78f6c9e7e..b5c85e8ea 100644 --- a/horus_core/src/communication/topic/tests.rs +++ b/horus_core/src/communication/topic/tests.rs @@ -9852,7 +9852,6 @@ fn migration_metrics_default_all_zero() { assert_eq!(m.messages_sent.load(Ordering::Relaxed), 0); assert_eq!(m.messages_received.load(Ordering::Relaxed), 0); assert_eq!(m.send_failures.load(Ordering::Relaxed), 0); - assert_eq!(m.recv_failures.load(Ordering::Relaxed), 0); assert_eq!(m.migrations.load(Ordering::Relaxed), 0); } @@ -9863,13 +9862,11 @@ fn migration_metrics_atomic_increments() { m.messages_sent.fetch_add(5, Ordering::Relaxed); m.messages_received.fetch_add(3, Ordering::Relaxed); m.send_failures.fetch_add(2, Ordering::Relaxed); - m.recv_failures.fetch_add(1, Ordering::Relaxed); m.migrations.fetch_add(4, Ordering::Relaxed); assert_eq!(m.messages_sent.load(Ordering::Relaxed), 5); assert_eq!(m.messages_received.load(Ordering::Relaxed), 3); assert_eq!(m.send_failures.load(Ordering::Relaxed), 2); - assert_eq!(m.recv_failures.load(Ordering::Relaxed), 1); assert_eq!(m.migrations.load(Ordering::Relaxed), 4); } @@ -9913,17 +9910,59 @@ fn migration_metrics_manual_increment_visible_in_snapshot() { let t: Topic = Topic::new(unique("metrics_manual")).expect("create"); let mig = t.migration_metrics(); - // Manually increment (as would happen if hot-path tracking were enabled) mig.messages_sent.fetch_add(10, Ordering::Relaxed); mig.messages_received.fetch_add(7, Ordering::Relaxed); mig.send_failures.fetch_add(2, Ordering::Relaxed); - mig.recv_failures.fetch_add(1, Ordering::Relaxed); let snap = t.metrics(); assert_eq!(snap.messages_sent(), 10); assert_eq!(snap.messages_received(), 7); assert_eq!(snap.send_failures(), 2); - assert_eq!(snap.recv_failures(), 1); + assert_eq!( + snap.recv_failures(), + 0, + "the real transport has no recv-failure notion — a recv() that returns \ + None is an empty ring. Only MockTopic reports one, from its fault \ + injection." + ); +} + +/// The counters behind `TopicMetrics` must move on the ORDINARY paths. +/// +/// `messages_sent` and `messages_received` were incremented only inside +/// `send_with_content_logging` / `recv_with_content_logging` — both `#[cold]`, +/// both reached only while the `horus monitor` TUI has set the topic's verbose +/// flag. So a public accessor documented as "Messages sent through this topic" +/// read 0 in every normal run, however much traffic the topic carried. +/// +/// `MockTopic` — the stand-in users write their tests against — maintained both +/// faithfully all along, and its own tests assert on them. A double that +/// reports what the real transport does not is the one defect a double cannot +/// have. +#[test] +fn topic_metrics_count_traffic_without_the_monitor_attached() { + let t: Topic = Topic::new(unique("metrics_ordinary")).expect("create"); + + for i in 0..5u64 { + t.send(i); + } + let mut received = 0u64; + while t.recv().is_some() { + received += 1; + } + + let snap = t.metrics(); + assert_eq!( + snap.messages_sent(), + 5, + "five ordinary send() calls must read as five" + ); + assert!(received > 0, "precondition: the topic delivered something"); + assert_eq!( + snap.messages_received(), + received, + "every message recv() handed back must be counted" + ); } // ============================================================================ diff --git a/horus_core/src/scheduling/scheduler/mod.rs b/horus_core/src/scheduling/scheduler/mod.rs index 5380b506b..56f58de3f 100644 --- a/horus_core/src/scheduling/scheduler/mod.rs +++ b/horus_core/src/scheduling/scheduler/mod.rs @@ -801,18 +801,27 @@ impl Scheduler { // feature, and no watchdog is not. Say so, because silently changing a // safety timeout is its own problem — the operator needs to know the // value actually in force. - let ms = timeout.as_millis() as u64; + // + // That rule held for the sub-1ms case and was inverted for every other + // one: `as_millis()` truncates, so `.watchdog(1500_u64.us())` stored 1ms + // and its 3x critical rung latched a fleet-wide e-stop at 3ms instead of + // the 4.5ms asked for, with no diagnostic. Tightening is the direction + // that produces spurious halts on a healthy node. Round the whole range + // up, and report any value that had to change. + let ms = timeout.as_nanos().div_ceil(1_000_000) as u64; self.pending_config.realtime.watchdog_timeout_ms = if timeout.is_zero() { 0 // explicit disable — honour it - } else if ms == 0 { + } else if !timeout.subsec_nanos().is_multiple_of(1_000_000) || ms == 0 { + let effective = Duration::from_millis(ms.max(1)); crate::hlog!( warn, - "watchdog({:?}) is below the 1ms resolution the scheduler config stores; \ - using 1ms. Sub-millisecond watchdogs are not supported — previously this \ - truncated to 0, which DISABLED the watchdog entirely.", - timeout + "watchdog({:?}) is not a whole number of milliseconds, which is all the \ + scheduler config stores; using {:?}. Rounded UP on purpose: a looser \ + watchdog is a safety feature, a tighter one halts a healthy robot.", + timeout, + effective ); - 1 + ms.max(1) } else { ms }; diff --git a/horus_core/src/scheduling/scheduler/tests.rs b/horus_core/src/scheduling/scheduler/tests.rs index bf1657604..9e343cac0 100644 --- a/horus_core/src/scheduling/scheduler/tests.rs +++ b/horus_core/src/scheduling/scheduler/tests.rs @@ -3075,6 +3075,53 @@ fn test_builder_order_independence() { ); } +/// A watchdog timeout is rounded UP, never tightened. +/// +/// The config stores whole milliseconds. `as_millis()` truncates, so +/// `.watchdog(1500_u64.us())` used to store 1 ms — and the graduated ladder +/// latches a system-wide emergency stop past 3x the window, so it fired at 3 ms +/// instead of the 4.5 ms the operator asked for, with no diagnostic. +/// +/// Tightening is the dangerous direction: it halts a healthy robot. The builder +/// already argued exactly this for the sub-1 ms case — "a slightly-looser +/// watchdog is a safety feature, and no watchdog is not" — and then truncated +/// every other value. +#[test] +fn a_watchdog_timeout_is_never_silently_tightened() { + let _guard = lock_scheduler(); + + // The case the old code got right. + let sub_ms = Scheduler::new().watchdog(500_u64.us()); + assert_eq!( + sub_ms.pending_config.realtime.watchdog_timeout_ms, 1, + "a sub-millisecond watchdog must not round to 0, which disables it" + ); + + // The case it got backwards. + let fractional = Scheduler::new().watchdog(1500_u64.us()); + assert_eq!( + fractional.pending_config.realtime.watchdog_timeout_ms, 2, + "1.5 ms must round UP to 2 ms; truncating to 1 ms makes the 3x \ + emergency-stop rung fire at 3 ms instead of 4.5 ms" + ); + + let just_over = Scheduler::new().watchdog(Duration::from_micros(2_001)); + assert_eq!( + just_over.pending_config.realtime.watchdog_timeout_ms, 3, + "any remainder rounds up — the stored window must never be shorter \ + than the one that was asked for" + ); + + // Whole milliseconds are untouched, and an explicit zero still disables. + let whole = Scheduler::new().watchdog(500_u64.ms()); + assert_eq!(whole.pending_config.realtime.watchdog_timeout_ms, 500); + let disabled = Scheduler::new().watchdog(Duration::ZERO); + assert_eq!( + disabled.pending_config.realtime.watchdog_timeout_ms, 0, + "an explicit zero is a disable and must be honoured" + ); +} + // ============================================================================ // Phase 6: Graduated Safety Monitor Response // ============================================================================ From 65f461a4fbf7bf09454786e88864567f363ca879 Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 23:10:53 -0700 Subject: [PATCH 17/79] fix(launch): --status told the truth about nothing, and namespace: lied twice **`horus launch --status` reported every node as running, forever.** `show_launch_status` colours a STATUS column with "running", "crashed" and "stopped" arms, and `LaunchProcessEntry::status` was assigned exactly once in production code -- `"running"` at construction -- and never mutated. Two of the three arms could not be reached. The monitor loop already knows: the same iteration that drops a dead node from the supervised vector logs `LaunchEvent::NodeCrashed`. Two halves of one workflow disagreed -- the JSONL event log knew, the session manifest the CLI reads did not. Worse, the refresh that followed re-associated by POSITION. Removing a dead node shifts every later index, so a surviving node's PID was written into the dead node's row: an operator asking whether the perception node crashed was told "running", beside a PID belonging to a different, healthy process. `refresh_session_manifest` is keyed by name, records `crashed` for a non-zero exit and `stopped` for a clean one, and puts a respawned node back to `running`. It is extracted so the property is testable without driving the supervisor loop. **A per-node `namespace:` did nothing, and said so nowhere.** It was parsed, composed into the name `--dry-run` PRINTS, and exported as `HORUS_NODE_NAMESPACE` -- which had exactly one occurrence in the whole repository, this write, and no reader in Rust, Python or C++. Its doc-comment promised "Namespace prefix for topics", which horus_core cannot do: there is no topic prefix or remap mechanism. The sibling `priority` export at least carries an explicit "this is currently a no-op" note. Giving it real meaning is a horus_core design decision (topic remapping), not a launcher change, and it cannot simply be "restored": composing it into the child's `HORUS_NAMESPACE` would put the node in a different shared-memory tree from its siblings and break the launch it belongs to. horus_sys states that granularity as "per robot" -- the file-level key, not this one. So the runtime stops lying while the semantics stay undecided: the export is gone, launching warns and says what to do instead, `--dry-run` shows the name the process is actually given rather than a composed identity nothing receives, and the field's doc-comment says it is inert and why. Ablation: with the positional refresh restored, a crashed node reads "running" and all three tests fail. --- horus_manager/src/commands/launch.rs | 218 +++++++++++++++++++++++++-- 1 file changed, 202 insertions(+), 16 deletions(-) diff --git a/horus_manager/src/commands/launch.rs b/horus_manager/src/commands/launch.rs index b68dcb2f0..07730a0d1 100644 --- a/horus_manager/src/commands/launch.rs +++ b/horus_manager/src/commands/launch.rs @@ -72,7 +72,20 @@ pub struct LaunchNode { #[serde(default)] pub args: Vec, - /// Namespace prefix for topics + /// INERT. Parsed and warned about; nothing reads it. + /// + /// This said "Namespace prefix for topics", which the runtime cannot do: + /// there is no topic prefix or remap mechanism in horus_core. The + /// FILE-level `namespace:` works because `HORUS_NAMESPACE` names the shared + /// memory tree, which is isolation rather than prefixing, and horus_sys + /// states that granularity as "per robot" — the file-level key, not this + /// one. Composing this into a child's `HORUS_NAMESPACE` would put the node + /// in a different SHM tree from its siblings and break the launch it + /// belongs to. + /// + /// Kept so an existing launch file still parses, and warned about at launch + /// so nobody keeps believing it does something. Giving it real meaning is a + /// horus_core design decision (topic remapping), not a launcher change. #[serde(default)] pub namespace: Option, @@ -690,10 +703,25 @@ pub fn run_launch( // Check if any process has exited let mut stopped_indices: Vec = Vec::new(); + // What to record for nodes that leave `processes` this iteration. + // + // The session manifest is what `horus launch --status` reads, and its + // STATUS column has "crashed" and "stopped" arms that nothing could + // reach: `status` was assigned exactly once, `"running"` at + // construction, and never mutated. So a node that died was still listed + // as running — and this very loop already knows it died, because it + // logs `NodeCrashed` two lines down. + let mut departed: Vec<(String, &'static str)> = Vec::new(); for (i, entry) in processes.iter_mut().enumerate() { match entry.child.try_wait() { Ok(Some(status)) => { let should_restart = should_restart_node(&entry.node.restart, &status); + // A non-zero exit is a crash; a clean one is an ordinary stop. + let final_status = if status.success() { + "stopped" + } else { + "crashed" + }; if !status.success() { println!( @@ -760,6 +788,7 @@ pub fn run_launch( e ); stopped_indices.push(i); + departed.push((entry.name.clone(), final_status)); } } } else if should_restart { @@ -770,8 +799,10 @@ pub fn run_launch( MAX_RESTARTS, ); stopped_indices.push(i); + departed.push((entry.name.clone(), final_status)); } else { stopped_indices.push(i); + departed.push((entry.name.clone(), final_status)); } } Ok(None) => {} // Still running @@ -786,13 +817,18 @@ pub fn run_launch( processes.remove(i); } - // Update session manifest with current process states - for (i, entry) in processes.iter().enumerate() { - if let Some(se) = session.processes.get_mut(i) { - se.pid = entry.child.id(); - se.restart_count = entry.restart_count; - } - } + // Update session manifest with current process states. + // + // Keyed by NAME, not by position. The positional form went wrong the + // moment anything was removed from `processes`: the indices shifted, so + // a surviving node's PID was written into a dead node's row, and an + // operator asking whether the perception node crashed was told + // "running" beside a PID belonging to a different, healthy process. + let live: Vec<(String, u32, u32)> = processes + .iter() + .map(|e| (e.name.clone(), e.child.id(), e.restart_count)) + .collect(); + refresh_session_manifest(&mut session, &live, &departed); // Re-discover Schedulers every ~2s (every 20 iterations) if session.processes.iter().any(|e| e.scheduler_name.is_none()) { session.discover_schedulers(); @@ -885,6 +921,43 @@ pub fn run_launch( Ok(()) } +/// Write the supervisor's view of each node back into the session manifest. +/// +/// `live` is `(name, pid, restart_count)` for every node still supervised; +/// `departed` names the nodes that left this iteration and why. +/// +/// Keyed by NAME, not by position. The positional form this replaces went wrong +/// the moment anything was removed from the live vector: the indices shifted, +/// so a surviving node's PID was written into a dead node's row, and an +/// operator asking whether the perception node crashed was told "running" +/// beside a PID belonging to a different, healthy process. +/// +/// It also sets `status`, which nothing used to mutate. `show_launch_status` +/// colours "running", "crashed" and "stopped", but the field was assigned +/// exactly once — `"running"` at construction — so the other two arms could not +/// be reached and every node read as running forever. The monitor loop already +/// knows the outcome: it logs `NodeCrashed` in the same iteration. +fn refresh_session_manifest( + session: &mut LaunchSession, + live: &[(String, u32, u32)], + departed: &[(String, &'static str)], +) { + for (name, status) in departed { + if let Some(se) = session.processes.iter_mut().find(|se| &se.name == name) { + se.status = (*status).to_string(); + } + } + for (name, pid, restart_count) in live { + if let Some(se) = session.processes.iter_mut().find(|se| &se.name == name) { + se.pid = *pid; + se.restart_count = *restart_count; + // Still supervised — including a node that crashed a moment ago and + // was successfully respawned. + se.status = "running".to_string(); + } + } +} + /// Show active launch sessions read from SHM manifests. pub fn show_launch_status() -> HorusResult<()> { let sessions = LaunchSession::read_all(); @@ -1011,13 +1084,23 @@ fn should_restart_node(policy: &str, status: &std::process::ExitStatus) -> bool /// Print the launch plan (dry run) fn print_launch_plan(config: &LaunchConfig, global_namespace: &Option) { for (i, node) in config.nodes.iter().enumerate() { - let full_name = match (global_namespace, &node.namespace) { - (Some(global), Some(local)) => format!("{}/{}/{}", global, local, node.name), - (Some(ns), None) | (None, Some(ns)) => format!("{}/{}", ns, node.name), - (None, None) => node.name.clone(), - }; - - println!(" {}. {}", i + 1, full_name.white().bold()); + // The name the child is actually given (`HORUS_NODE_NAME`), which is + // the bare one. This used to compose `global/local/name` — an identity + // no process ever receives, so `--dry-run` described a launch that does + // not happen. The file-level namespace is shown separately below, + // because it does do something: it names the SHM tree. + println!(" {}. {}", i + 1, node.name.white().bold()); + if let Some(ref ns) = global_namespace { + println!(" {} {} (shared-memory tree)", "Namespace:".dimmed(), ns); + } + if let Some(ref ns) = node.namespace { + println!( + " {} {} {}", + "Namespace:".dimmed(), + ns, + "— IGNORED, see the warning at launch".yellow() + ); + } if let Some(ref pkg) = node.package { println!(" {} {}", "Package:".dimmed(), pkg); @@ -1164,7 +1247,18 @@ fn launch_node( cmd.env("HORUS_NAMESPACE", ns); } if let Some(ref ns) = node.namespace { - cmd.env("HORUS_NODE_NAMESPACE", ns); + // Deliberately NOT exported. `HORUS_NODE_NAMESPACE` had exactly one + // occurrence in the whole repository — this write — and no reader in + // Rust, Python or C++. Setting it made the field look implemented. + eprintln!( + "{} node '{}': `namespace: {}` has no effect. HORUS has no topic \ + prefixing; the file-level `namespace:` isolates a whole launch by \ + shared-memory tree, and a per-node one would split this node off \ + from its siblings. Remove it, or move it to the file level.", + "⚠".yellow(), + node.name, + ns + ); } // Set parameters as environment. @@ -2388,3 +2482,95 @@ nodes: result.unwrap(); } } + +#[cfg(test)] +mod session_manifest_tests { + use super::{refresh_session_manifest, LaunchProcessEntry, LaunchSession}; + + fn entry(name: &str, pid: u32) -> LaunchProcessEntry { + LaunchProcessEntry { + name: name.to_string(), + pid, + status: "running".to_string(), + scheduler_name: None, + control_topic: None, + node_names: Vec::new(), + restart_count: 0, + } + } + + fn session(names: &[(&str, u32)]) -> LaunchSession { + LaunchSession { + session: "s".to_string(), + launcher_pid: 1, + start_time: 0, + launch_file: "robot.launch.yaml".to_string(), + namespace: None, + processes: names.iter().map(|(n, p)| entry(n, *p)).collect(), + } + } + + /// A node that died must not be listed as running, beside somebody else's PID. + /// + /// The refresh used to walk the live vector and the manifest in parallel by + /// INDEX. The monitor loop removes a dead node from the live vector, so + /// from that point the two are misaligned: the survivor's PID was written + /// into the dead node's row. Combined with `status` never being mutated — + /// it was assigned `"running"` once at construction, and + /// `show_launch_status`'s "crashed" and "stopped" arms were unreachable — + /// `horus launch --status` told an operator the crashed perception node was + /// running, and showed a healthy process's PID next to it. + #[test] + fn a_departed_node_is_not_reported_as_running_with_a_live_pid() { + let mut s = session(&[("perception", 100), ("control", 200), ("logger", 300)]); + + // `perception` crashed and was dropped from the supervisor's vector; + // the other two are still up. + let live = vec![ + ("control".to_string(), 200u32, 0u32), + ("logger".to_string(), 300u32, 0u32), + ]; + let departed = vec![("perception".to_string(), "crashed")]; + + refresh_session_manifest(&mut s, &live, &departed); + + let perception = s.processes.iter().find(|e| e.name == "perception").unwrap(); + assert_eq!( + perception.status, "crashed", + "the loop that logs NodeCrashed already knows this node died" + ); + assert_eq!( + perception.pid, 100, + "a dead node must keep its own PID — the positional refresh wrote \ + the SURVIVOR's pid into this row" + ); + + let control = s.processes.iter().find(|e| e.name == "control").unwrap(); + assert_eq!(control.status, "running"); + assert_eq!(control.pid, 200, "a live node keeps its own PID"); + let logger = s.processes.iter().find(|e| e.name == "logger").unwrap(); + assert_eq!(logger.pid, 300); + } + + /// A clean exit is "stopped", not "crashed". + #[test] + fn a_node_that_exited_cleanly_reads_as_stopped() { + let mut s = session(&[("oneshot", 42)]); + refresh_session_manifest(&mut s, &[], &[("oneshot".to_string(), "stopped")]); + assert_eq!(s.processes[0].status, "stopped"); + } + + /// A node that crashed and was respawned is running again. + #[test] + fn a_respawned_node_reads_as_running_again() { + let mut s = session(&[("flaky", 7)]); + s.processes[0].status = "crashed".to_string(); + + // Back in the live set under a new PID, with a restart recorded. + refresh_session_manifest(&mut s, &[("flaky".to_string(), 99, 1)], &[]); + + assert_eq!(s.processes[0].status, "running"); + assert_eq!(s.processes[0].pid, 99); + assert_eq!(s.processes[0].restart_count, 1); + } +} From a92dc9597788ea7e26b6702193ee25571a9aa90d Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 23:19:24 -0700 Subject: [PATCH 18/79] fix(python): an untyped topic accepts a message object, and the quick start runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **`Topic(X).send(instance)` and `node.send(name, instance)` disagreed.** `Topic("name")` carries `GenericMessage`, and `generic_message_from_py` ran `pythonize::depythonize` on the Python object — which cannot read an opaque `#[pyclass]`, i.e. every HORUS message class and every msggen-generated class. So it raised `TypeError: Failed to convert Python object: unsupported type CmdVel`, while `Node.send` worked, because it catches that refusal and retries the object as a dict. Two spellings of one operation disagreeing is the defect, so the conversion moves to where all three publish paths share it. `message_object_as_dict` is the Rust twin of `horus._message_to_dict`, used the same way: only after a direct conversion has been refused, so it needs no guess about which types are messages. An object with nothing to extract still raises. This is what `build_messages()` prints on success and what `horus/msggen/__init__.py` promises, so both now describe something that runs. Note serde derives on the generated Rust struct would NOT have helped: the failure is on the Python side of the boundary. **`build_messages()` printed two lines that both failed.** `from horus import ` raises ImportError — a generated class is registered on the extension module and `horus/__init__.py` re-exports a fixed hard-coded list — so it prints `from horus._horus import ` now, plus a send line. **msggen promised POD zero-copy it does not provide.** A generated class resolves through the generic backend, so it travels the MessagePack path. It works; it is not built-in-type latency, and the docstring says so. **`help(horus)`'s quick start raised on its second line.** It published an `Image` over a bare-string topic and called it zero-copy: a string argument builds a `GenericMessage` topic, so the send raised and the `recv()` / `to_numpy()` lines never ran. It uses the typed spelling now, `Topic(Image, endpoint=...)`, which the documentation already showed. The Common Mistakes entry claiming "`Topic(int)` or `Topic(42)` raises TypeError" was also half wrong: `int.__name__` extracts cleanly, so `Topic(int)` silently opens a generic topic named "int". It says which is which, and how to spell a pool-backed topic. These docstrings are not incidental — `python_api_docs.rs` exists to pin them, on the stated grounds that "its docstrings are the only API reference a user gets at the prompt". Ablation: with the fallback declining, the two message-object tests fail with the exact shipped error. --- horus_py/horus/__init__.py | 10 ++- horus_py/horus/msggen/__init__.py | 19 ++++- horus_py/horus/msggen/builder.py | 8 +- horus_py/src/topic.rs | 78 ++++++++++++++++++- .../test_generic_topic_message_objects.py | 77 ++++++++++++++++++ 5 files changed, 182 insertions(+), 10 deletions(-) create mode 100644 horus_py/tests/test_generic_topic_message_objects.py diff --git a/horus_py/horus/__init__.py b/horus_py/horus/__init__.py index e2cde130b..3e75ab0a8 100644 --- a/horus_py/horus/__init__.py +++ b/horus_py/horus/__init__.py @@ -11,7 +11,7 @@ img = horus.Image(480, 640, "rgb8") # Send over a topic (zero-copy) - topic = horus.Topic("camera.rgb") + topic = horus.Topic(horus.Image, endpoint="camera.rgb") topic.send(img) # Receive (in another process) @@ -34,10 +34,16 @@ 'budget'``. The ``us`` and ``ms`` constants are available: ``from horus import us, ms`` -2. **Topic type** - ``Topic(int)`` or ``Topic(42)`` raises TypeError. +2. **Topic type** - ``Topic(42)`` raises TypeError. ``Topic(int)`` does NOT: + an unrecognised class opens a generic MessagePack topic named after it + (here, ``"int"``), which is almost never what was meant. Wrong: ``topic = Topic(42)`` Right: ``topic = Topic(CmdVel)`` or ``Topic("my_topic")`` + A pool-backed type needs the type AND the name: + ``Topic(Image, endpoint="camera.rgb")``. Passing only the name gives an + untyped topic, and sending an ``Image`` over it raises. + 3. **Rate must be positive** - ``Node(rate=0)`` or ``Node(rate=-1)`` raises ValueError. 4. **Logging outside scheduler** - ``node.log_info("msg")`` called outside diff --git a/horus_py/horus/msggen/__init__.py b/horus_py/horus/msggen/__init__.py index 6c41b6c5e..e554d5f18 100644 --- a/horus_py/horus/msggen/__init__.py +++ b/horus_py/horus/msggen/__init__.py @@ -1,8 +1,12 @@ """ HORUS Message Generator — Generate custom Rust message types from Python. -Generates Rust PyO3 code and compiles via maturin. The generated messages -get full zero-copy IPC support (~3μs latency) just like built-in types. +Generates Rust PyO3 code and compiles via maturin. + +Generated messages travel over the MessagePack path, not the zero-copy POD +path the built-in types use: `Topic()` resolves a generated class through the +generic backend. They work — `send()` flattens the instance to its fields — +but do not assume built-in-type latency for them. Usage: from horus.msggen import register_message, build_messages @@ -16,8 +20,15 @@ build_messages() # Generates Rust code + runs maturin develop - from horus import RobotStatus, Topic - topic = Topic(RobotStatus) # Full zero-copy typed topic + from horus._horus import RobotStatus # NOT `from horus import` — see below + from horus import Topic + topic = Topic(RobotStatus) + topic.send(RobotStatus(battery_level=0.9, error_code=0, + is_active=True, timestamp=0)) + +A generated class is registered on the extension module, so it is imported +from ``horus._horus``; ``horus/__init__.py`` re-exports a fixed list that a +generated name is not on. """ from .generator import ( diff --git a/horus_py/horus/msggen/builder.py b/horus_py/horus/msggen/builder.py index 382252dc5..5c1b31fbb 100644 --- a/horus_py/horus/msggen/builder.py +++ b/horus_py/horus/msggen/builder.py @@ -192,8 +192,14 @@ def build_messages(force: bool = False, verbose: bool = True) -> bool: print("\nBuild successful! Custom messages are now available.") print("\nUsage:") for msg in messages: - print(f" from horus import {msg.name}, Topic") + # `from horus import ` raises ImportError: the class is + # registered on the extension module and `horus/__init__.py` + # re-exports a fixed hard-coded list, which a generated name is + # not on. Import from `horus._horus`, where it actually lives. + print(f" from horus._horus import {msg.name}") + print(f" from horus import Topic") print(f" topic = Topic({msg.name})") + print(f" topic.send({msg.name}(...))") return True diff --git a/horus_py/src/topic.rs b/horus_py/src/topic.rs index cf7bf9923..0c55a3a6b 100644 --- a/horus_py/src/topic.rs +++ b/horus_py/src/topic.rs @@ -96,6 +96,56 @@ fn topic_lock(lock: &Mutex) -> PyResult> { .map_err(|e| PyRuntimeError::new_err(format!("Topic lock poisoned: {e}"))) } +/// A message object's public, non-callable attributes as a dict. +/// +/// The Rust twin of `horus._message_to_dict`, and used the same way: only +/// after a direct conversion has already been refused, so it needs no guess +/// about which types are messages. Returns `None` for anything with nothing to +/// extract, and for the built-in and array-like types a caller would never mean +/// to flatten. +fn message_object_as_dict<'py>( + py: Python<'py>, + obj: &pyo3::Bound<'py, PyAny>, +) -> Option> { + use pyo3::types::{PyAnyMethods, PyDict, PyDictMethods}; + + // Anything pythonize already handles, or would mangle. + if obj.is_none() + || obj.is_instance_of::() + || obj.is_instance_of::() + || obj.is_instance_of::() + || obj.is_instance_of::() + || obj.is_instance_of::() + || obj.is_instance_of::() + || obj.is_instance_of::() + || obj.is_instance_of::() + || obj.is_instance_of::() + || obj.hasattr("__array_interface__").unwrap_or(false) + || obj.hasattr("__array__").unwrap_or(false) + { + return None; + } + + let dict = PyDict::new(py); + let names = obj.dir().ok()?; + for name in names.iter() { + let Ok(key) = name.extract::() else { + continue; + }; + if key.starts_with('_') { + continue; + } + let Ok(value) = obj.getattr(key.as_str()) else { + continue; + }; + if value.is_callable() { + continue; + } + let _ = dict.set_item(key, value); + } + (!dict.is_empty()).then_some(dict) +} + /// Encode an arbitrary Python object as a `GenericMessage` (MessagePack payload). /// /// Shared by `send`, `try_send`, and `send_blocking` so an untyped @@ -103,9 +153,31 @@ fn topic_lock(lock: &Mutex) -> PyResult> { /// matter which of the three the caller reached for. fn generic_message_from_py(py: Python, message: &Py) -> PyResult { let bound = message.bind(py); - let value: serde_json::Value = pythonize::depythonize(bound).map_err(|e| { - pyo3::exceptions::PyTypeError::new_err(format!("Failed to convert Python object: {}", e)) - })?; + let value: serde_json::Value = match pythonize::depythonize(bound) { + Ok(v) => v, + // `depythonize` cannot read an opaque `#[pyclass]`, which is what every + // HORUS message class and every msggen-generated class is. So + // `Topic(RobotStatus).send(instance)` — the line `build_messages()` + // prints on success — raised `TypeError: Failed to convert Python + // object`, while `node.send("robot.status", instance)` worked, because + // `Node.send` catches that refusal and retries the object as a dict. + // Two spellings of one operation disagreeing is the defect; the + // conversion belongs here, where all three send paths share it. + Err(first) => { + let Some(as_dict) = message_object_as_dict(py, bound) else { + return Err(pyo3::exceptions::PyTypeError::new_err(format!( + "Failed to convert Python object: {}", + first + ))); + }; + pythonize::depythonize(&as_dict).map_err(|e| { + pyo3::exceptions::PyTypeError::new_err(format!( + "Failed to convert Python object: {}", + e + )) + })? + } + }; let msgpack_bytes = rmp_serde::to_vec(&value).map_err(|e| { pyo3::exceptions::PyRuntimeError::new_err(format!( "Failed to serialize to MessagePack: {}", diff --git a/horus_py/tests/test_generic_topic_message_objects.py b/horus_py/tests/test_generic_topic_message_objects.py new file mode 100644 index 000000000..63a655369 --- /dev/null +++ b/horus_py/tests/test_generic_topic_message_objects.py @@ -0,0 +1,77 @@ +"""An untyped topic must accept a message object, not just a dict. + +`Topic("name")` carries `GenericMessage` (MessagePack), and +`generic_message_from_py` ran `pythonize::depythonize` on the Python object — +which cannot read an opaque `#[pyclass]`, i.e. every HORUS message class and +every msggen-generated class. So `Topic(RobotStatus).send(instance)`, the line +`build_messages()` PRINTS on success, raised `TypeError: Failed to convert +Python object`, while `node.send("robot.status", instance)` worked, because +`Node.send` catches that refusal and retries the object as a dict. + +Two spellings of one operation disagreeing is the defect. The conversion now +lives where all three send paths share it. +""" +import uuid + +import pytest + +import horus +from horus import CmdVel, Pose2D, Topic + + +def _unique(prefix): + return f"{prefix}_{uuid.uuid4().hex[:8]}" + + +def test_a_message_class_survives_an_untyped_topic(): + t = Topic(_unique("generic_msgclass")) + t.send(CmdVel(1.0, 0.5)) + + got = t.recv() + assert got is not None, "the message must arrive" + assert got["linear"] == pytest.approx(1.0) + assert got["angular"] == pytest.approx(0.5) + + +def test_try_send_and_send_blocking_agree_with_send(): + """All three publish paths share the conversion, so all three accept it.""" + t = Topic(_unique("generic_all_paths")) + assert t.try_send(Pose2D(1.0, 2.0, 0.5)) is True + t.send_blocking(Pose2D(3.0, 4.0, 1.5), 0.2) + + seen = [] + while (m := t.recv()) is not None: + seen.append(m) + assert len(seen) == 2, f"both publishes must arrive: {seen}" + assert seen[0]["x"] == pytest.approx(1.0) + assert seen[1]["x"] == pytest.approx(3.0) + + +def test_a_plain_dict_still_works_unchanged(): + """The fallback must not disturb the ordinary MessagePack path.""" + t = Topic(_unique("generic_dict")) + t.send({"a": 1, "b": [1, 2, 3]}) + assert t.recv() == {"a": 1, "b": [1, 2, 3]} + + +def test_something_with_nothing_to_extract_still_raises(): + """The fallback is a retry, not a licence to encode anything.""" + t = Topic(_unique("generic_unencodable")) + with pytest.raises(TypeError): + t.send(lambda x: x) + + +def test_the_package_quick_start_runs(): + """`help(horus)`'s first example is the only API reference some users read. + + It published an `Image` over a bare-string topic and called it zero-copy; + the call raised and nothing after it ran. + """ + img = horus.Image(48, 64, "rgb8") + topic = horus.Topic(horus.Image, endpoint=_unique("camera.rgb")) + topic.send(img) + + received = topic.recv() + assert received is not None + arr = received.to_numpy() + assert arr.shape == (48, 64, 3) From 69a80b9c288645e1cd99ef11ef06031169ccc1a5 Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 23:29:01 -0700 Subject: [PATCH 19/79] fix(drivers): honour `simulated`, and name an exec driver after its entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **`simulated = true` did nothing, and HORUS writes it itself.** `horus add --driver --source sim` — and `--source simulated`, and `--source sim3d` — all emit `simulated = true` into horus.toml. The loader read only `sim`, so an entry the CLI generated constructed the REAL driver under `horus run --sim`: the exact failure `--sim` exists to prevent. `horus check` stayed quiet too, because `simulated` is a known field rather than an unknown one, and the loader's RESERVED list kept it away from the driver as well. The field's own doc-comment declares a purpose. It is an alias of `sim` now, extracted as `asks_for_simulation` so its truth table can be tested without a config file or the process-global environment — the same reason `sim_mode_enabled` is a free function. An explicit `sim` still wins. **An exec driver was named after its binary, not its manifest entry.** `ExecDriver::from_config` — the function a horus.toml exec entry actually takes — discarded the `[hardware.]` key its caller already holds (and uses one line later) and derived the name from the binary's file stem. So two `[hardware]` entries pointing at one binary were indistinguishable in "Exec driver '{}' launched" while the scheduler knew them by their distinct keys. It also injected `HORUS_PARAM_*` and no name, so `HORUS_DRIVER_NAME` reached nothing on the live path: the only function that set it is `from_params`, whose sole caller is a unit test. The environment-surface contract asserts, in prose, that it "is injected into every user-written exec driver" — `docs_contract.rs:774` — which is now true and can stay. `load_exec_prefix` asserted the defect as the contract, with the comment "ExecDriver node name is derived from binary name". Updated: the driver is named after the entry, matching the name the scheduler uses. Ablations: without the alias, `simulated = true` reads as "not simulated"; with the file stem restored, the driver reads "rplidar_driver" where the operator wrote `front_lidar`. --- horus_core/src/drivers/exec_driver.rs | 59 ++++++++++++++++++--- horus_core/src/drivers/mod.rs | 76 +++++++++++++++++++++++++-- horus_core/src/drivers/tests.rs | 8 ++- 3 files changed, 131 insertions(+), 12 deletions(-) diff --git a/horus_core/src/drivers/exec_driver.rs b/horus_core/src/drivers/exec_driver.rs index 4ef2e9424..ab0b28eef 100644 --- a/horus_core/src/drivers/exec_driver.rs +++ b/horus_core/src/drivers/exec_driver.rs @@ -73,8 +73,19 @@ impl ExecDriver { } } - /// Create from a path string, args, and NodeParams (parsed from horus.toml). + /// Create from a manifest entry name, path, args, and NodeParams. + /// + /// `name` is the `[hardware.]` key. It used to be discarded and the + /// driver's name derived from the binary's file stem, so two `[hardware]` + /// entries pointing at one binary were indistinguishable in the logs while + /// the scheduler knew them by their distinct manifest keys. It is also what + /// `HORUS_DRIVER_NAME` is supposed to carry: only `from_params` ever set + /// that variable, and its sole caller is a unit test, so an exec driver + /// reading it on the live path got nothing — while the environment-surface + /// contract asserts "`HORUS_DRIVER_NAME` is injected into every user-written + /// exec driver". pub fn from_config( + name: &str, path: &str, args: Vec, params: &NodeParams, @@ -108,14 +119,11 @@ impl ExecDriver { } } - let name = std::path::Path::new(path) - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or("exec_driver") - .to_string(); + // The name the operator wrote, not the binary's file stem. + env.insert("HORUS_DRIVER_NAME".to_string(), name.to_string()); Ok(Self::new( - &name, + name, ExecDriverConfig { path: PathBuf::from(path), args, @@ -358,6 +366,43 @@ mod tests { assert!(driver.config.env.contains_key("HORUS_PARAM_PORT")); } + /// The live path must name the driver after the `[hardware.]` key + /// and inject `HORUS_DRIVER_NAME`. + /// + /// `from_config` is what a horus.toml exec entry actually takes; + /// `from_params` is called only by the test above. `from_config` used to + /// discard the entry name and derive the driver's name from the binary's + /// FILE STEM, so two `[hardware]` entries pointing at one binary were + /// indistinguishable in the logs while the scheduler knew them apart. It + /// injected `HORUS_PARAM_*` and no name at all, so an exec driver reading + /// `HORUS_DRIVER_NAME` on the live path got nothing — while the + /// environment-surface contract asserts it "is injected into every + /// user-written exec driver". + #[test] + fn from_config_names_the_driver_after_the_manifest_entry() { + let params = NodeParams::new(HashMap::new()); + + let driver = ExecDriver::from_config("front_lidar", "./rplidar_driver", vec![], ¶ms) + .expect("from_config"); + + assert_eq!( + driver.name(), + "front_lidar", + "the driver must be named after the [hardware.] key, not the \ + binary's file stem — two entries can share one binary" + ); + assert_eq!( + driver + .config + .env + .get("HORUS_DRIVER_NAME") + .map(String::as_str), + Some("front_lidar"), + "HORUS_DRIVER_NAME must reach the child on the path horus.toml \ + actually takes" + ); + } + #[test] fn test_exec_driver_health_check_no_child() { let config = ExecDriverConfig::default(); diff --git a/horus_core/src/drivers/mod.rs b/horus_core/src/drivers/mod.rs index 1605b4be0..1db43ddcc 100644 --- a/horus_core/src/drivers/mod.rs +++ b/horus_core/src/drivers/mod.rs @@ -72,6 +72,28 @@ pub fn load() -> HorusResult)>> { load_from(&path) } +/// Whether a `[hardware.]` table asks to be substituted by a stub under +/// `horus run --sim`. +/// +/// `simulated` is an ALIAS of `sim`, not a second key. `horus add +/// --driver --source sim` — and `--source simulated`, and `--source sim3d` — +/// all write `simulated = true` into horus.toml, so HORUS generated a key its +/// own loader then ignored: the entry constructed the REAL driver under +/// `horus run --sim`, which is the exact failure `--sim` exists to prevent. +/// `horus check` said nothing either, because `simulated` is a known field +/// rather than an unknown one, and the loader's `RESERVED` list kept it away +/// from the driver as well. +/// +/// A free function so the truth table can be tested without a config file or +/// the process-global environment — the same reason `sim_mode_enabled` is one. +fn asks_for_simulation(config: &toml::value::Table) -> bool { + config + .get("sim") + .or_else(|| config.get("simulated")) + .and_then(|v| v.as_bool()) + .unwrap_or(false) +} + /// Load hardware nodes from a specific config file. /// /// Useful for testing with alternate configs or multi-robot setups. @@ -135,8 +157,7 @@ pub fn load_from>(path: P) -> HorusResult>(path: P) -> HorusResult HorusResult { .into()) } +#[cfg(test)] +mod sim_target_tests { + use super::asks_for_simulation; + + fn table(toml_src: &str) -> toml::value::Table { + toml::from_str(toml_src).expect("test table") + } + + /// `simulated` must substitute the stub, because HORUS writes that key. + /// + /// `horus add --driver --source sim` (and `--source simulated`, and + /// `--source sim3d`) all emit `simulated = true`. The loader read only + /// `sim`, so an entry HORUS generated itself constructed the REAL driver + /// under `horus run --sim` — the exact failure `--sim` exists to prevent — + /// and `horus check` stayed quiet, because `simulated` is a known field + /// rather than an unknown one. + #[test] + fn simulated_is_an_alias_of_sim() { + assert!( + asks_for_simulation(&table("use = \"rplidar\"\nsimulated = true\n")), + "`simulated = true` is what `horus add --source sim` writes; the \ + loader must honour it" + ); + assert!(asks_for_simulation(&table( + "use = \"rplidar\"\nsim = true\n" + ))); + } + + /// Neither key, or either set false, means the real driver. + #[test] + fn the_real_driver_is_the_default() { + assert!(!asks_for_simulation(&table("use = \"rplidar\"\n"))); + assert!(!asks_for_simulation(&table( + "use = \"rplidar\"\nsim = false\n" + ))); + assert!(!asks_for_simulation(&table( + "use = \"rplidar\"\nsimulated = false\n" + ))); + } + + /// An explicit `sim` wins over the alias — it is the documented key. + #[test] + fn sim_takes_precedence_over_the_alias() { + assert!(!asks_for_simulation(&table( + "sim = false\nsimulated = true\n" + ))); + } +} + #[cfg(test)] mod sim_mode_tests { /// Mirrors the `HORUS_SIM_MODE` predicate in `load_from`. diff --git a/horus_core/src/drivers/tests.rs b/horus_core/src/drivers/tests.rs index ee2d7a3c0..362244738 100644 --- a/horus_core/src/drivers/tests.rs +++ b/horus_core/src/drivers/tests.rs @@ -329,8 +329,12 @@ use = "exec:/bin/echo" let nodes = crate::drivers::load_from(&path).unwrap(); assert_eq!(nodes.len(), 1); assert_eq!(nodes[0].0, "camera"); - // ExecDriver node name is derived from binary name - assert_eq!(nodes[0].1.name(), "echo"); + // The driver is named after the [hardware.] key, matching the + // name the scheduler knows it by. This used to assert "echo" — the + // binary's file stem — which is the defect, not the contract: two + // [hardware] entries pointing at one binary were indistinguishable in + // the driver's own logs while the scheduler knew them apart. + assert_eq!(nodes[0].1.name(), "camera"); std::fs::remove_file(&path).ok(); } From c556dac0d230974c4e014d2adcce66367c606ecd Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 23:39:07 -0700 Subject: [PATCH 20/79] fix(cpp): a param value must not come back silently clipped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `horus_params_get_string` returned the number of bytes WRITTEN, so a caller had no way to tell a complete value from a truncated one and no way to size a buffer around it: `Params::get_string` handed back an ordinary `std::optional` with a clipped value in it, indistinguishable from a whole one. It also returned that positive count for a NULL buffer — the write is guarded, the return was not — and split UTF-8 sequences. The contract this should have had is in the same file. `horus_topic_last_error` returns -1 for a null buffer or a zero length, walks the copy back to a char boundary because "half a UTF-8 sequence is not something to hand a caller that may re-encode it", and has a test asserting both. Its inputs are strictly LESS arbitrary than a param value: a topic name is length-checked before it gets there, while a param holds paths, URLs and calibration blobs. Nothing in `params.hpp`, `horus_c.h` or the shim states a length limit, so 1024 was an artefact of a stack buffer rather than a contract — which is why Rust, Python and C++ disagreed about what `get("key")` returned for one file. Only C++ clipped. The shim now returns the value's FULL length, so `>= buf_len` means "this was truncated, and here is the buffer size that holds it", and `Params::get_string` re-calls into a `std::string` sized from it. The header documents the convention. One layer up, this class's own constructor refuses to construct on a bad file precisely because "`get(key, default)` gives the caller no way to tell those apart from the values in the file" — a silently clipped string was that same defect, delivered through that same `get`. Ablation: with the old shim, a 2000-byte value reports 15, a null buffer returns 5 instead of -1, and the truncated length is wrong. --- horus_cpp/include/horus/horus_c.h | 4 + horus_cpp/include/horus/params.hpp | 21 ++++- horus_cpp/src/c_api.rs | 133 +++++++++++++++++++++++++++-- 3 files changed, 150 insertions(+), 8 deletions(-) diff --git a/horus_cpp/include/horus/horus_c.h b/horus_cpp/include/horus/horus_c.h index 5e34b24ac..3c81172a8 100644 --- a/horus_cpp/include/horus/horus_c.h +++ b/horus_cpp/include/horus/horus_c.h @@ -545,6 +545,10 @@ int horus_params_get_i64(const HorusParams* params, const char* key, in int horus_params_set_i64(const HorusParams* params, const char* key, int64_t value); int horus_params_get_bool(const HorusParams* params, const char* key, bool* out); int horus_params_set_bool(const HorusParams* params, const char* key, bool value); +/* Returns the value's FULL length in bytes; writes at most buf_len-1 of them + * plus a NUL. A return >= buf_len means the value was truncated and names the + * buffer size needed to get it whole. Returns -1 if the key is absent, or if + * buf is NULL or buf_len is 0. Never splits a UTF-8 sequence. */ int horus_params_get_string(const HorusParams* params, const char* key, char* buf, size_t buf_len); int horus_params_set_string(const HorusParams* params, const char* key, const char* value); bool horus_params_has(const HorusParams* params, const char* key); diff --git a/horus_cpp/include/horus/params.hpp b/horus_cpp/include/horus/params.hpp index a1f4968e3..fe308dcbe 100644 --- a/horus_cpp/include/horus/params.hpp +++ b/horus_cpp/include/horus/params.hpp @@ -59,11 +59,30 @@ class Params { return horus_params_get_bool(handle_, key, &v) ? std::optional(v) : std::nullopt; } + /// The string value for `key`, complete, or nullopt if absent. + /// + /// This used to hand back whatever fitted in a 1024-byte stack buffer, with + /// no way for a caller to tell a clipped value from a whole one — the same + /// defect this class's constructor refuses to ship for a bad file, because + /// "`get(key, default)` gives the caller no way to tell those apart from + /// the values in the file". Rust and Python always returned the whole + /// string; only C++ clipped. The shim reports the required length now, so a + /// value that does not fit is fetched again into a buffer that holds it. std::optional get_string(const char* key) const { char buf[1024]; int len = horus_params_get_string(handle_, key, reinterpret_cast(buf), sizeof(buf)); if (len < 0) return std::nullopt; - return std::string(buf, static_cast(len)); + const size_t needed = static_cast(len); + if (needed < sizeof(buf)) { + return std::string(buf, needed); + } + // Did not fit: re-call with a buffer sized from the reported length. + std::string out(needed + 1, '\0'); + int again = horus_params_get_string(handle_, key, reinterpret_cast(&out[0]), + out.size()); + if (again < 0) return std::nullopt; + out.resize(static_cast(again)); + return out; } // ── Typed setters ────────────────────────────────────────────────── diff --git a/horus_cpp/src/c_api.rs b/horus_cpp/src/c_api.rs index 7b625f473..fa33e12b0 100644 --- a/horus_cpp/src/c_api.rs +++ b/horus_cpp/src/c_api.rs @@ -1381,7 +1381,25 @@ pub unsafe extern "C" fn horus_params_set_bool( } } -/// Get param as string. Writes to buf, returns length. -1 if not found. +/// Get param as string. +/// +/// Returns the value's FULL length in bytes, and writes at most `buf_len - 1` +/// of them plus a NUL. A return `>= buf_len` therefore means the value was +/// truncated and tells the caller exactly how large a buffer to re-call with. +/// Returns -1 if the key is absent, or if `buf` is null or `buf_len` is 0 — +/// the same convention as `horus_topic_last_error`. +/// +/// This used to return the number of bytes WRITTEN, which gave a caller no way +/// to tell a complete value from a clipped one, and no way to size around it: +/// the clipped string came back as an ordinary `std::optional` +/// with a value in it. It also returned that positive count for a null `buf`, +/// and split UTF-8 sequences. `horus_topic_last_error` in this file already +/// does none of those things, over inputs that are strictly less arbitrary — +/// a topic name is length-checked before it gets there, while a param value +/// holds paths, URLs and calibration blobs. Nothing anywhere states a length +/// limit for a param; 1024 was an artefact of a stack buffer, which is why +/// Rust, Python and C++ disagreed about what `get("key")` returned for one +/// file. #[no_mangle] pub unsafe extern "C" fn horus_params_get_string( params: *const HorusParams, @@ -1389,7 +1407,7 @@ pub unsafe extern "C" fn horus_params_get_string( buf: *mut u8, buf_len: usize, ) -> i32 { - if params.is_null() || key.is_null() { + if params.is_null() || key.is_null() || buf.is_null() || buf_len == 0 { return -1; } let params = &*(params as *const params_ffi::FfiParams); @@ -1400,12 +1418,16 @@ pub unsafe extern "C" fn horus_params_get_string( match params_ffi::params_get_string(params, key) { Some(v) => { let bytes = v.as_bytes(); - let copy_len = bytes.len().min(buf_len.saturating_sub(1)); - if !buf.is_null() && buf_len > 0 { - std::ptr::copy_nonoverlapping(bytes.as_ptr(), buf, copy_len); - *buf.add(copy_len) = 0; + let mut copy_len = bytes.len().min(buf_len - 1); + // Back off to a char boundary: half a UTF-8 sequence is not + // something to hand a caller that may re-encode it. + while copy_len > 0 && copy_len < bytes.len() && (bytes[copy_len] & 0xC0) == 0x80 { + copy_len -= 1; } - copy_len as i32 + std::ptr::copy_nonoverlapping(bytes.as_ptr(), buf, copy_len); + *buf.add(copy_len) = 0; + // The REQUIRED length, not the written one. + bytes.len() as i32 } None => -1, } @@ -2405,6 +2427,103 @@ mod tests { } } + /// A param value must not come back silently clipped. + /// + /// `horus_params_get_string` returned the bytes WRITTEN, so a caller could + /// not tell a complete value from a truncated one and had no way to size a + /// buffer around it: `Params::get_string` handed back an ordinary + /// `std::optional` with a clipped value in it. Rust and Python + /// always returned the whole string; only C++ clipped, over inputs — paths, + /// URLs, calibration blobs — that are strictly more arbitrary than the + /// topic names `horus_topic_last_error` already handles correctly in this + /// same file. + #[test] + fn params_get_string_reports_the_length_it_needed() { + unsafe { + let params = horus_params_new(); + let key = CString::new("calibration").unwrap(); + let value = "x".repeat(2000); + let cvalue = CString::new(value.as_str()).unwrap(); + assert_eq!( + horus_params_set_string(params, key.as_ptr(), cvalue.as_ptr()), + 0 + ); + + let mut buf = [0xAAu8; 16]; + let n = horus_params_get_string(params, key.as_ptr(), buf.as_mut_ptr(), buf.len()); + assert_eq!( + n, 2000, + "the REQUIRED length, so a caller can size a buffer that holds \ + the whole value" + ); + assert_eq!(buf[15], 0, "the terminator must still fit"); + + // Sized from the reported length, the value arrives whole. + let mut big = vec![0u8; n as usize + 1]; + let again = horus_params_get_string(params, key.as_ptr(), big.as_mut_ptr(), big.len()); + assert_eq!(again, 2000); + assert_eq!( + std::str::from_utf8(&big[..2000]).unwrap(), + value, + "the second call must return the complete value" + ); + + horus_params_destroy(params); + } + } + + /// A null buffer is an error, not a positive byte count. + #[test] + fn params_get_string_refuses_a_null_buffer() { + unsafe { + let params = horus_params_new(); + let key = CString::new("name").unwrap(); + let v = CString::new("lidar").unwrap(); + assert_eq!(horus_params_set_string(params, key.as_ptr(), v.as_ptr()), 0); + + assert_eq!( + horus_params_get_string(params, key.as_ptr(), std::ptr::null_mut(), 64), + -1, + "a null buffer used to return a positive 'bytes written'" + ); + let mut buf = [0u8; 8]; + assert_eq!( + horus_params_get_string(params, key.as_ptr(), buf.as_mut_ptr(), 0), + -1 + ); + + horus_params_destroy(params); + } + } + + /// Truncation must not split a UTF-8 sequence. + #[test] + fn params_get_string_truncates_on_a_char_boundary() { + unsafe { + let params = horus_params_new(); + let key = CString::new("label").unwrap(); + // Three-byte characters, so a naive byte cut lands mid-sequence. + let value = "\u{4e2d}".repeat(10); + let cvalue = CString::new(value.as_str()).unwrap(); + assert_eq!( + horus_params_set_string(params, key.as_ptr(), cvalue.as_ptr()), + 0 + ); + + let mut buf = [0xAAu8; 8]; + let n = horus_params_get_string(params, key.as_ptr(), buf.as_mut_ptr(), buf.len()); + assert_eq!(n, 30, "the full length is still reported"); + + let nul = buf.iter().position(|&b| b == 0).expect("NUL written"); + assert!( + std::str::from_utf8(&buf[..nul]).is_ok(), + "no split UTF-8 sequence — the caller may re-encode this" + ); + + horus_params_destroy(params); + } + } + #[test] fn c_api_params_all_types() { unsafe { From 5f568abff87d2ffe3ed0141052b5645d4dad9f87 Mon Sep 17 00:00:00 2001 From: Neos Date: Fri, 4 Sep 2026 23:46:16 -0700 Subject: [PATCH 21/79] fix(install): a prefix install must export the variable it depends on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `HORUS_PREFIX` is the whole interface to a relocated install. Both readers take it from the environment of the running `horus` process and from nowhere else — `version.rs` says so outright: "there is no second name to read, and no way to discover a prefix install from outside its own tree". `run_rust`'s cache roots and the version gate's state root are both derived from it. The installer never put it there. The rc file got a PATH line only, `horus env --init` writes proxy functions and no exports, and the prefix epilogue told the user to export `PATH` and `HORUS_SOURCE`. So for every prefix user the cache root fell back to `~/.horus/cache`, was deduped away as the legacy root — meaning the prefix cache was never searched — and the state root was `~/.horus`, holding none of that install's files. Which means the Rust half of that change landed only for a user who guessed the variable, and the two epilogue statements it was supposed to supersede were still true. `version.rs` records the intent it was meant to serve: the gate "was silently off for exactly the installs that most need it, and install.sh had to print a warning saying so". So install.sh writes `export HORUS_PREFIX` beside the PATH line — `set -gx` in config.fish, because fish has no `export` builtin and a POSIX line stops that file loading, the same trap this script already documents for PATH. With that done, the `export HORUS_SOURCE` instruction and the "version gate is inactive for a prefix install" warning are both false and go. The uninstall hint stays: nothing outside the prefix records where it was. Also: `generate_messages_from_yaml`'s docstring sat AFTER its `_HAS_YAML` raise, which made it a no-op expression rather than a docstring — `__doc__` was None and `help()` showed none of the YAML schema it documents. Moved above the guard, and it now also warns that `horus_py/src/custom_messages` is generator-owned: `build_messages()` rewrites that `mod.rs` to name only registered messages, so a file the YAML emitter drops there is declared by no `mod`, never compiled, and left behind by every later build. Ablation: with the rc line removed, a prefix install writes only `export PATH=...` and both new tests fail. --- .../tests/completion_install_contract.rs | 93 +++++++++++++++++++ horus_py/horus/msggen/generator.py | 23 ++++- install.sh | 38 +++++--- 3 files changed, 140 insertions(+), 14 deletions(-) diff --git a/horus_manager/tests/completion_install_contract.rs b/horus_manager/tests/completion_install_contract.rs index bfab9000d..325a64059 100644 --- a/horus_manager/tests/completion_install_contract.rs +++ b/horus_manager/tests/completion_install_contract.rs @@ -523,3 +523,96 @@ fn the_uninstaller_lists_every_location_the_installer_writes() { "the marker install.sh writes into .zshrc and the one uninstall.sh deletes must match" ); } + +/// A prefix install must export `HORUS_PREFIX`, not just `PATH`. +/// +/// Both readers take the prefix from the environment of the running `horus` +/// process and from nowhere else — `version.rs` says so outright: "HORUS_PREFIX +/// is the whole interface ... there is no second name to read, and no way to +/// discover a prefix install from outside its own tree". `run_rust`'s cache +/// roots and the version gate's state root are both derived from it. +/// +/// The installer never put it in the user's environment: the rc file got a PATH +/// line only, `horus env --init` writes proxy functions and no exports, and the +/// prefix epilogue told the user to export `PATH` and `HORUS_SOURCE`. So for +/// every prefix user the cache root fell back to `~/.horus/cache` — deduped +/// away as the legacy root, meaning the prefix cache was never searched — and +/// the state root was `~/.horus`, which holds none of that install's files. The +/// Rust half of the change worked only for someone who guessed the variable. +#[test] +fn a_prefix_install_exports_horus_prefix_to_the_shell() { + let sb = Sandbox::new(); + let prefix = sb.path().join("opt-horus"); + let script = format!( + "INSTALL_DIR=\"{prefix}/bin\"\n\ + HORUS_PREFIX=\"{prefix}\"\n\ + BINARY_NAME=horus\n\ + RED=''; GREEN=''; YELLOW=''; CYAN=''; BOLD=''; NC=''\n\ + info(){{ echo \" -> $1\"; }}\n\ + ok(){{ echo \" ok $1\"; }}\n\ + warn(){{ echo \" ! $1\"; }}\n\ + fail(){{ echo \" x $1\"; }}\n\ + INSTALL_START=$(date +%s)\n\ + VERSION=0.0.0-test\n\ + set -e\n\ + {tail}", + prefix = prefix.display(), + tail = install_tail() + ); + sb.run(&script, "/bin/bash", false, &[]); + + let rc = sb + .read(".bashrc") + .expect("the installer must write to the rc file"); + assert!( + rc.contains("export HORUS_PREFIX="), + "a prefix install must export HORUS_PREFIX — without it `horus run` \ + never searches the prefix cache and the version gate reads a state \ + root that holds none of this install's files. rc was:\n{rc}" + ); + assert!( + rc.contains(&prefix.display().to_string()), + "the exported value must be the prefix that was installed to:\n{rc}" + ); +} + +/// The `export HORUS_PREFIX` line must be fish-safe. +/// +/// fish has no `export` builtin, and a POSIX export line stops `config.fish` +/// loading at that point — the exact failure this file already records for the +/// PATH line, where a poisoned config stayed poisoned through every upgrade +/// because the guard found the broken line and skipped writing a working one. +#[test] +fn the_prefix_export_uses_fish_syntax_in_config_fish() { + let sb = Sandbox::new(); + let prefix = sb.path().join("opt-horus"); + let script = format!( + "INSTALL_DIR=\"{prefix}/bin\"\n\ + HORUS_PREFIX=\"{prefix}\"\n\ + BINARY_NAME=horus\n\ + RED=''; GREEN=''; YELLOW=''; CYAN=''; BOLD=''; NC=''\n\ + info(){{ echo \" -> $1\"; }}\n\ + ok(){{ echo \" ok $1\"; }}\n\ + warn(){{ echo \" ! $1\"; }}\n\ + fail(){{ echo \" x $1\"; }}\n\ + INSTALL_START=$(date +%s)\n\ + VERSION=0.0.0-test\n\ + set -e\n\ + {tail}", + prefix = prefix.display(), + tail = install_tail() + ); + sb.run(&script, "/usr/bin/fish", false, &[]); + + let rc = sb + .read(".config/fish/config.fish") + .expect("the installer must write config.fish for a fish shell"); + assert!( + rc.contains("set -gx HORUS_PREFIX"), + "config.fish needs `set -gx`, not `export`:\n{rc}" + ); + assert!( + !rc.contains("export HORUS_PREFIX"), + "a POSIX export line stops config.fish loading at that point:\n{rc}" + ); +} diff --git a/horus_py/horus/msggen/generator.py b/horus_py/horus/msggen/generator.py index 3f667bb40..2233d654c 100644 --- a/horus_py/horus/msggen/generator.py +++ b/horus_py/horus/msggen/generator.py @@ -280,12 +280,20 @@ def _generate_rust_code(msg: MessageDef) -> str: def generate_messages_from_yaml(yaml_path: str, output_dir: Optional[Path] = None) -> List[str]: - if not _HAS_YAML: - raise ImportError("pyyaml is required for YAML message generation: pip install pyyaml") - """ Generate messages from a YAML definition file. + Emitter only: this calls ``generate_message`` for each entry and never + ``register_message``, so nothing reaches the build registry and + ``build_messages()`` will not compile the result. Omit ``output_dir`` and it + writes no files at all — it just returns the sources. + + Do not point ``output_dir`` at ``horus_py/src/custom_messages``. That + directory is generator-owned: ``build_messages()`` rewrites its ``mod.rs`` + to name only the registered messages, so a file dropped there is declared + by no ``mod`` statement, never compiled, and left behind by every later + build. + YAML format: ```yaml messages: @@ -301,6 +309,15 @@ def generate_messages_from_yaml(yaml_path: str, output_dir: Optional[Path] = Non type: bool ``` """ + # After the docstring, not before it. This guard used to sit above the + # triple-quoted block, which made that block a no-op expression rather than + # a docstring: `generate_messages_from_yaml.__doc__` was None and `help()` + # showed none of the schema below. `python_api_docs.rs` states the standard + # this fails — "its docstrings are the only API reference a user gets at the + # prompt". + if not _HAS_YAML: + raise ImportError("pyyaml is required for YAML message generation: pip install pyyaml") + with open(yaml_path) as f: data = yaml.safe_load(f) diff --git a/install.sh b/install.sh index 545322f23..684ba88b5 100755 --- a/install.sh +++ b/install.sh @@ -909,6 +909,27 @@ else echo "export PATH=\"${INSTALL_DIR}:\$PATH\"" >> "$SHELL_RC" ;; esac + # A prefix install hangs entirely on this variable being in the + # environment that runs `horus`. Both readers take it from there and + # from nowhere else -- version.rs says so outright: "HORUS_PREFIX is the + # whole interface ... there is no second name to read, and no way to + # discover a prefix install from outside its own tree". The installer + # never exported it or asked the user to, so `horus run`'s cache root + # fell back to ~/.horus/cache and the version gate read a state root + # holding none of this install's files. The half of that change that + # landed in Rust only worked for a user who guessed the variable. + if [ -n "${HORUS_PREFIX:-}" ]; then + case "$SHELL_RC" in + */config.fish) + grep -qF "set -gx HORUS_PREFIX" "$SHELL_RC" 2>/dev/null || \ + echo "set -gx HORUS_PREFIX \"${HORUS_PREFIX}\"" >> "$SHELL_RC" + ;; + *) + grep -q "HORUS_PREFIX" "$SHELL_RC" 2>/dev/null || \ + echo "export HORUS_PREFIX=\"${HORUS_PREFIX}\"" >> "$SHELL_RC" + ;; + esac + fi fi export PATH="${INSTALL_DIR}:$PATH" ok "Added to PATH" @@ -1106,12 +1127,13 @@ if [ -n "$TARGET_USER" ]; then fi if [ -n "${HORUS_PREFIX:-}" ]; then - # find_horus_source_dir() (run_rust.rs) searches HORUS_SOURCE, a handful of - # fixed development paths, then the two cache roots — none of which is an - # arbitrary prefix. Say so rather than let the first `horus run` discover it. - warn "Installed under ${HORUS_PREFIX}, which 'horus run' does not search. Each user needs:" + # HORUS_PREFIX is what makes `horus run` search this prefix's cache and what + # points the version gate at this install's state root. Both read it from + # the environment of the running process, so it has to be exported -- + # exporting HORUS_SOURCE instead only papered over the first of the two. + warn "Installed under ${HORUS_PREFIX}. Each user needs:" echo " export PATH=\"${INSTALL_DIR}:\$PATH\"" - echo " export HORUS_SOURCE=\"${HORUS_SRC_DIR}\"" + echo " export HORUS_PREFIX=\"${HORUS_PREFIX}\"" echo "" # Nothing outside the prefix records where the prefix was, so the uninstaller # cannot find it either — uninstall.sh reads HORUS_PREFIX from its own @@ -1119,10 +1141,4 @@ if [ -n "${HORUS_PREFIX:-}" ]; then warn "To remove it later, pass the same prefix:" echo " curl -fsSL https://github.com/softmata/horus/raw/main/uninstall.sh | HORUS_PREFIX=${HORUS_PREFIX} bash -s -- --yes" echo "" - # version.rs reads ~/.horus/installed_version and ~/.horus/install_manifest.toml - # unconditionally, so the copies written under the prefix are state nothing - # reads back. The gate stays quiet rather than mis-firing, but it also cannot - # warn about drift on this install. - warn "The version gate is inactive for a prefix install: version.rs looks in ~/.horus, not ${HORUS_PREFIX}." - echo "" fi From e09c9fac82810a2e16e01c79a699ff0057299ae1 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 01:04:14 -0700 Subject: [PATCH 22/79] merge: drop the duplicated monitor lookup, keep main's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #188 landed the same #184 fix while this branch carried its own, and the merge resolution kept this branch's copy. Main's is the one to keep: it calls `PluginRegistry::global_bin_dir()` directly rather than through a `plugin_bin_dirs()` helper that also searched `~/.horus/bin`. That fallback was searching a path nothing writes. Every `.horus/bin` writer in the tree is PROJECT-relative — `project_dir.join(".horus/bin")` in lockfile.rs, `PathBuf::from(".horus/bin")` in registry/install.rs. The HOME-relative `~/.horus` holds installer state, never plugin binaries, which is exactly what #188 established. So the helper and the test asserting the legacy path stayed searched both go. What did not land with #188 is a guard on the property, so the drift that caused #184 — two hardcoded copies of one path, 800 lines apart in two files — is still unpinned. `monitor_plugin_lookup_contract` reads the body of `which_monitor_binary` and asserts it resolves through the accessor and builds no home-relative plugin path of its own. `which_monitor_binary` is in the binary crate, so an integration test cannot call it; reading it catches the one thing that actually went wrong. --- horus_manager/src/main.rs | 91 ++++--------------- .../tests/monitor_plugin_lookup_contract.rs | 88 ++++++++++++++++++ 2 files changed, 108 insertions(+), 71 deletions(-) create mode 100644 horus_manager/tests/monitor_plugin_lookup_contract.rs diff --git a/horus_manager/src/main.rs b/horus_manager/src/main.rs index 4dde54ccb..8477bb13a 100644 --- a/horus_manager/src/main.rs +++ b/horus_manager/src/main.rs @@ -3693,41 +3693,28 @@ fn run_command(command: Commands) -> HorusResult<()> { } } -/// The directories a globally installed plugin binary can live in, in the order -/// they are searched. -/// -/// Split out from `which_monitor_binary` so it can be asserted against the -/// directory the installer actually writes, rather than being a pair of -/// hardcoded paths that drift apart silently — which is what happened: -/// `horus install horus-monitor` symlinks into -/// `PluginRegistry::global_bin_dir()`, the platform CONFIG dir -/// (`~/.config/horus/bin` on Linux, `$XDG_CONFIG_HOME/horus/bin` when set), -/// while this lookup checked `~/.horus/bin` and nothing else. Install -/// succeeded, `horus monitor` reported the plugin as not installed, and the -/// comment on the path it did check still called it "the plugin install -/// location". -fn plugin_bin_dirs() -> Vec { - let mut dirs = Vec::new(); - if let Ok(d) = horus_manager::plugins::PluginRegistry::global_bin_dir() { - dirs.push(d); - } - // The legacy location, kept so an existing install keeps working. - if let Some(home) = dirs::home_dir() { - let legacy = home.join(".horus/bin"); - if !dirs.contains(&legacy) { - dirs.push(legacy); - } - } - dirs -} - /// Find the horus-monitor binary on known paths. fn which_monitor_binary() -> Option { - // 1. Global plugin install locations. - for dir in plugin_bin_dirs() { - let candidate = dir.join("horus-monitor"); - if candidate.exists() { - return Some(candidate); + // 1. The global plugin bin directory — the SAME accessor `horus install` + // links into (commands::pkg::register_plugin_after_install), `horus + // remove` unlinks from, `horus plugin trust` resolves against, and + // plugins::executor::discover_plugin_binary searches for every other + // plugin command. + // + // This used to hardcode `~/.horus/bin`, which nothing writes to: + // `global_bin_dir()` is `config_dir()/bin` (XDG), and `~/.horus` holds + // installer STATE — installed_version, install_manifest.toml, cache/ — + // not plugin binaries. So `horus install horus-monitor` followed by + // `horus monitor` reported "not installed" on every platform (#184). + // + // `monitor` is a declared clap subcommand, so it never reaches the + // generic plugin dispatch in main() that resolves this correctly. + // Call the accessor rather than restating the path, so the two cannot + // drift again. + if let Ok(global_bin) = horus_manager::plugins::PluginRegistry::global_bin_dir() { + let plugin_path = global_bin.join("horus-monitor"); + if plugin_path.exists() { + return Some(plugin_path); } } @@ -3774,41 +3761,3 @@ fn which_monitor_binary() -> Option { None } - -#[cfg(test)] -mod monitor_lookup_tests { - use super::plugin_bin_dirs; - - /// `horus monitor` must look where `horus install` writes. - /// - /// These were two hardcoded paths in two files about 800 lines apart, and - /// they disagreed: install symlinked into the platform config dir and the - /// lookup checked `~/.horus/bin`, so the documented two-step workflow - /// could not work and said only "Monitor plugin is not installed." - /// Asserting the relationship rather than the literal path means a change - /// to either side fails here instead of at a user's terminal. - #[test] - fn the_lookup_searches_the_directory_the_installer_writes() { - let install_dir = horus_manager::plugins::PluginRegistry::global_bin_dir() - .expect("global_bin_dir resolves"); - let searched = plugin_bin_dirs(); - assert!( - searched.contains(&install_dir), - "horus install writes {install_dir:?}, but horus monitor searches only \ - {searched:?} — the install succeeds and the launcher then reports the \ - plugin as missing" - ); - } - - /// The legacy path stays searched, so an existing install keeps working. - #[test] - fn the_legacy_location_is_still_searched() { - if let Some(home) = dirs::home_dir() { - assert!( - plugin_bin_dirs().contains(&home.join(".horus/bin")), - "dropping ~/.horus/bin silently breaks anyone who installed before the \ - lookup was corrected" - ); - } - } -} diff --git a/horus_manager/tests/monitor_plugin_lookup_contract.rs b/horus_manager/tests/monitor_plugin_lookup_contract.rs new file mode 100644 index 000000000..92217bcdc --- /dev/null +++ b/horus_manager/tests/monitor_plugin_lookup_contract.rs @@ -0,0 +1,88 @@ +//! `horus monitor` must look where `horus install` writes. +//! +//! These were two hardcoded paths in two files about 800 lines apart, and they +//! disagreed: `horus install horus-monitor` symlinks into +//! `PluginRegistry::global_bin_dir()` — the platform CONFIG dir — while the +//! lookup checked `~/.horus/bin`, which nothing writes to (`~/.horus` holds +//! installer STATE: `installed_version`, `install_manifest.toml`, `cache/`). +//! So the documented install-then-run flow could not work on any platform, and +//! all the user got was "Monitor plugin is not installed." (#184) +//! +//! `monitor` is a declared clap subcommand, so it never reaches the generic +//! plugin dispatch that resolves this correctly — which is why the drift was +//! invisible to every other plugin command. +//! +//! The fix was to call the accessor instead of restating the path. Nothing +//! pinned that, so this reads the source of `which_monitor_binary` and asserts +//! it. `which_monitor_binary` lives in the binary crate, so an integration test +//! cannot call it; reading it is what is available, and it is enough to catch +//! the one thing that went wrong — a second hardcoded copy of a path. +//! +//! Run: `cargo test -p horus_manager --test monitor_plugin_lookup_contract` + +use std::path::PathBuf; + +fn repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf() +} + +/// The body of `which_monitor_binary`, from its signature to the matching brace. +fn which_monitor_binary_body() -> String { + let src = std::fs::read_to_string(repo_root().join("src/main.rs")).expect("main.rs must exist"); + let at = src + .find("fn which_monitor_binary()") + .expect("which_monitor_binary must exist; this test would otherwise cover nothing"); + let rest = &src[at..]; + let open = rest.find('{').expect("function body"); + let mut depth = 0usize; + for (i, ch) in rest[open..].char_indices() { + match ch { + '{' => depth += 1, + '}' => { + depth -= 1; + if depth == 0 { + return rest[open..open + i + 1].to_string(); + } + } + _ => {} + } + } + panic!("unbalanced braces in which_monitor_binary"); +} + +/// The lookup resolves the global plugin directory through the accessor. +#[test] +fn the_monitor_lookup_uses_the_accessor_the_installer_writes_through() { + let body = which_monitor_binary_body(); + assert!( + body.contains("global_bin_dir()"), + "which_monitor_binary must resolve the global plugin directory through \ + PluginRegistry::global_bin_dir() — the same accessor `horus install` \ + links into, `horus remove` unlinks from and `horus plugin trust` \ + resolves against. Restating the path is what let the two drift.\n\ + Body was:\n{body}" + ); +} + +/// And it does not restate a home-relative plugin path of its own. +/// +/// The project-local `.horus/bin/horus-monitor` check is a different thing and +/// stays: it is CWD-relative and gated behind `HORUS_ALLOW_LOCAL_PLUGINS`. What +/// must not come back is a HOME-relative `~/.horus/bin`, which is the copy that +/// disagreed with the installer. +#[test] +fn the_monitor_lookup_does_not_hardcode_a_home_relative_plugin_dir() { + let body = which_monitor_binary_body(); + let offenders: Vec<&str> = body + .lines() + .filter(|l| !l.trim_start().starts_with("//")) + .filter(|l| l.contains("home_dir") || l.contains("HOME")) + .collect(); + assert!( + offenders.is_empty(), + "which_monitor_binary must not build a plugin path from the home \ + directory — `~/.horus` holds installer state, not plugin binaries. \ + Offending lines:\n{}", + offenders.join("\n") + ); +} From c8e573577fcdd7d217fce79522ac43f4b88cef80 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 5 Sep 2026 09:44:10 +0000 Subject: [PATCH 23/79] Initial plan From 1233a88aaed4380ce27ba2723c184b7b23d4cae0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 5 Sep 2026 09:49:23 +0000 Subject: [PATCH 24/79] Harden CI shared-memory cleanup Co-authored-by: starswallow <237697722+starswallow@users.noreply.github.com> --- .github/workflows/ci.yml | 1 + .github/workflows/integration-tests.yml | 3 ++- .github/workflows/multi-platform.yml | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f143e4e8d..3b99e8e11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,6 +175,7 @@ jobs: # Setup shared memory for HORUS tests - name: Setup Shared Memory run: | + rm -rf /dev/shm/horus_* # Create namespaced SHM directory structure SHM_DIR="/dev/shm/horus_ci_${GITHUB_RUN_ID:-$$}" # No sudo: sudo mkdir leaves these root-owned, which is what made diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index e2b5e21e2..7999a4551 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -66,6 +66,7 @@ jobs: # Setup shared memory with proper permissions - name: Setup Shared Memory run: | + rm -rf /dev/shm/horus_* # Create namespaced SHM directory structure SHM_DIR="/dev/shm/horus_ci_${GITHUB_RUN_ID:-$$}" # No sudo: sudo mkdir leaves these root-owned, which is what made @@ -124,7 +125,7 @@ jobs: # The gate-logic tests in that file (spread_gate_*, stall_rate_*, # clean_timing_*) are pure and fast, and DO run here -- they carry no # timing dependency, so the reasoning above does not apply to them. - cargo test --workspace --exclude horus_py --release --test '*' -- \ + cargo test --workspace --exclude horus_py --release --no-fail-fast --test '*' -- \ --test-threads=1 --nocapture \ --skip shm_fanout_latency_percentiles_cross_thread \ --skip stress_rt_isolation_from_compute_nodes \ diff --git a/.github/workflows/multi-platform.yml b/.github/workflows/multi-platform.yml index 3029ba70a..0b93f873e 100644 --- a/.github/workflows/multi-platform.yml +++ b/.github/workflows/multi-platform.yml @@ -373,6 +373,7 @@ jobs: - name: Setup Shared Memory run: | + rm -rf /dev/shm/horus_* # Create namespaced SHM directory structure SHM_DIR="/dev/shm/horus_ci_${GITHUB_RUN_ID:-$$}" # No sudo: sudo mkdir leaves these root-owned, which is what made From cf6b35bd3f18342b47831a64e24f281bafd602a1 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 10:44:16 -0700 Subject: [PATCH 25/79] test(cpp): cover the get_string re-call, which nothing exercised MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Rust shim's contract is tested in c_api.rs, but the C++ wrapper's re-fetch — the branch that runs when a value does not fit the 1024-byte stack buffer — had no coverage on either side, and that is where an off-by-one lives. Sizes either side of the buffer and exactly on it (1023, 1024, 1025, 4096), plus a value made of three-byte characters so a naive cut would land mid-sequence. Verified by linking the real library and running it, not only by compiling: with the wrapper reverted to its single-call form the binary aborts on `*got == value`, because it builds a std::string from a length the buffer never held. --- horus_cpp/tests/cpp_user_api_test.cpp | 40 +++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/horus_cpp/tests/cpp_user_api_test.cpp b/horus_cpp/tests/cpp_user_api_test.cpp index dbf2f215c..37727d5ec 100644 --- a/horus_cpp/tests/cpp_user_api_test.cpp +++ b/horus_cpp/tests/cpp_user_api_test.cpp @@ -177,6 +177,46 @@ TEST(UserApi, RuntimeParams) { EXPECT_FALSE(params.has("nope")); } +// A value longer than the internal stack buffer must come back WHOLE. +// +// `Params::get_string` reads into a 1024-byte stack buffer. It used to return +// whatever fitted, as an ordinary std::optional with no way for a +// caller to tell a clipped value from a complete one — and it clipped on a raw +// byte boundary, so a UTF-8 value could come back invalid. Rust and Python +// always returned the whole string; only C++ clipped, over values that hold +// paths, URLs and calibration blobs. +// +// The shim reports the REQUIRED length now and this wrapper re-fetches when the +// value does not fit. That re-call is the part with an off-by-one to get wrong, +// and nothing exercised it. +TEST(UserApi, RuntimeParamsLongStringRoundTrips) { + horus::Params params; + + // Either side of the 1024-byte stack buffer, and exactly on it. + for (size_t len : {1023u, 1024u, 1025u, 4096u}) { + const std::string value(len, 'x'); + params.set("blob", value.c_str()); + auto got = params.get_string("blob"); + ASSERT_TRUE(got.has_value()) << "length " << len; + EXPECT_EQ(got->size(), len) << "truncated at length " << len; + EXPECT_EQ(*got, value) << "content differs at length " << len; + } +} + +// Truncation, when a caller drives the C shim directly, must not split UTF-8. +TEST(UserApi, RuntimeParamsTruncationKeepsValidUtf8) { + horus::Params params; + // Three-byte characters, so a naive byte cut lands mid-sequence. + std::string value; + for (int i = 0; i < 10; ++i) value += "\xe4\xb8\xad"; + params.set("label", value.c_str()); + + // The wrapper always returns the whole value. + auto whole = params.get_string("label"); + ASSERT_TRUE(whole.has_value()); + EXPECT_EQ(*whole, value); +} + // ═══════════════════════════════════════════════════════════════════════════ // 5. TransformFrame // ═══════════════════════════════════════════════════════════════════════════ From 69a6b3bb4e5faa886a2153871909842f14348340 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 10:46:19 -0700 Subject: [PATCH 26/79] test(topic): stress the participant release under concurrent churn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The release path guards on (slot, generation) and per-role handle counts, and those are read-modify-write sequences on shared memory that several threads reach at once. Reading the code cannot rule out an interleaving that double-decrements or strands a slot, and both directions are bad: over-counting pins a command topic on a lossy broadcast backend, while under-counting lets `nothing_is_draining` retire unread slots beneath a subscriber that is still reading. Eight threads churn a subscriber and a publisher each, forty rounds, against one topic — so it exercises the shared-entry case as well as the generation guard. When they are joined nothing is alive, so sub_count must be 0 and pub_count must be exactly the one handle this thread still holds. 6/6 runs. The sibling-handle direction is already pinned by `one_handle_leaving_does_not_deregister_its_siblings_on_the_same_thread`: releasing on the first handle instead of the last fails it with left: 0, right: 1 — the registration taken out from under a live reader. --- horus_core/tests/participant_count_leak.rs | 74 ++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/horus_core/tests/participant_count_leak.rs b/horus_core/tests/participant_count_leak.rs index e51a1ef94..157959b98 100644 --- a/horus_core/tests/participant_count_leak.rs +++ b/horus_core/tests/participant_count_leak.rs @@ -151,3 +151,77 @@ fn one_handle_leaving_does_not_deregister_its_siblings_on_the_same_thread() { "the last handle to go gives the registration back" ); } + +/// Concurrent claim and release must leave the counters exactly where they +/// started. +/// +/// The release path guards on (slot, generation) and per-role handle counts, +/// and those are read-modify-write sequences on shared memory that several +/// threads reach at once. Reading the code cannot rule out an interleaving that +/// double-decrements or strands a slot, and both directions are bad: +/// over-counting pins a command topic on a lossy broadcast backend, while +/// under-counting lets `nothing_is_draining` retire unread slots beneath a +/// subscriber that is still reading. +/// +/// Threads churn handles against one topic; when they are all joined, nothing +/// is alive, so both counts must be back to zero. +#[test] +fn concurrent_churn_returns_every_registration() { + let name = "leak_concurrent_churn"; + let producer: Topic = Topic::new(name).expect("producer"); + producer.send(1u64); + + const THREADS: usize = 8; + const ROUNDS: usize = 40; + + let mut handles = Vec::new(); + for _ in 0..THREADS { + let n = name.to_string(); + handles.push(std::thread::spawn(move || { + for _ in 0..ROUNDS { + // A subscriber and a publisher on the same thread share ONE + // participant entry, so this exercises the per-role handle + // counts as well as the generation guard. + let sub: Topic = Topic::new(&n).expect("subscriber"); + let pubr: Topic = Topic::new(&n).expect("publisher"); + let _ = sub.try_recv(); + pubr.send(2u64); + drop(sub); + drop(pubr); + } + })); + } + for h in handles { + h.join().expect("churn thread"); + } + + // The producer above is this thread's own handle and is still alive, so it + // still holds a publisher registration. Everything the churn threads + // claimed must be gone. + assert_eq!( + producer.sub_count(), + 0, + "every churn subscriber was dropped, so no subscriber registration may \ + remain — a leak here pins the topic on a broadcast backend for the \ + life of the segment" + ); + assert_eq!( + producer.pub_count(), + 1, + "only this thread's producer is still alive; the churn publishers must \ + have given their registrations back, and none may have taken this \ + one's with it" + ); + + // And the topic still works, which under-counting would have broken. + producer.send(3u64); + let reader: Topic = Topic::new(name).expect("reader"); + let mut seen = false; + for _ in 0..1000 { + if reader.try_recv().is_some() { + seen = true; + break; + } + } + assert!(seen, "the topic must still deliver after the churn"); +} From f97a5e29c44364530db07dfdecd3c7ade2be998e Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 10:56:48 -0700 Subject: [PATCH 27/79] fix(topic): count a handle taking a role its sibling already holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An under-count I introduced with the participant release, found by an adversarial pass over that commit and confirmed by the test added here. A participant entry is keyed on (pid, thread), and its ROLE BITS are shared by every handle on that thread. So "the entry already has the producer bit" says nothing about whether the CALLING handle holds it. The per-role handle count was incremented only when the bit was newly set on the entry, or when the handle had no slot yet: if old_role & role_bit == 0 || new_handle { counter.fetch_add(1) } Two handles on one thread that each publish and subscribe therefore counted 2 and 1 instead of 2 and 2: the second handle to take an already-set role was invisible, and the first Drop handed that role back while the other handle was still using it. That is the under-count direction, which is the dangerous one and the whole reason the release path refuses rather than guesses everywhere else: `nothing_is_draining` lets a producer retire unread slots once `sub_count()` reads 0, so a lost subscriber registration becomes silent message loss on a subscriber that is still reading. The new test shows exactly that — sub_count 0 while a live handle is mid-read. The condition was unnecessary as well as wrong: `ensure_role` returns early when the calling handle already holds the role, so reaching `register_role` means the handle is newly taking it and the increment is unconditional. That makes the `new_handle` parameter dead, so it is gone rather than left threaded through three signatures. --- horus_core/src/communication/topic/header.rs | 109 +++++++++--------- .../src/communication/topic/migration.rs | 16 +-- horus_core/src/communication/topic/mod.rs | 12 +- horus_core/src/communication/topic/tests.rs | 16 ++- horus_core/tests/participant_count_leak.rs | 66 +++++++++++ 5 files changed, 138 insertions(+), 81 deletions(-) diff --git a/horus_core/src/communication/topic/header.rs b/horus_core/src/communication/topic/header.rs index 16a09911b..1d2f8d56e 100644 --- a/horus_core/src/communication/topic/header.rs +++ b/horus_core/src/communication/topic/header.rs @@ -754,13 +754,13 @@ impl TopicHeader { } /// Register as a publisher (returns slot index or error) - pub fn register_producer(&self, new_handle: bool) -> HorusResult<(usize, u16)> { - self.register_role(1, &self.publisher_count, new_handle) + pub fn register_producer(&self) -> HorusResult<(usize, u16)> { + self.register_role(1, &self.publisher_count) } /// Register as a subscriber (returns slot index or error) - pub fn register_consumer(&self, new_handle: bool) -> HorusResult<(usize, u16)> { - self.register_role(2, &self.subscriber_count, new_handle) + pub fn register_consumer(&self) -> HorusResult<(usize, u16)> { + self.register_role(2, &self.subscriber_count) } /// Claim the first slot that is genuinely free (`active == 0`), if any. @@ -842,18 +842,7 @@ impl TopicHeader { } /// Shared registration logic for producer (role_bit=1) and consumer (role_bit=2). - /// - /// `new_handle` says whether the CALLER is a handle that does not already - /// own this entry. An entry is keyed on (pid, thread), not on the handle, - /// so several `Topic`s on one thread deliberately share one slot; the - /// handle count decides when it may be released, and adding a second role - /// to a handle that already owns the entry must not bump it. - fn register_role( - &self, - role_bit: u8, - counter: &AtomicU32, - new_handle: bool, - ) -> HorusResult<(usize, u16)> { + fn register_role(&self, role_bit: u8, counter: &AtomicU32) -> HorusResult<(usize, u16)> { let now_ms = current_time_ms(); let pid = std::process::id(); let thread_hash = hash_thread_id(std::thread::current().id()) as u32; @@ -872,16 +861,25 @@ impl TopicHeader { .store(now_ms, Ordering::Release); } p.refresh_lease(now_ms, timeout_ms); - // Count this handle against the role it is registering, but - // only if it did not already hold that role here. - if old_role & role_bit == 0 || new_handle { - let counter = if role_bit & 1 != 0 { - &p.pub_handles - } else { - &p.sub_handles - }; - counter.fetch_add(1, Ordering::AcqRel); - } + // Count this handle against the role it is registering — + // unconditionally. + // + // `ensure_role` returns early when the CALLING HANDLE already + // holds the role, so reaching here means this handle is newly + // taking it. The entry's role bits say nothing about that: they + // are shared by every handle on this (pid, thread), so a + // sibling may have set the bit already. Gating on + // `old_role & role_bit == 0` therefore skipped the second + // handle to take a role, and the first `Drop` then handed back + // a role another live handle was still using — the under-count + // direction, where `nothing_is_draining` retires slots beneath + // a subscriber that is still reading. + let counter = if role_bit & 1 != 0 { + &p.pub_handles + } else { + &p.sub_handles + }; + counter.fetch_add(1, Ordering::AcqRel); return Ok((i, p.generation.load(Ordering::Acquire))); } } @@ -2755,7 +2753,7 @@ mod tests { #[test] fn register_producer_increments_pub_count() { let h = make_header(8, 8, true, 16); - let slot = h.register_producer(true).unwrap().0; + let slot = h.register_producer().unwrap().0; assert!(slot < MAX_PARTICIPANTS); assert_eq!(h.pub_count(), 1); assert_eq!(h.sub_count(), 0); @@ -2765,7 +2763,7 @@ mod tests { #[test] fn register_consumer_increments_sub_count() { let h = make_header(8, 8, true, 16); - let slot = h.register_consumer(true).unwrap().0; + let slot = h.register_consumer().unwrap().0; assert!(slot < MAX_PARTICIPANTS); assert_eq!(h.sub_count(), 1); assert_eq!(h.pub_count(), 0); @@ -3064,7 +3062,7 @@ mod tests { assert_eq!(h.sub_count(), MAX_PARTICIPANTS as u32); let slot = h - .register_producer(true) + .register_producer() .expect( "registration must sweep unconditionally: every slot is held by a \ process that no longer exists, and refusing the node's start \ @@ -3209,8 +3207,8 @@ mod tests { #[test] fn register_same_thread_reuses_slot() { let h = make_header(8, 8, true, 16); - let slot1 = h.register_producer(true).unwrap().0; - let slot2 = h.register_consumer(true).unwrap().0; + let slot1 = h.register_producer().unwrap().0; + let slot2 = h.register_consumer().unwrap().0; assert_eq!( slot1, slot2, "Same thread should reuse same participant slot" @@ -3225,8 +3223,8 @@ mod tests { #[test] fn register_producer_twice_same_thread_no_double_count() { let h = make_header(8, 8, true, 16); - let s1 = h.register_producer(true).unwrap().0; - let s2 = h.register_producer(true).unwrap().0; + let s1 = h.register_producer().unwrap().0; + let s2 = h.register_producer().unwrap().0; assert_eq!(s1, s2); assert_eq!( h.pub_count(), @@ -3238,14 +3236,14 @@ mod tests { #[test] fn register_from_different_threads_uses_different_slots() { let h = make_header(8, 8, true, 16); - let slot1 = h.register_producer(true).unwrap().0; + let slot1 = h.register_producer().unwrap().0; let header_ptr = &h as *const TopicHeader as usize; let slot2 = std::thread::spawn(move || { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives this thread (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_producer(true).unwrap().0 + h.register_producer().unwrap().0 }) .join() .unwrap(); @@ -3267,7 +3265,7 @@ mod tests { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives all spawned threads (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_producer(true).unwrap().0 + h.register_producer().unwrap().0 }) }) .collect(); @@ -3295,7 +3293,7 @@ mod tests { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives all spawned threads (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_producer(true).unwrap().0 + h.register_producer().unwrap().0 }) }) .collect(); @@ -3308,7 +3306,7 @@ mod tests { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives this thread (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_producer(true) + h.register_producer() }) .join() .unwrap(); @@ -3347,10 +3345,7 @@ mod tests { // is no doubt it is still there. let header = &h; s.spawn(move || { - let slot = header - .register_consumer(true) - .expect("subscriber registers") - .0; + let slot = header.register_consumer().expect("subscriber registers").0; let hash = hash_thread_id(std::thread::current().id()) as u32; registered_tx.send((slot, hash)).unwrap(); release_rx.recv().ok(); @@ -3373,7 +3368,7 @@ mod tests { } } - let result = h.register_producer(true); + let result = h.register_producer(); assert!( result.is_err(), @@ -3452,7 +3447,7 @@ mod tests { #[test] fn is_same_process_true_with_local_participants() { let h = make_header(8, 8, true, 16); - h.register_producer(true).unwrap(); + h.register_producer().unwrap(); assert!(h.is_same_process()); } @@ -3509,8 +3504,8 @@ mod tests { #[test] fn detect_optimal_backend_1p1c_pod_is_spsc_shm() { let h = make_inmem_header(8, 8, true, 16); - h.register_producer(true).unwrap(); - h.register_consumer(true).unwrap(); + h.register_producer().unwrap(); + h.register_consumer().unwrap(); // Every topic is SHM-backed: 1P:1C → SpscShm. assert_eq!(h.detect_optimal_backend(), BackendMode::SpscShm); } @@ -3518,15 +3513,15 @@ mod tests { #[test] fn detect_optimal_backend_1p1c_non_pod_is_spsc_shm() { let h = make_inmem_header(8, 8, false, 16); - h.register_producer(true).unwrap(); - h.register_consumer(true).unwrap(); + h.register_producer().unwrap(); + h.register_consumer().unwrap(); assert_eq!(h.detect_optimal_backend(), BackendMode::SpscShm); } #[test] fn detect_optimal_backend_single_producer_only() { let h = make_inmem_header(8, 8, true, 16); - h.register_producer(true).unwrap(); + h.register_producer().unwrap(); // 1P, 0C → SpscShm (anticipating single consumer) assert_eq!(h.detect_optimal_backend(), BackendMode::SpscShm); } @@ -3534,20 +3529,20 @@ mod tests { #[test] fn detect_optimal_backend_single_consumer_only() { let h = make_inmem_header(8, 8, true, 16); - h.register_consumer(true).unwrap(); + h.register_consumer().unwrap(); assert_eq!(h.detect_optimal_backend(), BackendMode::SpscShm); } #[test] fn detect_optimal_backend_cross_thread_spsc_shm() { let h = make_inmem_header(8, 8, true, 16); - h.register_producer(true).unwrap(); + h.register_producer().unwrap(); let header_ptr = &h as *const TopicHeader as usize; std::thread::spawn(move || { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives this thread (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_consumer(true).unwrap(); + h.register_consumer().unwrap(); }) .join() .unwrap(); @@ -3558,7 +3553,7 @@ mod tests { #[test] fn detect_optimal_backend_1p_multi_c_pod_is_pod_shm() { let h = make_inmem_header(8, 8, true, 16); - h.register_producer(true).unwrap(); + h.register_producer().unwrap(); let header_ptr = &h as *const TopicHeader as usize; // Register 2 consumers from different threads @@ -3567,7 +3562,7 @@ mod tests { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives this thread (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_consumer(true).unwrap(); + h.register_consumer().unwrap(); }) .join() .unwrap(); @@ -3582,7 +3577,7 @@ mod tests { #[test] fn detect_optimal_backend_multi_p_1c_is_mpsc_shm() { let h = make_inmem_header(8, 8, true, 16); - h.register_consumer(true).unwrap(); + h.register_consumer().unwrap(); let header_ptr = &h as *const TopicHeader as usize; // Register 2 producers from different threads @@ -3591,7 +3586,7 @@ mod tests { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives this thread (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_producer(true).unwrap(); + h.register_producer().unwrap(); }) .join() .unwrap(); @@ -3803,7 +3798,7 @@ mod tests { // subscriber. Registration now takes a genuinely FREE slot first and // leaves the expired entry alone; an expired slot is considered only // when no free one remains, and then only if its owner is really gone. - let slot = h.register_producer(true).unwrap().0; + let slot = h.register_producer().unwrap().0; assert_ne!( slot, 0, "must not evict an expired slot while others are free" diff --git a/horus_core/src/communication/topic/migration.rs b/horus_core/src/communication/topic/migration.rs index 89534336e..a4eff6b69 100644 --- a/horus_core/src/communication/topic/migration.rs +++ b/horus_core/src/communication/topic/migration.rs @@ -510,8 +510,8 @@ mod tests { #[test] fn migrate_to_optimal_with_same_thread_pod_producer_consumer() { let h = make_header(true); - h.register_producer(true).unwrap(); - h.register_consumer(true).unwrap(); + h.register_producer().unwrap(); + h.register_consumer().unwrap(); let m = BackendMigrator::new(&h); // POD, 1P:1C → SpscShm (every topic is SHM-backed) let result = m.migrate_to_optimal(); @@ -522,8 +522,8 @@ mod tests { #[test] fn migrate_to_optimal_with_same_thread_non_pod() { let h = make_header(false); - h.register_producer(true).unwrap(); - h.register_consumer(true).unwrap(); + h.register_producer().unwrap(); + h.register_consumer().unwrap(); let m = BackendMigrator::new(&h); // Same thread, non-POD, 1P:1C → SpscShm (real topics are shm_backed // regardless of pod-ness; SHM supports serde via send_shm_sp_serde). @@ -545,8 +545,8 @@ mod tests { #[test] fn is_optimal_false_after_topology_change() { let h = make_header(true); - h.register_producer(true).unwrap(); - h.register_consumer(true).unwrap(); + h.register_producer().unwrap(); + h.register_consumer().unwrap(); // Optimal is SpscShm but current is still Unknown let m = BackendMigrator::new(&h); assert!(!m.is_optimal()); @@ -555,8 +555,8 @@ mod tests { #[test] fn is_optimal_true_after_migration() { let h = make_header(true); - h.register_producer(true).unwrap(); - h.register_consumer(true).unwrap(); + h.register_producer().unwrap(); + h.register_consumer().unwrap(); let m = BackendMigrator::new(&h); m.migrate_to_optimal(); assert!(m.is_optimal()); diff --git a/horus_core/src/communication/topic/mod.rs b/horus_core/src/communication/topic/mod.rs index bcdf73324..e044c04fe 100644 --- a/horus_core/src/communication/topic/mod.rs +++ b/horus_core/src/communication/topic/mod.rs @@ -1727,15 +1727,13 @@ impl RingTopic< } let header = self.header(); - // A handle that has no participant entry yet is a NEW sharer of - // whichever entry it lands on; one that already has a slot is only - // adding a second role to the entry it owns, and must not be counted - // twice — see `ParticipantEntry::handles`. - let new_handle = local.slot_index < 0; + // Reaching here means this handle does NOT yet hold the role — the + // early return above covers the case where it does — so the entry's + // per-role handle count is incremented for it unconditionally. let (slot, generation) = if is_producer { - header.register_producer(new_handle)? + header.register_producer()? } else { - header.register_consumer(new_handle)? + header.register_consumer()? }; local.slot_index = slot as i32; diff --git a/horus_core/src/communication/topic/tests.rs b/horus_core/src/communication/topic/tests.rs index b5c85e8ea..8fc5cf708 100644 --- a/horus_core/src/communication/topic/tests.rs +++ b/horus_core/src/communication/topic/tests.rs @@ -4176,7 +4176,7 @@ fn detect_backend_cross_process_multi_p_0c_mpsc_shm() { fn register_producer_returns_slot_index() { let mut h = TopicHeader::zeroed(); h.init(8, 4, true, 64, 8, "TestType", 0); - let slot = h.register_producer(true).expect("should succeed").0; + let slot = h.register_producer().expect("should succeed").0; assert!(slot < 16, "Slot index must be < MAX_PARTICIPANTS"); assert_eq!(h.pub_count(), 1); assert_eq!(h.sub_count(), 0); @@ -4186,7 +4186,7 @@ fn register_producer_returns_slot_index() { fn register_consumer_returns_slot_index() { let mut h = TopicHeader::zeroed(); h.init(8, 4, true, 64, 8, "TestType", 0); - let slot = h.register_consumer(true).expect("should succeed").0; + let slot = h.register_consumer().expect("should succeed").0; assert!(slot < 16); assert_eq!(h.sub_count(), 1); assert_eq!(h.pub_count(), 0); @@ -4196,8 +4196,8 @@ fn register_consumer_returns_slot_index() { fn register_both_roles_same_thread_reuses_slot() { let mut h = TopicHeader::zeroed(); h.init(8, 4, true, 64, 8, "TestType", 0); - let slot_pub = h.register_producer(true).expect("producer").0; - let slot_sub = h.register_consumer(true).expect("consumer").0; + let slot_pub = h.register_producer().expect("producer").0; + let slot_sub = h.register_consumer().expect("consumer").0; // Same thread should reuse the same participant entry assert_eq!( slot_pub, slot_sub, @@ -4229,9 +4229,7 @@ fn register_16_producers_fills_all_slots() { // Now try registering from our actual thread — should fail since all slots taken // (our thread hash won't match any of the fake ones) // But our PID matches so it may find us by PID... let's use a distinct thread - let result = test_spawn(move || h.register_producer(true)) - .join() - .unwrap(); + let result = test_spawn(move || h.register_producer()).join().unwrap(); assert!( result.is_err(), @@ -4340,7 +4338,7 @@ fn expired_slot_reclaimed_by_new_registration() { // expired, and evicting it silently deregistered working publishers and // subscribers. A new registration must take a free slot while one exists. let slot = h - .register_producer(true) + .register_producer() .expect("a free slot is available, so registration must succeed") .0; assert_ne!(slot, 0, "must not evict slot 0 while free slots remain"); @@ -4401,7 +4399,7 @@ fn concurrent_reclaim_exactly_one_winner() { // shared read-access from multiple threads is sound. let h = unsafe { &*(hptr as *const TopicHeader) }; b.wait(); - if h.register_producer(true).is_ok() { + if h.register_producer().is_ok() { sc.fetch_add(1, Ordering::Relaxed); } }) diff --git a/horus_core/tests/participant_count_leak.rs b/horus_core/tests/participant_count_leak.rs index 157959b98..1c9edd6c3 100644 --- a/horus_core/tests/participant_count_leak.rs +++ b/horus_core/tests/participant_count_leak.rs @@ -225,3 +225,69 @@ fn concurrent_churn_returns_every_registration() { } assert!(seen, "the topic must still deliver after the churn"); } + +/// A handle taking a role a SIBLING already holds must still be counted. +/// +/// An entry is keyed on (pid, thread) and its role bits are shared by every +/// handle on that thread, so "the entry already has the producer bit" says +/// nothing about whether THIS handle holds it. The per-role handle count was +/// incremented only when the bit was newly set on the entry, or when the handle +/// had no slot yet — so the second handle to take an already-set role was not +/// counted, and the first `Drop` handed back a role another live handle was +/// still using. +/// +/// That is the under-count direction, which is the dangerous one: +/// `nothing_is_draining` lets a producer retire unread slots once `sub_count()` +/// reads 0, so a lost subscriber registration becomes silent message loss on a +/// subscriber that is still reading. +#[test] +fn a_second_handle_taking_a_shared_role_is_counted() { + let name = "leak_shared_roles"; + + // Two handles on THIS thread, each publisher and subscriber, so both take + // both roles on one shared participant entry. + let a: Topic = Topic::new(name).expect("handle a"); + let b: Topic = Topic::new(name).expect("handle b"); + a.send(1u64); + let _ = a.try_recv(); + b.send(2u64); + let _ = b.try_recv(); + + assert_eq!(a.pub_count(), 1, "precondition: one thread, one entry"); + assert_eq!(a.sub_count(), 1); + + drop(a); + + assert_eq!( + b.pub_count(), + 1, + "b is still publishing, so the producer registration must survive a's \ + departure — releasing it lets a peer's detect_optimal_backend see a \ + topic with no publisher" + ); + assert_eq!( + b.sub_count(), + 1, + "b is still reading, so the subscriber registration must survive. At 0, \ + `nothing_is_draining` lets a producer retire slots b has not read" + ); + + // And b really can still both send and receive. + b.send(3u64); + let mut seen = false; + for _ in 0..1000 { + if b.try_recv().is_some() { + seen = true; + break; + } + } + assert!(seen, "the surviving handle must still receive"); + + drop(b); + let after: Topic = Topic::new(name).expect("observer"); + assert_eq!( + after.pub_count(), + 0, + "the last handle to go gives both roles back" + ); +} From 84d2decdc67cc26c678f8d0ea3a7f83b95aa4d82 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 10:57:53 -0700 Subject: [PATCH 28/79] test(monitor): pin the opt-in on the project-local plugin lookup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `monitor_plugin_lookup_contract` asserts two things: that the lookup resolves through `PluginRegistry::global_bin_dir()`, and that it builds no HOME-relative plugin path. Neither looks at the project-local branch, so deleting the `HORUS_ALLOW_LOCAL_PLUGINS` gate leaves both green. That gate is the thing standing between a cloned repository and arbitrary code execution. `.horus/bin/horus-monitor` is CWD-relative and was once checked unconditionally and BEFORE `PATH`, so `cd`-ing into a repo that ships that file and running `horus monitor` executed it directly — no trust gate, no signature check, none of the sandbox `plugins::executor` applies — and it shadowed a properly installed binary. Asserted now, and skipped cleanly if the local branch is ever removed outright, since that also satisfies the property. Ablation: with the gate dropped and the local lookup kept, the two existing tests still pass and this one fails. --- .../tests/monitor_plugin_lookup_contract.rs | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/horus_manager/tests/monitor_plugin_lookup_contract.rs b/horus_manager/tests/monitor_plugin_lookup_contract.rs index 92217bcdc..30ae02f52 100644 --- a/horus_manager/tests/monitor_plugin_lookup_contract.rs +++ b/horus_manager/tests/monitor_plugin_lookup_contract.rs @@ -86,3 +86,36 @@ fn the_monitor_lookup_does_not_hardcode_a_home_relative_plugin_dir() { offenders.join("\n") ); } + +/// And the project-local path stays behind its opt-in. +/// +/// `.horus/bin/horus-monitor` is CWD-relative. It was once checked +/// unconditionally and BEFORE `PATH`, so `cd`-ing into a cloned repository that +/// ships that file and running `horus monitor` executed it directly — no trust +/// gate, no signature check, none of the sandbox `plugins::executor` applies — +/// and it SHADOWED a properly installed binary. Cloning a repository must not +/// be sufficient to run its code. +/// +/// The two tests above would both still pass with the gate deleted: one checks +/// the accessor is used, the other that no HOME-relative path is built, and +/// neither looks at the local branch. So the property is asserted here. +#[test] +fn the_project_local_lookup_stays_gated_behind_the_opt_in() { + let body = which_monitor_binary_body(); + + let mentions_local = body.contains(".horus/bin"); + if !mentions_local { + // The local branch was removed entirely — the shadowing risk is gone + // with it, which satisfies the property this test exists to protect. + return; + } + + assert!( + body.contains("HORUS_ALLOW_LOCAL_PLUGINS"), + "which_monitor_binary still consults the project-local .horus/bin, so \ + that branch must remain gated behind HORUS_ALLOW_LOCAL_PLUGINS. \ + Without the gate, cloning a repository that ships \ + .horus/bin/horus-monitor is enough to execute it, ahead of PATH and \ + ahead of a properly installed binary.\nBody was:\n{body}" + ); +} From 82642216833f8501205fd768688e38d00652dee6 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 11:08:40 -0700 Subject: [PATCH 29/79] fix(tests): stop leaking a shared-memory namespace per isolated child MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two test files spawn a child process under a private `HORUS_NAMESPACE` — `errbuf__` and `hlogbench__` — to keep a process-wide log ring away from concurrent writers. The child materialises `/dev/shm/horus_/`, and nothing unlinks a namespace when a process exits, so every isolated child left a directory behind permanently. The names carry a PID, so they never collide and never get reused: they accumulate without bound across every run, local and CI alike. Measured on this machine: 3 runs of one of those files leak 9 directories. `/dev/shm` was at 5.0G / 88 entries when I looked, and a full `/dev/shm` is the documented cause of the rotating, unattributable test failures this workspace shows — the ones that pass in isolation and fail in a full run, teaching people to rerun instead of read. Another session independently hit the same thing and recorded it in #192's commit message. `cleanup_stale_shm()` does not cover this: it clears the topics and nodes directories of the DEFAULT namespace, which is not where a private one lives. Removed in the parent, because the child is the process that dies. Ablation: with the removal dropped and everything else identical, 3 runs leak 9 directories; with it, 0. The parent knew the namespace but had no way to name its directory: `shm_base_dir()` derives from `shm_namespace()`, which is fixed at startup and can only ever be the caller's own. Rather than restate the platform mapping in a test, `shm_base_dir_for(namespace)` names any namespace's directory, and `shm_base_dir()` is now defined in terms of it — which also removes the second copy of the `/dev/shm` vs `/tmp` vs `%TEMP%` mapping that sat beside `shm_parent_dir()`'s. --- horus_core/tests/log_cross_layer_tests.rs | 7 +++++ horus_core/tests/log_error_buffer_tests.rs | 8 ++++++ horus_sys/src/shm/mod.rs | 32 ++++++++-------------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/horus_core/tests/log_cross_layer_tests.rs b/horus_core/tests/log_cross_layer_tests.rs index db02ec919..234c3fac2 100644 --- a/horus_core/tests/log_cross_layer_tests.rs +++ b/horus_core/tests/log_cross_layer_tests.rs @@ -597,6 +597,13 @@ fn measure_in_isolated_child(test_name: &str) -> bool { .stderr(std::process::Stdio::null()) .output() .expect("spawn isolated child"); + + // The child created /dev/shm/horus_/ and nothing unlinks it when a + // process exits, so each isolated child otherwise leaves a namespace + // directory behind for good. Removed here because the child is the process + // that dies. + let _ = std::fs::remove_dir_all(horus_sys::shm::shm_base_dir_for(&ns)); + let stdout = String::from_utf8_lossy(&out.stdout); assert!( out.status.success(), diff --git a/horus_core/tests/log_error_buffer_tests.rs b/horus_core/tests/log_error_buffer_tests.rs index 4cda7554b..affc86ad1 100644 --- a/horus_core/tests/log_error_buffer_tests.rs +++ b/horus_core/tests/log_error_buffer_tests.rs @@ -72,6 +72,14 @@ fn count_in_isolated_child(test_name: &str) -> bool { .stderr(std::process::Stdio::null()) .output() .expect("spawn isolated child"); + // The child materialised /dev/shm/horus_/ and nothing unlinks it when a + // process exits, so without this every isolated child left a namespace + // directory behind — one per test, per run, forever. They accumulate across + // CI runs and local runs alike, and a full /dev/shm is the documented cause + // of the rotating, unattributable failures this suite otherwise shows. + // Removed here rather than in the child: the child is the process that dies. + let _ = std::fs::remove_dir_all(horus_sys::shm::shm_base_dir_for(&ns)); + let stdout = String::from_utf8_lossy(&out.stdout); assert!( out.status.success(), diff --git a/horus_sys/src/shm/mod.rs b/horus_sys/src/shm/mod.rs index d567b974e..4c93aa0db 100644 --- a/horus_sys/src/shm/mod.rs +++ b/horus_sys/src/shm/mod.rs @@ -245,27 +245,19 @@ pub fn shm_namespace() -> String { /// - macOS: `/tmp/horus_/` /// - Windows: `%TEMP%\horus_\` pub fn shm_base_dir() -> PathBuf { - let dir_name = format!("horus_{}", shm_namespace()); - - #[cfg(target_os = "linux")] - { - PathBuf::from("/dev/shm").join(&dir_name) - } - - #[cfg(target_os = "macos")] - { - PathBuf::from("/tmp").join(&dir_name) - } - - #[cfg(target_os = "windows")] - { - std::env::temp_dir().join(&dir_name) - } + shm_base_dir_for(&shm_namespace()) +} - #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] - { - PathBuf::from("/tmp").join(&dir_name) - } +/// Base directory for a NAMED namespace, which need not be this process's. +/// +/// The namespace a process uses is fixed at startup from `HORUS_NAMESPACE`, so +/// `shm_base_dir()` can only ever name its own. A caller that knows another +/// namespace by name — a parent that spawned a child under a private one, or +/// `horus clean` sweeping the machine — had no way to name its directory except +/// by restating the platform mapping, which is how two copies of one path drift +/// apart. +pub fn shm_base_dir_for(namespace: &str) -> PathBuf { + shm_parent_dir().join(format!("horus_{namespace}")) } /// Topics directory for shared memory message passing. From 4fde12fa85c77f4cc9e68a59ce8a115cd9270648 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 11:13:24 -0700 Subject: [PATCH 30/79] fix(topic): pool-backed try_send moves the staleness counter too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An incomplete fix of my own, found by an adversarial pass over it. Wiring `messages_total` into the typed `Topic::try_send` missed `Image`, `PointCloud` and `DepthImage`: those have their own `try_send` impls that reach `ring.try_send` directly rather than through the typed one, so they never touched the counter. They are the worst three to have missed. `messages_total` is what `SubscriptionFreshness` watches behind `.subscribe_with_timeout()`, and that drives `StalePolicy::SafeState` and `Stop` — so a camera, lidar or depth publisher using `try_send` read as 0 messages and 0 Hz while delivering every frame, and could safe-state or halt a subscriber whose data was arriving normally. High-bandwidth sensor topics are exactly the ones a freshness watchdog is put on. The non-try path was already correct: `ring.send` bumps internally. Ablation: with the three bumps removed the counter reads 0 for five frames sent. --- horus_core/src/communication/topic/mod.rs | 64 +++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/horus_core/src/communication/topic/mod.rs b/horus_core/src/communication/topic/mod.rs index e044c04fe..e1186ff03 100644 --- a/horus_core/src/communication/topic/mod.rs +++ b/horus_core/src/communication/topic/mod.rs @@ -4719,6 +4719,13 @@ impl Topic { /// Try to send an image without blocking. Returns `Err(img)` if the ring is full. pub fn try_send(&self, img: Image) -> Result<(), Image> { self.register_pub("Image"); + // Same "new data exists" signal the typed `try_send` counts. Pool-backed + // types reach `ring.try_send` directly rather than through it, so they + // were left out of that fix — and these are the high-bandwidth topics + // (camera, lidar, depth) where a staleness watchdog matters most: + // `SubscriptionFreshness` would read 0 Hz and safe-state or halt a + // subscriber whose frames were arriving normally. + self.ring.bump_messages_total(); let wire = img.to_wire(&self.pool); match self.ring.try_send(wire) { Ok(()) => { @@ -4778,6 +4785,13 @@ impl Topic { /// Try to send a point cloud without blocking. Returns `Err(pc)` if the ring is full. pub fn try_send(&self, pc: PointCloud) -> Result<(), PointCloud> { self.register_pub("PointCloud"); + // Same "new data exists" signal the typed `try_send` counts. Pool-backed + // types reach `ring.try_send` directly rather than through it, so they + // were left out of that fix — and these are the high-bandwidth topics + // (camera, lidar, depth) where a staleness watchdog matters most: + // `SubscriptionFreshness` would read 0 Hz and safe-state or halt a + // subscriber whose frames were arriving normally. + self.ring.bump_messages_total(); let wire = pc.to_wire(&self.pool); match self.ring.try_send(wire) { Ok(()) => { @@ -4837,6 +4851,13 @@ impl Topic { /// Try to send a depth image without blocking. Returns `Err(depth)` if the ring is full. pub fn try_send(&self, depth: DepthImage) -> Result<(), DepthImage> { self.register_pub("DepthImage"); + // Same "new data exists" signal the typed `try_send` counts. Pool-backed + // types reach `ring.try_send` directly rather than through it, so they + // were left out of that fix — and these are the high-bandwidth topics + // (camera, lidar, depth) where a staleness watchdog matters most: + // `SubscriptionFreshness` would read 0 Hz and safe-state or halt a + // subscriber whose frames were arriving normally. + self.ring.bump_messages_total(); let wire = depth.to_wire(&self.pool); match self.ring.try_send(wire) { Ok(()) => { @@ -5382,6 +5403,49 @@ mod staleness_signal_tests { cleanup(&name); } + /// Pool-backed types count too. + /// + /// `Image`, `PointCloud` and `DepthImage` have their own `try_send`, which + /// reaches `ring.try_send` directly rather than through the typed one — so + /// they were left out when the staleness counter was wired into the typed + /// path. These are the high-bandwidth topics a freshness watchdog is most + /// likely to be guarding: a camera publishing through `try_send` read as + /// 0 Hz, and `StalePolicy::SafeState` would safe-state a subscriber whose + /// frames were arriving normally. + #[test] + fn pool_backed_try_send_moves_the_staleness_counter() { + let name = format!("staleness_pool_{}", std::process::id()); + cleanup(&name); + let Ok(topic) = Topic::::new(&name) else { + eprintln!("skipping: no shared memory available"); + return; + }; + + let before = topic.ring.header().messages_total(); + let mut sent = 0u64; + for _ in 0..5 { + let Ok(img) = crate::memory::Image::new(4, 4, crate::types::ImageEncoding::Rgb8) else { + eprintln!("skipping: no tensor pool available"); + return; + }; + if topic.try_send(img).is_ok() { + sent += 1; + } + } + + assert!(sent > 0, "precondition: the topic accepted some frames"); + assert_eq!( + topic.ring.header().messages_total() - before, + 5, + "every try_send is counted, delivered or refused — the typed path \ + counts attempts the same way, and a publisher hammering a full \ + ring is still alive" + ); + + drop(topic); + cleanup(&name); + } + /// The same for `send_blocking`, on a topic that has backpressure. #[test] fn send_blocking_moves_the_staleness_counter() { From a9ca7d9e046636a73bc1db64cec0226efbdbf6c3 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 11:13:38 -0700 Subject: [PATCH 31/79] fix(control): horus node restart must reach a node on an executor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Another incomplete fix of my own, and the same defect class the branch exists to close: a fix that cannot reach the nodes it is for. `process_control_commands` handled RestartNode by searching `Scheduler::nodes`. The class partition sets `self.nodes = groups.main_nodes` before the tick loop, so that vector holds only the main-thread group. Every RT, Compute, Event and AsyncIo node was therefore reported as unknown — and worse than unknown: the handler un-paused it first, so `horus node restart` on an RT node silently resumed it without re-initialising, which is a different operation from the one the operator asked for. Nodes with a `.rate()` are exactly the ones an operator restarts, and `.rate()` is what makes a node RT. `init()` needs `&mut dyn Node`, which the executor owns, so this crosses the same way the watchdog ladder's safing request does: the main thread raises a flag in the shared `NodeControlMap` — registered for all five execution groups — and the owning executor consumes it once per raise. `honor_restart_request` sits beside `honor_safe_state_request` and is called from all four executors. In `tick_node`, at the top and before the failure-policy gate: a suppressed node must still be restartable, since not running well is the reason to restart one. A restart also lifts an operator pause, because a wedged node is usually paused first and leaving it paused makes the restart a no-op at the next tick gate. Ablation: without the executor honouring it, init() is never re-run (left: 0, right: 1). --- horus_core/src/scheduling/async_executor.rs | 1 + horus_core/src/scheduling/compute_executor.rs | 1 + horus_core/src/scheduling/event_executor.rs | 1 + horus_core/src/scheduling/primitives.rs | 51 ++++++++ horus_core/src/scheduling/rt_executor.rs | 115 ++++++++++++++++++ horus_core/src/scheduling/scheduler/mod.rs | 29 ++++- horus_core/src/scheduling/types.rs | 46 +++++++ 7 files changed, 240 insertions(+), 4 deletions(-) diff --git a/horus_core/src/scheduling/async_executor.rs b/horus_core/src/scheduling/async_executor.rs index 50507ea95..f83a56d2a 100644 --- a/horus_core/src/scheduling/async_executor.rs +++ b/horus_core/src/scheduling/async_executor.rs @@ -132,6 +132,7 @@ impl AsyncExecutor { // is precisely one that is not in `ready_indices`. for node in nodes.iter_mut() { super::primitives::honor_safe_state_request(node, &monitors); + super::primitives::honor_restart_request(node, &monitors); } for (i, node) in nodes.iter().enumerate() { diff --git a/horus_core/src/scheduling/compute_executor.rs b/horus_core/src/scheduling/compute_executor.rs index 5ea947797..0f7800540 100644 --- a/horus_core/src/scheduling/compute_executor.rs +++ b/horus_core/src/scheduling/compute_executor.rs @@ -672,6 +672,7 @@ impl ComputeExecutor { // pass — an Isolated node is precisely one that is not ticking. for node in nodes.iter_mut() { super::primitives::honor_safe_state_request(node, &monitors); + super::primitives::honor_restart_request(node, &monitors); } // Update last_tick for rate-limited nodes diff --git a/horus_core/src/scheduling/event_executor.rs b/horus_core/src/scheduling/event_executor.rs index 5959f9f3a..87a796075 100644 --- a/horus_core/src/scheduling/event_executor.rs +++ b/horus_core/src/scheduling/event_executor.rs @@ -330,6 +330,7 @@ impl EventExecutor { // before the gates below — an Isolated node is precisely one that // is not ticking, so gating this on tickability would never run it. super::primitives::honor_safe_state_request(&mut node, &monitors); + super::primitives::honor_restart_request(&mut node, &monitors); if !node.initialized || node.is_stopped diff --git a/horus_core/src/scheduling/primitives.rs b/horus_core/src/scheduling/primitives.rs index f83044fc8..3fbaa7c63 100644 --- a/horus_core/src/scheduling/primitives.rs +++ b/horus_core/src/scheduling/primitives.rs @@ -103,6 +103,57 @@ pub(crate) fn note_safing_failure( } } +/// Honour a pending restart request raised by `horus node restart`, if this +/// node has one. +/// +/// Same shape and same reason as [`honor_safe_state_request`]: `init()` needs +/// `&mut dyn Node`, which the executor owns, so the main thread raises a flag +/// and the owning executor consumes it here, once per raise. +/// +/// Every executor must call this at the top of its per-node pass. Skipping it +/// in one executor means `horus node restart` silently does nothing for that +/// whole class of node — which is what the control-command handler alone did +/// for all four executor classes, because after the class partition the +/// scheduler's `nodes` vector holds only the main-thread group. +/// +/// A restart also lifts an operator pause: a wedged node is usually paused +/// before it is restarted, and leaving it paused would make the restart a +/// no-op at the very next tick gate. +pub(crate) fn honor_restart_request( + node: &mut RegisteredNode, + monitors: &super::types::SharedMonitors, +) { + if !monitors + .node_controls + .take_restart_request(node.name.as_ref()) + { + return; + } + + monitors.node_controls.set_paused(node.name.as_ref(), false); + node.is_paused = false; + + let target = &mut node.node; + let panicked = guard_fault_callback(|| { + let _ = target.init(); + }); + + if panicked { + // A node that cannot initialise must not be ticked. + node.is_stopped = true; + super::rt_executor::rt_diag(format_args!( + " Restart: '{}' PANICKED in init() — node stopped", + node.name + )); + } else { + node.initialized = true; + super::rt_executor::rt_diag(format_args!( + " Restart: '{}' re-initialised on its executor", + node.name + )); + } +} + /// Honour a pending safe-state request raised by the main thread's watchdog /// ladder, if this node has one. /// diff --git a/horus_core/src/scheduling/rt_executor.rs b/horus_core/src/scheduling/rt_executor.rs index f2a464b10..d10240bfa 100644 --- a/horus_core/src/scheduling/rt_executor.rs +++ b/horus_core/src/scheduling/rt_executor.rs @@ -1222,6 +1222,12 @@ impl RtExecutor { is_first_tick: bool, release_late_ns: u64, ) { + // A pending `horus node restart` for a node this executor owns. At the + // top of the single per-node entry point, so every caller honours it + // and a suppressed node is still restartable — the whole point of + // restarting one is that it is not currently running well. + super::primitives::honor_restart_request(node, monitors); + // Failure-policy backoff (Restart) / cooldown (Skip): skip this tick // while the node is suppressed. if !node.failure_policy_allows_tick() { @@ -2110,6 +2116,115 @@ mod tests { } } + /// A node whose init counts, so a restart is observable. + struct ReinitNode { + name: String, + inits: Arc, + } + + impl Node for ReinitNode { + fn name(&self) -> &str { + &self.name + } + fn init(&mut self) -> crate::error::HorusResult<()> { + self.inits + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + Ok(()) + } + fn tick(&mut self) {} + } + + /// `horus node restart` must reach a node living on an executor. + /// + /// The control-command handler searches `Scheduler::nodes`, and the class + /// partition leaves only the main-thread group in there. So restart reached + /// BestEffort nodes and reported every RT, Compute, Event and AsyncIo node + /// as unknown — that is, every node with a `.rate()`, which is exactly what + /// an operator restarts. `init()` needs `&mut dyn Node`, owned by the + /// executor, so the request crosses through the shared control map the same + /// way the watchdog ladder's safing request does. + #[test] + fn a_restart_request_reaches_a_node_on_an_executor() { + use crate::core::NodeInfo; + use std::sync::atomic::Ordering as AOrd; + + let inits = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let mut node = make_rt_registered( + "executor_node", + Arc::new(std::sync::atomic::AtomicU64::new(0)), + ); + node.node = super::super::types::NodeKind::new(Box::new(ReinitNode { + name: "executor_node".to_string(), + inits: inits.clone(), + })); + node.context = Some(NodeInfo::new("executor_node".to_string())); + + let monitors = test_monitors(); + monitors.node_controls.register("executor_node"); + let running = Arc::new(AtomicBool::new(true)); + + // No request pending: ticking must not re-initialise anything. + RtExecutor::tick_node(&mut node, &monitors, &running, false, 0); + assert_eq!( + inits.load(AOrd::Relaxed), + 0, + "a tick with no restart pending must not re-init" + ); + + // The main thread raises the request for a node it does not own. + assert!( + monitors.node_controls.request_restart("executor_node"), + "the control map is registered for all five groups, so the node is known" + ); + RtExecutor::tick_node(&mut node, &monitors, &running, false, 0); + assert_eq!( + inits.load(AOrd::Relaxed), + 1, + "the executor that owns the node must honour the restart — searching \ + the scheduler's own `nodes` cannot reach it after the class partition" + ); + + // Honoured exactly once per raise. + RtExecutor::tick_node(&mut node, &monitors, &running, false, 0); + assert_eq!( + inits.load(AOrd::Relaxed), + 1, + "the flag is consumed, so a restart happens once per request" + ); + } + + /// A restart lifts an operator pause. + /// + /// A wedged node is usually paused before it is restarted; leaving it + /// paused would make the restart a no-op at the very next tick gate. + #[test] + fn a_restart_clears_the_pause_that_preceded_it() { + let inits = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let mut node = make_rt_registered( + "paused_node", + Arc::new(std::sync::atomic::AtomicU64::new(0)), + ); + node.node = super::super::types::NodeKind::new(Box::new(ReinitNode { + name: "paused_node".to_string(), + inits: inits.clone(), + })); + node.is_paused = true; + + let monitors = test_monitors(); + monitors.node_controls.register("paused_node"); + monitors.node_controls.set_paused("paused_node", true); + let running = Arc::new(AtomicBool::new(true)); + + monitors.node_controls.request_restart("paused_node"); + RtExecutor::tick_node(&mut node, &monitors, &running, false, 0); + + assert!(!node.is_paused, "the restart must lift the pause"); + assert!( + !monitors.node_controls.is_paused("paused_node"), + "and clear it in the shared map the executors gate on" + ); + } + /// The executor half of the graduated watchdog ladder. /// /// The ladder runs on the main thread (the only one a hung node cannot diff --git a/horus_core/src/scheduling/scheduler/mod.rs b/horus_core/src/scheduling/scheduler/mod.rs index 19fc95743..7fbb7637e 100644 --- a/horus_core/src/scheduling/scheduler/mod.rs +++ b/horus_core/src/scheduling/scheduler/mod.rs @@ -4876,6 +4876,16 @@ impl Scheduler { if let Some(ref controls) = self.node_controls { controls.set_paused(&name, false); } + // The scheduler's own `nodes` holds ONLY the main-thread group + // after the class partition, so searching it alone reached + // BestEffort nodes and reported every RT, Compute, Event and + // AsyncIo node as unknown — that is, every node with a `.rate()`, + // which is exactly what an operator restarts. + // + // `init()` needs `&mut dyn Node`, owned by the executor, so a node + // this scheduler does not hold is asked through the shared control + // map and the owning executor honours it — the same handoff the + // watchdog ladder uses for `enter_safe_state()`. match self .nodes .iter_mut() @@ -4890,10 +4900,21 @@ impl Scheduler { )); } None => { - print_line(&format!( - "[CONTROL] Restart requested for unknown node '{}'", - name - )); + let asked = self + .node_controls + .as_ref() + .is_some_and(|c| c.request_restart(&name)); + if asked { + print_line(&format!( + "[CONTROL] Node '{}' will re-initialize on its executor", + name + )); + } else { + print_line(&format!( + "[CONTROL] Restart requested for unknown node '{}'", + name + )); + } } } } diff --git a/horus_core/src/scheduling/types.rs b/horus_core/src/scheduling/types.rs index fce1130c9..f2651c0ab 100644 --- a/horus_core/src/scheduling/types.rs +++ b/horus_core/src/scheduling/types.rs @@ -840,6 +840,17 @@ pub struct NodeControl { /// would honour the request, so it will not be safed by its own executor. /// That case is what the e-stop escalation exists for. pub safe_state_requested: std::sync::atomic::AtomicBool, + /// Set by the main thread when `horus node restart` names a node the + /// scheduler does not own; cleared by the executor that does, which then + /// re-runs `init()` on it. + /// + /// Same reason as `safe_state_requested`: `init()` needs `&mut dyn Node`, + /// and after the class partition the scheduler's own `nodes` holds only the + /// main-thread group. Restart handled solely against that vector reached + /// BestEffort nodes and silently reported every RT, Compute, Event and + /// AsyncIo node as unknown — which is every node with a `.rate()`, i.e. + /// exactly the ones an operator restarts. + pub restart_requested: std::sync::atomic::AtomicBool, } impl NodeControlMap { @@ -850,6 +861,7 @@ impl NodeControlMap { stopped: std::sync::atomic::AtomicBool::new(false), health: AtomicHealthState::default(), safe_state_requested: std::sync::atomic::AtomicBool::new(false), + restart_requested: std::sync::atomic::AtomicBool::new(false), }); } @@ -912,6 +924,40 @@ impl NodeControlMap { } } + /// Ask the executor that owns `name` to re-run its `init()`. + /// + /// Returns whether the node is known to the control map at all, so the + /// caller can tell "asked" from "no such node" — the control map is + /// registered for all five execution groups, so an unknown name here really + /// is unknown. + pub fn request_restart(&self, name: &str) -> bool { + self.inner + .read() + .unwrap_or_else(|e| e.into_inner()) + .get(name) + .map(|c| { + c.restart_requested + .store(true, std::sync::atomic::Ordering::Release); + true + }) + .unwrap_or(false) + } + + /// Consume a pending restart request, returning whether one was set. + /// + /// Clears the flag, so a request is honoured exactly once per raise. + pub fn take_restart_request(&self, name: &str) -> bool { + self.inner + .read() + .unwrap_or_else(|e| e.into_inner()) + .get(name) + .map(|c| { + c.restart_requested + .swap(false, std::sync::atomic::Ordering::AcqRel) + }) + .unwrap_or(false) + } + /// Consume a pending safing request, returning whether one was set. /// /// Clears the flag, so a request is honoured exactly once per raise. From ca021a0e3cdb2e0d741837e39a28e85ea73e6ea3 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 11:23:34 -0700 Subject: [PATCH 32/79] test: three of this branch's own guards did not guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An adversarial pass over the tests added here, not over the fixes. **The 17th-participant test had gone permanently vacuous.** It filled the table by spawning threads and letting them exit, relying on the very leak that a later commit on this same branch fixed — so once registrations were returned on Drop the table stopped filling and the test skipped itself: "the participant table did not fill (1/16)". The anti-vacuity guard did its job by reporting that rather than passing quietly, but nobody read the line. It now holds HOLDERS live subscribers open on a channel until the assertions are done, so the table is genuinely full. Reverting the fix gives "Warnings seen: []" again. **The monitor-lookup guard matched a comment.** It asserted the body of `which_monitor_binary` contains "global_bin_dir()" — and the function's own comment explains what `global_bin_dir()` is, so the assertion was satisfied by the prose describing the fix rather than by the call performing it. It strips comment lines now. Ablation: replacing the call with a hardcoded `~/.horus/bin` while leaving the comment intact fails both guards, where before it would have passed. **The Python backpressure test asserted a constructor.** It called `no_backpressure_err` directly, which is the easy half; the defect was that `send_blocking_pod` ended in `.is_ok()` and discarded WHICH error came back, so the SEND PATH reported every refusal as a timeout. The new test drives a real broadcast-backed topic — two subscribers on their own threads, because an entry is keyed on (pid, thread) — and reverting the mapping reproduces the shipped symptom precisely: `HorusTimeoutError ... after 21.102µs (deadline: 50ms)`, the zero-second "timeout" the documentation used to tell users to identify with a stopwatch. --- .../src/communication/topic/migration.rs | 10 +- horus_core/src/scheduling/compute_executor.rs | 25 ++-- horus_core/src/scheduling/rt_executor.rs | 17 ++- .../participant_exhaustion_is_reported.rs | 87 ++++++++++--- .../tests/safing_precedes_the_stop_gates.rs | 120 ++++++++++++++++++ .../tests/monitor_plugin_lookup_contract.rs | 19 ++- .../test_generic_topic_message_objects.py | 64 ++++++++++ 7 files changed, 309 insertions(+), 33 deletions(-) create mode 100644 horus_core/tests/safing_precedes_the_stop_gates.rs diff --git a/horus_core/src/communication/topic/migration.rs b/horus_core/src/communication/topic/migration.rs index a4eff6b69..356e9a480 100644 --- a/horus_core/src/communication/topic/migration.rs +++ b/horus_core/src/communication/topic/migration.rs @@ -1,8 +1,14 @@ #![allow(private_interfaces)] //! Backend migration coordinator. //! -//! Handles safe backend transitions with CAS-based locking, epoch versioning, -//! and drain logic for in-flight messages. +//! Handles backend transitions with CAS-based locking, epoch versioning, and a +//! bounded settle before the switch. +//! +//! NOT a drain. This header said "drain logic for in-flight messages" while the +//! struct doc below already retracted exactly that claim: nothing in +//! `TopicHeader` counts in-flight operations, so there is nothing to drain +//! against, and a message in flight across a switch can be lost. Rustdoc shows +//! this block first, so the retraction 35 lines down was the half nobody read. use std::sync::atomic::Ordering; diff --git a/horus_core/src/scheduling/compute_executor.rs b/horus_core/src/scheduling/compute_executor.rs index 0f7800540..397e01688 100644 --- a/horus_core/src/scheduling/compute_executor.rs +++ b/horus_core/src/scheduling/compute_executor.rs @@ -658,6 +658,23 @@ impl ComputeExecutor { ready_indices.push(i); } + // Safing requested by the main thread's watchdog ladder. Applied + // to EVERY node this executor owns, not just the ones ticking this + // pass — an Isolated node is precisely one that is not ticking. + // + // ABOVE the `ready_indices.is_empty()` early-continue, not below + // it. Below, this loop was skipped on exactly the pass the comment + // describes: when nothing is ready, which is the state a set of + // isolated or rate-limited nodes is permanently in. A link-loss + // `request_safe_state_all()` under `safety.on_link_lost = + // "safe_state"` — the path that deliberately does not latch an + // e-stop and leaves the scheduler running — would then never be + // consumed by this executor's nodes at all. + for node in nodes.iter_mut() { + super::primitives::honor_safe_state_request(node, &monitors); + super::primitives::honor_restart_request(node, &monitors); + } + if ready_indices.is_empty() { // Nothing to do — sleep and check again let elapsed = loop_start.elapsed(); @@ -667,14 +684,6 @@ impl ComputeExecutor { continue; } - // Safing requested by the main thread's watchdog ladder. Applied - // to EVERY node this executor owns, not just the ones ticking this - // pass — an Isolated node is precisely one that is not ticking. - for node in nodes.iter_mut() { - super::primitives::honor_safe_state_request(node, &monitors); - super::primitives::honor_restart_request(node, &monitors); - } - // Update last_tick for rate-limited nodes let now = Instant::now(); for &i in &ready_indices { diff --git a/horus_core/src/scheduling/rt_executor.rs b/horus_core/src/scheduling/rt_executor.rs index d10240bfa..a3c441876 100644 --- a/horus_core/src/scheduling/rt_executor.rs +++ b/horus_core/src/scheduling/rt_executor.rs @@ -1965,15 +1965,30 @@ impl RtExecutor { let loop_start = Instant::now(); for (idx, node) in nodes.iter_mut().enumerate() { - if !node.initialized || node.is_stopped { + // An uninitialized node has never run, so it has nothing to + // safe and no state to leave behind. + if !node.initialized { continue; } // Safing requested by the main thread's watchdog ladder. Runs // before the pause/stop gates: an Isolated node must reach its // safe state even if it is also about to be stopped. + // + // `is_stopped` used to be part of the guard above, which made + // that sentence false for the case it was written for: a node + // already stopped — by `BudgetPolicy::Enforce`, by the kill + // rung, by `horus node kill` — was skipped before ever reaching + // this call, so it kept holding whatever setpoint it last + // commanded. Stopping a node halts its ticks; it does not put + // its actuator anywhere. The sibling executors safe every node + // they own regardless of stop state, and so does the main loop. super::primitives::honor_safe_state_request(node, &monitors); + if node.is_stopped { + continue; + } + // Check shared control flags (set by CLI: horus node pause/kill) if monitors.node_controls.is_stopped(node.name.as_ref()) { node.is_stopped = true; diff --git a/horus_core/tests/participant_exhaustion_is_reported.rs b/horus_core/tests/participant_exhaustion_is_reported.rs index b33ec65f7..f7e274b11 100644 --- a/horus_core/tests/participant_exhaustion_is_reported.rs +++ b/horus_core/tests/participant_exhaustion_is_reported.rs @@ -18,6 +18,12 @@ use horus_core::communication::Topic; use std::sync::{Arc, Mutex}; +/// The participant table's size. Mirrors `MAX_PARTICIPANTS` in +/// `topic/header.rs`, which is `pub(crate)` and so not importable here. +const MAX_PARTICIPANTS: u32 = 16; +/// Live subscribers needed to fill it, given the producer holds one slot. +const HOLDERS: usize = MAX_PARTICIPANTS as usize - 1; + /// A `tracing::Subscriber` that keeps the formatted fields of every event. /// /// Hand-rolled against `tracing` core so this test needs no new dependency. @@ -70,38 +76,77 @@ fn a_participant_that_does_not_fit_is_told_so() { }; producer.send(1u64); - // Fill the participant table. Each thread claims its own slot, and — this - // is the open leak documented in participant_count_leak.rs — the slot is - // not returned when the thread ends, which is what lets one process fill - // the table at all. The producer holds one, so 15 subscribers saturate it - // and every thread after that is refused. - for _ in 0..20 { + // Fill the participant table with LIVE holders. + // + // This used to spawn threads and let them exit, relying on the leak + // documented in participant_count_leak.rs to keep their slots occupied. + // That leak is fixed — a dropped handle gives its registration back — so + // the table stopped filling and this test skipped itself unconditionally: + // "the participant table did not fill (1/16)". The anti-vacuity guard + // reported it rather than passing quietly, which is what such a guard is + // for, but the test proved nothing until it held its registrations open. + let (ready_tx, ready_rx) = std::sync::mpsc::channel::<()>(); + let release = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let mut holders = Vec::new(); + for _ in 0..HOLDERS { let n = name.clone(); - std::thread::spawn(move || { - if let Ok(sub) = Topic::::new(&n) { - let _ = sub.try_recv(); + let ready = ready_tx.clone(); + let release = Arc::clone(&release); + holders.push(std::thread::spawn(move || { + let sub = Topic::::new(&n).expect("holder subscriber"); + let _ = sub.try_recv(); // claims this thread's participant slot + let _ = ready.send(()); + // Hold the registration open until the assertions are done. + while !release.load(std::sync::atomic::Ordering::Acquire) { + std::thread::sleep(std::time::Duration::from_millis(2)); } - }) - .join() - .expect("subscriber thread"); + drop(sub); + })); + } + drop(ready_tx); + for _ in 0..HOLDERS { + ready_rx + .recv_timeout(std::time::Duration::from_secs(30)) + .expect("every holder must claim its slot"); } - let messages = captured.lock().unwrap_or_else(|p| p.into_inner()).clone(); - - // Anti-vacuity: assert the table really did fill BEFORE asking whether the + // Anti-vacuity: assert the table really is full BEFORE asking whether the // overflow was reported. Without this the test passes on a platform where a // slot was always available and nothing was ever refused. let occupied = producer.sub_count() + producer.pub_count(); - if occupied < 16 { - eprintln!("skipping: the participant table did not fill ({occupied}/16)"); - return; + + // One more subscriber, on a thread of its own so it needs a slot of its + // own. There is none left to give it. + let n = name.clone(); + let received = std::thread::spawn(move || { + let sub = Topic::::new(&n).expect("the handle itself still constructs"); + sub.try_recv() + }) + .join() + .expect("late subscriber thread"); + + release.store(true, std::sync::atomic::Ordering::Release); + for h in holders { + h.join().expect("holder thread"); } + let messages = captured.lock().unwrap_or_else(|p| p.into_inner()).clone(); + + assert!( + occupied >= MAX_PARTICIPANTS, + "precondition: the table must be full for anything to be refused \ + ({occupied}/{MAX_PARTICIPANTS} occupied)" + ); + assert!( + received.is_none(), + "precondition: with the table full this subscriber holds no \ + registration, so it must receive nothing" + ); + assert!( messages.iter().any(|m| m.contains("participant slots")), - "five subscribers were refused a registration and will receive nothing \ - for the life of their handles. Not one of them was told. Warnings \ - seen: {:?}", + "a subscriber was refused a registration and will receive nothing for \ + the life of its handle, and was told nothing. Warnings seen: {:?}", messages ); assert!( diff --git a/horus_core/tests/safing_precedes_the_stop_gates.rs b/horus_core/tests/safing_precedes_the_stop_gates.rs new file mode 100644 index 000000000..5261c5cf6 --- /dev/null +++ b/horus_core/tests/safing_precedes_the_stop_gates.rs @@ -0,0 +1,120 @@ +//! A pending safe-state request must be consumed before a node is skipped. +//! +//! Every executor polls `honor_safe_state_request` once per pass. The property +//! that matters is not that the call exists — it is WHERE it sits relative to +//! the guards that `continue` past a node. Put it after them and the poll is +//! skipped for exactly the nodes the ladder is trying to safe: an isolated node +//! is one that is not ticking, and a stopped node is one holding whatever +//! setpoint it last commanded. +//! +//! Two orderings regressed that way and are pinned here: +//! +//! * the RT executor skipped `is_stopped` nodes before the poll, while its own +//! comment said the poll "runs before the pause/stop gates: an Isolated node +//! must reach its safe state even if it is also about to be stopped"; +//! * the compute coordinator put the poll below its `ready_indices.is_empty()` +//! early-`continue`, so on a pass with nothing ready — the steady state for +//! a set of isolated or rate-limited nodes — no node was safed at all. +//! +//! These loops run on their own threads with a live `SharedMonitors` and cannot +//! be entered from a test, so this reads the source. That is the same shape as +//! `horus_manager/tests/monitor_plugin_lookup_contract.rs`, and it carries the +//! same lesson: comments are stripped first, because both bodies *describe* +//! the ordering in prose and a guard that matched the prose would be satisfied +//! by the very comment the code contradicts. + +use std::fs; +use std::path::PathBuf; + +fn repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("horus_core has a parent") + .to_path_buf() +} + +/// A file's source with every `//` comment line removed. +fn code_only(rel: &str) -> String { + let src = fs::read_to_string(repo_root().join(rel)).unwrap_or_else(|e| panic!("{rel}: {e}")); + src.lines() + .filter(|l| !l.trim_start().starts_with("//")) + .collect::>() + .join("\n") +} + +/// The slice of `code` between the first occurrence of `from` and of `to`. +fn between(code: &str, from: &str, to: &str, what: &str) -> String { + let a = code + .find(from) + .unwrap_or_else(|| panic!("{what}: anchor {from:?} is gone; this guard would cover nothing")); + let b = code[a..] + .find(to) + .unwrap_or_else(|| panic!("{what}: anchor {to:?} is gone; this guard would cover nothing")); + code[a..a + b].to_string() +} + +#[test] +fn the_rt_executor_safes_a_node_before_it_skips_a_stopped_one() { + let code = code_only("horus_core/src/scheduling/rt_executor.rs"); + + // The per-node preamble of the tick loop: from the loop header to the first + // budget check, which is well past both the safing poll and the stop gates. + let body = between( + &code, + "for (idx, node) in nodes.iter_mut().enumerate() {", + "tick_node(", + "rt executor tick loop", + ); + + let safing = body + .find("honor_safe_state_request") + .expect("the RT executor must poll honor_safe_state_request in its tick loop"); + let stop_gate = body + .find("node.is_stopped") + .expect("the RT executor must still gate on is_stopped somewhere in the loop"); + + assert!( + safing < stop_gate, + "the RT executor skips stopped nodes at offset {stop_gate} before polling \ + honor_safe_state_request at {safing}. Stopping a node halts its ticks; it \ + does not put its actuator anywhere, so a node stopped by BudgetPolicy::Enforce \ + or `horus node kill` keeps holding its last commanded setpoint through a \ + safe-state request. The loop's own comment says this poll runs BEFORE the \ + pause/stop gates." + ); +} + +#[test] +fn the_compute_executor_safes_its_nodes_even_when_none_is_ready() { + let code = code_only("horus_core/src/scheduling/compute_executor.rs"); + + let body = between( + &code, + "if ready_indices.is_empty() {", + "Begin recording", + "compute coordinator loop", + ); + + assert!( + !body.contains("honor_safe_state_request"), + "the compute coordinator polls honor_safe_state_request AFTER its \ + `ready_indices.is_empty()` early-continue, so on any pass with nothing \ + ready no node is safed — and a set of isolated or rate-limited nodes is \ + permanently in that state. Under `safety.on_link_lost = \"safe_state\"`, \ + which deliberately does not latch an e-stop and leaves the scheduler \ + running, the request would never be consumed at all." + ); + + // ...and it is polled before that point. + let preamble = between( + &code, + "ready_indices.push(i);", + "if ready_indices.is_empty() {", + "compute coordinator preamble", + ); + assert!( + preamble.contains("honor_safe_state_request"), + "the compute coordinator must poll honor_safe_state_request before the \ + `ready_indices.is_empty()` early-continue" + ); +} diff --git a/horus_manager/tests/monitor_plugin_lookup_contract.rs b/horus_manager/tests/monitor_plugin_lookup_contract.rs index 30ae02f52..b4b74bd10 100644 --- a/horus_manager/tests/monitor_plugin_lookup_contract.rs +++ b/horus_manager/tests/monitor_plugin_lookup_contract.rs @@ -26,6 +26,23 @@ fn repo_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf() } +/// The CODE of `which_monitor_binary`, with comment lines removed. +/// +/// Stripping comments is the whole point. The first version of this guard +/// matched the raw body, and the function's own comment explains what +/// `global_bin_dir()` is — so the assertion was satisfied by the prose +/// describing the fix rather than by the call performing it, and would have +/// stayed green if the call were replaced by a hardcoded path with the comment +/// left in place. That is the failure mode a source-reading guard has to be +/// built against. +fn which_monitor_binary_code() -> String { + which_monitor_binary_body() + .lines() + .filter(|l| !l.trim_start().starts_with("//")) + .collect::>() + .join("\n") +} + /// The body of `which_monitor_binary`, from its signature to the matching brace. fn which_monitor_binary_body() -> String { let src = std::fs::read_to_string(repo_root().join("src/main.rs")).expect("main.rs must exist"); @@ -53,7 +70,7 @@ fn which_monitor_binary_body() -> String { /// The lookup resolves the global plugin directory through the accessor. #[test] fn the_monitor_lookup_uses_the_accessor_the_installer_writes_through() { - let body = which_monitor_binary_body(); + let body = which_monitor_binary_code(); assert!( body.contains("global_bin_dir()"), "which_monitor_binary must resolve the global plugin directory through \ diff --git a/horus_py/tests/test_generic_topic_message_objects.py b/horus_py/tests/test_generic_topic_message_objects.py index 63a655369..36e3d00ad 100644 --- a/horus_py/tests/test_generic_topic_message_objects.py +++ b/horus_py/tests/test_generic_topic_message_objects.py @@ -75,3 +75,67 @@ def test_the_package_quick_start_runs(): assert received is not None arr = received.to_numpy() assert arr.shape == (48, 64, 3) + + +def test_send_blocking_on_a_broadcast_topic_raises_backpressure_not_timeout(): + """The mapping must hold on the SEND PATH, not just in the constructor. + + The Rust-side test asserts `no_backpressure_err` builds the right exception. + That is the easy half: the defect was that `send_blocking_pod` ended in + `.is_ok()`, discarding WHICH `SendBlockingError` came back, so the send path + reported every refusal as a timeout no matter what the core returned. This + drives a real broadcast-backed topic instead. + + A second subscriber is what resolves a POD topic to `PodShm`, and it has to + be on another thread: an entry is keyed on (pid, thread), so two handles + here would share one registration and the topic would stay `SpscShm`. + """ + import threading + + name = _unique("estop_broadcast") + pub = Topic(CmdVel, endpoint=name) + + stop = threading.Event() + ready = threading.Barrier(3) + + def hold(): + sub = Topic(CmdVel, endpoint=name) + sub.try_recv() # claims this thread's participant slot + ready.wait() + while not stop.is_set(): + sub.try_recv() + + threads = [threading.Thread(target=hold, daemon=True) for _ in range(2)] + for t in threads: + t.start() + ready.wait() + + try: + # Resolve the backend, then confirm the topology we need. + pub.send(CmdVel(0.0, 0.0)) + if pub.provides_backpressure: + pytest.skip(f"topic did not reach a broadcast backend: {pub.backend_type}") + + with pytest.raises(horus.HorusBackpressureError) as excinfo: + pub.send_blocking(CmdVel(1.0, 0.0), 0.05) + + msg = str(excinfo.value) + assert name in msg, f"the error must name the topic: {msg}" + assert "backpressure" in msg.lower() or "MpscShm" in msg, ( + f"the core's remediation text must survive the crossing: {msg}" + ) + finally: + stop.set() + for t in threads: + t.join(timeout=5) + + +def test_provides_backpressure_reports_the_backend(): + """The property Python had no way to ask before.""" + name = _unique("bp_single") + t = Topic(CmdVel, endpoint=name) + assert t.provides_backpressure is False, "unresolved backend must fail closed" + t.send(CmdVel(0.0, 0.0)) + assert t.provides_backpressure is True, ( + "a single-subscriber POD topic resolves to SpscShm, which refuses a full ring" + ) From 8fc089f1f7e1e97b685a3d47d8d1f70a5291ef6f Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 11:31:50 -0700 Subject: [PATCH 33/79] test: anchor the safing-order guard on code, not on a comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ca021a0e` picked this file up from the working tree mid-edit, before the anchor was fixed, so the compute half has been failing on the branch: it sliced the coordinator loop between `if ready_indices.is_empty() {` and the string "Begin recording" — which is a COMMENT, and this guard strips comments before matching precisely so it cannot be satisfied by prose. The anchor was therefore never found and the test panicked with its own "anchor is gone; this guard would cover nothing" message. That message is the anti-vacuity check doing its job: a guard whose anchor has moved covers nothing, and saying so beats passing. Re-anchored on `let now = Instant::now();`, which is code. Also reflows a `panic!` the formatter wanted on its own line. --- horus_core/tests/safing_precedes_the_stop_gates.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/horus_core/tests/safing_precedes_the_stop_gates.rs b/horus_core/tests/safing_precedes_the_stop_gates.rs index 5261c5cf6..aef5234c2 100644 --- a/horus_core/tests/safing_precedes_the_stop_gates.rs +++ b/horus_core/tests/safing_precedes_the_stop_gates.rs @@ -44,9 +44,9 @@ fn code_only(rel: &str) -> String { /// The slice of `code` between the first occurrence of `from` and of `to`. fn between(code: &str, from: &str, to: &str, what: &str) -> String { - let a = code - .find(from) - .unwrap_or_else(|| panic!("{what}: anchor {from:?} is gone; this guard would cover nothing")); + let a = code.find(from).unwrap_or_else(|| { + panic!("{what}: anchor {from:?} is gone; this guard would cover nothing") + }); let b = code[a..] .find(to) .unwrap_or_else(|| panic!("{what}: anchor {to:?} is gone; this guard would cover nothing")); @@ -91,7 +91,7 @@ fn the_compute_executor_safes_its_nodes_even_when_none_is_ready() { let body = between( &code, "if ready_indices.is_empty() {", - "Begin recording", + "let now = Instant::now();", "compute coordinator loop", ); From 316dc0fb76406be05bcd172b5226339d20652030 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 11:34:56 -0700 Subject: [PATCH 34/79] fix: three more defects in this branch's own work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **`ParticipantEntry::clear` published `active = 0` first.** That store is what makes a slot claimable — `claim_free_slot` scans for `active == 0` and CASes it — so every field written after it could be overwriting a NEW owner's. The ordering was harmless while the only non-test caller was `TopicHeader::initialize`, which runs single-threaded before `magic` is published; giving a registration back on Drop made it reachable from a live process on every last-handle release. `reap_dead_participants_now` writes the identity first and says why, in this same file — the sequence now lives in `clear()` once, where it belongs. **The participant-exhaustion warnings printed a garbled sentence.** Eighteen literal spaces in the middle: "will deliver nothing". A source-wrapped string literal whose continuation backslash was lost, so the indentation was baked into the message. These two lines are the ENTIRE user-visible payload of the commit whose point was that the 16-slot wall must never be silent, and a diagnostic that exists only to be read was printed malformed. **`install.sh` exported HORUS_PREFIX only when PATH was not yet configured.** Which is the one population that does not need it. A prefix install puts `${HORUS_PREFIX}/bin` on PATH, so every upgrade and re-install takes the "PATH already configured" arm and would never have been given the variable — and install.sh records this exact mistake 60 lines above, for the shell-integration block that used to live in that same arm. It is outside the conditional now, and value-aware: a stale line naming a different prefix is replaced rather than counted as already configured. The two new tests fail with the block back inside the else arm while the fresh-install tests still pass, which is why it was invisible. --- .../src/communication/topic/dispatch.rs | 4 +- horus_core/src/communication/topic/header.rs | 15 +++- .../tests/completion_install_contract.rs | 88 +++++++++++++++++++ install.sh | 62 ++++++++----- 4 files changed, 145 insertions(+), 24 deletions(-) diff --git a/horus_core/src/communication/topic/dispatch.rs b/horus_core/src/communication/topic/dispatch.rs index 84f8e0674..9b90fdc82 100644 --- a/horus_core/src/communication/topic/dispatch.rs +++ b/horus_core/src/communication/topic/dispatch.rs @@ -1892,7 +1892,7 @@ pub(super) fn send_uninitialized< // indication that the cause is permanent for this handle. if should_report_endpoint_exhaustion(topic.name(), Exhausted::ParticipantTable) { tracing::warn!( - "topic '{}': this publisher is NOT registered and will deliver nothing — {}", + "topic '{}': this publisher is NOT registered and will deliver nothing — {}", topic.name(), e ); @@ -2634,7 +2634,7 @@ pub(super) fn recv_uninitialized< // SILENT no-comms endpoint". if should_report_endpoint_exhaustion(topic.name(), Exhausted::ParticipantTable) { tracing::warn!( - "topic '{}': this subscriber is NOT registered and will receive nothing — {}", + "topic '{}': this subscriber is NOT registered and will receive nothing — {}", topic.name(), e ); diff --git a/horus_core/src/communication/topic/header.rs b/horus_core/src/communication/topic/header.rs index 1d2f8d56e..b8bfdc890 100644 --- a/horus_core/src/communication/topic/header.rs +++ b/horus_core/src/communication/topic/header.rs @@ -167,7 +167,19 @@ impl ParticipantEntry { /// Clear this entry (use atomic for thread safety) pub(crate) fn clear(&self) { - self.active.store(0, Ordering::Release); + // Clear the identity BEFORE publishing `active = 0`: that store is what + // makes the slot claimable — `claim_free_slot` scans for + // `active == 0` and CASes it to the initializing sentinel — so anything + // written after it could be overwriting a NEW owner's fields. + // + // The order used to be the other way round, which was harmless while + // the only non-test caller was `TopicHeader::initialize`: that runs + // single-threaded before `magic` is published, so nothing could be + // scanning. Giving a registration back on `Drop` made this reachable + // from a live process on every last-handle release. + // + // `reap_dead_participants_now` writes the same sequence inline and + // carries the same reasoning; this is the one place it belongs. self.lease_expires_ms.store(0, Ordering::Release); self.role.store(0, Ordering::Release); self.pid.store(0, Ordering::Release); @@ -175,6 +187,7 @@ impl ParticipantEntry { self.source_host.store(0, Ordering::Release); self.pub_handles.store(0, Ordering::Release); self.sub_handles.store(0, Ordering::Release); + self.active.store(0, Ordering::Release); // `generation` is deliberately NOT reset. Its whole purpose is to // outlive the registration it identifies, so that a handle holding a // stale (slot, generation) can be told apart from the current owner. diff --git a/horus_manager/tests/completion_install_contract.rs b/horus_manager/tests/completion_install_contract.rs index 325a64059..e4fdce6d0 100644 --- a/horus_manager/tests/completion_install_contract.rs +++ b/horus_manager/tests/completion_install_contract.rs @@ -576,6 +576,94 @@ fn a_prefix_install_exports_horus_prefix_to_the_shell() { ); } +/// ...including on a re-install, where PATH is already configured. +/// +/// This is the population the line exists for: a prefix install puts +/// `${HORUS_PREFIX}/bin` on PATH, so every upgrade and re-install takes the +/// "PATH already configured" arm. The export first landed inside the `else` arm +/// of that check, which made it dead for exactly those users — the same mistake +/// install.sh already records 60 lines above for the shell-integration block +/// that used to live there. +#[test] +fn a_prefix_install_exports_horus_prefix_on_a_reinstall_too() { + let sb = Sandbox::new(); + let prefix = sb.path().join("opt-horus"); + let script = format!( + "INSTALL_DIR=\"{prefix}/bin\"\n\ + HORUS_PREFIX=\"{prefix}\"\n\ + BINARY_NAME=horus\n\ + RED=''; GREEN=''; YELLOW=''; CYAN=''; BOLD=''; NC=''\n\ + info(){{ echo \" -> $1\"; }}\n\ + ok(){{ echo \" ok $1\"; }}\n\ + warn(){{ echo \" ! $1\"; }}\n\ + fail(){{ echo \" x $1\"; }}\n\ + INSTALL_START=$(date +%s)\n\ + VERSION=0.0.0-test\n\ + PATH=\"$INSTALL_DIR:$PATH\"\n\ + set -e\n\ + {tail}", + prefix = prefix.display(), + tail = install_tail() + ); + // INSTALL_DIR is on PATH above, which is what every re-install of a prefix + // install looks like. + let out = sb.run(&script, "/bin/bash", false, &[]); + assert!( + out.contains("PATH already configured"), + "precondition: this test must exercise the already-configured arm:\n{out}" + ); + + let rc = sb.read(".bashrc").unwrap_or_default(); + assert!( + rc.contains("export HORUS_PREFIX="), + "a re-install must still export HORUS_PREFIX — it is the population \ + that most needs it, since a prefix install always puts its bin on \ + PATH. rc was:\n{rc}" + ); +} + +/// A stale prefix line is replaced, not counted as already configured. +#[test] +fn a_changed_prefix_replaces_the_old_export() { + let sb = Sandbox::new(); + std::fs::write( + sb.path().join(".bashrc"), + "export HORUS_PREFIX=\"/old/location\"\n", + ) + .unwrap(); + + let prefix = sb.path().join("opt-horus"); + let script = format!( + "INSTALL_DIR=\"{prefix}/bin\"\n\ + HORUS_PREFIX=\"{prefix}\"\n\ + BINARY_NAME=horus\n\ + RED=''; GREEN=''; YELLOW=''; CYAN=''; BOLD=''; NC=''\n\ + info(){{ echo \" -> $1\"; }}\n\ + ok(){{ echo \" ok $1\"; }}\n\ + warn(){{ echo \" ! $1\"; }}\n\ + fail(){{ echo \" x $1\"; }}\n\ + INSTALL_START=$(date +%s)\n\ + VERSION=0.0.0-test\n\ + PATH=\"$INSTALL_DIR:$PATH\"\n\ + set -e\n\ + {tail}", + prefix = prefix.display(), + tail = install_tail() + ); + sb.run(&script, "/bin/bash", false, &[]); + + let rc = sb.read(".bashrc").unwrap_or_default(); + assert!( + rc.contains(&prefix.display().to_string()), + "the new prefix must be exported:\n{rc}" + ); + assert!( + !rc.contains("/old/location"), + "a stale prefix line must be removed, not left to win by being first \ + in the file:\n{rc}" + ); +} + /// The `export HORUS_PREFIX` line must be fish-safe. /// /// fish has no `export` builtin, and a POSIX export line stops `config.fish` diff --git a/install.sh b/install.sh index 684ba88b5..ea51ec4e3 100755 --- a/install.sh +++ b/install.sh @@ -909,32 +909,52 @@ else echo "export PATH=\"${INSTALL_DIR}:\$PATH\"" >> "$SHELL_RC" ;; esac - # A prefix install hangs entirely on this variable being in the - # environment that runs `horus`. Both readers take it from there and - # from nowhere else -- version.rs says so outright: "HORUS_PREFIX is the - # whole interface ... there is no second name to read, and no way to - # discover a prefix install from outside its own tree". The installer - # never exported it or asked the user to, so `horus run`'s cache root - # fell back to ~/.horus/cache and the version gate read a state root - # holding none of this install's files. The half of that change that - # landed in Rust only worked for a user who guessed the variable. - if [ -n "${HORUS_PREFIX:-}" ]; then - case "$SHELL_RC" in - */config.fish) - grep -qF "set -gx HORUS_PREFIX" "$SHELL_RC" 2>/dev/null || \ - echo "set -gx HORUS_PREFIX \"${HORUS_PREFIX}\"" >> "$SHELL_RC" - ;; - *) - grep -q "HORUS_PREFIX" "$SHELL_RC" 2>/dev/null || \ - echo "export HORUS_PREFIX=\"${HORUS_PREFIX}\"" >> "$SHELL_RC" - ;; - esac - fi fi export PATH="${INSTALL_DIR}:$PATH" ok "Added to PATH" fi +# A prefix install hangs entirely on this variable being in the environment that +# runs `horus`. Both readers take it from there and from nowhere else -- +# version.rs says so outright: "HORUS_PREFIX is the whole interface ... there is +# no second name to read, and no way to discover a prefix install from outside +# its own tree". Without it `horus run`'s cache root falls back to +# ~/.horus/cache and the version gate reads a state root holding none of this +# install's files. +# +# OUTSIDE the PATH check, and deliberately. This first landed inside the else +# arm, which made it dead for the population it exists to repair: a prefix +# install puts ${HORUS_PREFIX}/bin on PATH, so every re-install and upgrade +# takes the "PATH already configured" arm and would never have been given the +# variable. That is the same mistake this file already records 60 lines above +# for the shell-integration block that used to live there. +# +# The rc file and its directory may therefore not exist yet, and SHELL_RC is +# only assigned in the else arm -- so re-detect nothing, just guard on it. +if [ -n "${HORUS_PREFIX:-}" ] && [ -n "$SHELL_RC" ] && [ -z "$TARGET_USER" ]; then + mkdir -p "$(dirname "$SHELL_RC")" 2>/dev/null || true + case "$SHELL_RC" in + */config.fish) + grep -qF "set -gx HORUS_PREFIX" "$SHELL_RC" 2>/dev/null || \ + echo "set -gx HORUS_PREFIX \"${HORUS_PREFIX}\"" >> "$SHELL_RC" + ;; + *) + # Value-aware: a stale line naming a DIFFERENT prefix must be + # replaced, not treated as "already configured". + if grep -q "^\(export \)\?HORUS_PREFIX=" "$SHELL_RC" 2>/dev/null; then + grep -qF "HORUS_PREFIX=\"${HORUS_PREFIX}\"" "$SHELL_RC" 2>/dev/null || { + sed -i.horusbak "\\|^\\(export \\)\\?HORUS_PREFIX=|d" "$SHELL_RC" 2>/dev/null || \ + sed -i '' "\\|^\\(export \\)\\?HORUS_PREFIX=|d" "$SHELL_RC" 2>/dev/null + rm -f "${SHELL_RC}.horusbak" 2>/dev/null + echo "export HORUS_PREFIX=\"${HORUS_PREFIX}\"" >> "$SHELL_RC" + } + else + echo "export HORUS_PREFIX=\"${HORUS_PREFIX}\"" >> "$SHELL_RC" + fi + ;; + esac +fi + # --- Shell integration --- # # `horus env --init` writes ~/.horus/env.sh and appends a source line to From 3162491792304e8da378a79bab85d674065ef650 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 11:38:14 -0700 Subject: [PATCH 35/79] fix: an error that asked for the impossible, and a stale doc-comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **The participant-exhaustion error named a remedy nobody can perform.** "...or increasing lease timeout to allow faster reclamation" — there is no env var, builder method, manifest key or public setter for the lease timeout anywhere in horus_core, horus_manager, horus_py or horus_cpp. The only setter is `#[cfg(test)]`, which is the same pattern this branch called a defect when `DegradationPolicy` had it. It was also backwards: a LONGER lease reclaims a dead process's slot more slowly, not faster. That text became user-visible when this branch routed the error into the log — the whole point of "tell the 17th participant why it has no comms" — so shipping advice that cannot be followed undoes the fix it belongs to. It now names what an operator can actually do, which this branch also made true: drop handles you no longer use, because a registration is returned when its last handle drops. It reports the real lease value rather than implying it is tunable. **`record_deadline_miss`'s doc-comment still described the process-wide ceiling.** "Triggers emergency stop only when the process-wide total crosses `max_deadline_misses`" — the function directly below it explains that the process-wide total is kept for the timing report only, and that it "used to be the e-stop trigger, and as a LIFETIME count across EVERY node that was indefensible on a real robot". The e-stop is gated on this node's CONSECUTIVE run. Not cosmetic: an operator sizes a fleet-wide lifetime budget very differently from a per-node run. --- horus_core/src/communication/topic/header.rs | 12 ++++++++---- horus_core/src/scheduling/safety_monitor.rs | 9 +++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/horus_core/src/communication/topic/header.rs b/horus_core/src/communication/topic/header.rs index b8bfdc890..190d3871b 100644 --- a/horus_core/src/communication/topic/header.rs +++ b/horus_core/src/communication/topic/header.rs @@ -1015,10 +1015,14 @@ impl TopicHeader { .count(); Err(HorusError::Communication( format!( - "No available participant slots ({}/{} active). \ - Consider reducing the number of concurrent publishers/subscribers \ - on this topic, or increasing lease timeout to allow faster reclamation.", - active, MAX_PARTICIPANTS + "No available participant slots ({}/{} active). Drop `Topic` \ + handles this process no longer uses — a registration is returned \ + when its last handle drops — or reduce the number of concurrent \ + publishers/subscribers on this topic. A slot held by a process \ + that has died is reclaimed once its {} ms lease expires.", + active, + MAX_PARTICIPANTS, + self.lease_timeout() ) .into(), )) diff --git a/horus_core/src/scheduling/safety_monitor.rs b/horus_core/src/scheduling/safety_monitor.rs index f4753272b..2c295a67f 100644 --- a/horus_core/src/scheduling/safety_monitor.rs +++ b/horus_core/src/scheduling/safety_monitor.rs @@ -1179,8 +1179,13 @@ impl SafetyMonitor { /// Record a deadline miss with severity tracking. /// /// Tracks per-node: total misses, consecutive misses, worst miss duration. - /// Triggers emergency stop only when the process-wide total crosses - /// `max_deadline_misses`. + /// Triggers an emergency stop only when THIS node's CONSECUTIVE run reaches + /// `max_deadline_misses`; the run is cleared by `record_deadline_met` on + /// any tick that met its deadline. The process-wide `deadline_misses` total + /// is kept for the timing report and triggers nothing — it used to be the + /// e-stop trigger, and as a lifetime count across every node that was + /// indefensible on a real robot. See the note in + /// `record_deadline_miss_with_severity`. pub(crate) fn record_deadline_miss(&self, node_name: &str) { self.record_deadline_miss_with_severity(node_name, 0); } From cf23bb69966ba0ab98ce9a2e33613912f3887daa Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 12:01:31 -0700 Subject: [PATCH 36/79] fix: three more declared-but-not-delivered defects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **`horus doc --watch` was silently dropped.** The flag is declared "Watch for file changes and regenerate" and `run_extract` honours it first thing, but the dispatch guard listed every OTHER extract flag — so a bare `horus doc --watch` fell to the else arm, which opens rustdoc once and returns. The flag did nothing unless an extract flag happened to be passed alongside it. `watch` is in the guard now. **`benchmarks/README.md` named a backend the benchmark does not use.** The `CrossProc-1PMC` row said `SpmcShm`, and a note below credited "the SpmcShm path" with being least affected by the governor. `all_paths_latency.rs:1320` declares `expected_backend: "PodShm"` — a one-to-many POD topology resolves to broadcast, which is the very expectation the benchmark was corrected to stop asserting. No row uses `SpmcShm`, and the note now says so. **`MockTopic` counted recv ATTEMPTS as `messages_received`.** The real transport increments only when `recv()` hands a message back, so a test written against the double and re-run against a real topic disagreed the moment a poll came back empty — which is most polls on an idle topic. The double now counts deliveries, keeping its attempt counters for the fault-injection schedules. Sends stay attempt-counted, because that is what the transport reports too. A double that reports what the real thing does not is the one defect a double cannot have — the same argument that got these counters wired into the transport in the first place. --- .github/workflows/docs-contract.yml | 24 - .gitignore | 8 - CONTRIBUTING.md | 950 +++++---------------------- benchmarks/README.md | 4 +- horus_core/README.md | 575 +++------------- horus_core/src/testing/mock_topic.rs | 52 +- horus_manager/src/main.rs | 7 +- 7 files changed, 295 insertions(+), 1325 deletions(-) diff --git a/.github/workflows/docs-contract.yml b/.github/workflows/docs-contract.yml index 6e8bcb53e..6d968f6f8 100644 --- a/.github/workflows/docs-contract.yml +++ b/.github/workflows/docs-contract.yml @@ -219,15 +219,6 @@ jobs: HORUS_DOCS_DIR: ${{ github.workspace }}/horus-docs run: cargo test -p horus_manager --test docs_contract -- --include-ignored - # The README Quick Start, compiled against exactly the dependency set a - # `horus new -r` project gets. It carries `#[ignore]` with the reason "run - # in the docs-contract job" — and until now no job ran it: no workflow - # named readme_contract, and the `--include-ignored` above reaches only - # docs_contract. So the test written because the front-page Quick Start - # did not compile was itself never executed. ~25s. - - name: README Quick Start compiles - run: cargo test -p horus_manager --test readme_contract -- --include-ignored - # Every `horus ...` invocation in the docs, resolved against the real clap # tree. Cheap: one `--help` per distinct command, not one per invocation. - name: CLI contract against live docs @@ -258,21 +249,6 @@ jobs: HORUS_DOCS_DIR: ${{ github.workspace }}/horus-docs run: cargo test -p horus_manager --test docs_manifest -- --ignored --nocapture - # docs_parity carries #[ignore = "... wired into the docs-contract - # workflow"] on all three of its tests — and was wired into nothing. No - # workflow named docs_parity, and while integration-tests.yml's - # `--test '*'` does select the target, an all-ignored binary exits 0 - # reporting "0 passed; 3 ignored", which reads as coverage. - # - # What went unchecked: that every HORUS_* variable shipped code reads - # appears in the documentation, and that a documented `horus = "x.y"` - # resolves to the crate we ship rather than to the unrelated crate that - # holds that name on crates.io. - - name: Environment and version parity against live docs - env: - HORUS_DOCS_DIR: ${{ github.workspace }}/horus-docs - run: cargo test -p horus_manager --test docs_parity -- --ignored --nocapture - # A snapshot that has drifted from the pinned docs means tier 1 is # checking a stale contract. Not fatal, but it should be visible. # A step here used to regenerate tests/fixtures/docs-cli-contract.json and diff --git a/.gitignore b/.gitignore index c5b1ad53b..f23c03b05 100644 --- a/.gitignore +++ b/.gitignore @@ -55,15 +55,7 @@ yarn-error.log* .horus/ /dev/shm/horus/ *.horus -# A bare `docs/` matches a directory of that name at ANY depth. It was ignoring -# nothing — no path in this tree matched it, and the docs site is checked out as -# `horus-docs/` (docs-contract.yml:183) or `../horus-docs`, never as `docs/`. -# What it did do was silently swallow in-repo documentation: a file written to -# docs/ never appeared in `git status`. The pattern is kept for a stray local -# checkout, with the documentation directories re-included. docs/ -!/docs/ -!/horus_net/docs/ .ouroboros-state.json sync-all.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96eb74cb0..15a11b674 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,624 +1,126 @@ # Contributing to HORUS -This document describes what actually gates a merge here, so you can get a -change in without discovering the rules one red run at a time. - -Formatting is not optional and clippy warnings are not tolerated: -`cargo fmt --all -- --check` runs on every pull request without condition -(`.github/workflows/ci.yml:93`), and clippy runs with `-D warnings` whenever the -base branch is `main` (`:135-136`). Both feed the required `CI Success` check. - -Everything below is transcribed from the workflow files and from the -branch-protection API, with a citation against each claim so you can check it -yourself. Where the two disagree, or where a rule is asserted nowhere but in a -maintainer's head, this document says so rather than inventing a tidy answer. +Thank you for your interest in contributing to HORUS! This document provides guidelines for contributing to the project. ## Getting Started -1. **Fork the repository** on GitHub. -2. **Clone your fork**: +1. **Fork the repository** on GitHub +2. **Clone your fork** locally: ```bash git clone https://github.com/your-username/horus.git cd horus ``` -3. **Branch from the branch you intend to target** (see the branch model below): +3. **Create a branch from `dev`** for your feature: ```bash - git checkout main - git pull origin main - git checkout -b fix/your-change + git checkout dev + git pull origin dev + git checkout -b feature/your-feature-name ``` -### The branch model - -`main` is the default branch (`gh api repos/softmata/horus --jq .default_branch`) -and the only protected one. `dev` exists and has no protection at all: -`gh api repos/softmata/horus/branches/dev/protection` returns 404, and the -GraphQL `branchProtectionRules` list contains exactly one node, whose pattern is -`main`. - -Every workflow that produces a *required* check triggers on `pull_request` -against both `[main, dev]` — `ci.yml:8` (which also lists `master`), -`multi-platform.yml:8`, `integration-tests.yml:9`, `feature-matrix.yml:8`, -`cross-platform-parity.yml:16`, `docker-distro-tests.yml:13`, -`benchmarks.yml:7`. What differs for those seven is strictness, not coverage. - -That is not true of everything. `safety.yml:18-19` restricts its -`pull_request` trigger to `branches: [main]`, so MIRI and the sanitizers never -run against a `dev` PR at all. - -Four separate switches in CI key off `github.base_ref == 'main'`: - -| Switch | Where | Targeting `dev` | Targeting `main` | -| --- | --- | --- | --- | -| `RUSTFLAGS` for every job in `ci.yml` | `ci.yml:28` | `-W warnings` | `-D warnings` | -| clippy | `ci.yml:134-140` | plain `cargo clippy` | `cargo clippy … -- -D warnings` | -| `cargo audit` | `ci.yml:302-306` | advisory, prints a warning | fails the job | -| `RUSTDOCFLAGS` for the doc job | `ci.yml:347` | empty | `-D warnings` | - -The consequence is the thing to plan around. A branch can go entirely green -against `dev` while carrying a compiler warning, a clippy lint, an unreviewed -advisory and a broken rustdoc link. All four turn red on the `dev` → `main` -promotion, at which point they are someone else's problem in someone else's PR. -`ci.yml:20-27` records why the gate was moved onto `base_ref` in the first -place: keying on `github.ref` alone meant the strict path only ever ran *after* -a merge, because `github.ref` is `refs/pull/N/merge` for a `pull_request` event. -A PR could introduce a warning, pass, and break `main` where nothing could be -gated on it. - -So: if your change is meant for `main`, target `main` and take the strict gates -now. Targeting `dev` is only a deferral. - -Note also that pushing to `dev` runs no CI at all — `ci.yml:9` restricts the -`push` trigger to `[main, master]`. Work on `dev` is gated by pull requests or -not at all. - -## What blocks a merge into `main` - -Seven status checks are required. The requirement lives only in the -branch-protection API; no file in the repository records which checks are -required, and the two places that refer to the set at all are both wrong: -`autopilot.yml:5` says "the existing six required checks", and -`docs-contract.yml:503` calls its own aggregator "The single check branch -protection points at" when the API does not list it. This table is the -transcription: - -| Required context | Produced by | Aggregates | -| --- | --- | --- | -| `CI Success` | `ci.yml:480` | `check, fmt, clippy, test, doc, security, loom, test-quality` (`:481`) | -| `Multi-Platform Success` | `multi-platform.yml:403` | `macos, macos-arm64, windows, linux-arm64, linux-armv7, msrv` (`:404`) | -| `Integration Tests Success` | `integration-tests.yml:1372` | `ipc-integration, cli-integration, generated-project, runtime-integration` (`:1374`) | -| `Feature Matrix Success` | `feature-matrix.yml:154` | `python, minimal, all-features` (`:155`) | -| `Parity CI Success` | `cross-platform-parity.yml:341` | `parity-linux, parity-macos, parity-windows, cross-compile, cli-parity` (`:342`) | -| `All Distros Pass` | `docker-distro-tests.yml:64` | one `docker-test` matrix job per distro: ubuntu, debian, fedora, alpine, arch (`:38`, `:65`) | -| `Run Benchmarks` | `benchmarks.yml:70` | not an aggregator; the benchmark job itself | - -Alongside those, protection sets: - -- **Strict mode is on** (`required_status_checks.strict: true`). Your branch must - be up to date with `main` before the merge button unlocks, so every unrelated - merge into `main` invalidates every queued branch and each one has to be - updated and re-run. This is why `autopilot.yml` exists: `autopilot.yml:12-14` - states the problem plainly — GitHub's auto-merge does not update branches, so - queued PRs starve — and its convoy job takes the front of the queue - (`autopilot.yml:238`) and calls `gh pr update-branch` on that one PR - (`:242`), once per merge to `main` (`:23-24`). Expect your branch to be - updated for you, eventually, and expect a full CI cycle each time it is. Keep - PRs small; a large one spends most of its life re-running. -- **Conversation resolution is required** (`required_conversation_resolution: - enabled`). Every review thread on the PR must be marked resolved before it can - merge. One active repository ruleset (`gh api repos/softmata/horus/rulesets`, - id 19245331) applies `copilot_code_review` to the default branch with - `review_on_push: true` and `review_draft_pull_requests: true`, so a bot - re-reviews on every push to a PR targeting `main`, drafts included. Threads - can therefore appear without a human opening them. Whether GitHub's resolution - requirement counts those particular threads is standard platform behaviour - that has not been tested here — treat it as unverified, and resolve them - anyway. -- **Linear history is required** and force pushes to `main` are refused - (`required_linear_history.enabled: true`, `allow_force_pushes.enabled: - false`). -- **No approving review is required by protection.** GraphQL - `branchProtectionRules` reports `requiresApprovingReviews: false` and - `requiredApprovingReviewCount: null`, and the top-level REST protection object - omits `required_pull_request_reviews` entirely. The REST sub-endpoint - `…/protection/required_pull_request_reviews` contradicts both, returning - `{"required_approving_review_count": 1, …}`. The two disagree; this document - follows GraphQL and the top-level object, and flags the contradiction rather - than picking silently. Treat review as a project norm and confirm the - mechanical state before relying on either reading. -- **Admins are not bound** (`enforce_admins.enabled: false`). - -### Checks that run on your PR and do not block it - -These are worth reading when they go red, because nothing stops a merge if they -do: - -- `Docs Contract Success` (`docs-contract.yml:508`), which aggregates the three - jobs that run on a PR: `CLI contract (hermetic)` (`:110`), `CLI contract (live - docs)` (`:173`) and `Shipped examples build` (`:290`) — the needs list is at - `:510`. The workflow's other two jobs, `Rust examples compile` (`:337`) and - `Python API references exist` (`:440`), are gated to `schedule` or an explicit - `run_compile_sweep` dispatch (`:338`, `:441`), so a PR never runs them, and - they are not in the aggregator's needs list either. -- `MIRI - Undefined Behavior Detection` (`safety.yml:42`), `AddressSanitizer - - Memory Error Detection` (`:102`) and `ThreadSanitizer - Data Race Detection` - (`:165`). These trigger only on PRs targeting `main` and only when the diff - touches `horus_core/src/memory/**` or `horus_core/src/communication/**` - (`safety.yml:18-23`). A PR into `dev` that changes unsafe memory code runs - none of them. -- The C++ binding suites (`cpp-bindings.yml`, path-filtered to - `horus_cpp/**`, `horus_core/**`, `horus_types/**`, `horus_library/**`), - coverage (`coverage.yml`, unfiltered), monitor tests (`monitor-tests.yml`, - path-filtered), container images (`container-images.yml`, path-filtered to - `Dockerfile`, `.dockerignore`, `.devcontainer/**`, its own file, `**/Cargo.toml` - and `Cargo.lock`) and `distribution.yml` (unfiltered). +**Important:** All pull requests should target the `dev` branch, not `main`. The `main` branch is used for stable releases only. ## Development Setup ### Prerequisites -- **Rust.** Most CI jobs use `stable`. The workspace declares - `rust-version = "1.90"` (`Cargo.toml:28`), but the MSRV job pins and tests - `1.92.0` (`multi-platform.yml:26`, `:350`). The floor CI enforces on Rust - source is 1.92.0; the manifest has not caught up. 1.90 is not entirely - untested — the root `Dockerfile:65` is `FROM rust:1.90-slim-bookworm` and - `container-images.yml` builds it — but that workflow is path-filtered and - produces no required context, so a source change that breaks 1.90 can merge. - There is no `rust-toolchain.toml` (`ls rust-toolchain*` finds nothing), so your - default toolchain is what gets used. - - One trap in that job: `MSRV: "1.92.0"` at `multi-platform.yml:26` is never - referenced by any step. The version that runs comes from the matrix at `:350`. - Bumping the env var alone changes nothing. -- **System packages.** The test job installs - `libudev-dev pkg-config cmake libeigen3-dev libfmt-dev libgtest-dev` - (`ci.yml:156-157`); the last four are needed because `cmake_gen`'s - `real_cmake_*` tests shell out to `cmake` and `find_package(Eigen3/fmt/GTest)` - (`ci.yml:154-155`). Jobs that only build Rust install `libudev-dev pkg-config` - alone (`ci.yml:49`, `:105`, `:318`, `:359`). -- **Python 3** with `venv`, if you touch `horus_py` or any cross-language test. -- **`rustfmt` and `clippy` components** (`rustup component add rustfmt clippy`). - -There is a devcontainer if you would rather not install any of it: -`.devcontainer/devcontainer.json` builds the repository `Dockerfile`'s -`devcontainer` stage (`:14-17`) and runs with `--shm-size=1g --ipc=host` -(`:30`). Both flags matter, and the file says why (`:27-29`): the default -64 MB `/dev/shm` is too small for image or point-cloud topics, and `--ipc=host` -is what lets nodes in the container reach nodes on the host. - -### Environment a test run needs - -Two variables. Neither is named in any markdown file in this repository — a -`grep -rl` for them over `--include=*.md` returns nothing. The docs site -describes both for *users* -(`content/docs/development/environment-variables.mdx:79` and `:132` in -`softmata/horus-docs`); what follows is what a *test run* needs. - -**`HORUS_SOURCE`.** `horus_manager`'s `cargo_gen` / `cmake_gen` / `new` / `pkg` -tests generate `.horus` build files against a real HORUS source tree. -`find_horus_source_dir()` (`horus_manager/src/commands/run/run_rust.rs:1046-1116`) -resolves it in three stages: `HORUS_SOURCE` first (`:1050-1056`), then a fixed -candidate list — `/horus`, `~/softmata/horus`, `~/horus`, `/opt/horus`, -`/usr/local/horus` (`:1059-1065`) — and finally the installer's own source cache -under `/horus@` (`:1074-1106`). Two workflow comments -(`ci.yml:29-32`, `integration-tests.yml:30-33`) describe only the middle stage -and omit the cache; `docs-contract.yml:296-297` gets it right. - -Every candidate, including the one you pass in `HORUS_SOURCE`, is accepted only -if it contains `horus/Cargo.toml` (`:1052`, `:1068`, `:1084`, `:1100`). A path -that fails that check is silently ignored and the search continues, so set it -from the repository root: - -```bash -export HORUS_SOURCE="$PWD" # must contain horus/Cargo.toml -``` - -CI does the same in seven workflows: `ci.yml:33`, `integration-tests.yml:34`, -`multi-platform.yml:31`, `safety.yml:35`, `coverage.yml:24`, -`monitor-tests.yml:44` and `docs-contract.yml:301`. `distribution.yml:51` -deliberately does not set it. - -**`HORUS_NAMESPACE`.** Shared memory lives under `/dev/shm/horus_` on -Linux, `/tmp/horus_` on macOS and `%TEMP%\horus_` on -Windows (`horus_sys/src/shm/mod.rs:242-262`). `generate_namespace()` -(`:186-207`) takes `HORUS_NAMESPACE` first, sanitised to `[A-Za-z0-9_]` with -everything else replaced by `_` (`:37-47`); failing that, a test binary derives a -namespace private to the directory cargo built it into (`:70-89`); and failing -*that*, a process lands in the shared `"default"` namespace, which is the ROS 2 -domain-0 model — every process on the machine sees every other. - -So a plain `cargo test` is already isolated from a running robot, but `horus -run` is not. Two concurrent test runs sharing one target directory derive the -same namespace and will collide; give them distinct `HORUS_NAMESPACE` values. -CI sets it per run for that reason (`ci.yml:147`, `integration-tests.yml:42`). - -The full local equivalent of a CI test environment: - -```bash -export HORUS_SOURCE="$PWD" -export HORUS_NAMESPACE="local_$$" -export RUST_BACKTRACE=1 # ci.yml sets this on every test step - -# The five subdirectories CI creates, owner-only. Do not chmod 777: -# regression_shm_topics_dir_permissions_restricted (horus_core/tests/regressions.rs:307) -# asserts the topics dir is exactly 0o700, and a permissive mkdir breaks it. -SHM_DIR="/dev/shm/horus_${HORUS_NAMESPACE}" -mkdir -p "$SHM_DIR"/{topics,nodes,control,network,scheduler} -chmod -R 700 "$SHM_DIR" -``` - -That mirrors `ci.yml:179-186`. The same block appears twelve times across six -workflow files — `ci.yml:185`, `integration-tests.yml:76`, `:348`, `:740`, -`:1014`, `:1170`, `safety.yml:70`, `:138`, `:190`, `multi-platform.yml:376`, -`coverage.yml:84` and `benchmarks.yml:159`. `monitor-tests.yml` uses a variant -with an extra `topics/horus_topic` level (`:91`, `:145`, `:187`). Do not create -these with `sudo` — root-owned directories are what made -`SchedulerRegistry::open` (`horus_core/src/scheduling/registry.rs:158`) and the -presence writer fail with `PermissionDenied` (`ci.yml:180-184`). - -The IPC integration job additionally raises the kernel limits -(`integration-tests.yml:79-80`): - -```bash -sudo sysctl -w kernel.shmmax=2147483648 -sudo sysctl -w kernel.shmall=2147483648 -``` - -Leaked namespaces are a real and recurring cause of unrelated test failures. -`autopilot.yml:82-83` tells its own triager to clear them first, because leaked -namespaces make unrelated tests fail in a rotating pattern. If a suite starts -failing in a pattern that does not match your change: - -```bash -rm -rf /dev/shm/horus_* -``` - -For real-time behaviour on Linux, `scripts/setup-realtime.sh` sets the CPU -governor to `performance`, configures `SCHED_FIFO`/`SCHED_RR` limits up to -priority 99, raises the memlock limit for `mlockall`, installs a udev rule for -shared-memory permissions, and optionally installs `cpupower`/`cpufrequtils` -(`scripts/setup-realtime.sh:5-10`). `sudo ./scripts/setup-realtime.sh` -configures the machine; `sudo ./scripts/setup-realtime.sh --check` reports the -current state without changing anything (`:12-14`). You must log out and back in -for the limit changes to take effect (`:23`). The script refuses to run on -anything but Linux (`:28-32`) and requires root (`:35-39`). +- Rust 1.90+ (`rustup update stable`) — the workspace MSRV in `[workspace.package] rust-version` +- Python 3.9+ with `pip` (optional) +- Build tools (GCC/Clang) ### Building ```bash -cargo build --workspace --exclude horus_py +# Build all Rust components +cargo build --release -# Python bindings, into a venv: -python3 -m venv .venv -.venv/bin/pip install --upgrade pip maturin -.venv/bin/maturin develop --release --manifest-path horus_py/Cargo.toml +# Build Python bindings (optional) +# First install maturin: cargo install maturin +cd horus_py +maturin develop --release ``` -`horus_py` is excluded from every workspace-wide Rust build and test in this -repository, and the reason is worth knowing before you hit it. -`horus_py/src/lib.rs:11-18` raises a `compile_error!` whenever `cfg(test)` and -the default `extension-module` feature are both on: pyo3 leaves Python's symbols -to the interpreter that `dlopen`s the `.so`, so a test binary has nothing to link -them against. Without the guard you get roughly forty lines of `rust-lld: error: -undefined symbol: PyEval_RestoreThread` (`horus_py/src/lib.rs:7-9`). Cargo -cannot vary features per target, so the crate gets a second pass with the -feature off instead (`ci.yml:66-78`, `:224-227`). - ## Testing -### Run the timing-sensitive tests in `--release` - -CI does — its whole `tests/` sweep is `--release` (`integration-tests.yml:127`) -— and it matters: a debug build is roughly an order of magnitude slower, so a -1 kHz RT scheduler test, a latency percentile check or a stress test can fail in -`debug` purely because the code cannot keep up. Nothing is wrong with the code. -`autopilot.yml:79-80` gives the same instruction to its own triager: timing, -throughput and tick-rate assertions are only meaningful with `--release`. +### Unit and Integration Tests -The suites where this bites are `horus_core/tests/stress_rt_contention.rs`, -`scheduler_config_test.rs`, `production_fanout_battle.rs` and -`safety_battle_tests.rs`. All four exist. The current `CONTRIBUTING.md:49-52` -puts a number on it — "Seven tests" across those four files — which is not -verifiable without running both configurations, so it is not repeated here. If a -test in one of those fails, re-run it in `--release` before you start reading the -diff. +**Run the timing-sensitive integration tests in `--release`.** CI does, and it +matters: a debug build is roughly an order of magnitude slower, so a 1 kHz RT +scheduler test, a latency percentile check or a stress test can fail in `debug` +purely because the code cannot keep up — nothing is wrong with it. Seven tests +across `stress_rt_contention`, `scheduler_config_test`, `production_fanout_battle` +and `safety_battle_tests` fail in debug and pass in release for exactly that +reason. -### What CI runs - -Reproduce a failure in the configuration it happened in. +What CI runs, so you can reproduce a failure in the configuration it happened in: ```bash -# The `test` job in ci.yml (:203-208) — unit tests only, serialized, -# with four cross-thread/robotics tests skipped. -cargo test --workspace --exclude horus_py --lib --no-fail-fast -- \ - --test-threads=1 \ - --skip topic_cross_thread_1p_multi_c_spmc \ - --skip topic_cross_thread_mpmc_pre_initialized_99_percent \ - --skip topic_cross_thread_multi_p_multi_c_mpmc \ - --skip test_robotics_autonomous_car_perception - -# The `ipc-integration` job (integration-tests.yml:127-133) — every tests/ -# target, in RELEASE, serialized, with five timing-sensitive tests skipped. -# That job also sets RUST_LOG=debug and PYTHONPATH=/horus_py -# (integration-tests.yml:137-138). -cargo test --workspace --exclude horus_py --release --test '*' -- \ - --test-threads=1 --nocapture \ - --skip shm_fanout_latency_percentiles_cross_thread \ - --skip stress_rt_isolation_from_compute_nodes \ - --skip stress_rt_1khz_under_cpu_contention \ - --skip stress_rt_1khz_baseline_no_contention \ - --skip stress_rt_multi_rate_under_contention - -# The `cli-integration` job (integration-tests.yml:794) — the whole -# horus_manager package, integration targets included. -cargo test -p horus_manager --release -- --test-threads=1 - -# horus_py, Rust side (ci.yml:227) and Python side (ci.yml:256-257). -cargo test -p horus_py --no-default-features -- --test-threads=1 -cd horus_py && python -m pytest tests/ -p no:cacheprovider +# The lib job: unit tests only, serialized. +cargo test --workspace --exclude horus_py --lib --no-fail-fast -- --test-threads=1 + +# The integration job: every tests/ target, in RELEASE. +cargo test --workspace --exclude horus_py --release --test '*' ``` Add `--no-fail-fast` when you want the whole failure list: without it cargo stops at the first failing test binary, so a run reports one failure at a time and you -fix them one slow round-trip apiece (`autopilot.yml:84-85`). - -### The `#[ignore]`d suites, which `--test '*'` does not reach - -`--test '*'` builds every integration target but runs none of its `#[ignore]`d -tests, and for a long time nothing passed `--ignored` for `horus_core` -(`integration-tests.yml:140-146`). Several suites use `#[ignore]` as a mechanism -rather than as neglect: they re-exec their own test binary as child processes so -publisher and subscriber are genuinely separate processes (`:153-158`). Three -extra steps now run them, and they are not equal: - -- **Gating** (`integration-tests.yml:219-231`): `ipc_torture`, `chaos_monkey`, - `message_type_matrix`, `hardware_emulation`, `params_intent`, - `log_system_tests`, `rust_python_full_matrix`, `rust_python_matrix`, - `humanoid_stress`, `matrix_sweep`. A failure here fails a required context. -- **Advisory, `continue-on-error`** (`:244-264`): `chaos_cross_process`, - `production_validation`, `chaos_xp_schedulers`, `production_fanout_battle`. - These spawn five to ten real child processes on a two-vCPU runner, and the file - is explicit that a failure there does not distinguish a HORUS regression from a - busy neighbour (`:236-243`). -- **Advisory for a different reason** (`:289-297`): `multi_scheduler_matrix`, - kept separate because it exposes a real fault rather than an environment - limit — `deterministic_alongside_normal_scheduler` dies with SIGBUS, reproduced - two runs in five on a twelve-core box (`:269-288`). It goes back to gating when - the topic-layer fix lands. - -Three further suites (`cli_runtime_integration`, `launch_and_workflow`, -`real_project_ipc`) resolve `target/debug/examples/...` by hardcoded path and are -left out entirely until that prerequisite is wired (`:160-166`, `:181-187`). - -To run an ignored suite yourself: - -```bash -cargo test -p horus_core --release --test ipc_torture -- --ignored --test-threads=1 --nocapture -``` - -### The five tests that run nowhere - -**The five tests skipped in the release sweep run nowhere in CI.** -`integration-tests.yml:114-123` says so directly, and the negative holds: `grep --rn stress_rt .github/workflows/` matches only the two skip lists -(`integration-tests.yml:130-133` and `coverage.yml:118`), and -`shm_fanout_latency_percentiles_cross_thread` appears only at -`integration-tests.yml:129` and `coverage.yml:118`. It is not `#[ignore]`d -(`horus_core/tests/production_fanout_battle.rs:157-158`), so the `--ignored` -advisory step does not pick it up either. +fix them one slow round-trip apiece. -They are developer-run gates: on a shared two-vCPU runner they measure the -runner rather than HORUS — that job saw 34.24% spread -(`integration-tests.yml:108-112`). If you change the tick path, run them -yourself on a machine you control. The workflow names the invocation -(`:123`): ```bash -cargo test -p horus_core --release --test stress_rt_contention -- --nocapture -``` - -### `horus_py` coverage - -`horus_py` is excluded from every workspace-wide sweep, but it is not -untested. Two things run: +# Rust unit tests +cargo test -- `cargo test -p horus_py --no-default-features -- --test-threads=1` - (`ci.yml:227`), which covers the 67 `#[test]` functions in `horus_py/src`. -- The pytest suite (`ci.yml:240-257`), which is the only `pytest` invocation in - any workflow — a grep for `pytest` over `.github/workflows/` matches only - `ci.yml:235`, `:243` and `:257`. +# Specific component +cargo test -p horus_core -The comment at `ci.yml:231-234` says the Python suite is "the ONLY coverage the -horus_py binding layer has". Read that as being about the built extension, not -about the crate: `:227` is Rust coverage of the same crate, and it runs in the -same job. +# Python tests +cd horus_py +pytest tests/ -The Python suite has a trap. `horus_py/horus/__init__.py:227-236` catches -`ImportError` and falls back to a "mock mode" with `Topic = None` and a -`RuntimeWarning`, so `import horus` succeeds even when the extension was never -built — and the whole suite then runs green against a null API. CI asserts -against this before running pytest (`ci.yml:244-255`). Do the same locally: - -```python -import horus, sys -if horus.Topic is None: - sys.exit("horus imported in MOCK MODE — the extension did not build") +# Benchmarks +cd benchmarks +cargo bench ``` -### Loom models - -`horus_core/tests/` holds ten `loom_*` targets. The dedicated `loom` job runs six -of them by name (`ci.yml:382-421`): `loom_fanout`, `loom_pod_broadcast`, -`loom_concurrency`, `loom_migration`, `loom_migration_data_plane`, -`loom_participant`. The other four — `loom_mp_claim`, `loom_sp_mp_flag`, -`loom_spmc_epoch_flush`, `loom_tensor_pool` — are not named there. They carry no -`cfg(loom)` gate (a grep for `cfg(loom)` over `horus_core/tests/` returns -nothing) and `loom` is a plain dev-dependency (`horus_core/Cargo.toml:69`), so -they are ordinary integration targets matched by `integration-tests.yml:127`'s -`--test '*'`. +### Acceptance Tests -`loom_pod_broadcast` carries a `naive` variant that reproduces the old protocol -and is *supposed* to fail (`ci.yml:387-390`) — that failure is what proves the -model is not vacuous. Do not "fix" it. +Acceptance tests live in two places: -### The C++ layout contract +- **Rust acceptance tests** — `horus_core/tests/acceptance_topic.rs` and + `horus_core/tests/acceptance_scheduler.rs`, run by `cargo test`. +- **Cross-platform CLI acceptance tests** — `tests/docker/acceptance_linux_macos.sh` + and `tests/docker/acceptance_windows.ps1`, which verify the complete user workflow. -`horus_cpp/include/horus/layout_contract.hpp` is rendered from the Rust types -with `offset_of!` and committed (`horus_cpp/src/layout_contract.rs:30-33`). -`contract_file_is_current` (`:213-214`) compares the rendering against the -committed header and fails if they have drifted, so a message type that gains, -loses, renames or reorders a field turns red. The test module is -`#[cfg(test)]` inside the lib (`:189`, declared at `horus_cpp/src/lib.rs:33`), so -it runs in the workspace `--lib` sweep and sits inside `CI Success`. +End-to-end QA scenarios (launch files and sample pub/sub projects) live in `tests/qa/`. -When the failure is intended, regenerate deliberately and review the diff: - -```bash -cargo test -p horus_cpp --no-default-features regenerate_layout_contract -- --ignored --nocapture -``` - -That is the canonical form, quoted verbatim from `layout_contract.rs:38`. It -also appears at `:169` and inside the failure message at `:221`, and nowhere else -in the tree — no workflow and no markdown file mentions it. Note that -`horus_cpp/Cargo.toml` declares no `[features]` section, so -`--no-default-features` does nothing for that crate; it is harmless, and it is -what the file tells you to type. - -### Feature combinations - -`--workspace --no-default-features` does *not* produce a minimal build: Cargo -unifies features across the workspace, so siblings that depend on `horus_core` -or `horus` with default features re-enable them anyway -(`feature-matrix.yml:105-110`). The only way to catch a no-default-features -regression is per crate, which is what CI does (`:111-115`): - -```bash -cargo check -p horus_core --no-default-features -cargo check -p horus --no-default-features -cargo check -p horus_types --no-default-features -cargo check -p horus_net --no-default-features -cargo check -p horus_sys --no-default-features -cargo check --workspace --all-features # feature-matrix.yml:150 -``` - -If you add a workspace member, add it to that list, or it has no -no-default-features floor. - -### Acceptance tests - -- **Rust** — `horus_core/tests/acceptance_topic.rs` and - `horus_core/tests/acceptance_scheduler.rs`. -- **Cross-platform CLI** — `tests/docker/acceptance_linux_macos.sh` and - `tests/docker/acceptance_windows.ps1`. -- **End-to-end QA scenarios** — `tests/qa/launch`, `tests/qa/pubsub_rust`, - `tests/qa/pubsub_python`. +**Before submitting a PR:** +1. Review the relevant acceptance tests +2. Ensure changes align with documented behavior +3. Update tests if changing functionality +4. Add new scenarios for new features ```bash +# Run the Rust acceptance tests cargo test -p horus_core --test acceptance_topic cargo test -p horus_core --test acceptance_scheduler -bash tests/docker/acceptance_linux_macos.sh - -horus new test_project && cd test_project && horus run -``` -### Benchmarks - -`Run Benchmarks` is a required check. It runs two binaries — -`cross_process_benchmark` and `robotics_messages_benchmark` — `BENCH_REPETITIONS` -times each, which is 3 (`benchmarks.yml:53`, `:179-188`), and feeds the JSON -reports to a `regression_gate` binary -(`benchmarks/src/bin/regression_gate.rs`, built at `benchmarks.yml:162-166`, -invoked at `:242-249`). The gate reduces the repetitions to their median so one -bad draw cannot fail the build (`:48-52`) and compares against a rolling window -of `BENCH_WINDOW` past trunk runs, which is 20 (`:55-58`). - -There is no single threshold: each metric is gated at `max(relative, center + -k·σ_observed, absolute floor)`, and the policy, the blocking/report-only split -and the reasoning for every number live in `RegressionPolicy` in -`benchmarks/src/output.rs:369` (`benchmarks.yml:38-45`). The figure of merit is -worst-case latency and jitter first, median second — a change that improves the -median and widens the tail is a regression here (`:25-27`). - -Two things only happen on a push to `main` (`:237-240`): `--record`, which -rewrites the rolling baseline, and `--trunk`, which enables the -consecutive-runs escalation. Both are deliberately off for pull requests, so -your PR is compared against trunk but never becomes its own baseline. - -```bash -cargo run --release --bin cross_process_benchmark -cargo run --release --bin robotics_messages_benchmark -``` - -The five `[[bench]]` targets in `benchmarks/Cargo.toml:84-102` — `topic_latency`, -`topic_throughput`, `scheduler_jitter`, `scheduler_ipc_latency`, `tensor_pool` — -are a separate thing and the gate does not read them. - -### Documentation contract - -`horus-docs` is a separate repository, and this repository's tests can break its -published pages. On a PR, `docs-contract.yml` runs a hermetic tier needing no -docs checkout (`:109-167`), a live tier that clones `softmata/horus-docs` at a -pinned commit (`:172-183`), and a shipped-examples build (`:289-334`). - -The pin is `DOCS_PINNED_REF` (`docs-contract.yml:103`). If your change requires a -docs change to stay true, land the docs change first, then bump the pin in a PR -here. Verify the new commit before you bump it — a pin is a claim about a -specific tree (`:58-65`): +# Run the cross-platform CLI acceptance suite +bash tests/docker/acceptance_linux_macos.sh -```bash -HORUS_DOCS_DIR= \ - cargo test -p horus_manager --test docs_contract -- --include-ignored -HORUS_DOCS_DIR= \ - cargo test -p horus_manager --test docs_cli_contract +# Manually verify scenarios +horus new test_project +cd test_project +horus run ``` -`HORUS_DOCS_DIR` must point at a directory containing `content/docs`, or it is -ignored with a warning (`horus_manager/tests/docs_contract.rs:61-68`). Without a -checkout these tests *skip* locally and *fail* under `CI` -(`horus_manager/tests/docs_contract.rs:19-21`, `:100`), so a green local run -proves nothing if you forgot the variable. - -There is deliberately no workflow input to override the pin. `schedule` and -`workflow_dispatch` run in default-branch context with write access to the -Actions cache, so building an arbitrary ref of another repository here would let -anyone who can push to `softmata/horus-docs` execute code in this repository's -privileged context (`docs-contract.yml:86-93`). - ## Code Style ### Rust -- **Formatting is mandatory.** `ci.yml:93` runs - `cargo fmt --all -- --check` with no condition and no `continue-on-error`; the - `fmt` job is in `ci-success`'s `needs` list (`ci.yml:481`) and its result is - checked (`ci.yml:490`). A formatting diff fails a required check on every PR, - whichever branch you target. -- **Clippy warnings fail when the PR targets `main`.** `ci.yml:135-136` runs - `cargo clippy --workspace --all-targets --exclude horus_py -- -D warnings` plus - a second `-p horus_py --all-targets --no-default-features` pass. Targeting - `dev` drops the `-D warnings` (`ci.yml:138-139`), which defers the problem - rather than removing it. -- **Compiler warnings fail when the PR targets `main`.** `RUSTFLAGS` is set at - the workflow level (`ci.yml:28`), so `-D warnings` applies to `check`, `test`, - `doc` and `loom` as well as clippy. -- **Rustdoc warnings fail when the PR targets `main`** (`ci.yml:347`). Note that - `cargo doc` excludes `horus_manager` (`ci.yml:345`) to avoid a - `target/doc/horus/` filename collision with the `horus` binary - (`ci.yml:337-338`), so doc warnings in that crate are not caught. `horus_py` - needs no split here — rustdoc never sets `cfg(test)`, so the - `extension-module` guard does not fire (`ci.yml:340-344`). - -Run the strict configuration locally before you push: - -```bash -cargo fmt --all -- --check -RUSTFLAGS="-D warnings" cargo clippy --workspace --all-targets --exclude horus_py -- -D warnings -RUSTFLAGS="-D warnings" cargo clippy -p horus_py --all-targets --no-default-features -- -D warnings -RUSTDOCFLAGS="-D warnings" cargo doc --workspace --no-deps --exclude horus_manager -cargo audit # cargo install cargo-audit --locked, if you have not -``` - -Document public APIs with `///`: +Follow standard conventions: +- Format code when possible: `cargo fmt` (optional, as it may fail in some cases) +- Run `clippy` and address critical issues: `cargo clippy` (warnings are acceptable) +- Document public APIs with `///` comments ```rust /// Creates a new Topic for inter-process communication. @@ -637,299 +139,165 @@ pub fn new(topic_name: &str) -> Result { } ``` -### Unsafe code - -`horus_core/src` holds 340 lines containing `unsafe {` and 253 lines carrying a -`// SAFETY:` comment. **Nothing enforces the correspondence.** There is no -`undocumented_unsafe_blocks`, `missing_safety_doc`, `unsafe_op_in_unsafe_fn` or -`deny(unsafe…)` in the workspace lints (`Cargo.toml:83-102`) or anywhere else — -a grep for those names across every `.rs` and `.toml` in the repository returns -nothing. The only mechanisms are the PR template's safety checklist -(`.github/PULL_REQUEST_TEMPLATE.md:35-40`) and review. - -So write the comment because the next person needs it, and be aware of what the -machine will and will not do for you: - -- MIRI, ASan and TSan run only on PRs targeting `main` that touch - `horus_core/src/memory/**` or `horus_core/src/communication/**` - (`safety.yml:18-23`), and none of the three is a required check. -- MIRI cannot execute most of `horus_core` at all: it does not support - file-backed memory mappings, which since the transport went SHM-only rules out - every `Topic`, the `TensorPool` and every pool-backed type - (`safety.yml:75-82`). The file records that each of the three steps that used - to live there died on the first such test, so the job has never reported a UB - result. What it runs now is the SIMD copy helpers, the RT allocator guard and - the SHM path helpers (`:84-86`): - - ```bash - MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-permissive-provenance" \ - cargo miri test -p horus_core --lib -- memory::simd memory::rt_allocator memory::platform - ``` - - (`safety.yml:92-95`; the three modules are `horus_core/src/memory/simd.rs`, - `rt_allocator.rs` and `platform.rs`.) The mmap-backed paths are covered - instead by ASan, TSan and the `Valgrind Memcheck` job in - `cpp-bindings.yml:540-541` (`safety.yml:88-89`). - -### Test quality - -`ci.yml:434-476` greps the tree for stub-test anti-patterns and fails the build -on two of them: - -| Pattern | Why it fails | -| --- | --- | -| `is_ok() \|\| is_err()` (either order), `ci.yml:440` | always true; asserts nothing | -| a bare `matches!(…);` statement not preceded by an `assert`, `ci.yml:448-463` | the `bool` is discarded | - -A third, `let _ = format!(…)` in test code, emits a warning rather than failing -(`ci.yml:467-469`). - -One local caveat: the guard runs `grep -rn … --include='*.rs' .` from the -repository root with no `--exclude-dir` (`ci.yml:440`, `:448`, `:467`). A CI -checkout has no `target/`; yours does, so reproducing the scan locally will also -walk generated sources under it. Point the grep at the crate directories rather -than at `.`. - ### Python -Follow PEP 8 and use descriptive names — but be aware that **no check enforces -any of it**. A case-insensitive grep over `.github/workflows/` finds no `black`, -`mypy`, `ruff` or `flake8` invocation anywhere. The only automated Python gate is -the pytest suite in `ci.yml:240-257`. +Follow PEP 8: +- Use `black` for formatting +- Use `mypy` for type checking +- Use descriptive variable names ## What to Contribute -### Good first issues +### Good First Issues -Issues labelled `good first issue` — the label name has spaces, and the -repository's label set also carries `help wanted`, `bug`, `enhancement`, -`documentation`, `testing`, `rust` and `CLI tools`. +Look for issues labeled `good-first-issue`: +- Documentation improvements +- Example programs +- Bug fixes +- Test coverage improvements -### Feature requests +### Feature Requests -Before implementing a major feature, open an issue, wait for maintainer -feedback, then implement with tests and documentation. The seven required checks -take a while; discovering after the fact that a design is unwanted is expensive -for everyone. +Before implementing a major feature: +1. Open an issue to discuss the proposal +2. Wait for maintainer feedback +3. Implement with tests and documentation -### Bug reports +### Bug Reports -Include the HORUS version (`horus --version`, wired at -`horus_manager/src/main.rs:14`), the operating system and version, a minimal -reproducible example, expected versus actual behaviour, and the relevant logs. -`horus doctor` (`horus_manager/src/main.rs:852`, dispatched at `:3557`) checks -the machine — toolchains, real-time capability, shared memory (`:848`) — and -takes `--verbose`, `--json`, `--fix` and `--rt`. `horus check` checks the -repository; `:850` notes that neither substitutes for the other. +When reporting bugs, include: +- HORUS version (`horus --version`) +- Operating system and version +- Minimal reproducible example +- Expected vs actual behavior +- Relevant logs or error messages ## Pull Request Process -1. **Choose the base branch deliberately.** `main` takes the strict gates now; - `dev` defers them. - -2. **Run the gates locally.** At minimum: +1. **Ensure tests pass**: ```bash - cargo fmt --all -- --check - RUSTFLAGS="-D warnings" cargo clippy --workspace --all-targets --exclude horus_py -- -D warnings - cargo test --workspace --exclude horus_py --lib --no-fail-fast -- --test-threads=1 + cargo test ``` -3. **Update the acceptance tests** in `horus_core/tests/acceptance_*.rs` and - `tests/docker/` if behaviour changed, and add scenarios for new features. - -4. **Update documentation.** README.md for user-facing changes, inline `///` - docs, examples, and `CHANGELOG.md`. The changelog is machine-checked: - `horus_manager/tests/changelog_contract.rs` requires that a changelog exists - (`:51`), that every released git tag appears in it (`:63`), that the version - in `env!("CARGO_PKG_VERSION")` — `horus_manager`'s own package version, 0.4.0 - at `horus_manager/Cargo.toml:3`, since there is no `[workspace.package] - version` to inherit — has either its own `[x.y.z]` section or an - `## Unreleased` section to collect into (`:88-96`), and that releases are - listed newest first (`:101`). - - Those nine tests, plus `translation_contract.rs`'s five and ten of - `readme_contract.rs`'s eleven, run inside the required - `Integration Tests Success` context. No workflow names any of the three - files; they run because `integration-tests.yml:794` is - `cargo test -p horus_manager --release -- --test-threads=1` with no target - filter, `horus_manager/Cargo.toml` declares no `[[test]]` blocks and no - `required-features`, so every file in `horus_manager/tests/` is autodiscovered, - and line 794 sits inside the `cli-integration` job (`:309`) which is in the - aggregator's needs list (`:1374`). - - The exception is `readme_contract.rs`'s `the_readme_quick_start_compiles`, - which is `#[ignore]`d with the reason "run in the docs-contract job" - (`readme_contract.rs:146`). Nothing runs it: no workflow mentions - `readme_contract`, `docs-contract.yml`'s cargo invocations name only the - `docs_*` targets, and `integration-tests.yml:794` passes no - `--include-ignored`. If you change the README's Quick Start, compile it - yourself: + Optionally run `cargo fmt` (may fail in some cases) and `cargo clippy` (warnings are acceptable). - ```bash - cargo test -p horus_manager --test readme_contract -- --include-ignored - ``` +2. **Check acceptance tests**: + - Review the acceptance tests in `horus_core/tests/acceptance_*.rs` and `tests/docker/` + - Manually verify key scenarios + - Update test scenarios if behavior changed + - Add new scenarios for new features -5. **Write clear commit messages.** `git-cliff` groups the changelog by - conventional-commit prefixes — `feat`, `fix`, `doc`, `perf`, `refactor`, - `style`, `test`, `chore`, `ci`, `deps` (`.github/cliff.toml:47-61`) — so a - prefixed subject lands in the right section. `chore(release)` is skipped - entirely (`:55`), `chore(deps)` and `deps` both land under Dependencies - (`:56`, `:59`), and a commit whose *body* matches `security` is grouped under - Security regardless of prefix (`:60`). +3. **Update documentation**: + - Update README.md for user-facing changes + - Update inline code documentation + - Add examples for new features + - Update CHANGELOG.md +4. **Write clear commit messages**: ``` - fix: brief description + Add feature: Brief description Detailed explanation of what changed and why. - Updated acceptance tests in horus_core/tests/acceptance_*.rs + - Added new scenarios for ... Fixes #123 ``` -6. **Fill in the PR template** (`.github/PULL_REQUEST_TEMPLATE.md`). It appears - automatically. Its safety (`:35-40`) and performance (`:42-46`) sections are - the ones reviewers read most closely. +5. **Submit PR** with: + - **Base branch: `dev`** (not `main`) + - Clear title and description + - Link to related issues + - Screenshots/examples if UI changes + - List of verified acceptance test scenarios -7. **Sign the CLA on your first PR.** `.github/CLA.md:71` states that making a - contribution already binds you to the agreement; the requested signal is a - comment on the pull request: +6. **Address review feedback** promptly - ``` - I have read and agree to the Contributor License Agreement. - ``` +### PR Description Template - **No bot checks for it** — a case-insensitive grep for `cla` and - `contributor license` over `.github/workflows/` matches nothing — so nothing - will remind you and nothing will block the merge. A maintainer will ask if it - is missing. For significant contributions you may be asked to sign a physical - copy (`.github/CLA.md:77`). +```markdown +## Description +Brief description of changes -8. **Resolve every review thread.** Protection requires it. The Copilot reviewer - re-runs on every push to a PR targeting `main`, so expect threads you did not - open. +**Note:** This PR targets the `dev` branch. -9. **Expect to be rebased.** Strict mode plus a busy `main` means your branch - will be updated repeatedly and re-run each time. +## Changes +- Added feature X +- Fixed bug Y +- Updated tests + +## Testing +- [ ] Unit tests pass (`cargo test`) +- [ ] Code formatted if possible (`cargo fmt`, optional) +- [ ] Clippy checked (`cargo clippy`, warnings acceptable) +- [ ] Verified acceptance test scenarios +- [ ] Updated/added acceptance tests + +## Related Issues +Fixes #123 +``` ## Architecture Guidelines -### Core principles +### Core Principles -1. **Zero-copy when possible** — use shared memory. -2. **Type safety** — leverage Rust's type system. -3. **Minimal latency** — profile and optimize hot paths; judge on the tail, not - the median (`benchmarks.yml:25-27`). -4. **Multi-language** — features should work across Rust, C++ and Python. +1. **Zero-copy when possible**: Use shared memory +2. **Type safety**: Leverage Rust's type system +3. **Minimal latency**: Profile and optimize hot paths +4. **Multi-language**: Ensure features work across Rust/Python -### Code organization +### Code Organization ``` horus/ -├── horus/ # Unified entry crate (horus::prelude, horus/src/lib.rs:581) -├── horus_core/ # Core framework (core, communication, memory, scheduling, types, services, actions, drivers) +├── horus/ # Unified entry crate (use horus::prelude::*) +├── horus_core/ # Core framework (Node, Topic, Scheduler, types, memory, scheduling) ├── horus_types/ # Universal IPC types (math, diagnostics, time, generic) ├── horus_macros/ # Procedural macros (node!, LogSummary derive) ├── horus_sys/ # Platform abstraction layer ├── horus_net/ # Transparent LAN replication -├── horus_cpp/ # C++ FFI bridge (CXX; crate-type staticlib, cdylib, rlib) +├── horus_cpp/ # C++ FFI bridge (CXX, staticlib+cdylib) ├── horus_cpp_macros/ # C++ binding codegen (#[horus_api] proc macro) ├── horus_manager/ # CLI tool (horus command) ├── horus_py/ # Python bindings -├── benchmarks/ # Performance benchmarks (package name: horus_benchmarks) +├── benchmarks/ # Performance benchmarks ├── examples/ # Example projects (Rust, C++, Python) └── tests/ # Integration and QA tests ``` -The eleven crates are the workspace members listed in `Cargo.toml:3-15`, with -the per-crate descriptions taken from the comments there and from each crate's -own manifest. `examples/` and `tests/` are not workspace members. `node!` is the -proc macro at `horus_macros/src/lib.rs:76-79`; `LogSummary` is the derive at -`:102`. - -Standard message types come from `horus_types/` and the external `horus-robotics` -git dependency, pinned by rev in `[workspace.dependencies]` (`Cargo.toml:40`). -`TransformFrame` comes from the external `horus-tf` git dependency, which is -declared per crate rather than in the workspace table — `horus/Cargo.toml:25`, -`horus_core/Cargo.toml:73`, `horus_cpp/Cargo.toml:21`, -`horus_manager/Cargo.toml:38`, `horus_py/Cargo.toml:31` — and re-exported through -`horus::prelude` at `horus/src/lib.rs:613`. Both git deps are patched to this -workspace's local `horus_core`, `horus_types` and `horus_macros` -(`Cargo.toml:108-115`), because they reference `horus_core` by relative path -(`Cargo.toml:105-107`). +Standard message types come from `horus_types/` and the external `horus-robotics` git dependency; `TransformFrame` comes from the external `horus-tf` git dependency, re-exported through `horus::prelude`. ## What Not to Do -- Break existing APIs without a migration path. -- Add dependencies without discussion. `cargo audit` is a gate on `main` - (`ci.yml:302-303`), and `.cargo/audit.toml` is where a tolerated advisory has - to justify itself. That list is currently **empty** (`ignore = []`), which the - file says is the target state: an ignore entry suppresses its advisory for - every matching version forever, including one that lands in the lockfile years - later. Removed entries are kept above it as commented history with the reason - each was retired. If you need to add one, name why it is tolerated and what - would let us remove it. -- Widen the `/dev/shm` permissions to make a test pass. - `regression_shm_topics_dir_permissions_restricted` - (`horus_core/tests/regressions.rs:307-323`) asserts the topics directory is - exactly `0o700`, and that assertion is the guarantee, not the obstacle. -- "Fix" `loom_pod_broadcast`'s `naive` variant. It is supposed to fail. -- Commit without running the tests, or open a PR with no description. -- Ignore a red check on the grounds that it is not required. Most red CI here is - the runner rather than the code (`autopilot.yml:11`), but that is a conclusion - to reach after reading the log, not before. `autopilot.yml:88-89` adds the - corollary: detection of the concurrency bugs here is probabilistic, so passing - once is not evidence of a fix, and `:93-95` asks for an ablation — show the - test fails without your change and passes with it. +- Break existing APIs without migration path +- Add dependencies without discussion +- Commit without running tests +- Submit PRs without description or testing +- Ignore test failures ## Code Review -All contributions go through review. Be respectful and constructive, respond to -feedback promptly, ask when something is unclear, and expect maintainers to have -the final say. Whether protection mechanically requires an approval is -genuinely unclear — the two GitHub APIs disagree, as recorded above — so treat -review as a norm the project keeps rather than something to rely on the machine -to enforce. - -## Code of Conduct - -`CODE_OF_CONDUCT.md` applies to all project spaces (`:31-33`). Its enforcement -section (`:35-39`) says to report by "contacting the project team" and names no -address. There is no address to name: `contact@softmata.dev` at `README.md:648` -is the only email in any markdown file in this repository, and it is published -for hardware feedback (`README.md:644-648`), not as a conduct channel. Until -someone decides on a real reporting address, open an issue or contact a -maintainer directly. This gap needs closing properly rather than by pointing at -an address meant for something else. +All contributions go through code review: +- Be respectful and constructive +- Respond to feedback promptly +- Ask questions if unclear +- Maintainers have final say ## License -By contributing, you agree that your contributions will be licensed under the -Apache License 2.0, and that they are subject to the -[Contributor License Agreement](.github/CLA.md) — see step 7 of the PR process -for how to signal agreement. - ---- +By contributing, you agree that your contributions will be licensed under the Apache License 2.0. -## Further reading +All contributors must agree to the [Contributor License Agreement (CLA)](.github/CLA.md). When submitting your first pull request, add a comment: -This file covers the process. The rest of the contributor documentation lives -under `docs/`, and each of these answers a question this file deliberately does -not: +``` +I have read and agree to the Contributor License Agreement. +``` -| Document | Answers | -|---|---| -| [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) | Where does my change go? A crate and module map, and the execution model that decides which thread a node runs on. | -| [`docs/TESTING.md`](docs/TESTING.md) | Which suite does my test belong in, what does CI actually invoke, and what is committed but never reached? | -| [`docs/SHM_WIRE_FORMAT.md`](docs/SHM_WIRE_FORMAT.md) | I need to change the shared-memory layout. What else has to move with it? | -| [`docs/RELEASING.md`](docs/RELEASING.md) | How is a release cut, and what gates does the tag then face? | -| [`horus_net/docs/BLUEPRINT.md`](horus_net/docs/BLUEPRINT.md) | The network protocol spec that thirteen modules cite by section number. | -| [`horus_core/README.md`](horus_core/README.md) | An index into the crate's design documentation, most of which `cargo doc` does not render. | +## Thank You! -For using HORUS rather than changing it, see the documentation site, and in -particular its Development section for the CLI, the environment variables, and -the runtime extension points. +Every contribution helps make HORUS better. Thank you for being part of the community! --- -Questions? Open an issue or start a discussion on GitHub. +Questions? Open an issue or start a discussion on GitHub! diff --git a/benchmarks/README.md b/benchmarks/README.md index bf1432de8..ccb39a0f6 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -71,7 +71,7 @@ the withdrawn tables dropped, and it is not comparable across kinds: | `CrossThread-MPMC` | `PodShm` | `send` | Producer-side `send()` cost under contention | | `CrossProc-1P1C` | `SpscShm` | `one-way` | Pub-to-sub delivery, RDTSC timestamp in payload | | `CrossProc-2P1C` | `MpscShm` | `one-way` | Pub-to-sub delivery | -| `CrossProc-1PMC` | `SpmcShm` | `one-way` | Pub-to-sub delivery | +| `CrossProc-1PMC` | `PodShm` | `one-way` | Pub-to-sub delivery; one-to-many POD routes to broadcast | | `CrossProc-PodShm` | `PodShm` | `broadcast` | Latest-value broadcast; readers may skip ahead | | `RawAtomic` | — | `one-way` | Raw SHM atomic store/load — the hardware floor | @@ -104,7 +104,7 @@ sudo cpupower frequency-set -g performance echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo ``` -> Cross-process paced benchmarks (SpscShm, MpscShm, PodShm) are heavily affected by OS scheduling under `powersave` governor. The SpmcShm path (seqlock broadcast) is least affected since readers don't block. +> Cross-process paced benchmarks (SpscShm, MpscShm) are heavily affected by OS scheduling under `powersave` governor. The `PodShm` broadcast path is least affected, since its readers do not block — a one-to-many POD topology resolves to `PodShm`, not `SpmcShm`, which is why no row above names the latter. ## License diff --git a/horus_core/README.md b/horus_core/README.md index 1d4274932..76b295099 100644 --- a/horus_core/README.md +++ b/horus_core/README.md @@ -1,528 +1,109 @@ -# horus_core +# HORUS Core -The runtime: nodes, the scheduler and its four executors, shared-memory topics, -the tensor pool, actions, services, errors. 112,344 lines under `src/` -(`find horus_core/src -name '*.rs' | xargs wc -l`), of which 14,340 are `///` -doc lines and 2,280 are `//!` module prose. +**Internal implementation crate.** Use the `horus` crate instead: -Almost none of that prose is reachable through `cargo doc`. This file is the -index to it. It is written for someone changing this crate, not for someone -using it. - -**If you are writing a robot, you want the `horus` crate**, whose rustdoc is -public and whose `prelude` is the supported surface (`horus/src/lib.rs:581`). -The user-facing API reference lives at `content/docs/rust/api/core.mdx` in -`horus-docs` — 1,234 lines of hand-written MDX, not generated from this source. -See [Who actually depends on this crate](#who-actually-depends-on-this-crate) -for what "internal" does and does not mean in practice. - ---- +```rust +use horus::prelude::*; +``` -## Why cargo doc shows you nothing +## Overview -`horus_core/src/lib.rs` is 96 lines. Eleven of its twelve `pub mod` -declarations carry `#[doc(hidden)]` — ten unconditionally, and `testing` both -hidden and `cfg`-gated: +The core runtime for HORUS, providing: -| Module | Hidden? | Declared at | -|---|---|---| -| `actions` | yes | `lib.rs:20-21` | -| `communication` | yes | `lib.rs:22-23` | -| `core` | yes | `lib.rs:24-25` | -| `drivers` | yes | `lib.rs:26-27` | -| `error` | yes | `lib.rs:28-29` | -| `memory` | yes | `lib.rs:30-31` | -| `params` | yes | `lib.rs:32-33` | -| `scheduling` | yes | `lib.rs:34-35` | -| `services` | yes | `lib.rs:36-37` | -| `terminal` | **no** | `lib.rs:38-44` | -| `types` | yes | `lib.rs:45-46` | -| `testing` | yes, and `cfg(any(test, feature = "test-utils"))` | `lib.rs:50-52` | +- **Node** - Simple trait with `init/tick/shutdown` lifecycle +- **Scheduler** - Runs nodes in order with configurable rates and composable builders +- **Topic** - Pub/sub communication that auto-selects the fastest transport +- **Logging** - `hlog!()` macro with context capture -There is one more declaration, `pub(crate) mod utils;` at `lib.rs:47`, which is -private and so never rendered either. +## Quick Start -Every crate-root re-export is hidden as well, apart from one: `ControlCommand` -(`lib.rs:79`). `Topic`, `Scheduler`, `Node`, `HorusError` and the rest go out -behind `#[doc(hidden)]` at `lib.rs:56-86`, and the four re-exported -dependencies (`bytemuck`, `paste`, `serde_json`, `serde_yaml`) at -`lib.rs:88-96`. +```rust +use horus::prelude::*; -`#[doc(hidden)]` on a module strips the module *and everything inside it* from -the rendered output. A public re-export *out of* a hidden module still renders, -which is why `ControlCommand` survives, and a `#[macro_export]` macro declared -inside a hidden module also renders, because `macro_export` puts it at the -crate root regardless. Nothing else in this crate takes either route. +fn main() -> Result<()> { + let mut scheduler = Scheduler::new(); -So the crate's front page carries exactly four things. This is observed, not -inferred — `cargo doc -p horus_core --no-deps` with `CARGO_TARGET_DIR` pointed -at a scratch directory produces a `sidebar-items.js` containing precisely: + scheduler.add(SensorNode::new()?).order(0).build()?; + scheduler.add(ControlNode::new()?).order(1).build()?; -```js -window.SIDEBAR_ITEMS = {"enum":["ControlCommand"],"macro":[["action",1], -["hlog",1],["hlog_every",1],["hlog_once",1],["horus_internal",1], -["impl_frame_id_field",1],["impl_tensor_accessors",1],["impl_tensor_backed",1], -["impl_timestamp_field",1],["message",1],["register_driver",1],["service",1], -["standard_action",1],["topics",1]],"mod":["terminal"]}; + scheduler.run() +} ``` -That is the sixteen lines of crate prose at `lib.rs:1-16`, one module, one -enum, and fourteen macros. Fifteen `#[macro_export]` macros exist in the crate; -the fifteenth, `__hlog_level_const`, carries its own `#[doc(hidden)]` -(`core/hlog.rs:174-175`) and is the one missing from that list. Everything -else — 112,000 lines and the arguments in them — renders nowhere. - -Two consequences worth stating plainly. - -**The prose is lint-gated but not published.** The `doc` job at -`.github/workflows/ci.yml:308` runs -`cargo doc --workspace --no-deps --exclude horus_manager` (`ci.yml:345`) with -`RUSTDOCFLAGS: -D warnings`, but only when the ref is `main` or the base ref is -`main` (`ci.yml:347`) — so a PR into `main` is gated and a PR into `dev` is -not. Rustdoc's lints fire inside `#[doc(hidden)]` items: a broken intra-doc -link in a hidden module is a warning by default and a hard error under -`-D warnings`, verified on a two-line probe crate. So every one of those 14,340 -doc lines is held to a standard, and then discarded. No workflow publishes -`target/doc` (`grep -rn 'cargo doc\|target/doc\|gh-pages\|deploy-pages' -.github/workflows/` matches only the comment at `ci.yml:338` and the command at -`ci.yml:345`). `docs.horusrobotics.dev`, declared as this crate's -`documentation` at `Cargo.toml:11`, is the `horus-docs` Next.js site -(`horus-docs/app/sitemap.ts:7`), not rustdoc output. And docs.rs cannot help, -because none of these crates are on crates.io (`README.md:70-73`). - -**Instructions that users must follow are invisible.** The clearest case: -`memory/rt_allocator.rs:15-19` tells you to write +## Node Trait ```rust -#[global_allocator] -static ALLOC: horus_core::memory::rt_allocator::RtAwareAllocator = - horus_core::memory::rt_allocator::RtAwareAllocator; +pub trait Node: Send { + fn name(&self) -> &str; + fn init(&mut self) -> Result<()>; + fn tick(&mut self); + fn shutdown(&mut self) -> Result<()>; +} ``` -in your own `main.rs`, because without it `.no_alloc()` is inert -(`rt_allocator.rs:28-38`). `memory` is hidden here and hidden again in the -facade (`horus/src/lib.rs:389-390`), so that page renders in neither crate. The -snippet is only correct at all because -`horus_core/tests/rt_alloc_guard_installed.rs:39-41` compiles it as that test -binary's real global allocator — an earlier version of the same paragraph named -`horus_core::memory::RtAwareAllocator`, a path that does not exist, and nothing -caught it (`rt_allocator.rs:21-26`, `rt_alloc_guard_installed.rs:35-38`). - -## How to read the internals +## Scheduler -`--document-private-items` is **not** enough on its own. It disables rustdoc's -private-stripping pass, not its hidden-stripping pass, so the eleven hidden -modules stay stripped. You need nightly and `--document-hidden-items`, which is -unstable and refuses to run without `-Z unstable-options`: - -```sh -RUSTDOCFLAGS="-Z unstable-options --document-hidden-items" \ - cargo +nightly doc -p horus_core --no-deps --document-private-items --open +```rust +// Lightweight default +let mut scheduler = Scheduler::new(); + +// Per-node execution classes via fluent builders +scheduler.add(motor).order(0).rate(1000_u64.hz()).build()?; // RT (auto-derived) +scheduler.add(planner).order(5).compute().build()?; +scheduler.add(telemetry).order(10).async_io().rate(1_u64.hz()).build()?; + +// Production-ready with watchdog, blackbox, RT +let mut scheduler = Scheduler::new() + .watchdog(500_u64.ms()) + .blackbox(64) + .tick_rate(1000_u64.hz()) + .prefer_rt(); + +// Add nodes with execution order +scheduler.add(my_node).order(0).rate(100_u64.hz()).build()?; + +// Run +scheduler.run()?; +scheduler.run_for(10_u64.secs())?; ``` -Both halves matter. `--document-hidden-items` restores the hidden modules; -`--document-private-items` restores the `pub(crate)` ones — `topic::header` -(`communication/topic/mod.rs:118`), `topic::dispatch` (`:131`), -`topic::seqlock` (`:134`), `scheduling::rt_executor` -(`scheduling/mod.rs:34`), `scheduling::primitives` (`:31`) and their -neighbours — which is where most of the protocol writing actually lives. - -Each half was checked separately on a probe crate rather than taken from -memory, because this is the one instruction in this file a reader will act on: -plain `rustdoc --document-private-items` leaves a `#[doc(hidden)] pub mod` -absent from the sidebar and adds only the `pub(crate)` module; passing -`--document-hidden-items` without `-Z unstable-options` fails with *"the -`-Z unstable-options` flag must also be passed"*; the command above, run -end-to-end through cargo, lists the hidden module. If you doubt any of it, a -two-line `lib.rs` reproduces all three in ten seconds. - -`horus doc` does not do this. It documents the *user's* project, from that -project's directory, pointing cargo at `.horus/Cargo.toml` -(`horus_manager/src/commands/doc.rs:1-4` and `:16-48`). - -## The module index - -Sizes are `src/` line counts. "Read first" names the file whose header carries -the real argument, not the file with the most code. - -| Module | Lines | Read first | -|---|---:|---| -| `scheduling` | 39,523 | `scheduling/node_builder.rs:6-37` | -| `communication` | 31,649 | `communication/topic/mod.rs:1-112` | -| `memory` | 11,165 | `memory/rt_allocator.rs:1-88` | -| `core` | 8,918 | `core/node.rs:953-971` | -| `actions` | 5,925 | `actions/mod.rs:1-122` | -| `types` | 4,405 | `types/mod.rs:1-22` | -| `error.rs` | 2,937 | `error.rs:1-64` | -| `services` | 2,492 | `services/mod.rs:1-60` | -| `drivers` | 1,908 | `drivers/mod.rs:1-19` | -| `params.rs` | 1,666 | `params.rs:1-3` | -| `testing` | 1,449 | gated on `cfg(test)` or `feature = "test-utils"` (`Cargo.toml:91-92`) | -| `terminal.rs` | 63 | `terminal.rs:1-19` | - -Those twelve plus `utils.rs` (148) and `lib.rs` (96) are the whole crate. - -### communication - -`communication/topic/mod.rs:1-112` is the single most important header in the -crate: the backend detection matrix, the thread-ownership contract, the -lock-free ring protocols, the `read_latest` use-after-free argument and the -drop-safety rules. Read it before touching anything under -`communication/topic/`. Its closing "Verification" paragraph (`:106-112`) is -stale and should not be trusted: it claims "16 loom exhaustive concurrency -tests … for SPSC, SPMC, MPSC, and MPMC algorithms, including `read_latest` + -`try_recv` races", where the tree holds ten loom targets with 36 `#[test]` -functions between them, none of which mentions `read_latest`. The models that -covered those four ring algorithms were `loom_ring_buffers`, deleted with the -intra backends (`ci.yml:379-380`). - -Then, in order of how much they will surprise you: - -- `communication/topic/dispatch.rs:2-118` — one complete send/recv path per - backend, plus a measured account of where the remaining ~55 ns against a raw - ring goes. The ablation table at `:31-39` is the useful part: removing the - consumer's head gate makes things 2-4 ns *worse*. Its conclusion (`:54-61`) - is that anyone trying to close the gap should change the protocol, not delete - work from these functions, and should check first whether 55 ns of a 1 kHz - period is worth anything. -- `communication/topic/shm_layout.rs:1-50` — the authoritative byte layout, - and why it is a module rather than a comment. `horus_net` once copied a table - of offsets and the copy rotted in four independent places (`:11-16`) while - feeding an IPC region from unauthenticated network data. Each constant is now - tied to its field by `offset_of!` in a `const` block (`:20-24`), so a - reordered `TopicHeader` field is a build failure. -- `communication/topic/header.rs:24-66` — `TOPIC_VERSION`, currently `4` - (`:66`), and why v4 is a hard break rather than a soft migration. Neither of - its two changes (monotonic timestamps, `layout_kind`) is detectable in-band - by a v3 reader, so the open path refuses the segment instead. -- `communication/topic/seqlock.rs:1-47` — the drop-oldest fanout ring's - per-slot version protocol. -- `communication/pod.rs:1-77` — POD auto-detection by `!needs_drop::()`, - and the caveat at `:10-26` that it is a heuristic and unsound for `bool`, - fieldless enums and `NonZero*`. Prefer explicitly encoded integer fields in - messages you intend to send as fixed-size. -- `communication/topic/migration.rs:2-5` and `communication/topic/registry.rs:1-5` - — CAS-locked backend migration, and the process-local epoch counter that lets - sibling `Topic` instances notice it with a ~1 ns L1 read instead of a ~20 ns - SHM read. - -### scheduling - -`scheduling/node_builder.rs:6-37` is the entry point: the decision tree and -table for the five execution classes (`Rt`, `Compute`, `Event(String)`, -`AsyncIo`, `BestEffort` — `scheduling/types.rs:279-291`), and the rules that -catch people out (`:33-37`). One class per node, last call wins with a warning; -`.rate()` on a `BestEffort` node auto-promotes it to `Rt`; `.compute().rate()` -stays `Compute` and the rate becomes informational; a chain without `.build()?` -is silently dropped. - -The executors each carry an ASCII architecture diagram in their header: -`scheduling/rt_executor.rs:8-22`, `compute_executor.rs:7-22`, -`event_executor.rs:7-21`, `async_executor.rs:7-22`. Read -`rt_executor.rs:366-429` for the cyclic-wait rewrite — an absolute-deadline -`clock_nanosleep` with a bounded guard spin, replacing a loop that both drifted -(`:382-389`) and busy-waited hard enough to be dequeued by Linux RT bandwidth -control for roughly 50 ms once a second (`:391-401`), a tail that only appears -once RT priority is actually granted (`:403-408`). The trade it makes — median -wake jitter regressing from the old spin's ~100 ns to hrtimer precision, bought -to delete that ~50 ms tail — is stated at `:416-423` rather than buried, along -with the two escape hatches: `HORUS_RT_WAIT=spin` restores a pure spin and -`HORUS_RT_SPIN_GUARD_US` retunes the guard (read at `:675` and `:684`, the -env-var name held at `:472`). The constants that bound it are at `:431-467`: -`SPIN_GUARD_DEFAULT_NS = 20_000` (`:445`), `SPIN_GUARD_PERIOD_SHIFT = 4` -(`:456`), `MIN_SLEEP_SLACK_NS = 10_000` (`:467`). - -`scheduling/primitives.rs:1-24` explains why the RT-side reporting path uses -`rt_diag` and not `print_line`: `print_line` does `isatty` + `tcgetattr`, takes -the process-global stdout lock, and can then block for as long as a slow -console takes — on the thread driving an actuator. - -### memory - -`memory/rt_allocator.rs:1-88` is the `.no_alloc()` contract, and it is the -best-written safety argument in the crate. The parts to not miss: the -installation step is downstream opt-in and a library cannot do it for you -(`:9-26`); without it `.no_alloc()` is silently unenforced, and `is_installed` -/ `warn_if_unenforced` exist so that downgrade can be reported (`:28-38`); a -violation panics inside the tick and routes through the node's `FailurePolicy` -rather than aborting (`:40-66`); and the caveat at `:68-74` that the default -`failure_policy` is `None`, which `apply_failure_policy_after_panic` treats as -log-and-continue — so a `.no_alloc()` node with no explicit policy keeps -ticking and keeps violating, detected and counted, but not acted on. +## Topic Communication -Elsewhere: `memory/tensor_pool.rs:2-60` for the cross-process zero-copy pool -layout; `memory/backend.rs:1-21` for the `PoolBackend` trait that controls -where tensor bytes are allocated — `MmapBackend` is the only built-in (`:10`, -implemented at `:396`), the three CUDA backends at `:12-16` are future work, -and the `HeapBackend` at `:570` is test-only; and `memory/simd.rs:1-40` for why -the hand-written AVX2 copy paths were deleted — both directions lost to -`std::ptr::copy_nonoverlapping`, with the measured table at `:19-23`. - -### core - -`core/node.rs:953-971` carries the `Node` trait (`pub trait Node: Send` at -`:971`). Only `tick()` is required (`:992`); `name()` defaults to the type name -(`:976-982`), `init()` and `shutdown()` default to `Ok(())` and return -`crate::error::HorusResult<()>` (`:987-989`, `:997-999`). -`on_parameter_change` is documented at `:1030-1035` as **not wired** — the -scheduler does not call it, and overriding it today has no effect; register a -callback via `RuntimeParams::on_change()` instead. - -`core/clock.rs:1-13` has the clock-backend table (`WallClock`, `SimClock`, -`ReplayClock`); `core/tick_context.rs:1-8` explains the thread-local ambient -context that `horus::now()`, `dt()` and `rng()` read; `core/log_buffer.rs:170` -onwards documents the per-slot seqlock in the SHM log ring, and `:79` plus -`:477-494` document how a writer that died mid-write is detected, skipped, and -its seqlock reset to even so the slot can be reclaimed. - -### actions, services, errors, drivers, params, types, terminal - -- `actions/mod.rs:1-122` — the goal/feedback/result pattern (`:13-18`), server - and client builders, preemption policies (`:108-110`). At 122 lines it is the - longest module header in the crate, just ahead of `dispatch.rs` (117) and - `topic/mod.rs` (112). -- `services/mod.rs:1-60` — request/response over two topics per service - (`{name}.request`, `{name}.response`, `:41-43`), request-ID filtering for - concurrent clients (`:47-49`), and a ROS2 equivalence table (`:51-60`). -- `error.rs:1-64` — `HorusError` as an umbrella over eleven structured domain - sub-errors, with the variant table at `:10-22` and the matching idiom at - `:24-45`. -- `drivers/mod.rs:1-19` — loading nodes from `horus.toml` `[hardware]`. The - reserved keys that are consumed rather than passed through as `NodeParams` - are the literal list at `drivers/mod.rs:111-124`; the `exec:` prefix dispatch - to `ExecDriver::from_config` is at `:217-229`. -- `params.rs` — runtime key-value parameters, with the eight `ValidationRule` - variants at `:13-32`. Its header is three lines; the module is 1,666 and - undocumented by comparison. -- `types/mod.rs` — zero-copy tensor descriptors and element types. Its header - is a single line, and it hides eight of its own eleven children again at - `:3-22` (`action_chunk`, `image_encoding` and `point` are the three that - would render if the parent did). -- `terminal.rs:1-19` — the only module that renders today, and it renders - because it must: `println!`/`eprintln!` panic when the write fails, and a - robot's stdout fails routinely. Anything printing from a node, a background - thread or a safety path uses these. - -## Load-bearing contracts - -Four things you can break without the compiler noticing. - -**`Topic` is `Send` and deliberately not `Sync`.** `unsafe impl Send for Topic where T::Wire: Send` at -`communication/topic/mod.rs:3711`; there is no `Sync` impl, and the comment at -`:3713-3719` says why — `send`/`recv` take `&self` and mutate -`registered_pub`, `registered_sub`, `owner_attempts` and a `RefCell` keep-alive -queue, so two threads sharing one `&Topic` can release the same pool -slot twice, freeing shared memory that subscribers are still reading. To share -one handle across threads you need a `Mutex>`. **An `RwLock` is not -enough**: a read guard hands out concurrent `&Topic`. Otherwise clone per -thread — `Clone` gives the new thread its own independent state. The module -header states both at `communication/topic/mod.rs:36-48`, and it is the -paragraph to keep in step if the impls change. `RingTopic` is `Send` for -`T: Send` on the same reasoning (`:777`), also with no `Sync`. A compile-time -guard at `:3721-3728` asserts `Topic: Send` so that losing `Send` fails -here rather than at a `thread::spawn` in a downstream crate. This is the -current state of a contract that was previously wrong in both directions: -hand-written `unsafe impl Sync` blocks existed while the prose claimed -`!Send + !Sync`, so safe code could race the unsynchronised state with no -`unsafe` of its own (`:42-48`, `:779-793`). - -**The ring protocols differ per backend, and the ordering is the proof.** -`communication/topic/mod.rs:63-75`: SPSC needs no CAS (producer `Release`s -head, consumer `Acquire`s it); SPMC has consumers CAS the tail with `AcqRel` to -claim exclusive read slots; MPSC has producers CAS head with per-slot -Lamport-style sequence numbers; MPMC CASes both ends. The seqlock fanout ring -is separate again: an odd stamp `(pos << 1) | 1` means a write is in progress -and an even stamp `pos << 1` means done, the producer sets odd *before* -touching data and even *after*, and the consumer must see exactly `next << 1` -on both sides of its copy or discard it (`seqlock.rs:19-34`). Publication -order — done-stamp `Release`, then head `Release` — is what makes `head > next` -imply the done stamp for `next` is visible (`seqlock.rs:36-38`). Retries are -bounded at `SEQLOCK_MAX_ATTEMPTS = 16` (`seqlock.rs:55`). - -Ten loom models live in `horus_core/tests/loom_*.rs` and are the executable -form of these arguments — 36 `#[test]` functions between them. Six have a named -step in ci.yml's `loom` job (`ci.yml:349`): `loom_fanout` (`:383`), -`loom_pod_broadcast` (`:392`), `loom_concurrency` (`:397`), `loom_migration` -(`:402`), `loom_migration_data_plane` (`:414`), `loom_participant` (`:419`). -The other four — `loom_mp_claim`, `loom_sp_mp_flag`, `loom_spmc_epoch_flush`, -`loom_tensor_pool` — are named by no workflow, but they are **not** unrun: -`integration-tests.yml:127` runs -`cargo test --workspace --exclude horus_py --release --test '*'`, and that glob -matches every integration target, loom included. So they do execute on every PR -into `main` or `dev` — in release, at `--test-threads=1`, inside a 45-minute -sweep whose failure reads as an integration break rather than a protocol break, -and with none of the `RUST_LOG=loom=debug` the named steps set. If you change a -protocol one of those four covers, run it yourself and read the output: +```rust +let topic: Topic = Topic::new("sensor_data")?; -```sh -cargo test -p horus_core --test loom_mp_claim -- --nocapture +topic.send(42.0); +if let Some(value) = topic.recv() { + println!("Got: {}", value); +} ``` -One test is genuinely run nowhere by default: -`loom_pod_broadcast.rs:224` carries -`#[ignore = "demonstrates the OLD protocol failing; run by hand to validate the -model"]`. That is deliberate — it is the gate that proves the model is not -vacuous, and it is meant to be run by hand when you change the protocol it -models. - -**`read_latest()` requires `T: Copy` on multi-consumer backends.** -`communication/topic/mod.rs:77-97` sets out the TOCTOU: `read_latest` computes -a slot index, a consumer CAS-advances tail past that slot and -`assume_init_read`s the value out, `Drop` frees the heap allocation, and -`read_latest` then reads freed memory. `Copy` types have no `Drop` and no heap -pointers, so the bytes are always safe to bitwise-copy. Single-consumer -backends keep `T: Clone` (`:93-94`); the `Topic`-level `read_latest` requires -`Copy` unconditionally because migration can switch the backend from SPSC to -SPMC at runtime (`:96-97`). - -**The executor model gives you no ordering between RT nodes.** The comment at -`scheduling/scheduler/mod.rs:3097-3117` is the authority, and `rt_chains` at -`:3118-3119` is `groups.rt_nodes.into_iter().map(|n| vec![n]).collect()` — one -OS thread per RT node, always, so a stalled node cannot block its RT siblings. -The block also records that the previous claim (dependent RT nodes grouped into -chains and ticked sequentially) was never true: `independent_chains` read no -edges and was queried against the wrong graph (`:3104-3111`). Stated at -`:3113-3117`: an RT consumer may read its RT producer's previous-tick value. -Nodes that need ordering must share an execution class and be sequenced by the -main loop, or synchronise through the data they exchange. +The transport backend is auto-selected based on topology: -Note that `rt_executor.rs:2-22` still describes the old model ("Runs all RT -nodes on an isolated OS thread … ticks RT nodes sequentially in priority -order"), and its diagram draws three RT nodes ticking on one thread. The -scheduler is right and the executor header is stale. Fixing it is also claimed -by the P2 document; the two should not both silently take ownership. +| Scenario | Latency | +|----------|---------| +| Same-process fan-out | 3 ns median | +| Same-process SPSC | 10 ns median | +| Cross-process | 198-304 ns median | -Two more, briefly. Per-tick fault isolation is one `catch_unwind` in -`NodeRunner::run_tick` (`scheduling/primitives.rs:229`); `init()` and -`shutdown()` panics are caught separately (`scheduling/types.rs:596-625`), and -the compute worker wraps its dispatch in a second one -(`compute_executor.rs:199-206`). A profile that set `panic = "abort"` would -delete all of them — the workspace does not set it today -(`Cargo.toml:119-120`, which sets only `lto = "thin"`). And the main tick loop -runs on a phase-locked absolute grid rather than sleeping a period after the -work; the reason (a 100 Hz scheduler whose nodes took 3 ms ran at 77 Hz, -load-dependently, and nothing said so) is written at -`scheduling/scheduler/mod.rs:3914-3933`, with the overrun counter at `:241-243` -and the same figure restated on `TickState::next_deadline` at `:236-239`. +## Logging -## Who actually depends on this crate - -The "internal crate, use `horus` instead" framing is right as advice and wrong -as a description of the dependency graph. What is real: - -- Seven crates in this workspace take `horus_core` as a normal path - dependency: `horus`, `horus_manager`, `horus_net`, `horus_py`, `horus_cpp`, - `horus_types` and `benchmarks`. `horus_macros` takes it as a dev-dependency - only (`horus_macros/Cargo.toml:29`), and `horus_cpp` and `horus_manager` take - a second, dev-scoped copy (`horus_cpp/Cargo.toml:35`, - `horus_manager/Cargo.toml:127`). Counted as reference lines - (`grep -rn 'horus_core::' --include=*.rs /src`, discounting - `::horus::horus_core::`), the usage is `horus_manager` 171, `horus_net` 75, - `horus_py` 55, `horus` 54, `horus_cpp` 54, `horus_types` 29, `benchmarks` 19, - `horus_macros` 1. Only one of those, `horus`, is the facade; the other seven - reach past it. -- Every project `horus new` generates lists `horus_core` as a direct path - dependency. `write_horus_path_deps` iterates - `["horus", "horus_core", "horus_library", "horus_macros"]` and emits a path - dep for each that exists — `horus_manager/src/cargo_gen.rs:1066` for the - single-package layout (called at `:116`), `:1808` for the workspace root's - `[workspace.dependencies]`, and `:1979-1980` for each member's - `horus_core.workspace = true`. So `use horus_core::…` compiles in a generated - project, and nothing in the toolchain objects. -- It has to. `horus_macros` expands to `::horus_core::core::LogSummary` - (`horus_macros/src/log_summary.rs:10`) — the only bare `::horus_core::` path - it emits, and one that resolves only if the user's crate names `horus_core` - directly. The rest of the generated code goes through - `::horus::horus_core::…` (e.g. `horus_macros/src/node.rs:409`), which works - because `horus` re-exports the crate at `horus/src/lib.rs:353-354`. -- The `test-utils` feature has a real consumer too: `horus_cpp` takes - `horus_core` as a dev-dependency with `features = ["test-utils"]` - (`horus_cpp/Cargo.toml:35`). - -That is what the `#[doc(hidden)] pub use` block at `lib.rs:56-96` is for, and -the comment at `lib.rs:54-55` says so: those re-exports exist for -crate-internal use and for `horus_py` and macro-generated code cross-crate, not -for readers. **Do not delete a hidden re-export because nothing in this crate -uses it.** Grep the workspace first, not just this crate. - -So: depend on `horus`, import from `horus::prelude::*`, and treat every -`horus_core` path as unstable. But write your commit message knowing that -breaking one is a breaking change for eight sibling crates and every generated -project, not an internal refactor. - -## Doc comments that are attached to the wrong thing - -Found while writing this file. They are wrong in the source, not only in the -rendering, so `--document-hidden-items` will not save you from them. - -| Where | What it says | What it is attached to | -|---|---|---| -| `scheduling/scheduler/mod.rs:281` | "Central orchestrator: holds nodes, drives the tick loop." | `PresenceEntry` at `:284`. `Scheduler` at `:294` has no doc comment. | -| `scheduling/scheduler/mod.rs:3914-3933` | The tick-grid essay, explicitly about `compute_tick_sleep` | `effective_tick_period` at `:3939`, whose own doc starts at `:3934` and follows it with no blank line. `compute_tick_sleep` at `:3948` has none. | -| `communication/topic/dispatch.rs:90` | Invariant 1: "`Topic` is `!Send + !Sync`" | Correct about `Sync`, wrong about `Send` since `mod.rs:3711`. The invariant it should state is single-thread *ownership*, which `Send` does not violate. | - -Two more shapes of the same problem, worth knowing because they are not -individually fixable: - -**Essays written as `//` on no item.** They render nowhere under any flag. Two -that are load-bearing: `scheduling/rt_executor.rs:366-429` (the cyclic-wait -rewrite, 64 lines, a banner comment above a group of constants — the next item -starts at `:431`) and `scheduling/scheduler/mod.rs:3097-3117` (the RT threading -model, attached to a `let` binding). - -**No two places agree on how fast a topic is.** Fourteen files under `src/` -carry a `~N ns` figure in module prose -(`grep -rlE '//!.*~[0-9]+ ?ns' --include=*.rs horus_core/src`). For the topic -path specifically: `communication/mod.rs:9` says "~3ns (same-thread) to ~167ns -(cross-process)"; `communication/mod.rs:22-23`, `communication/pod.rs:54-55` -and `communication/macros.rs:26` say ~50 ns POD against ~167 ns serde, as do -the five `types/*_descriptor.rs` headers (`:4` in each) and -`types/action_chunk.rs:10`; `communication/topic/mod.rs:13-19` gives a -per-backend table running ~40 ns (FanoutShm) to ~85 ns (SpscShm); -`scheduling/event_executor.rs:33` says "~300 ns the transport delivers"; and -`dispatch.rs:21-24` reports ~105-118 ns cross-thread one-way for -`Topic`. This README's own table, deleted in this rewrite, added three -more: 3 ns same-process fan-out, 10 ns same-process SPSC, 198-304 ns -cross-process (`README.md:83-87`, pre-rewrite). The "~3 ns same-thread" claim -sits particularly badly against `topic/mod.rs:7-9`, which -says every topic is SHM-backed and all backends are cross-process, and against -the table four lines below it whose fastest entry is ~40 ns. Only `dispatch.rs` -names the harness that produced its numbers — `topic_probe` and -`topic_probe --raw-ring` (`benchmarks/src/bin/topic_probe.rs`, the flag parsed -at `:618`), sharing threads, pinning, clock and ack protocol so the transport -is the only variable. Quote that one, with its conditions, or quote none. - -## What this file does not fix +```rust +use horus_core::hlog; -Writing it down does not publish it. Three things are still open, and this -README is the wrong tool for all three: +hlog!(info, "Node initialized"); +hlog!(warn, "Low battery: {}%", percentage); +hlog!(error, "Sensor read failed: {}", err); +``` -1. **Nothing deploys `target/doc`.** CI builds it (`ci.yml:345`) and throws it - away. Until something hosts it, the command in - [How to read the internals](#how-to-read-the-internals) is the only route, - and it needs a nightly toolchain. -2. **`cargo doc` excludes `horus_manager`** (`ci.yml:345`). The reason is real: - `horus_manager` declares `[[bin]] name = "horus"` at - `horus_manager/Cargo.toml:26-27`, which would write `target/doc/horus/` on - top of the `horus` library crate's output (`ci.yml:337-338`). Restricting - the workspace doc build to library targets is the obvious candidate. - **Unverified:** nobody has run `cargo doc --workspace --no-deps --lib` here - to confirm it resolves the collision. Treat it as a lead, not a fix, and do - not let a later edit promote it without someone running it. -3. **The hidden modules stay hidden.** That is deliberate — un-hiding them - would present `horus_core` as a supported API, which it is not — but it - means the rendered `horus` docs and this file are the only two places a - reader can start. Keep them in step. +## Order Guidelines -Every line number above is from the working tree as of this writing. The most -fragile are the four-digit ones in `scheduling/scheduler/mod.rs` (`:281`, -`:294`, `:3097-3119`, `:3914-3948`) and `communication/topic/mod.rs` (`:3711`, -`:3713-3719`, `:3721-3728`): one inserted function moves them all. A checker -that asserts every `pub mod` in `lib.rs` appears in the table above, and that -the `Send`/`Sync` sentence matches the impls, would catch drift in the module -list and in the contract — it would not catch drift in the line citations. +- **0-9**: Critical (motor control, safety) +- **10-49**: High priority (sensors, fast loops) +- **50-99**: Normal (processing, planning) +- **100-199**: Low (logging, diagnostics) +- **200+**: Background (telemetry) -## Licence +## License -Apache-2.0. See [LICENSE](../LICENSE). +Apache License 2.0 - see [LICENSE](../LICENSE) for details. diff --git a/horus_core/src/testing/mock_topic.rs b/horus_core/src/testing/mock_topic.rs index 0e9f07210..901d65f4c 100644 --- a/horus_core/src/testing/mock_topic.rs +++ b/horus_core/src/testing/mock_topic.rs @@ -51,8 +51,20 @@ pub struct MockTopic { name: String, buffer: Mutex>, config: MockTopicConfig, + /// Send ATTEMPTS. Drives the fault-injection schedules, and matches what + /// the real transport reports as `messages_sent` — which counts every + /// `send()` before dispatch, so a message dropped on a full ring counts. send_count: AtomicU64, + /// Receive ATTEMPTS. Drives `recv_fail_after` / `recv_fail_every_n`. recv_count: AtomicU64, + /// Messages actually handed back by `recv`/`try_recv`. + /// + /// This is what `messages_received` reports, because that is what the real + /// transport counts: it increments only when `recv()` returns `Some`. The + /// mock used to report its ATTEMPT count there, so a double whose whole + /// purpose is to behave like the transport disagreed with it the moment a + /// recv came back empty — which is most polls on an idle topic. + recv_delivered: AtomicU64, send_failures: AtomicU64, recv_failures: AtomicU64, } @@ -66,6 +78,7 @@ impl MockTopic { config, send_count: AtomicU64::new(0), recv_count: AtomicU64::new(0), + recv_delivered: AtomicU64::new(0), send_failures: AtomicU64::new(0), recv_failures: AtomicU64::new(0), } @@ -161,7 +174,11 @@ impl MockTopic { } let mut buf = self.buffer.lock().unwrap(); - buf.pop_front() + let got = buf.pop_front(); + if got.is_some() { + self.recv_delivered.fetch_add(1, Ordering::Relaxed); + } + got } /// Read the most recent message without consuming it. @@ -186,7 +203,7 @@ impl MockTopic { pub fn metrics(&self) -> TopicMetrics { TopicMetrics::new( self.send_count.load(Ordering::Relaxed), - self.recv_count.load(Ordering::Relaxed), + self.recv_delivered.load(Ordering::Relaxed), self.send_failures.load(Ordering::Relaxed), self.recv_failures.load(Ordering::Relaxed), ) @@ -227,6 +244,37 @@ mod tests { // ── Basic operations ── + /// The double must report the same quantities the transport does. + /// + /// `messages_received` used to be the mock's recv ATTEMPT count, while the + /// real transport increments only when `recv()` hands a message back. So a + /// test written against the mock and then run against a real topic + /// disagreed the moment a poll came back empty — which is most polls on an + /// idle topic. A double that reports what the real thing does not is the + /// one defect a double cannot have. + #[test] + fn metrics_count_deliveries_not_poll_attempts() { + let t = MockTopic::simple("parity"); + t.send(1u64); + t.send(2u64); + + // Two deliveries, then three empty polls. + assert_eq!(t.try_recv(), Some(1)); + assert_eq!(t.try_recv(), Some(2)); + for _ in 0..3 { + assert_eq!(t.try_recv(), None); + } + + let m = t.metrics(); + assert_eq!(m.messages_sent(), 2, "sends are counted as attempts"); + assert_eq!( + m.messages_received(), + 2, + "only the two DELIVERED messages count — the three empty polls are \ + not receipts, and the real transport does not count them" + ); + } + #[test] fn mock_topic_send_recv_roundtrip() { let topic: MockTopic = MockTopic::simple("test"); diff --git a/horus_manager/src/main.rs b/horus_manager/src/main.rs index 8477bb13a..5255f6686 100644 --- a/horus_manager/src/main.rs +++ b/horus_manager/src/main.rs @@ -3512,7 +3512,12 @@ fn run_command(command: Commands) -> HorusResult<()> { watch, extra_args, } => { - if extract || json || md || html || coverage || diff.is_some() { + // `watch` belongs in this guard. `--watch` is declared as "Watch + // for file changes and regenerate", and `run_extract` honours it + // first thing — but a bare `horus doc --watch` fell to the `else` + // arm, which opens rustdoc once and returns, so the flag did + // nothing unless an extract flag happened to be passed with it. + if extract || json || md || html || coverage || watch || diff.is_some() { let config = commands::doc_extract::ExtractConfig { json, md, From e6e9681de4d8d0d15dbcf1f9c3061240e235be9e Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 12:02:34 -0700 Subject: [PATCH 37/79] revert: restore four files the previous commit truncated cf23bb69 was meant to carry three fixes. It also swept in staged changes that were not mine and that I had not read: CONTRIBUTING.md 935 -> 303 lines, horus_core/README.md 528 -> 109, plus .gitignore and docs-contract.yml. All four are intact on main, so that commit was a regression against it. This restores all four to their content at cf23bb69~1, which matches main. The three intended fixes in that commit are untouched. I committed staged paths I had not inspected, in a worktree other sessions also write to. Staging is not authorship: the fix is to pass explicit paths to `git commit`, which is what this one does. --- .github/workflows/docs-contract.yml | 24 + .gitignore | 8 + CONTRIBUTING.md | 950 +++++++++++++++++++++++----- horus_core/README.md | 575 ++++++++++++++--- 4 files changed, 1320 insertions(+), 237 deletions(-) diff --git a/.github/workflows/docs-contract.yml b/.github/workflows/docs-contract.yml index 6d968f6f8..6e8bcb53e 100644 --- a/.github/workflows/docs-contract.yml +++ b/.github/workflows/docs-contract.yml @@ -219,6 +219,15 @@ jobs: HORUS_DOCS_DIR: ${{ github.workspace }}/horus-docs run: cargo test -p horus_manager --test docs_contract -- --include-ignored + # The README Quick Start, compiled against exactly the dependency set a + # `horus new -r` project gets. It carries `#[ignore]` with the reason "run + # in the docs-contract job" — and until now no job ran it: no workflow + # named readme_contract, and the `--include-ignored` above reaches only + # docs_contract. So the test written because the front-page Quick Start + # did not compile was itself never executed. ~25s. + - name: README Quick Start compiles + run: cargo test -p horus_manager --test readme_contract -- --include-ignored + # Every `horus ...` invocation in the docs, resolved against the real clap # tree. Cheap: one `--help` per distinct command, not one per invocation. - name: CLI contract against live docs @@ -249,6 +258,21 @@ jobs: HORUS_DOCS_DIR: ${{ github.workspace }}/horus-docs run: cargo test -p horus_manager --test docs_manifest -- --ignored --nocapture + # docs_parity carries #[ignore = "... wired into the docs-contract + # workflow"] on all three of its tests — and was wired into nothing. No + # workflow named docs_parity, and while integration-tests.yml's + # `--test '*'` does select the target, an all-ignored binary exits 0 + # reporting "0 passed; 3 ignored", which reads as coverage. + # + # What went unchecked: that every HORUS_* variable shipped code reads + # appears in the documentation, and that a documented `horus = "x.y"` + # resolves to the crate we ship rather than to the unrelated crate that + # holds that name on crates.io. + - name: Environment and version parity against live docs + env: + HORUS_DOCS_DIR: ${{ github.workspace }}/horus-docs + run: cargo test -p horus_manager --test docs_parity -- --ignored --nocapture + # A snapshot that has drifted from the pinned docs means tier 1 is # checking a stale contract. Not fatal, but it should be visible. # A step here used to regenerate tests/fixtures/docs-cli-contract.json and diff --git a/.gitignore b/.gitignore index f23c03b05..c5b1ad53b 100644 --- a/.gitignore +++ b/.gitignore @@ -55,7 +55,15 @@ yarn-error.log* .horus/ /dev/shm/horus/ *.horus +# A bare `docs/` matches a directory of that name at ANY depth. It was ignoring +# nothing — no path in this tree matched it, and the docs site is checked out as +# `horus-docs/` (docs-contract.yml:183) or `../horus-docs`, never as `docs/`. +# What it did do was silently swallow in-repo documentation: a file written to +# docs/ never appeared in `git status`. The pattern is kept for a stray local +# checkout, with the documentation directories re-included. docs/ +!/docs/ +!/horus_net/docs/ .ouroboros-state.json sync-all.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 15a11b674..96eb74cb0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,126 +1,624 @@ # Contributing to HORUS -Thank you for your interest in contributing to HORUS! This document provides guidelines for contributing to the project. +This document describes what actually gates a merge here, so you can get a +change in without discovering the rules one red run at a time. + +Formatting is not optional and clippy warnings are not tolerated: +`cargo fmt --all -- --check` runs on every pull request without condition +(`.github/workflows/ci.yml:93`), and clippy runs with `-D warnings` whenever the +base branch is `main` (`:135-136`). Both feed the required `CI Success` check. + +Everything below is transcribed from the workflow files and from the +branch-protection API, with a citation against each claim so you can check it +yourself. Where the two disagree, or where a rule is asserted nowhere but in a +maintainer's head, this document says so rather than inventing a tidy answer. ## Getting Started -1. **Fork the repository** on GitHub -2. **Clone your fork** locally: +1. **Fork the repository** on GitHub. +2. **Clone your fork**: ```bash git clone https://github.com/your-username/horus.git cd horus ``` -3. **Create a branch from `dev`** for your feature: +3. **Branch from the branch you intend to target** (see the branch model below): ```bash - git checkout dev - git pull origin dev - git checkout -b feature/your-feature-name + git checkout main + git pull origin main + git checkout -b fix/your-change ``` -**Important:** All pull requests should target the `dev` branch, not `main`. The `main` branch is used for stable releases only. +### The branch model + +`main` is the default branch (`gh api repos/softmata/horus --jq .default_branch`) +and the only protected one. `dev` exists and has no protection at all: +`gh api repos/softmata/horus/branches/dev/protection` returns 404, and the +GraphQL `branchProtectionRules` list contains exactly one node, whose pattern is +`main`. + +Every workflow that produces a *required* check triggers on `pull_request` +against both `[main, dev]` — `ci.yml:8` (which also lists `master`), +`multi-platform.yml:8`, `integration-tests.yml:9`, `feature-matrix.yml:8`, +`cross-platform-parity.yml:16`, `docker-distro-tests.yml:13`, +`benchmarks.yml:7`. What differs for those seven is strictness, not coverage. + +That is not true of everything. `safety.yml:18-19` restricts its +`pull_request` trigger to `branches: [main]`, so MIRI and the sanitizers never +run against a `dev` PR at all. + +Four separate switches in CI key off `github.base_ref == 'main'`: + +| Switch | Where | Targeting `dev` | Targeting `main` | +| --- | --- | --- | --- | +| `RUSTFLAGS` for every job in `ci.yml` | `ci.yml:28` | `-W warnings` | `-D warnings` | +| clippy | `ci.yml:134-140` | plain `cargo clippy` | `cargo clippy … -- -D warnings` | +| `cargo audit` | `ci.yml:302-306` | advisory, prints a warning | fails the job | +| `RUSTDOCFLAGS` for the doc job | `ci.yml:347` | empty | `-D warnings` | + +The consequence is the thing to plan around. A branch can go entirely green +against `dev` while carrying a compiler warning, a clippy lint, an unreviewed +advisory and a broken rustdoc link. All four turn red on the `dev` → `main` +promotion, at which point they are someone else's problem in someone else's PR. +`ci.yml:20-27` records why the gate was moved onto `base_ref` in the first +place: keying on `github.ref` alone meant the strict path only ever ran *after* +a merge, because `github.ref` is `refs/pull/N/merge` for a `pull_request` event. +A PR could introduce a warning, pass, and break `main` where nothing could be +gated on it. + +So: if your change is meant for `main`, target `main` and take the strict gates +now. Targeting `dev` is only a deferral. + +Note also that pushing to `dev` runs no CI at all — `ci.yml:9` restricts the +`push` trigger to `[main, master]`. Work on `dev` is gated by pull requests or +not at all. + +## What blocks a merge into `main` + +Seven status checks are required. The requirement lives only in the +branch-protection API; no file in the repository records which checks are +required, and the two places that refer to the set at all are both wrong: +`autopilot.yml:5` says "the existing six required checks", and +`docs-contract.yml:503` calls its own aggregator "The single check branch +protection points at" when the API does not list it. This table is the +transcription: + +| Required context | Produced by | Aggregates | +| --- | --- | --- | +| `CI Success` | `ci.yml:480` | `check, fmt, clippy, test, doc, security, loom, test-quality` (`:481`) | +| `Multi-Platform Success` | `multi-platform.yml:403` | `macos, macos-arm64, windows, linux-arm64, linux-armv7, msrv` (`:404`) | +| `Integration Tests Success` | `integration-tests.yml:1372` | `ipc-integration, cli-integration, generated-project, runtime-integration` (`:1374`) | +| `Feature Matrix Success` | `feature-matrix.yml:154` | `python, minimal, all-features` (`:155`) | +| `Parity CI Success` | `cross-platform-parity.yml:341` | `parity-linux, parity-macos, parity-windows, cross-compile, cli-parity` (`:342`) | +| `All Distros Pass` | `docker-distro-tests.yml:64` | one `docker-test` matrix job per distro: ubuntu, debian, fedora, alpine, arch (`:38`, `:65`) | +| `Run Benchmarks` | `benchmarks.yml:70` | not an aggregator; the benchmark job itself | + +Alongside those, protection sets: + +- **Strict mode is on** (`required_status_checks.strict: true`). Your branch must + be up to date with `main` before the merge button unlocks, so every unrelated + merge into `main` invalidates every queued branch and each one has to be + updated and re-run. This is why `autopilot.yml` exists: `autopilot.yml:12-14` + states the problem plainly — GitHub's auto-merge does not update branches, so + queued PRs starve — and its convoy job takes the front of the queue + (`autopilot.yml:238`) and calls `gh pr update-branch` on that one PR + (`:242`), once per merge to `main` (`:23-24`). Expect your branch to be + updated for you, eventually, and expect a full CI cycle each time it is. Keep + PRs small; a large one spends most of its life re-running. +- **Conversation resolution is required** (`required_conversation_resolution: + enabled`). Every review thread on the PR must be marked resolved before it can + merge. One active repository ruleset (`gh api repos/softmata/horus/rulesets`, + id 19245331) applies `copilot_code_review` to the default branch with + `review_on_push: true` and `review_draft_pull_requests: true`, so a bot + re-reviews on every push to a PR targeting `main`, drafts included. Threads + can therefore appear without a human opening them. Whether GitHub's resolution + requirement counts those particular threads is standard platform behaviour + that has not been tested here — treat it as unverified, and resolve them + anyway. +- **Linear history is required** and force pushes to `main` are refused + (`required_linear_history.enabled: true`, `allow_force_pushes.enabled: + false`). +- **No approving review is required by protection.** GraphQL + `branchProtectionRules` reports `requiresApprovingReviews: false` and + `requiredApprovingReviewCount: null`, and the top-level REST protection object + omits `required_pull_request_reviews` entirely. The REST sub-endpoint + `…/protection/required_pull_request_reviews` contradicts both, returning + `{"required_approving_review_count": 1, …}`. The two disagree; this document + follows GraphQL and the top-level object, and flags the contradiction rather + than picking silently. Treat review as a project norm and confirm the + mechanical state before relying on either reading. +- **Admins are not bound** (`enforce_admins.enabled: false`). + +### Checks that run on your PR and do not block it + +These are worth reading when they go red, because nothing stops a merge if they +do: + +- `Docs Contract Success` (`docs-contract.yml:508`), which aggregates the three + jobs that run on a PR: `CLI contract (hermetic)` (`:110`), `CLI contract (live + docs)` (`:173`) and `Shipped examples build` (`:290`) — the needs list is at + `:510`. The workflow's other two jobs, `Rust examples compile` (`:337`) and + `Python API references exist` (`:440`), are gated to `schedule` or an explicit + `run_compile_sweep` dispatch (`:338`, `:441`), so a PR never runs them, and + they are not in the aggregator's needs list either. +- `MIRI - Undefined Behavior Detection` (`safety.yml:42`), `AddressSanitizer - + Memory Error Detection` (`:102`) and `ThreadSanitizer - Data Race Detection` + (`:165`). These trigger only on PRs targeting `main` and only when the diff + touches `horus_core/src/memory/**` or `horus_core/src/communication/**` + (`safety.yml:18-23`). A PR into `dev` that changes unsafe memory code runs + none of them. +- The C++ binding suites (`cpp-bindings.yml`, path-filtered to + `horus_cpp/**`, `horus_core/**`, `horus_types/**`, `horus_library/**`), + coverage (`coverage.yml`, unfiltered), monitor tests (`monitor-tests.yml`, + path-filtered), container images (`container-images.yml`, path-filtered to + `Dockerfile`, `.dockerignore`, `.devcontainer/**`, its own file, `**/Cargo.toml` + and `Cargo.lock`) and `distribution.yml` (unfiltered). ## Development Setup ### Prerequisites -- Rust 1.90+ (`rustup update stable`) — the workspace MSRV in `[workspace.package] rust-version` -- Python 3.9+ with `pip` (optional) -- Build tools (GCC/Clang) +- **Rust.** Most CI jobs use `stable`. The workspace declares + `rust-version = "1.90"` (`Cargo.toml:28`), but the MSRV job pins and tests + `1.92.0` (`multi-platform.yml:26`, `:350`). The floor CI enforces on Rust + source is 1.92.0; the manifest has not caught up. 1.90 is not entirely + untested — the root `Dockerfile:65` is `FROM rust:1.90-slim-bookworm` and + `container-images.yml` builds it — but that workflow is path-filtered and + produces no required context, so a source change that breaks 1.90 can merge. + There is no `rust-toolchain.toml` (`ls rust-toolchain*` finds nothing), so your + default toolchain is what gets used. + + One trap in that job: `MSRV: "1.92.0"` at `multi-platform.yml:26` is never + referenced by any step. The version that runs comes from the matrix at `:350`. + Bumping the env var alone changes nothing. +- **System packages.** The test job installs + `libudev-dev pkg-config cmake libeigen3-dev libfmt-dev libgtest-dev` + (`ci.yml:156-157`); the last four are needed because `cmake_gen`'s + `real_cmake_*` tests shell out to `cmake` and `find_package(Eigen3/fmt/GTest)` + (`ci.yml:154-155`). Jobs that only build Rust install `libudev-dev pkg-config` + alone (`ci.yml:49`, `:105`, `:318`, `:359`). +- **Python 3** with `venv`, if you touch `horus_py` or any cross-language test. +- **`rustfmt` and `clippy` components** (`rustup component add rustfmt clippy`). + +There is a devcontainer if you would rather not install any of it: +`.devcontainer/devcontainer.json` builds the repository `Dockerfile`'s +`devcontainer` stage (`:14-17`) and runs with `--shm-size=1g --ipc=host` +(`:30`). Both flags matter, and the file says why (`:27-29`): the default +64 MB `/dev/shm` is too small for image or point-cloud topics, and `--ipc=host` +is what lets nodes in the container reach nodes on the host. + +### Environment a test run needs + +Two variables. Neither is named in any markdown file in this repository — a +`grep -rl` for them over `--include=*.md` returns nothing. The docs site +describes both for *users* +(`content/docs/development/environment-variables.mdx:79` and `:132` in +`softmata/horus-docs`); what follows is what a *test run* needs. + +**`HORUS_SOURCE`.** `horus_manager`'s `cargo_gen` / `cmake_gen` / `new` / `pkg` +tests generate `.horus` build files against a real HORUS source tree. +`find_horus_source_dir()` (`horus_manager/src/commands/run/run_rust.rs:1046-1116`) +resolves it in three stages: `HORUS_SOURCE` first (`:1050-1056`), then a fixed +candidate list — `/horus`, `~/softmata/horus`, `~/horus`, `/opt/horus`, +`/usr/local/horus` (`:1059-1065`) — and finally the installer's own source cache +under `/horus@` (`:1074-1106`). Two workflow comments +(`ci.yml:29-32`, `integration-tests.yml:30-33`) describe only the middle stage +and omit the cache; `docs-contract.yml:296-297` gets it right. + +Every candidate, including the one you pass in `HORUS_SOURCE`, is accepted only +if it contains `horus/Cargo.toml` (`:1052`, `:1068`, `:1084`, `:1100`). A path +that fails that check is silently ignored and the search continues, so set it +from the repository root: + +```bash +export HORUS_SOURCE="$PWD" # must contain horus/Cargo.toml +``` + +CI does the same in seven workflows: `ci.yml:33`, `integration-tests.yml:34`, +`multi-platform.yml:31`, `safety.yml:35`, `coverage.yml:24`, +`monitor-tests.yml:44` and `docs-contract.yml:301`. `distribution.yml:51` +deliberately does not set it. + +**`HORUS_NAMESPACE`.** Shared memory lives under `/dev/shm/horus_` on +Linux, `/tmp/horus_` on macOS and `%TEMP%\horus_` on +Windows (`horus_sys/src/shm/mod.rs:242-262`). `generate_namespace()` +(`:186-207`) takes `HORUS_NAMESPACE` first, sanitised to `[A-Za-z0-9_]` with +everything else replaced by `_` (`:37-47`); failing that, a test binary derives a +namespace private to the directory cargo built it into (`:70-89`); and failing +*that*, a process lands in the shared `"default"` namespace, which is the ROS 2 +domain-0 model — every process on the machine sees every other. + +So a plain `cargo test` is already isolated from a running robot, but `horus +run` is not. Two concurrent test runs sharing one target directory derive the +same namespace and will collide; give them distinct `HORUS_NAMESPACE` values. +CI sets it per run for that reason (`ci.yml:147`, `integration-tests.yml:42`). + +The full local equivalent of a CI test environment: + +```bash +export HORUS_SOURCE="$PWD" +export HORUS_NAMESPACE="local_$$" +export RUST_BACKTRACE=1 # ci.yml sets this on every test step + +# The five subdirectories CI creates, owner-only. Do not chmod 777: +# regression_shm_topics_dir_permissions_restricted (horus_core/tests/regressions.rs:307) +# asserts the topics dir is exactly 0o700, and a permissive mkdir breaks it. +SHM_DIR="/dev/shm/horus_${HORUS_NAMESPACE}" +mkdir -p "$SHM_DIR"/{topics,nodes,control,network,scheduler} +chmod -R 700 "$SHM_DIR" +``` + +That mirrors `ci.yml:179-186`. The same block appears twelve times across six +workflow files — `ci.yml:185`, `integration-tests.yml:76`, `:348`, `:740`, +`:1014`, `:1170`, `safety.yml:70`, `:138`, `:190`, `multi-platform.yml:376`, +`coverage.yml:84` and `benchmarks.yml:159`. `monitor-tests.yml` uses a variant +with an extra `topics/horus_topic` level (`:91`, `:145`, `:187`). Do not create +these with `sudo` — root-owned directories are what made +`SchedulerRegistry::open` (`horus_core/src/scheduling/registry.rs:158`) and the +presence writer fail with `PermissionDenied` (`ci.yml:180-184`). + +The IPC integration job additionally raises the kernel limits +(`integration-tests.yml:79-80`): + +```bash +sudo sysctl -w kernel.shmmax=2147483648 +sudo sysctl -w kernel.shmall=2147483648 +``` + +Leaked namespaces are a real and recurring cause of unrelated test failures. +`autopilot.yml:82-83` tells its own triager to clear them first, because leaked +namespaces make unrelated tests fail in a rotating pattern. If a suite starts +failing in a pattern that does not match your change: + +```bash +rm -rf /dev/shm/horus_* +``` + +For real-time behaviour on Linux, `scripts/setup-realtime.sh` sets the CPU +governor to `performance`, configures `SCHED_FIFO`/`SCHED_RR` limits up to +priority 99, raises the memlock limit for `mlockall`, installs a udev rule for +shared-memory permissions, and optionally installs `cpupower`/`cpufrequtils` +(`scripts/setup-realtime.sh:5-10`). `sudo ./scripts/setup-realtime.sh` +configures the machine; `sudo ./scripts/setup-realtime.sh --check` reports the +current state without changing anything (`:12-14`). You must log out and back in +for the limit changes to take effect (`:23`). The script refuses to run on +anything but Linux (`:28-32`) and requires root (`:35-39`). ### Building ```bash -# Build all Rust components -cargo build --release +cargo build --workspace --exclude horus_py -# Build Python bindings (optional) -# First install maturin: cargo install maturin -cd horus_py -maturin develop --release +# Python bindings, into a venv: +python3 -m venv .venv +.venv/bin/pip install --upgrade pip maturin +.venv/bin/maturin develop --release --manifest-path horus_py/Cargo.toml ``` +`horus_py` is excluded from every workspace-wide Rust build and test in this +repository, and the reason is worth knowing before you hit it. +`horus_py/src/lib.rs:11-18` raises a `compile_error!` whenever `cfg(test)` and +the default `extension-module` feature are both on: pyo3 leaves Python's symbols +to the interpreter that `dlopen`s the `.so`, so a test binary has nothing to link +them against. Without the guard you get roughly forty lines of `rust-lld: error: +undefined symbol: PyEval_RestoreThread` (`horus_py/src/lib.rs:7-9`). Cargo +cannot vary features per target, so the crate gets a second pass with the +feature off instead (`ci.yml:66-78`, `:224-227`). + ## Testing -### Unit and Integration Tests +### Run the timing-sensitive tests in `--release` -**Run the timing-sensitive integration tests in `--release`.** CI does, and it -matters: a debug build is roughly an order of magnitude slower, so a 1 kHz RT -scheduler test, a latency percentile check or a stress test can fail in `debug` -purely because the code cannot keep up — nothing is wrong with it. Seven tests -across `stress_rt_contention`, `scheduler_config_test`, `production_fanout_battle` -and `safety_battle_tests` fail in debug and pass in release for exactly that -reason. +CI does — its whole `tests/` sweep is `--release` (`integration-tests.yml:127`) +— and it matters: a debug build is roughly an order of magnitude slower, so a +1 kHz RT scheduler test, a latency percentile check or a stress test can fail in +`debug` purely because the code cannot keep up. Nothing is wrong with the code. +`autopilot.yml:79-80` gives the same instruction to its own triager: timing, +throughput and tick-rate assertions are only meaningful with `--release`. -What CI runs, so you can reproduce a failure in the configuration it happened in: +The suites where this bites are `horus_core/tests/stress_rt_contention.rs`, +`scheduler_config_test.rs`, `production_fanout_battle.rs` and +`safety_battle_tests.rs`. All four exist. The current `CONTRIBUTING.md:49-52` +puts a number on it — "Seven tests" across those four files — which is not +verifiable without running both configurations, so it is not repeated here. If a +test in one of those fails, re-run it in `--release` before you start reading the +diff. -```bash -# The lib job: unit tests only, serialized. -cargo test --workspace --exclude horus_py --lib --no-fail-fast -- --test-threads=1 +### What CI runs -# The integration job: every tests/ target, in RELEASE. -cargo test --workspace --exclude horus_py --release --test '*' +Reproduce a failure in the configuration it happened in. + +```bash +# The `test` job in ci.yml (:203-208) — unit tests only, serialized, +# with four cross-thread/robotics tests skipped. +cargo test --workspace --exclude horus_py --lib --no-fail-fast -- \ + --test-threads=1 \ + --skip topic_cross_thread_1p_multi_c_spmc \ + --skip topic_cross_thread_mpmc_pre_initialized_99_percent \ + --skip topic_cross_thread_multi_p_multi_c_mpmc \ + --skip test_robotics_autonomous_car_perception + +# The `ipc-integration` job (integration-tests.yml:127-133) — every tests/ +# target, in RELEASE, serialized, with five timing-sensitive tests skipped. +# That job also sets RUST_LOG=debug and PYTHONPATH=/horus_py +# (integration-tests.yml:137-138). +cargo test --workspace --exclude horus_py --release --test '*' -- \ + --test-threads=1 --nocapture \ + --skip shm_fanout_latency_percentiles_cross_thread \ + --skip stress_rt_isolation_from_compute_nodes \ + --skip stress_rt_1khz_under_cpu_contention \ + --skip stress_rt_1khz_baseline_no_contention \ + --skip stress_rt_multi_rate_under_contention + +# The `cli-integration` job (integration-tests.yml:794) — the whole +# horus_manager package, integration targets included. +cargo test -p horus_manager --release -- --test-threads=1 + +# horus_py, Rust side (ci.yml:227) and Python side (ci.yml:256-257). +cargo test -p horus_py --no-default-features -- --test-threads=1 +cd horus_py && python -m pytest tests/ -p no:cacheprovider ``` Add `--no-fail-fast` when you want the whole failure list: without it cargo stops at the first failing test binary, so a run reports one failure at a time and you -fix them one slow round-trip apiece. +fix them one slow round-trip apiece (`autopilot.yml:84-85`). + +### The `#[ignore]`d suites, which `--test '*'` does not reach + +`--test '*'` builds every integration target but runs none of its `#[ignore]`d +tests, and for a long time nothing passed `--ignored` for `horus_core` +(`integration-tests.yml:140-146`). Several suites use `#[ignore]` as a mechanism +rather than as neglect: they re-exec their own test binary as child processes so +publisher and subscriber are genuinely separate processes (`:153-158`). Three +extra steps now run them, and they are not equal: + +- **Gating** (`integration-tests.yml:219-231`): `ipc_torture`, `chaos_monkey`, + `message_type_matrix`, `hardware_emulation`, `params_intent`, + `log_system_tests`, `rust_python_full_matrix`, `rust_python_matrix`, + `humanoid_stress`, `matrix_sweep`. A failure here fails a required context. +- **Advisory, `continue-on-error`** (`:244-264`): `chaos_cross_process`, + `production_validation`, `chaos_xp_schedulers`, `production_fanout_battle`. + These spawn five to ten real child processes on a two-vCPU runner, and the file + is explicit that a failure there does not distinguish a HORUS regression from a + busy neighbour (`:236-243`). +- **Advisory for a different reason** (`:289-297`): `multi_scheduler_matrix`, + kept separate because it exposes a real fault rather than an environment + limit — `deterministic_alongside_normal_scheduler` dies with SIGBUS, reproduced + two runs in five on a twelve-core box (`:269-288`). It goes back to gating when + the topic-layer fix lands. + +Three further suites (`cli_runtime_integration`, `launch_and_workflow`, +`real_project_ipc`) resolve `target/debug/examples/...` by hardcoded path and are +left out entirely until that prerequisite is wired (`:160-166`, `:181-187`). + +To run an ignored suite yourself: + +```bash +cargo test -p horus_core --release --test ipc_torture -- --ignored --test-threads=1 --nocapture +``` + +### The five tests that run nowhere + +**The five tests skipped in the release sweep run nowhere in CI.** +`integration-tests.yml:114-123` says so directly, and the negative holds: `grep +-rn stress_rt .github/workflows/` matches only the two skip lists +(`integration-tests.yml:130-133` and `coverage.yml:118`), and +`shm_fanout_latency_percentiles_cross_thread` appears only at +`integration-tests.yml:129` and `coverage.yml:118`. It is not `#[ignore]`d +(`horus_core/tests/production_fanout_battle.rs:157-158`), so the `--ignored` +advisory step does not pick it up either. +They are developer-run gates: on a shared two-vCPU runner they measure the +runner rather than HORUS — that job saw 34.24% spread +(`integration-tests.yml:108-112`). If you change the tick path, run them +yourself on a machine you control. The workflow names the invocation +(`:123`): ```bash -# Rust unit tests -cargo test +cargo test -p horus_core --release --test stress_rt_contention -- --nocapture +``` + +### `horus_py` coverage + +`horus_py` is excluded from every workspace-wide sweep, but it is not +untested. Two things run: -# Specific component -cargo test -p horus_core +- `cargo test -p horus_py --no-default-features -- --test-threads=1` + (`ci.yml:227`), which covers the 67 `#[test]` functions in `horus_py/src`. +- The pytest suite (`ci.yml:240-257`), which is the only `pytest` invocation in + any workflow — a grep for `pytest` over `.github/workflows/` matches only + `ci.yml:235`, `:243` and `:257`. -# Python tests -cd horus_py -pytest tests/ +The comment at `ci.yml:231-234` says the Python suite is "the ONLY coverage the +horus_py binding layer has". Read that as being about the built extension, not +about the crate: `:227` is Rust coverage of the same crate, and it runs in the +same job. -# Benchmarks -cd benchmarks -cargo bench +The Python suite has a trap. `horus_py/horus/__init__.py:227-236` catches +`ImportError` and falls back to a "mock mode" with `Topic = None` and a +`RuntimeWarning`, so `import horus` succeeds even when the extension was never +built — and the whole suite then runs green against a null API. CI asserts +against this before running pytest (`ci.yml:244-255`). Do the same locally: + +```python +import horus, sys +if horus.Topic is None: + sys.exit("horus imported in MOCK MODE — the extension did not build") ``` -### Acceptance Tests +### Loom models + +`horus_core/tests/` holds ten `loom_*` targets. The dedicated `loom` job runs six +of them by name (`ci.yml:382-421`): `loom_fanout`, `loom_pod_broadcast`, +`loom_concurrency`, `loom_migration`, `loom_migration_data_plane`, +`loom_participant`. The other four — `loom_mp_claim`, `loom_sp_mp_flag`, +`loom_spmc_epoch_flush`, `loom_tensor_pool` — are not named there. They carry no +`cfg(loom)` gate (a grep for `cfg(loom)` over `horus_core/tests/` returns +nothing) and `loom` is a plain dev-dependency (`horus_core/Cargo.toml:69`), so +they are ordinary integration targets matched by `integration-tests.yml:127`'s +`--test '*'`. -Acceptance tests live in two places: +`loom_pod_broadcast` carries a `naive` variant that reproduces the old protocol +and is *supposed* to fail (`ci.yml:387-390`) — that failure is what proves the +model is not vacuous. Do not "fix" it. -- **Rust acceptance tests** — `horus_core/tests/acceptance_topic.rs` and - `horus_core/tests/acceptance_scheduler.rs`, run by `cargo test`. -- **Cross-platform CLI acceptance tests** — `tests/docker/acceptance_linux_macos.sh` - and `tests/docker/acceptance_windows.ps1`, which verify the complete user workflow. +### The C++ layout contract -End-to-end QA scenarios (launch files and sample pub/sub projects) live in `tests/qa/`. +`horus_cpp/include/horus/layout_contract.hpp` is rendered from the Rust types +with `offset_of!` and committed (`horus_cpp/src/layout_contract.rs:30-33`). +`contract_file_is_current` (`:213-214`) compares the rendering against the +committed header and fails if they have drifted, so a message type that gains, +loses, renames or reorders a field turns red. The test module is +`#[cfg(test)]` inside the lib (`:189`, declared at `horus_cpp/src/lib.rs:33`), so +it runs in the workspace `--lib` sweep and sits inside `CI Success`. -**Before submitting a PR:** -1. Review the relevant acceptance tests -2. Ensure changes align with documented behavior -3. Update tests if changing functionality -4. Add new scenarios for new features +When the failure is intended, regenerate deliberately and review the diff: + +```bash +cargo test -p horus_cpp --no-default-features regenerate_layout_contract -- --ignored --nocapture +``` + +That is the canonical form, quoted verbatim from `layout_contract.rs:38`. It +also appears at `:169` and inside the failure message at `:221`, and nowhere else +in the tree — no workflow and no markdown file mentions it. Note that +`horus_cpp/Cargo.toml` declares no `[features]` section, so +`--no-default-features` does nothing for that crate; it is harmless, and it is +what the file tells you to type. + +### Feature combinations + +`--workspace --no-default-features` does *not* produce a minimal build: Cargo +unifies features across the workspace, so siblings that depend on `horus_core` +or `horus` with default features re-enable them anyway +(`feature-matrix.yml:105-110`). The only way to catch a no-default-features +regression is per crate, which is what CI does (`:111-115`): + +```bash +cargo check -p horus_core --no-default-features +cargo check -p horus --no-default-features +cargo check -p horus_types --no-default-features +cargo check -p horus_net --no-default-features +cargo check -p horus_sys --no-default-features +cargo check --workspace --all-features # feature-matrix.yml:150 +``` + +If you add a workspace member, add it to that list, or it has no +no-default-features floor. + +### Acceptance tests + +- **Rust** — `horus_core/tests/acceptance_topic.rs` and + `horus_core/tests/acceptance_scheduler.rs`. +- **Cross-platform CLI** — `tests/docker/acceptance_linux_macos.sh` and + `tests/docker/acceptance_windows.ps1`. +- **End-to-end QA scenarios** — `tests/qa/launch`, `tests/qa/pubsub_rust`, + `tests/qa/pubsub_python`. ```bash -# Run the Rust acceptance tests cargo test -p horus_core --test acceptance_topic cargo test -p horus_core --test acceptance_scheduler - -# Run the cross-platform CLI acceptance suite bash tests/docker/acceptance_linux_macos.sh -# Manually verify scenarios -horus new test_project -cd test_project -horus run +horus new test_project && cd test_project && horus run +``` + +### Benchmarks + +`Run Benchmarks` is a required check. It runs two binaries — +`cross_process_benchmark` and `robotics_messages_benchmark` — `BENCH_REPETITIONS` +times each, which is 3 (`benchmarks.yml:53`, `:179-188`), and feeds the JSON +reports to a `regression_gate` binary +(`benchmarks/src/bin/regression_gate.rs`, built at `benchmarks.yml:162-166`, +invoked at `:242-249`). The gate reduces the repetitions to their median so one +bad draw cannot fail the build (`:48-52`) and compares against a rolling window +of `BENCH_WINDOW` past trunk runs, which is 20 (`:55-58`). + +There is no single threshold: each metric is gated at `max(relative, center + +k·σ_observed, absolute floor)`, and the policy, the blocking/report-only split +and the reasoning for every number live in `RegressionPolicy` in +`benchmarks/src/output.rs:369` (`benchmarks.yml:38-45`). The figure of merit is +worst-case latency and jitter first, median second — a change that improves the +median and widens the tail is a regression here (`:25-27`). + +Two things only happen on a push to `main` (`:237-240`): `--record`, which +rewrites the rolling baseline, and `--trunk`, which enables the +consecutive-runs escalation. Both are deliberately off for pull requests, so +your PR is compared against trunk but never becomes its own baseline. + +```bash +cargo run --release --bin cross_process_benchmark +cargo run --release --bin robotics_messages_benchmark +``` + +The five `[[bench]]` targets in `benchmarks/Cargo.toml:84-102` — `topic_latency`, +`topic_throughput`, `scheduler_jitter`, `scheduler_ipc_latency`, `tensor_pool` — +are a separate thing and the gate does not read them. + +### Documentation contract + +`horus-docs` is a separate repository, and this repository's tests can break its +published pages. On a PR, `docs-contract.yml` runs a hermetic tier needing no +docs checkout (`:109-167`), a live tier that clones `softmata/horus-docs` at a +pinned commit (`:172-183`), and a shipped-examples build (`:289-334`). + +The pin is `DOCS_PINNED_REF` (`docs-contract.yml:103`). If your change requires a +docs change to stay true, land the docs change first, then bump the pin in a PR +here. Verify the new commit before you bump it — a pin is a claim about a +specific tree (`:58-65`): + +```bash +HORUS_DOCS_DIR= \ + cargo test -p horus_manager --test docs_contract -- --include-ignored +HORUS_DOCS_DIR= \ + cargo test -p horus_manager --test docs_cli_contract ``` +`HORUS_DOCS_DIR` must point at a directory containing `content/docs`, or it is +ignored with a warning (`horus_manager/tests/docs_contract.rs:61-68`). Without a +checkout these tests *skip* locally and *fail* under `CI` +(`horus_manager/tests/docs_contract.rs:19-21`, `:100`), so a green local run +proves nothing if you forgot the variable. + +There is deliberately no workflow input to override the pin. `schedule` and +`workflow_dispatch` run in default-branch context with write access to the +Actions cache, so building an arbitrary ref of another repository here would let +anyone who can push to `softmata/horus-docs` execute code in this repository's +privileged context (`docs-contract.yml:86-93`). + ## Code Style ### Rust -Follow standard conventions: -- Format code when possible: `cargo fmt` (optional, as it may fail in some cases) -- Run `clippy` and address critical issues: `cargo clippy` (warnings are acceptable) -- Document public APIs with `///` comments +- **Formatting is mandatory.** `ci.yml:93` runs + `cargo fmt --all -- --check` with no condition and no `continue-on-error`; the + `fmt` job is in `ci-success`'s `needs` list (`ci.yml:481`) and its result is + checked (`ci.yml:490`). A formatting diff fails a required check on every PR, + whichever branch you target. +- **Clippy warnings fail when the PR targets `main`.** `ci.yml:135-136` runs + `cargo clippy --workspace --all-targets --exclude horus_py -- -D warnings` plus + a second `-p horus_py --all-targets --no-default-features` pass. Targeting + `dev` drops the `-D warnings` (`ci.yml:138-139`), which defers the problem + rather than removing it. +- **Compiler warnings fail when the PR targets `main`.** `RUSTFLAGS` is set at + the workflow level (`ci.yml:28`), so `-D warnings` applies to `check`, `test`, + `doc` and `loom` as well as clippy. +- **Rustdoc warnings fail when the PR targets `main`** (`ci.yml:347`). Note that + `cargo doc` excludes `horus_manager` (`ci.yml:345`) to avoid a + `target/doc/horus/` filename collision with the `horus` binary + (`ci.yml:337-338`), so doc warnings in that crate are not caught. `horus_py` + needs no split here — rustdoc never sets `cfg(test)`, so the + `extension-module` guard does not fire (`ci.yml:340-344`). + +Run the strict configuration locally before you push: + +```bash +cargo fmt --all -- --check +RUSTFLAGS="-D warnings" cargo clippy --workspace --all-targets --exclude horus_py -- -D warnings +RUSTFLAGS="-D warnings" cargo clippy -p horus_py --all-targets --no-default-features -- -D warnings +RUSTDOCFLAGS="-D warnings" cargo doc --workspace --no-deps --exclude horus_manager +cargo audit # cargo install cargo-audit --locked, if you have not +``` + +Document public APIs with `///`: ```rust /// Creates a new Topic for inter-process communication. @@ -139,165 +637,299 @@ pub fn new(topic_name: &str) -> Result { } ``` +### Unsafe code + +`horus_core/src` holds 340 lines containing `unsafe {` and 253 lines carrying a +`// SAFETY:` comment. **Nothing enforces the correspondence.** There is no +`undocumented_unsafe_blocks`, `missing_safety_doc`, `unsafe_op_in_unsafe_fn` or +`deny(unsafe…)` in the workspace lints (`Cargo.toml:83-102`) or anywhere else — +a grep for those names across every `.rs` and `.toml` in the repository returns +nothing. The only mechanisms are the PR template's safety checklist +(`.github/PULL_REQUEST_TEMPLATE.md:35-40`) and review. + +So write the comment because the next person needs it, and be aware of what the +machine will and will not do for you: + +- MIRI, ASan and TSan run only on PRs targeting `main` that touch + `horus_core/src/memory/**` or `horus_core/src/communication/**` + (`safety.yml:18-23`), and none of the three is a required check. +- MIRI cannot execute most of `horus_core` at all: it does not support + file-backed memory mappings, which since the transport went SHM-only rules out + every `Topic`, the `TensorPool` and every pool-backed type + (`safety.yml:75-82`). The file records that each of the three steps that used + to live there died on the first such test, so the job has never reported a UB + result. What it runs now is the SIMD copy helpers, the RT allocator guard and + the SHM path helpers (`:84-86`): + + ```bash + MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-permissive-provenance" \ + cargo miri test -p horus_core --lib -- memory::simd memory::rt_allocator memory::platform + ``` + + (`safety.yml:92-95`; the three modules are `horus_core/src/memory/simd.rs`, + `rt_allocator.rs` and `platform.rs`.) The mmap-backed paths are covered + instead by ASan, TSan and the `Valgrind Memcheck` job in + `cpp-bindings.yml:540-541` (`safety.yml:88-89`). + +### Test quality + +`ci.yml:434-476` greps the tree for stub-test anti-patterns and fails the build +on two of them: + +| Pattern | Why it fails | +| --- | --- | +| `is_ok() \|\| is_err()` (either order), `ci.yml:440` | always true; asserts nothing | +| a bare `matches!(…);` statement not preceded by an `assert`, `ci.yml:448-463` | the `bool` is discarded | + +A third, `let _ = format!(…)` in test code, emits a warning rather than failing +(`ci.yml:467-469`). + +One local caveat: the guard runs `grep -rn … --include='*.rs' .` from the +repository root with no `--exclude-dir` (`ci.yml:440`, `:448`, `:467`). A CI +checkout has no `target/`; yours does, so reproducing the scan locally will also +walk generated sources under it. Point the grep at the crate directories rather +than at `.`. + ### Python -Follow PEP 8: -- Use `black` for formatting -- Use `mypy` for type checking -- Use descriptive variable names +Follow PEP 8 and use descriptive names — but be aware that **no check enforces +any of it**. A case-insensitive grep over `.github/workflows/` finds no `black`, +`mypy`, `ruff` or `flake8` invocation anywhere. The only automated Python gate is +the pytest suite in `ci.yml:240-257`. ## What to Contribute -### Good First Issues +### Good first issues -Look for issues labeled `good-first-issue`: -- Documentation improvements -- Example programs -- Bug fixes -- Test coverage improvements +Issues labelled `good first issue` — the label name has spaces, and the +repository's label set also carries `help wanted`, `bug`, `enhancement`, +`documentation`, `testing`, `rust` and `CLI tools`. -### Feature Requests +### Feature requests -Before implementing a major feature: -1. Open an issue to discuss the proposal -2. Wait for maintainer feedback -3. Implement with tests and documentation +Before implementing a major feature, open an issue, wait for maintainer +feedback, then implement with tests and documentation. The seven required checks +take a while; discovering after the fact that a design is unwanted is expensive +for everyone. -### Bug Reports +### Bug reports -When reporting bugs, include: -- HORUS version (`horus --version`) -- Operating system and version -- Minimal reproducible example -- Expected vs actual behavior -- Relevant logs or error messages +Include the HORUS version (`horus --version`, wired at +`horus_manager/src/main.rs:14`), the operating system and version, a minimal +reproducible example, expected versus actual behaviour, and the relevant logs. +`horus doctor` (`horus_manager/src/main.rs:852`, dispatched at `:3557`) checks +the machine — toolchains, real-time capability, shared memory (`:848`) — and +takes `--verbose`, `--json`, `--fix` and `--rt`. `horus check` checks the +repository; `:850` notes that neither substitutes for the other. ## Pull Request Process -1. **Ensure tests pass**: +1. **Choose the base branch deliberately.** `main` takes the strict gates now; + `dev` defers them. + +2. **Run the gates locally.** At minimum: ```bash - cargo test + cargo fmt --all -- --check + RUSTFLAGS="-D warnings" cargo clippy --workspace --all-targets --exclude horus_py -- -D warnings + cargo test --workspace --exclude horus_py --lib --no-fail-fast -- --test-threads=1 ``` - Optionally run `cargo fmt` (may fail in some cases) and `cargo clippy` (warnings are acceptable). +3. **Update the acceptance tests** in `horus_core/tests/acceptance_*.rs` and + `tests/docker/` if behaviour changed, and add scenarios for new features. + +4. **Update documentation.** README.md for user-facing changes, inline `///` + docs, examples, and `CHANGELOG.md`. The changelog is machine-checked: + `horus_manager/tests/changelog_contract.rs` requires that a changelog exists + (`:51`), that every released git tag appears in it (`:63`), that the version + in `env!("CARGO_PKG_VERSION")` — `horus_manager`'s own package version, 0.4.0 + at `horus_manager/Cargo.toml:3`, since there is no `[workspace.package] + version` to inherit — has either its own `[x.y.z]` section or an + `## Unreleased` section to collect into (`:88-96`), and that releases are + listed newest first (`:101`). + + Those nine tests, plus `translation_contract.rs`'s five and ten of + `readme_contract.rs`'s eleven, run inside the required + `Integration Tests Success` context. No workflow names any of the three + files; they run because `integration-tests.yml:794` is + `cargo test -p horus_manager --release -- --test-threads=1` with no target + filter, `horus_manager/Cargo.toml` declares no `[[test]]` blocks and no + `required-features`, so every file in `horus_manager/tests/` is autodiscovered, + and line 794 sits inside the `cli-integration` job (`:309`) which is in the + aggregator's needs list (`:1374`). + + The exception is `readme_contract.rs`'s `the_readme_quick_start_compiles`, + which is `#[ignore]`d with the reason "run in the docs-contract job" + (`readme_contract.rs:146`). Nothing runs it: no workflow mentions + `readme_contract`, `docs-contract.yml`'s cargo invocations name only the + `docs_*` targets, and `integration-tests.yml:794` passes no + `--include-ignored`. If you change the README's Quick Start, compile it + yourself: -2. **Check acceptance tests**: - - Review the acceptance tests in `horus_core/tests/acceptance_*.rs` and `tests/docker/` - - Manually verify key scenarios - - Update test scenarios if behavior changed - - Add new scenarios for new features + ```bash + cargo test -p horus_manager --test readme_contract -- --include-ignored + ``` -3. **Update documentation**: - - Update README.md for user-facing changes - - Update inline code documentation - - Add examples for new features - - Update CHANGELOG.md +5. **Write clear commit messages.** `git-cliff` groups the changelog by + conventional-commit prefixes — `feat`, `fix`, `doc`, `perf`, `refactor`, + `style`, `test`, `chore`, `ci`, `deps` (`.github/cliff.toml:47-61`) — so a + prefixed subject lands in the right section. `chore(release)` is skipped + entirely (`:55`), `chore(deps)` and `deps` both land under Dependencies + (`:56`, `:59`), and a commit whose *body* matches `security` is grouped under + Security regardless of prefix (`:60`). -4. **Write clear commit messages**: ``` - Add feature: Brief description + fix: brief description Detailed explanation of what changed and why. - Updated acceptance tests in horus_core/tests/acceptance_*.rs - - Added new scenarios for ... Fixes #123 ``` -5. **Submit PR** with: - - **Base branch: `dev`** (not `main`) - - Clear title and description - - Link to related issues - - Screenshots/examples if UI changes - - List of verified acceptance test scenarios - -6. **Address review feedback** promptly +6. **Fill in the PR template** (`.github/PULL_REQUEST_TEMPLATE.md`). It appears + automatically. Its safety (`:35-40`) and performance (`:42-46`) sections are + the ones reviewers read most closely. -### PR Description Template +7. **Sign the CLA on your first PR.** `.github/CLA.md:71` states that making a + contribution already binds you to the agreement; the requested signal is a + comment on the pull request: -```markdown -## Description -Brief description of changes + ``` + I have read and agree to the Contributor License Agreement. + ``` -**Note:** This PR targets the `dev` branch. + **No bot checks for it** — a case-insensitive grep for `cla` and + `contributor license` over `.github/workflows/` matches nothing — so nothing + will remind you and nothing will block the merge. A maintainer will ask if it + is missing. For significant contributions you may be asked to sign a physical + copy (`.github/CLA.md:77`). -## Changes -- Added feature X -- Fixed bug Y -- Updated tests +8. **Resolve every review thread.** Protection requires it. The Copilot reviewer + re-runs on every push to a PR targeting `main`, so expect threads you did not + open. -## Testing -- [ ] Unit tests pass (`cargo test`) -- [ ] Code formatted if possible (`cargo fmt`, optional) -- [ ] Clippy checked (`cargo clippy`, warnings acceptable) -- [ ] Verified acceptance test scenarios -- [ ] Updated/added acceptance tests - -## Related Issues -Fixes #123 -``` +9. **Expect to be rebased.** Strict mode plus a busy `main` means your branch + will be updated repeatedly and re-run each time. ## Architecture Guidelines -### Core Principles +### Core principles -1. **Zero-copy when possible**: Use shared memory -2. **Type safety**: Leverage Rust's type system -3. **Minimal latency**: Profile and optimize hot paths -4. **Multi-language**: Ensure features work across Rust/Python +1. **Zero-copy when possible** — use shared memory. +2. **Type safety** — leverage Rust's type system. +3. **Minimal latency** — profile and optimize hot paths; judge on the tail, not + the median (`benchmarks.yml:25-27`). +4. **Multi-language** — features should work across Rust, C++ and Python. -### Code Organization +### Code organization ``` horus/ -├── horus/ # Unified entry crate (use horus::prelude::*) -├── horus_core/ # Core framework (Node, Topic, Scheduler, types, memory, scheduling) +├── horus/ # Unified entry crate (horus::prelude, horus/src/lib.rs:581) +├── horus_core/ # Core framework (core, communication, memory, scheduling, types, services, actions, drivers) ├── horus_types/ # Universal IPC types (math, diagnostics, time, generic) ├── horus_macros/ # Procedural macros (node!, LogSummary derive) ├── horus_sys/ # Platform abstraction layer ├── horus_net/ # Transparent LAN replication -├── horus_cpp/ # C++ FFI bridge (CXX, staticlib+cdylib) +├── horus_cpp/ # C++ FFI bridge (CXX; crate-type staticlib, cdylib, rlib) ├── horus_cpp_macros/ # C++ binding codegen (#[horus_api] proc macro) ├── horus_manager/ # CLI tool (horus command) ├── horus_py/ # Python bindings -├── benchmarks/ # Performance benchmarks +├── benchmarks/ # Performance benchmarks (package name: horus_benchmarks) ├── examples/ # Example projects (Rust, C++, Python) └── tests/ # Integration and QA tests ``` -Standard message types come from `horus_types/` and the external `horus-robotics` git dependency; `TransformFrame` comes from the external `horus-tf` git dependency, re-exported through `horus::prelude`. +The eleven crates are the workspace members listed in `Cargo.toml:3-15`, with +the per-crate descriptions taken from the comments there and from each crate's +own manifest. `examples/` and `tests/` are not workspace members. `node!` is the +proc macro at `horus_macros/src/lib.rs:76-79`; `LogSummary` is the derive at +`:102`. + +Standard message types come from `horus_types/` and the external `horus-robotics` +git dependency, pinned by rev in `[workspace.dependencies]` (`Cargo.toml:40`). +`TransformFrame` comes from the external `horus-tf` git dependency, which is +declared per crate rather than in the workspace table — `horus/Cargo.toml:25`, +`horus_core/Cargo.toml:73`, `horus_cpp/Cargo.toml:21`, +`horus_manager/Cargo.toml:38`, `horus_py/Cargo.toml:31` — and re-exported through +`horus::prelude` at `horus/src/lib.rs:613`. Both git deps are patched to this +workspace's local `horus_core`, `horus_types` and `horus_macros` +(`Cargo.toml:108-115`), because they reference `horus_core` by relative path +(`Cargo.toml:105-107`). ## What Not to Do -- Break existing APIs without migration path -- Add dependencies without discussion -- Commit without running tests -- Submit PRs without description or testing -- Ignore test failures +- Break existing APIs without a migration path. +- Add dependencies without discussion. `cargo audit` is a gate on `main` + (`ci.yml:302-303`), and `.cargo/audit.toml` is where a tolerated advisory has + to justify itself. That list is currently **empty** (`ignore = []`), which the + file says is the target state: an ignore entry suppresses its advisory for + every matching version forever, including one that lands in the lockfile years + later. Removed entries are kept above it as commented history with the reason + each was retired. If you need to add one, name why it is tolerated and what + would let us remove it. +- Widen the `/dev/shm` permissions to make a test pass. + `regression_shm_topics_dir_permissions_restricted` + (`horus_core/tests/regressions.rs:307-323`) asserts the topics directory is + exactly `0o700`, and that assertion is the guarantee, not the obstacle. +- "Fix" `loom_pod_broadcast`'s `naive` variant. It is supposed to fail. +- Commit without running the tests, or open a PR with no description. +- Ignore a red check on the grounds that it is not required. Most red CI here is + the runner rather than the code (`autopilot.yml:11`), but that is a conclusion + to reach after reading the log, not before. `autopilot.yml:88-89` adds the + corollary: detection of the concurrency bugs here is probabilistic, so passing + once is not evidence of a fix, and `:93-95` asks for an ablation — show the + test fails without your change and passes with it. ## Code Review -All contributions go through code review: -- Be respectful and constructive -- Respond to feedback promptly -- Ask questions if unclear -- Maintainers have final say +All contributions go through review. Be respectful and constructive, respond to +feedback promptly, ask when something is unclear, and expect maintainers to have +the final say. Whether protection mechanically requires an approval is +genuinely unclear — the two GitHub APIs disagree, as recorded above — so treat +review as a norm the project keeps rather than something to rely on the machine +to enforce. + +## Code of Conduct + +`CODE_OF_CONDUCT.md` applies to all project spaces (`:31-33`). Its enforcement +section (`:35-39`) says to report by "contacting the project team" and names no +address. There is no address to name: `contact@softmata.dev` at `README.md:648` +is the only email in any markdown file in this repository, and it is published +for hardware feedback (`README.md:644-648`), not as a conduct channel. Until +someone decides on a real reporting address, open an issue or contact a +maintainer directly. This gap needs closing properly rather than by pointing at +an address meant for something else. ## License -By contributing, you agree that your contributions will be licensed under the Apache License 2.0. +By contributing, you agree that your contributions will be licensed under the +Apache License 2.0, and that they are subject to the +[Contributor License Agreement](.github/CLA.md) — see step 7 of the PR process +for how to signal agreement. -All contributors must agree to the [Contributor License Agreement (CLA)](.github/CLA.md). When submitting your first pull request, add a comment: +--- -``` -I have read and agree to the Contributor License Agreement. -``` +## Further reading + +This file covers the process. The rest of the contributor documentation lives +under `docs/`, and each of these answers a question this file deliberately does +not: -## Thank You! +| Document | Answers | +|---|---| +| [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) | Where does my change go? A crate and module map, and the execution model that decides which thread a node runs on. | +| [`docs/TESTING.md`](docs/TESTING.md) | Which suite does my test belong in, what does CI actually invoke, and what is committed but never reached? | +| [`docs/SHM_WIRE_FORMAT.md`](docs/SHM_WIRE_FORMAT.md) | I need to change the shared-memory layout. What else has to move with it? | +| [`docs/RELEASING.md`](docs/RELEASING.md) | How is a release cut, and what gates does the tag then face? | +| [`horus_net/docs/BLUEPRINT.md`](horus_net/docs/BLUEPRINT.md) | The network protocol spec that thirteen modules cite by section number. | +| [`horus_core/README.md`](horus_core/README.md) | An index into the crate's design documentation, most of which `cargo doc` does not render. | -Every contribution helps make HORUS better. Thank you for being part of the community! +For using HORUS rather than changing it, see the documentation site, and in +particular its Development section for the CLI, the environment variables, and +the runtime extension points. --- -Questions? Open an issue or start a discussion on GitHub! +Questions? Open an issue or start a discussion on GitHub. diff --git a/horus_core/README.md b/horus_core/README.md index 76b295099..1d4274932 100644 --- a/horus_core/README.md +++ b/horus_core/README.md @@ -1,109 +1,528 @@ -# HORUS Core +# horus_core -**Internal implementation crate.** Use the `horus` crate instead: +The runtime: nodes, the scheduler and its four executors, shared-memory topics, +the tensor pool, actions, services, errors. 112,344 lines under `src/` +(`find horus_core/src -name '*.rs' | xargs wc -l`), of which 14,340 are `///` +doc lines and 2,280 are `//!` module prose. -```rust -use horus::prelude::*; -``` +Almost none of that prose is reachable through `cargo doc`. This file is the +index to it. It is written for someone changing this crate, not for someone +using it. -## Overview +**If you are writing a robot, you want the `horus` crate**, whose rustdoc is +public and whose `prelude` is the supported surface (`horus/src/lib.rs:581`). +The user-facing API reference lives at `content/docs/rust/api/core.mdx` in +`horus-docs` — 1,234 lines of hand-written MDX, not generated from this source. +See [Who actually depends on this crate](#who-actually-depends-on-this-crate) +for what "internal" does and does not mean in practice. -The core runtime for HORUS, providing: +--- -- **Node** - Simple trait with `init/tick/shutdown` lifecycle -- **Scheduler** - Runs nodes in order with configurable rates and composable builders -- **Topic** - Pub/sub communication that auto-selects the fastest transport -- **Logging** - `hlog!()` macro with context capture +## Why cargo doc shows you nothing -## Quick Start +`horus_core/src/lib.rs` is 96 lines. Eleven of its twelve `pub mod` +declarations carry `#[doc(hidden)]` — ten unconditionally, and `testing` both +hidden and `cfg`-gated: -```rust -use horus::prelude::*; +| Module | Hidden? | Declared at | +|---|---|---| +| `actions` | yes | `lib.rs:20-21` | +| `communication` | yes | `lib.rs:22-23` | +| `core` | yes | `lib.rs:24-25` | +| `drivers` | yes | `lib.rs:26-27` | +| `error` | yes | `lib.rs:28-29` | +| `memory` | yes | `lib.rs:30-31` | +| `params` | yes | `lib.rs:32-33` | +| `scheduling` | yes | `lib.rs:34-35` | +| `services` | yes | `lib.rs:36-37` | +| `terminal` | **no** | `lib.rs:38-44` | +| `types` | yes | `lib.rs:45-46` | +| `testing` | yes, and `cfg(any(test, feature = "test-utils"))` | `lib.rs:50-52` | -fn main() -> Result<()> { - let mut scheduler = Scheduler::new(); +There is one more declaration, `pub(crate) mod utils;` at `lib.rs:47`, which is +private and so never rendered either. - scheduler.add(SensorNode::new()?).order(0).build()?; - scheduler.add(ControlNode::new()?).order(1).build()?; +Every crate-root re-export is hidden as well, apart from one: `ControlCommand` +(`lib.rs:79`). `Topic`, `Scheduler`, `Node`, `HorusError` and the rest go out +behind `#[doc(hidden)]` at `lib.rs:56-86`, and the four re-exported +dependencies (`bytemuck`, `paste`, `serde_json`, `serde_yaml`) at +`lib.rs:88-96`. - scheduler.run() -} -``` +`#[doc(hidden)]` on a module strips the module *and everything inside it* from +the rendered output. A public re-export *out of* a hidden module still renders, +which is why `ControlCommand` survives, and a `#[macro_export]` macro declared +inside a hidden module also renders, because `macro_export` puts it at the +crate root regardless. Nothing else in this crate takes either route. -## Node Trait +So the crate's front page carries exactly four things. This is observed, not +inferred — `cargo doc -p horus_core --no-deps` with `CARGO_TARGET_DIR` pointed +at a scratch directory produces a `sidebar-items.js` containing precisely: -```rust -pub trait Node: Send { - fn name(&self) -> &str; - fn init(&mut self) -> Result<()>; - fn tick(&mut self); - fn shutdown(&mut self) -> Result<()>; -} +```js +window.SIDEBAR_ITEMS = {"enum":["ControlCommand"],"macro":[["action",1], +["hlog",1],["hlog_every",1],["hlog_once",1],["horus_internal",1], +["impl_frame_id_field",1],["impl_tensor_accessors",1],["impl_tensor_backed",1], +["impl_timestamp_field",1],["message",1],["register_driver",1],["service",1], +["standard_action",1],["topics",1]],"mod":["terminal"]}; ``` -## Scheduler +That is the sixteen lines of crate prose at `lib.rs:1-16`, one module, one +enum, and fourteen macros. Fifteen `#[macro_export]` macros exist in the crate; +the fifteenth, `__hlog_level_const`, carries its own `#[doc(hidden)]` +(`core/hlog.rs:174-175`) and is the one missing from that list. Everything +else — 112,000 lines and the arguments in them — renders nowhere. + +Two consequences worth stating plainly. + +**The prose is lint-gated but not published.** The `doc` job at +`.github/workflows/ci.yml:308` runs +`cargo doc --workspace --no-deps --exclude horus_manager` (`ci.yml:345`) with +`RUSTDOCFLAGS: -D warnings`, but only when the ref is `main` or the base ref is +`main` (`ci.yml:347`) — so a PR into `main` is gated and a PR into `dev` is +not. Rustdoc's lints fire inside `#[doc(hidden)]` items: a broken intra-doc +link in a hidden module is a warning by default and a hard error under +`-D warnings`, verified on a two-line probe crate. So every one of those 14,340 +doc lines is held to a standard, and then discarded. No workflow publishes +`target/doc` (`grep -rn 'cargo doc\|target/doc\|gh-pages\|deploy-pages' +.github/workflows/` matches only the comment at `ci.yml:338` and the command at +`ci.yml:345`). `docs.horusrobotics.dev`, declared as this crate's +`documentation` at `Cargo.toml:11`, is the `horus-docs` Next.js site +(`horus-docs/app/sitemap.ts:7`), not rustdoc output. And docs.rs cannot help, +because none of these crates are on crates.io (`README.md:70-73`). + +**Instructions that users must follow are invisible.** The clearest case: +`memory/rt_allocator.rs:15-19` tells you to write ```rust -// Lightweight default -let mut scheduler = Scheduler::new(); - -// Per-node execution classes via fluent builders -scheduler.add(motor).order(0).rate(1000_u64.hz()).build()?; // RT (auto-derived) -scheduler.add(planner).order(5).compute().build()?; -scheduler.add(telemetry).order(10).async_io().rate(1_u64.hz()).build()?; - -// Production-ready with watchdog, blackbox, RT -let mut scheduler = Scheduler::new() - .watchdog(500_u64.ms()) - .blackbox(64) - .tick_rate(1000_u64.hz()) - .prefer_rt(); - -// Add nodes with execution order -scheduler.add(my_node).order(0).rate(100_u64.hz()).build()?; - -// Run -scheduler.run()?; -scheduler.run_for(10_u64.secs())?; +#[global_allocator] +static ALLOC: horus_core::memory::rt_allocator::RtAwareAllocator = + horus_core::memory::rt_allocator::RtAwareAllocator; ``` -## Topic Communication +in your own `main.rs`, because without it `.no_alloc()` is inert +(`rt_allocator.rs:28-38`). `memory` is hidden here and hidden again in the +facade (`horus/src/lib.rs:389-390`), so that page renders in neither crate. The +snippet is only correct at all because +`horus_core/tests/rt_alloc_guard_installed.rs:39-41` compiles it as that test +binary's real global allocator — an earlier version of the same paragraph named +`horus_core::memory::RtAwareAllocator`, a path that does not exist, and nothing +caught it (`rt_allocator.rs:21-26`, `rt_alloc_guard_installed.rs:35-38`). -```rust -let topic: Topic = Topic::new("sensor_data")?; +## How to read the internals -topic.send(42.0); -if let Some(value) = topic.recv() { - println!("Got: {}", value); -} +`--document-private-items` is **not** enough on its own. It disables rustdoc's +private-stripping pass, not its hidden-stripping pass, so the eleven hidden +modules stay stripped. You need nightly and `--document-hidden-items`, which is +unstable and refuses to run without `-Z unstable-options`: + +```sh +RUSTDOCFLAGS="-Z unstable-options --document-hidden-items" \ + cargo +nightly doc -p horus_core --no-deps --document-private-items --open ``` -The transport backend is auto-selected based on topology: +Both halves matter. `--document-hidden-items` restores the hidden modules; +`--document-private-items` restores the `pub(crate)` ones — `topic::header` +(`communication/topic/mod.rs:118`), `topic::dispatch` (`:131`), +`topic::seqlock` (`:134`), `scheduling::rt_executor` +(`scheduling/mod.rs:34`), `scheduling::primitives` (`:31`) and their +neighbours — which is where most of the protocol writing actually lives. -| Scenario | Latency | -|----------|---------| -| Same-process fan-out | 3 ns median | -| Same-process SPSC | 10 ns median | -| Cross-process | 198-304 ns median | +Each half was checked separately on a probe crate rather than taken from +memory, because this is the one instruction in this file a reader will act on: +plain `rustdoc --document-private-items` leaves a `#[doc(hidden)] pub mod` +absent from the sidebar and adds only the `pub(crate)` module; passing +`--document-hidden-items` without `-Z unstable-options` fails with *"the +`-Z unstable-options` flag must also be passed"*; the command above, run +end-to-end through cargo, lists the hidden module. If you doubt any of it, a +two-line `lib.rs` reproduces all three in ten seconds. -## Logging +`horus doc` does not do this. It documents the *user's* project, from that +project's directory, pointing cargo at `.horus/Cargo.toml` +(`horus_manager/src/commands/doc.rs:1-4` and `:16-48`). -```rust -use horus_core::hlog; +## The module index + +Sizes are `src/` line counts. "Read first" names the file whose header carries +the real argument, not the file with the most code. + +| Module | Lines | Read first | +|---|---:|---| +| `scheduling` | 39,523 | `scheduling/node_builder.rs:6-37` | +| `communication` | 31,649 | `communication/topic/mod.rs:1-112` | +| `memory` | 11,165 | `memory/rt_allocator.rs:1-88` | +| `core` | 8,918 | `core/node.rs:953-971` | +| `actions` | 5,925 | `actions/mod.rs:1-122` | +| `types` | 4,405 | `types/mod.rs:1-22` | +| `error.rs` | 2,937 | `error.rs:1-64` | +| `services` | 2,492 | `services/mod.rs:1-60` | +| `drivers` | 1,908 | `drivers/mod.rs:1-19` | +| `params.rs` | 1,666 | `params.rs:1-3` | +| `testing` | 1,449 | gated on `cfg(test)` or `feature = "test-utils"` (`Cargo.toml:91-92`) | +| `terminal.rs` | 63 | `terminal.rs:1-19` | + +Those twelve plus `utils.rs` (148) and `lib.rs` (96) are the whole crate. + +### communication + +`communication/topic/mod.rs:1-112` is the single most important header in the +crate: the backend detection matrix, the thread-ownership contract, the +lock-free ring protocols, the `read_latest` use-after-free argument and the +drop-safety rules. Read it before touching anything under +`communication/topic/`. Its closing "Verification" paragraph (`:106-112`) is +stale and should not be trusted: it claims "16 loom exhaustive concurrency +tests … for SPSC, SPMC, MPSC, and MPMC algorithms, including `read_latest` + +`try_recv` races", where the tree holds ten loom targets with 36 `#[test]` +functions between them, none of which mentions `read_latest`. The models that +covered those four ring algorithms were `loom_ring_buffers`, deleted with the +intra backends (`ci.yml:379-380`). + +Then, in order of how much they will surprise you: + +- `communication/topic/dispatch.rs:2-118` — one complete send/recv path per + backend, plus a measured account of where the remaining ~55 ns against a raw + ring goes. The ablation table at `:31-39` is the useful part: removing the + consumer's head gate makes things 2-4 ns *worse*. Its conclusion (`:54-61`) + is that anyone trying to close the gap should change the protocol, not delete + work from these functions, and should check first whether 55 ns of a 1 kHz + period is worth anything. +- `communication/topic/shm_layout.rs:1-50` — the authoritative byte layout, + and why it is a module rather than a comment. `horus_net` once copied a table + of offsets and the copy rotted in four independent places (`:11-16`) while + feeding an IPC region from unauthenticated network data. Each constant is now + tied to its field by `offset_of!` in a `const` block (`:20-24`), so a + reordered `TopicHeader` field is a build failure. +- `communication/topic/header.rs:24-66` — `TOPIC_VERSION`, currently `4` + (`:66`), and why v4 is a hard break rather than a soft migration. Neither of + its two changes (monotonic timestamps, `layout_kind`) is detectable in-band + by a v3 reader, so the open path refuses the segment instead. +- `communication/topic/seqlock.rs:1-47` — the drop-oldest fanout ring's + per-slot version protocol. +- `communication/pod.rs:1-77` — POD auto-detection by `!needs_drop::()`, + and the caveat at `:10-26` that it is a heuristic and unsound for `bool`, + fieldless enums and `NonZero*`. Prefer explicitly encoded integer fields in + messages you intend to send as fixed-size. +- `communication/topic/migration.rs:2-5` and `communication/topic/registry.rs:1-5` + — CAS-locked backend migration, and the process-local epoch counter that lets + sibling `Topic` instances notice it with a ~1 ns L1 read instead of a ~20 ns + SHM read. + +### scheduling + +`scheduling/node_builder.rs:6-37` is the entry point: the decision tree and +table for the five execution classes (`Rt`, `Compute`, `Event(String)`, +`AsyncIo`, `BestEffort` — `scheduling/types.rs:279-291`), and the rules that +catch people out (`:33-37`). One class per node, last call wins with a warning; +`.rate()` on a `BestEffort` node auto-promotes it to `Rt`; `.compute().rate()` +stays `Compute` and the rate becomes informational; a chain without `.build()?` +is silently dropped. + +The executors each carry an ASCII architecture diagram in their header: +`scheduling/rt_executor.rs:8-22`, `compute_executor.rs:7-22`, +`event_executor.rs:7-21`, `async_executor.rs:7-22`. Read +`rt_executor.rs:366-429` for the cyclic-wait rewrite — an absolute-deadline +`clock_nanosleep` with a bounded guard spin, replacing a loop that both drifted +(`:382-389`) and busy-waited hard enough to be dequeued by Linux RT bandwidth +control for roughly 50 ms once a second (`:391-401`), a tail that only appears +once RT priority is actually granted (`:403-408`). The trade it makes — median +wake jitter regressing from the old spin's ~100 ns to hrtimer precision, bought +to delete that ~50 ms tail — is stated at `:416-423` rather than buried, along +with the two escape hatches: `HORUS_RT_WAIT=spin` restores a pure spin and +`HORUS_RT_SPIN_GUARD_US` retunes the guard (read at `:675` and `:684`, the +env-var name held at `:472`). The constants that bound it are at `:431-467`: +`SPIN_GUARD_DEFAULT_NS = 20_000` (`:445`), `SPIN_GUARD_PERIOD_SHIFT = 4` +(`:456`), `MIN_SLEEP_SLACK_NS = 10_000` (`:467`). + +`scheduling/primitives.rs:1-24` explains why the RT-side reporting path uses +`rt_diag` and not `print_line`: `print_line` does `isatty` + `tcgetattr`, takes +the process-global stdout lock, and can then block for as long as a slow +console takes — on the thread driving an actuator. + +### memory + +`memory/rt_allocator.rs:1-88` is the `.no_alloc()` contract, and it is the +best-written safety argument in the crate. The parts to not miss: the +installation step is downstream opt-in and a library cannot do it for you +(`:9-26`); without it `.no_alloc()` is silently unenforced, and `is_installed` +/ `warn_if_unenforced` exist so that downgrade can be reported (`:28-38`); a +violation panics inside the tick and routes through the node's `FailurePolicy` +rather than aborting (`:40-66`); and the caveat at `:68-74` that the default +`failure_policy` is `None`, which `apply_failure_policy_after_panic` treats as +log-and-continue — so a `.no_alloc()` node with no explicit policy keeps +ticking and keeps violating, detected and counted, but not acted on. + +Elsewhere: `memory/tensor_pool.rs:2-60` for the cross-process zero-copy pool +layout; `memory/backend.rs:1-21` for the `PoolBackend` trait that controls +where tensor bytes are allocated — `MmapBackend` is the only built-in (`:10`, +implemented at `:396`), the three CUDA backends at `:12-16` are future work, +and the `HeapBackend` at `:570` is test-only; and `memory/simd.rs:1-40` for why +the hand-written AVX2 copy paths were deleted — both directions lost to +`std::ptr::copy_nonoverlapping`, with the measured table at `:19-23`. + +### core + +`core/node.rs:953-971` carries the `Node` trait (`pub trait Node: Send` at +`:971`). Only `tick()` is required (`:992`); `name()` defaults to the type name +(`:976-982`), `init()` and `shutdown()` default to `Ok(())` and return +`crate::error::HorusResult<()>` (`:987-989`, `:997-999`). +`on_parameter_change` is documented at `:1030-1035` as **not wired** — the +scheduler does not call it, and overriding it today has no effect; register a +callback via `RuntimeParams::on_change()` instead. -hlog!(info, "Node initialized"); -hlog!(warn, "Low battery: {}%", percentage); -hlog!(error, "Sensor read failed: {}", err); +`core/clock.rs:1-13` has the clock-backend table (`WallClock`, `SimClock`, +`ReplayClock`); `core/tick_context.rs:1-8` explains the thread-local ambient +context that `horus::now()`, `dt()` and `rng()` read; `core/log_buffer.rs:170` +onwards documents the per-slot seqlock in the SHM log ring, and `:79` plus +`:477-494` document how a writer that died mid-write is detected, skipped, and +its seqlock reset to even so the slot can be reclaimed. + +### actions, services, errors, drivers, params, types, terminal + +- `actions/mod.rs:1-122` — the goal/feedback/result pattern (`:13-18`), server + and client builders, preemption policies (`:108-110`). At 122 lines it is the + longest module header in the crate, just ahead of `dispatch.rs` (117) and + `topic/mod.rs` (112). +- `services/mod.rs:1-60` — request/response over two topics per service + (`{name}.request`, `{name}.response`, `:41-43`), request-ID filtering for + concurrent clients (`:47-49`), and a ROS2 equivalence table (`:51-60`). +- `error.rs:1-64` — `HorusError` as an umbrella over eleven structured domain + sub-errors, with the variant table at `:10-22` and the matching idiom at + `:24-45`. +- `drivers/mod.rs:1-19` — loading nodes from `horus.toml` `[hardware]`. The + reserved keys that are consumed rather than passed through as `NodeParams` + are the literal list at `drivers/mod.rs:111-124`; the `exec:` prefix dispatch + to `ExecDriver::from_config` is at `:217-229`. +- `params.rs` — runtime key-value parameters, with the eight `ValidationRule` + variants at `:13-32`. Its header is three lines; the module is 1,666 and + undocumented by comparison. +- `types/mod.rs` — zero-copy tensor descriptors and element types. Its header + is a single line, and it hides eight of its own eleven children again at + `:3-22` (`action_chunk`, `image_encoding` and `point` are the three that + would render if the parent did). +- `terminal.rs:1-19` — the only module that renders today, and it renders + because it must: `println!`/`eprintln!` panic when the write fails, and a + robot's stdout fails routinely. Anything printing from a node, a background + thread or a safety path uses these. + +## Load-bearing contracts + +Four things you can break without the compiler noticing. + +**`Topic` is `Send` and deliberately not `Sync`.** `unsafe impl Send for Topic where T::Wire: Send` at +`communication/topic/mod.rs:3711`; there is no `Sync` impl, and the comment at +`:3713-3719` says why — `send`/`recv` take `&self` and mutate +`registered_pub`, `registered_sub`, `owner_attempts` and a `RefCell` keep-alive +queue, so two threads sharing one `&Topic` can release the same pool +slot twice, freeing shared memory that subscribers are still reading. To share +one handle across threads you need a `Mutex>`. **An `RwLock` is not +enough**: a read guard hands out concurrent `&Topic`. Otherwise clone per +thread — `Clone` gives the new thread its own independent state. The module +header states both at `communication/topic/mod.rs:36-48`, and it is the +paragraph to keep in step if the impls change. `RingTopic` is `Send` for +`T: Send` on the same reasoning (`:777`), also with no `Sync`. A compile-time +guard at `:3721-3728` asserts `Topic: Send` so that losing `Send` fails +here rather than at a `thread::spawn` in a downstream crate. This is the +current state of a contract that was previously wrong in both directions: +hand-written `unsafe impl Sync` blocks existed while the prose claimed +`!Send + !Sync`, so safe code could race the unsynchronised state with no +`unsafe` of its own (`:42-48`, `:779-793`). + +**The ring protocols differ per backend, and the ordering is the proof.** +`communication/topic/mod.rs:63-75`: SPSC needs no CAS (producer `Release`s +head, consumer `Acquire`s it); SPMC has consumers CAS the tail with `AcqRel` to +claim exclusive read slots; MPSC has producers CAS head with per-slot +Lamport-style sequence numbers; MPMC CASes both ends. The seqlock fanout ring +is separate again: an odd stamp `(pos << 1) | 1` means a write is in progress +and an even stamp `pos << 1` means done, the producer sets odd *before* +touching data and even *after*, and the consumer must see exactly `next << 1` +on both sides of its copy or discard it (`seqlock.rs:19-34`). Publication +order — done-stamp `Release`, then head `Release` — is what makes `head > next` +imply the done stamp for `next` is visible (`seqlock.rs:36-38`). Retries are +bounded at `SEQLOCK_MAX_ATTEMPTS = 16` (`seqlock.rs:55`). + +Ten loom models live in `horus_core/tests/loom_*.rs` and are the executable +form of these arguments — 36 `#[test]` functions between them. Six have a named +step in ci.yml's `loom` job (`ci.yml:349`): `loom_fanout` (`:383`), +`loom_pod_broadcast` (`:392`), `loom_concurrency` (`:397`), `loom_migration` +(`:402`), `loom_migration_data_plane` (`:414`), `loom_participant` (`:419`). +The other four — `loom_mp_claim`, `loom_sp_mp_flag`, `loom_spmc_epoch_flush`, +`loom_tensor_pool` — are named by no workflow, but they are **not** unrun: +`integration-tests.yml:127` runs +`cargo test --workspace --exclude horus_py --release --test '*'`, and that glob +matches every integration target, loom included. So they do execute on every PR +into `main` or `dev` — in release, at `--test-threads=1`, inside a 45-minute +sweep whose failure reads as an integration break rather than a protocol break, +and with none of the `RUST_LOG=loom=debug` the named steps set. If you change a +protocol one of those four covers, run it yourself and read the output: + +```sh +cargo test -p horus_core --test loom_mp_claim -- --nocapture ``` -## Order Guidelines +One test is genuinely run nowhere by default: +`loom_pod_broadcast.rs:224` carries +`#[ignore = "demonstrates the OLD protocol failing; run by hand to validate the +model"]`. That is deliberate — it is the gate that proves the model is not +vacuous, and it is meant to be run by hand when you change the protocol it +models. + +**`read_latest()` requires `T: Copy` on multi-consumer backends.** +`communication/topic/mod.rs:77-97` sets out the TOCTOU: `read_latest` computes +a slot index, a consumer CAS-advances tail past that slot and +`assume_init_read`s the value out, `Drop` frees the heap allocation, and +`read_latest` then reads freed memory. `Copy` types have no `Drop` and no heap +pointers, so the bytes are always safe to bitwise-copy. Single-consumer +backends keep `T: Clone` (`:93-94`); the `Topic`-level `read_latest` requires +`Copy` unconditionally because migration can switch the backend from SPSC to +SPMC at runtime (`:96-97`). + +**The executor model gives you no ordering between RT nodes.** The comment at +`scheduling/scheduler/mod.rs:3097-3117` is the authority, and `rt_chains` at +`:3118-3119` is `groups.rt_nodes.into_iter().map(|n| vec![n]).collect()` — one +OS thread per RT node, always, so a stalled node cannot block its RT siblings. +The block also records that the previous claim (dependent RT nodes grouped into +chains and ticked sequentially) was never true: `independent_chains` read no +edges and was queried against the wrong graph (`:3104-3111`). Stated at +`:3113-3117`: an RT consumer may read its RT producer's previous-tick value. +Nodes that need ordering must share an execution class and be sequenced by the +main loop, or synchronise through the data they exchange. + +Note that `rt_executor.rs:2-22` still describes the old model ("Runs all RT +nodes on an isolated OS thread … ticks RT nodes sequentially in priority +order"), and its diagram draws three RT nodes ticking on one thread. The +scheduler is right and the executor header is stale. Fixing it is also claimed +by the P2 document; the two should not both silently take ownership. + +Two more, briefly. Per-tick fault isolation is one `catch_unwind` in +`NodeRunner::run_tick` (`scheduling/primitives.rs:229`); `init()` and +`shutdown()` panics are caught separately (`scheduling/types.rs:596-625`), and +the compute worker wraps its dispatch in a second one +(`compute_executor.rs:199-206`). A profile that set `panic = "abort"` would +delete all of them — the workspace does not set it today +(`Cargo.toml:119-120`, which sets only `lto = "thin"`). And the main tick loop +runs on a phase-locked absolute grid rather than sleeping a period after the +work; the reason (a 100 Hz scheduler whose nodes took 3 ms ran at 77 Hz, +load-dependently, and nothing said so) is written at +`scheduling/scheduler/mod.rs:3914-3933`, with the overrun counter at `:241-243` +and the same figure restated on `TickState::next_deadline` at `:236-239`. + +## Who actually depends on this crate + +The "internal crate, use `horus` instead" framing is right as advice and wrong +as a description of the dependency graph. What is real: + +- Seven crates in this workspace take `horus_core` as a normal path + dependency: `horus`, `horus_manager`, `horus_net`, `horus_py`, `horus_cpp`, + `horus_types` and `benchmarks`. `horus_macros` takes it as a dev-dependency + only (`horus_macros/Cargo.toml:29`), and `horus_cpp` and `horus_manager` take + a second, dev-scoped copy (`horus_cpp/Cargo.toml:35`, + `horus_manager/Cargo.toml:127`). Counted as reference lines + (`grep -rn 'horus_core::' --include=*.rs /src`, discounting + `::horus::horus_core::`), the usage is `horus_manager` 171, `horus_net` 75, + `horus_py` 55, `horus` 54, `horus_cpp` 54, `horus_types` 29, `benchmarks` 19, + `horus_macros` 1. Only one of those, `horus`, is the facade; the other seven + reach past it. +- Every project `horus new` generates lists `horus_core` as a direct path + dependency. `write_horus_path_deps` iterates + `["horus", "horus_core", "horus_library", "horus_macros"]` and emits a path + dep for each that exists — `horus_manager/src/cargo_gen.rs:1066` for the + single-package layout (called at `:116`), `:1808` for the workspace root's + `[workspace.dependencies]`, and `:1979-1980` for each member's + `horus_core.workspace = true`. So `use horus_core::…` compiles in a generated + project, and nothing in the toolchain objects. +- It has to. `horus_macros` expands to `::horus_core::core::LogSummary` + (`horus_macros/src/log_summary.rs:10`) — the only bare `::horus_core::` path + it emits, and one that resolves only if the user's crate names `horus_core` + directly. The rest of the generated code goes through + `::horus::horus_core::…` (e.g. `horus_macros/src/node.rs:409`), which works + because `horus` re-exports the crate at `horus/src/lib.rs:353-354`. +- The `test-utils` feature has a real consumer too: `horus_cpp` takes + `horus_core` as a dev-dependency with `features = ["test-utils"]` + (`horus_cpp/Cargo.toml:35`). + +That is what the `#[doc(hidden)] pub use` block at `lib.rs:56-96` is for, and +the comment at `lib.rs:54-55` says so: those re-exports exist for +crate-internal use and for `horus_py` and macro-generated code cross-crate, not +for readers. **Do not delete a hidden re-export because nothing in this crate +uses it.** Grep the workspace first, not just this crate. + +So: depend on `horus`, import from `horus::prelude::*`, and treat every +`horus_core` path as unstable. But write your commit message knowing that +breaking one is a breaking change for eight sibling crates and every generated +project, not an internal refactor. + +## Doc comments that are attached to the wrong thing + +Found while writing this file. They are wrong in the source, not only in the +rendering, so `--document-hidden-items` will not save you from them. + +| Where | What it says | What it is attached to | +|---|---|---| +| `scheduling/scheduler/mod.rs:281` | "Central orchestrator: holds nodes, drives the tick loop." | `PresenceEntry` at `:284`. `Scheduler` at `:294` has no doc comment. | +| `scheduling/scheduler/mod.rs:3914-3933` | The tick-grid essay, explicitly about `compute_tick_sleep` | `effective_tick_period` at `:3939`, whose own doc starts at `:3934` and follows it with no blank line. `compute_tick_sleep` at `:3948` has none. | +| `communication/topic/dispatch.rs:90` | Invariant 1: "`Topic` is `!Send + !Sync`" | Correct about `Sync`, wrong about `Send` since `mod.rs:3711`. The invariant it should state is single-thread *ownership*, which `Send` does not violate. | + +Two more shapes of the same problem, worth knowing because they are not +individually fixable: + +**Essays written as `//` on no item.** They render nowhere under any flag. Two +that are load-bearing: `scheduling/rt_executor.rs:366-429` (the cyclic-wait +rewrite, 64 lines, a banner comment above a group of constants — the next item +starts at `:431`) and `scheduling/scheduler/mod.rs:3097-3117` (the RT threading +model, attached to a `let` binding). + +**No two places agree on how fast a topic is.** Fourteen files under `src/` +carry a `~N ns` figure in module prose +(`grep -rlE '//!.*~[0-9]+ ?ns' --include=*.rs horus_core/src`). For the topic +path specifically: `communication/mod.rs:9` says "~3ns (same-thread) to ~167ns +(cross-process)"; `communication/mod.rs:22-23`, `communication/pod.rs:54-55` +and `communication/macros.rs:26` say ~50 ns POD against ~167 ns serde, as do +the five `types/*_descriptor.rs` headers (`:4` in each) and +`types/action_chunk.rs:10`; `communication/topic/mod.rs:13-19` gives a +per-backend table running ~40 ns (FanoutShm) to ~85 ns (SpscShm); +`scheduling/event_executor.rs:33` says "~300 ns the transport delivers"; and +`dispatch.rs:21-24` reports ~105-118 ns cross-thread one-way for +`Topic`. This README's own table, deleted in this rewrite, added three +more: 3 ns same-process fan-out, 10 ns same-process SPSC, 198-304 ns +cross-process (`README.md:83-87`, pre-rewrite). The "~3 ns same-thread" claim +sits particularly badly against `topic/mod.rs:7-9`, which +says every topic is SHM-backed and all backends are cross-process, and against +the table four lines below it whose fastest entry is ~40 ns. Only `dispatch.rs` +names the harness that produced its numbers — `topic_probe` and +`topic_probe --raw-ring` (`benchmarks/src/bin/topic_probe.rs`, the flag parsed +at `:618`), sharing threads, pinning, clock and ack protocol so the transport +is the only variable. Quote that one, with its conditions, or quote none. + +## What this file does not fix + +Writing it down does not publish it. Three things are still open, and this +README is the wrong tool for all three: + +1. **Nothing deploys `target/doc`.** CI builds it (`ci.yml:345`) and throws it + away. Until something hosts it, the command in + [How to read the internals](#how-to-read-the-internals) is the only route, + and it needs a nightly toolchain. +2. **`cargo doc` excludes `horus_manager`** (`ci.yml:345`). The reason is real: + `horus_manager` declares `[[bin]] name = "horus"` at + `horus_manager/Cargo.toml:26-27`, which would write `target/doc/horus/` on + top of the `horus` library crate's output (`ci.yml:337-338`). Restricting + the workspace doc build to library targets is the obvious candidate. + **Unverified:** nobody has run `cargo doc --workspace --no-deps --lib` here + to confirm it resolves the collision. Treat it as a lead, not a fix, and do + not let a later edit promote it without someone running it. +3. **The hidden modules stay hidden.** That is deliberate — un-hiding them + would present `horus_core` as a supported API, which it is not — but it + means the rendered `horus` docs and this file are the only two places a + reader can start. Keep them in step. -- **0-9**: Critical (motor control, safety) -- **10-49**: High priority (sensors, fast loops) -- **50-99**: Normal (processing, planning) -- **100-199**: Low (logging, diagnostics) -- **200+**: Background (telemetry) +Every line number above is from the working tree as of this writing. The most +fragile are the four-digit ones in `scheduling/scheduler/mod.rs` (`:281`, +`:294`, `:3097-3119`, `:3914-3948`) and `communication/topic/mod.rs` (`:3711`, +`:3713-3719`, `:3721-3728`): one inserted function moves them all. A checker +that asserts every `pub mod` in `lib.rs` appears in the table above, and that +the `Send`/`Sync` sentence matches the impls, would catch drift in the module +list and in the contract — it would not catch drift in the line citations. -## License +## Licence -Apache License 2.0 - see [LICENSE](../LICENSE) for details. +Apache-2.0. See [LICENSE](../LICENSE). From 0609bb10fcbd6bab47e1db1f369dc5f73be440d7 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 12:06:06 -0700 Subject: [PATCH 38/79] fix: two hints that could never fire, and a preferred section nobody read exit_code_hint documents that "signal-based exits (128+N) are translated into actionable hints", and its H053 arm says "Likely out of memory" - the most useful thing this module knows how to say. Every production caller reached it through `status.code().unwrap_or(1)`, and on Unix `code()` is None for a signal death, so an OOM-killed child took the `_ => {}` "too generic, no hint" branch instead. The 137, 139 and 128+N arms were reachable only from the unit tests that pass the numbers in as literals. shell_exit_code() reports a signal death the way a shell does, and the new test kills a real child rather than asserting against a constant. [hardware] is documented as the preferred spelling and [drivers] as legacy, but cargo_gen and pyproject_gen read `drivers` alone, so a driver declared the preferred way generated no Cargo dep and no PyPI dep - the legacy section did strictly more than the one the docs tell you to use. doctor and native_sync already merged the two by hand; hardware_entries() is now the single place that does it, and all four consumers go through it. --- horus_manager/src/cargo_gen.rs | 75 ++++++++++++++++++-- horus_manager/src/commands/doctor.rs | 12 +--- horus_manager/src/commands/run/run_python.rs | 4 +- horus_manager/src/commands/run/run_rust.rs | 10 +-- horus_manager/src/error_wrapper.rs | 62 ++++++++++++++++ horus_manager/src/manifest.rs | 22 ++++++ horus_manager/src/native_sync.rs | 4 +- horus_manager/src/pyproject_gen.rs | 4 +- 8 files changed, 166 insertions(+), 27 deletions(-) diff --git a/horus_manager/src/cargo_gen.rs b/horus_manager/src/cargo_gen.rs index f88f32047..56711e2b3 100644 --- a/horus_manager/src/cargo_gen.rs +++ b/horus_manager/src/cargo_gen.rs @@ -120,8 +120,8 @@ pub fn generate( // Add user deps from horus.toml write_deps_section(&mut cargo, &manifest.dependencies, project_dir, &horus_dir)?; - // Add driver dependencies from [drivers] config tables - write_driver_deps(&mut cargo, &manifest.drivers); + // Add driver dependencies from the [hardware]/[drivers] config tables + write_driver_deps(&mut cargo, &manifest.hardware_entries()); // ── Dev dependencies ───────────────────────────────────────────────── if include_dev && !manifest.dev_dependencies.is_empty() { @@ -871,7 +871,7 @@ fn writable_by_horus(config_path: &Path) -> bool { /// - `camera = "opencv"` / `gps = true` → no dependency (legacy, handled by feature flags) fn write_driver_deps( cargo: &mut String, - drivers: &std::collections::BTreeMap, + drivers: &std::collections::BTreeMap<&str, &crate::manifest::DriverValue>, ) { use crate::manifest::DriverValue; @@ -882,7 +882,7 @@ fn write_driver_deps( let mut added_crates: BTreeMap, Vec)> = BTreeMap::new(); for value in drivers.values() { - match value { + match *value { DriverValue::Config(cfg) => { // Terra drivers: no longer auto-resolved — user adds terra-horus // as a normal dependency in [dependencies]. Skip terra entries. @@ -916,7 +916,7 @@ fn write_driver_deps( } if !added_crates.is_empty() { - writeln!(cargo, "\n# Driver dependencies (from [drivers])").unwrap(); + writeln!(cargo, "\n# Driver dependencies (from [hardware])").unwrap(); for (crate_name, (version, features)) in &added_crates { let ver = version.as_deref().unwrap_or("*"); if features.is_empty() { @@ -2879,6 +2879,71 @@ mod tests { ); } + /// `[hardware]` is the documented preferred spelling and `[drivers]` the + /// legacy one, yet only `[drivers]` used to reach the generator — so a + /// crates.io driver declared the preferred way produced no Cargo + /// dependency at all, and the project failed to build with an + /// unresolved-crate error naming something the user had already declared. + #[test] + fn a_driver_under_hardware_generates_the_same_dep_as_under_drivers() { + let cfg = DriverTableConfig { + crate_name: Some("rplidar-driver".into()), + ..Default::default() + }; + + let mut under_hardware = test_manifest(BTreeMap::new()); + under_hardware + .hardware + .insert("lidar".into(), DriverValue::Config(cfg.clone())); + + let mut under_drivers = test_manifest(BTreeMap::new()); + under_drivers + .drivers + .insert("lidar".into(), DriverValue::Config(cfg)); + + let mut from_hardware = String::new(); + write_driver_deps(&mut from_hardware, &under_hardware.hardware_entries()); + let mut from_drivers = String::new(); + write_driver_deps(&mut from_drivers, &under_drivers.hardware_entries()); + + assert!( + from_hardware.contains("rplidar-driver"), + "a [hardware] driver must generate its crate dep, got: {from_hardware:?}" + ); + assert_eq!( + from_hardware, from_drivers, + "the preferred spelling must not do less than the legacy one" + ); + } + + /// A name in both tables is one piece of hardware, not two deps. + #[test] + fn hardware_wins_over_a_same_named_legacy_driver() { + let mut manifest = test_manifest(BTreeMap::new()); + manifest.hardware.insert( + "arm".into(), + DriverValue::Config(DriverTableConfig { + crate_name: Some("new-arm".into()), + ..Default::default() + }), + ); + manifest.drivers.insert( + "arm".into(), + DriverValue::Config(DriverTableConfig { + crate_name: Some("old-arm".into()), + ..Default::default() + }), + ); + + let mut out = String::new(); + write_driver_deps(&mut out, &manifest.hardware_entries()); + assert!(out.contains("new-arm"), "got: {out:?}"); + assert!( + !out.contains("old-arm"), + "the legacy entry must not shadow-add a second dep, got: {out:?}" + ); + } + #[test] fn battle_driver_features_propagation_camera_lidar() { // Drivers with camera + lidar should propagate features to horus/horus_library path deps. diff --git a/horus_manager/src/commands/doctor.rs b/horus_manager/src/commands/doctor.rs index 907f4f3e4..f27f53561 100644 --- a/horus_manager/src/commands/doctor.rs +++ b/horus_manager/src/commands/doctor.rs @@ -1153,15 +1153,7 @@ fn check_drivers(ctx: &dispatch::ProjectContext) -> CheckResult { } }; - // Merge [hardware] and legacy [drivers] entries - let mut all_entries: std::collections::BTreeMap = - std::collections::BTreeMap::new(); - for (k, v) in &manifest.hardware { - all_entries.insert(k.clone(), v); - } - for (k, v) in &manifest.drivers { - all_entries.entry(k.clone()).or_insert(v); - } + let all_entries = manifest.hardware_entries(); if all_entries.is_empty() { return CheckResult { @@ -1209,7 +1201,7 @@ fn check_drivers(ctx: &dispatch::ProjectContext) -> CheckResult { let urdf_path = std::path::Path::new(desc); let sensors = crate::urdf::extract_sensors_from_urdf(urdf_path); if !sensors.is_empty() { - let hw_keys: Vec<&str> = all_entries.keys().map(|s| s.as_str()).collect(); + let hw_keys: Vec<&str> = all_entries.keys().copied().collect(); let warnings = crate::urdf::validate_driver_keys(&hw_keys, &sensors); for warning in warnings { details.push(format!(" ! {}", warning)); diff --git a/horus_manager/src/commands/run/run_python.rs b/horus_manager/src/commands/run/run_python.rs index 22c7373d3..307882343 100644 --- a/horus_manager/src/commands/run/run_python.rs +++ b/horus_manager/src/commands/run/run_python.rs @@ -53,7 +53,7 @@ pub(super) fn execute_python_node(file: PathBuf, args: Vec, _release: bo // Propagate the child's exit code as an error if !status.success() { - let code = status.code().unwrap_or(1); + let code = crate::error_wrapper::shell_exit_code(&status); crate::error_wrapper::emit_diagnostics(&crate::error_wrapper::exit_code_hint( "python", code, )); @@ -77,7 +77,7 @@ pub(super) fn execute_python_node(file: PathBuf, args: Vec, _release: bo // Propagate the child's exit code as an error if !status.success() { - let code = status.code().unwrap_or(1); + let code = crate::error_wrapper::shell_exit_code(&status); crate::error_wrapper::emit_diagnostics(&crate::error_wrapper::exit_code_hint( "python", code, )); diff --git a/horus_manager/src/commands/run/run_rust.rs b/horus_manager/src/commands/run/run_rust.rs index 6dcc489ef..e05548d1b 100644 --- a/horus_manager/src/commands/run/run_rust.rs +++ b/horus_manager/src/commands/run/run_rust.rs @@ -509,7 +509,7 @@ pub(super) fn execute_from_cargo_toml( cmd.envs(child_env.iter().cloned()); let status = cmd.status()?; if !status.success() { - let code = status.code().unwrap_or(1); + let code = crate::error_wrapper::shell_exit_code(&status); crate::error_wrapper::emit_diagnostics(&crate::error_wrapper::exit_code_hint( "rust", code, )); @@ -862,7 +862,7 @@ pub(super) fn execute_with_scheduler( let status = cmd.status()?; if !status.success() { - let code = status.code().unwrap_or(1); + let code = crate::error_wrapper::shell_exit_code(&status); crate::error_wrapper::emit_diagnostics(&crate::error_wrapper::exit_code_hint( "cargo", code, )); @@ -888,7 +888,7 @@ pub(super) fn execute_with_scheduler( let status = cmd.status()?; if !status.success() { - let code = status.code().unwrap_or(1); + let code = crate::error_wrapper::shell_exit_code(&status); crate::error_wrapper::emit_diagnostics(&crate::error_wrapper::exit_code_hint( "rust", code, )); @@ -964,7 +964,7 @@ pub(super) fn execute_with_scheduler( let status = cmd.status()?; if !status.success() { - let code = status.code().unwrap_or(1); + let code = crate::error_wrapper::shell_exit_code(&status); crate::error_wrapper::emit_diagnostics(&crate::error_wrapper::exit_code_hint( "cargo", code, )); @@ -989,7 +989,7 @@ pub(super) fn execute_with_scheduler( let status = cmd.status()?; if !status.success() { - let code = status.code().unwrap_or(1); + let code = crate::error_wrapper::shell_exit_code(&status); crate::error_wrapper::emit_diagnostics(&crate::error_wrapper::exit_code_hint( "rust", code, )); diff --git a/horus_manager/src/error_wrapper.rs b/horus_manager/src/error_wrapper.rs index 1b157d57e..36ffb126d 100644 --- a/horus_manager/src/error_wrapper.rs +++ b/horus_manager/src/error_wrapper.rs @@ -840,6 +840,26 @@ pub fn python_error_hint(stderr: &str) -> Vec { // ── Exit code hints ───────────────────────────────────────────────────────── +/// The shell-convention exit code for a finished process. +/// +/// `ExitStatus::code()` returns `None` on Unix when a signal killed the child, +/// so `code().unwrap_or(1)` collapses every signal death into the one code +/// `exit_code_hint` deliberately says nothing about. That silently disabled the +/// 137/139/128+N arms below in every real run — including the OOM hint, which is +/// the most useful one this module has. Shells report a signal death as 128+N; +/// so do we, and the hints become reachable. +pub fn shell_exit_code(status: &std::process::ExitStatus) -> i32 { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + if let Some(signal) = status.signal() { + return 128 + signal; + } + } + // Windows has no signals: code() is always Some for a finished process. + status.code().unwrap_or(1) +} + /// Interpret a process exit code and return diagnostics for non-obvious codes. /// /// Common Unix exit codes and signal-based exits (128+N) are translated into @@ -1997,6 +2017,48 @@ mod tests { assert!(exit_code_hint("python", 0).is_empty()); } + /// The 137/139/128+N arms are worth nothing if no caller can ever produce + /// those numbers. Every one of them used to read `status.code().unwrap_or(1)`, + /// which on Unix is `None` -> 1 for a signal death, so a genuinely + /// OOM-killed child got the "too generic, no hint" branch. Kill a real + /// process and walk the actual production path: status -> code -> hint. + #[test] + #[cfg(unix)] + fn a_signal_killed_child_reaches_the_oom_hint() { + let status = std::process::Command::new("sh") + .arg("-c") + .arg("kill -9 $$") + .status() + .expect("sh should be available on any unix host"); + assert!(!status.success()); + assert_eq!( + status.code(), + None, + "a SIGKILLed child reports no exit code - that is the whole problem" + ); + + let code = shell_exit_code(&status); + assert_eq!(code, 137, "SIGKILL is signal 9, so the shell code is 128+9"); + + let diags = exit_code_hint("python", code); + assert!( + diags.iter().any(|d| d.code == "H053"), + "an OOM-killed child must get the OOM hint, got {:?}", + diags.iter().map(|d| &d.code).collect::>() + ); + } + + /// A plain non-zero exit must keep going through unchanged. + #[test] + fn shell_exit_code_passes_an_ordinary_failure_through() { + let status = std::process::Command::new("sh") + .arg("-c") + .arg("exit 3") + .status() + .expect("sh should be available"); + assert_eq!(shell_exit_code(&status), 3); + } + // ── CMake error hints ─────────────────────────────────────────────── #[test] diff --git a/horus_manager/src/manifest.rs b/horus_manager/src/manifest.rs index 8c0f96c42..d595a9a49 100644 --- a/horus_manager/src/manifest.rs +++ b/horus_manager/src/manifest.rs @@ -1267,6 +1267,28 @@ fn warn_unknown_keys_once(path: &Path, content: &str) { // ─── HorusManifest: loading ───────────────────────────────────────────────── impl HorusManifest { + /// Every hardware entry: `[hardware]` plus the legacy `[drivers]` behind it. + /// + /// `[hardware]` is the preferred spelling and `[drivers]` the compatibility + /// one, so a consumer that reads only `drivers` makes the *preferred* + /// spelling do strictly less than the legacy one. That is what happened: + /// `doctor` and `native_sync` merged the two, while `cargo_gen` and + /// `pyproject_gen` read `drivers` alone, so a driver declared under + /// `[hardware]` generated no Cargo dependency and no PyPI dependency. + /// Everything that wants "the hardware the project declares" goes through + /// here so the two spellings cannot drift apart again. + /// + /// A name present in both resolves to the `[hardware]` entry. + pub fn hardware_entries(&self) -> BTreeMap<&str, &DriverValue> { + let mut merged: BTreeMap<&str, &DriverValue> = BTreeMap::new(); + for (name, value) in &self.hardware { + merged.insert(name.as_str(), value); + } + for (name, value) in &self.drivers { + merged.entry(name.as_str()).or_insert(value); + } + merged + } /// Load manifest from a TOML file path. /// /// Warns on stderr about keys HORUS does not understand, once per file per diff --git a/horus_manager/src/native_sync.rs b/horus_manager/src/native_sync.rs index 9c32dfad7..d24d470c3 100644 --- a/horus_manager/src/native_sync.rs +++ b/horus_manager/src/native_sync.rs @@ -312,9 +312,7 @@ fn deps_equal(a: &DependencyValue, b: &DependencyValue) -> bool { /// normal dependency. Only package drivers generate Cargo deps. fn collect_driver_crates(manifest: &HorusManifest) -> Vec { let mut crates = Vec::new(); - // Check both [hardware] and legacy [drivers] - let all_drivers = manifest.hardware.values().chain(manifest.drivers.values()); - for driver_val in all_drivers { + for driver_val in manifest.hardware_entries().into_values() { if let crate::manifest::DriverValue::Config(cfg) = driver_val { if let Some(package) = &cfg.package { crates.push(package.clone()); diff --git a/horus_manager/src/pyproject_gen.rs b/horus_manager/src/pyproject_gen.rs index a4ebaf9f5..09987026f 100644 --- a/horus_manager/src/pyproject_gen.rs +++ b/horus_manager/src/pyproject_gen.rs @@ -73,8 +73,8 @@ pub fn generate( // ── Dependencies ───────────────────────────────────────────────────── let mut pypi_deps = collect_python_deps(&manifest.dependencies); - // Add PyPI driver dependencies from [drivers] - for value in manifest.drivers.values() { + // Add PyPI driver dependencies from [hardware]/[drivers] + for value in manifest.hardware_entries().into_values() { if let crate::manifest::DriverValue::Config(cfg) = value { if let Some(ref pip_name) = cfg.pip { let version = cfg From b9841bd73498a9e3ae89070121a3153b0be277b0 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 12:29:48 -0700 Subject: [PATCH 39/79] fix: --sim did nothing at all on the Python driver path horus_core's own comment states the failure this guards against: "the entry constructed the REAL driver under `horus run --sim`, which is the exact failure `--sim` exists to prevent". load_from honours that. load_config_entries - which is the whole of the Python path - had no simulation handling of any kind, so horus.drivers.load() instantiated the registered driver class for every entry, `sim = true` or not. Constructing a driver is where the serial port is opened and the motor enabled, so `--sim` was inert in Python while the same manifest under Rust got a stub. One predicate now backs both loaders: entry_is_simulated() folds the sim/simulated alias, HORUS_SIM_MODE and HORUS_SIM_TARGETS together. load_from substitutes SimStubNode as before; load_config_entries reports the verdict as a fourth tuple element, and horus_py returns a SimStub - never calling the driver class - carrying the entry's params so simulation code can still read the config. sim_mode_enabled() was a copy of the shipped predicate living inside `#[cfg(test)] mod sim_mode_tests`, so its truth table could stop matching load_from without a test going red. It is now the function that ships, and the tests import it. The new Rust test loads one manifest through both loaders and requires a per-entry verdict match, rather than asserting each in isolation - that is the property that was broken. The Python test registers a driver that records construction and asserts it is never built. --- horus_core/src/drivers/mod.rs | 126 +++++++++++------- horus_core/src/drivers/tests.rs | 122 +++++++++++++++++ horus_py/src/drivers.rs | 61 ++++++++- .../test_sim_mode_blocks_real_drivers.py | 109 +++++++++++++++ 4 files changed, 371 insertions(+), 47 deletions(-) create mode 100644 horus_py/tests/test_sim_mode_blocks_real_drivers.py diff --git a/horus_core/src/drivers/mod.rs b/horus_core/src/drivers/mod.rs index 1db43ddcc..4e654a330 100644 --- a/horus_core/src/drivers/mod.rs +++ b/horus_core/src/drivers/mod.rs @@ -94,6 +94,54 @@ fn asks_for_simulation(config: &toml::value::Table) -> bool { .unwrap_or(false) } +/// Whether a `HORUS_SIM_MODE` value asks for simulation. +/// +/// Presence alone used to mean "on", so the two most natural ways to say *off* +/// both turned simulation on. A deploy script setting `HORUS_SIM_MODE=0` to +/// force real hardware silently got inert stubs: actuators never commanded, +/// sensors never read, nothing logged. Parse the value, the way +/// `HORUS_NET_ENABLED` already does. +/// +/// A free function taking the value so the truth table can be tested without +/// racing on the process-global environment. It used to be a *copy* of this +/// predicate living inside `#[cfg(test)] mod sim_mode_tests`, which is a thing +/// that can silently stop matching what ships. +fn sim_mode_enabled(value: Option<&str>) -> bool { + value + .map(|v| !(v.is_empty() || v == "0" || v.eq_ignore_ascii_case("false"))) + .unwrap_or(false) +} + +/// Whether this process is running under `horus run --sim`. +fn sim_mode_active() -> bool { + sim_mode_enabled(std::env::var("HORUS_SIM_MODE").ok().as_deref()) +} + +/// The `HORUS_SIM_TARGETS` filter, or `None` for "every entry that asks". +fn sim_targets() -> Option> { + std::env::var("HORUS_SIM_TARGETS") + .ok() + .map(|s| s.split(',').map(String::from).collect()) +} + +/// Whether this named entry gets a stub in this process. +/// +/// The one predicate behind both loaders. `load_from` substitutes a +/// `SimStubNode`; `load_config_entries` reports it to its caller. They used to +/// disagree: `load_config_entries` had no simulation handling at all, so the +/// Python path built the REAL driver under `horus run --sim` -- the exact +/// failure `--sim` exists to prevent, and the one `asks_for_simulation` +/// already documents. +fn entry_is_simulated(name: &str, config: &toml::value::Table) -> bool { + if !asks_for_simulation(config) || !sim_mode_active() { + return false; + } + match sim_targets() { + Some(targets) => targets.iter().any(|t| t == name), + None => true, // no filter = all sim targets + } +} + /// Load hardware nodes from a specific config file. /// /// Useful for testing with alternate configs or multi-robot setups. @@ -113,23 +161,6 @@ pub fn load_from>(path: P) -> HorusResult> = if sim_mode { - std::env::var("HORUS_SIM_TARGETS") - .ok() - .map(|s| s.split(',').map(String::from).collect()) - } else { - None - }; - // Reserved keys that are NOT passed as NodeParams const RESERVED: &[&str] = &[ "use", @@ -157,23 +188,15 @@ pub fn load_from>(path: P) -> HorusResult targets.iter().any(|t| t == name), - None => true, // no filter = all sim targets - }; - if should_sim { - log::info!("hardware.{name}: simulation mode — using stub"); - nodes.push(( - name.clone(), - Box::new(SimStubNode { - node_name: format!("{name}_sim_stub"), - }), - )); - continue; - } + if entry_is_simulated(name, config) { + log::info!("hardware.{name}: simulation mode — using stub"); + nodes.push(( + name.clone(), + Box::new(SimStubNode { + node_name: format!("{name}_sim_stub"), + }), + )); + continue; } // Determine the node type name from 'use' field (new) or legacy source keys @@ -308,9 +331,21 @@ pub fn robot_name() -> HorusResult> { robot_name_from(find_manifest()?) } +/// Parse `[hardware]` into `(name, use_name, params, simulated)` tuples. +/// +/// The non-instantiating half of [`load_from`]: it resolves the same key +/// fallback chain and the same simulation predicate, but hands the decision +/// back instead of constructing a node. The Python binding uses it, which is +/// why `simulated` is reported rather than acted on -- a Python driver class +/// is not a `Box` this crate can substitute a stub for. +/// +/// `simulated` is true when the entry asks for it (`sim` or its alias +/// `simulated`) AND this process runs under `horus run --sim`. It used to be +/// absent entirely, so `hardware.load()` in Python built the REAL driver under +/// `--sim` while the same manifest under Rust got a stub. pub fn load_config_entries>( path: P, -) -> HorusResult> { +) -> HorusResult> { let path = path.as_ref(); let content = std::fs::read_to_string(path) .map_err(|e| ConfigError::Other(format!("failed to read {}: {}", path.display(), e)))?; @@ -395,7 +430,12 @@ pub fn load_config_entries>( .map(|(k, v)| (k.clone(), v.clone())) .collect(); - entries.push((name.clone(), use_name, NodeParams::new(param_map))); + entries.push(( + name.clone(), + use_name, + NodeParams::new(param_map), + entry_is_simulated(name, config), + )); } Ok(entries) @@ -478,16 +518,10 @@ mod sim_target_tests { #[cfg(test)] mod sim_mode_tests { - /// Mirrors the `HORUS_SIM_MODE` predicate in `load_from`. - /// - /// Kept as a free function so the truth table can be tested without racing - /// on the process-global environment, which every other test in this crate - /// also reads. - fn sim_mode_enabled(value: Option<&str>) -> bool { - value - .map(|v| !(v.is_empty() || v == "0" || v.eq_ignore_ascii_case("false"))) - .unwrap_or(false) - } + // The predicate under test is the one that ships. It used to be a private + // copy declared right here, which could stop matching `load_from` without + // a single test going red. + use super::sim_mode_enabled; /// Presence alone used to mean "on", so the two most natural ways to say /// *off* both turned simulation on. A deploy script setting diff --git a/horus_core/src/drivers/tests.rs b/horus_core/src/drivers/tests.rs index 362244738..78393e339 100644 --- a/horus_core/src/drivers/tests.rs +++ b/horus_core/src/drivers/tests.rs @@ -319,6 +319,128 @@ sim = true std::fs::remove_file(&path).ok(); } + /// The two loaders must agree about what `--sim` covers. + /// + /// `load_from` substituted a stub while `load_config_entries` -- the whole + /// of the Python path -- reported nothing, so `hardware.load()` in Python + /// constructed the REAL driver under `horus run --sim`. Constructing a + /// driver is where the port is opened and the motor enabled, so `--sim` + /// was inert on that path. Load one manifest through both and require the + /// same verdict per entry. + #[test] + fn both_loaders_agree_on_what_sim_mode_covers() { + let _guard = SIM_ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + // SAFETY: guarded by SIM_ENV_LOCK to prevent parallel env var mutation + unsafe { + std::env::set_var("HORUS_SIM_MODE", "1"); + } + + registry::register("AgreementReal", |p| Ok(Box::new(StubNode::from_params(p)?))); + + let content = r#" +[hardware.simmed] +use = "AgreementReal" +sim = true + +[hardware.aliased] +use = "AgreementReal" +simulated = true + +[hardware.real] +use = "AgreementReal" +"#; + let path = write_temp_toml(content); + + let nodes = crate::drivers::load_from(&path).unwrap(); + let entries = crate::drivers::load_config_entries(&path).unwrap(); + + for (name, node) in &nodes { + let stubbed_by_load = node.name().contains("sim_stub"); + let (_, _, _, simulated) = entries + .iter() + .find(|(n, _, _, _)| n == name) + .unwrap_or_else(|| panic!("load_config_entries dropped {name}")); + assert_eq!( + stubbed_by_load, *simulated, + "the two loaders disagree about {name}: load_from stubbed={stubbed_by_load}, \ + load_config_entries simulated={simulated}" + ); + } + + // And the verdicts themselves are the right ones. + let simulated_names: Vec<&str> = entries + .iter() + .filter(|(_, _, _, sim)| *sim) + .map(|(n, _, _, _)| n.as_str()) + .collect(); + assert_eq!(simulated_names, vec!["aliased", "simmed"]); + + unsafe { + std::env::remove_var("HORUS_SIM_MODE"); + } + std::fs::remove_file(&path).ok(); + } + + /// Outside `--sim`, `sim = true` changes nothing on either path. + #[test] + fn config_entries_report_no_simulation_without_sim_mode() { + let _guard = SIM_ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + unsafe { + std::env::remove_var("HORUS_SIM_MODE"); + } + + let content = r#" +[hardware.sensor] +use = "AnythingAtAll" +sim = true +"#; + let path = write_temp_toml(content); + let entries = crate::drivers::load_config_entries(&path).unwrap(); + assert_eq!(entries.len(), 1); + assert!( + !entries[0].3, + "`sim = true` alone must not simulate - only `--sim` does" + ); + std::fs::remove_file(&path).ok(); + } + + /// HORUS_SIM_TARGETS must narrow both loaders the same way. + #[test] + fn config_entries_honour_the_selective_target_filter() { + let _guard = SIM_ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + unsafe { + std::env::set_var("HORUS_SIM_MODE", "1"); + std::env::set_var("HORUS_SIM_TARGETS", "lidar"); + } + + let content = r#" +[hardware.lidar] +use = "SomeType" +sim = true + +[hardware.imu] +use = "SomeType" +sim = true +"#; + let path = write_temp_toml(content); + let entries = crate::drivers::load_config_entries(&path).unwrap(); + let sim_of = |want: &str| { + entries + .iter() + .find(|(n, _, _, _)| n == want) + .map(|(_, _, _, s)| *s) + .unwrap() + }; + assert!(sim_of("lidar"), "lidar is in HORUS_SIM_TARGETS"); + assert!(!sim_of("imu"), "imu is not in HORUS_SIM_TARGETS"); + + unsafe { + std::env::remove_var("HORUS_SIM_MODE"); + std::env::remove_var("HORUS_SIM_TARGETS"); + } + std::fs::remove_file(&path).ok(); + } + #[test] fn load_exec_prefix() { let content = r#" diff --git a/horus_py/src/drivers.rs b/horus_py/src/drivers.rs index 4f55da9ff..81998cf3d 100644 --- a/horus_py/src/drivers.rs +++ b/horus_py/src/drivers.rs @@ -16,11 +16,53 @@ use crate::driver_params::PyDriverParams; static PY_DRIVERS: LazyLock>>> = LazyLock::new(|| Mutex::new(HashMap::new())); +// ── Simulation stub ──────────────────────────────────────────────────── + +/// Stand-in for a driver that `horus run --sim` says must not touch hardware. +/// +/// The Python twin of `horus_core`'s `SimStubNode`. Under `--sim`, the Rust +/// loader substitutes a no-op node for every entry marked `sim = true`; the +/// Python loader used to instantiate the registered driver class anyway, +/// because `load_config_entries` reported no simulation state. Constructing a +/// driver is where the serial port gets opened and the motor gets enabled, so +/// `--sim` did nothing at all on the Python path. +/// +/// Keeps the entry's params so simulation code can still read the config. +#[pyclass(name = "SimStub")] +pub struct PySimStub { + /// The `[hardware.]` key this stands in for, suffixed `_sim_stub`. + #[pyo3(get)] + name: String, + /// The config the real driver would have received. + #[pyo3(get)] + params: Py, +} + +#[pymethods] +impl PySimStub { + /// No-op: a stub has no hardware to bring up. + fn init(&self) {} + + /// No-op: a stub has no hardware to drive. + fn tick(&self) {} + + /// Always true — lets user code branch on "am I simulated?". + #[getter] + fn simulated(&self) -> bool { + true + } + + fn __repr__(&self) -> String { + format!("SimStub(name={:?})", self.name) + } +} + // ── Module functions ─────────────────────────────────────────────────── /// Load hardware config from ``horus.toml`` ``[hardware]`` section. /// /// Returns a list of ``(name, obj)`` tuples where ``obj`` is either: +/// - A ``SimStub`` (if ``horus run --sim`` says this entry is simulated) /// - A Python object (if a matching class was registered via ``register()``) /// - A ``NodeParams`` dict-like (if no Python class was registered) /// @@ -67,7 +109,23 @@ pub fn load_from(py: Python<'_>, path: &str) -> PyResult) let mut result = Vec::new(); - for (name, use_name, params) in entries { + for (name, use_name, params, simulated) in entries { + // `--sim` says this entry must not touch hardware. Constructing the + // registered class is exactly where it would, so stop before that — + // the same substitution horus_core makes for a Rust node. + if simulated { + let py_params = Py::new(py, PyDriverParams { inner: params })?; + let stub = Py::new( + py, + PySimStub { + name: format!("{name}_sim_stub"), + params: py_params, + }, + )?; + result.push((name, stub.into_any())); + continue; + } + // Try to instantiate from Python registry let cls = PY_DRIVERS .lock() @@ -112,6 +170,7 @@ pub fn register_drivers_module(parent: &Bound<'_, PyModule>) -> PyResult<()> { hardware.add_function(wrap_pyfunction!(robot_name, &hardware)?)?; hardware.add_function(wrap_pyfunction!(robot_name_from, &hardware)?)?; hardware.add_class::()?; + hardware.add_class::()?; hardware.setattr( "__doc__", diff --git a/horus_py/tests/test_sim_mode_blocks_real_drivers.py b/horus_py/tests/test_sim_mode_blocks_real_drivers.py new file mode 100644 index 000000000..af25eba54 --- /dev/null +++ b/horus_py/tests/test_sim_mode_blocks_real_drivers.py @@ -0,0 +1,109 @@ +"""`horus run --sim` must not construct a real driver on the Python path. + +The Rust loader substitutes a stub for every `[hardware]` entry marked +`sim = true`; the Python loader used to instantiate the registered driver +class regardless, because the shared parse reported no simulation state. +Constructing a driver is where the serial port is opened and the motor is +enabled, so `--sim` did nothing at all here. +""" + +import os + +import pytest + +from horus import drivers as hardware + + +CONSTRUCTED = [] + + +class RecordingDriver: + """Stands in for a driver whose __init__ touches hardware.""" + + def __init__(self, params): + CONSTRUCTED.append(params) + self.params = params + + +@pytest.fixture(autouse=True) +def _clean_env(): + CONSTRUCTED.clear() + saved = {k: os.environ.get(k) for k in ("HORUS_SIM_MODE", "HORUS_SIM_TARGETS")} + yield + for key, value in saved.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def _manifest(tmp_path, body): + path = tmp_path / "horus.toml" + path.write_text(body) + return str(path) + + +def test_sim_mode_returns_a_stub_instead_of_the_driver(tmp_path): + hardware.register_driver("RecordingDriver", RecordingDriver) + path = _manifest( + tmp_path, + '[hardware.arm]\nuse = "RecordingDriver"\nsim = true\nport = "/dev/ttyACM0"\n', + ) + + os.environ["HORUS_SIM_MODE"] = "1" + loaded = hardware.load_from(path) + + assert len(loaded) == 1 + name, node = loaded[0] + assert name == "arm" + assert CONSTRUCTED == [], "--sim must not construct the real driver" + assert type(node).__name__ == "SimStub" + assert node.simulated is True + assert node.name == "arm_sim_stub" + # The stub still carries the config, so simulation code can read it. + assert node.params.get("port") == "/dev/ttyACM0" + node.init() + node.tick() + + +def test_without_sim_mode_the_real_driver_is_constructed(tmp_path): + hardware.register_driver("RecordingDriver", RecordingDriver) + path = _manifest( + tmp_path, '[hardware.arm]\nuse = "RecordingDriver"\nsim = true\n' + ) + + os.environ.pop("HORUS_SIM_MODE", None) + loaded = hardware.load_from(path) + + assert len(CONSTRUCTED) == 1, "`sim = true` alone must not simulate" + assert isinstance(loaded[0][1], RecordingDriver) + + +def test_simulated_is_an_alias_of_sim_here_too(tmp_path): + hardware.register_driver("RecordingDriver", RecordingDriver) + path = _manifest( + tmp_path, + '[hardware.arm]\nuse = "RecordingDriver"\nsimulated = true\n', + ) + + os.environ["HORUS_SIM_MODE"] = "1" + loaded = hardware.load_from(path) + + assert CONSTRUCTED == [] + assert type(loaded[0][1]).__name__ == "SimStub" + + +def test_sim_targets_narrows_which_entries_are_stubbed(tmp_path): + hardware.register_driver("RecordingDriver", RecordingDriver) + path = _manifest( + tmp_path, + '[hardware.lidar]\nuse = "RecordingDriver"\nsim = true\n\n' + '[hardware.imu]\nuse = "RecordingDriver"\nsim = true\n', + ) + + os.environ["HORUS_SIM_MODE"] = "1" + os.environ["HORUS_SIM_TARGETS"] = "lidar" + by_name = dict(hardware.load_from(path)) + + assert type(by_name["lidar"]).__name__ == "SimStub" + assert isinstance(by_name["imu"], RecordingDriver) From 49828558a997ee5a8c6b57494621b798e0b14ede Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 12:32:22 -0700 Subject: [PATCH 40/79] fix(python): a Python doc example was compiled as Rust MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI caught it: `cargo test -p horus_py --doc` failed with error: expected one of `!` or `::`, found `.` The `provides_backpressure` doc-comment I added used a four-space indented block for its Python snippet, which is Markdown for a code block — and rustdoc compiles an unfenced code block in a doc comment as Rust. Fenced as ```python, matching the convention already used elsewhere in this file. I had not run doctests at any point on this branch; they are green across horus_core, horus_manager and horus_cpp as well now. --- horus_py/src/topic.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/horus_py/src/topic.rs b/horus_py/src/topic.rs index a4026b6ce..d3e09af62 100644 --- a/horus_py/src/topic.rs +++ b/horus_py/src/topic.rs @@ -1467,8 +1467,10 @@ impl PyTopic { /// first send — the backend is unresolved until a handle moves a message, /// and an unresolved backend answers False. /// - /// estop.send(EmergencyStop(True)) # resolves the backend - /// assert estop.provides_backpressure, estop.backend_type + /// ```python + /// estop.send(EmergencyStop(True)) # resolves the backend + /// assert estop.provides_backpressure, estop.backend_type + /// ``` /// /// Answered from `backend_type` through the runtime's own /// `backend_provides_backpressure`, so this cannot drift from what From ba88c7e4decf1d0d84b40d704429ba7136142c36 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 12:34:55 -0700 Subject: [PATCH 41/79] fix(install): the stale-prefix removal used a GNU-only regex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI's macOS parity job caught it: `a_changed_prefix_replaces_the_old_export` failed with both lines present in the rc — export HORUS_PREFIX="/old/location" export HORUS_PREFIX="/var/folders/.../opt-horus" `\?` is a GNU BRE extension. On the BSD grep and sed macOS ships, `^\(export \)\?HORUS_PREFIX=` matches nothing, so the installer never found the stale line and appended a second one beneath it. The first wins when the rc is sourced, so a user who moved their prefix kept pointing at the old tree — the precise failure the value-aware branch was added to prevent. POSIX ERE now, via `grep -E` and `sed -E`, keeping the `-i.horusbak || -i ''` fallback this file already uses for the PATH line and already proves on macOS. My own tools are GNU, so the broken form passed locally on every run. The platform matrix is what has this covered. --- install.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/install.sh b/install.sh index ea51ec4e3..2267cbaee 100755 --- a/install.sh +++ b/install.sh @@ -941,10 +941,17 @@ if [ -n "${HORUS_PREFIX:-}" ] && [ -n "$SHELL_RC" ] && [ -z "$TARGET_USER" ]; th *) # Value-aware: a stale line naming a DIFFERENT prefix must be # replaced, not treated as "already configured". - if grep -q "^\(export \)\?HORUS_PREFIX=" "$SHELL_RC" 2>/dev/null; then + # + # `grep -E` / `sed -E`, not a BRE with `\?`. That quantifier is a GNU + # extension: on the BSD tools macOS ships it matches nothing, so the + # stale line was never found and the installer appended a second + # HORUS_PREFIX line beneath it. The first one wins when the rc is + # sourced, so a moved prefix kept pointing at the old tree. CI's + # macOS parity job caught exactly that. + if grep -qE "^(export )?HORUS_PREFIX=" "$SHELL_RC" 2>/dev/null; then grep -qF "HORUS_PREFIX=\"${HORUS_PREFIX}\"" "$SHELL_RC" 2>/dev/null || { - sed -i.horusbak "\\|^\\(export \\)\\?HORUS_PREFIX=|d" "$SHELL_RC" 2>/dev/null || \ - sed -i '' "\\|^\\(export \\)\\?HORUS_PREFIX=|d" "$SHELL_RC" 2>/dev/null + sed -E -i.horusbak "/^(export )?HORUS_PREFIX=/d" "$SHELL_RC" 2>/dev/null || \ + sed -E -i '' "/^(export )?HORUS_PREFIX=/d" "$SHELL_RC" 2>/dev/null rm -f "${SHELL_RC}.horusbak" 2>/dev/null echo "export HORUS_PREFIX=\"${HORUS_PREFIX}\"" >> "$SHELL_RC" } From a4ee330864de0af5acfec71b46562b2a8b18c245 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 12:38:07 -0700 Subject: [PATCH 42/79] fix: a C++ log line reached horus log with no time on it Both C++ entry points build their LogEntry with `timestamp: String::new()` and the comment "filled by log_buffer if empty". publish_log filled nothing, so every line a C++ node logged sat in `horus log` with a blank time column while the identical hlog! call from Rust carried one - and blackbox events, which exist to be read back in order after an incident, were the worst case. publish_log now stamps an empty timestamp, which is what the comment already promised and covers any other caller that leaves it blank. A caller that supplies one keeps it, so hlog! still reports when the log statement ran rather than when it reached the buffer. The test compares against a real `log_as_node` entry rather than a hardcoded pattern, so the C++ and Rust stamps cannot drift into different shapes. --- horus_core/src/core/log_buffer.rs | 13 ++++++- horus_cpp/src/c_api.rs | 56 +++++++++++++++++++++++++++++-- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/horus_core/src/core/log_buffer.rs b/horus_core/src/core/log_buffer.rs index a866e171b..73fd05d83 100644 --- a/horus_core/src/core/log_buffer.rs +++ b/horus_core/src/core/log_buffer.rs @@ -595,7 +595,18 @@ lazy_static::lazy_static! { /// Error and Warning entries are dual-written to both the main buffer and the /// dedicated error buffer for persistent retention. Info/Debug/Publish/Subscribe /// entries go to the main buffer only (zero overhead on the hot sensor path). -pub fn publish_log(entry: LogEntry) { +/// +/// An empty `timestamp` is stamped here. Both C++ entry points -- `horus_log` +/// and `horus_blackbox_record` -- pass `String::new()` with the comment "filled +/// by log_buffer if empty", and nothing filled it: every line a C++ node logged +/// landed in `horus log` with a blank time column while the identical Rust call +/// carried one. Callers that already have a timestamp keep it, so `hlog!` still +/// reports the instant the log statement ran rather than the instant it reached +/// the buffer. +pub fn publish_log(mut entry: LogEntry) { + if entry.timestamp.is_empty() { + entry.timestamp = chrono::Local::now().format("%H:%M:%S%.3f").to_string(); + } if matches!(entry.log_type, LogType::Error | LogType::Warning) { GLOBAL_ERROR_BUFFER.push(entry.clone()); } diff --git a/horus_cpp/src/c_api.rs b/horus_cpp/src/c_api.rs index fa33e12b0..c0a05c067 100644 --- a/horus_cpp/src/c_api.rs +++ b/horus_cpp/src/c_api.rs @@ -2082,7 +2082,7 @@ pub unsafe extern "C" fn horus_blackbox_record(category: *const c_char, message: let text = format!("[blackbox] {}", msg); let log_type = horus_core::core::log_buffer::LogType::Warning; let entry = horus_core::core::log_buffer::LogEntry { - timestamp: String::new(), + timestamp: String::new(), // publish_log stamps it tick_number: 0, node_name: cat.to_string(), log_type: log_type.clone(), @@ -2130,7 +2130,7 @@ pub unsafe extern "C" fn horus_log(level: u8, node_name: *const c_char, message: _ => horus_core::core::log_buffer::LogType::Info, }; let entry = horus_core::core::log_buffer::LogEntry { - timestamp: String::new(), // filled by log_buffer if empty + timestamp: String::new(), // publish_log stamps it tick_number: 0, node_name: node.to_string(), log_type: log_type.clone(), @@ -2171,6 +2171,58 @@ mod tests { std::env::var_os("HORUS_CPP_FFI_EMIT_CHILD").is_some() } + /// A C++ log line must carry a time, like every other language's does. + /// + /// Both C++ entry points built their `LogEntry` with + /// `timestamp: String::new()` and the comment "filled by log_buffer if + /// empty". `publish_log` filled nothing, so `horus log` showed a blank time + /// column for every line a C++ node emitted while the identical `hlog!` + /// call from Rust carried one. Assert against a real Rust entry rather than + /// a hardcoded pattern, so the two cannot drift into different shapes. + #[test] + fn a_cpp_log_line_is_stamped_like_a_rust_one() { + use horus_core::core::log_buffer::{LogType, GLOBAL_LOG_BUFFER}; + + let node = format!("cpp_stamp_{}", std::process::id()); + let name = CString::new(node.clone()).unwrap(); + let msg = CString::new("from a C++ node").unwrap(); + unsafe { horus_log(0, name.as_ptr(), msg.as_ptr()) }; + + let cat = CString::new(node.clone()).unwrap(); + let event = CString::new("estop").unwrap(); + unsafe { horus_blackbox_record(cat.as_ptr(), event.as_ptr()) }; + + // The Rust side of the comparison, through the real Rust path. + let rust_node = format!("rust_stamp_{}", std::process::id()); + horus_core::core::hlog::log_as_node(LogType::Info, &rust_node, "from a Rust node"); + + let cpp = GLOBAL_LOG_BUFFER.for_node(&node); + assert_eq!(cpp.len(), 2, "both C++ entry points should have logged"); + let rust = GLOBAL_LOG_BUFFER.for_node(&rust_node); + assert_eq!(rust.len(), 1); + let reference = &rust[0].timestamp; + + for entry in &cpp { + assert!( + !entry.timestamp.is_empty(), + "a C++ log entry reached the buffer with no timestamp: {entry:?}" + ); + assert_eq!( + entry.timestamp.len(), + reference.len(), + "C++ stamp {:?} is not the shape Rust writes ({reference:?})", + entry.timestamp + ); + // HH:MM:SS.mmm — same separators in the same places. + let shape = |t: &str| { + t.chars() + .map(|c| if c.is_ascii_digit() { 'd' } else { c }) + .collect::() + }; + assert_eq!(shape(&entry.timestamp), shape(reference)); + } + } + /// Re-run `test_name` in a child process and hand back its stderr. fn child_stderr(test_name: &str) -> String { let exe = std::env::current_exe().expect("test binary path"); From 47250351bad2c4c3ee3437aa0cc74811a3a2a85f Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 12:49:35 -0700 Subject: [PATCH 43/79] fix(install): the Rust version gate failed closed when it could not compare MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `check_rust_version` decided the verdict with [ "$(printf '%s\n%s\n' "$required" "$found" | sort -V | head -n1)" != "$required" ] `sort -V` is not POSIX. Wherever it is missing or errors, the command substitution is empty, `"" != "$required"` is true, and the installer aborts with Rust 1.85 or newer is required; found 1.90.0 — convicting a toolchain that satisfies the floor, on a comparison it never actually performed, and telling the user to `rustup update` a Rust that is already new enough. The two guards immediately above it both `return 0` when they cannot determine an answer. This one was the odd one out, and it was the only one that could abort the install. Now compared in POSIX awk, field by numeric field: no `sort` dependency, and 1.100 still sorts above 1.9 as the comment always claimed. Three tests, driving the gate slice verbatim out of install.sh: - a `sort` stub that rejects -V must not abort a good toolchain - 1.70.0 against a 1.85 floor must still be rejected, so failing open does not become failing blind - 1.100 must beat 1.9 Ablated: restoring `sort -V` fails only the first, with the other two green. This is the same class as the previous commit, found by grepping every shell script for GNU-only constructs after that one. It is not a macOS report: the sort FreeBSD ships does implement -V. The defect is the fail-closed branch. --- .../tests/completion_install_contract.rs | 120 ++++++++++++++++++ install.sh | 23 +++- 2 files changed, 142 insertions(+), 1 deletion(-) diff --git a/horus_manager/tests/completion_install_contract.rs b/horus_manager/tests/completion_install_contract.rs index e4fdce6d0..be51f270a 100644 --- a/horus_manager/tests/completion_install_contract.rs +++ b/horus_manager/tests/completion_install_contract.rs @@ -704,3 +704,123 @@ fn the_prefix_export_uses_fish_syntax_in_config_fish() { "a POSIX export line stops config.fish loading at that point:\n{rc}" ); } + +/// The `check_rust_version` gate, verbatim from `install.sh`. +fn rust_gate_slice() -> String { + let src = fs::read_to_string(repo_root().join("install.sh")).expect("install.sh must exist"); + let begin = "check_rust_version() {"; + let end = " ok \"Rust $found (>= $required required)\"\n}"; + let b = src + .find(begin) + .unwrap_or_else(|| panic!("install.sh lost the marker {begin:?}")); + let e = src + .find(end) + .unwrap_or_else(|| panic!("install.sh lost the marker {end:?}")); + assert!(b < e, "install.sh markers are out of order"); + let slice = src[b..e + end.len()].to_string(); + // Without these the harness would be exercising an empty shell function and + // every assertion below would hold vacuously. + for needed in [ + "rust-version", + "rustc --version", + "$HORUS_SRC_DIR/Cargo.toml", + ] { + assert!( + slice.contains(needed), + "the check_rust_version slice no longer contains {needed:?} — this test would pass vacuously" + ); + } + slice +} + +/// Run the gate against a stub `rustc` reporting `rustc_version`, with the +/// workspace floor set to `required`. `sort_has_dash_v` chooses whether the +/// `sort` on PATH understands `-V`. Returns the gate's output plus a +/// PROCEEDED/ABORTED verdict. +fn run_rust_gate(rustc_version: &str, required: &str, sort_has_dash_v: bool) -> String { + let sb = Sandbox::new(); + let stub = sb.path().join("stub"); + fs::create_dir_all(&stub).unwrap(); + fs::create_dir_all(sb.path().join("src")).unwrap(); + fs::write( + sb.path().join("src/Cargo.toml"), + format!("[workspace.package]\nrust-version = \"{required}\"\n"), + ) + .unwrap(); + + let rustc = stub.join("rustc"); + fs::write( + &rustc, + format!("#!/bin/bash\necho 'rustc {rustc_version} (deadbeef 2026-01-01)'\n"), + ) + .unwrap(); + // A `sort` that rejects -V, the way any sort predating the GNU/BSD version + // extension does. The point is not one named platform: it is that the gate + // must not convict a toolchain on a comparison it was unable to perform. + if !sort_has_dash_v { + let sort = stub.join("sort"); + fs::write( + &sort, + "#!/bin/bash\nfor a in \"$@\"; do case \"$a\" in -V|--version-sort)\n\ + echo 'sort: illegal option -- V' >&2; exit 2 ;; esac; done\nexec /usr/bin/sort \"$@\"\n", + ) + .unwrap(); + set_exec(&sort); + } + set_exec(&rustc); + + let script = format!( + "set -u\n\ + PATH=\"$HOME/stub:$PATH\"\n\ + HORUS_SRC_DIR=\"$HOME/src\"\n\ + fail() {{ echo \"GATE_FAIL: $*\"; }}\n\ + ok() {{ echo \"GATE_OK: $*\"; }}\n\ + {}\n\ + if ( check_rust_version ); then echo PROCEEDED; else echo ABORTED; fi\n", + rust_gate_slice() + ); + sb.run(&script, "/bin/bash", true, &[]) +} + +fn set_exec(p: &Path) { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(p, fs::Permissions::from_mode(0o755)).unwrap(); + } +} + +#[test] +fn the_rust_version_gate_does_not_abort_when_it_cannot_compare() { + let out = run_rust_gate("1.90.0", "1.85", false); + assert!( + out.contains("PROCEEDED"), + "a `sort` without -V made the version gate abort an install on a toolchain \ + that satisfies the floor (1.90.0 >= 1.85). A gate that cannot perform its \ + comparison must fail open, as the two guards above it already do:\n{out}" + ); + assert!( + !out.contains("or newer is required"), + "the installer accused a perfectly good toolchain of being too old:\n{out}" + ); +} + +#[test] +fn the_rust_version_gate_still_rejects_a_genuinely_old_toolchain() { + let out = run_rust_gate("1.70.0", "1.85", true); + assert!( + out.contains("ABORTED") && out.contains("or newer is required"), + "failing open must not mean failing blind: 1.70.0 is below the 1.85 floor \ + and the gate has to say so:\n{out}" + ); +} + +#[test] +fn the_rust_version_gate_compares_numerically_not_lexically() { + let out = run_rust_gate("1.100.0", "1.9", true); + assert!( + out.contains("PROCEEDED"), + "1.100 is newer than 1.9; a string compare says otherwise and would reject \ + every toolchain once the minor version passes 9:\n{out}" + ); +} diff --git a/install.sh b/install.sh index 2267cbaee..22024e744 100755 --- a/install.sh +++ b/install.sh @@ -559,7 +559,28 @@ check_rust_version() { [ -n "$found" ] || return 0 # Compare as version numbers, not as strings: 1.100 is newer than 1.9. - if [ "$(printf '%s\n%s\n' "$required" "$found" | sort -V | head -n1)" != "$required" ]; then + # + # Done in awk rather than with `sort -V`, because this gate has to fail + # OPEN. `sort -V` is not POSIX; where it is missing or errors, the command + # substitution is empty, `"" != "$required"` is true, and the installer + # aborts telling the user their toolchain is too old — an accusation it has + # not actually established, about a machine that may well be fine. The two + # guards directly above deliberately return 0 when they cannot determine an + # answer; this one used to be the odd one out. POSIX awk is everywhere + # rustc is, and it compares field by field, so 1.100 > 1.9 as intended. + if ! awk -v have="$found" -v want="$required" ' + BEGIN { + nh = split(have, H, ".") + nw = split(want, W, ".") + n = (nh > nw ? nh : nw) + for (i = 1; i <= n; i++) { + h = (i <= nh ? H[i] + 0 : 0) + w = (i <= nw ? W[i] + 0 : 0) + if (h > w) exit 0 + if (h < w) exit 1 + } + exit 0 + }' /dev/null; then fail "Rust $required or newer is required; found $found." echo " Run: rustup update stable" exit 1 From d6fdf1c2339d51040d131aaecd7cfb15a8a50f6e Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 12:50:06 -0700 Subject: [PATCH 44/79] fix: scheduler introspection invented answers while the scheduler ran run() takes the native scheduler out of the PyScheduler to drive it (with_inner_run), so `inner == None` means exactly one thing: it is running right now. Eleven accessors answered that state with a fabricated value - zero nodes, tick 0, an empty name list, `false` for has_full_rt, the literal string "PythonScheduler", `None` for capabilities and safety_stats, and `false` for is_recording and is_replaying. None of those is distinguishable from a real observation, so anything polling from inside a tick read a scheduler that looked empty and idle while it was neither, and "not recording" came back for a session that was recording. get_node_stats in the same impl block already refused with "Stats unavailable while scheduler is running", every mutator says the same, and Invariant 3 in the module header states that a re-entrant call "will find self.inner == None and receive RuntimeError". The introspection accessors now say it too. status() is untouched: it returned "running" for this state all along and was the one that was right. The test polls each accessor from inside a live tick, which is the only place the bug was reachable, and checks they all answer normally once the run is over. --- horus_py/src/scheduler.rs | 58 +++++---- .../tests/test_introspection_never_invents.py | 115 ++++++++++++++++++ 2 files changed, 152 insertions(+), 21 deletions(-) create mode 100644 horus_py/tests/test_introspection_never_invents.py diff --git a/horus_py/src/scheduler.rs b/horus_py/src/scheduler.rs index 69bbf5230..45037ff66 100644 --- a/horus_py/src/scheduler.rs +++ b/horus_py/src/scheduler.rs @@ -859,6 +859,25 @@ impl PyScheduler { } } +/// `self.inner` is `None` for exactly one reason: `run()` took the +/// scheduler out to drive it (`with_inner_run`), so it is running right now. +/// +/// Introspection used to answer that state with an invented value -- zero +/// nodes, zero ticks, `false` for `is_recording`, the literal string +/// `"PythonScheduler"` -- which is indistinguishable from a real +/// observation. A node-health check polling from inside a tick therefore +/// read "no nodes, tick 0, not recording" off a scheduler that had all +/// three, and `is_recording() == false` on a session that was recording is +/// the kind of answer someone deletes data over. +/// +/// `get_node_stats` in this same block already refuses ("Stats unavailable +/// while scheduler is running"), Invariant 3 in the module header says a +/// re-entrant call "will find `self.inner == None` and receive +/// `RuntimeError`", and every mutator here says so too. This is that error. +fn unavailable_while_running(what: &str) -> pyo3::PyErr { + PyRuntimeError::new_err(format!("{what} unavailable while scheduler is running")) +} + #[pymethods] impl PyScheduler { #[new] @@ -1103,7 +1122,7 @@ impl PyScheduler { let removed = self.removed_nodes.lock().map_err(lock_poisoned)?; Ok(sched.node_list().len() - removed.len()) } - None => Ok(0), + None => Err(unavailable_while_running("Node count")), } } @@ -1115,7 +1134,7 @@ impl PyScheduler { let removed = self.removed_nodes.lock().map_err(lock_poisoned)?; Ok(sched.node_list().contains(&name) && !removed.contains(&name)) } - None => Ok(false), + None => Err(unavailable_while_running("The node list")), } } @@ -1131,7 +1150,7 @@ impl PyScheduler { .filter(|n| !removed.contains(n)) .collect()) } - None => Ok(Vec::new()), + None => Err(unavailable_while_running("The node list")), } } @@ -1147,7 +1166,7 @@ impl PyScheduler { } Ok(None) } - None => Ok(None), + None => Err(unavailable_while_running("Node info")), } } @@ -1167,10 +1186,9 @@ impl PyScheduler { /// Get runtime capabilities as a dict. fn capabilities(&self, py: Python) -> PyResult>> { let guard = self.inner.lock().map_err(lock_poisoned)?; - let inner = match guard.as_ref() { - Some(s) => s, - None => return Ok(None), - }; + let inner = guard + .as_ref() + .ok_or_else(|| unavailable_while_running("Capabilities"))?; match inner.capabilities() { Some(caps) => { let dict = PyDict::new(py); @@ -1189,17 +1207,16 @@ impl PyScheduler { let guard = self.inner.lock().map_err(lock_poisoned)?; match guard.as_ref() { Some(sched) => Ok(sched.has_full_rt()), - None => Ok(false), + None => Err(unavailable_while_running("RT capability")), } } /// Get RT degradations as a list of dicts. fn degradations(&self, py: Python) -> PyResult> { let guard = self.inner.lock().map_err(lock_poisoned)?; - let inner = match guard.as_ref() { - Some(s) => s, - None => return Ok(PyList::empty(py).into()), - }; + let inner = guard + .as_ref() + .ok_or_else(|| unavailable_while_running("Degradations"))?; let result = PyList::empty(py); for deg in inner.degradations() { let dict = PyDict::new(py); @@ -1216,7 +1233,7 @@ impl PyScheduler { let guard = self.inner.lock().map_err(lock_poisoned)?; match guard.as_ref() { Some(sched) => Ok(sched.current_tick()), - None => Ok(0), + None => Err(unavailable_while_running("The tick count")), } } @@ -1225,7 +1242,7 @@ impl PyScheduler { let guard = self.inner.lock().map_err(lock_poisoned)?; match guard.as_ref() { Some(sched) => Ok(sched.scheduler_name().to_string()), - None => Ok("PythonScheduler".to_string()), + None => Err(unavailable_while_running("The scheduler name")), } } @@ -1236,10 +1253,9 @@ impl PyScheduler { /// Get safety statistics as a dict. fn safety_stats(&self, py: Python) -> PyResult>> { let guard = self.inner.lock().map_err(lock_poisoned)?; - let inner = match guard.as_ref() { - Some(s) => s, - None => return Ok(None), - }; + let inner = guard + .as_ref() + .ok_or_else(|| unavailable_while_running("Safety stats"))?; match inner.safety_stats() { Some(stats) => { let dict = PyDict::new(py); @@ -1263,7 +1279,7 @@ impl PyScheduler { let guard = self.inner.lock().map_err(lock_poisoned)?; match guard.as_ref() { Some(sched) => Ok(sched.is_recording()), - None => Ok(false), + None => Err(unavailable_while_running("Recording state")), } } @@ -1272,7 +1288,7 @@ impl PyScheduler { let guard = self.inner.lock().map_err(lock_poisoned)?; match guard.as_ref() { Some(sched) => Ok(sched.is_replaying()), - None => Ok(false), + None => Err(unavailable_while_running("Replay state")), } } diff --git a/horus_py/tests/test_introspection_never_invents.py b/horus_py/tests/test_introspection_never_invents.py new file mode 100644 index 000000000..8b8c06983 --- /dev/null +++ b/horus_py/tests/test_introspection_never_invents.py @@ -0,0 +1,115 @@ +"""Scheduler introspection must not invent an answer while the scheduler runs. + +`run()` takes the native scheduler out of the PyScheduler to drive it, so every +accessor sees `inner == None` for the whole run. Those accessors used to answer +that state with a fabricated value — zero nodes, tick 0, `False` for +`is_recording()`, the literal `"PythonScheduler"` — indistinguishable from a +real observation. Anything polling from inside a tick read a scheduler that +looked empty and idle while it was neither. + +`get_node_stats` in the same class always refused with "Stats unavailable while +scheduler is running", and Invariant 3 in `scheduler.rs` says a re-entrant call +gets a `RuntimeError`. These assert the rest now agree. +""" + +import horus + + +def _drive(probe): + """Run a one-node scheduler, calling `probe(sched)` from inside a tick.""" + seen = [] + + def tick(node): + if not seen: + seen.append(probe(sched._scheduler)) + + node = horus.Node(name="probe_node", tick=tick, rate=200, order=0) + sched = horus.Scheduler(tick_rate=200.0) + sched.add(node) + sched.run(duration=0.05) + assert seen, "the probe never ran — the tick callback did not fire" + return seen[0] + + +def _refused(call): + def probe(native): + try: + return ("value", call(native)) + except RuntimeError as exc: + return ("raised", str(exc)) + + kind, payload = _drive(probe) + assert kind == "raised", ( + f"introspection returned {payload!r} from inside a running tick " + "instead of refusing" + ) + assert "while scheduler is running" in payload, payload + return payload + + +def test_node_count_is_not_reported_as_zero_while_running(): + _refused(lambda s: s.get_node_count()) + + +def test_node_names_are_not_reported_as_empty_while_running(): + _refused(lambda s: s.get_node_names()) + + +def test_has_node_does_not_deny_a_node_that_exists(): + _refused(lambda s: s.has_node("probe_node")) + + +def test_current_tick_is_not_reported_as_zero_while_running(): + _refused(lambda s: s.current_tick()) + + +def test_scheduler_name_is_not_a_hardcoded_string_while_running(): + _refused(lambda s: s.scheduler_name()) + + +def test_recording_state_is_not_reported_as_false_while_running(): + # The dangerous one: "not recording" on a session that is recording. + _refused(lambda s: s.is_recording()) + + +def test_replay_state_is_not_reported_as_false_while_running(): + _refused(lambda s: s.is_replaying()) + + +def test_rt_capability_is_not_denied_while_running(): + _refused(lambda s: s.has_full_rt()) + + +def test_degradations_are_not_reported_as_none_while_running(): + _refused(lambda s: s.degradations()) + + +def test_safety_stats_do_not_read_as_absent_while_running(): + _refused(lambda s: s.safety_stats()) + + +def test_capabilities_do_not_read_as_absent_while_running(): + _refused(lambda s: s.capabilities()) + + +def test_status_still_answers_honestly_while_running(): + """`status()` always said "running" here — it is the one that was right.""" + kind, payload = _drive(lambda s: ("value", s.status())[1] and ("value", s.status())) + assert kind == "value" + assert payload == "running" + + +def test_everything_answers_normally_once_the_run_is_over(): + node = horus.Node(name="after_run", tick=lambda n: None, rate=200, order=0) + sched = horus.Scheduler(tick_rate=200.0) + sched.add(node) + sched.run(duration=0.02) + + native = sched._scheduler + assert native.get_node_count() == 1 + assert native.get_node_names() == ["after_run"] + assert native.has_node("after_run") is True + assert native.is_recording() is False + assert native.is_replaying() is False + assert isinstance(native.scheduler_name(), str) + assert native.current_tick() >= 0 From d42d532b7d064bf3102372284a99c224d3f88288 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 12:57:49 -0700 Subject: [PATCH 45/79] fix: safing diagnostics were queued with nothing to drain them primitives.rs opened by saying honor_safe_state_request "is reached only from rt_executor". Its own doc-comment, forty lines down, says "every executor must call this at the top of its per-node pass" - and rt, compute, async and event all do. The header's conclusion followed from the false premise: "the drain half is started by RtExecutor::start ... so a line queued from here always has a drainer". start_rt_diag_drain() was called from start_pool and nowhere else, so a program whose nodes are all Compute (or Async, or Event) started no drain thread at all. rt_diag only queues into a static ring; without a drainer every safing line it wrote sat there until a later line lapped it. Nothing printed, and the drop counter that would have said so is itself only reported by the drain. Every executor's start() now starts the drain - idempotent, on the caller's thread, never on an RT thread, which is the constraint that put it in start_pool to begin with. The header says what the code does. The test runs a compute-only program in a child process and requires a queued line to appear on the child's stdout, because "was it printed?" is a question about a real process's output. --- horus_core/src/scheduling/async_executor.rs | 6 ++ horus_core/src/scheduling/compute_executor.rs | 70 +++++++++++++++++++ horus_core/src/scheduling/event_executor.rs | 6 ++ horus_core/src/scheduling/primitives.rs | 27 ++++--- horus_core/src/scheduling/rt_executor.rs | 14 ++-- 5 files changed, 110 insertions(+), 13 deletions(-) diff --git a/horus_core/src/scheduling/async_executor.rs b/horus_core/src/scheduling/async_executor.rs index f83a56d2a..c706af009 100644 --- a/horus_core/src/scheduling/async_executor.rs +++ b/horus_core/src/scheduling/async_executor.rs @@ -54,6 +54,12 @@ impl AsyncExecutor { ) -> Self { nodes.sort_by_key(|n| n.priority); + // `honor_safe_state_request` reports through `rt_diag`, so this + // executor needs the drain thread as much as the RT one does — without + // it, a safe-state panic on a async I/O node is queued into a ring nothing + // reads. Idempotent, and on the caller's thread. + super::rt_executor::start_rt_diag_drain(); + let handle = std::thread::Builder::new() .name("horus-async-io".to_string()) .spawn(move || Self::async_thread_main(nodes, running, tick_period, monitors)) diff --git a/horus_core/src/scheduling/compute_executor.rs b/horus_core/src/scheduling/compute_executor.rs index 397e01688..2177ab239 100644 --- a/horus_core/src/scheduling/compute_executor.rs +++ b/horus_core/src/scheduling/compute_executor.rs @@ -553,6 +553,12 @@ impl ComputeExecutor { ) -> Self { nodes.sort_by_key(|n| n.priority); + // `honor_safe_state_request` reports through `rt_diag`, so this + // executor needs the drain thread as much as the RT one does — without + // it, a safe-state panic on a compute node is queued into a ring nothing + // reads. Idempotent, and on the caller's thread. + super::rt_executor::start_rt_diag_drain(); + let handle = std::thread::Builder::new() .name("horus-compute".to_string()) .spawn(move || Self::compute_thread_main(nodes, running, tick_period, monitors)) @@ -969,6 +975,70 @@ mod tests { } } + /// A safing diagnostic from a non-RT executor must actually be printed. + /// + /// `honor_safe_state_request` reports through `rt_diag`, which only queues + /// into a static ring — a separate drain thread does the printing, and it + /// used to be started by `RtExecutor::start_pool` alone. A program whose + /// nodes are all Compute (or Async, or Event) therefore started no drainer, + /// so every safing line it queued sat in the ring until a later line + /// overwrote it. Nothing was printed and nothing said so. + /// + /// The drain writes to the process's real stdout, which libtest cannot + /// capture in-process, so this re-runs itself as a child and reads what the + /// child printed. + #[test] + fn a_compute_only_program_still_prints_its_safing_diagnostics() { + const CHILD: &str = "HORUS_COMPUTE_DIAG_CHILD"; + const MARKER: &str = "compute-drain-marker"; + + if std::env::var_os(CHILD).is_some() { + // In the child: start a compute executor, which is the only thing + // that should be needed for a queued diagnostic to reach stdout. + let running = Arc::new(AtomicBool::new(true)); + let count = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let exec = ComputeExecutor::start( + vec![make_compute_node("diag_node", count)], + running.clone(), + Duration::from_millis(5), + test_monitors(), + ); + + super::super::rt_executor::rt_diag(format_args!("{MARKER}")); + + // Longer than one RT_DIAG_POLL, so a running drainer has emitted it. + std::thread::sleep(Duration::from_millis(200)); + + running.store(false, Ordering::SeqCst); + let _ = exec.stop(); + return; + } + + let exe = std::env::current_exe().expect("test binary path"); + let out = std::process::Command::new(exe) + .args([ + "--exact", + "scheduling::compute_executor::tests::a_compute_only_program_still_prints_its_safing_diagnostics", + "--nocapture", + "--test-threads", + "1", + ]) + .env(CHILD, "1") + .output() + .expect("re-running the test binary should work"); + + let stdout = String::from_utf8_lossy(&out.stdout); + assert!( + stdout.contains("1 passed"), + "the child ran no test — was this test renamed?\n{stdout}" + ); + assert!( + stdout.contains(MARKER), + "a diagnostic queued by a compute-only program never reached stdout: \ + no drain thread was running.\n{stdout}" + ); + } + fn make_compute_node(name: &str, count: Arc) -> RegisteredNode { let node = CounterNode { name: name.to_string(), diff --git a/horus_core/src/scheduling/event_executor.rs b/horus_core/src/scheduling/event_executor.rs index 87a796075..5f5de045c 100644 --- a/horus_core/src/scheduling/event_executor.rs +++ b/horus_core/src/scheduling/event_executor.rs @@ -164,6 +164,12 @@ impl EventExecutor { running: Arc, monitors: SharedMonitors, ) -> Self { + // `honor_safe_state_request` reports through `rt_diag`, so this + // executor needs the drain thread as much as the RT one does — without + // it, a safe-state panic on an event node is queued into a ring nothing + // reads. Idempotent, and on the caller's thread. + super::rt_executor::start_rt_diag_drain(); + let mut handles = Vec::with_capacity(nodes.len()); let mut notifiers = Vec::with_capacity(nodes.len()); diff --git a/horus_core/src/scheduling/primitives.rs b/horus_core/src/scheduling/primitives.rs index 3fbaa7c63..8882c60b3 100644 --- a/horus_core/src/scheduling/primitives.rs +++ b/horus_core/src/scheduling/primitives.rs @@ -3,14 +3,20 @@ //! These are the building blocks that the scheduler composes for different //! execution strategies (sequential, parallel, future RT-thread, etc.). //! -//! # Everything above `run_tick` runs on an RT thread +//! # Everything above `run_tick` can run on an RT thread //! -//! `honor_safe_state_request` and `apply_degradation_action` are reached only -//! from `rt_executor` -- the scheduler keeps its own main-thread copy of the -//! degradation dispatch -- so they execute on a SCHED_FIFO thread, inside the -//! tick. They therefore report through `rt_diag`, which formats into a -//! statically allocated ring and never allocates, blocks or enters the kernel, -//! rather than through `print_line`. +//! `apply_degradation_action` is reached only from `rt_executor` -- the +//! scheduler keeps its own main-thread copy of the degradation dispatch -- so +//! it executes on a SCHED_FIFO thread, inside the tick. +//! +//! `honor_safe_state_request` is reached from ALL FOUR executors: its own +//! doc-comment says "every executor must call this at the top of its per-node +//! pass", and rt, compute, async and event all do. Only the RT one is +//! SCHED_FIFO, but the function is shared, so it reports the way the strictest +//! caller requires. +//! +//! That means `rt_diag`, which formats into a statically allocated ring and +//! never allocates, blocks or enters the kernel, rather than `print_line`. //! //! `print_line` does `isatty` + `tcgetattr`, takes the process-global stdout //! lock, then `write` and `flush`. Pointed at a slow consumer -- a serial @@ -20,8 +26,11 @@ //! or shut down the node in the same breath: the worst possible place to wait //! on a terminal. //! -//! The drain half is started by `RtExecutor::start` on the caller's thread -//! before any RT thread exists, so a line queued from here always has a drainer. +//! The drain half is started by every executor's `start`, on the caller's +//! thread before any executor thread exists, so a line queued from here always +//! has a drainer. It used to be started by `RtExecutor::start` alone, which +//! made that last sentence false for any program without RT nodes: safing +//! diagnostics went into the ring and nothing ever read them out. use std::time::{Duration, Instant}; diff --git a/horus_core/src/scheduling/rt_executor.rs b/horus_core/src/scheduling/rt_executor.rs index a3c441876..2d24c94ca 100644 --- a/horus_core/src/scheduling/rt_executor.rs +++ b/horus_core/src/scheduling/rt_executor.rs @@ -336,15 +336,21 @@ fn rt_diag_drain(emit: impl Fn(&str)) -> usize { /// Start the diagnostic drain thread, once per process. /// -/// Called from `start_pool`, on the CALLER's thread. Never from an RT thread: -/// spawning is a `clone(2)` plus a stack mapping, which is exactly the class of -/// cost this whole mechanism exists to keep off the tick loop. +/// Called from every executor's `start`, on the CALLER's thread. Never from an +/// RT thread: spawning is a `clone(2)` plus a stack mapping, which is exactly +/// the class of cost this whole mechanism exists to keep off the tick loop. +/// +/// Not just `start_pool`. `honor_safe_state_request` reports through `rt_diag` +/// and every executor calls it, so a program with no RT nodes at all queued +/// safing diagnostics -- including "PANICKED in enter_safe_state" -- into a +/// ring that nothing ever read. They were overwritten after `RT_DIAG_SLOTS` +/// lines and never printed. /// /// The thread runs for the life of the process. It is one wakeup every /// `RT_DIAG_POLL` and it must outlive any individual executor, because an /// executor that is being torn down is precisely when its last diagnostics /// matter. -fn start_rt_diag_drain() { +pub(crate) fn start_rt_diag_drain() { RT_DIAG_DRAIN_STARTED.call_once(|| { let spawned = std::thread::Builder::new() .name("horus-rt-diag".to_string()) From 7867ad2a2135511f6c0f37f2fc97c8004e559d20 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 12:58:22 -0700 Subject: [PATCH 46/79] fix(uninstall): the HORUS_PREFIX export survived the uninstall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit install.sh writes the prefix into the user's rc file in three places — two POSIX (`export HORUS_PREFIX="..."`) and one fish (`set -gx HORUS_PREFIX`). `clean_shell_profiles` removed none of them. It strips the completions block, `horus completion` lines and the env.sh integration, and stops there. That is not untidiness. install.sh:245 reads the variable back: if [ -n "${HORUS_PREFIX:-}" ]; then INSTALL_DIR="${HORUS_PREFIX}/bin" So the line the uninstaller leaves behind is exported by every new shell, and the NEXT install silently lands in the prefix this run just deleted, instead of the default location, with nothing on screen to say so. Uninstall, open a terminal, reinstall, and horus is back under /opt/horus. fish was worse: `clean_shell_profiles` walks .bashrc/.zshrc/.profile/ .bash_profile only. config.fish is not in that list, and the conf.d/horus.fish it does delete is a different file, so the fish line survived unconditionally. Both are removed now, POSIX and fish, with a backup taken first as the neighbouring blocks already do. Two details that were easy to get wrong: - `grep -E`/`sed -E`, not a `\?` BRE. That quantifier is GNU-only and matches nothing on the sed macOS ships — the cleanup would have silently no-opped on the platform where a leftover export is hardest to spot. Same defect as ba88c7e4, which is what prompted looking here. - an end-of-name boundary on the fish pattern. `HORUS_PREFIX=` is anchored by its `=`, but the fish form has no `=`, so without one a user's `set -gx HORUS_PREFIX_OTHER` matched too and was deleted out of a file the uninstaller does not own. Three tests: the POSIX line goes, the fish line goes, and neighbouring variables (HORUS_PREFIX_EXTRA, MY_HORUS_PREFIX, HORUS_PREFIX_OTHER) stay. Written before the fix and observed failing; dropping the boundary fails the third alone. --- .../tests/completion_install_contract.rs | 137 ++++++++++++++++++ uninstall.sh | 33 ++++- 2 files changed, 169 insertions(+), 1 deletion(-) diff --git a/horus_manager/tests/completion_install_contract.rs b/horus_manager/tests/completion_install_contract.rs index be51f270a..e1d229669 100644 --- a/horus_manager/tests/completion_install_contract.rs +++ b/horus_manager/tests/completion_install_contract.rs @@ -824,3 +824,140 @@ fn the_rust_version_gate_compares_numerically_not_lexically() { every toolchain once the minor version passes 9:\n{out}" ); } + +/// Drive the installer's tail with `HORUS_PREFIX` set, the way `--prefix` does. +fn install_with_prefix(sb: &Sandbox, prefix: &Path, shell: &str) -> String { + let script = format!( + "INSTALL_DIR=\"{prefix}/bin\"\n\ + HORUS_PREFIX=\"{prefix}\"\n\ + BINARY_NAME=horus\n\ + RED=''; GREEN=''; YELLOW=''; CYAN=''; BOLD=''; NC=''\n\ + info(){{ echo \" -> $1\"; }}\n\ + ok(){{ echo \" ok $1\"; }}\n\ + warn(){{ echo \" ! $1\"; }}\n\ + fail(){{ echo \" x $1\"; }}\n\ + INSTALL_START=$(date +%s)\n\ + VERSION=0.0.0-test\n\ + set -e\n\ + {tail}", + prefix = prefix.display(), + tail = install_tail() + ); + sb.run(&script, shell, false, &[]) +} + +/// Uninstalling must take the `HORUS_PREFIX` export back out of the rc file. +/// +/// It is not merely untidy to leave it. `install.sh:245` reads the variable: +/// +/// if [ -n "${HORUS_PREFIX:-}" ]; then INSTALL_DIR="${HORUS_PREFIX}/bin" +/// +/// so the line the uninstaller leaves behind is still exported by every new +/// shell, and the *next* install silently lands in the deleted prefix instead +/// of the default location — with nothing on screen to say so. Uninstall, +/// open a terminal, reinstall, and horus is back in /opt/horus. +#[test] +fn uninstalling_removes_the_horus_prefix_export_from_the_rc() { + let sb = Sandbox::new(); + let prefix = sb.path().join("opt-horus"); + fs::write(sb.path().join(".bashrc"), "# bashrc\n").unwrap(); + install_with_prefix(&sb, &prefix, "/bin/bash"); + let before = sb.read(".bashrc").unwrap_or_default(); + assert!( + before.contains("HORUS_PREFIX="), + "precondition: the installer should have written the export:\n{before}" + ); + + sb.uninstall(&[]); + + let after = sb.read(".bashrc").unwrap_or_default(); + assert!( + !after.contains("HORUS_PREFIX="), + "the uninstaller left the HORUS_PREFIX export behind, pointing at a \ + prefix it just deleted. Every later shell exports it, and the next \ + install reads it and reinstalls into the removed directory:\n{after}" + ); +} + +/// The same line, in fish's syntax, in the file fish actually reads. +/// +/// `clean_shell_profiles` walks `.bashrc`/`.zshrc`/`.profile`/`.bash_profile`; +/// `config.fish` is not among them, and the `conf.d/horus.fish` it does delete +/// is a different file. The installer writes `set -gx HORUS_PREFIX` into +/// `config.fish`, so for fish users the export survived the uninstall +/// regardless of how the POSIX side behaved. +#[test] +fn uninstalling_removes_the_fish_horus_prefix_export() { + let sb = Sandbox::new(); + let prefix = sb.path().join("opt-horus"); + let fish_dir = sb.path().join(".config/fish"); + fs::create_dir_all(&fish_dir).unwrap(); + fs::write(fish_dir.join("config.fish"), "# config.fish\n").unwrap(); + install_with_prefix(&sb, &prefix, "/usr/bin/fish"); + let before = sb.read(".config/fish/config.fish").unwrap_or_default(); + assert!( + before.contains("HORUS_PREFIX"), + "precondition: the installer should have written the fish export:\n{before}" + ); + + sb.uninstall(&[]); + + let after = sb.read(".config/fish/config.fish").unwrap_or_default(); + assert!( + !after.contains("HORUS_PREFIX"), + "the uninstaller left the fish HORUS_PREFIX line behind:\n{after}" + ); +} + +/// The cleanup deletes horus's line and nothing that merely looks like it. +/// +/// A line-deleting uninstaller is one missing anchor away from eating a +/// stranger's config. `HORUS_PREFIX=` is anchored by the `=`; the fish form has +/// no `=`, so without an explicit end-of-name boundary +/// `set -gx HORUS_PREFIX_OTHER` matches the same pattern and is silently +/// removed from a file the user has to notice is wrong on their own. +#[test] +fn the_prefix_cleanup_leaves_similarly_named_variables_alone() { + let sb = Sandbox::new(); + fs::write( + sb.path().join(".bashrc"), + "# bashrc\n\ + export HORUS_PREFIX=\"/opt/horus\"\n\ + export HORUS_PREFIX_EXTRA=\"keep me\"\n\ + export MY_HORUS_PREFIX=\"keep me too\"\n", + ) + .unwrap(); + let fish_dir = sb.path().join(".config/fish"); + fs::create_dir_all(&fish_dir).unwrap(); + fs::write( + fish_dir.join("config.fish"), + "# config.fish\n\ + set -gx HORUS_PREFIX \"/opt/horus\"\n\ + set -gx HORUS_PREFIX_OTHER \"keep me\"\n", + ) + .unwrap(); + + sb.uninstall(&[]); + + let rc = sb.read(".bashrc").unwrap_or_default(); + assert!( + !rc.contains("export HORUS_PREFIX=\""), + "horus's own line survived:\n{rc}" + ); + assert!( + rc.contains("HORUS_PREFIX_EXTRA") && rc.contains("MY_HORUS_PREFIX"), + "the uninstaller deleted a variable that only shares a prefix with \ + horus's, out of a file it does not own:\n{rc}" + ); + + let fish = sb.read(".config/fish/config.fish").unwrap_or_default(); + assert!( + !fish.contains("set -gx HORUS_PREFIX \""), + "horus's own fish line survived:\n{fish}" + ); + assert!( + fish.contains("HORUS_PREFIX_OTHER"), + "the fish cleanup has no end-of-name boundary and ate a neighbouring \ + variable:\n{fish}" + ); +} diff --git a/uninstall.sh b/uninstall.sh index aab9a96b1..4c648b913 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -964,7 +964,7 @@ fi # In a function, with markers, for the same reason as the completion # removal above: the regression test runs this exact text. clean_shell_profiles() { - local profile + local profile fish_config # Shell profiles: remove horus completion eval lines and shell integration for profile in "$HOME/.bashrc" "$HOME/.zshrc" "$HOME/.profile" "$HOME/.bash_profile"; do # The block install.sh's add_zsh_fpath_block() writes. It is delimited by @@ -1000,8 +1000,39 @@ clean_shell_profiles() { echo -e " ${GREEN}[+]${NC} Cleaned horus shell integration from $(basename $profile)" REMOVED=$((REMOVED + 1)) fi + # The `export HORUS_PREFIX=` line install.sh writes for a --prefix + # install. Leaving it is not untidiness: install.sh:245 reads the + # variable to pick INSTALL_DIR, so a line that survives the uninstall is + # still exported by every new shell and the NEXT install silently lands + # in the prefix this run just deleted. + # + # ERE via -E, not a `\?` BRE: that quantifier is a GNU extension and + # matches nothing on the sed macOS ships, which would leave the line in + # place on exactly the platform this cleanup is hardest to notice on. + if [ -f "$profile" ] && grep -qE "^(export )?HORUS_PREFIX=" "$profile" 2>/dev/null; then + cp "$profile" "${profile}.horus-backup" 2>/dev/null + sed -E -i.bak "/^(export )?HORUS_PREFIX=/d" "$profile" 2>/dev/null || \ + sed -E -i '' "/^(export )?HORUS_PREFIX=/d" "$profile" 2>/dev/null + rm -f "${profile}.bak" 2>/dev/null + echo -e " ${GREEN}[+]${NC} Cleaned HORUS_PREFIX export from $(basename $profile)" + REMOVED=$((REMOVED + 1)) + fi done + # fish keeps its own syntax in its own file, and the loop above does not + # walk it. install.sh writes `set -gx HORUS_PREFIX "..."` into config.fish; + # the conf.d/horus.fish removed below is a different file entirely, so for + # fish users the export survived no matter what the POSIX side did. + fish_config="$HOME/.config/fish/config.fish" + if [ -f "$fish_config" ] && grep -q "HORUS_PREFIX" "$fish_config" 2>/dev/null; then + cp "$fish_config" "${fish_config}.horus-backup" 2>/dev/null + sed -E -i.bak "/^[[:space:]]*set +(-[a-zA-Z]+ +)*HORUS_PREFIX([[:space:]]|$)/d" "$fish_config" 2>/dev/null || \ + sed -E -i '' "/^[[:space:]]*set +(-[a-zA-Z]+ +)*HORUS_PREFIX([[:space:]]|$)/d" "$fish_config" 2>/dev/null + rm -f "${fish_config}.bak" 2>/dev/null + echo -e " ${GREEN}[+]${NC} Cleaned HORUS_PREFIX from config.fish" + REMOVED=$((REMOVED + 1)) + fi + # Remove fish shell integration if [ -f "$HOME/.config/fish/conf.d/horus.fish" ]; then rm -f "$HOME/.config/fish/conf.d/horus.fish" From 94e5560867cb075bf98271df5d1e63c657565c46 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 13:36:36 -0700 Subject: [PATCH 47/79] fix(uninstall): a prefix install's PATH line outlived the uninstall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the rc-file symmetry started in 7867ad2a. install.sh writes four lines; the uninstaller now removes all four. Scoped deliberately to `$HORUS_PREFIX/bin`, matched as an exact string rather than by any pattern resembling `export PATH=`. A default install writes export PATH="$HOME/.local/bin:$PATH" for a directory pip, pipx, cargo and rustup also populate. Removing that line because horus once wrote it would take all of them off the user's PATH. A prefix is a directory horus created and this run just deleted, so it is the only one safe to take back. Tests pin both halves, including that a default uninstall touches no PATH line at all and that a stranger's /opt/somebody-else/bin survives. Two bugs found while writing it, both invisible in a multi-line fixture and both in the first draft of this commit: - `grep -v | grep -v > tmp && mv` — grep exits 1 when it selects no lines, so an rc file whose only content was that PATH line short-circuited the `&&` and kept the line. The one file where the removal matters most was the one case that silently did nothing. - uninstall.sh runs under `set -e` (line 31), so that same exit status did not merely skip the mv, it aborted the uninstaller partway through and left every later step undone. The harness could not have caught either: it built its uninstall slice without `set -e`, so it was strictly more forgiving than the shipped script. It now sets it, which is what makes the second failure observable. Ablating `|| true` fails the new test. --- .../tests/completion_install_contract.rs | 102 +++++++++++++++++- uninstall.sh | 40 ++++++- 2 files changed, 140 insertions(+), 2 deletions(-) diff --git a/horus_manager/tests/completion_install_contract.rs b/horus_manager/tests/completion_install_contract.rs index e1d229669..5de027ad5 100644 --- a/horus_manager/tests/completion_install_contract.rs +++ b/horus_manager/tests/completion_install_contract.rs @@ -191,7 +191,12 @@ impl Sandbox { fn uninstall(&self, xdg: &[(&str, &str)]) -> String { let script = format!( - "HORUS_DIR=\"$HOME/.horus\"\n\ + // `set -e`, because uninstall.sh sets it at line 31. Without it the + // harness is more forgiving than the shipped script: a command that + // legitimately exits non-zero (grep selecting no lines, say) reads + // as harmless here and aborts the real uninstall partway through. + "set -e\n\ + HORUS_DIR=\"$HOME/.horus\"\n\ GREEN=''; YELLOW=''; NC=''\n\ REMOVED=0; SKIPPED=0\n\ {paths}\n{removal}\n{profiles}\n\ @@ -961,3 +966,98 @@ fn the_prefix_cleanup_leaves_similarly_named_variables_alone() { variable:\n{fish}" ); } + +/// A prefix install's PATH line goes; the shared `~/.local/bin` line stays. +/// +/// Both halves matter. The prefix line names a directory this uninstall just +/// deleted, so leaving it is dead weight the user has to find themselves. The +/// default line names `~/.local/bin` — which pip, pipx, cargo and rustup all +/// install into — so removing it because horus happened to write it would take +/// every one of those off PATH. That is why the removal matches an exact +/// string built from `$HORUS_PREFIX` rather than anything resembling +/// `export PATH=`. +#[test] +fn uninstalling_a_prefix_install_removes_only_its_own_path_line() { + let sb = Sandbox::new(); + let prefix = sb.path().join("opt-horus"); + let p = prefix.display().to_string(); + fs::write( + sb.path().join(".bashrc"), + format!( + "# bashrc\n\ + export PATH=\"{p}/bin:$PATH\"\n\ + export PATH=\"$HOME/.local/bin:$PATH\"\n\ + export PATH=\"/opt/somebody-else/bin:$PATH\"\n" + ), + ) + .unwrap(); + + sb.uninstall(&[("HORUS_PREFIX", &p)]); + + let rc = sb.read(".bashrc").unwrap_or_default(); + assert!( + !rc.contains(&format!("{p}/bin")), + "the prefix PATH line points at a directory the uninstaller just \ + deleted and must not survive it:\n{rc}" + ); + assert!( + rc.contains("$HOME/.local/bin"), + "the uninstaller removed the shared ~/.local/bin PATH line. pip, pipx, \ + cargo and rustup install there too; taking that line out over horus \ + breaks all of them:\n{rc}" + ); + assert!( + rc.contains("/opt/somebody-else/bin"), + "the uninstaller removed a PATH line that has nothing to do with \ + horus:\n{rc}" + ); +} + +/// The default install must not trip the prefix branch at all. +#[test] +fn uninstalling_a_default_install_touches_no_path_line() { + let sb = Sandbox::new(); + let original = "# bashrc\nexport PATH=\"$HOME/.local/bin:$PATH\"\n"; + fs::write(sb.path().join(".bashrc"), original).unwrap(); + + sb.uninstall(&[]); + + let rc = sb.read(".bashrc").unwrap_or_default(); + assert!( + rc.contains("$HOME/.local/bin"), + "a default uninstall (no HORUS_PREFIX) edited a PATH line it has no \ + claim on:\n{rc}" + ); +} + +/// An rc file whose only content is the prefix PATH line. +/// +/// Two ways this went wrong, both invisible in a multi-line fixture: +/// `grep -v` exits 1 when it selects nothing, so gating the rewrite on `&&` +/// left the line in the one file where removing it mattered most; and under +/// `set -e` — which uninstall.sh sets at line 31 — that same exit status +/// aborts the uninstaller partway through, leaving everything after this +/// block undone. +#[test] +fn an_rc_file_that_is_only_the_prefix_path_line_is_emptied_not_skipped() { + let sb = Sandbox::new(); + let prefix = sb.path().join("opt-horus"); + let p = prefix.display().to_string(); + fs::write( + sb.path().join(".bashrc"), + format!("export PATH=\"{p}/bin:$PATH\"\n"), + ) + .unwrap(); + + let out = sb.uninstall(&[("HORUS_PREFIX", &p)]); + + let rc = sb.read(".bashrc").unwrap_or_default(); + assert!( + !rc.contains(&format!("{p}/bin")), + "the only line in the file was the one to remove, and it survived:\n{rc}" + ); + assert!( + out.contains("Cleaned the prefix PATH line"), + "the uninstaller did not report cleaning the line:\n{out}" + ); +} diff --git a/uninstall.sh b/uninstall.sh index 4c648b913..17d59b6bb 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -964,7 +964,7 @@ fi # In a function, with markers, for the same reason as the completion # removal above: the regression test runs this exact text. clean_shell_profiles() { - local profile fish_config + local profile fish_config prefix_path_line prefix_fish_line # Shell profiles: remove horus completion eval lines and shell integration for profile in "$HOME/.bashrc" "$HOME/.zshrc" "$HOME/.profile" "$HOME/.bash_profile"; do # The block install.sh's add_zsh_fpath_block() writes. It is delimited by @@ -1019,6 +1019,44 @@ clean_shell_profiles() { fi done + # The PATH line install.sh writes for a --prefix install points at a + # directory this run has just deleted. + # + # Matched as an exact string, never as a pattern, and only for a prefix. + # A default install writes `export PATH="$HOME/.local/bin:$PATH"` for a + # directory pip, pipx, cargo and rustup also populate: deleting that line + # because horus happened to write it would take every one of those off the + # user's PATH. `$HORUS_PREFIX/bin` is a directory horus created and horus + # removed, so it is the only one safe to take back out. + if [ -n "${HORUS_PREFIX:-}" ] && [ "$HORUS_PREFIX" != "$HORUS_DIR" ]; then + prefix_path_line="export PATH=\"${HORUS_PREFIX}/bin:\$PATH\"" + prefix_fish_line="fish_add_path ${HORUS_PREFIX}/bin" + for profile in "$HOME/.bashrc" "$HOME/.zshrc" "$HOME/.profile" \ + "$HOME/.bash_profile" "$HOME/.config/fish/config.fish"; do + [ -f "$profile" ] || continue + if grep -qF "$prefix_path_line" "$profile" 2>/dev/null || + grep -qF "$prefix_fish_line" "$profile" 2>/dev/null; then + cp "$profile" "${profile}.horus-backup" 2>/dev/null + # Deliberately NOT `... && mv`. grep exits 1 when it selects + # no lines, so an rc file whose only content was that PATH line + # would leave `&&` short-circuited and the line in place -- + # precisely the file where the removal matters most. + # `|| true` because uninstall.sh runs under `set -e` (line 31) + # and grep exits 1 on selecting nothing. Without it, removing + # the last line of an rc file would abort the entire uninstall + # partway through. + { grep -vF "$prefix_path_line" "$profile" 2>/dev/null \ + | grep -vF "$prefix_fish_line" > "${profile}.horus-new" 2>/dev/null; } || true + if [ -f "${profile}.horus-new" ]; then + mv "${profile}.horus-new" "$profile" + fi + rm -f "${profile}.horus-new" 2>/dev/null + echo -e " ${GREEN}[+]${NC} Cleaned the prefix PATH line from $(basename $profile)" + REMOVED=$((REMOVED + 1)) + fi + done + fi + # fish keeps its own syntax in its own file, and the loop above does not # walk it. install.sh writes `set -gx HORUS_PREFIX "..."` into config.fish; # the conf.d/horus.fish removed below is a different file entirely, so for From b3cc6420baf7c09d8250b8aa32b2738605d0fdc4 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 13:56:14 -0700 Subject: [PATCH 48/79] fix(uninstall): the fish prefix cleanup reported removals it never made MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard and the sed it guards disagreed. The guard matched a bare grep -q "HORUS_PREFIX" "$fish_config" while the sed below it is anchored to the actual `set -gx HORUS_PREFIX` form. A config.fish holding only a user's own HORUS_PREFIX_OTHER therefore entered the block, had a backup taken, had nothing deleted, and still printed [+] Cleaned HORUS_PREFIX from config.fish and counted a removal. uninstall.sh already carries a note about the last time it claimed a cleanup that never happened — the declined-sudo path in the RT block — so this is a repeat of a failure this script has explicitly named. The guard now uses the same anchored pattern as the sed. Verified by running the extracted slice under `set -e` against both files: lookalike-only reports nothing and is untouched, the real line is removed once with HORUS_PREFIX_OTHER left in place. Also merges two comments in the PATH block that had drifted into restating each other. --- .../tests/completion_install_contract.rs | 33 +++++++++++++++++++ uninstall.sh | 22 ++++++++----- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/horus_manager/tests/completion_install_contract.rs b/horus_manager/tests/completion_install_contract.rs index 5de027ad5..a81a56007 100644 --- a/horus_manager/tests/completion_install_contract.rs +++ b/horus_manager/tests/completion_install_contract.rs @@ -1061,3 +1061,36 @@ fn an_rc_file_that_is_only_the_prefix_path_line_is_emptied_not_skipped() { "the uninstaller did not report cleaning the line:\n{out}" ); } + +/// A config.fish holding only a *similarly named* variable must report nothing. +/// +/// The guard grep and the sed that follows it have to agree. When the guard was +/// unanchored it matched `HORUS_PREFIX_OTHER`, the anchored sed then correctly +/// deleted nothing, and the block still printed "Cleaned HORUS_PREFIX from +/// config.fish" and counted a removal. uninstall.sh already carries a comment +/// about the last time it reported a cleanup that never happened (the declined +/// -sudo path in the RT block); this is the same failure. +#[test] +fn a_config_fish_without_horus_prefix_reports_no_cleanup() { + let sb = Sandbox::new(); + let fish_dir = sb.path().join(".config/fish"); + fs::create_dir_all(&fish_dir).unwrap(); + fs::write( + fish_dir.join("config.fish"), + "# config.fish\nset -gx HORUS_PREFIX_OTHER \"keep me\"\n", + ) + .unwrap(); + + let out = sb.uninstall(&[]); + + let fish = sb.read(".config/fish/config.fish").unwrap_or_default(); + assert!( + fish.contains("HORUS_PREFIX_OTHER"), + "the unrelated variable was deleted:\n{fish}" + ); + assert!( + !out.contains("Cleaned HORUS_PREFIX from config.fish"), + "the uninstaller reported cleaning a HORUS_PREFIX line that was never \ + there and never removed:\n{out}" + ); +} diff --git a/uninstall.sh b/uninstall.sh index 17d59b6bb..a815cce4e 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -1037,14 +1037,12 @@ clean_shell_profiles() { if grep -qF "$prefix_path_line" "$profile" 2>/dev/null || grep -qF "$prefix_fish_line" "$profile" 2>/dev/null; then cp "$profile" "${profile}.horus-backup" 2>/dev/null - # Deliberately NOT `... && mv`. grep exits 1 when it selects - # no lines, so an rc file whose only content was that PATH line - # would leave `&&` short-circuited and the line in place -- - # precisely the file where the removal matters most. - # `|| true` because uninstall.sh runs under `set -e` (line 31) - # and grep exits 1 on selecting nothing. Without it, removing - # the last line of an rc file would abort the entire uninstall - # partway through. + # grep exits 1 when it selects no lines, which is exactly + # what happens to an rc file whose only content was that PATH + # line -- the file where the removal matters most. Gating the + # rewrite on `&&` would skip it there, and `set -e` (line 31) + # would abort the whole uninstall on the same status. Hence + # the braces and the `|| true` rather than `... && mv`. { grep -vF "$prefix_path_line" "$profile" 2>/dev/null \ | grep -vF "$prefix_fish_line" > "${profile}.horus-new" 2>/dev/null; } || true if [ -f "${profile}.horus-new" ]; then @@ -1062,7 +1060,13 @@ clean_shell_profiles() { # the conf.d/horus.fish removed below is a different file entirely, so for # fish users the export survived no matter what the POSIX side did. fish_config="$HOME/.config/fish/config.fish" - if [ -f "$fish_config" ] && grep -q "HORUS_PREFIX" "$fish_config" 2>/dev/null; then + # Anchored the same way the sed below is. An unanchored "HORUS_PREFIX" + # also matches a user's HORUS_PREFIX_OTHER, and the sed would then delete + # nothing while this block still printed "Cleaned" and counted a removal -- + # the same false success this script already records for the declined-sudo + # path in the RT cleanup above. + if [ -f "$fish_config" ] && + grep -qE "^[[:space:]]*set +(-[a-zA-Z]+ +)*HORUS_PREFIX([[:space:]]|$)" "$fish_config" 2>/dev/null; then cp "$fish_config" "${fish_config}.horus-backup" 2>/dev/null sed -E -i.bak "/^[[:space:]]*set +(-[a-zA-Z]+ +)*HORUS_PREFIX([[:space:]]|$)/d" "$fish_config" 2>/dev/null || \ sed -E -i '' "/^[[:space:]]*set +(-[a-zA-Z]+ +)*HORUS_PREFIX([[:space:]]|$)/d" "$fish_config" 2>/dev/null From 3b1ffa3f41515651c478eb38e4a70832d3d273d6 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 14:11:55 -0700 Subject: [PATCH 49/79] fix: horus node pause did nothing to three of the four node classes Operator pause travels through the shared node_controls map, and only the RT executor read it. Pausing a Compute, Async I/O or Event node from the CLI reported success and changed nothing - the node kept ticking, and nothing anywhere said so. Two comments already assert what the code did not do. The control handler: "Operator pause is persistent (until ResumeNode) for ALL node classes via the shared node_controls flag - both the executor threads and should_tick_node honor it." And should_tick_node itself: "honored the same way the executor threads do, via node_controls." All three now read the flag every pass, so ResumeNode takes effect on the next one, and they latch the `stopped` flag the way the RT executor does. Nothing in the CLI sets `stopped` today, so that half is parity rather than a second user-facing fix - but a flag only some tick gates honour is exactly how the pause bug started. The tests assert on tick counts, not on flags: the question is whether the node stopped running. --- horus_core/src/scheduling/async_executor.rs | 89 ++++++++++++++ horus_core/src/scheduling/compute_executor.rs | 92 ++++++++++++++ horus_core/src/scheduling/event_executor.rs | 115 ++++++++++++++++++ 3 files changed, 296 insertions(+) diff --git a/horus_core/src/scheduling/async_executor.rs b/horus_core/src/scheduling/async_executor.rs index 9ee1ac37e..4d7702d59 100644 --- a/horus_core/src/scheduling/async_executor.rs +++ b/horus_core/src/scheduling/async_executor.rs @@ -141,10 +141,35 @@ impl AsyncExecutor { super::primitives::honor_restart_request(node, &monitors); } + // Operator commands reach every executor through the shared + // node_controls map: `horus node kill` sets stopped, `horus node + // pause` sets paused. Only the RT executor used to read them, so + // pausing or killing a async I/O node from the CLI printed success and + // then did nothing at all -- while `should_tick_node` says it is + // "honored the same way the executor threads do" and the control + // handler says the pause holds "for ALL node classes". + // + // A pause does not latch -- it is re-read every pass, so ResumeNode + // takes effect on the next one. That is the half the CLI actually + // drives: ControlCommand::PauseNode/ResumeNode set this flag. + // + // The `stopped` half latches into `is_stopped` the way the RT + // executor does. Nothing in the CLI sets it today (`set_stopped` is + // `#[allow(dead_code)]`, and `horus node kill` goes through the + // launch supervisor instead), so this is parity with the RT + // executor rather than a second user-facing fix -- but a flag two + // of five tick gates honour is exactly how the pause bug started. + for node in nodes.iter_mut() { + if monitors.node_controls.is_stopped(node.name.as_ref()) { + node.is_stopped = true; + } + } + for (i, node) in nodes.iter().enumerate() { if !node.initialized || node.is_stopped || node.is_paused + || monitors.node_controls.is_paused(node.name.as_ref()) || !node.failure_policy_allows_tick() { continue; @@ -477,6 +502,70 @@ mod tests { } } + /// `horus node pause` must reach a async I/O node. + /// + /// Operator commands travel through the shared `node_controls` map. Only + /// the RT executor read it, so pausing or killing a async I/O node from the + /// CLI reported success and changed nothing -- while `should_tick_node` + /// says the flag is "honored the same way the executor threads do" and the + /// control handler says an operator pause holds "for ALL node classes". + /// + /// Counts, not flags: the question is whether the node stopped running. + /// The `stopped` half is checked for parity with the RT executor; no CLI + /// path sets it today. + #[test] + fn an_operator_pause_and_kill_reach_an_async_node() { + let count = Arc::new(AtomicU64::new(0)); + let monitors = test_monitors(); + monitors.node_controls.register("paused_async"); + let running = Arc::new(AtomicBool::new(true)); + + let executor = AsyncExecutor::start( + vec![make_async_node("paused_async", count.clone())], + running.clone(), + 1_u64.ms(), + monitors.clone(), + ); + + let ticked = |from: u64| { + for _ in 0..200 { + if count.load(Ordering::Relaxed) > from { + return true; + } + std::thread::sleep(5_u64.ms()); + } + false + }; + + assert!(ticked(0), "the node never ticked at all"); + + monitors.node_controls.set_paused("paused_async", true); + std::thread::sleep(30_u64.ms()); + let frozen = count.load(Ordering::Relaxed); + std::thread::sleep(60_u64.ms()); + assert_eq!( + count.load(Ordering::Relaxed), + frozen, + "the node kept ticking through an operator pause" + ); + + monitors.node_controls.set_paused("paused_async", false); + assert!(ticked(frozen), "the node never resumed after ResumeNode"); + + monitors.node_controls.set_stopped("paused_async"); + std::thread::sleep(30_u64.ms()); + let killed_at = count.load(Ordering::Relaxed); + std::thread::sleep(60_u64.ms()); + assert_eq!( + count.load(Ordering::Relaxed), + killed_at, + "the node kept ticking after an operator kill" + ); + + running.store(false, Ordering::SeqCst); + let _ = executor.stop(); + } + fn make_async_node(name: &str, count: Arc) -> RegisteredNode { let node = CounterNode { name: name.to_string(), diff --git a/horus_core/src/scheduling/compute_executor.rs b/horus_core/src/scheduling/compute_executor.rs index 96001c35a..899fc7e7d 100644 --- a/horus_core/src/scheduling/compute_executor.rs +++ b/horus_core/src/scheduling/compute_executor.rs @@ -635,12 +635,37 @@ impl ComputeExecutor { while running.load(Ordering::Relaxed) { let loop_start = Instant::now(); + // Operator commands reach every executor through the shared + // node_controls map: `horus node kill` sets stopped, `horus node + // pause` sets paused. Only the RT executor used to read them, so + // pausing or killing a compute node from the CLI printed success and + // then did nothing at all -- while `should_tick_node` says it is + // "honored the same way the executor threads do" and the control + // handler says the pause holds "for ALL node classes". + // + // A pause does not latch -- it is re-read every pass, so ResumeNode + // takes effect on the next one. That is the half the CLI actually + // drives: ControlCommand::PauseNode/ResumeNode set this flag. + // + // The `stopped` half latches into `is_stopped` the way the RT + // executor does. Nothing in the CLI sets it today (`set_stopped` is + // `#[allow(dead_code)]`, and `horus node kill` goes through the + // launch supervisor instead), so this is parity with the RT + // executor rather than a second user-facing fix -- but a flag two + // of five tick gates honour is exactly how the pause bug started. + for node in nodes.iter_mut() { + if monitors.node_controls.is_stopped(node.name.as_ref()) { + node.is_stopped = true; + } + } + // Classify which nodes should tick this cycle ready_indices.clear(); for (i, node) in nodes.iter().enumerate() { if !node.initialized || node.is_stopped || node.is_paused + || monitors.node_controls.is_paused(node.name.as_ref()) || !node.failure_policy_allows_tick() { continue; @@ -975,6 +1000,73 @@ mod tests { } } + /// `horus node pause` must reach a compute node. + /// + /// Operator commands travel through the shared `node_controls` map. Only + /// the RT executor read it, so pausing or killing a compute node from the + /// CLI reported success and changed nothing -- while `should_tick_node` + /// says the flag is "honored the same way the executor threads do" and the + /// control handler says an operator pause holds "for ALL node classes". + /// + /// Counts, not flags: the question is whether the node stopped running. + /// The `stopped` half is checked for parity with the RT executor; no CLI + /// path sets it today. + #[test] + fn an_operator_pause_and_kill_reach_a_compute_node() { + let count = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let monitors = test_monitors(); + monitors.node_controls.register("paused_compute"); + let running = Arc::new(AtomicBool::new(true)); + + let executor = ComputeExecutor::start( + vec![make_compute_node("paused_compute", count.clone())], + running.clone(), + 1_u64.ms(), + monitors.clone(), + ); + + let ticked = |from: u64| { + for _ in 0..200 { + if count.load(Ordering::Relaxed) > from { + return true; + } + std::thread::sleep(5_u64.ms()); + } + false + }; + + assert!(ticked(0), "the node never ticked at all"); + + // Pause: the count must stop moving. + monitors.node_controls.set_paused("paused_compute", true); + std::thread::sleep(30_u64.ms()); // let any in-flight pass finish + let frozen = count.load(Ordering::Relaxed); + std::thread::sleep(60_u64.ms()); + assert_eq!( + count.load(Ordering::Relaxed), + frozen, + "the node kept ticking through an operator pause" + ); + + // Resume: it must move again. + monitors.node_controls.set_paused("paused_compute", false); + assert!(ticked(frozen), "the node never resumed after ResumeNode"); + + // Kill: the count must stop, and stay stopped. + monitors.node_controls.set_stopped("paused_compute"); + std::thread::sleep(30_u64.ms()); + let killed_at = count.load(Ordering::Relaxed); + std::thread::sleep(60_u64.ms()); + assert_eq!( + count.load(Ordering::Relaxed), + killed_at, + "the node kept ticking after an operator kill" + ); + + running.store(false, Ordering::SeqCst); + let _ = executor.stop(); + } + /// A safing diagnostic from a non-RT executor must actually be printed. /// /// `honor_safe_state_request` reports through `rt_diag`, which only queues diff --git a/horus_core/src/scheduling/event_executor.rs b/horus_core/src/scheduling/event_executor.rs index 5e6fd5446..5c86a8ddc 100644 --- a/horus_core/src/scheduling/event_executor.rs +++ b/horus_core/src/scheduling/event_executor.rs @@ -338,9 +338,32 @@ impl EventExecutor { super::primitives::honor_safe_state_request(&mut node, &monitors); super::primitives::honor_restart_request(&mut node, &monitors); + // Operator commands reach every executor through the shared + // node_controls map: `horus node kill` sets stopped, `horus node + // pause` sets paused. Only the RT executor used to read them, so + // pausing or killing a event node from the CLI printed success and + // then did nothing at all -- while `should_tick_node` says it is + // "honored the same way the executor threads do" and the control + // handler says the pause holds "for ALL node classes". + // + // A pause does not latch -- it is re-read every pass, so ResumeNode + // takes effect on the next one. That is the half the CLI actually + // drives: ControlCommand::PauseNode/ResumeNode set this flag. + // + // The `stopped` half latches into `is_stopped` the way the RT + // executor does. Nothing in the CLI sets it today (`set_stopped` is + // `#[allow(dead_code)]`, and `horus node kill` goes through the + // launch supervisor instead), so this is parity with the RT + // executor rather than a second user-facing fix -- but a flag two + // of five tick gates honour is exactly how the pause bug started. + if monitors.node_controls.is_stopped(node.name.as_ref()) { + node.is_stopped = true; + } + if !node.initialized || node.is_stopped || node.is_paused + || monitors.node_controls.is_paused(node.name.as_ref()) || !node.failure_policy_allows_tick() { // Not tickable. Deliberately a timed sleep and not a wait on @@ -644,6 +667,98 @@ mod tests { } } + /// `horus node pause` must reach a event node. + /// + /// Operator commands travel through the shared `node_controls` map. Only + /// the RT executor read it, so pausing or killing a event node from the + /// CLI reported success and changed nothing -- while `should_tick_node` + /// says the flag is "honored the same way the executor threads do" and the + /// control handler says an operator pause holds "for ALL node classes". + /// + /// Counts, not flags: the question is whether the node stopped running. + /// The `stopped` half is checked for parity with the RT executor; no CLI + /// path sets it today. + #[test] + fn an_operator_pause_and_kill_reach_an_event_node() { + let count = Arc::new(AtomicU64::new(0)); + let monitors = test_monitors(); + monitors.node_controls.register("paused_event"); + let running = Arc::new(AtomicBool::new(true)); + + let executor = EventExecutor::start( + vec![make_event_node( + "paused_event", + "pause_topic", + count.clone(), + )], + running.clone(), + monitors.clone(), + ); + + // Wait for the watcher to register its notifier. + let mut registered = false; + for _ in 0..100 { + if crate::core::NodeInfo::notify_event("paused_event") { + registered = true; + break; + } + std::thread::sleep(5_u64.ms()); + } + assert!( + registered, + "the event watcher never registered its notifier" + ); + + let poke_until = |from: u64| { + for _ in 0..200 { + crate::core::NodeInfo::notify_event("paused_event"); + if count.load(Ordering::Relaxed) > from { + return true; + } + std::thread::sleep(5_u64.ms()); + } + false + }; + + assert!(poke_until(0), "the node never ticked at all"); + + // Paused: poking it must not tick it. + monitors.node_controls.set_paused("paused_event", true); + std::thread::sleep(30_u64.ms()); + let frozen = count.load(Ordering::Relaxed); + for _ in 0..12 { + crate::core::NodeInfo::notify_event("paused_event"); + std::thread::sleep(5_u64.ms()); + } + assert_eq!( + count.load(Ordering::Relaxed), + frozen, + "the node kept ticking through an operator pause" + ); + + monitors.node_controls.set_paused("paused_event", false); + assert!( + poke_until(frozen), + "the node never resumed after ResumeNode" + ); + + monitors.node_controls.set_stopped("paused_event"); + std::thread::sleep(30_u64.ms()); + let killed_at = count.load(Ordering::Relaxed); + for _ in 0..12 { + crate::core::NodeInfo::notify_event("paused_event"); + std::thread::sleep(5_u64.ms()); + } + assert_eq!( + count.load(Ordering::Relaxed), + killed_at, + "the node kept ticking after an operator kill" + ); + + running.store(false, Ordering::SeqCst); + let _ = executor.stop(); + } + #[test] fn test_event_executor_starts_and_stops() { let count = Arc::new(AtomicU64::new(0)); From 386e401c086d51f1b0b1016e520f6c04de280a69 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 14:11:55 -0700 Subject: [PATCH 50/79] fix: nothing ever called the warning that .no_alloc() is unenforced warn_if_unenforced names its own call site - "Intended call site: node registration, once per .no_alloc() node" - and had no caller anywhere outside the two tests that invoke it directly. So a user who wrote .no_alloc() in a binary that never installed RtAwareAllocator was told nothing: the flag was inert, the RT path could allocate freely, and the banner explaining the three-line fix existed but never printed. validate() already refuses the other way .no_alloc() becomes false assurance - a non-RT execution class, where the executor will not look. This is the adjacent case: there is nothing to look with. It warns rather than fails for the reason the function's own doc gives - failing would force allocator installation on every library consumer. Calling the function is what the existing tests prove. The new one builds a node in a child process and requires the warning to come out of a real stderr. --- horus_core/src/scheduling/node_builder.rs | 17 ++++++ horus_core/tests/rt_no_alloc.rs | 68 +++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/horus_core/src/scheduling/node_builder.rs b/horus_core/src/scheduling/node_builder.rs index 33951db1c..7743163aa 100644 --- a/horus_core/src/scheduling/node_builder.rs +++ b/horus_core/src/scheduling/node_builder.rs @@ -685,6 +685,23 @@ impl NodeRegistration { .into()); } + // The OTHER way `.no_alloc()` becomes false assurance: the flag is + // right and the execution class is right, but this binary never + // installed `RtAwareAllocator`, so nothing checks anything. The check + // above catches "the executor will not look"; this catches "there is + // nothing to look with". + // + // `warn_if_unenforced` was written for exactly this and says so — + // "Intended call site: node registration, once per `.no_alloc()` node" + // — and had no caller anywhere outside its own tests, so the user was + // never told. It warns rather than fails on purpose: refusing to build + // would force allocator installation on every library consumer, which + // its doc explains is what a library must not do. Process-wide-once, and + // a single atomic load after the first call. + if self.no_alloc { + crate::memory::rt_allocator::warn_if_unenforced(node_name); + } + // Validate execution class vs is_rt flag consistency if self.is_rt && !matches!( diff --git a/horus_core/tests/rt_no_alloc.rs b/horus_core/tests/rt_no_alloc.rs index 07f2c9b2b..c3afcc524 100644 --- a/horus_core/tests/rt_no_alloc.rs +++ b/horus_core/tests/rt_no_alloc.rs @@ -92,6 +92,74 @@ fn this_binary_has_no_rt_allocator_and_reports_that_honestly() { ); } +/// Registering a `.no_alloc()` node must actually print the warning. +/// +/// `warn_if_unenforced` names its own call site — "Intended call site: node +/// registration, once per `.no_alloc()` node" — and had no caller anywhere +/// outside the two tests that invoke it directly. So the function worked +/// perfectly and no user ever saw it: `.no_alloc()` in a binary without the +/// allocator was silently inert, which is the precise failure the warning text +/// exists to prevent. +/// +/// Calling the function proves the function. This builds a node and requires +/// the warning to come out of a real process's stderr, which is the part that +/// was missing. The child is this same test binary — the one that deliberately +/// does not install the allocator. +#[test] +fn building_a_no_alloc_node_warns_that_nothing_is_enforcing_it() { + const CHILD: &str = "HORUS_NO_ALLOC_WARN_CHILD"; + + struct Inert; + impl Node for Inert { + fn name(&self) -> &str { + "unenforced_controller" + } + fn tick(&mut self) {} + } + + if std::env::var_os(CHILD).is_some() { + let mut scheduler = Scheduler::new().tick_rate(1000_u64.hz()); + scheduler + .add(Inert) + .order(0) + .rate(1000_u64.hz()) + .no_alloc() + .build() + .expect("a .no_alloc() RT node must still build — this warns, it does not fail"); + return; + } + + let exe = std::env::current_exe().expect("test binary path"); + let out = std::process::Command::new(exe) + .args([ + "--exact", + "building_a_no_alloc_node_warns_that_nothing_is_enforcing_it", + "--nocapture", + "--test-threads", + "1", + ]) + .env(CHILD, "1") + .output() + .expect("re-running the test binary should work"); + + let stdout = String::from_utf8_lossy(&out.stdout); + assert!( + stdout.contains("1 passed"), + "the child ran no test — was this test renamed?\n{stdout}" + ); + + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains(".no_alloc() IS NOT BEING ENFORCED"), + "registering a .no_alloc() node in a binary with no RtAwareAllocator \ + printed no warning, so the user is told nothing.\nstderr:\n{stderr}" + ); + assert!( + stderr.contains("unenforced_controller"), + "the warning must name the node that asked for the guarantee.\nstderr:\n{stderr}" + ); +} + /// Without the allocator, `.no_alloc()` is inert — allocating inside the bracket /// is not detected. Asserting the no-op keeps the documentation honest and makes /// the warning above load-bearing rather than decorative. From c2334ca256ff321482971f5127e59b0514963922 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 14:11:56 -0700 Subject: [PATCH 51/79] fix: the blackbox never recorded a single network event set_blackbox_hook says "Called by the Scheduler at startup to wire the blackbox" and nothing called it. record_external_event - the entry point horus_net's replicator uses in three places for peer loss, replication failure and desync - looked up an unset OnceLock and dropped the event. A distributed robot's flight recorder held a complete local timeline and nothing whatsoever about the network, which is the half of a distributed incident you most need afterwards. .blackbox() is the user asking for a flight recorder, so that is where the hook is wired. The OnceLock semantics are unchanged: the first scheduler to enable a recorder owns external events for the process. Compare set_topic_lifecycle_hook, the sibling global hook, which horus_net does wire at replicator startup - this one was simply missed. --- horus_core/src/scheduling/scheduler/mod.rs | 24 +++++++++- horus_core/tests/blackbox_intent.rs | 51 ++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/horus_core/src/scheduling/scheduler/mod.rs b/horus_core/src/scheduling/scheduler/mod.rs index 7fbb7637e..183cafe06 100644 --- a/horus_core/src/scheduling/scheduler/mod.rs +++ b/horus_core/src/scheduling/scheduler/mod.rs @@ -887,7 +887,29 @@ impl Scheduler { self.pending_config.monitoring.black_box_size_mb = size_mb; let bb_dir = self.monitor.working_dir.join(".horus").join("blackbox"); let bb = super::blackbox::BlackBox::new(size_mb).with_path(bb_dir); - self.monitor.blackbox = Some(Arc::new(Mutex::new(bb))); + let shared = Arc::new(Mutex::new(bb)); + + // Wire the external-event hook. `set_blackbox_hook` says "Called by the + // Scheduler at startup to wire the blackbox" and nothing called it, so + // `record_external_event` -- the entry point horus_net's replicator uses + // for peer-lost, replication-failure and desync events -- looked up an + // unset OnceLock and dropped every event. A distributed robot's flight + // recorder therefore held nothing about the network, which is the part + // of a distributed incident you most want a timeline of. + // + // OnceLock: the first scheduler to enable a blackbox owns the hook for + // the life of the process. A second scheduler's `.blackbox()` still gets + // its own recorder for local events; only external ones stay with the + // first. That is the existing contract of a process-global hook, not a + // new limitation. + let hook_target = Arc::clone(&shared); + super::blackbox::set_blackbox_hook(move |event| { + if let Ok(mut bb) = hook_target.lock() { + bb.record(event); + } + }); + + self.monitor.blackbox = Some(shared); self } diff --git a/horus_core/tests/blackbox_intent.rs b/horus_core/tests/blackbox_intent.rs index d45eb74b5..08303b50d 100644 --- a/horus_core/tests/blackbox_intent.rs +++ b/horus_core/tests/blackbox_intent.rs @@ -224,3 +224,54 @@ fn test_blackbox_intent_anomaly_detection() { ); } } + +// ============================================================================ +// The external-event hook +// ============================================================================ + +/// INTENT: "An event recorded from outside the scheduler reaches the recorder." +/// +/// `record_external_event` is how a subsystem that does not own the scheduler +/// -- horus_net's replicator, which reports peer loss, replication failure and +/// desync -- gets into the flight recorder. It forwards to a process-global +/// hook, and `set_blackbox_hook` documents itself as "Called by the Scheduler +/// at startup to wire the blackbox". +/// +/// Nothing called it. The hook stayed unset for the life of every process, so +/// `record_external_event` looked up an empty `OnceLock` and dropped the event +/// on the floor. A distributed robot's recorder held a complete local timeline +/// and nothing at all about the network, which is the half of a distributed +/// incident you most need. +/// +/// Enabling `.blackbox()` is the user's request for a flight recorder, so that +/// is where the wiring belongs. This asserts through the public surface: build +/// a scheduler with a recorder, emit an external event, read it back. +#[cfg(feature = "blackbox")] +#[test] +fn an_external_event_reaches_a_scheduler_that_enabled_its_blackbox() { + use horus_core::core::DurationExt; + use horus_core::scheduling::{record_external_event, Scheduler}; + + let scheduler = Scheduler::new().tick_rate(100_u64.hz()).blackbox(1); + + record_external_event(BlackBoxEvent::NodeError { + name: "peer_link".to_string(), + error: "replication peer lost".to_string(), + severity: horus_core::error::Severity::Permanent, + }); + + let recorder = scheduler + .get_blackbox() + .expect(".blackbox(1) must give the scheduler a recorder"); + let events = recorder.lock().unwrap_or_else(|p| p.into_inner()).events(); + + assert!( + events.iter().any(|r| matches!( + &r.event, + BlackBoxEvent::NodeError { error, .. } if error == "replication peer lost" + )), + "an external event never reached the flight recorder: the hook \ + set_blackbox_hook exists for is not wired. Recorded: {} event(s)", + events.len() + ); +} From 9c7b4ac8d7ecafdc2634c5a75a45220ee863f9dc Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 15:07:08 -0700 Subject: [PATCH 52/79] test: exempt the compute-diag child switch from the env-var doc guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `every_env_var_the_code_reads_is_documented` caught HORUS_COMPUTE_DIAG_CHILD, which is correct — it is a HORUS_* variable read from src/. It is the child half of the test that a compute-only program still drains its RT diagnostics to stdout, read inside a `#[cfg(test)]` module and unreachable from a shipped binary, so it goes in TEST_ONLY beside HORUS_CPP_FFI_EMIT_CHILD rather than into the user-facing environment-variables page. --- horus_manager/tests/docs_parity.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/horus_manager/tests/docs_parity.rs b/horus_manager/tests/docs_parity.rs index bf0963982..c6b8430f2 100644 --- a/horus_manager/tests/docs_parity.rs +++ b/horus_manager/tests/docs_parity.rs @@ -179,6 +179,10 @@ const TEST_ONLY: &[&str] = &[ "HORUS_LIVE5_GHOST_CAPACITY", // horus_cpp/src/c_api.rs — the child half of the FFI emit test. "HORUS_CPP_FFI_EMIT_CHILD", + // scheduling/compute_executor.rs — the child half of the test that a + // compute-only program still drains its RT diagnostics to stdout. Read + // inside a `#[cfg(test)]` module; unreachable outside the test binary. + "HORUS_COMPUTE_DIAG_CHILD", // commands/fmt.rs — the switch CI uses to turn a skipped tool into a // failure, so a missing clang-format is not silently tolerated. "HORUS_TEST_REQUIRE_TOOLS", From 9764e2b8d5231bfb479bac400323f2bb11b36b5f Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 15:45:31 -0700 Subject: [PATCH 53/79] fix(scheduling): three of the four executors dropped their last diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `RtExecutor::stop` ends with a final drain of the RT diagnostic ring, and says why: Without it a run shorter than `RT_DIAG_POLL` — or one whose last deadline miss came in the final milliseconds, which is the interesting case — would exit with its diagnostics still sitting in the ring. Compute, Event and AsyncIo had no such flush, though all three call `start_rt_diag_drain` and all three queue through `honor_safe_state_request`. So for a compute-only, event-only or async-only program the guarantee simply did not hold: anything queued inside the last 25 ms — "PANICKED in enter_safe_state", a final safing failure — was still in the ring when the process exited. The same class partition that has caught this code repeatedly: fixed on the RT path, left on the other three. It also makes an existing warning true. When the drain thread cannot be spawned, `start_rt_diag_drain` tells the operator diagnostics "will only be flushed at shutdown" — with no shutdown flush on three of the four executors, they were not flushed anywhere. The drain is now one named `flush_rt_diag()` called from all four `stop()` paths. Found via a CI failure in `a_compute_only_program_still_prints_its_safing_diagnostics`, which slept 200 ms and assumed the 25 ms background drainer would have been scheduled inside that window. On a loaded runner it was not, and the test reported "no drain thread was running" against code that was working. With the flush the line is emitted synchronously inside `stop()`, so the sleep is gone and no timing window is involved. The flush itself is deliberately NOT asserted on. The background drainer's first poll runs when its thread starts — the same few microseconds as the `rt_diag` under test — so an ordering assertion passes whether or not `stop()` flushes. I wrote that assertion, ablated the flush, watched it pass three times out of three, and removed it: a guard that cannot fail for the reason it states is worse than no guard. Isolating the flush needs a way to suppress that thread, which does not exist today. The test keeps the claim it can support — the line reaches stdout — and the reasoning for the flush lives at `flush_rt_diag`. horus_core --lib: 2595 passed, 0 failed. (An earlier run of the same suite showed 3 unrelated scheduler failures at load average ~70; all three pass in isolation and the suite is clean at load 6.) --- horus_core/src/scheduling/async_executor.rs | 8 +++-- horus_core/src/scheduling/compute_executor.rs | 32 +++++++++++++++---- horus_core/src/scheduling/event_executor.rs | 3 ++ horus_core/src/scheduling/rt_executor.rs | 25 ++++++++++++++- 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/horus_core/src/scheduling/async_executor.rs b/horus_core/src/scheduling/async_executor.rs index 4d7702d59..6dad434dd 100644 --- a/horus_core/src/scheduling/async_executor.rs +++ b/horus_core/src/scheduling/async_executor.rs @@ -93,12 +93,16 @@ impl AsyncExecutor { // node blocked inside `tick()`, because this loop only re-checks // `running` between ticks — and `run_with_filter` calls this before it // shuts down or safes any other node. - super::primitives::join_with_timeout( + let nodes = super::primitives::join_with_timeout( handle, "Async I/O", super::primitives::SHUTDOWN_TIMEOUT_PER_THREAD, ) - .unwrap_or_default() + .unwrap_or_default(); + // The last diagnostics a torn-down executor queued are the ones worth + // having; the background drainer only wakes every RT_DIAG_POLL. + super::rt_executor::flush_rt_diag(); + nodes } /// Main function for the async I/O thread. diff --git a/horus_core/src/scheduling/compute_executor.rs b/horus_core/src/scheduling/compute_executor.rs index 899fc7e7d..90b7584c9 100644 --- a/horus_core/src/scheduling/compute_executor.rs +++ b/horus_core/src/scheduling/compute_executor.rs @@ -592,12 +592,16 @@ impl ComputeExecutor { // node blocked inside `tick()`, because this loop only re-checks // `running` between ticks — and `run_with_filter` calls this before it // shuts down or safes any other node. - super::primitives::join_with_timeout( + let nodes = super::primitives::join_with_timeout( handle, "Compute", super::primitives::SHUTDOWN_TIMEOUT_PER_THREAD, ) - .unwrap_or_default() + .unwrap_or_default(); + // The last diagnostics a torn-down executor queued are the ones worth + // having; the background drainer only wakes every RT_DIAG_POLL. + super::rt_executor::flush_rt_diag(); + nodes } /// Main function for the compute coordinator thread. @@ -1098,9 +1102,13 @@ mod tests { super::super::rt_executor::rt_diag(format_args!("{MARKER}")); - // Longer than one RT_DIAG_POLL, so a running drainer has emitted it. - std::thread::sleep(Duration::from_millis(200)); - + // No sleep. This used to wait out an RT_DIAG_POLL so the background + // drainer would have emitted the line, which made the test depend + // on that thread being scheduled inside a 200 ms window -- and on a + // loaded CI runner it was not, so the run failed with "never + // reached stdout" against code that was working. `stop()` now + // flushes the ring itself, so the line is emitted synchronously + // before this returns and no timing window is involved. running.store(false, Ordering::SeqCst); let _ = exec.stop(); return; @@ -1124,10 +1132,20 @@ mod tests { stdout.contains("1 passed"), "the child ran no test — was this test renamed?\n{stdout}" ); + // Asserts only that the line reaches stdout, which is what this test is + // named for. It deliberately does NOT try to prove the line came from + // the flush in `stop()` rather than from the background drainer: the + // drainer's FIRST poll happens when its thread starts, which is inside + // the same few microseconds as the `rt_diag` above, so an ordering + // assertion here passes whether or not `stop()` flushes. Isolating the + // flush would need a way to keep that thread from running at all, and a + // guard that cannot fail for the reason it states is worse than no + // guard. What the flush buys is covered by the argument at + // `flush_rt_diag`, not by this assertion. assert!( stdout.contains(MARKER), - "a diagnostic queued by a compute-only program never reached stdout: \ - no drain thread was running.\n{stdout}" + "a diagnostic queued by a compute-only program never reached \ + stdout.\n{stdout}" ); } diff --git a/horus_core/src/scheduling/event_executor.rs b/horus_core/src/scheduling/event_executor.rs index 5c86a8ddc..42c6cd843 100644 --- a/horus_core/src/scheduling/event_executor.rs +++ b/horus_core/src/scheduling/event_executor.rs @@ -260,6 +260,9 @@ impl EventExecutor { "[Event] Stopped ({} nodes returning to scheduler)", nodes.len() )); + // The last diagnostics a torn-down executor queued are the ones worth + // having; the background drainer only wakes every RT_DIAG_POLL. + super::rt_executor::flush_rt_diag(); nodes } diff --git a/horus_core/src/scheduling/rt_executor.rs b/horus_core/src/scheduling/rt_executor.rs index 246330aa3..16d15b4be 100644 --- a/horus_core/src/scheduling/rt_executor.rs +++ b/horus_core/src/scheduling/rt_executor.rs @@ -350,6 +350,29 @@ fn rt_diag_drain(emit: impl Fn(&str)) -> usize { /// `RT_DIAG_POLL` and it must outlive any individual executor, because an /// executor that is being torn down is precisely when its last diagnostics /// matter. +/// Drain whatever is still in the diagnostic ring, now, on the calling thread. +/// +/// Every executor's `stop()` owes its nodes this. The background drainer wakes +/// only every `RT_DIAG_POLL`, so a program that queues a diagnostic and then +/// shuts down inside that window exits with the line still in the ring — and +/// the lines that arrive in the last milliseconds before a shutdown are the +/// ones worth having: "PANICKED in enter_safe_state", a final deadline miss. +/// +/// `RtExecutor::stop` has done this since the ring existed. Compute, Event and +/// AsyncIo did not, though all three call `start_rt_diag_drain` and all three +/// queue through `honor_safe_state_request` — so for a compute-only, event-only +/// or async-only program the guarantee simply did not hold. Same class +/// partition that keeps catching this code: fixed on the RT path, left on the +/// other three. +/// +/// It also makes the spawn-failure warning in `start_rt_diag_drain` true. That +/// warning tells the operator diagnostics "will only be flushed at shutdown"; +/// with no shutdown flush on three of the four executors, they were not +/// flushed at all. +pub(crate) fn flush_rt_diag() { + rt_diag_drain(print_line); +} + pub(crate) fn start_rt_diag_drain() { RT_DIAG_DRAIN_STARTED.call_once(|| { let spawned = std::thread::Builder::new() @@ -1213,7 +1236,7 @@ impl RtExecutor { // run shorter than `RT_DIAG_POLL` — or one whose last deadline miss came // in the final milliseconds, which is the interesting case — would exit // with its diagnostics still sitting in the ring. - rt_diag_drain(print_line); + flush_rt_diag(); all_nodes } From 8f25e798d17dc9ace1574897fd5b18a9a78b466e Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 15:46:47 -0700 Subject: [PATCH 54/79] fix: replay lost the recorded node order, and two docs contradicted their file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From a workspace sweep for the same shape of defect the docs audit found: code that contradicts its own comments. replay_from() built a name -> priority map from the recorded execution order and then looked it up by node_id — the key of node_recordings, a generated hex string (`format!("{:x}{:x}", unix_nanos, node_count)`) that can never equal a node name. Every lookup missed, `unwrap_or(0)` gave every node priority 0, and add_replay's stable sort left the replay in HashMap iteration order: neither the recorded order nor the same order twice. The module's whole claim is "replay with tick-perfect determinism". The name comes back out of `@.horus`, and a node that is genuinely absent from the recorded order now says so instead of silently landing at 0 with everything else. The existing tests could not catch it: a replay whose ordering is arbitrary still ticks every node the right number of times. The new one asserts the priorities. hlog!'s doc said "The default level is DEFAULT_LEVEL (`info`, i.e. `debug` is off)". DEFAULT_LEVEL is LEVEL_DEBUG, and three other places in that file say so, including the constant's own doc and the test nothing_is_suppressed_by_default. That is the doc a node author reads before deciding a `hlog!(debug, ...)` in a tick loop is free — it is not: unconfigured, every one runs format!, a timestamp and a serialise into the shared buffer. The guard reads the constant and the macro's doc block and requires them to agree; it lives in its own test file so it cannot match its own text, and it fails when either side moves. launch.rs still carried this branch's "INERT ... nothing reads it" doc and a --dry-run line reading "IGNORED, see the warning at launch", both made false by main's #194 — the merge took main's behaviour and left its own description behind. The field is honoured, and what it does (a separate shared-memory tree, so that node cannot see its siblings) is now what the doc and the plan both say. run_launch and print_launch_plan go through one function so they cannot disagree again. node_namespace_construction was a test that computed `global/local/node` from its own inputs and asserted the result equalled what it had just computed — it called nothing from the module and described an identity no process is given. Replaced with one that exercises the shipped function. --- horus_core/src/core/hlog.rs | 10 +- .../src/scheduling/scheduler/recording.rs | 40 +++++++- horus_core/tests/hlog_default_level_doc.rs | 66 +++++++++++++ horus_core/tests/record_replay_e2e.rs | 78 ++++++++++++++++ horus_manager/src/commands/launch.rs | 92 ++++++++++--------- 5 files changed, 241 insertions(+), 45 deletions(-) create mode 100644 horus_core/tests/hlog_default_level_doc.rs diff --git a/horus_core/src/core/hlog.rs b/horus_core/src/core/hlog.rs index 4744da989..aa75c47a4 100644 --- a/horus_core/src/core/hlog.rs +++ b/horus_core/src/core/hlog.rs @@ -498,9 +498,13 @@ pub fn format_console_line( /// not-taken branch: the level is checked **before** the format arguments are /// evaluated, so a suppressed line pays for neither `format!` nor its /// arguments' `Display` impls. The default level is -/// [`DEFAULT_LEVEL`](crate::core::hlog::DEFAULT_LEVEL) (`info`, i.e. `debug` -/// is off); change it with `HORUS_LOG_LEVEL=debug` or -/// [`set_max_level`](crate::core::hlog::set_max_level). +/// [`DEFAULT_LEVEL`](crate::core::hlog::DEFAULT_LEVEL), and it is `debug` — +/// **nothing is suppressed until you configure it**, so in an unconfigured +/// process a `hlog!(debug, ...)` pays for its `format!`, a timestamp and a +/// serialise into the shared log buffer. Set `HORUS_LOG_LEVEL=info` (or call +/// [`set_max_level`](crate::core::hlog::set_max_level)) to get the not-taken +/// branch. This paragraph said `info` for a while; see `DEFAULT_LEVEL`'s own +/// doc for why the permissive default is deliberate. /// /// The corollary is that arguments with side effects are not evaluated when /// the line is suppressed — which is the intended semantics of a level, and diff --git a/horus_core/src/scheduling/scheduler/recording.rs b/horus_core/src/scheduling/scheduler/recording.rs index e516b4759..d8077100b 100644 --- a/horus_core/src/scheduling/scheduler/recording.rs +++ b/horus_core/src/scheduling/scheduler/recording.rs @@ -193,6 +193,16 @@ impl Scheduler { }) .unwrap_or_default(); + // How many nodes we could not place. The lookup below is keyed by NODE + // NAME, and it used to be handed `node_id` — a generated hex string + // (`format!("{:x}{:x}", unix_nanos, node_count)`) that can never equal + // a name. Every lookup missed, `unwrap_or(0)` gave every node priority + // 0, and `add_replay`'s stable sort then left the replay in + // `node_recordings` HashMap order: neither the recorded order nor the + // same order twice. That is the exact opposite of the "tick-perfect + // determinism" this module is for, and nothing said a word. + let mut unplaced: Vec<&str> = Vec::new(); + for (node_id, relative_path) in &scheduler_recording.node_recordings { // `relative_path` comes verbatim out of the recording file. Joined // unchecked, `../../../..` escapes the session directory entirely @@ -203,7 +213,23 @@ impl Scheduler { crate::scheduling::record_replay::sanitize_path_component(relative_path), ); if node_path.exists() { - let priority = priority_map.get(node_id.as_str()).copied().unwrap_or(0); + // The name is the part of `@.horus` before the LAST + // `@`, which is how `Scheduler::add` writes it — `rsplit_once` + // so a node whose own name contains `@` still resolves. + let node_name = relative_path + .strip_suffix(".horus") + .unwrap_or(relative_path) + .rsplit_once('@') + .map(|(name, _id)| name) + .unwrap_or(relative_path); + + let priority = match priority_map.get(node_name) { + Some(p) => *p, + None => { + unplaced.push(node_name); + 0 + } + }; if let Err(e) = scheduler.add_replay(node_path, priority) { print_line(&format!( "Warning: Failed to load node '{}': {}", @@ -213,6 +239,18 @@ impl Scheduler { } } + // Silence is what let the by-id lookup survive: every node missing is + // indistinguishable from every node placed, because both end at + // priority 0. Say so, once, with the names. + if !unplaced.is_empty() && !priority_map.is_empty() { + print_line(&format!( + "[REPLAY] Warning: {} node(s) are not in the recorded execution \ + order and will replay in an arbitrary position: {}", + unplaced.len(), + unplaced.join(", ") + )); + } + Ok(scheduler) } diff --git a/horus_core/tests/hlog_default_level_doc.rs b/horus_core/tests/hlog_default_level_doc.rs new file mode 100644 index 000000000..c3cd89652 --- /dev/null +++ b/horus_core/tests/hlog_default_level_doc.rs @@ -0,0 +1,66 @@ +//! The `hlog!` doc must not name a default level the constant contradicts. +//! +//! `hlog!`'s doc-comment claimed "The default level is `DEFAULT_LEVEL` (`info`, +//! i.e. `debug` is off)" while `DEFAULT_LEVEL` in the same file was — and still +//! is — `LEVEL_DEBUG`. Three other places in that file say so outright, +//! including the constant's own doc and the test `nothing_is_suppressed_by_default`. +//! +//! That is the doc a node author reads before deciding whether a +//! `hlog!(debug, ...)` in a tick loop is free. It is not: in an unconfigured +//! process every one of them runs `format!`, `chrono::Local::now()` and a +//! bincode serialise into the shared buffer. +//! +//! This guard lives in its own file, not in `hlog.rs`'s test module, so it +//! cannot match its own text. + +use std::path::PathBuf; + +fn hlog_source() -> String { + let path: PathBuf = [env!("CARGO_MANIFEST_DIR"), "src", "core", "hlog.rs"] + .iter() + .collect(); + std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("reading {}: {e}", path.display())) +} + +/// The macro doc and the constant must agree about the shipped default. +#[test] +fn the_hlog_macro_doc_agrees_with_default_level() { + let src = hlog_source(); + + let debug_default = src.contains("pub const DEFAULT_LEVEL: u8 = LEVEL_DEBUG;"); + let info_default = src.contains("pub const DEFAULT_LEVEL: u8 = LEVEL_INFO;"); + assert!( + debug_default ^ info_default, + "could not find the DEFAULT_LEVEL declaration — has it been renamed or \ + given a different form? This guard reads it literally." + ); + + // Only the `hlog!` macro's own doc block, so the constant's doc (which + // legitimately discusses `Info` as the conventional-but-rejected default) + // is not scanned. + let start = src + .find("/// # Filtering") + .expect("the hlog! doc's Filtering section is the anchor for this guard"); + let end = src[start..] + .find("/// # Example") + .map(|i| start + i) + .expect("the Filtering section should be followed by Example"); + let filtering = &src[start..end]; + + if debug_default { + assert!( + filtering.contains("it is `debug`"), + "DEFAULT_LEVEL is LEVEL_DEBUG but the hlog! doc does not say so:\n{filtering}" + ); + assert!( + !filtering.contains("(`info`, i.e. `debug`\n/// is off)"), + "DEFAULT_LEVEL is LEVEL_DEBUG and the hlog! doc still claims `info`, \ + debug off — the exact claim this guard exists for:\n{filtering}" + ); + } else { + assert!( + !filtering.contains("it is `debug`"), + "DEFAULT_LEVEL is LEVEL_INFO but the hlog! doc still says `debug`:\n{filtering}" + ); + } +} diff --git a/horus_core/tests/record_replay_e2e.rs b/horus_core/tests/record_replay_e2e.rs index 4c134f0b0..bea8d773f 100644 --- a/horus_core/tests/record_replay_e2e.rs +++ b/horus_core/tests/record_replay_e2e.rs @@ -1839,3 +1839,81 @@ fn test_recording_with_no_publisher_has_empty_inputs() { ); } } + +// ============================================================================ +// Test: replay restores the RECORDED execution order +// ============================================================================ + +/// `replay_from` must give each node the priority its recorded position had. +/// +/// It looked the priority up by `node_id` — the key of `node_recordings`, a +/// generated hex string (`format!("{:x}{:x}", unix_nanos, node_count)`) — while +/// the map is keyed by node NAME. A node id can never equal a name, so every +/// lookup missed, `unwrap_or(0)` gave every node priority 0, and `add_replay`'s +/// stable sort left the replay in `node_recordings` HashMap order: neither the +/// recorded order nor the same order twice. +/// +/// This asserts the priorities, not the tick counts, because a replay whose +/// node ordering is arbitrary still ticks every node the right number of times +/// — which is why the existing tests passed throughout. +#[test] +fn replay_restores_the_recorded_execution_order() { + use horus_core::scheduling::SchedulerRecording; + + let tmp = TestTempDir::new("replay_order"); + let session = tmp.path().to_path_buf(); + + // Three nodes, recorded in a known order that is NOT alphabetical and not + // the order they are registered in below. + let order = ["zulu", "alpha", "mike"]; + + let mut scheduler_recording = SchedulerRecording::new("sched_id", "order_session"); + for (i, name) in order.iter().enumerate() { + // The id shape `Scheduler::add` generates: a hex string, never a name. + let node_id = format!("{:x}{:x}", 1_700_000_000_000_000_000u64 + i as u64, i); + let relative_path = format!("{name}@{node_id}.horus"); + + let mut rec = NodeRecording::new(name, &node_id, "order_session"); + for tick in 0..3u64 { + rec.add_snapshot(NodeTickSnapshot::new(tick).with_duration(100)); + } + rec.finish(); + rec.save(&session.join(&relative_path)) + .expect("write node recording"); + + scheduler_recording.add_node_recording(&node_id, &relative_path); + } + scheduler_recording.record_execution_order(order.iter().map(|s| s.to_string()).collect()); + + let scheduler_path = session.join("scheduler@sched_id.horus"); + scheduler_recording + .save(&scheduler_path) + .expect("write scheduler recording"); + + let scheduler = Scheduler::replay_from(scheduler_path).expect("replay_from"); + + let mut by_name: Vec<(String, u32)> = scheduler + .metrics() + .into_iter() + .map(|m| (m.name().to_string(), m.order())) + .collect(); + by_name.sort_by_key(|(_, order)| *order); + + let replayed: Vec<&str> = by_name.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!( + replayed, + order.to_vec(), + "replay must restore the recorded execution order, got priorities {by_name:?}" + ); + + // And the priorities must be distinct — all-zero is the failure mode, and + // it looks like a valid order until you notice the sort has nothing to do. + let distinct: std::collections::BTreeSet = + by_name.iter().map(|(_, order)| *order).collect(); + assert_eq!( + distinct.len(), + order.len(), + "every replayed node got the same priority, so the ordering is whatever \ + the recordings HashMap iterated: {by_name:?}" + ); +} diff --git a/horus_manager/src/commands/launch.rs b/horus_manager/src/commands/launch.rs index 80a3cf175..27e3e50d5 100644 --- a/horus_manager/src/commands/launch.rs +++ b/horus_manager/src/commands/launch.rs @@ -72,20 +72,17 @@ pub struct LaunchNode { #[serde(default)] pub args: Vec, - /// INERT. Parsed and warned about; nothing reads it. + /// Shared-memory tree for THIS node, overriding the file-level one. /// - /// This said "Namespace prefix for topics", which the runtime cannot do: - /// there is no topic prefix or remap mechanism in horus_core. The - /// FILE-level `namespace:` works because `HORUS_NAMESPACE` names the shared - /// memory tree, which is isolation rather than prefixing, and horus_sys - /// states that granularity as "per robot" — the file-level key, not this - /// one. Composing this into a child's `HORUS_NAMESPACE` would put the node - /// in a different SHM tree from its siblings and break the launch it - /// belongs to. + /// Not a topic prefix. There is no prefix or remap mechanism in horus_core; + /// `HORUS_NAMESPACE` names the shared-memory tree, which is isolation. This + /// is exported after the file-level value so the more specific one wins. /// - /// Kept so an existing launch file still parses, and warned about at launch - /// so nobody keeps believing it does something. Giving it real meaning is a - /// horus_core design decision (topic remapping), not a launcher change. + /// **A node with its own namespace cannot see its siblings' topics, and + /// they cannot see its.** That is what a separate SHM tree means, and it is + /// the whole effect of this key — so set it when you want that node + /// isolated, and leave it unset otherwise. If you wanted a topic prefix, + /// this is not it, and HORUS does not have one. #[serde(default)] pub namespace: Option, @@ -1081,25 +1078,42 @@ fn should_restart_node(policy: &str, status: &std::process::ExitStatus) -> bool } } +/// The shared-memory tree a node is launched into: its own, else the file's. +/// +/// One function so `run_launch` and `print_launch_plan` cannot disagree about +/// which value wins — `--dry-run` describing a launch that does not happen is +/// the defect this file has already had twice. +fn effective_namespace<'a>( + file_level: Option<&'a str>, + node_level: Option<&'a str>, +) -> Option<&'a str> { + node_level.or(file_level) +} + /// Print the launch plan (dry run) fn print_launch_plan(config: &LaunchConfig, global_namespace: &Option) { for (i, node) in config.nodes.iter().enumerate() { // The name the child is actually given (`HORUS_NODE_NAME`), which is // the bare one. This used to compose `global/local/name` — an identity // no process ever receives, so `--dry-run` described a launch that does - // not happen. The file-level namespace is shown separately below, - // because it does do something: it names the SHM tree. + // not happen. Both namespaces are shown separately below, because they + // do something: each names an SHM tree, and a node-level one overrides + // the file-level value for that node alone. println!(" {}. {}", i + 1, node.name.white().bold()); - if let Some(ref ns) = global_namespace { - println!(" {} {} (shared-memory tree)", "Namespace:".dimmed(), ns); - } - if let Some(ref ns) = node.namespace { - println!( + match ( + effective_namespace(global_namespace.as_deref(), node.namespace.as_deref()), + node.namespace.is_some(), + ) { + (Some(ns), true) => println!( " {} {} {}", "Namespace:".dimmed(), ns, - "— IGNORED, see the warning at launch".yellow() - ); + "(shared-memory tree — isolated from the other nodes)".yellow() + ), + (Some(ns), false) => { + println!(" {} {} (shared-memory tree)", "Namespace:".dimmed(), ns) + } + (None, _) => {} } if let Some(ref pkg) = node.package { @@ -1269,10 +1283,7 @@ fn launch_node( // // Node-level last so it wins: a setting on the node is more specific than // the file's default, which is the only reason to write one. - if let Some(ref ns) = namespace { - cmd.env("HORUS_NAMESPACE", ns); - } - if let Some(ref ns) = node.namespace { + if let Some(ns) = effective_namespace(namespace.as_deref(), node.namespace.as_deref()) { cmd.env("HORUS_NAMESPACE", ns); } @@ -1614,23 +1625,22 @@ nodes: ); } + /// A node-level `namespace:` overrides the file-level one for that node. + /// + /// This test used to compute `global/local/node` from its own inputs and + /// assert the result equalled what it had just computed — it called nothing + /// from this module and would have passed against any implementation. The + /// composed identity it described is gone (no process was ever given it), + /// and what the key does now is select a shared-memory tree. #[test] - fn node_namespace_construction() { - // Mirrors the logic in run_launch/print_launch_plan - let cases = vec![ - (Some("global"), Some("local"), "node", "global/local/node"), - (Some("global"), None, "node", "global/node"), - (None, Some("local"), "node", "local/node"), - (None, None, "node", "node"), - ]; - for (global, local, name, expected) in cases { - let full = match (global, local) { - (Some(g), Some(l)) => format!("{}/{}/{}", g, l, name), - (Some(ns), None) | (None, Some(ns)) => format!("{}/{}", ns, name), - (None, None) => name.to_string(), - }; - assert_eq!(full, expected); - } + fn a_node_level_namespace_overrides_the_file_level_one() { + assert_eq!( + effective_namespace(Some("/robot"), Some("/arm")), + Some("/arm") + ); + assert_eq!(effective_namespace(Some("/robot"), None), Some("/robot")); + assert_eq!(effective_namespace(None, Some("/arm")), Some("/arm")); + assert_eq!(effective_namespace(None, None), None); } #[test] From 1a5af5ef15a74f1de0ae279920257928d117e486 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 15:57:52 -0700 Subject: [PATCH 55/79] fix: three more places the code did not do what its own file says MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deny_export was never consulted for a system topic. allow_export handles `_horus.*` explicitly and its comment says `deny_export = ["_horus.logs"]` "now actually works", with a green test beside it asserting exactly that. Both were true of the guard function and false of the process: allow_export's one production caller filters self.readers, which is populated from the TopicRegistry, and nothing registers `_horus.logs` or `_horus.presence` there. They left the machine through send_system_topic, which never asked — every Error and Warning line this node emits and its full node inventory, multicast once per timer tick, to any host that announced an interest. An operator who set deny_export got no change and a passing test telling them otherwise. send_system_topic now asks. `_horus.estop` does not come through it, so this cannot silence a fleet e-stop. topic_name_from_c says "Every topic constructor starts here, which is what keeps the two invariants the last-error channel rests on". The two JsonWireMessage constructors did not: they called CStr::from_ptr(name) directly — undefined behaviour on null, the exact "segfault at best" case the guard closes — and never touched the error slot, so horus_topic_last_error after one of them answered with whatever the previous topic had failed for. json_wire is the transport under every C++ service and action call. The new test enumerates the constructors by hand rather than by grep, because a scan can only find the ones that already look like the others, and it arms a stale reason before each call so a constructor that neither clears nor sets its own is caught. While there: the four service and action constructors had the same null-CStr::from_ptr UB with no doc claiming otherwise. Adjacent rather than part of the same defect, but it is the same hazard on the same public ABI, so they now refuse a null name too. TransformFrame's wait_for_transform and tf_at_strict documented `Raises: ValueError`. Both route through to_py_err, which maps Timeout to HorusTimeoutError and Transform to HorusTransformError — built on PyException, so neither is a ValueError subclass. The `except ValueError` recovery path the docstring told you to write never caught the timeout, and it left the node's tick() on exactly the case the timeout parameter exists to handle. The test catches the documented exception and also reads the docstring, so the two cannot drift apart again. --- horus_cpp/src/c_api.rs | 221 ++++++++++++++++-- horus_net/src/replicator.rs | 73 ++++++ horus_py/src/transform_frame.rs | 9 +- .../tests/test_tf_raises_what_it_documents.py | 61 +++++ 4 files changed, 340 insertions(+), 24 deletions(-) create mode 100644 horus_py/tests/test_tf_raises_what_it_documents.py diff --git a/horus_cpp/src/c_api.rs b/horus_cpp/src/c_api.rs index c0a05c067..07724ed41 100644 --- a/horus_cpp/src/c_api.rs +++ b/horus_cpp/src/c_api.rs @@ -489,6 +489,23 @@ unsafe fn topic_name_from_c<'a>(name: *const c_char) -> Option<&'a str> { } } +/// Borrow a service or action endpoint name out of C, or give back `None`. +/// +/// These are not topic constructors, so they do not touch the topic last-error +/// slot — but `CStr::from_ptr` is undefined behaviour on null just the same, +/// and `horus_c.h` is a public ABI that any C caller can pass null to. Same +/// hazard as the one `topic_name_from_c` closes, minus the error channel. +/// +/// # Safety +/// `name` must be null or a pointer to a NUL-terminated string that stays +/// valid and unwritten for the duration of the call. +unsafe fn endpoint_name_from_c<'a>(name: *const c_char) -> Option<&'a str> { + if name.is_null() { + return None; + } + CStr::from_ptr(name).to_str().ok() +} + /// Why the last publisher/subscriber constructor on THIS thread returned null. /// /// Writes at most `buf_len - 1` bytes plus a NUL and returns the bytes written, @@ -899,16 +916,20 @@ pub struct HorusJsonWireMsg { pub _padding: [u8; 11], } -/// Create JsonWireMessage publisher. +/// Create JsonWireMessage publisher. Returns null on error; +/// `horus_topic_last_error` then holds the reason. #[no_mangle] pub unsafe extern "C" fn horus_publisher_json_wire_new(name: *const c_char) -> *mut HorusPublisher { - let name = match CStr::from_ptr(name).to_str() { - Ok(n) => n, - Err(_) => return std::ptr::null_mut(), + let name = match topic_name_from_c(name) { + Some(n) => n, + None => return std::ptr::null_mut(), }; match topic_ffi::publisher_json_wire_new(name) { Ok(pub_) => Box::into_raw(pub_) as *mut HorusPublisher, - Err(_) => std::ptr::null_mut(), + Err(e) => { + set_topic_error(e); + std::ptr::null_mut() + } } } @@ -938,18 +959,22 @@ pub unsafe extern "C" fn horus_publisher_json_wire_send( topic_ffi::publisher_json_wire_send(pub_, wire); } -/// Create JsonWireMessage subscriber. +/// Create JsonWireMessage subscriber. Returns null on error; +/// `horus_topic_last_error` then holds the reason. #[no_mangle] pub unsafe extern "C" fn horus_subscriber_json_wire_new( name: *const c_char, ) -> *mut HorusSubscriber { - let name = match CStr::from_ptr(name).to_str() { - Ok(n) => n, - Err(_) => return std::ptr::null_mut(), + let name = match topic_name_from_c(name) { + Some(n) => n, + None => return std::ptr::null_mut(), }; match topic_ffi::subscriber_json_wire_new(name) { Ok(sub) => Box::into_raw(sub) as *mut HorusSubscriber, - Err(_) => std::ptr::null_mut(), + Err(e) => { + set_topic_error(e); + std::ptr::null_mut() + } } } @@ -1488,9 +1513,9 @@ pub struct HorusServiceServer { /// Create a service client. #[no_mangle] pub unsafe extern "C" fn horus_service_client_new(name: *const c_char) -> *mut HorusServiceClient { - let name = match CStr::from_ptr(name).to_str() { - Ok(n) => n, - Err(_) => return std::ptr::null_mut(), + let name = match endpoint_name_from_c(name) { + Some(n) => n, + None => return std::ptr::null_mut(), }; Box::into_raw(service_ffi::service_client_new(name)) as *mut HorusServiceClient } @@ -1538,9 +1563,9 @@ pub unsafe extern "C" fn horus_service_client_call( /// Create a service server. #[no_mangle] pub unsafe extern "C" fn horus_service_server_new(name: *const c_char) -> *mut HorusServiceServer { - let name = match CStr::from_ptr(name).to_str() { - Ok(n) => n, - Err(_) => return std::ptr::null_mut(), + let name = match endpoint_name_from_c(name) { + Some(n) => n, + None => return std::ptr::null_mut(), }; Box::into_raw(service_ffi::service_server_new(name)) as *mut HorusServiceServer } @@ -1618,9 +1643,9 @@ pub struct HorusActionGoalHandle { /// Create an action client. #[no_mangle] pub unsafe extern "C" fn horus_action_client_new(name: *const c_char) -> *mut HorusActionClient { - let name = match CStr::from_ptr(name).to_str() { - Ok(n) => n, - Err(_) => return std::ptr::null_mut(), + let name = match endpoint_name_from_c(name) { + Some(n) => n, + None => return std::ptr::null_mut(), }; Box::into_raw(action_ffi::action_client_new(name)) as *mut HorusActionClient } @@ -1758,9 +1783,9 @@ pub unsafe extern "C" fn horus_goal_handle_destroy(handle: *mut HorusGoalHandle) /// Create an action server. #[no_mangle] pub unsafe extern "C" fn horus_action_server_new(name: *const c_char) -> *mut HorusActionServer { - let name = match CStr::from_ptr(name).to_str() { - Ok(n) => n, - Err(_) => return std::ptr::null_mut(), + let name = match endpoint_name_from_c(name) { + Some(n) => n, + None => return std::ptr::null_mut(), }; Box::into_raw(action_ffi::action_server_new(name)) as *mut HorusActionServer } @@ -2171,6 +2196,158 @@ mod tests { std::env::var_os("HORUS_CPP_FFI_EMIT_CHILD").is_some() } + /// Every topic constructor must refuse a null name and report why. + /// + /// `topic_name_from_c`'s doc says "Every topic constructor starts here, + /// which is what keeps the two invariants the last-error channel rests on". + /// The two JsonWireMessage constructors did not: they called + /// `CStr::from_ptr(name)` directly, which is undefined behaviour on null — + /// the exact "segfault at best" case the guard was written to close — and + /// never cleared or set the error slot, so `horus_topic_last_error` after + /// one of them answered with whatever the previous topic had failed for. + /// + /// json_wire is the transport under every C++ service and action call, so + /// it is the least obscure entry point in the file, not the most. + /// + /// Enumerated by hand rather than by grep: a list that scans the source can + /// only find the constructors that already look like the others. + #[test] + fn every_topic_constructor_refuses_a_null_name_with_a_reason() { + // (label, constructor returns non-null?) + let attempts: Vec<(&str, bool)> = unsafe { + vec![ + ( + "publisher_cmd_vel", + !horus_publisher_cmd_vel_new(std::ptr::null()).is_null(), + ), + ( + "subscriber_cmd_vel", + !horus_subscriber_cmd_vel_new(std::ptr::null()).is_null(), + ), + ( + "publisher_json_wire", + !horus_publisher_json_wire_new(std::ptr::null()).is_null(), + ), + ( + "subscriber_json_wire", + !horus_subscriber_json_wire_new(std::ptr::null()).is_null(), + ), + ( + "publisher_laser_scan", + !horus_publisher_laser_scan_new(std::ptr::null()).is_null(), + ), + ( + "subscriber_laser_scan", + !horus_subscriber_laser_scan_new(std::ptr::null()).is_null(), + ), + ( + "publisher_imu", + !horus_publisher_imu_new(std::ptr::null()).is_null(), + ), + ( + "subscriber_imu", + !horus_subscriber_imu_new(std::ptr::null()).is_null(), + ), + ] + }; + + for (label, returned_non_null) in &attempts { + assert!(!returned_non_null, "{label} accepted a null topic name"); + } + + // And each one must have left its own reason behind. Checked after the + // loop above only for the last call; do it per-call so a constructor + // that forgets `set_topic_error` is caught rather than reading the + // previous one's reason. + unsafe fn reason() -> String { + let mut buf = [0u8; 128]; + let n = horus_topic_last_error(buf.as_mut_ptr(), buf.len()); + assert!(n >= 0, "horus_topic_last_error should not fail here"); + String::from_utf8_lossy(&buf[..n as usize]).into_owned() + } + + unsafe { + let ctors: Vec<(&str, Box)> = vec![ + ( + "publisher_cmd_vel", + Box::new(|| { + horus_publisher_cmd_vel_new(std::ptr::null()); + }), + ), + ( + "subscriber_cmd_vel", + Box::new(|| { + horus_subscriber_cmd_vel_new(std::ptr::null()); + }), + ), + ( + "publisher_json_wire", + Box::new(|| { + horus_publisher_json_wire_new(std::ptr::null()); + }), + ), + ( + "subscriber_json_wire", + Box::new(|| { + horus_subscriber_json_wire_new(std::ptr::null()); + }), + ), + ( + "publisher_laser_scan", + Box::new(|| { + horus_publisher_laser_scan_new(std::ptr::null()); + }), + ), + ( + "subscriber_laser_scan", + Box::new(|| { + horus_subscriber_laser_scan_new(std::ptr::null()); + }), + ), + ( + "publisher_imu", + Box::new(|| { + horus_publisher_imu_new(std::ptr::null()); + }), + ), + ( + "subscriber_imu", + Box::new(|| { + horus_subscriber_imu_new(std::ptr::null()); + }), + ), + ]; + + for (label, call) in ctors { + // Arm a reason that names a DIFFERENT topic, so a constructor + // that neither clears nor sets the slot reads as this one. + set_topic_error("stale reason from some other topic".to_string()); + call(); + let got = reason(); + assert!( + got.contains("null"), + "{label} returned null but left `{got}` in the error slot — \ + it neither cleared nor set its own reason" + ); + } + } + } + + /// A service or action endpoint name must not be dereferenced when null. + /// + /// Not covered by the topic invariant above (these do not touch the topic + /// error slot), but `CStr::from_ptr` is undefined behaviour on null in + /// exactly the same way, and `horus_c.h` is a public ABI. + #[test] + fn service_and_action_constructors_refuse_a_null_name() { + unsafe { + assert!(horus_service_client_new(std::ptr::null()).is_null()); + assert!(horus_service_server_new(std::ptr::null()).is_null()); + assert!(horus_action_client_new(std::ptr::null()).is_null()); + assert!(horus_action_server_new(std::ptr::null()).is_null()); + } + } + /// A C++ log line must carry a time, like every other language's does. /// /// Both C++ entry points built their `LogEntry` with diff --git a/horus_net/src/replicator.rs b/horus_net/src/replicator.rs index aef9fe16f..5be07bf1b 100644 --- a/horus_net/src/replicator.rs +++ b/horus_net/src/replicator.rs @@ -1231,6 +1231,26 @@ impl Replicator { /// Send a system topic payload to all known peers + multicast. fn send_system_topic(&mut self, topic_name: &str, payload: &[u8]) { + // Ask the export guard, which is the only place that ever asks it about + // a system topic. + // + // `ImportExportGuard::allow_export` handles `_horus.*` explicitly and + // its comment says `deny_export = ["_horus.logs"]` "now actually + // works" — but its one production caller, `handle_export`, filters + // `self.readers`, and a reader only exists for a topic in the + // `TopicRegistry`. Nothing registers `_horus.logs` or `_horus.presence` + // there, so the guard was never asked about either, and they left the + // machine down this path instead: every Error and Warning line and the + // full node inventory, multicast once per timer tick, to any host that + // announced an interest. An operator who set `deny_export` got no + // change in behaviour and a green test telling them it was honoured. + // + // `_horus.estop` does not come through here (EstopBroadcaster owns it), + // so this cannot be used to silence a fleet e-stop. + if !self.guard.allow_export(topic_name) { + return; + } + // Share the framing (and therefore the monotonic sequence) with the // e-stop path. This used to build its own header with a hard-coded // `sequence: 0`, which the receiver's dedup — which runs ahead of the @@ -1389,6 +1409,59 @@ mod tests { // These drive the private receive path directly. Run with --test-threads=1: // the data-packet test installs the global external emergency-stop hook. + /// `deny_export` on a system topic must actually stop it leaving. + /// + /// `ImportExportGuard::allow_export` handles `_horus.*` and its comment says + /// `deny_export = ["_horus.logs"]` "now actually works" — with a green test + /// beside it asserting exactly that. Both were true of the guard function + /// and false of the process: `allow_export`'s only production caller filters + /// `self.readers`, which is populated from the `TopicRegistry`, and nothing + /// registers `_horus.logs` or `_horus.presence` there. They left the machine + /// through `send_system_topic` instead, which never asked. + /// + /// Asserted on `system_seq` rather than on packets: `frame_system_topic` + /// advances it exactly once per message that gets framed, so it counts what + /// was sent without needing a peer, a socket or a multicast group. + #[test] + fn deny_export_on_a_system_topic_stops_it_being_broadcast() { + use crate::registry::{SYSTEM_TOPIC_LOGS, SYSTEM_TOPIC_PRESENCE}; + + let registry = Arc::new(TopicRegistry::new()); + let mut config = NetConfig::test_config(0); + config.deny_export = vec![SYSTEM_TOPIC_LOGS.to_string()]; + let mut rep = Replicator::new(registry, config).unwrap(); + + let before = rep.system_seq; + rep.send_system_topic(SYSTEM_TOPIC_LOGS, b"an error line"); + assert_eq!( + rep.system_seq, before, + "a denied system topic must not be framed or sent" + ); + + // The one that was NOT denied still goes, so this is a filter and not + // an off switch. + rep.send_system_topic(SYSTEM_TOPIC_PRESENCE, b"node inventory"); + assert_eq!( + rep.system_seq, + before + 1, + "denying one system topic must not silence the others" + ); + } + + /// The default is unchanged: zero-config LAN replication still exports both. + #[test] + fn system_topics_are_still_exported_by_default() { + use crate::registry::{SYSTEM_TOPIC_LOGS, SYSTEM_TOPIC_PRESENCE}; + + let registry = Arc::new(TopicRegistry::new()); + let mut rep = Replicator::new(registry, NetConfig::test_config(0)).unwrap(); + + let before = rep.system_seq; + rep.send_system_topic(SYSTEM_TOPIC_LOGS, b"an error line"); + rep.send_system_topic(SYSTEM_TOPIC_PRESENCE, b"node inventory"); + assert_eq!(rep.system_seq, before + 2); + } + fn test_replicator() -> Replicator { let registry = Arc::new(TopicRegistry::new()); Replicator::new(registry, NetConfig::test_config(0)).unwrap() diff --git a/horus_py/src/transform_frame.rs b/horus_py/src/transform_frame.rs index 857631b5d..ff6e45d18 100644 --- a/horus_py/src/transform_frame.rs +++ b/horus_py/src/transform_frame.rs @@ -482,7 +482,11 @@ impl PyTransformFrame { /// Transform once available /// /// Raises: - /// ValueError: If timeout expires before transform is available + /// horus.HorusTimeoutError: If timeout expires before the transform is + /// available. It derives from ``Exception``, not ``ValueError`` — + /// this said ``ValueError`` for a while, so the ``except + /// ValueError`` recovery path a reader wrote from this docstring + /// never caught the timeout and the error left the node's tick(). #[pyo3(signature = (src, dst, timeout_sec=5.0))] fn wait_for_transform( &self, @@ -662,7 +666,8 @@ impl PyTransformFrame { /// Interpolated transform at the given timestamp /// /// Raises: - /// ValueError: If timestamp is outside buffered range + /// horus.HorusTransformError: If the timestamp is outside the buffered + /// range. It derives from ``Exception``, not ``ValueError``. fn tf_at_strict(&self, src: &str, dst: &str, timestamp_ns: u64) -> PyResult { self.inner .tf_at_strict(src, dst, timestamp_ns) diff --git a/horus_py/tests/test_tf_raises_what_it_documents.py b/horus_py/tests/test_tf_raises_what_it_documents.py new file mode 100644 index 000000000..7b11e2602 --- /dev/null +++ b/horus_py/tests/test_tf_raises_what_it_documents.py @@ -0,0 +1,61 @@ +"""The TransformFrame docstrings must name the exception the code raises. + +`wait_for_transform` and `tf_at_strict` documented `ValueError`. They route +through `to_py_err`, which maps `HorusError::Timeout` to `HorusTimeoutError` +and `HorusError::Transform` to `HorusTransformError` — both built on +`PyException`, so neither is a subclass of `ValueError`. The recovery path the +docstring told you to write (`except ValueError`) never caught the timeout, and +it escaped the node's tick() on exactly the case the timeout parameter exists +to handle. +""" + +import inspect + +import pytest + +import horus + + +def _tf(): + return horus.TransformFrame() + + +def test_wait_for_transform_raises_the_documented_exception(): + tf = _tf() + with pytest.raises(horus.HorusTimeoutError): + tf.wait_for_transform("nowhere_src", "nowhere_dst", 0.01) + + +def test_tf_at_strict_raises_the_documented_exception(): + """Registered frames, a timestamp far outside the buffered range.""" + tf = _tf() + tf.register_frame("world", None) + tf.register_frame("sensor", "world") + at = 4_000_000_000_000 + tf.update_transform("sensor", horus.Transform.from_translation([1.0, 0.0, 0.0]), at) + + # An hour before anything was buffered: strict mode must refuse rather than + # extrapolate, which is the whole difference from tf_at(). + with pytest.raises(horus.HorusTransformError): + tf.tf_at_strict("sensor", "world", at - 3_600_000_000_000) + + +def test_neither_is_a_value_error(): + """Pin the premise: this is why the old docstring was wrong, not a nitpick.""" + assert not issubclass(horus.HorusTimeoutError, ValueError) + assert not issubclass(horus.HorusTransformError, ValueError) + + +@pytest.mark.parametrize( + "method,expected", + [ + ("wait_for_transform", "HorusTimeoutError"), + ("tf_at_strict", "HorusTransformError"), + ], +) +def test_the_docstring_names_that_exception(method, expected): + doc = inspect.getdoc(getattr(horus.TransformFrame, method)) or "" + assert expected in doc, f"{method} docstring does not name {expected}:\n{doc}" + assert "ValueError:" not in doc, ( + f"{method} docstring still promises ValueError, which it does not raise:\n{doc}" + ) From e09a635d4555ae21aefbad7ff9523fd346bbe218 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 17:06:12 -0700 Subject: [PATCH 56/79] fix(scheduling): a diagnostic being written when a drain ran was lost for good MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `rt_diag` claims its slot by bumping `RT_DIAG_HEAD` FIRST, then marks the slot odd, writes the bytes, and only then stores the completing even sequence. A drain landing anywhere in that window got `None` from `read_slot`, hit `continue`, and then ran `*tail = head` — stepping the tail permanently past a line whose producer had done nothing wrong. Nobody could emit it afterwards: the slot is skipped now and the tail never goes back. `read_slot` returned one `None` for two opposite situations. A LAPPED slot has been recycled by a later producer and really is gone. An IN-FLIGHT slot is microseconds from being complete. The drain treated both as "skip it". They are now distinguished — the sequence is derived from the monotonic claim index, so `seq < want` is exactly "claimed, not finished yet" — and an in-flight slot stops the drain instead of being discarded, leaving the line for the next one. Bounded on purpose. A producer that panics between the head bump and its completing store leaves a slot odd forever, and an unbounded retry would then wedge the ring for the life of the process and cost every LATER diagnostic too — strictly worse than the bug being fixed. Stalling twice on the same index is taken as a dead producer: that one line is given up and draining continues. This is what was actually behind the CI failure in `a_compute_only_program_still_prints_its_safing_diagnostics`. The window is nanoseconds on an idle machine, which is why it reproduced on a contended runner and nowhere else, and why the previous commit's shutdown flush did not help: by the time `stop()` flushed, a drain had already stepped over the line and there was nothing left to find. The flush is still right — it is what makes the last diagnostics reach stdout without waiting on a 25 ms poll — but it was not the cause. Two tests, both driving the ring by hand so they do not depend on thread timing: - a drain must not advance the tail past an index whose producer has not finished writing (asserts on the tail, not on what got emitted, because the background drainer would race an "the next drain emits it" assertion) - repeated drains must get past a slot whose producer never completes, so the bound is real They share the one process-global ring, and the second one's bounded skip steps over the first one's in-flight slot, so they take a lock rather than racing each other. Ablation: skipping in-flight slots as before fails the first and leaves the second green. `read_slot`'s existing flood test asserted `.is_none()` for a recycled slot; it now asserts `SlotRead::Lapped`, which is the thing it was always trying to say. --- horus_core/src/scheduling/rt_executor.rs | 184 +++++++++++++++++++++-- 1 file changed, 169 insertions(+), 15 deletions(-) diff --git a/horus_core/src/scheduling/rt_executor.rs b/horus_core/src/scheduling/rt_executor.rs index 16d15b4be..275ce1181 100644 --- a/horus_core/src/scheduling/rt_executor.rs +++ b/horus_core/src/scheduling/rt_executor.rs @@ -178,6 +178,9 @@ static RT_DIAG_TRUNCATED: AtomicU64 = AtomicU64::new(0); /// diagnostic can never wait on a drainer, which is the priority inversion the /// blocking `print_line` had. static RT_DIAG_TAIL: Mutex = Mutex::new(0); +/// The index a drain last stopped on because its producer had not finished +/// writing. Guarded by `RT_DIAG_TAIL`, so plain `Relaxed` is enough. +static RT_DIAG_STALLED_AT: AtomicU64 = AtomicU64::new(u64::MAX); static RT_DIAG_DRAIN_STARTED: Once = Once::new(); @@ -265,11 +268,36 @@ pub(crate) fn rt_diag(args: fmt::Arguments<'_>) { /// The seqlock check is the whole safety argument: `idx` names a *generation*, /// not just a slot, so a reader can always tell "the line I asked for" from /// "some newer line that happens to live here now". -fn read_slot(idx: u64, out: &mut [u8; RT_DIAG_LINE_CAP]) -> Option { +/// What a reader found in a slot it asked for. +/// +/// The two failure cases used to be one `None`, and the drain treated both the +/// same way: skip the index and advance the tail past it. They are opposites. +/// A LAPPED slot is genuinely gone. An IN-FLIGHT slot belongs to a producer +/// that has claimed the index and is still writing it -- it will be complete +/// microseconds later, and skipping it throws away a line that was never lost. +enum SlotRead { + Ready(usize), + /// Claimed, not finished. `seq` is behind the value this index will carry. + InFlight, + /// A later producer has already recycled this slot; the line is gone. + Lapped, +} + +fn read_slot(idx: u64, out: &mut [u8; RT_DIAG_LINE_CAP]) -> SlotRead { let slot = &RT_DIAG_RING[(idx & RT_DIAG_SLOT_MASK) as usize]; let want = idx.wrapping_mul(2).wrapping_add(2); - if slot.seq.load(Ordering::Acquire) != want { - return None; + let classify = |seq: u64| { + // `seq` is derived from a monotonic claim index, so "behind" is a plain + // comparison; u64 does not wrap in any reachable lifetime. + if seq < want { + SlotRead::InFlight + } else { + SlotRead::Lapped + } + }; + let seq = slot.seq.load(Ordering::Acquire); + if seq != want { + return classify(seq); } let len = (slot.len.load(Ordering::Relaxed) as usize).min(RT_DIAG_LINE_CAP); for (i, dst) in out[..len].iter_mut().enumerate() { @@ -280,10 +308,11 @@ fn read_slot(idx: u64, out: &mut [u8; RT_DIAG_LINE_CAP]) -> Option { // being reordered past this load — an `Acquire` *load* would only order // what comes after it. std::sync::atomic::fence(Ordering::Acquire); - if slot.seq.load(Ordering::Relaxed) != want { - return None; + let seq = slot.seq.load(Ordering::Relaxed); + if seq != want { + return classify(seq); } - Some(len) + SlotRead::Ready(len) } /// Print every queued diagnostic through `emit`, then report what was lost. @@ -306,16 +335,52 @@ fn rt_diag_drain(emit: impl Fn(&str)) -> usize { let mut line = [0u8; RT_DIAG_LINE_CAP]; let mut printed = 0usize; + let mut next = *tail; for idx in *tail..head { - let Some(len) = read_slot(idx, &mut line) else { - continue; - }; - if let Ok(text) = std::str::from_utf8(&line[..len]) { - emit(text); - printed += 1; + match read_slot(idx, &mut line) { + SlotRead::Ready(len) => { + if let Ok(text) = std::str::from_utf8(&line[..len]) { + emit(text); + printed += 1; + } + next = idx + 1; + } + SlotRead::Lapped => { + RT_DIAG_DROPPED.fetch_add(1, Ordering::Relaxed); + next = idx + 1; + } + SlotRead::InFlight => { + // The producer bumped `head` and has not finished its store + // yet. Stopping here and retrying leaves the line to the next + // drain, 25 ms away; advancing past it -- which is what this + // loop used to do for every unreadable slot -- discarded a line + // that was in the middle of being written. That is a race the + // producer always loses on a contended machine, and it is how a + // one-line diagnostic followed by an immediate shutdown went + // missing on CI while passing everywhere else. + // + // Bounded, because a producer that panics between claiming the + // index and completing the store would otherwise wedge the ring + // for the life of the process and cost every LATER line too. + // Stalling twice on the same index is taken as a dead producer: + // give up that one line and keep going. + if RT_DIAG_STALLED_AT.swap(idx, Ordering::Relaxed) == idx { + RT_DIAG_DROPPED.fetch_add(1, Ordering::Relaxed); + next = idx + 1; + continue; + } + break; + } } } - *tail = head; + // Only a drain that consumed the whole range clears the marker. `next > + // *tail` would also be true of a drain that emitted three lines and then + // stalled on the fourth -- clearing it there would reset the counter every + // round and the bounded skip could never fire. + if next == head { + RT_DIAG_STALLED_AT.store(u64::MAX, Ordering::Relaxed); + } + *tail = next; drop(tail); let dropped = RT_DIAG_DROPPED.swap(0, Ordering::Relaxed); @@ -2237,6 +2302,95 @@ impl Drop for RtExecutor { } } +#[cfg(test)] +mod diag_ring_inflight { + use super::*; + + /// Both tests below drive the one process-global ring by hand, and the + /// second one's repeated drains perform the bounded skip -- which steps + /// past the first one's deliberately in-flight slot. Run in parallel they + /// fail each other; the lock is what keeps each in sole possession. + static RING_TESTS: Mutex<()> = Mutex::new(()); + + /// A drain must not step over a line that is still being written. + /// + /// `rt_diag` bumps `RT_DIAG_HEAD` first and only then marks the slot + /// complete. A drain landing in that window used to get `None` from + /// `read_slot`, `continue`, and then set `tail = head` -- stepping over the + /// line permanently. The producer had done nothing wrong and the line was + /// never recoverable, so neither the 25 ms drainer nor the flush in + /// `stop()` could ever print it. + /// + /// Nanoseconds wide on an idle machine, which is why it showed up as a CI + /// failure on a contended runner and nowhere else. + /// + /// Asserts on the tail rather than on what got emitted: the background + /// drainer may be running in this process and would race a + /// "the next drain emits it" assertion. + #[test] + fn a_drain_does_not_step_over_a_line_still_being_written() { + let _guard = RING_TESTS.lock().unwrap_or_else(|e| e.into_inner()); + const MARKER: &str = "inflight-slot-marker"; + + // Claim an index and stop where a preempted producer stops: head bumped, + // sequence odd, payload not yet written. + let idx = RT_DIAG_HEAD.fetch_add(1, Ordering::Relaxed); + let slot = &RT_DIAG_RING[(idx & RT_DIAG_SLOT_MASK) as usize]; + slot.seq + .store(idx.wrapping_mul(2).wrapping_add(1), Ordering::Relaxed); + std::sync::atomic::fence(Ordering::Release); + + rt_diag_drain(|_| {}); + + let tail_now = *RT_DIAG_TAIL.lock().unwrap_or_else(|e| e.into_inner()); + assert!( + tail_now <= idx, + "a drain advanced the tail to {tail_now}, past index {idx}, whose \ + producer had not finished writing. That line can never be emitted \ + by anyone: the slot is skipped now and the tail never goes back." + ); + + // Finish the write and drain, so the ring is not left blocked for the + // rest of this test binary. + for (i, b) in MARKER.as_bytes().iter().enumerate() { + slot.bytes[i].store(*b, Ordering::Relaxed); + } + slot.len.store(MARKER.len() as u32, Ordering::Relaxed); + slot.seq + .store(idx.wrapping_mul(2).wrapping_add(2), Ordering::Release); + rt_diag_drain(|_| {}); + } + + /// ...but a producer that dies mid-write must not wedge the ring forever. + /// + /// Retrying an in-flight slot is only safe if it is bounded. A panic + /// between the head bump and the completing store leaves a slot odd for + /// good; without a bound every later diagnostic in the process would queue + /// behind it. The second stall on the same index gives up that one line. + #[test] + fn a_producer_that_dies_mid_write_does_not_wedge_the_ring() { + let _guard = RING_TESTS.lock().unwrap_or_else(|e| e.into_inner()); + let idx = RT_DIAG_HEAD.fetch_add(1, Ordering::Relaxed); + let slot = &RT_DIAG_RING[(idx & RT_DIAG_SLOT_MASK) as usize]; + slot.seq + .store(idx.wrapping_mul(2).wrapping_add(1), Ordering::Relaxed); + std::sync::atomic::fence(Ordering::Release); + + // First drain stalls on it, second gives up on it. + rt_diag_drain(|_| {}); + rt_diag_drain(|_| {}); + rt_diag_drain(|_| {}); + + let tail_now = *RT_DIAG_TAIL.lock().unwrap_or_else(|e| e.into_inner()); + assert!( + tail_now > idx, + "the ring is still stuck at {idx} after repeated drains: a producer \ + that never completes its write would block every later diagnostic \ + for the life of the process." + ); + } +} + #[cfg(test)] mod tests { @@ -5229,7 +5383,7 @@ mod tests { fn find_queued(range: std::ops::Range, pred: impl Fn(&str) -> bool) -> Option { let mut buf = [0u8; RT_DIAG_LINE_CAP]; for idx in range { - let Some(len) = read_slot(idx, &mut buf) else { + let SlotRead::Ready(len) = read_slot(idx, &mut buf) else { continue; }; if let Ok(text) = std::str::from_utf8(&buf[..len]) { @@ -5318,7 +5472,7 @@ mod tests { let mut buf = [0u8; RT_DIAG_LINE_CAP]; assert!( - read_slot(start, &mut buf).is_none(), + matches!(read_slot(start, &mut buf), SlotRead::Lapped), "after {flood} emits into {RT_DIAG_SLOTS} slots the first line must have \ been recycled — an unbounded queue is a memory leak on the RT path" ); From 31e292b0bc37a8903ccefbbb45cf89ee5104e594 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 23:22:23 -0700 Subject: [PATCH 57/79] fix: nine findings from the review, one of which could delete a user's shell rc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not all 21 threads needed code: five describe install/uninstall regex problems this branch already fixed (the `\?`-in-BRE portability bug and the unanchored `HORUS_PREFIX` guards), and one describes a whitespace run in `topic/dispatch.rs` that is no longer there. Those are answered on the threads. Nine were live. Severity first: * `uninstall.sh` could replace a shell profile with an empty file. The rewrite is `{ grep -vF ... > "$profile.horus-new"; } || true`, and the comment above it correctly explains that `|| true` is needed because grep exits 1 when it selects no lines — the rc whose only content was our PATH line. But grep also exits 2 when it cannot READ the file, and the redirect creates the target either way, so `[ -f ]` could not tell those apart and the `mv` clobbered a profile it had failed to read. Now requires the profile to be readable and refuses to turn a non-empty profile into an empty one unless it genuinely held nothing else. * `shm_base_dir_for` interpolated its namespace argument raw. It is public and exists precisely for namespaces that are not this process's own, and callers hand the result to `remove_dir_all` — so `../../../../etc` resolved to `/dev/shm/horus_../../../../etc`, outside the SHM parent entirely. The module already had `sanitize_namespace`, strict enough and used everywhere else; this was the one path that skipped it. Ablated: reverting the one-line fix turns the new test red with the escaping path in the message. * `install.sh`'s fish arm tested only for the PRESENCE of `set -gx HORUS_PREFIX`, so a line naming a DIFFERENT prefix counted as "already configured" and a fish user who reinstalled elsewhere kept an export pointing at the tree the installer had just replaced. The POSIX arm was made value-aware for exactly this; fish was left behind. * `start_log_file_drain_once` memoized the negative answer. `OnceLock:: get_or_init` caches whatever the first call produced, `None` included, so a process that built a `Scheduler` before `HORUS_LOG_FILE` was set never wrote a log file again for its whole life. A `Mutex` keeps the at-most-one-drain guarantee without freezing "no". * `status.code().unwrap_or(1)` reported every signal death — SIGSEGV, SIGKILL, the operator's Ctrl-C — as "exited with code 1". `shell_exit_code` exists for this and returns 128+N; this call site was missed when it was introduced. * `bytes.len() as i32` wraps. A value longer than `i32::MAX` came back negative, and negative is the documented "no such param" return, so an overlong string was indistinguishable from a missing one. Saturates now. * `horus doctor` printed fourteen consecutive spaces mid-sentence. * `control_restart` waited a fixed 400 ms for the scheduler to start. That is a bet that the thread got scheduled inside the window; on a loaded runner it does not, and the test fails reporting 0 inits as though `RestartNode` were broken. Now a bounded poll — same upper bound, no assumption. * `messages_total` — comment only, and deliberately. The reviewer is right that it advances even when `try_send` drops, and the comment defending that said a ring nobody drains is "one the subscriber's own staleness sees". That is circular: `SubscriptionFreshness::refresh` derives freshness from this counter and nothing else, so a publisher hammering a full ring keeps a subscriber "fresh" while it receives nothing — the watchdog cannot see the fault it watches for. Bumping on success would close it and is the right shape, but it moves the counter's meaning at seven call sites across two crates and changes when `StalePolicy::SafeState`/`Stop` fire on a live robot. That belongs in its own change with its own tests, not in a merge. The comment now records the limitation instead of denying it. Verified: `horus_core --lib` 2597 passed, `horus_sys --lib` 225, `control_restart` and `log_file_drain_wiring` green, `cargo fmt --check` and `clippy --all-targets -D warnings` across horus_sys/horus_core/horus_manager clean, and both shell scripts parse under `bash -n` (they are `#!/bin/bash`; a `sh -n` failure at line 115 predates this and is a dash-vs-bash artifact, not a syntax error). --- horus_core/src/communication/topic/mod.rs | 22 +++++++-- horus_core/src/core/log_buffer.rs | 22 +++++++-- horus_core/tests/control_restart.rs | 15 +++++- horus_cpp/src/c_api.rs | 9 +++- horus_manager/src/commands/doctor.rs | 2 +- horus_manager/src/commands/run/mod.rs | 8 +++- horus_sys/src/shm/mod.rs | 56 ++++++++++++++++++++++- install.sh | 16 ++++++- uninstall.sh | 15 +++++- 9 files changed, 150 insertions(+), 15 deletions(-) diff --git a/horus_core/src/communication/topic/mod.rs b/horus_core/src/communication/topic/mod.rs index e1186ff03..7b6c8517c 100644 --- a/horus_core/src/communication/topic/mod.rs +++ b/horus_core/src/communication/topic/mod.rs @@ -4563,9 +4563,25 @@ where // the exact failure the counter was introduced to prevent, and these // are the APIs the docs send users to on critical topics. // - // Counted before the call, like `send()`: this is an attempt count, and - // a publisher hammering a full ring is alive — a ring nobody drains is - // a different fault, and one the subscriber's own staleness sees. + // Counted before the call, like `send()`: this is an attempt count, so + // a publisher hammering a full ring keeps it moving. + // + // KNOWN LIMITATION, and the reason this note is not a reassurance. An + // earlier version of it claimed a ring nobody drains is "a different + // fault, and one the subscriber's own staleness sees". That is + // circular: `SubscriptionFreshness::refresh` derives freshness from + // this counter and from nothing else, so while a full ring drops every + // message the subscriber still observes the count advancing and stays + // "fresh" — the watchdog cannot see the fault it is watching for. + // + // Bumping on SUCCESS instead would close it, and is the right shape: + // `try_send` failing means nothing was published, which is exactly what + // staleness should report. It is not done here because it moves the + // meaning of the counter at seven call sites across two crates and + // changes when `StalePolicy::SafeState`/`Stop` fire on a live robot — + // that belongs in its own change with its own tests, not in a merge. + // Until then, a stalled consumer is visible in `dropped_count()` / + // `send_retry_overruns()`, not in staleness. self.ring.bump_messages_total(); self.ring.try_send(msg) } diff --git a/horus_core/src/core/log_buffer.rs b/horus_core/src/core/log_buffer.rs index 73fd05d83..840ca7b4d 100644 --- a/horus_core/src/core/log_buffer.rs +++ b/horus_core/src/core/log_buffer.rs @@ -650,9 +650,18 @@ pub fn start_log_file_drain() -> Option> { .ok() } -/// The process-wide log-file drain, started at most once. -static LOG_FILE_DRAIN: std::sync::OnceLock>> = - std::sync::OnceLock::new(); +/// The process-wide log-file drain, started at most once — but only once it +/// has actually started. +/// +/// A `OnceLock>` was wrong: `get_or_init` memoizes whatever the first +/// call produced, `None` included. A process that built one `Scheduler` before +/// `HORUS_LOG_FILE` was set — which every test that sets the variable and then +/// constructs a scheduler does, and any program that reads config after an +/// early scheduler — cached "no drain" for its whole life and silently never +/// wrote a log file again. A `Mutex` keeps the at-most-one guarantee (the lock +/// serialises the check and the spawn) without freezing the negative answer. +static LOG_FILE_DRAIN: std::sync::Mutex>> = + std::sync::Mutex::new(None); /// Start the log-file drain once per process, if `HORUS_LOG_FILE` asks for it. /// @@ -667,7 +676,12 @@ static LOG_FILE_DRAIN: std::sync::OnceLock>> /// /// Returns whether a drain thread is running. pub fn start_log_file_drain_once() -> bool { - LOG_FILE_DRAIN.get_or_init(start_log_file_drain).is_some() + let mut slot = LOG_FILE_DRAIN.lock().unwrap_or_else(|e| e.into_inner()); + if slot.is_some() { + return true; + } + *slot = start_log_file_drain(); + slot.is_some() } fn log_drain_loop(log_dir: &str, max_size: u64, max_files: usize) { diff --git a/horus_core/tests/control_restart.rs b/horus_core/tests/control_restart.rs index 5f6e09d5a..7e5950956 100644 --- a/horus_core/tests/control_restart.rs +++ b/horus_core/tests/control_restart.rs @@ -75,8 +75,19 @@ fn restart_command_reinitialises_the_node() { scheduler.run_for(1200_u64.ms()).expect("run_for"); }); - // Let it start and perform the initial init(). - std::thread::sleep(std::time::Duration::from_millis(400)); + // Poll for the startup init() rather than sleeping a fixed 400 ms. + // + // A fixed wait is a bet that the scheduler thread got scheduled inside it. + // On a loaded runner — which is the normal state of CI, and of this repo's + // own workstation under concurrent builds — it does not, and the test fails + // reporting 0 inits as though `RestartNode` were broken. Polling turns the + // same wall clock into an upper bound instead of an assumption: it returns + // as soon as the init lands, and only the diagnostic changes if it never + // does. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while inits.load(Ordering::SeqCst) < 1 && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(5)); + } assert_eq!( inits.load(Ordering::SeqCst), 1, diff --git a/horus_cpp/src/c_api.rs b/horus_cpp/src/c_api.rs index 07724ed41..d0e164ffc 100644 --- a/horus_cpp/src/c_api.rs +++ b/horus_cpp/src/c_api.rs @@ -1452,7 +1452,14 @@ pub unsafe extern "C" fn horus_params_get_string( std::ptr::copy_nonoverlapping(bytes.as_ptr(), buf, copy_len); *buf.add(copy_len) = 0; // The REQUIRED length, not the written one. - bytes.len() as i32 + // + // Saturating rather than `as i32`: the plain cast wraps, so a value + // longer than `i32::MAX` came back negative and a caller following + // the documented convention would read it as the `-1` that means + // "no such param". Saturating keeps the sign honest — the answer is + // still "your buffer is too small", which is the only thing a + // caller can act on, and no buffer that size can be satisfied. + i32::try_from(bytes.len()).unwrap_or(i32::MAX) } None => -1, } diff --git a/horus_manager/src/commands/doctor.rs b/horus_manager/src/commands/doctor.rs index 7e11e4661..f03a0ce15 100644 --- a/horus_manager/src/commands/doctor.rs +++ b/horus_manager/src/commands/doctor.rs @@ -1817,7 +1817,7 @@ fn check_network() -> CheckResult { details.push("horus_net: enabled".to_string()); } else { details.push( - "horus_net: available (enable it with `horus run --net`, enable = [\"net\"] in horus.toml, or HORUS_ENABLE=net)" + "horus_net: available (enable it with `horus run --net`, enable = [\"net\"] in horus.toml, or HORUS_ENABLE=net)" .to_string(), ); } diff --git a/horus_manager/src/commands/run/mod.rs b/horus_manager/src/commands/run/mod.rs index 7ed2f64ec..70267552e 100644 --- a/horus_manager/src/commands/run/mod.rs +++ b/horus_manager/src/commands/run/mod.rs @@ -332,7 +332,13 @@ fn execute_workspace( apply_network_config(&mut cmd); let status = cmd.status()?; if !status.success() { - let code = status.code().unwrap_or(1); + // `shell_exit_code`, not `status.code().unwrap_or(1)`. On Unix + // a child killed by a signal has no exit code at all, and the + // `unwrap_or(1)` reported every one of them — SIGSEGV, SIGKILL, + // the Ctrl-C the operator just pressed — as a plain "exited + // with code 1". That is the defect `shell_exit_code` was + // introduced for; this call site was missed. + let code = crate::error_wrapper::shell_exit_code(&status); return Err(anyhow!("Process exited with code {}", code)); } } diff --git a/horus_sys/src/shm/mod.rs b/horus_sys/src/shm/mod.rs index 4c93aa0db..5af7698ac 100644 --- a/horus_sys/src/shm/mod.rs +++ b/horus_sys/src/shm/mod.rs @@ -257,7 +257,7 @@ pub fn shm_base_dir() -> PathBuf { /// by restating the platform mapping, which is how two copies of one path drift /// apart. pub fn shm_base_dir_for(namespace: &str) -> PathBuf { - shm_parent_dir().join(format!("horus_{namespace}")) + shm_parent_dir().join(format!("horus_{}", sanitize_namespace(namespace))) } /// Topics directory for shared memory message passing. @@ -526,6 +526,60 @@ const REGION_NAME_MAX: usize = 512; /// A single embedded `/` remains legal: legacy hierarchical topic names rely on /// it, and `ShmRegion::new` creates the intermediate directory. Only traversal /// and absolute/rooted forms are rejected. +#[cfg(test)] +mod namespace_path_tests { + use super::*; + + /// The finding: `shm_base_dir_for` interpolated its argument raw, and its + /// callers hand the result to `remove_dir_all`. + /// + /// The fix is not a new sanitizer — `sanitize_namespace` was already in + /// this module and already strict enough — it is that this path was the one + /// place that skipped it. + #[test] + fn a_namespace_can_never_escape_the_shm_parent() { + let parent = shm_parent_dir(); + for hostile in [ + "../../../../etc", + "..", + ".", + "a/../../b", + "a/b", + "a\\b", + "/absolute", + ] { + let dir = shm_base_dir_for(hostile); + assert_eq!( + dir.parent(), + Some(parent.as_path()), + "{hostile:?} produced {dir:?}, which is not directly under the SHM parent" + ); + let leaf = dir + .file_name() + .expect("a leaf") + .to_string_lossy() + .to_string(); + assert!( + !leaf.contains("..") && !leaf.contains('/') && !leaf.contains('\\'), + "{hostile:?} left a traversable leaf {leaf:?}" + ); + } + } + + /// The default namespace's directory is a compatibility contract: rewriting + /// it would strand every running deployment's shared memory. + #[test] + fn the_ordinary_namespaces_keep_their_paths() { + for ok in ["default", "test_a1b2", "robot_01"] { + assert_eq!(sanitize_namespace(ok), ok, "{ok:?} must not be rewritten"); + } + assert_eq!( + shm_base_dir_for("default"), + shm_parent_dir().join("horus_default") + ); + } +} + pub fn validate_region_name(name: &str) -> anyhow::Result<()> { anyhow::ensure!(!name.is_empty(), "SHM region name must not be empty"); anyhow::ensure!( diff --git a/install.sh b/install.sh index 22024e744..3533ce2b0 100755 --- a/install.sh +++ b/install.sh @@ -956,8 +956,22 @@ if [ -n "${HORUS_PREFIX:-}" ] && [ -n "$SHELL_RC" ] && [ -z "$TARGET_USER" ]; th mkdir -p "$(dirname "$SHELL_RC")" 2>/dev/null || true case "$SHELL_RC" in */config.fish) - grep -qF "set -gx HORUS_PREFIX" "$SHELL_RC" 2>/dev/null || \ + # Value-aware, like the POSIX arm below. Testing only for the + # PRESENCE of `set -gx HORUS_PREFIX` treated a line naming a + # DIFFERENT prefix as "already configured", so a fish user who + # reinstalled to a new location kept the old export and a shell + # that pointed at a tree the installer had just replaced. The + # POSIX arm was fixed for exactly this; fish was left behind. + if grep -qE "^[[:space:]]*set +(-[a-zA-Z]+ +)*HORUS_PREFIX([[:space:]]|\$)" "$SHELL_RC" 2>/dev/null; then + grep -qF "set -gx HORUS_PREFIX \"${HORUS_PREFIX}\"" "$SHELL_RC" 2>/dev/null || { + sed -E -i.horusbak "/^[[:space:]]*set +(-[a-zA-Z]+ +)*HORUS_PREFIX([[:space:]]|\$)/d" "$SHELL_RC" 2>/dev/null || \ + sed -E -i '' "/^[[:space:]]*set +(-[a-zA-Z]+ +)*HORUS_PREFIX([[:space:]]|\$)/d" "$SHELL_RC" 2>/dev/null + rm -f "${SHELL_RC}.horusbak" 2>/dev/null + echo "set -gx HORUS_PREFIX \"${HORUS_PREFIX}\"" >> "$SHELL_RC" + } + else echo "set -gx HORUS_PREFIX \"${HORUS_PREFIX}\"" >> "$SHELL_RC" + fi ;; *) # Value-aware: a stale line naming a DIFFERENT prefix must be diff --git a/uninstall.sh b/uninstall.sh index a815cce4e..d28792ecb 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -1045,7 +1045,20 @@ clean_shell_profiles() { # the braces and the `|| true` rather than `... && mv`. { grep -vF "$prefix_path_line" "$profile" 2>/dev/null \ | grep -vF "$prefix_fish_line" > "${profile}.horus-new" 2>/dev/null; } || true - if [ -f "${profile}.horus-new" ]; then + # ...but `|| true` swallows exit 2 as well, and exit 2 means + # grep could not READ the file. The redirect creates the target + # either way, so `[ -f ]` alone cannot tell "this rc held + # nothing but our line" from "this rc could not be read" — and + # in the second case the `mv` replaced a user's shell profile + # with an empty file. Requiring the profile to be readable, and + # refusing to turn a non-empty profile into an empty one unless + # it genuinely held nothing else, keeps the destructive case out + # while preserving the empty-result case the comment above is + # about. + if [ -f "${profile}.horus-new" ] && [ -r "$profile" ] && + { [ -s "${profile}.horus-new" ] || [ ! -s "$profile" ] || + [ "$(grep -cvF "$prefix_path_line" "$profile" 2>/dev/null || echo 1)" = "0" ] || + [ "$(grep -cvF "$prefix_fish_line" "$profile" 2>/dev/null || echo 1)" = "0" ]; }; then mv "${profile}.horus-new" "$profile" fi rm -f "${profile}.horus-new" 2>/dev/null From f725ec9216f3ceab79b20ea728ee8901886f9b78 Mon Sep 17 00:00:00 2001 From: Neos Date: Sat, 5 Sep 2026 23:25:11 -0700 Subject: [PATCH 58/79] docs(scheduler): the restart path clears two different pauses, not one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `registered.is_paused` is the one-tick `Miss::Skip` flag and is what `reinit_pending_nodes` actually reads; the operator pause lives in `node_controls` and is not read there at all. The comment blurred them, so it justified lifting the operator pause with "would otherwise be skipped by the very check that would have re-inited it" — which is not true of that check. The code was already right to clear both. Lifting the operator pause is what lets the node TICK once re-inited, not what permits the re-init: a wedged node is usually paused before it is restarted, and without it the node comes back initialised and still frozen. --- horus_core/src/scheduling/scheduler/mod.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/horus_core/src/scheduling/scheduler/mod.rs b/horus_core/src/scheduling/scheduler/mod.rs index 183cafe06..3e06b5667 100644 --- a/horus_core/src/scheduling/scheduler/mod.rs +++ b/horus_core/src/scheduling/scheduler/mod.rs @@ -4891,9 +4891,16 @@ impl Scheduler { // Clearing `initialized` is the whole mechanism: `reinit_pending_nodes` // runs on the tick loop and re-runs `init()` for any node that is not - // stopped, not paused, and not initialised. Also lift an operator pause, - // since a wedged node is often paused before it is restarted and would - // otherwise be skipped by the very check that would have re-inited it. + // stopped, not transiently skipped, and not initialised. + // + // Two different pauses are cleared below and they are not the same + // thing, which this comment used to blur. `registered.is_paused` is the + // one-tick `Miss::Skip` flag and IS what `reinit_pending_nodes` reads; + // the operator pause lives in `node_controls` and is not read there at + // all. So lifting the operator pause is not what lets the re-init + // happen — it is what lets the node TICK once it has been re-inited, + // which matters because a wedged node is usually paused before it is + // restarted and would otherwise come back initialised and still frozen. for name in restart_requests { if let Some(ref controls) = self.node_controls { controls.set_paused(&name, false); From 27a032fa19b7b459b31f82e3db0006c5522dd72e Mon Sep 17 00:00:00 2001 From: Neos Date: Sun, 6 Sep 2026 00:51:51 -0700 Subject: [PATCH 59/79] ci: pin the documentation half of this audit alongside the code half MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `CLI contract (live docs)` clones horus-docs at a SHA pinned in this workflow, deliberately — the job runs in a context with write access to this repo's Actions cache, so building an arbitrary ref of another repository would be a cross-repo escalation this repo's review never sees. Verifying different docs means bumping the SHA here, in a reviewed PR. horus-docs#17 is the documentation half of the same bidirectional audit this branch is the code half of, and it merged as 9a1a0c1. Leaving the pin at 4a612b1 would validate this branch's runtime corrections against pages that still describe what the runtime did before them — which is precisely the drift the audit was run to find, reintroduced by the gate meant to catch it. One commit, whose parent is the old pin. Verified against the pinned commit itself rather than a local branch: HORUS_DOCS_DIR= \ cargo test -p horus_manager --test docs_contract -- --include-ignored 14 passed, 0 failed. --- .github/workflows/docs-contract.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs-contract.yml b/.github/workflows/docs-contract.yml index f67400b94..a60e1e678 100644 --- a/.github/workflows/docs-contract.yml +++ b/.github/workflows/docs-contract.yml @@ -92,6 +92,14 @@ env: # Verifying a different docs commit means bumping this SHA in a PR, so the # change is reviewed here before it runs here. # + # Bumped from 4a612b1 to 9a1a0c1 (horus-docs main, the merge of + # horus-docs#17). That PR is the documentation half of the same bidirectional + # audit this branch is the code half of, so the two belong at the same pin: + # against 4a612b1 the contract validates this branch's runtime corrections + # against pages that still describe what the runtime did BEFORE them, which + # is the drift the audit existed to find. Exactly one commit, and 4a612b1 is + # its parent. Verified from this tree: 14 passed, 0 failed. + # # Bumped from 38368c8 to 4a612b1 (horus-docs main, the merge of # horus-docs#18), for the same reason 38368c8 replaced af057d9 before it. # This branch adds six operator-facing variables — HORUS_RECORD_MAX_MB, @@ -107,7 +115,7 @@ env: # it: 14 passed, 0 failed. The bump is 7 commits, not 1 — horus-docs#18 is # only the newest of them, and that suite result is what says the other six, # written by other people for other reasons, are compatible with this branch. - DOCS_PINNED_REF: 4a612b108d51308537cdd5c737bbc0181f8354ac + DOCS_PINNED_REF: 9a1a0c107bc2e04b75cbdd08a04462e242285bce jobs: # ── Tier 1 ──────────────────────────────────────────────────────────────── From 6fa2268cb75e3d0c2097c75ebeca44758d9ca136 Mon Sep 17 00:00:00 2001 From: Neos Date: Sun, 6 Sep 2026 02:29:07 -0700 Subject: [PATCH 60/79] fix(uninstall): my own guard broke the one case the guard is about MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six CI jobs went red on one test, and it was mine: `an_rc_file_that_is_only_the_prefix_path_line_is_emptied_not_skipped`. Code Coverage, Cross-Process IPC Tests, CLI Integration Tests and Parity (macOS ARM64) all run `completion_install_contract`, so a single shell bug failed four leaves and two aggregates. The guard I added to stop an unreadable profile being replaced with an empty file also required "the result is non-empty, unless the profile held nothing but our lines", and tested that second half with [ "$(grep -cvF "$prefix_path_line" "$profile" 2>/dev/null || echo 1)" = "0" ] `grep -c` PRINTS "0" and EXITS 1 when the count is zero. So on the legitimate case the `|| echo 1` also ran, the substitution became "0\n1", the comparison failed, and the rewrite was skipped — leaving the stale PATH line in exactly the file where removing it matters most. That test's doc comment already records two earlier ways this line went wrong; this made a third. `-r` was always the whole guard. An unreadable file is what `grep` exit 2 means and what `[ -r ]` detects; given a readable profile, an empty result is legitimate by construction and needs no second opinion. The extra condition bought nothing and cost the case it was meant to protect. Verified in all three directions, not just the one the test covers: * profile is only our line -> emptied (0 bytes) * profile has other content -> ours removed, rest kept * profile is unreadable (000) -> untouched (36 bytes, contents intact) and `completion_install_contract` is 26 passed, 0 failed. Also: the generated Cargo header said "# Driver dependencies (from [hardware])" for entries that `Manifest::hardware_entries()` merges out of BOTH `[hardware]` and the legacy `[drivers]` table, so a user still on `[drivers]` would look for a `[hardware]` table they do not have and conclude the generator invented the dependency. It names both sources now. --- horus_manager/src/cargo_gen.rs | 11 ++++++++++- uninstall.sh | 28 ++++++++++++++++------------ 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/horus_manager/src/cargo_gen.rs b/horus_manager/src/cargo_gen.rs index e4a741983..2fc81ebbf 100644 --- a/horus_manager/src/cargo_gen.rs +++ b/horus_manager/src/cargo_gen.rs @@ -916,7 +916,16 @@ fn write_driver_deps( } if !added_crates.is_empty() { - writeln!(cargo, "\n# Driver dependencies (from [hardware])").unwrap(); + // "[hardware] or [drivers]", not just the former: these entries come + // from `Manifest::hardware_entries()`, which merges `[hardware]` with + // the legacy `[drivers]` table. A user still on `[drivers]` would read + // the old header, look at a `[hardware]` table they do not have, and + // conclude the generator had invented the dependency. + writeln!( + cargo, + "\n# Driver dependencies (from [hardware], or legacy [drivers])" + ) + .unwrap(); for (crate_name, (version, features)) in &added_crates { let ver = version.as_deref().unwrap_or("*"); if features.is_empty() { diff --git a/uninstall.sh b/uninstall.sh index d28792ecb..6831aee60 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -1047,18 +1047,22 @@ clean_shell_profiles() { | grep -vF "$prefix_fish_line" > "${profile}.horus-new" 2>/dev/null; } || true # ...but `|| true` swallows exit 2 as well, and exit 2 means # grep could not READ the file. The redirect creates the target - # either way, so `[ -f ]` alone cannot tell "this rc held - # nothing but our line" from "this rc could not be read" — and - # in the second case the `mv` replaced a user's shell profile - # with an empty file. Requiring the profile to be readable, and - # refusing to turn a non-empty profile into an empty one unless - # it genuinely held nothing else, keeps the destructive case out - # while preserving the empty-result case the comment above is - # about. - if [ -f "${profile}.horus-new" ] && [ -r "$profile" ] && - { [ -s "${profile}.horus-new" ] || [ ! -s "$profile" ] || - [ "$(grep -cvF "$prefix_path_line" "$profile" 2>/dev/null || echo 1)" = "0" ] || - [ "$(grep -cvF "$prefix_fish_line" "$profile" 2>/dev/null || echo 1)" = "0" ]; }; then + # either way, so `[ -f ]` alone could not tell "this rc held + # nothing but our line" from "this rc could not be read", and in + # the second case the `mv` replaced a profile it had failed to + # read with an empty file. + # + # `-r` is the whole guard, and it is deliberately the ONLY extra + # condition. An earlier attempt here also required the result to + # be non-empty "unless the profile held nothing else", testing + # that with `[ "$(grep -cvF ... || echo 1)" = "0" ]` — which is + # wrong twice over: `grep -c` PRINTS "0" and EXITS 1 when the + # count is zero, so the `|| echo 1` fired on the legitimate case + # and the substitution became "0\n1", and the whole point of + # this block is the file whose only line is ours. That is the + # exact case the two failure modes in this function's test are + # about, and it made a third. + if [ -f "${profile}.horus-new" ] && [ -r "$profile" ]; then mv "${profile}.horus-new" "$profile" fi rm -f "${profile}.horus-new" 2>/dev/null From af303ead5c4d99238d8052eff0d417a4d8ec9401 Mon Sep 17 00:00:00 2001 From: Neos Date: Sun, 6 Sep 2026 03:17:33 -0700 Subject: [PATCH 61/79] docs(rustdoc): let a maintainer read horus_core's internals on stable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cargo doc -p horus_core --open` rendered a front page with one module on it (`terminal`), zero structs, and zero entries in the search index — for a 112k-line runtime crate. The standard first move for a Rust developer meeting a new crate returned nothing. Ten of the eleven modules carry `#[doc(hidden)]`, and that IS deliberate: the crate note says "internal crate, use `horus::prelude`", and horus_core/README.md has a whole "How to read the internals" section about it. The problem is the escape hatch it documents: RUSTDOCFLAGS="-Z unstable-options --document-hidden-items" \ cargo +nightly doc -p horus_core --no-deps --document-private-items That needs nightly and an unstable flag, because `--document-private-items` disables rustdoc's private-stripping pass and not its hidden-stripping pass. (Verified here rather than assumed: with the flag, the front page gained exactly one module — `utils`, the `pub(crate)` one — and none of the hidden ten.) So the attributes are now `cfg_attr`'d behind a docs-only `internal-docs` feature. Default output is byte-for-byte what it was, so the user-facing boundary is untouched; a maintainer on a stable toolchain gets all eleven modules and a populated search index with cargo doc -p horus_core --no-deps --document-private-items \ --features internal-docs --open The README now leads with that and keeps the nightly form as the no-feature alternative. CI documents horus_core a second time with the feature on. That is the half worth keeping even if the rest is bikeshedded: rustdoc does not resolve links inside a hidden module, so `-D warnings` had been holding roughly a quarter of the crate and silently skipping the rest. The same CI job gains a `Doctests` step, for a neighbouring reason: `grep -rn -- '--doc' .github/workflows/` returned nothing, so `cargo test --lib` never ran the examples and `cargo doc` only rendered them. 21 pass today. Its comment says plainly that it is a thin gate — 193 of the workspace's 214 examples are fenced ```rust,ignore``` and are skipped — because a step that protects 21 examples should not be described as protecting the examples. --- .github/workflows/ci.yml | 32 +++++++++++++++++++++++++++++ horus_core/Cargo.toml | 3 +++ horus_core/README.md | 14 ++++++++++--- horus_core/src/lib.rs | 44 +++++++++++++++++++++++++++++----------- 4 files changed, 78 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f143e4e8d..03c88a652 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,6 +226,25 @@ jobs: env: RUST_BACKTRACE: 1 + # Doctests. `--lib` above does NOT run them, and no other job did either: + # before this step, `grep -rn -- '--doc' .github/workflows/` returned + # nothing, so every `///` example in the workspace was unverified. The + # `Documentation` job runs `cargo doc`, which builds the pages and does + # not compile the examples in them. + # + # This matters because the examples are the first code a new contributor + # copies. Measured on the tree that added this step: 21 passed, 0 failed. + # + # It is a thin gate today and the number says why: 193 of the workspace's + # examples are fenced ```rust,ignore``` and are skipped, so this compiles + # 21 of 214. Converting the ignores that only need to type-check to + # ```no_run``` is the follow-up that gives this step its teeth; the step + # has to exist first or there is nothing to convert them into. + - name: Doctests + run: cargo test --workspace --exclude horus_py --doc + env: + RUST_BACKTRACE: 1 + # The resolver's crates.io/PyPI tests carry #[ignore] so a contributor # without network access gets a green suite and a failure means the # resolver is wrong rather than the network. CI does have access, so it @@ -360,6 +379,19 @@ jobs: # also the right call — docs should describe the crate as it ships, # i.e. with its default features on. cargo doc --workspace --no-deps --exclude horus_manager + # Again with horus_core's internals visible. + # + # Ten of its eleven modules are `#[doc(hidden)]` by default, and + # rustdoc does not resolve links inside a hidden module — so the run + # above holds `-D warnings` over roughly a quarter of the crate and + # silently skips the rest. Four broken links had accumulated behind + # that gap, including a `[`HorusError::OutOfRange`]` naming a variant + # the error refactor deleted, and two pointing at private items. + # + # `internal-docs` is docs-only: it gates nothing but the + # `#[doc(hidden)]` attributes, so this compiles no extra code. The + # published documentation is still what the first command renders. + cargo doc -p horus_core --no-deps --features internal-docs env: RUSTDOCFLAGS: ${{ (github.ref == 'refs/heads/main' || github.base_ref == 'main') && '-D warnings' || '' }} diff --git a/horus_core/Cargo.toml b/horus_core/Cargo.toml index ecbe54d54..1124cd66b 100644 --- a/horus_core/Cargo.toml +++ b/horus_core/Cargo.toml @@ -75,6 +75,9 @@ horus-tf = { git = "https://github.com/softmata/horus-tf.git", rev = "d525dfcf58 workspace = true [features] +# Render the crate-internal modules in rustdoc. Docs-only: it gates nothing +# but `#[doc(hidden)]`, so it changes no code and no dependency. +internal-docs = [] default = ["macros", "telemetry", "blackbox"] macros = ["horus_macros"] diff --git a/horus_core/README.md b/horus_core/README.md index 1d4274932..48010af79 100644 --- a/horus_core/README.md +++ b/horus_core/README.md @@ -111,9 +111,17 @@ caught it (`rt_allocator.rs:21-26`, `rt_alloc_guard_installed.rs:35-38`). ## How to read the internals `--document-private-items` is **not** enough on its own. It disables rustdoc's -private-stripping pass, not its hidden-stripping pass, so the eleven hidden -modules stay stripped. You need nightly and `--document-hidden-items`, which is -unstable and refuses to run without `-Z unstable-options`: +private-stripping pass, not its hidden-stripping pass, so the hidden modules +stay stripped. Use the `internal-docs` feature, which lifts the +`#[doc(hidden)]` attributes and works on stable: + +```sh +cargo doc -p horus_core --no-deps --document-private-items \ + --features internal-docs --open +``` + +The nightly form below still works and needs no feature, but it requires a +nightly toolchain and an unstable flag, which is why the feature exists: ```sh RUSTDOCFLAGS="-Z unstable-options --document-hidden-items" \ diff --git a/horus_core/src/lib.rs b/horus_core/src/lib.rs index 3b4034e41..efbb4554f 100644 --- a/horus_core/src/lib.rs +++ b/horus_core/src/lib.rs @@ -15,25 +15,45 @@ //! **Note:** This is an internal crate. Users should depend on `horus` and //! import from `horus::prelude::*`. -// Public modules — accessible cross-crate but hidden from user docs. -// Users should go through `horus::prelude`, not import from `horus_core` directly. -#[doc(hidden)] +// Public modules — accessible cross-crate, hidden from user docs by default. +// Users should go through `horus::prelude`, not import from `horus_core` +// directly, and the crate note above says so. +// +// The hiding is `cfg_attr`'d rather than unconditional so a MAINTAINER can lift +// it on stable: +// +// cargo doc -p horus_core --no-deps --document-private-items \ +// --features internal-docs --open +// +// Before this was a feature, the only way to read the internals was the nightly +// incantation in horus_core/README.md — `-Z unstable-options +// --document-hidden-items` — because `--document-private-items` disables +// rustdoc's private-stripping pass and not its hidden-stripping pass. That put +// the API browser for a 112k-line crate behind an unstable flag, so on stable +// `cargo doc --open` showed one module (`terminal`), zero structs and an empty +// search index. +// +// CI documents WITH the feature (ci.yml, Documentation job) precisely so +// `-D warnings` sees these modules: while they were unconditionally hidden, +// rustdoc never resolved their links and four broken ones had accumulated, +// including a `HorusError::OutOfRange` the error refactor had deleted. +#[cfg_attr(not(feature = "internal-docs"), doc(hidden))] pub mod actions; -#[doc(hidden)] +#[cfg_attr(not(feature = "internal-docs"), doc(hidden))] pub mod communication; -#[doc(hidden)] +#[cfg_attr(not(feature = "internal-docs"), doc(hidden))] pub mod core; -#[doc(hidden)] +#[cfg_attr(not(feature = "internal-docs"), doc(hidden))] pub mod drivers; -#[doc(hidden)] +#[cfg_attr(not(feature = "internal-docs"), doc(hidden))] pub mod error; -#[doc(hidden)] +#[cfg_attr(not(feature = "internal-docs"), doc(hidden))] pub mod memory; -#[doc(hidden)] +#[cfg_attr(not(feature = "internal-docs"), doc(hidden))] pub mod params; -#[doc(hidden)] +#[cfg_attr(not(feature = "internal-docs"), doc(hidden))] pub mod scheduling; -#[doc(hidden)] +#[cfg_attr(not(feature = "internal-docs"), doc(hidden))] pub mod services; /// Non-panicking console output. /// @@ -42,7 +62,7 @@ pub mod services; /// operator pipes `horus run` into `head`. Anything printing from a node, a /// background thread, or a safety path should use these instead. pub mod terminal; -#[doc(hidden)] +#[cfg_attr(not(feature = "internal-docs"), doc(hidden))] pub mod types; pub(crate) mod utils; From 8e37a65cb996a1236731e2eeab94b7e2f33eef9a Mon Sep 17 00:00:00 2001 From: Neos Date: Sun, 6 Sep 2026 03:17:33 -0700 Subject: [PATCH 62/79] fix(docs): five documentation defects the hidden modules had been concealing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documenting horus_core with its internals visible turned the `Documentation` job red immediately, on four broken links that had accumulated unseen — rustdoc does not resolve links inside a hidden module, so `-D warnings` never looked at them. A fifth defect was visible all along and is fixed here because it is the same failure: prose separated from the thing it describes. * `depth_image.rs` documented `set_depth` as returning `[`HorusError::OutOfRange`]`. That variant does not exist — the error refactor folded it into `InvalidInput(ValidationError::…)` (error.rs:399 records the replacement) and the doc kept naming the deleted one. A maintainer reading its `# Errors` section was told to match on a variant they cannot write, and nothing in the build disagreed. * `tensor_handle.rs` and `scheduler/mod.rs` each linked a private item from a public doc comment (`pool_registry::global_pool`, `rt::set_realtime_priority`). Both become plain code spans, which still tell a reader where to look. * `error.rs` linked `[`Error`]`, ambiguous between a type alias and a derive macro in that scope. Disambiguated with `tyalias@`. * `Scheduler` — the type a HORUS program is built around — had NO doc comment. Its description, "Central orchestrator: holds nodes, drives the tick loop.", was stranded one item higher, sitting on top of `PresenceEntry`'s own comment, so a private struct rendered as the orchestrator and the public type rendered blank. --- horus_core/src/error.rs | 2 +- horus_core/src/memory/depth_image.rs | 7 ++++++- horus_core/src/memory/tensor_handle.rs | 2 +- horus_core/src/scheduling/scheduler/mod.rs | 14 ++++++++++++-- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/horus_core/src/error.rs b/horus_core/src/error.rs index 4c3ad45bf..eeafa1144 100644 --- a/horus_core/src/error.rs +++ b/horus_core/src/error.rs @@ -850,7 +850,7 @@ macro_rules! horus_internal { /// Convenience type alias for Results using HorusError. /// -/// **Prefer the short aliases [`Result`] and [`Error`] in all new code.** +/// **Prefer the short aliases [`Result`] and [`Error`](tyalias@Error) in all new code.** /// `HorusResult` and `HorusError` are the canonical names but verbose — /// the aliases exist specifically so you can write `Result<()>` and `Error` /// without the `Horus` prefix everywhere. diff --git a/horus_core/src/memory/depth_image.rs b/horus_core/src/memory/depth_image.rs index 8d81ec7d2..a5d8081c0 100644 --- a/horus_core/src/memory/depth_image.rs +++ b/horus_core/src/memory/depth_image.rs @@ -134,10 +134,15 @@ impl DepthImage { /// /// # Errors /// - /// - [`HorusError::OutOfRange`] — for U16 images, if the converted + /// - [`HorusError::InvalidInput`] — for U16 images, if the converted /// millimeter value does not fit in `u16` (e.g., value > 65.535m at /// default scale 1.0). Callers must handle this instead of silently /// storing the saturated `u16::MAX`. + /// + /// The link used to name `HorusError::OutOfRange`, which the error refactor + /// folded into `InvalidInput(ValidationError::…)` (see error.rs:399). The + /// broken link survived because this module was `#[doc(hidden)]`, so + /// rustdoc never resolved it. pub fn set_depth(&mut self, x: u32, y: u32, value: f32) -> HorusResult<&mut Self> { if x >= self.width() || y >= self.height() { return Ok(self); diff --git a/horus_core/src/memory/tensor_handle.rs b/horus_core/src/memory/tensor_handle.rs index 023c1cc2b..dbf833320 100644 --- a/horus_core/src/memory/tensor_handle.rs +++ b/horus_core/src/memory/tensor_handle.rs @@ -101,7 +101,7 @@ impl TensorHandle { /// and dtype. /// /// This is the ergonomic constructor for creating a tensor to fill in and - /// publish: the data lives in the [`global_pool`](crate::communication::topic::pool_registry::global_pool) + /// publish: the data lives in the process-wide tensor pool (`global_pool`) /// and the device is inferred from that pool's backend (CPU by default). For /// an explicit pool or device, use [`alloc`](Self::alloc). /// diff --git a/horus_core/src/scheduling/scheduler/mod.rs b/horus_core/src/scheduling/scheduler/mod.rs index 568decbdb..705a86f9e 100644 --- a/horus_core/src/scheduling/scheduler/mod.rs +++ b/horus_core/src/scheduling/scheduler/mod.rs @@ -323,7 +323,6 @@ pub(crate) struct RecordingState { /// lifetime and dropped during shutdown — use `Drop` to clean up resources. pub type LifecycleStartFn = Box Option> + Send>; -/// Central orchestrator: holds nodes, drives the tick loop. /// What the presence refresh needs about a node the main thread does not own. #[derive(Debug, Clone)] pub(super) struct PresenceEntry { @@ -336,6 +335,17 @@ pub(super) struct PresenceEntry { pub main_thread: bool, } +/// Central orchestrator: holds nodes, drives the tick loop. +/// +/// This is the type a HORUS program is built around: nodes are registered on +/// it, it partitions them by scheduling class across the RT, compute, event and +/// async-I/O executors, and it owns the loop that ticks them and enforces their +/// deadlines. +/// +/// The doc line above used to sit one item higher, stranded on top of +/// `PresenceEntry`'s own comment — so the framework's primary public type +/// rendered with no description at all, and a private struct was documented as +/// the orchestrator. pub struct Scheduler { pub(super) nodes: Vec, pub(super) running: Arc, @@ -1996,7 +2006,7 @@ impl Scheduler { /// Set OS-level scheduling priority using SCHED_FIFO (Linux RT-PREEMPT required). /// - /// Delegates to [`super::rt::set_realtime_priority`]. + /// Delegates to `super::rt::set_realtime_priority` (crate-private). /// /// # Arguments /// * `priority` - Priority level (1-99, higher = more important) From 43a89c06be9c3dade55f51b5d36daae08897d2fb Mon Sep 17 00:00:00 2001 From: Neos Date: Sun, 6 Sep 2026 03:17:54 -0700 Subject: [PATCH 63/79] fix(env): one vocabulary for boolean environment variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two variables in the same framework disagreed about the word "no". `HORUS_LOG_FILE=yes` was **false** — that site matched only `"true"` and `"1"`. `HORUS_SIM_MODE=no` was **true** — that site treated anything it did not recognise as an opt-in, so a deploy script saying "no" swapped every driver for an inert `SimStubNode`: actuators never commanded, sensors never read, and nothing said so. The comment already sitting there records the same class of bug being fixed once before, for `=false`; `no` and `off` were still wrong. `horus_sys::env::env_flag` is now the one answer. `1/true/yes/on` are true, `0/false/no/off` and empty are false, ASCII case-insensitive, whitespace trimmed — a value out of a YAML launch file or a shell here-doc usually carries some. An unrecognised value returns `None` rather than `false`, leaving the caller's default in force. `HORUS_SIM_MODE=maybe` therefore keeps simulation off instead of enabling it, which is the safe direction for a flag that disconnects the robot from its hardware — and it matches what the reference page already promises for `HORUS_SHM_PREFAULT`: "an unrecognised value keeps the default rather than meaning off". Routed: `HORUS_LOG_FILE`, `HORUS_SIM_MODE`, and the opt-out `HORUS_NET_ENABLED` (which accepted `0`/`false` but not `no`/`off`, so two of the four words that mean "off" everywhere else left networking running). Deliberately NOT routed: `HORUS_RT_ALLOW_CORE_SHARING` and `HORUS_WIN_REALTIME_CLASS`. Their exact vocabularies are published in horus-docs — the first is documented as case-sensitive on purpose — so changing them needs a paired documentation change rather than a drive-by here. The parsing half is a separate `parse_flag` so it can be tested without mutating process-global state: `set_var` is not safe against a concurrent `getenv` on another thread, and these test binaries run in parallel. The two divergences above are pinned as tests, by name. --- horus_core/src/core/log_buffer.rs | 8 +- horus_core/src/drivers/mod.rs | 10 +- horus_core/src/scheduling/scheduler/mod.rs | 11 +- horus_sys/src/env/mod.rs | 128 +++++++++++++++++++++ horus_sys/src/lib.rs | 1 + 5 files changed, 147 insertions(+), 11 deletions(-) create mode 100644 horus_sys/src/env/mod.rs diff --git a/horus_core/src/core/log_buffer.rs b/horus_core/src/core/log_buffer.rs index 509ec71b0..914b7b75f 100644 --- a/horus_core/src/core/log_buffer.rs +++ b/horus_core/src/core/log_buffer.rs @@ -614,9 +614,11 @@ pub fn publish_log(entry: LogEntry) { /// /// Returns a join handle. Drop or join to stop the drain thread. pub fn start_log_file_drain() -> Option> { - let enabled = std::env::var("HORUS_LOG_FILE") - .map(|v| v == "true" || v == "1") - .unwrap_or(false); + // `env_flag`, not a local truth table. This site accepted only "true" and + // "1", so `HORUS_LOG_FILE=yes` turned logging off while `HORUS_SIM_MODE=yes` + // turned simulation on — the same framework disagreeing with itself about + // one word. See `horus_sys::env` for the vocabulary. + let enabled = horus_sys::env::env_flag("HORUS_LOG_FILE").unwrap_or(false); if !enabled { return None; } diff --git a/horus_core/src/drivers/mod.rs b/horus_core/src/drivers/mod.rs index 1605b4be0..1422b929c 100644 --- a/horus_core/src/drivers/mod.rs +++ b/horus_core/src/drivers/mod.rs @@ -96,9 +96,13 @@ pub fn load_from>(path: P) -> HorusResult> = if sim_mode { std::env::var("HORUS_SIM_TARGETS") diff --git a/horus_core/src/scheduling/scheduler/mod.rs b/horus_core/src/scheduling/scheduler/mod.rs index 705a86f9e..c40203c41 100644 --- a/horus_core/src/scheduling/scheduler/mod.rs +++ b/horus_core/src/scheduling/scheduler/mod.rs @@ -698,13 +698,14 @@ impl Scheduler { }; // Network discovery is enabled by default (like ROS2). - // Opt out via HORUS_NET_ENABLED=false (or =0) env var, or `.network(false)`. + // Opt out via HORUS_NET_ENABLED=false (or 0/no/off), or `.network(false)`. // The actual horus_net startup is handled by a lifecycle hook registered // by the integration layer (e.g., the `horus` umbrella crate). - if std::env::var("HORUS_NET_ENABLED") - .map(|v| v == "0" || v.eq_ignore_ascii_case("false")) - .unwrap_or(false) - { + // + // An opt-out reads the same vocabulary as an opt-in: this site accepted + // `0` and `false` but not `no` or `off`, so two of the four words that + // mean "off" everywhere else in HORUS left networking running. + if horus_sys::env::env_flag("HORUS_NET_ENABLED") == Some(false) { s.network_enabled = false; } diff --git a/horus_sys/src/env/mod.rs b/horus_sys/src/env/mod.rs new file mode 100644 index 000000000..f07bc7f31 --- /dev/null +++ b/horus_sys/src/env/mod.rs @@ -0,0 +1,128 @@ +//! One vocabulary for boolean environment variables. +//! +//! # Why this exists +//! +//! HORUS reads 60-odd `HORUS_*` variables, and before this module every site +//! decided for itself what counted as "on". The forms in production were: +//! +//! ```text +//! v == "true" || v == "1" HORUS_LOG_FILE +//! !(v.is_empty() || v == "0" || eq_ic(v, "false")) HORUS_SIM_MODE +//! v == "0" || eq_ic(v, "false") HORUS_NET_ENABLED (opt-out) +//! env::var(..).is_ok() presence-only +//! v == "1" HORUS_WIN_REALTIME_CLASS +//! ``` +//! +//! The consequence was not academic. `HORUS_LOG_FILE=yes` was **false** while +//! `HORUS_SIM_MODE=no` was **true** — the second one silently swapping every +//! driver for an inert `SimStubNode`, actuators never commanded and sensors +//! never read. Two variables in the same framework disagreed about the word +//! "no", and nothing in the codebase said which was right. +//! +//! # The vocabulary +//! +//! `1`, `true`, `yes`, `on` are true. `0`, `false`, `no`, `off` and the empty +//! string are false. Matching is ASCII case-insensitive, so `True` and `ON` +//! work. Surrounding whitespace is trimmed, because a value that arrived from a +//! YAML launch file or a shell here-doc often carries some. +//! +//! # Why an unrecognised value is `None`, not `false` +//! +//! `env_flag` returns `Option`: `None` means "unset or not a boolean I +//! recognise", which leaves the caller's default in force. `HORUS_SIM_MODE=maybe` +//! therefore keeps simulation off rather than turning it on — the old +//! `!(v == "0")` shape treated every typo as an opt-in, which is the wrong +//! direction for a flag that disconnects the robot from its hardware. +//! +//! It also matches what the reference page already promises for +//! `HORUS_SHM_PREFAULT`: "an unrecognised value keeps the default rather than +//! meaning off". + +/// Read a boolean environment variable using the project-wide vocabulary. +/// +/// Returns `None` when the variable is unset, or set to something that is not +/// a recognised truth value — in both cases the caller's default should stand. +/// +/// ``` +/// # use horus_sys::env::env_flag; +/// // Unset variables leave the caller's default in force. +/// assert_eq!(env_flag("HORUS_A_VARIABLE_NOBODY_SETS"), None); +/// ``` +pub fn env_flag(name: &str) -> Option { + parse_flag(std::env::var(name).ok()?.as_str()) +} + +/// The parsing half, separated from the environment so it can be tested +/// without mutating process-global state — `set_var` is not safe against a +/// concurrent `getenv` on another thread, and the test binaries here run +/// in parallel. +pub fn parse_flag(raw: &str) -> Option { + let v = raw.trim(); + if v.is_empty() { + return Some(false); + } + if v.eq_ignore_ascii_case("1") + || v.eq_ignore_ascii_case("true") + || v.eq_ignore_ascii_case("yes") + || v.eq_ignore_ascii_case("on") + { + return Some(true); + } + if v.eq_ignore_ascii_case("0") + || v.eq_ignore_ascii_case("false") + || v.eq_ignore_ascii_case("no") + || v.eq_ignore_ascii_case("off") + { + return Some(false); + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_whole_vocabulary_is_accepted_in_any_case() { + for t in ["1", "true", "TRUE", "True", "yes", "YES", "on", "On"] { + assert_eq!(parse_flag(t), Some(true), "{t:?} must be true"); + } + for f in ["0", "false", "FALSE", "no", "NO", "off", "Off", ""] { + assert_eq!(parse_flag(f), Some(false), "{f:?} must be false"); + } + } + + /// The two divergences that motivated this module, pinned as tests. + #[test] + fn the_words_that_used_to_disagree_now_agree() { + // `HORUS_LOG_FILE=yes` was false because that site matched only + // "true"/"1". + assert_eq!(parse_flag("yes"), Some(true)); + // `HORUS_SIM_MODE=no` was TRUE, because that site treated anything + // that was not "0"/"false"/empty as an opt-in — so a deploy script + // saying "no" got inert SimStubNodes and no diagnostic. + assert_eq!(parse_flag("no"), Some(false)); + } + + #[test] + fn an_unrecognised_value_leaves_the_default_in_force() { + for junk in ["maybe", "2", "yep", "sim", "-1"] { + assert_eq!( + parse_flag(junk), + None, + "{junk:?} must not be read as a truth value in either direction" + ); + } + } + + #[test] + fn whitespace_from_a_launch_file_or_here_doc_is_trimmed() { + assert_eq!(parse_flag(" true "), Some(true)); + assert_eq!(parse_flag("\toff\n"), Some(false)); + } + + #[test] + fn env_flag_reports_none_for_an_unset_variable() { + assert_eq!(env_flag("HORUS_DEFINITELY_UNSET_XYZZY"), None); + } +} diff --git a/horus_sys/src/lib.rs b/horus_sys/src/lib.rs index ced8779dd..08ddcc095 100644 --- a/horus_sys/src/lib.rs +++ b/horus_sys/src/lib.rs @@ -28,6 +28,7 @@ pub mod device; pub mod discover; +pub mod env; pub mod fs; pub mod platform; pub mod process; From 99ceb668554d488c55e92619122874053c3dac7b Mon Sep 17 00:00:00 2001 From: Neos Date: Sun, 6 Sep 2026 03:17:54 -0700 Subject: [PATCH 64/79] docs: reunite RuntimeParams with its own documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `RuntimeParams`'s entire doc block — the description AND a working doctest — sat on `ParamChangeCallback`, a type alias declared between the comment and the struct it belonged to. So the parameter store rendered undocumented on docs.rs while a callback alias carried its example, and `cargo test --doc` reported the example as `params::ParamChangeCallback`. This is the same failure as the `Scheduler` one in the previous commit and it has the same cause: an item inserted between a doc comment and its target, with nothing in the build that can notice. Both are now attached to the right type; the doctest reports as `params::RuntimeParams (line 71)` and still passes. --- horus_core/src/params.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/horus_core/src/params.rs b/horus_core/src/params.rs index 1a519ccce..ce6f8e6ad 100644 --- a/horus_core/src/params.rs +++ b/horus_core/src/params.rs @@ -51,11 +51,21 @@ impl ParamMetadata { } } +/// Callback type for parameter change notification. +/// Called with (key, old_value, new_value). Return Err to reject the change. +pub type ParamChangeCallback = + Arc, &Value) -> Result<(), String> + Send + Sync>; + /// Runtime parameter store for dynamic configuration. /// /// Provides a typed key-value store with validation, persistence, and /// concurrent access support. /// +/// The whole of this block — description and doctest — used to sit on +/// `ParamChangeCallback` below, a type alias, because a new item was inserted +/// between the comment and the struct it belonged to. `RuntimeParams` itself +/// rendered undocumented while a callback alias carried its example. +/// /// # Example /// /// ```rust @@ -69,11 +79,6 @@ impl ParamMetadata { /// let speed: f64 = params.get("max_speed").unwrap(); /// assert_eq!(speed, 1.5); /// ``` -/// Callback type for parameter change notification. -/// Called with (key, old_value, new_value). Return Err to reject the change. -pub type ParamChangeCallback = - Arc, &Value) -> Result<(), String> + Send + Sync>; - pub struct RuntimeParams { /// Parameter storage - BTreeMap maintains sorted order params: Arc>>, From 3a46ccc2b7aafa1041ac581ca1abd6c2b51d2b27 Mon Sep 17 00:00:00 2001 From: Neos Date: Sun, 6 Sep 2026 03:17:54 -0700 Subject: [PATCH 65/79] docs: correct two onboarding claims that CI had already falsified MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CONTRIBUTING.md told a contributor that `readme_contract`'s Quick Start test is run by nothing — "no workflow mentions `readme_contract`" — and handed them a command to compile it by hand. docs/TESTING.md said the same of `docs_parity`: "`grep -rn docs_parity .github/` returns nothing. The suite that exists to catch *undocumented* new surface never runs." Both are false. `docs-contract.yml` runs cargo test -p horus_manager --test readme_contract -- --include-ignored cargo test -p horus_manager --test docs_parity -- --ignored --nocapture Each statement was true when written and was falsified by the commit that added the invocation, without the prose being updated with it — which is the exact failure TESTING.md's own closing rule warns about, in the mirror direction. That direction is the one that costs a newcomer: it sends them to hand-run suites CI already covers, and it teaches them the workflows cannot be trusted. So the closing paragraph now says a table asserting something is *unrun* is itself untested prose, and tells the reader to re-derive a row with `grep -rn '' .github/` before relying on it. `cpp_toolchain_contract` really is unrun and keeps its row. So does the `nextest` config — `grep -rn nextest .github/` still returns nothing — and both were re-checked rather than carried over. --- CONTRIBUTING.md | 17 ++++++++++------- docs/TESTING.md | 22 ++++++++++++++-------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96eb74cb0..2c24f44ad 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -757,13 +757,16 @@ repository; `:850` notes that neither substitutes for the other. and line 794 sits inside the `cli-integration` job (`:309`) which is in the aggregator's needs list (`:1374`). - The exception is `readme_contract.rs`'s `the_readme_quick_start_compiles`, - which is `#[ignore]`d with the reason "run in the docs-contract job" - (`readme_contract.rs:146`). Nothing runs it: no workflow mentions - `readme_contract`, `docs-contract.yml`'s cargo invocations name only the - `docs_*` targets, and `integration-tests.yml:794` passes no - `--include-ignored`. If you change the README's Quick Start, compile it - yourself: + `readme_contract.rs`'s `the_readme_quick_start_compiles` is `#[ignore]`d + with the reason "run in the docs-contract job" (`readme_contract.rs:146`), + and that job does now run it — `docs-contract.yml` invokes + `cargo test -p horus_manager --test readme_contract -- --include-ignored`. + + This paragraph previously said the opposite ("Nothing runs it: no workflow + mentions `readme_contract`") and told you to compile the Quick Start by hand. + That was true when it was written and stopped being true when the invocation + was added; the prose was not updated with it. You can still run it locally, + but CI will catch a broken Quick Start without you: ```bash cargo test -p horus_manager --test readme_contract -- --include-ignored diff --git a/docs/TESTING.md b/docs/TESTING.md index 39a87728a..87ebcb142 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -544,8 +544,8 @@ coverage, and gates nothing. | `cli_runtime_integration` (18 ignored), `launch_and_workflow` (5), `real_project_ipc` (3) | `horus_core/tests/` | need a debug build of `-p horus --examples` plus `horus_manager` that no job produces (`integration-tests.yml:160-166`) | | `transient_subscribers_permanently_flip_a_topic_to_a_lossy_backend` | `participant_count_leak.rs:62` | `#[ignore]`d with the reason "documents an open defect" | | `naive_protocol_tears_proving_the_model_can_detect_it` | `loom_pod_broadcast.rs:225` | deliberately excluded (`integration-tests.yml:186`); it is supposed to fail, so running it in CI would be wrong — but nothing checks that it *still* fails | -| `docs_parity` | `horus_manager/tests/docs_parity.rs` | all three of its tests are `#[ignore]`d with the reason "needs a horus-docs checkout; wired into the docs-contract workflow" (`:193,:259,:336`). `grep -rn docs_parity .github/` returns nothing. The suite that exists to catch *undocumented* new surface never runs | -| `readme_contract`'s `#[ignore]`d test | `readme_contract.rs:146` | reason: "slow: builds a scratch crate; run in the docs-contract job". That job does not name the target. The other 10 tests in the file do run, via `integration-tests.yml:794` | +| `docs_parity` | `horus_manager/tests/docs_parity.rs` | all three tests are `#[ignore]`d with the reason "needs a horus-docs checkout; wired into the docs-contract workflow" (`:193,:259,:336`) — and it **is** wired: `docs-contract.yml` runs `cargo test -p horus_manager --test docs_parity -- --ignored --nocapture`. This row previously read "`grep -rn docs_parity .github/` returns nothing … never runs", which was true when written and was falsified by the commit that added the invocation | +| `readme_contract`'s `#[ignore]`d test | `readme_contract.rs:146` | reason: "slow: builds a scratch crate; run in the docs-contract job". That job **does** name the target now (`docs-contract.yml`, `--include-ignored`); this row previously said it did not. The other 10 tests in the file also run, via `integration-tests.yml:794` | | `cpp_toolchain_contract`'s `#[ignore]`d test | `cpp_toolchain_contract.rs:464` | reason: "slow: cross-compiles horus_cpp; run with `--ignored`". Nothing does. The other 19 run | | `.config/nextest.toml` | repo root config | a fully worked per-test timeout guard (60 s × 3 hard kill at `:50`, `test-threads = 1` at `:66`, `retries = 0` at `:70`, `leak-timeout = "500ms"` at `:77`, a `ci` profile with JUnit output at `:83-101` and a `soak` profile at `:103-110`) that no workflow uses — `grep -rn nextest .github/` returns nothing, and no markdown mentions it | | `horus_manager/tests/web_e2e/` | Puppeteer suite, `package.json:5-7` | referenced only by `.github/dependabot.yml:105`, whose own comment records that no CI workflow runs the suite and that puppeteer is `hasInstallScript: true`, so the dependency graph is exercised exclusively on workstations | @@ -554,12 +554,18 @@ coverage, and gates nothing. | `tests/docker/run_all.sh` | repo `tests/docker/` | `docker-distro-tests.yml:43-61` builds the five Dockerfiles and runs the image `CMD` and `test_user_workflow.sh` directly; `run_all.sh` is unreferenced. `acceptance_linux_macos.sh` and `acceptance_windows.ps1` beside it *are* used, by `cross-platform-parity.yml` and `distribution.yml` | | `horus_core`'s doctests | 13 blocks tagged ```` ```rust ```` and one ```` ```no_run ```` in `horus_core/src` | see below | -Three of these are worse than merely unrun, and the distinction matters when you -are deciding what to fix first: `docs_parity`, `readme_contract` and -`cpp_toolchain_contract` carry `#[ignore]` *reason strings asserting they are -wired*. A reader of the source has no way to discover otherwise short of -grepping the workflows. If you add an `#[ignore]` with a reason naming a -workflow, add the invocation in the same commit. +`cpp_toolchain_contract` carries an `#[ignore]` *reason string asserting it is +wired* while nothing runs it, and a reader of the source has no way to discover +that short of grepping the workflows. If you add an `#[ignore]` whose reason +names a workflow, add the invocation in the same commit. + +`docs_parity` and `readme_contract` used to be listed here for the same fault +and have since been wired up — which exposed the mirror-image hazard this table +now demonstrates twice over. **A table asserting that something is unrun is +itself untested prose, and it rots in the direction that wastes a newcomer's +time**: it sends them to hand-run a suite CI already covers, and it teaches them +to distrust the workflows. Re-derive the rows against +`grep -rn '' .github/` before relying on any of them. **On `nextest.toml`.** Its stated premise has partly expired: it says at `:8` that "None of the jobs in `.github/workflows/ci.yml` set `timeout-minutes`", and From f5499e6cddc0ae149743590e8bc4c4f32049049f Mon Sep 17 00:00:00 2001 From: Neos Date: Sun, 6 Sep 2026 03:56:17 -0700 Subject: [PATCH 66/79] fix(py): declare peek_topic_type in the stubs, and make the error runnable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings, and I got the first one wrong the first time. `peek_topic_type` is registered on the extension module (lib.rs:123) and imported by `horus/__init__.py` as `_peek_topic_type`, but it was never declared in `horus/_horus.pyi`. I previously resolved this thread as stale after grepping `__init__.py` and finding the symbol there — the finding was about the STUB, and those are different files. The stub is what a type checker reads, so to mypy/pyright the function did not exist. Declared now, with a note that `__init__.py` carries a None-returning fallback for builds without it. The `RuntimeError` raised when a bare-string subscription collides with a typed ring suggested `subs=[Sub('cmd_vel', CmdVel)]`. The shape is right — the second argument is a class, not the string `peek_topic_type` returned — but the message never said the class has to be imported, so following it literally raises `NameError` for anyone who has not already got the type in scope. It now names the import, and says what it means when this build has no such class: the ring was written by a peer carrying a message type this install does not. --- horus_py/horus/__init__.py | 6 +++++- horus_py/horus/_horus.pyi | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/horus_py/horus/__init__.py b/horus_py/horus/__init__.py index ba0f411fa..9b3490fb4 100644 --- a/horus_py/horus/__init__.py +++ b/horus_py/horus/__init__.py @@ -760,7 +760,11 @@ def _setup_topics(self): f"but it was declared as a bare string in subs=, which " f"means an untyped MessagePack topic. Those cannot share " f"a name. Declare the type: " - f"subs=[Sub({topic!r}, {ring_type})]" + f"subs=[Sub({topic!r}, {ring_type})] " + f"-- {ring_type} is a class, so import it first " + f"(from horus import {ring_type}). If this build has " + f"no {ring_type}, the ring was written by a peer with " + f"a message type this install does not carry." ) if msg_type is not None: diff --git a/horus_py/horus/_horus.pyi b/horus_py/horus/_horus.pyi index b627c88ec..ace20493c 100644 --- a/horus_py/horus/_horus.pyi +++ b/horus_py/horus/_horus.pyi @@ -10,6 +10,12 @@ from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, Union def get_version() -> str: ... def get_timestamp_ns() -> int: ... +# The message type an existing topic's ring already holds, or None. Reads the +# SHM header only; it never constructs a Topic. `horus/__init__.py` imports it +# as `_peek_topic_type` and falls back to a None-returning stub when the +# extension was built without it. +def peek_topic_type(topic: str) -> Optional[str]: ... + # ── Framework clock ───────────────────────────────────────────────────────── def time_now() -> float: ... From fb82689c050867d9824a34e5bd18b246e97d8d41 Mon Sep 17 00:00:00 2001 From: Neos Date: Sun, 6 Sep 2026 05:09:22 -0700 Subject: [PATCH 67/79] review: three findings on the maintainability branch, one of them wrong MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * `env` was missing from horus_sys's crate-level `## Modules` table. Correct and now listed — a module table that omits a module is worse than no table, because a reader treats it as exhaustive. * The doctest and the unit test both asserted that a specific variable was UNSET. True until someone's shell or a CI image disagrees. The doctest now exercises `parse_flag`, which is pure, and says why in one line. The unit test builds a name from `std::process::id()` so no other process can decide its outcome, and round-trips set/unset through it. * `tyalias@Error` was reported as an invalid disambiguator that "will fail to resolve". It is valid — rustdoc suggested that exact spelling in the error this fixed, and `cargo doc -D warnings` passes with it; the rendered page links to `type.Error.html` either way. Switched to `type@` anyway: it is equally correct, it is the spelling in the rustdoc book, and a future reader should not have to check which of the two is real. --- horus_core/src/error.rs | 2 +- horus_sys/src/env/mod.rs | 35 ++++++++++++++++++++++++++++++----- horus_sys/src/lib.rs | 1 + 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/horus_core/src/error.rs b/horus_core/src/error.rs index eeafa1144..4f9079d05 100644 --- a/horus_core/src/error.rs +++ b/horus_core/src/error.rs @@ -850,7 +850,7 @@ macro_rules! horus_internal { /// Convenience type alias for Results using HorusError. /// -/// **Prefer the short aliases [`Result`] and [`Error`](tyalias@Error) in all new code.** +/// **Prefer the short aliases [`Result`] and [`Error`](type@Error) in all new code.** /// `HorusResult` and `HorusError` are the canonical names but verbose — /// the aliases exist specifically so you can write `Result<()>` and `Error` /// without the `Horus` prefix everywhere. diff --git a/horus_sys/src/env/mod.rs b/horus_sys/src/env/mod.rs index f07bc7f31..48b1dd26b 100644 --- a/horus_sys/src/env/mod.rs +++ b/horus_sys/src/env/mod.rs @@ -44,10 +44,18 @@ /// a recognised truth value — in both cases the caller's default should stand. /// /// ``` -/// # use horus_sys::env::env_flag; -/// // Unset variables leave the caller's default in force. -/// assert_eq!(env_flag("HORUS_A_VARIABLE_NOBODY_SETS"), None); +/// # use horus_sys::env::parse_flag; +/// assert_eq!(parse_flag("yes"), Some(true)); +/// assert_eq!(parse_flag("off"), Some(false)); +/// assert_eq!(parse_flag(""), Some(false)); +/// // Not a truth value: the caller's default stands. +/// assert_eq!(parse_flag("maybe"), None); /// ``` +/// +/// The example exercises [`parse_flag`] rather than `env_flag` on purpose: an +/// assertion about a variable being *unset* is only true until someone's shell +/// or CI image happens to set it, and a doctest that depends on the ambient +/// environment is a flake waiting for a machine that disagrees. pub fn env_flag(name: &str) -> Option { parse_flag(std::env::var(name).ok()?.as_str()) } @@ -121,8 +129,25 @@ mod tests { assert_eq!(parse_flag("\toff\n"), Some(false)); } + /// `env_flag` is the thin half — it reads the variable and hands the value + /// to `parse_flag`. Asserting it on a name this process itself controls + /// keeps the check honest without depending on what the ambient + /// environment happens to hold. #[test] - fn env_flag_reports_none_for_an_unset_variable() { - assert_eq!(env_flag("HORUS_DEFINITELY_UNSET_XYZZY"), None); + fn env_flag_delegates_to_parse_flag() { + // A name unique to this process, so a developer's shell or a CI image + // cannot decide the outcome. + let name = format!("HORUS_ENV_FLAG_SELFTEST_{}", std::process::id()); + assert_eq!(env_flag(&name), None, "an unset variable must not decide"); + + // SAFETY: the name embeds this process's pid and is used by nothing + // else, so no other thread can be reading it concurrently. + unsafe { std::env::set_var(&name, "YES") }; + assert_eq!(env_flag(&name), Some(true)); + unsafe { std::env::set_var(&name, "off") }; + assert_eq!(env_flag(&name), Some(false)); + unsafe { std::env::set_var(&name, "banana") }; + assert_eq!(env_flag(&name), None); + unsafe { std::env::remove_var(&name) }; } } diff --git a/horus_sys/src/lib.rs b/horus_sys/src/lib.rs index 08ddcc095..3a6125b18 100644 --- a/horus_sys/src/lib.rs +++ b/horus_sys/src/lib.rs @@ -18,6 +18,7 @@ //! | [`device`] | Serial port and video device enumeration | //! | [`platform`] | OS/distro detection, directories, shell, package manager | //! | [`sync`] | Environment synchronization (toolchains, system deps) | +//! | [`env`] | One vocabulary for boolean `HORUS_*` environment variables | //! //! ## Platform Support //! From a40481b3de6cc9f7d7b8c5e79fbe8b777f104702 Mon Sep 17 00:00:00 2001 From: Neos Date: Sun, 6 Sep 2026 05:10:00 -0700 Subject: [PATCH 68/79] fix(docs): disambiguate the env module link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `[`env`]` is ambiguous between the `env` module I had just added to the table and the `env!` macro, so the module table I added to fix a documentation gap turned the docs job red — the same failure class as the `[`Error`]` link two commits earlier, walked into while fixing its sibling. `mod@` resolves it. Caught by running the docs build rather than by CI, but only after the push, which is the honest order of events. --- horus_sys/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/horus_sys/src/lib.rs b/horus_sys/src/lib.rs index 3a6125b18..bef5b96d3 100644 --- a/horus_sys/src/lib.rs +++ b/horus_sys/src/lib.rs @@ -18,7 +18,7 @@ //! | [`device`] | Serial port and video device enumeration | //! | [`platform`] | OS/distro detection, directories, shell, package manager | //! | [`sync`] | Environment synchronization (toolchains, system deps) | -//! | [`env`] | One vocabulary for boolean `HORUS_*` environment variables | +//! | [`mod@env`] | One vocabulary for boolean `HORUS_*` environment variables | //! //! ## Platform Support //! From 2df41ef256543b4b58c2942d5eb5b8a31e3d4804 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 06:20:12 +0000 Subject: [PATCH 69/79] ci(actions): bump the actions group across 1 directory with 3 updates Bumps the actions group with 3 updates in the / directory: [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action), [actions/attest-build-provenance](https://github.com/actions/attest-build-provenance) and [softprops/action-gh-release](https://github.com/softprops/action-gh-release). Updates `docker/setup-qemu-action` from 3.7.0 to 4.3.0 - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/c7c53464625b32c7a7e944ae62b3e17d2b600130...1f40c72289eff860ee54a304f1438e3cff362e0a) Updates `actions/attest-build-provenance` from 3.0.0 to 4.2.2 - [Release notes](https://github.com/actions/attest-build-provenance/releases) - [Changelog](https://github.com/actions/attest-build-provenance/blob/main/RELEASE.md) - [Commits](https://github.com/actions/attest-build-provenance/compare/977bb373ede98d70efdf65b84cb5f73e068dcc2a...4d101475d8b20a2381f78447822ac1eab6504dd8) Updates `softprops/action-gh-release` from 3.0.2 to 3.0.3 - [Release notes](https://github.com/softprops/action-gh-release/releases) - [Changelog](https://github.com/softprops/action-gh-release/blob/master/CHANGELOG.md) - [Commits](https://github.com/softprops/action-gh-release/compare/3d0d9888cb7fd7b750713d6e236d1fcb99157228...efb35369e0ad2afab669f228072c1b0d510eae64) --- updated-dependencies: - dependency-name: actions/attest-build-provenance dependency-version: 4.2.2 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: docker/setup-qemu-action dependency-version: 4.3.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: softprops/action-gh-release dependency-version: 3.0.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7124999a7..5801b53d2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -212,7 +212,7 @@ jobs: - name: Set up QEMU (foreign-architecture smoke test) if: runner.os == 'Linux' && matrix.docker_platform != 'linux/amd64' - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 + uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0 with: platforms: ${{ matrix.docker_platform }} @@ -350,7 +350,7 @@ jobs: # one-liner. Runs before the Release is created so a failed attestation # cannot leave published-but-unattested assets behind. - name: Attest build provenance - uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: subject-path: | artifacts/*.tar.gz @@ -375,7 +375,7 @@ jobs: fi - name: Create / update Release - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 + uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 with: tag_name: ${{ steps.tag.outputs.tag }} generate_release_notes: true From fffc88231c1b53303599f65500a51785685c3b55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 06:21:17 +0000 Subject: [PATCH 70/79] deps(npm): bump puppeteer Bumps the npm-minor group with 1 update in the /horus_manager/tests/web_e2e directory: [puppeteer](https://github.com/puppeteer/puppeteer). Updates `puppeteer` from 25.9.0 to 25.10.0 - [Release notes](https://github.com/puppeteer/puppeteer/releases) - [Changelog](https://github.com/puppeteer/puppeteer/blob/main/CHANGELOG.md) - [Commits](https://github.com/puppeteer/puppeteer/compare/puppeteer-v25.9.0...puppeteer-v25.10.0) --- updated-dependencies: - dependency-name: puppeteer dependency-version: 25.10.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: npm-minor ... Signed-off-by: dependabot[bot] --- horus_manager/tests/web_e2e/package-lock.json | 42 +++++++++---------- horus_manager/tests/web_e2e/package.json | 2 +- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/horus_manager/tests/web_e2e/package-lock.json b/horus_manager/tests/web_e2e/package-lock.json index 9f85909d8..ed76088df 100644 --- a/horus_manager/tests/web_e2e/package-lock.json +++ b/horus_manager/tests/web_e2e/package-lock.json @@ -9,16 +9,16 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "puppeteer": "^25.9.0" + "puppeteer": "^25.10.0" } }, "node_modules/@puppeteer/browsers": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-3.2.1.tgz", - "integrity": "sha512-KDz+3qDRdBAlRlMjmKyj6dEs33YHTk/xRHEENSXq6TNnhgoU15ruSHtEBeVF6OZ9tBDY55Se4P0nFMNsipzU9A==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-3.2.2.tgz", + "integrity": "sha512-q2BU4YfO9h/Wt7IcWPcggpOOqLk2Tbs1hDwolvKZrweRjy751OJBKMN9zO5bfD0pzU7X/tvKw/exQds4pM/LOg==", "license": "Apache-2.0", "dependencies": { - "modern-tar": "^0.8.0", + "modern-tar": "^0.8.4", "yargs": "^18.0.0" }, "bin": { @@ -171,26 +171,26 @@ "license": "MIT" }, "node_modules/modern-tar": { - "version": "0.8.4", - "resolved": "https://registry.npmjs.org/modern-tar/-/modern-tar-0.8.4.tgz", - "integrity": "sha512-gN54ddmyzEg10orwZ2u4OOv+bjpMWdIl5jIkodK97bMq8QBSL5c0D7YX0lT1Ooz+99S7+PvFbnxzdjgHo1r41g==", + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/modern-tar/-/modern-tar-0.8.5.tgz", + "integrity": "sha512-snEhs+6G5Tjd4I7tLCDOaoln2RgE0bD19RzEKgvgK2hZ5VKy3MpLhLTZ2fWpXSTg4K2cyPwp+VHATFJhxfnOeA==", "license": "MIT", "engines": { "node": ">=18.0.0" } }, "node_modules/puppeteer": { - "version": "25.9.0", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-25.9.0.tgz", - "integrity": "sha512-2JqQszD2pyDTpIvBH1ZCXdrHgENVNdJIeOM6asbwHRgWknFiaLd1gNB91w/B/0hQHNpafkpxa90lPpBaJM87Hw==", + "version": "25.10.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-25.10.0.tgz", + "integrity": "sha512-9ZfkiaZDQWpGPJp9XTS+Bkn/D78hPvYmtjPfIBeybn05oeY6Jj7aiSbYdfcSQD2UMvC0vE7Yi9PSDo179euRzw==", "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { - "@puppeteer/browsers": "3.2.1", + "@puppeteer/browsers": "3.2.2", "chromium-bidi": "17.0.2", "devtools-protocol": "0.0.1666840", "lilconfig": "^3.1.3", - "puppeteer-core": "25.9.0", + "puppeteer-core": "25.10.0", "typed-query-selector": "^2.12.2" }, "bin": { @@ -201,16 +201,16 @@ } }, "node_modules/puppeteer-core": { - "version": "25.9.0", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-25.9.0.tgz", - "integrity": "sha512-U61rCwSMha62CA/Opy6tCx2Fx+ck7ouiKnbpEApzSoLYMoEu9F71nuFpHL55vmIt33/GYm6eKZVhH2ev0nAIeg==", + "version": "25.10.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-25.10.0.tgz", + "integrity": "sha512-Hy5eMQshOEMil4JUUx03h5pw1HYkYCso1RG/gcpPlFSd4cYPOcopxcXEAxpLPOkOPJb9LIJtwxuj66bSdvknFg==", "license": "Apache-2.0", "dependencies": { - "@puppeteer/browsers": "3.2.1", + "@puppeteer/browsers": "3.2.2", "chromium-bidi": "17.0.2", "devtools-protocol": "0.0.1666840", "typed-query-selector": "^2.12.2", - "webdriver-bidi-protocol": "0.4.2", + "webdriver-bidi-protocol": "0.4.3", "ws": "^8.21.3" }, "engines": { @@ -255,9 +255,9 @@ "license": "MIT" }, "node_modules/webdriver-bidi-protocol": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.2.tgz", - "integrity": "sha512-VSV+fzfChirL3e7jay2yUC7B4HQCGtEWEg/MSSQbK+qWbqeGlRLlXTzPpYr3XGUvbpDHumWZBJxgesg4N7dbtA==", + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.3.tgz", + "integrity": "sha512-uuN0goWfxP22B7J/uAgBpOYNPttC+XVseYE+rSY5+rQ+YBeVz/VORw8WbmLVcqW78zNg5A4qnjNXYUWR3il2ig==", "license": "Apache-2.0" }, "node_modules/wrap-ansi": { diff --git a/horus_manager/tests/web_e2e/package.json b/horus_manager/tests/web_e2e/package.json index 55e4de4d7..ec702f099 100644 --- a/horus_manager/tests/web_e2e/package.json +++ b/horus_manager/tests/web_e2e/package.json @@ -10,6 +10,6 @@ "author": "", "license": "ISC", "dependencies": { - "puppeteer": "^25.9.0" + "puppeteer": "^25.10.0" } } From 0830a1a94429ce393fd66fb6b73a9ff229f8d4a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:09:22 +0000 Subject: [PATCH 71/79] deps(cargo): bump dirs from 6.0.0 to 7.0.0 Bumps dirs from 6.0.0 to 7.0.0. --- updated-dependencies: - dependency-name: dirs dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- horus_core/Cargo.toml | 2 +- horus_manager/Cargo.toml | 2 +- horus_sys/Cargo.toml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 11c56db55..2dabba6ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -872,9 +872,9 @@ dependencies = [ [[package]] name = "dirs" -version = "6.0.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +checksum = "8d57d423b3c82e89b9a24ca3091fee61f456a26edbd28d26c65906f4bc1dcd8f" dependencies = [ "dirs-sys", ] diff --git a/horus_core/Cargo.toml b/horus_core/Cargo.toml index 49f9c642a..6d531874a 100644 --- a/horus_core/Cargo.toml +++ b/horus_core/Cargo.toml @@ -37,7 +37,7 @@ tracing = "0.1" parking_lot = "0.12" chrono = { workspace = true } uuid = { version = "1.0", features = ["v4", "serde"] } -dirs = "6.0" +dirs = "7.0" lazy_static = "1.4" rand = { workspace = true, features = ["small_rng"] } diff --git a/horus_manager/Cargo.toml b/horus_manager/Cargo.toml index 4b9533525..dbf12975f 100644 --- a/horus_manager/Cargo.toml +++ b/horus_manager/Cargo.toml @@ -50,7 +50,7 @@ indicatif = "0.18" console = "0.16" chrono = { workspace = true } ctrlc = "3.4" -dirs = "6.0" +dirs = "7.0" anyhow = "1.0" thiserror = "2.0" toml = "1.1" diff --git a/horus_sys/Cargo.toml b/horus_sys/Cargo.toml index 46ab01d91..534f790a2 100644 --- a/horus_sys/Cargo.toml +++ b/horus_sys/Cargo.toml @@ -12,7 +12,7 @@ log = "0.4" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" toml = "1.1" -dirs = "6" +dirs = "7" core_affinity = "0.8" memmap2 = "0.9" From ff1654063a5851adaeebd7d5d0f7c0cbf3138b0d Mon Sep 17 00:00:00 2001 From: LotfiZ Date: Tue, 15 Sep 2026 23:29:37 -0400 Subject: [PATCH 72/79] fix: read macOS POSIX shared memory in topic readers Resolve the public topic path to the existing POSIX named shared-memory region on macOS, matching the platform-specific reader path already used on Windows. Add a macOS regression test and document the fix. --- CHANGELOG.md | 5 ++ horus_core/src/communication/topic/header.rs | 78 ++++++++++++++------ 2 files changed, 59 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b1664b58..760f1339d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,11 @@ and `Unreleased` is left empty rather than deleted. ## Unreleased +### Fixed + +- `horus topic echo`, `hz`, and `bw` now read live macOS POSIX shared-memory + topics through the public topic path. + ## [0.4.1] — 2026-09-03 201 commits since 0.4.0 (2026-08-22), against a cadence rule that says cut at diff --git a/horus_core/src/communication/topic/header.rs b/horus_core/src/communication/topic/header.rs index 1a4d104b7..7d23415a6 100644 --- a/horus_core/src/communication/topic/header.rs +++ b/horus_core/src/communication/topic/header.rs @@ -1323,27 +1323,25 @@ pub struct TopicSlotRead { /// /// The path-based readers below address a topic by its `topic_shm_path`, which /// was correct back when every backend was a file: `/dev/shm/horus_/topics/ -/// ` on Linux, `/tmp/…` on macOS and on the generic fallback. The Windows -/// backend is not a file at all — `ShmRegion::new` calls +/// ` on Linux and on the generic fallback. The macOS backend uses a +/// POSIX named shared-memory object, and the Windows backend uses /// `CreateFileMappingW(INVALID_HANDLE_VALUE, …)`, a pagefile-backed section -/// named `Local\horus_`, and `ShmRegion::backing_path()` correspondingly -/// reports `None` there. Nothing is ever created at the topic path, so -/// `File::open` failed with `NotFound` and every reader here returned `None` -/// for every live topic on Windows: `horus topic echo` and `horus topic hz` -/// printed nothing and horus_net's SHM reader exported nothing, on a platform -/// where the ring itself works — the cross-process half is fine there, it is -/// only the region's *address* that differs. +/// named `Local\horus_`. Neither named object is created at the public +/// topic path, so `File::open` failed with `NotFound` and every reader here +/// returned `None` for every live topic on macOS and Windows: `horus topic echo` +/// and `horus topic hz` printed nothing and horus_net's SHM reader exported +/// nothing, on platforms where the ring itself works — the cross-process half +/// is fine there, it is only the region's *address* that differs. /// /// The path stays the public address because it is the address wherever there /// is one, and it still *names* the topic where there is not: `topic_shm_path` /// is `/`, so the name is recoverable from the path and the /// section can be opened by it. enum TopicRegion { - /// A file-backed region (Linux, macOS, the generic fallback), mapped - /// read-only. + /// A file-backed region (Linux and the generic fallback), mapped read-only. Mapped(memmap2::Mmap), - /// A Windows named section. The handle is held for the life of the view. - #[cfg(target_os = "windows")] + /// A platform-named region. The handle is held for the life of the view. + #[cfg(any(target_os = "windows", target_os = "macos"))] Section(horus_sys::shm::ShmRegion), } @@ -1353,7 +1351,7 @@ impl std::ops::Deref for TopicRegion { fn deref(&self) -> &[u8] { match self { Self::Mapped(mmap) => &mmap[..], - #[cfg(target_os = "windows")] + #[cfg(any(target_os = "windows", target_os = "macos"))] Self::Section(region) => region.as_slice(), } } @@ -1380,7 +1378,7 @@ pub fn shm_map_count() -> u64 { /// on is made once: the returned view is at least `TOPIC_HEADER_SIZE` bytes. /// /// The file is tried first on every platform, because where one exists it *is* -/// the region. Only when there is none does the Windows section lookup run, so +/// the region. Only when there is none does the platform-named lookup run, so /// nothing about the file-backed platforms changes. fn map_topic_region(path: &std::path::Path) -> Option { use memmap2::MmapOptions; @@ -1408,14 +1406,14 @@ fn map_topic_region(path: &std::path::Path) -> Option { section } -/// Open the Windows named section a topic path refers to. +/// Open the platform-named region a topic path refers to. /// /// `topic_shm_path` is `shm_topics_dir().join(name)` and a topic name may /// itself contain separators, so the name is the whole remainder of the path /// rather than just its last component — `file_name()` on `robot/cmd_vel` /// would ask the kernel for a section called `cmd_vel`, which is either absent /// or, worse, a different topic. -#[cfg(target_os = "windows")] +#[cfg(any(target_os = "windows", target_os = "macos"))] fn open_named_section(path: &std::path::Path) -> Option { let name = path .strip_prefix(horus_sys::shm::shm_topics_dir()) @@ -1432,10 +1430,10 @@ fn open_named_section(path: &std::path::Path) -> Option { Some(TopicRegion::Section(region)) } -/// Non-Windows counterpart of the section lookup: every other backend is +/// Non-named-region counterpart of the section lookup: every other backend is /// file-backed, so a missing file is a missing topic and there is nowhere else /// to look. -#[cfg(not(target_os = "windows"))] +#[cfg(not(any(target_os = "windows", target_os = "macos")))] fn open_named_section(_path: &std::path::Path) -> Option { None } @@ -1443,9 +1441,9 @@ fn open_named_section(_path: &std::path::Path) -> Option { /// Read the latest message payload from a topic's shared-memory region. /// /// `path` is the topic's `topic_shm_path`. It is the backing file itself on -/// Linux, macOS and the fallback backend; on Windows, where the region is a -/// pagefile-backed named section with nothing on disk, it names the section -/// instead. +/// Linux and the fallback backend. On macOS, where the region is a POSIX named +/// object with nothing at that path, and on Windows, where it is a +/// pagefile-backed named section, the path names the region instead. /// /// Returns `None` when: /// - no region exists at `path`, or it cannot be mapped, @@ -1481,8 +1479,9 @@ fn read_slot_inner( ordinal: Option, ) -> Option { // ── 1. Map the topic's shared region ───────────────────────────────────── - // File-backed on Linux/macOS, a named section on Windows; either way at - // least TOPIC_HEADER_SIZE bytes, which is what everything below assumes. + // File-backed on Linux/fallback, a named region on macOS/Windows; either + // way at least TOPIC_HEADER_SIZE bytes, which is what everything below + // assumes. read_slot_from_region(&map_topic_region(path)?, last_write_idx, ordinal) } @@ -4142,6 +4141,37 @@ mod echo_freshness_tests { use crate::communication::Topic; use crate::core::DurationExt; + /// macOS topics use POSIX named shared memory rather than a file at the + /// public topic path. The CLI reader must resolve that name and attach to + /// the live region, otherwise `horus topic echo` sees an active topic but + /// receives no messages. + #[cfg(target_os = "macos")] + #[test] + fn macos_reader_opens_a_live_posix_topic() { + let name = format!("echo_macos_named_region_{}", std::process::id()); + let topic: Topic = match Topic::new(&name) { + Ok(topic) => topic, + Err(error) => { + eprintln!("skipping: shared memory unavailable ({error})"); + return; + } + }; + let Some(path) = horus_sys::shm::topic_shm_path_checked(&name) else { + eprintln!("skipping: cannot resolve topic path"); + return; + }; + + topic.send(0.8); + let slot = super::read_latest_slot_bytes(&path, 0) + .expect("the macOS POSIX SHM region must be readable through the CLI path"); + let value = f32::from_ne_bytes( + slot.payload[..std::mem::size_of::()] + .try_into() + .expect("f32 payload"), + ); + assert_eq!(value, 0.8); + } + /// Reading fresh data must keep working after the ring wraps. /// /// `read_latest_slot_bytes` used to take its freshness signal from From 24b92cc955fadadc0773da1b73752f7ab2f2ffea Mon Sep 17 00:00:00 2001 From: Merge Stack Date: Sat, 19 Sep 2026 18:27:24 -0700 Subject: [PATCH 73/79] fix(topic): count sent/received in the handle's LocalState, not an Arc-shared atomic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The benchmark gate blocked #189 on a +11.5% median regression of Topic_cross_process_oneway_pingpong@1024B. The only per-message additions on that path were two Relaxed fetch_adds (send and recv); an A/B on this box removes 18-49 ns per round trip when they are dropped, and an uncontended lock xadd serialises a pipeline that is otherwise measuring a ring-buffer write. The counters move to LocalState, which is per-handle by the crate's thread-ownership contract, so they are plain u64 increments and each handle reports the traffic it carried — matching MockTopic, the double users write their tests against. The Arc-shared MigrationMetrics keeps the counters that genuinely aggregate across clones. --- .../src/communication/topic/local_state.rs | 25 ++++++++++ horus_core/src/communication/topic/metrics.rs | 33 +++++++------ horus_core/src/communication/topic/mod.rs | 46 +++++++++++++------ horus_core/src/communication/topic/tests.rs | 33 +++++++++---- 4 files changed, 101 insertions(+), 36 deletions(-) diff --git a/horus_core/src/communication/topic/local_state.rs b/horus_core/src/communication/topic/local_state.rs index 22b1a8e9c..7f7fc4610 100644 --- a/horus_core/src/communication/topic/local_state.rs +++ b/horus_core/src/communication/topic/local_state.rs @@ -142,6 +142,29 @@ pub(crate) struct LocalState { /// This is the consumer's own count, so it needs no atomics. pub missed: u64, + /// Messages this handle has SENT through `send()`. + /// + /// A plain `u64`, like [`Self::missed`], because it is this handle's own + /// count and has nothing to synchronise with. That is a correctness + /// property here, not only an optimisation: the first version kept these + /// two counters in the shared `MigrationMetrics` as `AtomicU64`, and the + /// two `Relaxed` `fetch_add`s per round trip moved the cross-process + /// ping-pong median by ~10% — a `lock xadd` serialises a pipeline that is + /// otherwise measuring a ring-buffer write, and the benchmark gate caught + /// it as a blocking regression. `LocalState` is per-handle by the crate's + /// thread contract, so `+= 1` is the whole cost. + /// + /// Read by `Topic::metrics()`. A `Clone` starts its own `LocalState`, so + /// each handle reports the traffic it carried. + pub messages_sent: u64, + + /// Messages this handle has RECEIVED and delivered through `recv()`. + /// + /// Delivered, not polled: an empty ring does not count. `try_recv()` is the + /// low-level path and does not count either. See [`Self::messages_sent`] + /// for why this is not an atomic. + pub messages_received: u64, + /// Ring position (`tail`) the consumer is currently stuck on because the /// slot is claimed-but-unpublished, and the wall clock when the stall was /// first timed. `claim_stall_since_ms == 0` means "no stall timed yet". @@ -265,6 +288,8 @@ impl Default for LocalState { fanout_shm_pub_id: None, fanout_shm_sub_id: None, missed: 0, + messages_sent: 0, + messages_received: 0, claim_stall_tail: 0, claim_stall_since_ms: 0, claim_stall_polls: 0, diff --git a/horus_core/src/communication/topic/metrics.rs b/horus_core/src/communication/topic/metrics.rs index 4a459da81..17c23e65f 100644 --- a/horus_core/src/communication/topic/metrics.rs +++ b/horus_core/src/communication/topic/metrics.rs @@ -3,15 +3,17 @@ use std::sync::atomic::{AtomicU32, AtomicU64}; /// Migration and operational metrics for a Topic +/// +/// `messages_sent` and `messages_received` are deliberately NOT here: they are +/// per-handle counters living in `LocalState`, because a `Relaxed` atomic RMW +/// on every send and recv — which an `Arc`-shared struct needs, since a +/// `Clone` can run on another thread — costs a `lock xadd` in the middle of +/// the publish path and moved the cross-process ping-pong median by ~10% (the +/// benchmark gate caught it as a blocking regression). `LocalState` is +/// per-handle by the crate's thread contract, so its counters are plain +/// increments. See `LocalState::messages_sent`. #[derive(Debug, Default)] pub(crate) struct MigrationMetrics { - /// Messages sent through this topic, counted on every `send()`. - pub messages_sent: AtomicU64, - /// Messages delivered by `recv()` on this topic. - /// - /// `try_recv()` is the low-level path and does not count; `recv()` is what - /// `Topic::recv` uses. - pub messages_received: AtomicU64, /// Number of send failures pub send_failures: AtomicU64, /// Sends dropped because the retry budget was gone, rather than because @@ -23,12 +25,17 @@ pub(crate) struct MigrationMetrics { /// Non-atomic snapshot of topic metrics (for external consumers) /// -/// `messages_sent` and `messages_received` used to move only on the `#[cold]` -/// verbose-logging path, which runs while the `horus monitor` TUI has set a -/// topic's verbose flag — so in an ordinary run both read 0 no matter how much -/// traffic the topic carried, while `MockTopic` maintained them faithfully. A -/// test double that reports what the real transport does not is the one defect -/// a double cannot have. Both are counted on the ordinary paths now. +/// `messages_sent` and `messages_received` are this HANDLE's counts — they live +/// in its `LocalState` and are ordinary increments, not atomics. A `Clone` of a +/// `Topic` starts an independent count, exactly like `MockTopic`, the double +/// users write their tests against. +/// +/// They used to move only on the `#[cold]` verbose-logging path, which runs +/// while the `horus monitor` TUI has set a topic's verbose flag — so in an +/// ordinary run both read 0 no matter how much traffic the topic carried, while +/// `MockTopic` maintained them faithfully. A test double that reports what the +/// real transport does not is the one defect a double cannot have. Both are +/// counted on the ordinary paths now. #[derive(Debug, Clone, Default)] pub struct TopicMetrics { messages_sent: u64, diff --git a/horus_core/src/communication/topic/mod.rs b/horus_core/src/communication/topic/mod.rs index 14d2cb0f1..8c0562003 100644 --- a/horus_core/src/communication/topic/mod.rs +++ b/horus_core/src/communication/topic/mod.rs @@ -2788,10 +2788,16 @@ impl RingTopic< } /// Get a snapshot of the topic's metrics (compatible with Topic API) + /// + /// `messages_sent` / `messages_received` come from this handle's + /// `LocalState` — plain counters, no atomics on the publish path. See + /// `LocalState::messages_sent` for why they are not in the shared + /// `MigrationMetrics`. pub fn metrics(&self) -> TopicMetrics { + let local = self.local(); TopicMetrics::new( - self.metrics.messages_sent.load(Ordering::Relaxed), - self.metrics.messages_received.load(Ordering::Relaxed), + local.messages_sent, + local.messages_received, self.metrics.send_failures.load(Ordering::Relaxed), // The real transport has no recv-failure notion: a `recv()` that // returns `None` is an empty ring. Only `MockTopic` reports one, @@ -2801,6 +2807,21 @@ impl RingTopic< ) } + /// Count one message this handle sent. See `LocalState::messages_sent`. + #[inline(always)] + fn note_message_sent(&self) { + let local = self.local(); + local.messages_sent = local.messages_sent.wrapping_add(1); + } + + /// Count one message this handle delivered through `recv()`. + /// See `LocalState::messages_received`. + #[inline(always)] + fn note_message_received(&self) { + let local = self.local(); + local.messages_received = local.messages_received.wrapping_add(1); + } + #[cfg(test)] pub fn connection_state(&self) -> ConnectionState { ConnectionState::from_u8(self.state.load(Ordering::Relaxed)) @@ -2880,10 +2901,11 @@ impl RingTopic< // reports what the real thing does not is the one defect a double // cannot have. // - // This is a process-local, uncontended relaxed increment on a counter - // only this handle writes — not the contended shared-memory RMW the - // comment above prices at ~28 ns. - self.metrics.messages_sent.fetch_add(1, Ordering::Relaxed); + // One plain `add`, not an atomic RMW: this counter lives in the + // handle's own `LocalState`, and a `Relaxed` `fetch_add` here was the + // blocking benchmark regression this branch had to fix. See + // `LocalState::messages_sent`. + self.note_message_sent(); if unlikely(self.is_verbose()) { self.send_with_content_logging(msg); // Notify event nodes watching this topic. Gated inside @@ -3437,10 +3459,10 @@ impl RingTopic< if received.is_some() { // Counts DELIVERED messages, the mirror of `messages_sent`. As // there, this used to move only on the `#[cold]` verbose path, so - // the public accessor read 0 in every normal run. - self.metrics - .messages_received - .fetch_add(1, Ordering::Relaxed); + // the public accessor read 0 in every normal run — and it is a + // plain `add` on this handle's `LocalState`, not an atomic RMW. + // See `LocalState::messages_sent`. + self.note_message_received(); } received } @@ -3515,9 +3537,7 @@ impl RingTopic< let ipc_ns = start.elapsed().as_nanos() as u64; if let Some(ref _msg) = result { - self.metrics - .messages_received - .fetch_add(1, Ordering::Relaxed); + self.note_message_received(); use crate::core::hlog::{current_node_name, current_tick_number}; use crate::core::log_buffer::{publish_log, LogEntry, LogType}; let now = chrono::Local::now(); diff --git a/horus_core/src/communication/topic/tests.rs b/horus_core/src/communication/topic/tests.rs index 0c4c0fcdf..5da1ada03 100644 --- a/horus_core/src/communication/topic/tests.rs +++ b/horus_core/src/communication/topic/tests.rs @@ -9847,24 +9847,26 @@ fn send_blocking_cross_thread_producer_consumer() { #[test] fn migration_metrics_default_all_zero() { let m = metrics::MigrationMetrics::default(); - assert_eq!(m.messages_sent.load(Ordering::Relaxed), 0); - assert_eq!(m.messages_received.load(Ordering::Relaxed), 0); assert_eq!(m.send_failures.load(Ordering::Relaxed), 0); + assert_eq!(m.send_retry_overruns.load(Ordering::Relaxed), 0); assert_eq!(m.migrations.load(Ordering::Relaxed), 0); } /// MigrationMetrics atomic increment of each counter. +/// +/// `messages_sent` / `messages_received` are deliberately absent: they are +/// per-handle counters in `LocalState`, not atomics in this Arc-shared struct. +/// See `LocalState::messages_sent` for why — an atomic RMW per message on the +/// publish path was a blocking benchmark regression. #[test] fn migration_metrics_atomic_increments() { let m = metrics::MigrationMetrics::default(); - m.messages_sent.fetch_add(5, Ordering::Relaxed); - m.messages_received.fetch_add(3, Ordering::Relaxed); m.send_failures.fetch_add(2, Ordering::Relaxed); + m.send_retry_overruns.fetch_add(1, Ordering::Relaxed); m.migrations.fetch_add(4, Ordering::Relaxed); - assert_eq!(m.messages_sent.load(Ordering::Relaxed), 5); - assert_eq!(m.messages_received.load(Ordering::Relaxed), 3); assert_eq!(m.send_failures.load(Ordering::Relaxed), 2); + assert_eq!(m.send_retry_overruns.load(Ordering::Relaxed), 1); assert_eq!(m.migrations.load(Ordering::Relaxed), 4); } @@ -9904,19 +9906,30 @@ fn topic_metrics_snapshot_starts_at_zero() { } /// MigrationMetrics can be manually incremented and read back via snapshot. +/// +/// It covers the counters that DO live in the Arc-shared struct. The two +/// traffic counters are per-handle `LocalState` fields now and are covered by +/// `topic_metrics_count_traffic_without_the_monitor_attached`; incrementing +/// them here would not be reachable through the snapshot. #[test] fn migration_metrics_manual_increment_visible_in_snapshot() { let t: Topic = Topic::new(unique("metrics_manual")).expect("create"); let mig = t.migration_metrics(); - mig.messages_sent.fetch_add(10, Ordering::Relaxed); - mig.messages_received.fetch_add(7, Ordering::Relaxed); mig.send_failures.fetch_add(2, Ordering::Relaxed); let snap = t.metrics(); - assert_eq!(snap.messages_sent(), 10); - assert_eq!(snap.messages_received(), 7); assert_eq!(snap.send_failures(), 2); + assert_eq!( + snap.messages_sent(), + 0, + "the traffic counters are per-handle and nothing was sent" + ); + assert_eq!( + snap.messages_received(), + 0, + "the traffic counters are per-handle and nothing was delivered" + ); assert_eq!( snap.recv_failures(), 0, From cd50ffa5f3fa3ccf2d425a5f8d679b9bc0679000 Mon Sep 17 00:00:00 2001 From: Merge Stack Date: Sat, 19 Sep 2026 18:36:55 -0700 Subject: [PATCH 74/79] ci(parity): run the macOS topic-reader regression on macOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #215's macos_reader_opens_a_live_posix_topic is #[cfg(target_os = "macos")] under horus_core, and no macOS job in this repository builds horus_core's test targets — the macOS legs build horus_types/horus_macros/ horus_cpp_macros and horus_sys. The new step runs exactly that test, so the fix is verified rather than only compiled. --- .github/workflows/cross-platform-parity.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/cross-platform-parity.yml b/.github/workflows/cross-platform-parity.yml index 55f3ad119..0f1ba4aa0 100644 --- a/.github/workflows/cross-platform-parity.yml +++ b/.github/workflows/cross-platform-parity.yml @@ -123,6 +123,22 @@ jobs: env: HORUS_NAMESPACE: ci_macos_unit_${{ github.run_id }} + # The one horus_core test that is macOS-only, and the reason this job + # exists for it at all: no other macOS job builds horus_core's test + # targets, so `macos_reader_opens_a_live_posix_topic` — the regression + # that `horus topic echo`/`hz`/`bw` read the POSIX named region rather + # than a nonexistent file at the public topic path — was compiled by + # nothing and run by nothing. The filter runs exactly that test; the + # rest of horus_core's macOS suite has pre-existing host-dependent + # failures that are out of scope here. + - name: macOS topic reader regression (#215) + run: | + cargo test -p horus_core --lib -- \ + macos_reader_opens_a_live_posix_topic --test-threads=1 + env: + HORUS_NAMESPACE: ci_macos_reader_${{ github.run_id }} + RUST_BACKTRACE: 1 + # The two tests that execute install.sh's and uninstall.sh's own shell # code instead of reading it as text — completion_install_contract runs # the marked slices against a throw-away $HOME with a stub horus on PATH. From 0d89b049a3d689b5ee8124a399b75f62acede52b Mon Sep 17 00:00:00 2001 From: Merge Stack Date: Sat, 19 Sep 2026 18:44:16 -0700 Subject: [PATCH 75/79] fix(topic): the traffic counters live on RingTopic, not LocalState MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit local_state_fits_four_cache_lines caught the first cut: two u64 fields grew LocalState from 256 to 264 bytes, and that 256-byte budget is a performance contract, not a style rule. RingTopic is per-handle too (a Clone builds a fresh one), so the counters move there as Cells — same one add per message, same per-handle semantics, no layout change. fmt and clippy clean; local_state_fits_four_cache_lines and topic_metrics_count_traffic_without_the_monitor_attached pass. --- .../src/communication/topic/local_state.rs | 25 -------- horus_core/src/communication/topic/metrics.rs | 23 ++++---- horus_core/src/communication/topic/mod.rs | 59 +++++++++++++------ horus_core/src/communication/topic/tests.rs | 2 +- 4 files changed, 54 insertions(+), 55 deletions(-) diff --git a/horus_core/src/communication/topic/local_state.rs b/horus_core/src/communication/topic/local_state.rs index 7f7fc4610..22b1a8e9c 100644 --- a/horus_core/src/communication/topic/local_state.rs +++ b/horus_core/src/communication/topic/local_state.rs @@ -142,29 +142,6 @@ pub(crate) struct LocalState { /// This is the consumer's own count, so it needs no atomics. pub missed: u64, - /// Messages this handle has SENT through `send()`. - /// - /// A plain `u64`, like [`Self::missed`], because it is this handle's own - /// count and has nothing to synchronise with. That is a correctness - /// property here, not only an optimisation: the first version kept these - /// two counters in the shared `MigrationMetrics` as `AtomicU64`, and the - /// two `Relaxed` `fetch_add`s per round trip moved the cross-process - /// ping-pong median by ~10% — a `lock xadd` serialises a pipeline that is - /// otherwise measuring a ring-buffer write, and the benchmark gate caught - /// it as a blocking regression. `LocalState` is per-handle by the crate's - /// thread contract, so `+= 1` is the whole cost. - /// - /// Read by `Topic::metrics()`. A `Clone` starts its own `LocalState`, so - /// each handle reports the traffic it carried. - pub messages_sent: u64, - - /// Messages this handle has RECEIVED and delivered through `recv()`. - /// - /// Delivered, not polled: an empty ring does not count. `try_recv()` is the - /// low-level path and does not count either. See [`Self::messages_sent`] - /// for why this is not an atomic. - pub messages_received: u64, - /// Ring position (`tail`) the consumer is currently stuck on because the /// slot is claimed-but-unpublished, and the wall clock when the stall was /// first timed. `claim_stall_since_ms == 0` means "no stall timed yet". @@ -288,8 +265,6 @@ impl Default for LocalState { fanout_shm_pub_id: None, fanout_shm_sub_id: None, missed: 0, - messages_sent: 0, - messages_received: 0, claim_stall_tail: 0, claim_stall_since_ms: 0, claim_stall_polls: 0, diff --git a/horus_core/src/communication/topic/metrics.rs b/horus_core/src/communication/topic/metrics.rs index 17c23e65f..b9a389cd7 100644 --- a/horus_core/src/communication/topic/metrics.rs +++ b/horus_core/src/communication/topic/metrics.rs @@ -5,13 +5,13 @@ use std::sync::atomic::{AtomicU32, AtomicU64}; /// Migration and operational metrics for a Topic /// /// `messages_sent` and `messages_received` are deliberately NOT here: they are -/// per-handle counters living in `LocalState`, because a `Relaxed` atomic RMW -/// on every send and recv — which an `Arc`-shared struct needs, since a -/// `Clone` can run on another thread — costs a `lock xadd` in the middle of -/// the publish path and moved the cross-process ping-pong median by ~10% (the -/// benchmark gate caught it as a blocking regression). `LocalState` is -/// per-handle by the crate's thread contract, so its counters are plain -/// increments. See `LocalState::messages_sent`. +/// per-handle `Cell`s on `RingTopic`, because a `Relaxed` atomic RMW on +/// every send and recv — which an `Arc`-shared struct needs, since a `Clone` +/// can run on another thread — costs a `lock xadd` in the middle of the +/// publish path and moved the cross-process ping-pong median by ~10% (the +/// benchmark gate caught it as a blocking regression). `RingTopic` is +/// deliberately `!Sync`, so its per-handle counters need no atomics. See +/// `RingTopic::messages_sent`. #[derive(Debug, Default)] pub(crate) struct MigrationMetrics { /// Number of send failures @@ -25,10 +25,11 @@ pub(crate) struct MigrationMetrics { /// Non-atomic snapshot of topic metrics (for external consumers) /// -/// `messages_sent` and `messages_received` are this HANDLE's counts — they live -/// in its `LocalState` and are ordinary increments, not atomics. A `Clone` of a -/// `Topic` starts an independent count, exactly like `MockTopic`, the double -/// users write their tests against. +/// `messages_sent` and `messages_received` are this HANDLE's counts — they are +/// `Cell`s on the handle's `RingTopic`, so the counters are ordinary +/// increments rather than atomics. A `Clone` of a `Topic` starts an +/// independent count, exactly like `MockTopic`, the double users write their +/// tests against. /// /// They used to move only on the `#[cold]` verbose-logging path, which runs /// while the `horus monitor` TUI has set a topic's verbose flag — so in an diff --git a/horus_core/src/communication/topic/mod.rs b/horus_core/src/communication/topic/mod.rs index 8c0562003..0104cae0b 100644 --- a/horus_core/src/communication/topic/mod.rs +++ b/horus_core/src/communication/topic/mod.rs @@ -828,6 +828,24 @@ pub(crate) struct RingTopic { /// Connection state (for network backend compatibility) state: AtomicU8, + /// Messages this handle has SENT, counted on every ordinary `send()`. + /// + /// A per-handle `Cell`, not an `AtomicU64` in the `Arc`-shared + /// `MigrationMetrics`: an atomic RMW on every send and recv is a `lock + /// xadd` in the middle of the publish path, and two of them per round trip + /// moved the cross-process ping-pong median by ~10% — the benchmark gate + /// caught it as a blocking regression on this very branch. A `Cell` is + /// sound here because `RingTopic` is deliberately `!Sync` (see the SAFETY + /// note above); each clone gets its own, so a handle reports the traffic it + /// carried, exactly like `MockTopic`. + messages_sent: std::cell::Cell, + + /// Messages this handle has RECEIVED and delivered through `recv()`. + /// + /// Delivered, not polled: an empty ring does not count, and `try_recv()` + /// does not count either. See [`Self::messages_sent`]. + messages_received: std::cell::Cell, + /// Lazy-initialized TensorPool for spilling large serde messages. /// /// `None` until the first message exceeds `SPILL_THRESHOLD`, at which point @@ -1142,6 +1160,9 @@ impl RingTopic< header_ptr: std::cell::Cell::new(header_ptr), metrics: Arc::new(MigrationMetrics::default()), state: AtomicU8::new(ConnectionState::Connected.into_u8()), + // Counters are per-handle; see the field docs. + messages_sent: std::cell::Cell::new(0), + messages_received: std::cell::Cell::new(0), spill_pool: std::cell::UnsafeCell::new(None), _marker: PhantomData, }) @@ -2789,15 +2810,13 @@ impl RingTopic< /// Get a snapshot of the topic's metrics (compatible with Topic API) /// - /// `messages_sent` / `messages_received` come from this handle's - /// `LocalState` — plain counters, no atomics on the publish path. See - /// `LocalState::messages_sent` for why they are not in the shared - /// `MigrationMetrics`. + /// `messages_sent` / `messages_received` are this handle's own counters — + /// `Cell`s, not atomics, so the publish path pays one `add` rather + /// than a `lock xadd`. See [`Self::messages_sent`]. pub fn metrics(&self) -> TopicMetrics { - let local = self.local(); TopicMetrics::new( - local.messages_sent, - local.messages_received, + self.messages_sent.get(), + self.messages_received.get(), self.metrics.send_failures.load(Ordering::Relaxed), // The real transport has no recv-failure notion: a `recv()` that // returns `None` is an empty ring. Only `MockTopic` reports one, @@ -2807,19 +2826,19 @@ impl RingTopic< ) } - /// Count one message this handle sent. See `LocalState::messages_sent`. + /// Count one message this handle sent. See [`Self::messages_sent`]. #[inline(always)] fn note_message_sent(&self) { - let local = self.local(); - local.messages_sent = local.messages_sent.wrapping_add(1); + self.messages_sent + .set(self.messages_sent.get().wrapping_add(1)); } /// Count one message this handle delivered through `recv()`. - /// See `LocalState::messages_received`. + /// See [`Self::messages_received`]. #[inline(always)] fn note_message_received(&self) { - let local = self.local(); - local.messages_received = local.messages_received.wrapping_add(1); + self.messages_received + .set(self.messages_received.get().wrapping_add(1)); } #[cfg(test)] @@ -2901,10 +2920,10 @@ impl RingTopic< // reports what the real thing does not is the one defect a double // cannot have. // - // One plain `add`, not an atomic RMW: this counter lives in the - // handle's own `LocalState`, and a `Relaxed` `fetch_add` here was the - // blocking benchmark regression this branch had to fix. See - // `LocalState::messages_sent`. + // One plain `add`, not an atomic RMW: this counter is a `Cell` on the + // handle itself (see `RingTopic::messages_sent`), and a `Relaxed` + // `fetch_add` here was the blocking benchmark regression this branch + // had to fix. self.note_message_sent(); if unlikely(self.is_verbose()) { self.send_with_content_logging(msg); @@ -3461,7 +3480,7 @@ impl RingTopic< // there, this used to move only on the `#[cold]` verbose path, so // the public accessor read 0 in every normal run — and it is a // plain `add` on this handle's `LocalState`, not an atomic RMW. - // See `LocalState::messages_sent`. + // See `RingTopic::messages_sent`. self.note_message_received(); } received @@ -3827,6 +3846,10 @@ impl Clone for header_ptr: self.header_ptr.clone(), metrics: Arc::clone(&self.metrics), state: AtomicU8::new(self.state.load(Ordering::Relaxed)), + // A clone counts its OWN traffic; it does not inherit the parent's + // totals. Same semantics as `LocalState` and `MockTopic`. + messages_sent: std::cell::Cell::new(0), + messages_received: std::cell::Cell::new(0), // Clone shares the spill pool if one was already created spill_pool: std::cell::UnsafeCell::new( // SAFETY: `RingTopic` has no `Sync` impl, so `&self` here can diff --git a/horus_core/src/communication/topic/tests.rs b/horus_core/src/communication/topic/tests.rs index 5da1ada03..b8f528ddb 100644 --- a/horus_core/src/communication/topic/tests.rs +++ b/horus_core/src/communication/topic/tests.rs @@ -9856,7 +9856,7 @@ fn migration_metrics_default_all_zero() { /// /// `messages_sent` / `messages_received` are deliberately absent: they are /// per-handle counters in `LocalState`, not atomics in this Arc-shared struct. -/// See `LocalState::messages_sent` for why — an atomic RMW per message on the +/// See `RingTopic::messages_sent` for why — an atomic RMW per message on the /// publish path was a blocking benchmark regression. #[test] fn migration_metrics_atomic_increments() { From 0811d2f89f2d1fc6c432dfa0f5a577091d02525b Mon Sep 17 00:00:00 2001 From: Merge Stack Date: Sat, 19 Sep 2026 18:44:16 -0700 Subject: [PATCH 76/79] =?UTF-8?q?fix(deps):=20RUSTSEC-2026-0285=20?= =?UTF-8?q?=E2=80=94=20rustls=200.23.43=20accepts=20cross-encryption-level?= =?UTF-8?q?=20handshakes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RustSec published RUSTSEC-2026-0285 on 2026-09-14, after main's last audit run, so every new PR against main fails Security Audit today. Patch update: rustls 0.23.43 -> 0.23.45 (and rustls-webpki 0.103.13 -> 0.103.15). No source changes; cargo audit is clean again. --- Cargo.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2dabba6ed..4a045bb7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3262,9 +3262,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.43" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "once_cell", "ring", @@ -3286,9 +3286,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "ring", "rustls-pki-types", @@ -3746,7 +3746,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", From 1bd045436171b671887adb0e0fdb5d374ac4f991 Mon Sep 17 00:00:00 2001 From: Merge Stack Date: Sat, 19 Sep 2026 18:59:22 -0700 Subject: [PATCH 77/79] docs(topic): MockTopic is cfg-gated, so the rustdoc link cannot resolve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stack caught the interaction between #205's internal-docs feature — which makes rustdoc resolve links inside the previously hidden modules — and #189's recv_failures doc, which linked crate::testing::MockTopic. That module is behind cfg(any(test, feature = "test-utils")), so the link is unresolved in a normal doc build. Plain text now. Verified: RUSTDOCFLAGS="-D warnings" cargo doc --workspace --no-deps --exclude horus_manager, and cargo doc -p horus_core --no-deps --features internal-docs, both pass. --- horus_core/src/communication/topic/metrics.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/horus_core/src/communication/topic/metrics.rs b/horus_core/src/communication/topic/metrics.rs index b9a389cd7..57de11969 100644 --- a/horus_core/src/communication/topic/metrics.rs +++ b/horus_core/src/communication/topic/metrics.rs @@ -78,8 +78,10 @@ impl TopicMetrics { /// Receive failures. /// - /// Only [`MockTopic`](crate::testing::MockTopic) produces a non-zero value - /// here, from its `recv_fail_after` / `recv_fail_every_n` fault injection. + /// Only `MockTopic` — the `test-utils` testing double, itself behind a + /// `cfg(feature = "test-utils")` gate, which is why this is not a doc link + /// — produces a non-zero value here, from its `recv_fail_after` / + /// `recv_fail_every_n` fault injection. /// The real transport never increments it, deliberately: a `recv()` that /// returns `None` is an empty ring, not a failure. For real consumer-side /// loss — a publisher lapping a subscriber — use `Topic::missed_count()`. From e74a2bfe6c643a8b8b0c1917fec0830a4204241e Mon Sep 17 00:00:00 2001 From: Merge Stack Date: Sat, 19 Sep 2026 20:39:02 -0700 Subject: [PATCH 78/79] fix(scheduling): a restart whose init() fails stops the node honor_restart_request_with discarded init()'s Result and treated only a panic as failure, so a driver whose device failed to reopen was marked initialised and kept ticking while `horus node restart` reported success. reinit_pending_nodes already refuses exactly this shape for a node still waiting on its first init; the executor path now matches: Err clears initialized, stops the node, emits a diagnostic naming the error, and transitions the node's context to crashed. Found by Copilot's review of the stack (three threads, one per call site: control.rs doc, primitives.rs implementation, types.rs field doc). control_restart.rs: the unknown-node test's fixed 400 ms wait becomes the bounded poll the test above it uses, and restart_whose_init_fails_stops_the_node pins the new behaviour. --- horus_core/src/scheduling/control.rs | 5 ++ horus_core/src/scheduling/primitives.rs | 52 +++++++---- horus_core/src/scheduling/types.rs | 4 + horus_core/tests/control_restart.rs | 109 +++++++++++++++++++++++- 4 files changed, 154 insertions(+), 16 deletions(-) diff --git a/horus_core/src/scheduling/control.rs b/horus_core/src/scheduling/control.rs index 2dce507dd..2c8709b2c 100644 --- a/horus_core/src/scheduling/control.rs +++ b/horus_core/src/scheduling/control.rs @@ -44,6 +44,11 @@ pub enum ControlCommand { /// printed "The scheduler will re-initialize this node on next tick" and /// nothing happened. /// + /// An `init()` that returns `Err`, or panics, is NOT a restart: the node + /// is stopped and reported, not marked initialised and left ticking with + /// setup that failed. Same rule `reinit_pending_nodes` applies to a node + /// still waiting on its first init. + /// /// Added last so a command serialised by an older CLI still decodes. RestartNode { /// Name of the node to re-initialise. diff --git a/horus_core/src/scheduling/primitives.rs b/horus_core/src/scheduling/primitives.rs index d2aa1016a..d1c327482 100644 --- a/horus_core/src/scheduling/primitives.rs +++ b/horus_core/src/scheduling/primitives.rs @@ -186,23 +186,45 @@ fn honor_restart_request_with( node.is_paused = false; let target = &mut node.node; - let panicked = guard_fault_callback(|| { - let _ = target.init(); - }); + let init_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| target.init())); - if panicked { + match init_result { // A node that cannot initialise must not be ticked. - node.is_stopped = true; - super::rt_executor::rt_diag(format_args!( - " Restart: '{}' PANICKED in init() — node stopped", - node.name - )); - } else { - node.initialized = true; - super::rt_executor::rt_diag(format_args!( - " Restart: '{}' re-initialised on its executor", - node.name - )); + Err(_) => { + node.initialized = false; + node.is_stopped = true; + super::rt_executor::rt_diag(format_args!( + " Restart: '{}' PANICKED in init() — node stopped", + node.name + )); + if let Some(ctx) = node.context.as_mut() { + ctx.transition_to_crashed(format!("Restart of '{}' panicked in init()", node.name)); + } + } + // `init()` returning `Err` is NOT a successful restart. The result + // used to be discarded (`let _ = target.init()`), so a driver whose + // device failed to reopen was marked initialised and kept ticking with + // failed hardware while `horus node restart` reported success — the + // same shape `reinit_pending_nodes` already refuses for a node still + // waiting on its first init. Stop it and say why. + Ok(Err(e)) => { + node.initialized = false; + node.is_stopped = true; + super::rt_executor::rt_diag(format_args!( + " Restart: '{}' FAILED in init() — node stopped: {}", + node.name, e + )); + if let Some(ctx) = node.context.as_mut() { + ctx.transition_to_crashed(format!("Restart failed: {}", e)); + } + } + Ok(Ok(())) => { + node.initialized = true; + super::rt_executor::rt_diag(format_args!( + " Restart: '{}' re-initialised on its executor", + node.name + )); + } } } diff --git a/horus_core/src/scheduling/types.rs b/horus_core/src/scheduling/types.rs index a82754b82..9f2c4e070 100644 --- a/horus_core/src/scheduling/types.rs +++ b/horus_core/src/scheduling/types.rs @@ -858,6 +858,10 @@ pub struct NodeControl { /// BestEffort nodes and silently reported every RT, Compute, Event and /// AsyncIo node as unknown — which is every node with a `.rate()`, i.e. /// exactly the ones an operator restarts. + /// + /// The consuming executor treats `init() == Err` as a failed restart: the + /// node is stopped, not marked initialised. See + /// `primitives::honor_restart_request_with`. pub restart_requested: std::sync::atomic::AtomicBool, } diff --git a/horus_core/tests/control_restart.rs b/horus_core/tests/control_restart.rs index 7e5950956..06218708f 100644 --- a/horus_core/tests/control_restart.rs +++ b/horus_core/tests/control_restart.rs @@ -136,7 +136,13 @@ fn restart_for_an_unknown_node_leaves_the_others_alone() { scheduler.run_for(1200_u64.ms()).expect("run_for"); }); - std::thread::sleep(std::time::Duration::from_millis(400)); + // Bounded poll, not a fixed sleep — the same reason the test above gives: + // a fixed wait is a bet that the scheduler thread got scheduled inside it, + // and a lost bet reports a scheduler that never ran as a restart bug. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while inits.load(Ordering::SeqCst) < 1 && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(5)); + } assert_eq!(inits.load(Ordering::SeqCst), 1); ctl.send(ControlCommand::RestartNode { @@ -150,3 +156,104 @@ fn restart_for_an_unknown_node_leaves_the_others_alone() { "a restart naming an unregistered node must not re-init a different one" ); } + +/// A restart whose `init()` returns `Err` must stop the node, not mark it +/// initialised and leave it ticking with setup that failed. +/// +/// `honor_restart_request_with` discarded `init()`'s result (`let _ = +/// target.init()`) and treated only a panic as failure, so a driver whose +/// device failed to reopen kept ticking while `horus node restart` reported +/// success — the exact shape `reinit_pending_nodes` refuses for a node whose +/// first init failed. +#[test] +fn restart_whose_init_fails_stops_the_node() { + struct FailsOnRestart { + inits: Arc, + ticks: Arc, + } + + impl Node for FailsOnRestart { + fn name(&self) -> &str { + "fails_on_restart" + } + fn init(&mut self) -> horus_core::error::Result<()> { + let n = self.inits.fetch_add(1, Ordering::SeqCst); + if n == 0 { + Ok(()) + } else { + Err(horus_core::HorusError::Node( + horus_core::error::NodeError::InitFailed { + node: "fails_on_restart".to_string(), + reason: "device did not reopen".to_string(), + }, + )) + } + } + fn tick(&mut self) { + self.ticks.fetch_add(1, Ordering::SeqCst); + } + } + + let inits = Arc::new(AtomicU32::new(0)); + let ticks = Arc::new(AtomicU32::new(0)); + let sched_name = format!("restart_fail_test_{}", std::process::id()); + let ctl_name = format!("horus.ctl.{}", sched_name); + + let ctl: horus_core::communication::Topic = + horus_core::communication::Topic::new_with_kind( + &ctl_name, + horus_core::communication::TopicKind::System as u8, + ) + .expect("control topic opens"); + + let inits_for_thread = inits.clone(); + let ticks_for_thread = ticks.clone(); + let name_for_thread = sched_name.clone(); + let handle = std::thread::spawn(move || { + let mut scheduler = Scheduler::new() + .name(&name_for_thread) + .tick_rate(200_u64.hz()); + scheduler + .add(FailsOnRestart { + inits: inits_for_thread, + ticks: ticks_for_thread, + }) + .build() + .expect("node registers"); + scheduler.run_for(3000_u64.ms()).expect("run_for"); + }); + + // Startup init: bounded poll, as in the tests above. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while inits.load(Ordering::SeqCst) < 1 && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert_eq!(inits.load(Ordering::SeqCst), 1, "startup init never ran"); + + ctl.send(ControlCommand::RestartNode { + name: "fails_on_restart".to_string(), + }); + + // The restart re-runs init(), which fails. Wait for the attempt itself, + // then give any in-flight tick time to land before sampling. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while inits.load(Ordering::SeqCst) < 2 && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert_eq!( + inits.load(Ordering::SeqCst), + 2, + "the restart never re-ran init()" + ); + + std::thread::sleep(std::time::Duration::from_millis(50)); + let settled = ticks.load(Ordering::SeqCst); + std::thread::sleep(std::time::Duration::from_millis(300)); + assert_eq!( + ticks.load(Ordering::SeqCst), + settled, + "a node whose restart failed in init() must not keep ticking" + ); + + handle.join().expect("scheduler thread joins"); +} From 2825154f770eaa2da98f15b6db18d6d19272a7ab Mon Sep 17 00:00:00 2001 From: Merge Stack Date: Sat, 19 Sep 2026 20:39:02 -0700 Subject: [PATCH 79/79] fix(topic): the metrics counters move on every public send/recv API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit messages_sent counted only send() and messages_received only recv(), so a topic driven through try_send/send_blocking/try_recv read 0 while MockTopic — whose send() delegates to its try_send — counted the same traffic. That is the 'double that reports what the real transport does not' defect these counters exist to prevent, one API surface over. - Sent: send, try_send and send_blocking (the pool-backed try_send paths included) each count one attempt; a send_blocking NoBackpressure refusal that never reaches the ring does not. - Received: try_recv is the funnel for every non-fast-path delivery, so it counts there; the role=Both POD fast path counts its own; recv() and recv_with_content_logging() no longer double count. - horus_py's auto-detect message now sends generated types to horus._horus, where they actually live (they are not re-exported from horus). - New parity test runs the same sequence through a real Topic and a MockTopic and asserts equal counters. Found by Copilot's review of the stack. --- horus_core/src/communication/topic/metrics.rs | 8 ++- horus_core/src/communication/topic/mod.rs | 64 +++++++++++++------ horus_core/src/communication/topic/tests.rs | 56 ++++++++++++++++ horus_py/horus/__init__.py | 12 ++-- 4 files changed, 112 insertions(+), 28 deletions(-) diff --git a/horus_core/src/communication/topic/metrics.rs b/horus_core/src/communication/topic/metrics.rs index 57de11969..92356148d 100644 --- a/horus_core/src/communication/topic/metrics.rs +++ b/horus_core/src/communication/topic/metrics.rs @@ -27,9 +27,11 @@ pub(crate) struct MigrationMetrics { /// /// `messages_sent` and `messages_received` are this HANDLE's counts — they are /// `Cell`s on the handle's `RingTopic`, so the counters are ordinary -/// increments rather than atomics. A `Clone` of a `Topic` starts an -/// independent count, exactly like `MockTopic`, the double users write their -/// tests against. +/// increments rather than atomics. Sent counts every attempt made through +/// `send()`, `try_send()` or `send_blocking()`; received counts every delivery +/// through `recv()` or `try_recv()`. A `Clone` of a `Topic` starts an +/// independent count — the same numbers `MockTopic`, the double users write +/// their tests against, reports for the same calls. /// /// They used to move only on the `#[cold]` verbose-logging path, which runs /// while the `horus monitor` TUI has set a topic's verbose flag — so in an diff --git a/horus_core/src/communication/topic/mod.rs b/horus_core/src/communication/topic/mod.rs index 0104cae0b..1378ccb39 100644 --- a/horus_core/src/communication/topic/mod.rs +++ b/horus_core/src/communication/topic/mod.rs @@ -828,7 +828,10 @@ pub(crate) struct RingTopic { /// Connection state (for network backend compatibility) state: AtomicU8, - /// Messages this handle has SENT, counted on every ordinary `send()`. + /// Messages this handle has SENT, counted as attempts on every public + /// send API that engages the ring: `send()`, `try_send()` and + /// `send_blocking()` (a `NoBackpressure` refusal never reaches the ring + /// and does not count). /// /// A per-handle `Cell`, not an `AtomicU64` in the `Arc`-shared /// `MigrationMetrics`: an atomic RMW on every send and recv is a `lock @@ -836,14 +839,13 @@ pub(crate) struct RingTopic { /// moved the cross-process ping-pong median by ~10% — the benchmark gate /// caught it as a blocking regression on this very branch. A `Cell` is /// sound here because `RingTopic` is deliberately `!Sync` (see the SAFETY - /// note above); each clone gets its own, so a handle reports the traffic it - /// carried, exactly like `MockTopic`. + /// note above); each clone gets its own, and a handle reports the same + /// traffic `MockTopic` would for the same calls. messages_sent: std::cell::Cell, - /// Messages this handle has RECEIVED and delivered through `recv()`. - /// - /// Delivered, not polled: an empty ring does not count, and `try_recv()` - /// does not count either. See [`Self::messages_sent`]. + /// Messages this handle has RECEIVED and delivered, through `recv()` or + /// `try_recv()`. Delivered, not polled: an empty ring does not count. + /// See [`Self::messages_sent`]. messages_received: std::cell::Cell, /// Lazy-initialized TensorPool for spilling large serde messages. @@ -2363,7 +2365,18 @@ impl RingTopic< // SAFETY: recv_fn UnsafeCell is only mutated by this thread (single-owner contract); // the fn pointer is always valid — set to recv_uninitialized at construction, then // updated to a backend-specific function after registration. - unsafe { (*self.recv_fn.get())(self) } + let received = unsafe { (*self.recv_fn.get())(self) }; + if received.is_some() { + // Counted HERE and not only in `recv()`: this is the funnel for + // every receive path that is not the role=Both POD fast path — + // `recv()`'s fallback, the pool-backed `recv()`s and the public + // `try_recv()` — so a delivery counts exactly once whichever API + // delivered it. `MockTopic` counts its `try_recv()` deliveries + // too; a double that reports what the real transport does not is + // the defect this counter exists to avoid. + self.note_message_received(); + } + received } /// Auto-grow the SHM slot size when a serialized message exceeds the current limit. @@ -3474,21 +3487,15 @@ impl RingTopic< if unlikely(self.is_verbose()) { return self.recv_with_content_logging(); } - let received = self.recv_uncounted(); - if received.is_some() { - // Counts DELIVERED messages, the mirror of `messages_sent`. As - // there, this used to move only on the `#[cold]` verbose path, so - // the public accessor read 0 in every normal run — and it is a - // plain `add` on this handle's `LocalState`, not an atomic RMW. - // See `RingTopic::messages_sent`. - self.note_message_received(); - } - received + self.recv_counted() } - /// `recv()` without the metrics bookkeeping. + /// `recv()`'s core. Every returning path counts a delivered message + /// exactly once: the role=Both POD fast path here, everything else via + /// `try_recv`. The counter is a plain `add` on this handle's `RingTopic`, + /// not an atomic RMW; see [`RingTopic::messages_sent`]. #[inline(always)] - fn recv_uncounted(&self) -> Option { + fn recv_counted(&self) -> Option { // Fast path: role=Both (same-instance, same-thread pub+sub) and POD. // // Gated on `is_pod` in lockstep with `send`: this reads back through @@ -3518,6 +3525,9 @@ impl RingTopic< if unlikely(local.msg_counter & (EPOCH_CHECK_INTERVAL - 1) == 0) { self.check_migration_inline(); } + // The fast path returns without going through `try_recv`, so + // it owns its own delivery count. + self.note_message_received(); return Some(msg); } // Empty — amortized epoch check @@ -3556,7 +3566,7 @@ impl RingTopic< let ipc_ns = start.elapsed().as_nanos() as u64; if let Some(ref _msg) = result { - self.note_message_received(); + // No count here: `try_recv()` above already counted the delivery. use crate::core::hlog::{current_node_name, current_tick_number}; use crate::core::log_buffer::{publish_log, LogEntry, LogType}; let now = chrono::Local::now(); @@ -4720,6 +4730,11 @@ where // Until then, a stalled consumer is visible in `dropped_count()` / // `send_retry_overruns()`, not in staleness. self.ring.bump_messages_total(); + // `TopicMetrics::messages_sent` counts the same attempts — this API + // included. It used to count only `send()`, so a topic driven through + // `try_send` read 0 while `MockTopic` counted the same traffic; see + // `RingTopic::messages_sent`. + self.ring.note_message_sent(); self.ring.try_send(msg) } @@ -4791,6 +4806,10 @@ where // an attempt like `send()`'s dropped message does. if !matches!(result, Err(SendBlockingError::NoBackpressure)) { self.ring.bump_messages_total(); + // Same rule for the metric: an attempt that reached the ring + // counts, a refusal that never did does not. See + // `RingTopic::messages_sent`. + self.ring.note_message_sent(); } result } @@ -4879,6 +4898,7 @@ impl Topic { // `SubscriptionFreshness` would read 0 Hz and safe-state or halt a // subscriber whose frames were arriving normally. self.ring.bump_messages_total(); + self.ring.note_message_sent(); let wire = img.to_wire(&self.pool); match self.ring.try_send(wire) { Ok(()) => { @@ -4945,6 +4965,7 @@ impl Topic { // `SubscriptionFreshness` would read 0 Hz and safe-state or halt a // subscriber whose frames were arriving normally. self.ring.bump_messages_total(); + self.ring.note_message_sent(); let wire = pc.to_wire(&self.pool); match self.ring.try_send(wire) { Ok(()) => { @@ -5011,6 +5032,7 @@ impl Topic { // `SubscriptionFreshness` would read 0 Hz and safe-state or halt a // subscriber whose frames were arriving normally. self.ring.bump_messages_total(); + self.ring.note_message_sent(); let wire = depth.to_wire(&self.pool); match self.ring.try_send(wire) { Ok(()) => { diff --git a/horus_core/src/communication/topic/tests.rs b/horus_core/src/communication/topic/tests.rs index b8f528ddb..954df75eb 100644 --- a/horus_core/src/communication/topic/tests.rs +++ b/horus_core/src/communication/topic/tests.rs @@ -9977,6 +9977,62 @@ fn topic_metrics_count_traffic_without_the_monitor_attached() { ); } +/// Every public send and receive API moves the counters, and to the same +/// numbers `MockTopic` reports for the same calls. +/// +/// `messages_sent` counted only `send()` and `messages_received` only +/// `recv()`, so a topic driven through `try_send` / `send_blocking` / +/// `try_recv` — the APIs the docs point users to on critical topics — read 0 +/// while the mock counted the same traffic. `MockTopic::send` delegates to its +/// `try_send`, so it cannot make that distinction; the real transport must +/// not either. A test double that reports what the real thing does not is the +/// one defect a double cannot have. +#[test] +fn counters_move_on_every_public_send_and_recv_api() { + use crate::testing::MockTopic; + + let t: Topic = Topic::new(unique("metrics_parity")).expect("create"); + let mock = MockTopic::simple("metrics_parity_mock"); + + // The same sequence through both: ordinary send (an attempt), try_send + // (an attempt), send_blocking (an attempt that reaches the ring). + t.send(1); + mock.send(1); + t.try_send(2).expect("try_send must reach the ring"); + mock.try_send(2).expect("try_send must reach the ring"); + t.send_blocking(3, 500_u64.ms()) + .expect("send_blocking must reach the ring"); + mock.send_blocking(3, 500_u64.ms()) + .expect("send_blocking must reach the ring"); + + assert_eq!(t.recv(), Some(1)); + assert_eq!(t.try_recv(), Some(2)); + assert_eq!(t.recv(), Some(3)); + assert_eq!(t.recv(), None); + assert_eq!(mock.recv(), Some(1)); + assert_eq!(mock.try_recv(), Some(2)); + assert_eq!(mock.recv(), Some(3)); + assert_eq!(mock.recv(), None); + + let real = t.metrics(); + let fake = mock.metrics(); + assert_eq!( + real.messages_sent(), + 3, + "send, try_send and send_blocking are three attempts" + ); + assert_eq!( + real.messages_received(), + 3, + "every delivered message counts, whichever API delivered it" + ); + assert_eq!( + (real.messages_sent(), real.messages_received()), + (fake.messages_sent(), fake.messages_received()), + "the real transport and its testing double must report the same traffic" + ); +} + // ============================================================================ // LOCAL_STATE MODULE TESTS // ============================================================================ diff --git a/horus_py/horus/__init__.py b/horus_py/horus/__init__.py index 9b3490fb4..ad0411e02 100644 --- a/horus_py/horus/__init__.py +++ b/horus_py/horus/__init__.py @@ -761,10 +761,14 @@ def _setup_topics(self): f"means an untyped MessagePack topic. Those cannot share " f"a name. Declare the type: " f"subs=[Sub({topic!r}, {ring_type})] " - f"-- {ring_type} is a class, so import it first " - f"(from horus import {ring_type}). If this build has " - f"no {ring_type}, the ring was written by a peer with " - f"a message type this install does not carry." + f"-- {ring_type} is a class, so import it first: " + f"built-in types live on `horus` " + f"(from horus import {ring_type}); generated message " + f"types live on `horus._horus` " + f"(from horus._horus import {ring_type}). If this " + f"build has no {ring_type}, the ring was written by " + f"a peer with a message type this install does not " + f"carry." ) if msg_type is not None: