diff --git a/.github/workflows/docs-contract.yml b/.github/workflows/docs-contract.yml index f67400b94..a60e1e678 100644 --- a/.github/workflows/docs-contract.yml +++ b/.github/workflows/docs-contract.yml @@ -92,6 +92,14 @@ env: # Verifying a different docs commit means bumping this SHA in a PR, so the # change is reviewed here before it runs here. # + # Bumped from 4a612b1 to 9a1a0c1 (horus-docs main, the merge of + # horus-docs#17). That PR is the documentation half of the same bidirectional + # audit this branch is the code half of, so the two belong at the same pin: + # against 4a612b1 the contract validates this branch's runtime corrections + # against pages that still describe what the runtime did BEFORE them, which + # is the drift the audit existed to find. Exactly one commit, and 4a612b1 is + # its parent. Verified from this tree: 14 passed, 0 failed. + # # Bumped from 38368c8 to 4a612b1 (horus-docs main, the merge of # horus-docs#18), for the same reason 38368c8 replaced af057d9 before it. # This branch adds six operator-facing variables — HORUS_RECORD_MAX_MB, @@ -107,7 +115,7 @@ env: # it: 14 passed, 0 failed. The bump is 7 commits, not 1 — horus-docs#18 is # only the newest of them, and that suite result is what says the other six, # written by other people for other reasons, are compatible with this branch. - DOCS_PINNED_REF: 4a612b108d51308537cdd5c737bbc0181f8354ac + DOCS_PINNED_REF: 9a1a0c107bc2e04b75cbdd08a04462e242285bce jobs: # ── Tier 1 ──────────────────────────────────────────────────────────────── diff --git a/benchmarks/README.md b/benchmarks/README.md index bf1432de8..ccb39a0f6 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -71,7 +71,7 @@ the withdrawn tables dropped, and it is not comparable across kinds: | `CrossThread-MPMC` | `PodShm` | `send` | Producer-side `send()` cost under contention | | `CrossProc-1P1C` | `SpscShm` | `one-way` | Pub-to-sub delivery, RDTSC timestamp in payload | | `CrossProc-2P1C` | `MpscShm` | `one-way` | Pub-to-sub delivery | -| `CrossProc-1PMC` | `SpmcShm` | `one-way` | Pub-to-sub delivery | +| `CrossProc-1PMC` | `PodShm` | `one-way` | Pub-to-sub delivery; one-to-many POD routes to broadcast | | `CrossProc-PodShm` | `PodShm` | `broadcast` | Latest-value broadcast; readers may skip ahead | | `RawAtomic` | — | `one-way` | Raw SHM atomic store/load — the hardware floor | @@ -104,7 +104,7 @@ sudo cpupower frequency-set -g performance echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo ``` -> Cross-process paced benchmarks (SpscShm, MpscShm, PodShm) are heavily affected by OS scheduling under `powersave` governor. The SpmcShm path (seqlock broadcast) is least affected since readers don't block. +> Cross-process paced benchmarks (SpscShm, MpscShm) are heavily affected by OS scheduling under `powersave` governor. The `PodShm` broadcast path is least affected, since its readers do not block — a one-to-many POD topology resolves to `PodShm`, not `SpmcShm`, which is why no row above names the latter. ## License diff --git a/horus_core/src/communication/mod.rs b/horus_core/src/communication/mod.rs index 823062341..bd137224a 100644 --- a/horus_core/src/communication/mod.rs +++ b/horus_core/src/communication/mod.rs @@ -41,8 +41,9 @@ pub use topic::{set_topic_verbose, TOPIC_VERBOSE_OFFSET}; // Ring-buffer inspector for CLI tools (`horus topic echo`) #[doc(hidden)] pub use topic::{ - read_latest_slot_bytes, read_slots_since, read_topic_header_info, read_topic_messages_total, - read_topic_sequence, shm_map_count, TopicHeaderInfo, TopicReader, TopicSlotRead, + peek_topic_type_name, read_latest_slot_bytes, read_slots_since, read_topic_header_info, + read_topic_messages_total, read_topic_sequence, shm_map_count, TopicHeaderInfo, TopicReader, + TopicSlotRead, }; // Topic lifecycle hook for horus_net network replication diff --git a/horus_core/src/communication/topic/dispatch.rs b/horus_core/src/communication/topic/dispatch.rs index 3ac37e8b0..9b90fdc82 100644 --- a/horus_core/src/communication/topic/dispatch.rs +++ b/horus_core/src/communication/topic/dispatch.rs @@ -1199,7 +1199,7 @@ pub(super) fn send_fanout_shm { // COMM-H1: same as the send path — loud, rate-limited, never silent. - if should_report_endpoint_exhaustion(topic.name()) { + if should_report_endpoint_exhaustion(topic.name(), Exhausted::FanoutEndpoints) { tracing::warn!( "FanoutShm topic '{}': all {} subscriber endpoint slots are \ live — this subscriber has NO comms until a slot frees \ @@ -1887,7 +1887,16 @@ pub(super) fn send_uninitialized< topic: &RingTopic, msg: T, ) -> Result<(), T> { - if topic.ensure_producer().is_err() { + if let Err(e) = topic.ensure_producer() { + // As `recv_uninitialized`: the caller gets its message back with no + // indication that the cause is permanent for this handle. + if should_report_endpoint_exhaustion(topic.name(), Exhausted::ParticipantTable) { + tracing::warn!( + "topic '{}': this publisher is NOT registered and will deliver nothing — {}", + topic.name(), + e + ); + } return Err(msg); } // SAFETY: ensure_producer() → initialize_backend() → set_dispatch_fn_ptrs() @@ -2612,7 +2621,24 @@ pub(super) fn recv_uninitialized< >( topic: &RingTopic, ) -> Option { - if topic.ensure_consumer().is_err() { + if let Err(e) = topic.ensure_consumer() { + // Registration failed — almost always the 16-slot participant table + // being full. The role stays `Unregistered`, so every later `recv()` + // re-enters this cold path and fails identically: `None` forever, from + // a subscriber that looks merely idle. Discarding the error here made + // the message `register_role` builds — which names the limit and says + // what to do about it — reach nothing at all. + // + // The same 16-slot wall one layer down is reported, deliberately, by + // this same rate limiter: "never let 'all 16 slots live' become a + // SILENT no-comms endpoint". + if should_report_endpoint_exhaustion(topic.name(), Exhausted::ParticipantTable) { + tracing::warn!( + "topic '{}': this subscriber is NOT registered and will receive nothing — {}", + topic.name(), + e + ); + } return None; } // SAFETY: ensure_consumer() → initialize_backend() → set_dispatch_fn_ptrs() @@ -2632,13 +2658,20 @@ pub(super) fn recv_uninitialized< /// Keyed by topic and rate-limited per topic instead, so a hot loop still emits /// once a minute rather than once ever, and a second topic hitting the same /// wall is never masked by the first. -fn should_report_endpoint_exhaustion(topic: &str) -> bool { +/// +/// Keyed by `reason` as well as topic, for the same argument one level down: a +/// topic can hit two different walls — its FanoutShm endpoint bitmap and the +/// 16-slot participant table — and they have different causes and different +/// fixes. Sharing one bucket would let whichever fired first silence the other +/// for a minute, which is the masking this function exists to prevent. +pub(super) fn should_report_endpoint_exhaustion(topic: &str, reason: Exhausted) -> bool { use std::collections::HashMap; use std::sync::Mutex; use std::time::{Duration, Instant}; const QUIET: Duration = Duration::from_secs(60); - static LAST: Mutex>> = Mutex::new(None); + #[allow(clippy::type_complexity)] + static LAST: Mutex>> = Mutex::new(None); let mut guard = match LAST.lock() { Ok(g) => g, @@ -2646,15 +2679,27 @@ fn should_report_endpoint_exhaustion(topic: &str) -> bool { }; let seen = guard.get_or_insert_with(HashMap::new); let now = Instant::now(); - match seen.get(topic) { + let key = (topic.to_string(), reason); + match seen.get(&key) { Some(t) if now.duration_since(*t) < QUIET => false, _ => { - seen.insert(topic.to_string(), now); + seen.insert(key, now); true } } } +/// Which wall a topic ran into. Rate-limited separately; see +/// `should_report_endpoint_exhaustion`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(super) enum Exhausted { + /// The FanoutShm publisher or subscriber endpoint bitmap is full. + FanoutEndpoints, + /// The topic's 16-slot participant table is full, so this handle holds no + /// registration and will never send or receive anything. + ParticipantTable, +} + #[cfg(test)] mod tests { use super::*; diff --git a/horus_core/src/communication/topic/header.rs b/horus_core/src/communication/topic/header.rs index 1a4d104b7..6f3ee0de8 100644 --- a/horus_core/src/communication/topic/header.rs +++ b/horus_core/src/communication/topic/header.rs @@ -8,7 +8,7 @@ use std::collections::HashMap; use std::mem; use std::mem::offset_of; -use std::sync::atomic::{AtomicU32, AtomicU64, AtomicU8, Ordering}; +use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8, Ordering}; use std::sync::{Mutex, OnceLock}; use crate::error::{HorusError, HorusResult}; @@ -104,8 +104,41 @@ pub(crate) struct ParticipantEntry { /// 0 = local process (default). Non-zero = low byte of remote peer_id_hash. /// Set by horus_net ShmRingWriter when writing network-received data. pub(crate) source_host: AtomicU8, - /// Padding for alignment - pub(crate) _pad: [u8; 5], + /// Padding to align `generation` on a 2-byte boundary. + pub(crate) _pad0: [u8; 1], + /// Bumped every time this slot is CLAIMED by a new owner. + /// + /// A handle records (slot, generation) when it registers, so `Drop` can + /// tell its own registration from a later one that reused the same slot. + /// Without it a release has only the slot index to go on, and + /// `register_role`'s pass 3 can hand that index to another thread while + /// the old handle is still alive — charging the drop to somebody else's + /// registration. Under-counting is the dangerous direction: + /// `nothing_is_draining` lets a producer retire unread slots once + /// `sub_count()` reads 0, so a lost registration becomes silent message + /// loss on a subscriber that is still reading. + /// + /// Carved out of `_pad`, so no existing field moves and the entry is still + /// 24 bytes. + pub(crate) generation: AtomicU16, + /// How many live handles on this thread hold the PRODUCER role here. + /// + /// `register_role` keys an entry on (pid, thread), not on the handle, so + /// several `Topic` handles on one thread deliberately share one slot and + /// one increment of `publisher_count`. The role may only be given back + /// when the LAST of them goes, which a (slot, generation) match alone + /// cannot tell: the first handle to drop would clear an entry two others + /// are still using. + /// + /// Counted PER ROLE because one thread publishing and subscribing to the + /// same topic is a supported configuration — it is the `role = Both` fast + /// path. A single count could not express "the last subscriber left while a + /// publisher stays", and would hold `subscriber_count` up for as long as + /// the publisher lived. + pub(crate) pub_handles: AtomicU8, + /// How many live handles on this thread hold the CONSUMER role here. + /// See [`Self::pub_handles`]. + pub(crate) sub_handles: AtomicU8, /// When this lease runs out, in [`current_time_ms`] milliseconds — /// MONOTONIC since boot, NOT since the Unix epoch. Written by the owning /// process and compared by any other process on the host; see @@ -134,12 +167,30 @@ impl ParticipantEntry { /// Clear this entry (use atomic for thread safety) pub(crate) fn clear(&self) { - self.active.store(0, Ordering::Release); + // Clear the identity BEFORE publishing `active = 0`: that store is what + // makes the slot claimable — `claim_free_slot` scans for + // `active == 0` and CASes it to the initializing sentinel — so anything + // written after it could be overwriting a NEW owner's fields. + // + // The order used to be the other way round, which was harmless while + // the only non-test caller was `TopicHeader::initialize`: that runs + // single-threaded before `magic` is published, so nothing could be + // scanning. Giving a registration back on `Drop` made this reachable + // from a live process on every last-handle release. + // + // `reap_dead_participants_now` writes the same sequence inline and + // carries the same reasoning; this is the one place it belongs. self.lease_expires_ms.store(0, Ordering::Release); self.role.store(0, Ordering::Release); self.pid.store(0, Ordering::Release); self.thread_id_hash.store(0, Ordering::Release); self.source_host.store(0, Ordering::Release); + self.pub_handles.store(0, Ordering::Release); + self.sub_handles.store(0, Ordering::Release); + self.active.store(0, Ordering::Release); + // `generation` is deliberately NOT reset. Its whole purpose is to + // outlive the registration it identifies, so that a handle holding a + // stale (slot, generation) can be told apart from the current owner. } } @@ -482,7 +533,10 @@ impl TopicHeader { role: AtomicU8::new(0), active: AtomicU8::new(0), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(0), }), } @@ -713,12 +767,12 @@ impl TopicHeader { } /// Register as a publisher (returns slot index or error) - pub fn register_producer(&self) -> HorusResult { + pub fn register_producer(&self) -> HorusResult<(usize, u16)> { self.register_role(1, &self.publisher_count) } /// Register as a subscriber (returns slot index or error) - pub fn register_consumer(&self) -> HorusResult { + pub fn register_consumer(&self) -> HorusResult<(usize, u16)> { self.register_role(2, &self.subscriber_count) } @@ -736,7 +790,7 @@ impl TopicHeader { thread_hash: u32, now_ms: u64, timeout_ms: u64, - ) -> Option { + ) -> Option<(usize, u16)> { for (i, p) in self.participants.iter().enumerate() { if p.active.load(Ordering::Acquire) != 0 { continue; @@ -748,8 +802,9 @@ impl TopicHeader { { continue; } - self.finish_claim(p, role_bit, counter, pid, thread_hash, now_ms, timeout_ms); - return Some(i); + let generation = + self.finish_claim(p, role_bit, counter, pid, thread_hash, now_ms, timeout_ms); + return Some((i, generation)); } None } @@ -766,7 +821,7 @@ impl TopicHeader { thread_hash: u32, now_ms: u64, timeout_ms: u64, - ) { + ) -> u16 { // The one place a thread takes ownership of a slot, so the one place to // record it as a live owner: the last-resort pass in `register_role` // refuses to reclaim an entry whose thread is still running. @@ -779,13 +834,28 @@ impl TopicHeader { self.total_participants.fetch_add(1, Ordering::AcqRel); self.last_topology_change_ms .store(now_ms, Ordering::Release); + // A new owner, so a new generation: any handle still holding the + // previous (slot, generation) must not be able to release this one. + // Wrapping is fine — a release compares for equality, and 65536 claims + // of one slot would have to happen while a single stale handle lived. + let generation = p.generation.fetch_add(1, Ordering::AcqRel).wrapping_add(1); + // Exactly one handle owns this role at this point; see + // `ParticipantEntry::pub_handles`. + if role_bit & 1 != 0 { + p.pub_handles.store(1, Ordering::Release); + p.sub_handles.store(0, Ordering::Release); + } else { + p.sub_handles.store(1, Ordering::Release); + p.pub_handles.store(0, Ordering::Release); + } // Finalize: transition from initializing (2) to active (1). // This makes the slot visible with all fields properly set. p.active.store(1, Ordering::Release); + generation } /// Shared registration logic for producer (role_bit=1) and consumer (role_bit=2). - fn register_role(&self, role_bit: u8, counter: &AtomicU32) -> HorusResult { + fn register_role(&self, role_bit: u8, counter: &AtomicU32) -> HorusResult<(usize, u16)> { let now_ms = current_time_ms(); let pid = std::process::id(); let thread_hash = hash_thread_id(std::thread::current().id()) as u32; @@ -804,7 +874,26 @@ impl TopicHeader { .store(now_ms, Ordering::Release); } p.refresh_lease(now_ms, timeout_ms); - return Ok(i); + // Count this handle against the role it is registering — + // unconditionally. + // + // `ensure_role` returns early when the CALLING HANDLE already + // holds the role, so reaching here means this handle is newly + // taking it. The entry's role bits say nothing about that: they + // are shared by every handle on this (pid, thread), so a + // sibling may have set the bit already. Gating on + // `old_role & role_bit == 0` therefore skipped the second + // handle to take a role, and the first `Drop` then handed back + // a role another live handle was still using — the under-count + // direction, where `nothing_is_draining` retires slots beneath + // a subscriber that is still reading. + let counter = if role_bit & 1 != 0 { + &p.pub_handles + } else { + &p.sub_handles + }; + counter.fetch_add(1, Ordering::AcqRel); + return Ok((i, p.generation.load(Ordering::Acquire))); } } @@ -828,10 +917,10 @@ impl TopicHeader { // Pass 1: genuinely free slots only. No counter decrement: nothing was // registered here. - if let Some(i) = + if let Some(claim) = self.claim_free_slot(role_bit, counter, pid, thread_hash, now_ms, timeout_ms) { - return Ok(i); + return Ok(claim); } // Pass 2: nothing free — retire participants whose process is actually @@ -845,10 +934,10 @@ impl TopicHeader { // be a new failure introduced by an optimisation. Registration is a // startup-shaped event, so the probes are affordable exactly here. let _ = self.reap_dead_participants_now(now_ms); - if let Some(i) = + if let Some(claim) = self.claim_free_slot(role_bit, counter, pid, thread_hash, now_ms, timeout_ms) { - return Ok(i); + return Ok(claim); } // Pass 3, last resort: a stale entry belonging to a thread of THIS @@ -912,8 +1001,9 @@ impl TopicHeader { decrement_to_floor(&self.subscriber_count); } - self.finish_claim(p, role_bit, counter, pid, thread_hash, now_ms, timeout_ms); - return Ok(i); + let generation = + self.finish_claim(p, role_bit, counter, pid, thread_hash, now_ms, timeout_ms); + return Ok((i, generation)); } // All 16 participant slots occupied by live processes. @@ -925,15 +1015,96 @@ impl TopicHeader { .count(); Err(HorusError::Communication( format!( - "No available participant slots ({}/{} active). \ - Consider reducing the number of concurrent publishers/subscribers \ - on this topic, or increasing lease timeout to allow faster reclamation.", - active, MAX_PARTICIPANTS + "No available participant slots ({}/{} active). Drop `Topic` \ + handles this process no longer uses — a registration is returned \ + when its last handle drops — or reduce the number of concurrent \ + publishers/subscribers on this topic. A slot held by a process \ + that has died is reclaimed once its {} ms lease expires.", + active, + MAX_PARTICIPANTS, + self.lease_timeout() ) .into(), )) } + /// Give back a registration this handle owns. + /// + /// Called from `Drop for RingTopic`. Until it existed, nothing inside a + /// LIVING process ever returned a registration: the only decrements were in + /// `reap_dead_participants_now`, which wants an expired lease AND a dead + /// pid, and in `register_role`'s last-resort pass, which only runs once all + /// 16 slots are occupied. So the counts only ever grew — and + /// `detect_optimal_backend` reads them, so a transient subscriber on a + /// fresh thread (a worker pool, a scoped task, a one-shot diagnostic read) + /// permanently converted a command topic from backpressured `SpscShm` to + /// overwrite-oldest `PodShm`, with nothing alive to justify it and no path + /// back for the life of the segment. + /// + /// Releasing is only safe with unambiguous ownership, which is why this + /// takes a generation as well as a slot. A slot is NOT owned for a handle's + /// lifetime: `register_role`'s pass 3 reclaims slots whose thread has + /// ended, so an index alone can name somebody else's registration by the + /// time it is given back. Under-counting is the dangerous direction — + /// `nothing_is_draining` lets a producer retire unread slots once + /// `sub_count()` reads 0, so a lost registration becomes silent message loss + /// on a subscriber that is still reading, which is strictly worse than the + /// over-count being fixed. Every check below refuses rather than guesses. + pub(crate) fn release_participant( + &self, + slot: usize, + generation: u16, + held_producer: bool, + held_consumer: bool, + ) { + let Some(p) = self.participants.get(slot) else { + return; + }; + // Someone else's registration now, or already gone. + if p.active.load(Ordering::Acquire) != 1 + || p.generation.load(Ordering::Acquire) != generation + || p.pid.load(Ordering::Acquire) != std::process::id() + { + return; + } + + // Give back each role this handle actually held, and only when it was + // the last handle on this thread holding it. `checked_sub` refuses on a + // count already at zero — a double release — rather than wrapping and + // decrementing a shared counter twice. + let mut changed = false; + if held_producer + && p.pub_handles + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |n| n.checked_sub(1)) + == Ok(1) + { + p.role.fetch_and(!1, Ordering::AcqRel); + decrement_to_floor(&self.publisher_count); + changed = true; + } + if held_consumer + && p.sub_handles + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |n| n.checked_sub(1)) + == Ok(1) + { + p.role.fetch_and(!2, Ordering::AcqRel); + decrement_to_floor(&self.subscriber_count); + changed = true; + } + if !changed { + return; + } + self.last_topology_change_ms + .store(current_time_ms(), Ordering::Release); + + // The slot itself is only free once no handle holds either role on it. + if p.pub_handles.load(Ordering::Acquire) == 0 && p.sub_handles.load(Ordering::Acquire) == 0 + { + decrement_to_floor(&self.total_participants); + p.clear(); + } + } + /// Drop the registrations of participants whose process no longer exists. /// /// `subscriber_count` and `publisher_count` had exactly one decrement in @@ -1237,6 +1408,34 @@ impl TopicHeader { /// is purely a function of topology (producer/consumer counts) and whether /// the message type is POD. pub fn detect_optimal_backend(&self) -> BackendMode { + let desired = self.topology_backend(); + // Never NARROW a topic that is already running wider. + // + // Until participant registrations were given back on `Drop`, the counts + // could only grow, so this could not happen and nothing guarded it. + // Now that a departing subscriber really does decrement `sub_count`, a + // two-subscriber broadcast topic whose second subscriber exits would + // otherwise migrate back to `SpscShm` — and a backwards migration in + // the middle of a stream loses the messages already in the wider ring, + // because the migrator's drain does not actually drain: it has nothing + // to observe and always succeeds. Observed directly while building + // this: with the downgrade allowed, the SURVIVING subscriber of two + // received 198 of 200 messages and ended up back on `SpscShm` + // mid-stream. + // + // Widening on a join is safe and stays immediate. Narrowing has to wait + // for a drain that is real; until then the safe answer is the wider + // path, which is also the one the runtime took before the counts could + // fall at all. + let current = self.mode(); + if backend_width(current) > backend_width(desired) { + return current; + } + desired + } + + /// The backend this topology alone would choose, ignoring what is running. + fn topology_backend(&self) -> BackendMode { let pubs = self.pub_count(); let subs = self.sub_count(); let is_pod = self.is_pod_type(); @@ -1265,6 +1464,21 @@ impl TopicHeader { } } +/// How wide a backend is, for the "never narrow a running topic" rule in +/// [`TopicHeader::detect_optimal_backend`]. +/// +/// Only the ordering matters, not the numbers: a broadcast backend carries +/// every message to every subscriber, a multi-producer ring admits several +/// writers, and `SpscShm` is the narrowest real path. +fn backend_width(mode: BackendMode) -> u8 { + match mode { + BackendMode::Unknown => 0, + BackendMode::SpscShm => 1, + BackendMode::MpscShm | BackendMode::SpmcShm => 2, + BackendMode::PodShm | BackendMode::FanoutShm => 3, + } +} + // ============================================================================ // Public debug flag API (for external tools like the TUI monitor) // ============================================================================ @@ -2034,6 +2248,24 @@ pub(crate) fn type_name_as_stored(name: &str) -> &str { &name[..end] } +/// The message type an existing topic's ring already holds, by topic NAME. +/// +/// Reads only the header — it never constructs a `Topic`, which is the point. +/// Asking "what type is this ring?" by opening it is circular: opening requires +/// choosing a type, and choosing the wrong one is refused by the size check in +/// `RingTopic::new` (correctly — a name-based exemption once let a +/// `Topic` join a `Topic` ring and write megabytes past +/// the end of the mapping). So a caller that wants to pick the type from the +/// ring has to read the header without attaching, and this is that. +/// +/// `None` if the topic does not exist, the name is not a legal region name, the +/// region has no valid header, or the header records no type. +pub fn peek_topic_type_name(name: &str) -> Option { + let path = horus_sys::shm::topic_shm_path_checked(name)?; + let info = read_topic_header_info(&path)?; + (!info.type_name.is_empty()).then_some(info.type_name) +} + pub fn read_topic_header_info(path: &std::path::Path) -> Option { let mmap = map_topic_region(path)?; let base = mmap.as_ptr(); @@ -2536,7 +2768,10 @@ mod tests { role: AtomicU8::new(0), active: AtomicU8::new(0), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(0), }; assert!(p.is_lease_expired(1000)); @@ -2550,7 +2785,10 @@ mod tests { role: AtomicU8::new(0), active: AtomicU8::new(0), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(5000), }; assert!(!p.is_lease_expired(4999)); @@ -2564,7 +2802,10 @@ mod tests { role: AtomicU8::new(0), active: AtomicU8::new(0), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(5000), }; assert!(p.is_lease_expired(5001)); @@ -2578,7 +2819,10 @@ mod tests { role: AtomicU8::new(0), active: AtomicU8::new(0), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(5000), }; // now_ms == expires → not expired (> check, not >=) @@ -2593,7 +2837,10 @@ mod tests { role: AtomicU8::new(0), active: AtomicU8::new(0), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(0), }; p.refresh_lease(1000, 5000); @@ -2610,7 +2857,10 @@ mod tests { role: AtomicU8::new(3), active: AtomicU8::new(1), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(99999), }; p.clear(); @@ -2623,10 +2873,70 @@ mod tests { // ── Registration ──────────────────────────────────────────────────── + /// A topic already running a wider backend must not narrow when a + /// participant leaves. + /// + /// Before registrations were given back on `Drop`, the counts could only + /// grow, so nothing could narrow and nothing guarded it. Now a departing + /// subscriber really does decrement `sub_count`, and a backwards migration + /// mid-stream loses whatever is already in the wider ring — the migrator's + /// drain has nothing to observe and always succeeds. Measured while + /// building this: with the downgrade allowed, the surviving subscriber of + /// two received 198 of 200 messages and ended up back on `SpscShm`. + #[test] + fn a_running_topic_never_narrows_when_a_participant_leaves() { + let h = make_header(8, 8, true, 16); + h.publisher_count.store(1, Ordering::Release); + h.subscriber_count.store(2, Ordering::Release); + + let wide = h.detect_optimal_backend(); + assert_eq!( + wide, + BackendMode::PodShm, + "two subscribers on a POD topic is a broadcast topology" + ); + // Publish it, the way a migration would. + h.backend_mode.store(wide as u8, Ordering::Release); + + // One subscriber leaves. The topology alone would now say SpscShm. + h.subscriber_count.store(1, Ordering::Release); + assert_eq!( + h.topology_backend(), + BackendMode::SpscShm, + "precondition: the topology really has narrowed, so this test is \ + not asserting a no-op" + ); + assert_eq!( + h.detect_optimal_backend(), + BackendMode::PodShm, + "the running topic must stay on the wider backend — narrowing \ + mid-stream drops the messages already in the wider ring for the \ + subscriber that is still reading" + ); + } + + /// Widening on a join stays immediate — the rule is one-directional. + #[test] + fn a_running_topic_still_widens_when_a_participant_joins() { + let h = make_header(8, 8, true, 16); + h.publisher_count.store(1, Ordering::Release); + h.subscriber_count.store(1, Ordering::Release); + let narrow = h.detect_optimal_backend(); + assert_eq!(narrow, BackendMode::SpscShm); + h.backend_mode.store(narrow as u8, Ordering::Release); + + h.subscriber_count.store(2, Ordering::Release); + assert_eq!( + h.detect_optimal_backend(), + BackendMode::PodShm, + "a second subscriber must widen the path immediately" + ); + } + #[test] fn register_producer_increments_pub_count() { let h = make_header(8, 8, true, 16); - let slot = h.register_producer().unwrap(); + let slot = h.register_producer().unwrap().0; assert!(slot < MAX_PARTICIPANTS); assert_eq!(h.pub_count(), 1); assert_eq!(h.sub_count(), 0); @@ -2636,7 +2946,7 @@ mod tests { #[test] fn register_consumer_increments_sub_count() { let h = make_header(8, 8, true, 16); - let slot = h.register_consumer().unwrap(); + let slot = h.register_consumer().unwrap().0; assert!(slot < MAX_PARTICIPANTS); assert_eq!(h.sub_count(), 1); assert_eq!(h.pub_count(), 0); @@ -2934,12 +3244,15 @@ mod tests { } assert_eq!(h.sub_count(), MAX_PARTICIPANTS as u32); - let slot = h.register_producer().expect( - "registration must sweep unconditionally: every slot is held by a \ + let slot = h + .register_producer() + .expect( + "registration must sweep unconditionally: every slot is held by a \ process that no longer exists, and refusing the node's start \ because another process swept the table moments ago would be a \ failure the throttle invented", - ); + ) + .0; assert!(slot < MAX_PARTICIPANTS); assert_eq!( h.pub_count(), @@ -3077,8 +3390,8 @@ mod tests { #[test] fn register_same_thread_reuses_slot() { let h = make_header(8, 8, true, 16); - let slot1 = h.register_producer().unwrap(); - let slot2 = h.register_consumer().unwrap(); + let slot1 = h.register_producer().unwrap().0; + let slot2 = h.register_consumer().unwrap().0; assert_eq!( slot1, slot2, "Same thread should reuse same participant slot" @@ -3093,8 +3406,8 @@ mod tests { #[test] fn register_producer_twice_same_thread_no_double_count() { let h = make_header(8, 8, true, 16); - let s1 = h.register_producer().unwrap(); - let s2 = h.register_producer().unwrap(); + let s1 = h.register_producer().unwrap().0; + let s2 = h.register_producer().unwrap().0; assert_eq!(s1, s2); assert_eq!( h.pub_count(), @@ -3106,14 +3419,14 @@ mod tests { #[test] fn register_from_different_threads_uses_different_slots() { let h = make_header(8, 8, true, 16); - let slot1 = h.register_producer().unwrap(); + let slot1 = h.register_producer().unwrap().0; let header_ptr = &h as *const TopicHeader as usize; let slot2 = std::thread::spawn(move || { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives this thread (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_producer().unwrap() + h.register_producer().unwrap().0 }) .join() .unwrap(); @@ -3135,7 +3448,7 @@ mod tests { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives all spawned threads (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_producer().unwrap() + h.register_producer().unwrap().0 }) }) .collect(); @@ -3163,7 +3476,7 @@ mod tests { // SAFETY: header_ptr was derived from a valid stack-allocated TopicHeader that // outlives all spawned threads (main thread joins before h is dropped). let h = unsafe { &*(header_ptr as *const TopicHeader) }; - h.register_producer().unwrap() + h.register_producer().unwrap().0 }) }) .collect(); @@ -3215,7 +3528,7 @@ mod tests { // is no doubt it is still there. let header = &h; s.spawn(move || { - let slot = header.register_consumer().expect("subscriber registers"); + let slot = header.register_consumer().expect("subscriber registers").0; let hash = hash_thread_id(std::thread::current().id()) as u32; registered_tx.send((slot, hash)).unwrap(); release_rx.recv().ok(); @@ -3668,7 +3981,7 @@ mod tests { // subscriber. Registration now takes a genuinely FREE slot first and // leaves the expired entry alone; an expired slot is considered only // when no free one remains, and then only if its owner is really gone. - let slot = h.register_producer().unwrap(); + let slot = h.register_producer().unwrap().0; assert_ne!( slot, 0, "must not evict an expired slot while others are free" @@ -3709,7 +4022,10 @@ mod tests { role: AtomicU8::new(0), active: AtomicU8::new(0), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(0), }; // Realistic large timestamp (year ~2100 in ms) + large timeout diff --git a/horus_core/src/communication/topic/local_state.rs b/horus_core/src/communication/topic/local_state.rs index 29275ff98..22b1a8e9c 100644 --- a/horus_core/src/communication/topic/local_state.rs +++ b/horus_core/src/communication/topic/local_state.rs @@ -106,6 +106,11 @@ pub(crate) struct LocalState { /// Our slot index in the participant array (-1 if not registered) pub slot_index: i32, + /// Generation of the participant entry at `slot_index` when this handle + /// claimed it. Together they name a registration unambiguously, so `Drop` + /// can give back its own and never a later one that reused the slot. + pub participant_generation: u16, + /// Slot size for serialized messages (non-POD) pub slot_size: usize, @@ -254,6 +259,7 @@ impl Default for LocalState { cached_seq_ptr: std::ptr::null_mut(), cached_colo_stride: 0, slot_index: -1, + participant_generation: 0, slot_size: DEFAULT_SLOT_SIZE, cached_epoch: 0, fanout_shm_pub_id: None, diff --git a/horus_core/src/communication/topic/metrics.rs b/horus_core/src/communication/topic/metrics.rs index 52c7c2439..4a459da81 100644 --- a/horus_core/src/communication/topic/metrics.rs +++ b/horus_core/src/communication/topic/metrics.rs @@ -5,22 +5,30 @@ use std::sync::atomic::{AtomicU32, AtomicU64}; /// Migration and operational metrics for a Topic #[derive(Debug, Default)] pub(crate) struct MigrationMetrics { - /// Messages sent through this topic + /// Messages sent through this topic, counted on every `send()`. pub messages_sent: AtomicU64, - /// Messages received through this topic + /// Messages delivered by `recv()` on this topic. + /// + /// `try_recv()` is the low-level path and does not count; `recv()` is what + /// `Topic::recv` uses. pub messages_received: AtomicU64, /// Number of send failures pub send_failures: AtomicU64, /// Sends dropped because the retry budget was gone, rather than because /// the ring stayed full for the whole retry. pub send_retry_overruns: AtomicU64, - /// Number of receive failures - pub recv_failures: AtomicU64, /// Number of backend migrations performed pub migrations: AtomicU32, } /// Non-atomic snapshot of topic metrics (for external consumers) +/// +/// `messages_sent` and `messages_received` used to move only on the `#[cold]` +/// verbose-logging path, which runs while the `horus monitor` TUI has set a +/// topic's verbose flag — so in an ordinary run both read 0 no matter how much +/// traffic the topic carried, while `MockTopic` maintained them faithfully. A +/// test double that reports what the real transport does not is the one defect +/// a double cannot have. Both are counted on the ordinary paths now. #[derive(Debug, Clone, Default)] pub struct TopicMetrics { messages_sent: u64, @@ -60,6 +68,13 @@ impl TopicMetrics { self.send_failures } + /// Receive failures. + /// + /// Only [`MockTopic`](crate::testing::MockTopic) produces a non-zero value + /// here, from its `recv_fail_after` / `recv_fail_every_n` fault injection. + /// The real transport never increments it, deliberately: a `recv()` that + /// returns `None` is an empty ring, not a failure. For real consumer-side + /// loss — a publisher lapping a subscriber — use `Topic::missed_count()`. pub fn recv_failures(&self) -> u64 { self.recv_failures } diff --git a/horus_core/src/communication/topic/migration.rs b/horus_core/src/communication/topic/migration.rs index 2043257a8..3e624a650 100644 --- a/horus_core/src/communication/topic/migration.rs +++ b/horus_core/src/communication/topic/migration.rs @@ -1,8 +1,14 @@ #![allow(private_interfaces)] //! Backend migration coordinator. //! -//! Handles safe backend transitions with CAS-based locking, epoch versioning, -//! and drain logic for in-flight messages. +//! Handles backend transitions with CAS-based locking, epoch versioning, and a +//! bounded settle before the switch. +//! +//! NOT a drain. This header said "drain logic for in-flight messages" while the +//! struct doc below already retracted exactly that claim: nothing in +//! `TopicHeader` counts in-flight operations, so there is nothing to drain +//! against, and a message in flight across a switch can be lost. Rustdoc shows +//! this block first, so the retraction 35 lines down was the half nobody read. use std::sync::atomic::Ordering; @@ -24,8 +30,6 @@ pub(crate) enum MigrationResult { AlreadyInProgress, /// Migration lock acquisition failed (contention) LockContention, - /// Migration failed due to error - Failed, } // ============================================================================ @@ -34,10 +38,21 @@ pub(crate) enum MigrationResult { /// Backend migration coordinator. /// -/// Handles safe backend transitions with: +/// Handles backend transitions with: /// - CAS-based locking to prevent concurrent migrations /// - Epoch versioning for reader/writer coordination -/// - Drain logic to ensure no in-flight message loss +/// - A bounded settle before the switch +/// +/// It does NOT guarantee that no message is lost across a switch, and this +/// used to say it did ("Drain logic to ensure no in-flight message loss"). +/// Nothing in `TopicHeader` counts in-flight operations, so there is nothing +/// to drain against — see [`Self::settle_before_switch`]. A real guarantee +/// needs an in-flight counter incremented and decremented on the send and +/// recv hot paths of a real-time framework, which is a deliberate design +/// decision with a measurable cost. Until that is taken, the honest statement +/// is that a topology change switches backends under a lock and a message in +/// flight across the switch can be lost. The topic's own test suite already +/// works around it, pre-warming backends "to avoid migration losses". pub(crate) struct BackendMigrator<'a> { header: &'a TopicHeader, /// Spin count before yielding during drain @@ -90,24 +105,15 @@ impl<'a> BackendMigrator<'a> { /// Perform the actual migration (must hold lock). /// - /// Drains in-flight operations before switching the backend mode. - /// If the first drain attempt times out, retries once before failing. + /// Settles briefly, then switches the backend mode. + /// + /// This used to say it "drains in-flight operations before switching" and + /// "retries once before failing". Both halves guarded a return value that + /// no longer exists: `settle_before_switch` cannot observe an in-flight + /// operation and so cannot report one, which made the retry and the + /// `Failed` result unreachable code describing a guarantee nothing made. fn perform_migration(&self, new_mode: BackendMode) -> MigrationResult { - if !self.drain_in_flight() { - tracing::warn!( - "migration drain timed out on first attempt, retrying (target={:?})", - new_mode - ); - // One retry: in-flight ops are <200ns, so a second attempt should succeed - // unless there is a genuine stall (e.g., thread suspended by debugger). - if !self.drain_in_flight() { - tracing::warn!( - "migration drain timed out on retry, aborting migration (target={:?})", - new_mode - ); - return MigrationResult::Failed; - } - } + self.settle_before_switch(); // IMPORTANT: Store backend_mode BEFORE incrementing epoch. // The epoch increment is the "publication fence" — observers that @@ -133,13 +139,14 @@ impl<'a> BackendMigrator<'a> { MigrationResult::Success { new_epoch } } - /// Grace period for in-flight operations to complete before migration. + /// Bounded settle before the backend mode is switched. /// - /// Since send/recv are lock-free and complete in <200ns, a short spin - /// followed by yields gives in-progress operations time to finish. - /// Returns `false` if the drain timeout is exceeded, indicating that - /// migration should be retried rather than proceeding unsafely. - fn drain_in_flight(&self) -> bool { + /// NOT a drain, despite what this was called. Send and recv are lock-free + /// and complete in under 200 ns, so a bounded spin plus a few yields gives + /// an in-progress operation a good chance to finish — but "a good chance" + /// is the whole of it. Nothing here can confirm one finished, and nothing + /// here can fail. + fn settle_before_switch(&self) { // Give any in-flight send/recv time to finish. They complete in under // 200 ns, so a bounded spin plus a few yields is the whole mechanism. // @@ -190,7 +197,7 @@ impl<'a> BackendMigrator<'a> { std::sync::atomic::fence(Ordering::SeqCst); std::hint::spin_loop(); if start.elapsed() > budget { - return true; + return; } } // Yield to let preempted producer threads complete. @@ -202,11 +209,10 @@ impl<'a> BackendMigrator<'a> { // with a deadline. Leaving one of the two in the broken shape // guarantees the next person copies the wrong one. if start.elapsed() > budget { - return true; + return; } std::thread::yield_now(); } - true } /// Force a migration to the detected optimal backend. @@ -226,6 +232,49 @@ impl<'a> BackendMigrator<'a> { #[cfg(test)] mod tests { use super::*; + + /// The settle is bounded, and that bound is the only thing it guarantees. + /// + /// It cannot observe an in-flight operation — nothing in `TopicHeader` + /// counts them — so it can neither confirm a drain nor report a failure. + /// What it can do is not spin forever, and that part matters: removing the + /// budget made `send_blocking` block measurably longer under load (a test + /// asserting an immediate return went from failing 2 runs in 8 to 7 in 8). + #[test] + fn the_settle_is_bounded() { + let header = make_header(true); + let migrator = BackendMigrator::new(&header); + + let started = std::time::Instant::now(); + migrator.settle_before_switch(); + let elapsed = started.elapsed(); + + assert!( + elapsed < std::time::Duration::from_millis(250), + "the settle must stay bounded; it took {elapsed:?}" + ); + } + + /// A migration cannot report failure, so nothing may branch on one. + /// + /// `perform_migration` used to retry a drain and return + /// `MigrationResult::Failed` when it "timed out" — guarding a return value + /// `drain_in_flight` had stopped producing, so both the retry and the + /// result were unreachable code standing for a guarantee nothing made. + /// Callers wrote handlers for a case that could not occur. + #[test] + fn an_uncontended_migration_always_completes() { + let header = make_header(true); + header.publisher_count.store(1, Ordering::Release); + header.subscriber_count.store(2, Ordering::Release); + let migrator = BackendMigrator::new(&header); + + // Single-threaded, so there is nothing to contend with. + match migrator.migrate_to_optimal() { + MigrationResult::Success { .. } | MigrationResult::NotNeeded => {} + other => panic!("an uncontended migration reported {other:?}"), + } + } use std::sync::atomic::Ordering; fn make_header(is_pod: bool) -> TopicHeader { @@ -243,7 +292,6 @@ mod tests { MigrationResult::NotNeeded, MigrationResult::AlreadyInProgress, MigrationResult::LockContention, - MigrationResult::Failed, ]; for (i, a) in results.iter().enumerate() { for (j, b) in results.iter().enumerate() { diff --git a/horus_core/src/communication/topic/mod.rs b/horus_core/src/communication/topic/mod.rs index dc2516369..14d2cb0f1 100644 --- a/horus_core/src/communication/topic/mod.rs +++ b/horus_core/src/communication/topic/mod.rs @@ -561,9 +561,9 @@ pub(crate) use header::{TOPIC_MAGIC, TOPIC_VERSION}; // Public debug flag API for external tools (TUI monitor) #[doc(hidden)] pub use header::{ - read_latest_slot_bytes, read_slots_since, read_topic_header_info, read_topic_messages_total, - read_topic_sequence, set_topic_verbose, shm_map_count, TopicHeaderInfo, TopicKind, TopicReader, - TopicSlotRead, TOPIC_VERBOSE_OFFSET, + peek_topic_type_name, read_latest_slot_bytes, read_slots_since, read_topic_header_info, + read_topic_messages_total, read_topic_sequence, set_topic_verbose, shm_map_count, + TopicHeaderInfo, TopicKind, TopicReader, TopicSlotRead, TOPIC_VERBOSE_OFFSET, }; use local_state::LocalState; pub(crate) use metrics::MigrationMetrics; @@ -611,6 +611,31 @@ pub enum SendBlockingError { NoBackpressure, } +/// Whether the backend named by [`Topic::backend_name`] refuses a send when its +/// ring is full. +/// +/// Keyed by NAME rather than by the private `BackendMode`, because the bindings +/// are where this question actually gets asked and a name is all they can see: +/// `backend_type` is a string over both the Python and C++ FFI. Answering it +/// there by re-deriving the mapping is how a binding drifts from the runtime, so +/// `provides_backpressure()` is defined in terms of this too and the two cannot +/// disagree. +/// +/// - `SpscShm`, `MpscShm`, `SpmcShm` — ring-full is a real `Err`: the send paths +/// compare the claimed sequence against `header.tail` and refuse. +/// - `PodShm`, `FanoutShm` — broadcast, overwrite-oldest by construction, so +/// there is no full condition to report. +/// - Anything else, including `Unknown`, fails closed. `Unknown` is the state +/// EVERY topic is in until its first send or recv — including at the "assert +/// this at startup" moment the doc comment above recommends. Answering `true` +/// there made the assertion pass on a topic that was about to resolve to +/// `PodShm` and silently drop, so the check a caller writes to catch a lossy +/// e-stop channel could not catch one. An unresolved backend is not a promise +/// of backpressure. +pub fn backend_provides_backpressure(backend_name: &str) -> bool { + matches!(backend_name, "SpscShm" | "MpscShm" | "SpmcShm") +} + impl From for crate::error::HorusError { fn from(err: SendBlockingError) -> Self { crate::error::HorusError::Communication(crate::error::CommunicationError::TopicFull { @@ -1474,6 +1499,17 @@ impl RingTopic< unsafe { &*(self.storage.as_ptr() as *const TopicHeader) } } + /// Bump the "new data exists" counter the staleness watchdog reads. + /// + /// `RingTopic::send` does this inline. It lives here as well so the public + /// `Topic::try_send` / `Topic::send_blocking` can do it exactly once per + /// call: it must NOT move into `RingTopic::try_send`, which + /// `send_lossy_retry` calls up to ~70 times for a single message. + #[inline(always)] + fn bump_messages_total(&self) { + self.header().messages_total.fetch_add(1, Ordering::Relaxed); + } + /// Check if verbose content logging is enabled via the SHM header flag. /// Uses the stable `header_ptr` (not `LocalState::cached_header_ptr`, which /// is repurposed by the role=Both same-instance fast path). @@ -1712,13 +1748,17 @@ impl RingTopic< } let header = self.header(); - let slot = if is_producer { + // Reaching here means this handle does NOT yet hold the role — the + // early return above covers the case where it does — so the entry's + // per-role handle count is incremented for it unconditionally. + let (slot, generation) = if is_producer { header.register_producer()? } else { header.register_consumer()? }; local.slot_index = slot as i32; + local.participant_generation = generation; local.cached_header_ptr = self.storage.as_ptr() as *const TopicHeader; local.cached_epoch = header.migration_epoch.load(Ordering::Acquire); // Sync BEFORE deriving the data pointer, and derive it from the synced @@ -1913,10 +1953,6 @@ impl RingTopic< local.cached_mode = header.mode(); break; } - MigrationResult::Failed => { - local.cached_mode = header.mode(); - break; - } } } local.cached_mode = header.mode(); @@ -2757,7 +2793,10 @@ impl RingTopic< self.metrics.messages_sent.load(Ordering::Relaxed), self.metrics.messages_received.load(Ordering::Relaxed), self.metrics.send_failures.load(Ordering::Relaxed), - self.metrics.recv_failures.load(Ordering::Relaxed), + // The real transport has no recv-failure notion: a `recv()` that + // returns `None` is an empty ring. Only `MockTopic` reports one, + // from its fault injection. See `TopicMetrics::recv_failures`. + 0, self.metrics.send_retry_overruns.load(Ordering::Relaxed), ) } @@ -2832,6 +2871,19 @@ impl RingTopic< // ~28ns of ~95ns, and it should be done on purpose rather than by // flipping this line. self.header().messages_total.fetch_add(1, Ordering::Relaxed); + // The per-handle counter behind `TopicMetrics::messages_sent`, which was + // incremented ONLY inside `send_with_content_logging` — a `#[cold]` path + // reached only while the `horus monitor` TUI has set the topic's verbose + // flag. So the public accessor read 0 in every normal run, however much + // traffic the topic carried, while `MockTopic` (the stand-in users write + // their tests against) maintained it faithfully. A test double that + // reports what the real thing does not is the one defect a double + // cannot have. + // + // This is a process-local, uncontended relaxed increment on a counter + // only this handle writes — not the contended shared-memory RMW the + // comment above prices at ~28 ns. + self.metrics.messages_sent.fetch_add(1, Ordering::Relaxed); if unlikely(self.is_verbose()) { self.send_with_content_logging(msg); // Notify event nodes watching this topic. Gated inside @@ -2977,7 +3029,9 @@ impl RingTopic< self.send_lossy(msg); let ipc_ns = start.elapsed().as_nanos() as u64; - self.metrics.messages_sent.fetch_add(1, Ordering::Relaxed); + // NOT counted here: `send()` counts every send before dispatching to + // this cold path, so incrementing again would double-count while the + // monitor's verbose flag is set. self.state .store(ConnectionState::Connected.into_u8(), Ordering::Relaxed); use crate::core::hlog::{current_node_name, current_tick_number}; @@ -3379,6 +3433,21 @@ impl RingTopic< if unlikely(self.is_verbose()) { return self.recv_with_content_logging(); } + let received = self.recv_uncounted(); + if received.is_some() { + // Counts DELIVERED messages, the mirror of `messages_sent`. As + // there, this used to move only on the `#[cold]` verbose path, so + // the public accessor read 0 in every normal run. + self.metrics + .messages_received + .fetch_add(1, Ordering::Relaxed); + } + received + } + + /// `recv()` without the metrics bookkeeping. + #[inline(always)] + fn recv_uncounted(&self) -> Option { // Fast path: role=Both (same-instance, same-thread pub+sub) and POD. // // Gated on `is_pod` in lockstep with `send`: this reads back through @@ -3510,8 +3579,22 @@ impl RingTopic< T: Copy, { // Ensure we're registered as a consumer so the header is initialized - if self.local().role == TopicRole::Unregistered && self.ensure_consumer().is_err() { - return None; + if self.local().role == TopicRole::Unregistered { + if let Err(e) = self.ensure_consumer() { + // See `dispatch::recv_uninitialized`: silence here is a + // subscriber that reads `None` forever and looks idle. + if dispatch::should_report_endpoint_exhaustion( + self.name(), + dispatch::Exhausted::ParticipantTable, + ) { + tracing::warn!( + "topic '{}': this reader is NOT registered and will see nothing — {}", + self.name(), + e + ); + } + return None; + } } // Role==Both AND POD uses the local fast path (LocalState head/tail @@ -3666,22 +3749,7 @@ impl RingTopic< /// ); /// ``` pub fn provides_backpressure(&self) -> bool { - match self.mode() { - // Ring-full is a real Err on these: the mp/sp send paths compare the - // claimed sequence against `header.tail` and refuse. - BackendMode::SpscShm | BackendMode::MpscShm | BackendMode::SpmcShm => true, - // Broadcast: overwrite-oldest by construction, no full condition. - BackendMode::PodShm | BackendMode::FanoutShm => false, - // Fail closed. `Unknown` means the backend has not been resolved - // yet, which is the state EVERY topic is in until its first - // send/recv — including at the "assert this at startup" moment the - // doc comment above recommends. Answering `true` there made the - // assertion pass on a topic that was about to resolve to PodShm and - // silently drop, so the check a caller writes to catch a lossy - // e-stop channel could not catch one. An unresolved backend is not - // a promise of backpressure. - BackendMode::Unknown => false, - } + backend_provides_backpressure(self.backend_name()) } #[cfg(test)] @@ -3785,6 +3853,34 @@ impl Drop for RingTopic { local.fanout_sub_lock.take(); } + // Give the participant registration back. + // + // Nothing inside a living process used to do this, so the counts only + // ever grew — and `detect_optimal_backend` reads them, so one transient + // subscriber on a fresh thread permanently converted a backpressured + // command topic to overwrite-oldest, for the life of the segment. + // + // `release_participant` refuses unless the (slot, generation) still + // names THIS registration and this is the last handle sharing it. It + // has to: a slot is not owned for a handle's lifetime, and + // under-counting is the dangerous direction. + if local.slot_index >= 0 { + // `Drop` carries no bounds on `T`, so `header()` (which does) is + // not callable here. Same mapping, same invariant: storage was + // sized and aligned for a `TopicHeader` in the constructor. + let header = unsafe { &*(self.storage.as_ptr() as *const header::TopicHeader) }; + // Give back only the roles this handle actually registered — one + // thread can hold a publisher and a subscriber handle on the same + // topic (that is the role=Both fast path), and they share an entry. + header.release_participant( + local.slot_index as usize, + local.participant_generation, + local.role.can_send(), + local.role.can_recv(), + ); + local.slot_index = -1; + } + // Notify network layer (horus_net) that this topic instance is dropped notify_topic_lifecycle(TopicLifecycleEvent::Dropped { name: self.name.clone(), @@ -4552,6 +4648,35 @@ where // A backpressure-aware publisher is still a publisher; this had no // registration block, so `try_send`-only nodes were unattributed. self.register_pub_lazy(); + // `messages_total` is not a metric — `SubscriptionFreshness` watches it + // as the "new data exists" signal behind `.subscribe_with_timeout()`, + // and that drives `StalePolicy::SafeState` and `Stop`. Only `send()` + // used to bump it, so a topic driven entirely by `try_send` read as 0 + // messages and 0 Hz while carrying full traffic, and could safe-state + // or halt a subscriber node whose data was arriving normally. That is + // the exact failure the counter was introduced to prevent, and these + // are the APIs the docs send users to on critical topics. + // + // Counted before the call, like `send()`: this is an attempt count, so + // a publisher hammering a full ring keeps it moving. + // + // KNOWN LIMITATION, and the reason this note is not a reassurance. An + // earlier version of it claimed a ring nobody drains is "a different + // fault, and one the subscriber's own staleness sees". That is + // circular: `SubscriptionFreshness::refresh` derives freshness from + // this counter and from nothing else, so while a full ring drops every + // message the subscriber still observes the count advancing and stays + // "fresh" — the watchdog cannot see the fault it is watching for. + // + // Bumping on SUCCESS instead would close it, and is the right shape: + // `try_send` failing means nothing was published, which is exactly what + // staleness should report. It is not done here because it moves the + // meaning of the counter at seven call sites across two crates and + // changes when `StalePolicy::SafeState`/`Stop` fire on a live robot — + // that belongs in its own change with its own tests, not in a merge. + // Until then, a stalled consumer is visible in `dropped_count()` / + // `send_retry_overruns()`, not in staleness. + self.ring.bump_messages_total(); self.ring.try_send(msg) } @@ -4614,7 +4739,17 @@ where timeout: std::time::Duration, ) -> Result<(), SendBlockingError> { self.register_pub_lazy(); - self.ring.send_blocking(msg, timeout) + let result = self.ring.send_blocking(msg, timeout); + // Same "new data exists" signal as `try_send` above — but NOT for + // `NoBackpressure`. That refusal is returned before the send is even + // attempted, so nothing reached the ring; counting it would report a + // healthy flow on a topic delivering nothing, which is worse than the + // silence it replaced. A `Timeout` did engage the ring, so it counts as + // an attempt like `send()`'s dropped message does. + if !matches!(result, Err(SendBlockingError::NoBackpressure)) { + self.ring.bump_messages_total(); + } + result } /// Low-level receive without logging/recording hooks. @@ -4694,6 +4829,13 @@ impl Topic { /// Try to send an image without blocking. Returns `Err(img)` if the ring is full. pub fn try_send(&self, img: Image) -> Result<(), Image> { self.register_pub("Image"); + // Same "new data exists" signal the typed `try_send` counts. Pool-backed + // types reach `ring.try_send` directly rather than through it, so they + // were left out of that fix — and these are the high-bandwidth topics + // (camera, lidar, depth) where a staleness watchdog matters most: + // `SubscriptionFreshness` would read 0 Hz and safe-state or halt a + // subscriber whose frames were arriving normally. + self.ring.bump_messages_total(); let wire = img.to_wire(&self.pool); match self.ring.try_send(wire) { Ok(()) => { @@ -4753,6 +4895,13 @@ impl Topic { /// Try to send a point cloud without blocking. Returns `Err(pc)` if the ring is full. pub fn try_send(&self, pc: PointCloud) -> Result<(), PointCloud> { self.register_pub("PointCloud"); + // Same "new data exists" signal the typed `try_send` counts. Pool-backed + // types reach `ring.try_send` directly rather than through it, so they + // were left out of that fix — and these are the high-bandwidth topics + // (camera, lidar, depth) where a staleness watchdog matters most: + // `SubscriptionFreshness` would read 0 Hz and safe-state or halt a + // subscriber whose frames were arriving normally. + self.ring.bump_messages_total(); let wire = pc.to_wire(&self.pool); match self.ring.try_send(wire) { Ok(()) => { @@ -4812,6 +4961,13 @@ impl Topic { /// Try to send a depth image without blocking. Returns `Err(depth)` if the ring is full. pub fn try_send(&self, depth: DepthImage) -> Result<(), DepthImage> { self.register_pub("DepthImage"); + // Same "new data exists" signal the typed `try_send` counts. Pool-backed + // types reach `ring.try_send` directly rather than through it, so they + // were left out of that fix — and these are the high-bandwidth topics + // (camera, lidar, depth) where a staleness watchdog matters most: + // `SubscriptionFreshness` would read 0 Hz and safe-state or halt a + // subscriber whose frames were arriving normally. + self.ring.bump_messages_total(); let wire = depth.to_wire(&self.pool); match self.ring.try_send(wire) { Ok(()) => { @@ -5262,3 +5418,178 @@ mod untrusted_ring_geometry_tests { cleanup(&name); } } + +#[cfg(test)] +mod staleness_signal_tests { + use super::{SendBlockingError, Topic}; + use std::time::Duration; + + /// `backend_provides_backpressure` must answer for every backend there is, + /// and answer `false` for anything it does not recognise. + /// + /// The Python and C++ bindings only ever see the backend NAME, so this + /// string table is what they ask. A backend added without a row here would + /// silently report "no backpressure" — which fails closed, but would make + /// `send_blocking` refuse on a backend that actually supports it. The match + /// below is exhaustive over `BackendMode`, so adding a variant stops + /// compiling here rather than going unnoticed. + #[test] + fn every_backend_name_has_a_backpressure_answer() { + use super::backend_provides_backpressure; + use super::types::BackendMode; + + let expected = [ + (BackendMode::Unknown, false), + (BackendMode::PodShm, false), + (BackendMode::FanoutShm, false), + (BackendMode::SpscShm, true), + (BackendMode::SpmcShm, true), + (BackendMode::MpscShm, true), + ]; + + for (mode, provides) in expected { + // Mirrors `RingTopic::backend_name`; exhaustive on purpose. + let name = match mode { + BackendMode::Unknown => "Unknown", + BackendMode::PodShm => "PodShm", + BackendMode::SpscShm => "SpscShm", + BackendMode::SpmcShm => "SpmcShm", + BackendMode::MpscShm => "MpscShm", + BackendMode::FanoutShm => "FanoutShm", + }; + assert_eq!( + backend_provides_backpressure(name), + provides, + "{name} is on the wrong side of the backpressure table" + ); + } + + assert!( + !backend_provides_backpressure("SomethingNewAndUnknown"), + "an unrecognised backend must fail closed — an unresolved backend \ + is not a promise of backpressure" + ); + } + + fn cleanup(name: &str) { + if let Some(path) = horus_sys::shm::topic_shm_path_checked(name) { + let _ = std::fs::remove_file(path); + } + } + + /// `messages_total` is the staleness watchdog's "new data exists" signal. + /// + /// `SubscriptionFreshness::refresh` watches it behind + /// `.subscribe_with_timeout()`, and its answer drives + /// `StalePolicy::SafeState` and `StalePolicy::Stop`. Only `RingTopic::send` + /// bumped it, so a topic published entirely through `try_send` read as 0 + /// messages and 0 Hz while carrying every message — and could safe-state or + /// halt a subscriber whose data was arriving normally. That is precisely + /// the failure the counter exists to prevent, and `try_send` / + /// `send_blocking` are the two APIs the docs point users to on critical + /// topics. + #[test] + fn try_send_moves_the_staleness_counter() { + let name = format!("staleness_try_send_{}", std::process::id()); + cleanup(&name); + let Ok(topic) = Topic::::new(&name) else { + eprintln!("skipping: no shared memory available"); + return; + }; + + assert_eq!(topic.ring.header().messages_total(), 0); + for i in 0..10u64 { + let _ = topic.try_send(i); + } + + assert_eq!( + topic.ring.header().messages_total(), + 10, + "ten try_send calls delivered ten messages; a watchdog reading this \ + counter must not conclude the topic is dead" + ); + + drop(topic); + cleanup(&name); + } + + /// Pool-backed types count too. + /// + /// `Image`, `PointCloud` and `DepthImage` have their own `try_send`, which + /// reaches `ring.try_send` directly rather than through the typed one — so + /// they were left out when the staleness counter was wired into the typed + /// path. These are the high-bandwidth topics a freshness watchdog is most + /// likely to be guarding: a camera publishing through `try_send` read as + /// 0 Hz, and `StalePolicy::SafeState` would safe-state a subscriber whose + /// frames were arriving normally. + #[test] + fn pool_backed_try_send_moves_the_staleness_counter() { + let name = format!("staleness_pool_{}", std::process::id()); + cleanup(&name); + let Ok(topic) = Topic::::new(&name) else { + eprintln!("skipping: no shared memory available"); + return; + }; + + let before = topic.ring.header().messages_total(); + let mut sent = 0u64; + for _ in 0..5 { + let Ok(img) = crate::memory::Image::new(4, 4, crate::types::ImageEncoding::Rgb8) else { + eprintln!("skipping: no tensor pool available"); + return; + }; + if topic.try_send(img).is_ok() { + sent += 1; + } + } + + assert!(sent > 0, "precondition: the topic accepted some frames"); + assert_eq!( + topic.ring.header().messages_total() - before, + 5, + "every try_send is counted, delivered or refused — the typed path \ + counts attempts the same way, and a publisher hammering a full \ + ring is still alive" + ); + + drop(topic); + cleanup(&name); + } + + /// The same for `send_blocking`, on a topic that has backpressure. + #[test] + fn send_blocking_moves_the_staleness_counter() { + let name = format!("staleness_send_blocking_{}", std::process::id()); + cleanup(&name); + let Ok(topic) = Topic::::new(&name) else { + eprintln!("skipping: no shared memory available"); + return; + }; + + let mut delivered = 0u64; + for i in 0..10u64 { + match topic.send_blocking(i, Duration::from_millis(50)) { + Ok(()) => delivered += 1, + // A broadcast backend refuses before it ever touches the ring. + Err(SendBlockingError::NoBackpressure) => {} + Err(SendBlockingError::Timeout) => delivered += 1, + } + } + + assert_eq!( + topic.ring.header().messages_total(), + delivered, + "every send_blocking that engaged the ring must move the counter, \ + and a NoBackpressure refusal — which never reaches the ring — must \ + not, or a topic delivering nothing would read as healthy" + ); + assert!( + delivered > 0, + "precondition: a single-subscriber topic has backpressure, so these \ + sends must have engaged the ring" + ); + + drop(topic); + cleanup(&name); + } +} diff --git a/horus_core/src/communication/topic/tests.rs b/horus_core/src/communication/topic/tests.rs index 6c6509ede..0c4c0fcdf 100644 --- a/horus_core/src/communication/topic/tests.rs +++ b/horus_core/src/communication/topic/tests.rs @@ -15,7 +15,7 @@ use super::header::ParticipantEntry; use super::*; use std::mem; -use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicU8, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU16, AtomicU32, AtomicU64, AtomicU8, Ordering}; use std::sync::{Arc, Barrier}; use std::thread; use std::time::{Duration, Instant}; @@ -4176,7 +4176,7 @@ fn detect_backend_cross_process_multi_p_0c_mpsc_shm() { fn register_producer_returns_slot_index() { let mut h = TopicHeader::zeroed(); h.init(8, 4, true, 64, 8, "TestType", 0); - let slot = h.register_producer().expect("should succeed"); + let slot = h.register_producer().expect("should succeed").0; assert!(slot < 16, "Slot index must be < MAX_PARTICIPANTS"); assert_eq!(h.pub_count(), 1); assert_eq!(h.sub_count(), 0); @@ -4186,7 +4186,7 @@ fn register_producer_returns_slot_index() { fn register_consumer_returns_slot_index() { let mut h = TopicHeader::zeroed(); h.init(8, 4, true, 64, 8, "TestType", 0); - let slot = h.register_consumer().expect("should succeed"); + let slot = h.register_consumer().expect("should succeed").0; assert!(slot < 16); assert_eq!(h.sub_count(), 1); assert_eq!(h.pub_count(), 0); @@ -4196,8 +4196,8 @@ fn register_consumer_returns_slot_index() { fn register_both_roles_same_thread_reuses_slot() { let mut h = TopicHeader::zeroed(); h.init(8, 4, true, 64, 8, "TestType", 0); - let slot_pub = h.register_producer().expect("producer"); - let slot_sub = h.register_consumer().expect("consumer"); + let slot_pub = h.register_producer().expect("producer").0; + let slot_sub = h.register_consumer().expect("consumer").0; // Same thread should reuse the same participant entry assert_eq!( slot_pub, slot_sub, @@ -4249,7 +4249,10 @@ fn participant_entry_lease_not_expired_when_fresh() { role: AtomicU8::new(1), active: AtomicU8::new(1), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(current_time_ms() + 60_000), }; assert!(!p.is_lease_expired(current_time_ms())); @@ -4263,7 +4266,10 @@ fn participant_entry_lease_expired_after_timeout() { role: AtomicU8::new(1), active: AtomicU8::new(1), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(current_time_ms().saturating_sub(1000)), }; assert!(p.is_lease_expired(current_time_ms())); @@ -4277,7 +4283,10 @@ fn participant_entry_zero_lease_is_expired() { role: AtomicU8::new(1), active: AtomicU8::new(1), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(0), }; assert!(p.is_lease_expired(current_time_ms())); @@ -4291,7 +4300,10 @@ fn participant_entry_refresh_extends_lease() { role: AtomicU8::new(1), active: AtomicU8::new(1), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(current_time_ms().saturating_sub(1000)), }; assert!(p.is_lease_expired(current_time_ms())); @@ -4327,7 +4339,8 @@ fn expired_slot_reclaimed_by_new_registration() { // subscribers. A new registration must take a free slot while one exists. let slot = h .register_producer() - .expect("a free slot is available, so registration must succeed"); + .expect("a free slot is available, so registration must succeed") + .0; assert_ne!(slot, 0, "must not evict slot 0 while free slots remain"); assert_eq!( p.pid.load(Ordering::Acquire), @@ -4415,7 +4428,10 @@ fn participant_clear_resets_all_fields() { role: AtomicU8::new(3), active: AtomicU8::new(1), source_host: AtomicU8::new(0), - _pad: [0; 5], + _pad0: [0; 1], + generation: AtomicU16::new(0), + pub_handles: AtomicU8::new(0), + sub_handles: AtomicU8::new(0), lease_expires_ms: AtomicU64::new(999999), }; @@ -9834,7 +9850,6 @@ fn migration_metrics_default_all_zero() { assert_eq!(m.messages_sent.load(Ordering::Relaxed), 0); assert_eq!(m.messages_received.load(Ordering::Relaxed), 0); assert_eq!(m.send_failures.load(Ordering::Relaxed), 0); - assert_eq!(m.recv_failures.load(Ordering::Relaxed), 0); assert_eq!(m.migrations.load(Ordering::Relaxed), 0); } @@ -9845,13 +9860,11 @@ fn migration_metrics_atomic_increments() { m.messages_sent.fetch_add(5, Ordering::Relaxed); m.messages_received.fetch_add(3, Ordering::Relaxed); m.send_failures.fetch_add(2, Ordering::Relaxed); - m.recv_failures.fetch_add(1, Ordering::Relaxed); m.migrations.fetch_add(4, Ordering::Relaxed); assert_eq!(m.messages_sent.load(Ordering::Relaxed), 5); assert_eq!(m.messages_received.load(Ordering::Relaxed), 3); assert_eq!(m.send_failures.load(Ordering::Relaxed), 2); - assert_eq!(m.recv_failures.load(Ordering::Relaxed), 1); assert_eq!(m.migrations.load(Ordering::Relaxed), 4); } @@ -9896,17 +9909,59 @@ fn migration_metrics_manual_increment_visible_in_snapshot() { let t: Topic = Topic::new(unique("metrics_manual")).expect("create"); let mig = t.migration_metrics(); - // Manually increment (as would happen if hot-path tracking were enabled) mig.messages_sent.fetch_add(10, Ordering::Relaxed); mig.messages_received.fetch_add(7, Ordering::Relaxed); mig.send_failures.fetch_add(2, Ordering::Relaxed); - mig.recv_failures.fetch_add(1, Ordering::Relaxed); let snap = t.metrics(); assert_eq!(snap.messages_sent(), 10); assert_eq!(snap.messages_received(), 7); assert_eq!(snap.send_failures(), 2); - assert_eq!(snap.recv_failures(), 1); + assert_eq!( + snap.recv_failures(), + 0, + "the real transport has no recv-failure notion — a recv() that returns \ + None is an empty ring. Only MockTopic reports one, from its fault \ + injection." + ); +} + +/// The counters behind `TopicMetrics` must move on the ORDINARY paths. +/// +/// `messages_sent` and `messages_received` were incremented only inside +/// `send_with_content_logging` / `recv_with_content_logging` — both `#[cold]`, +/// both reached only while the `horus monitor` TUI has set the topic's verbose +/// flag. So a public accessor documented as "Messages sent through this topic" +/// read 0 in every normal run, however much traffic the topic carried. +/// +/// `MockTopic` — the stand-in users write their tests against — maintained both +/// faithfully all along, and its own tests assert on them. A double that +/// reports what the real transport does not is the one defect a double cannot +/// have. +#[test] +fn topic_metrics_count_traffic_without_the_monitor_attached() { + let t: Topic = Topic::new(unique("metrics_ordinary")).expect("create"); + + for i in 0..5u64 { + t.send(i); + } + let mut received = 0u64; + while t.recv().is_some() { + received += 1; + } + + let snap = t.metrics(); + assert_eq!( + snap.messages_sent(), + 5, + "five ordinary send() calls must read as five" + ); + assert!(received > 0, "precondition: the topic delivered something"); + assert_eq!( + snap.messages_received(), + received, + "every message recv() handed back must be counted" + ); } // ============================================================================ @@ -12482,10 +12537,6 @@ fn no_message_is_lost_without_being_counted() { treat this as a real failure, not a skip." ); } - MigrationResult::Failed => { - eprintln!("skipping {mode:?}: not available in this build"); - continue; - } } backends_checked += 1; trigger_shm_dispatch(&name); diff --git a/horus_core/src/core/hlog.rs b/horus_core/src/core/hlog.rs index 86361483e..80f9ab05f 100644 --- a/horus_core/src/core/hlog.rs +++ b/horus_core/src/core/hlog.rs @@ -606,9 +606,13 @@ pub fn format_console_line( /// not-taken branch: the level is checked **before** the format arguments are /// evaluated, so a suppressed line pays for neither `format!` nor its /// arguments' `Display` impls. The default level is -/// [`DEFAULT_LEVEL`](crate::core::hlog::DEFAULT_LEVEL) (`info`, i.e. `debug` -/// is off); change it with `HORUS_LOG_LEVEL=debug` or -/// [`set_max_level`](crate::core::hlog::set_max_level). +/// [`DEFAULT_LEVEL`](crate::core::hlog::DEFAULT_LEVEL), and it is `debug` — +/// **nothing is suppressed until you configure it**, so in an unconfigured +/// process a `hlog!(debug, ...)` pays for its `format!`, a timestamp and a +/// serialise into the shared log buffer. Set `HORUS_LOG_LEVEL=info` (or call +/// [`set_max_level`](crate::core::hlog::set_max_level)) to get the not-taken +/// branch. This paragraph said `info` for a while; see `DEFAULT_LEVEL`'s own +/// doc for why the permissive default is deliberate. /// /// The corollary is that arguments with side effects are not evaluated when /// the line is suppressed — which is the intended semantics of a level, and diff --git a/horus_core/src/core/log_buffer.rs b/horus_core/src/core/log_buffer.rs index 509ec71b0..da423a41f 100644 --- a/horus_core/src/core/log_buffer.rs +++ b/horus_core/src/core/log_buffer.rs @@ -595,7 +595,18 @@ lazy_static::lazy_static! { /// Error and Warning entries are dual-written to both the main buffer and the /// dedicated error buffer for persistent retention. Info/Debug/Publish/Subscribe /// entries go to the main buffer only (zero overhead on the hot sensor path). -pub fn publish_log(entry: LogEntry) { +/// +/// An empty `timestamp` is stamped here. Both C++ entry points -- `horus_log` +/// and `horus_blackbox_record` -- pass `String::new()` with the comment "filled +/// by log_buffer if empty", and nothing filled it: every line a C++ node logged +/// landed in `horus log` with a blank time column while the identical Rust call +/// carried one. Callers that already have a timestamp keep it, so `hlog!` still +/// reports the instant the log statement ran rather than the instant it reached +/// the buffer. +pub fn publish_log(mut entry: LogEntry) { + if entry.timestamp.is_empty() { + entry.timestamp = chrono::Local::now().format("%H:%M:%S%.3f").to_string(); + } if matches!(entry.log_type, LogType::Error | LogType::Warning) { GLOBAL_ERROR_BUFFER.push(entry.clone()); } @@ -637,6 +648,40 @@ pub fn start_log_file_drain() -> Option> { .ok() } +/// The process-wide log-file drain, started at most once — but only once it +/// has actually started. +/// +/// A `OnceLock>` was wrong: `get_or_init` memoizes whatever the first +/// call produced, `None` included. A process that built one `Scheduler` before +/// `HORUS_LOG_FILE` was set — which every test that sets the variable and then +/// constructs a scheduler does, and any program that reads config after an +/// early scheduler — cached "no drain" for its whole life and silently never +/// wrote a log file again. A `Mutex` keeps the at-most-one guarantee (the lock +/// serialises the check and the spawn) without freezing the negative answer. +static LOG_FILE_DRAIN: std::sync::Mutex>> = + std::sync::Mutex::new(None); + +/// Start the log-file drain once per process, if `HORUS_LOG_FILE` asks for it. +/// +/// `Scheduler::new` calls this unconditionally: `start_log_file_drain` reads +/// the variable itself and returns `None` when it is unset, so a process that +/// has not opted in pays one env lookup and spawns nothing. +/// +/// Once, because a process may build several `Scheduler`s (tests do, and so +/// does any program that runs one scheduler after another). Each drain thread +/// tracks its own `last_idx` into the same global ring, so N of them would +/// write every entry N times into the same file. +/// +/// Returns whether a drain thread is running. +pub fn start_log_file_drain_once() -> bool { + let mut slot = LOG_FILE_DRAIN.lock().unwrap_or_else(|e| e.into_inner()); + if slot.is_some() { + return true; + } + *slot = start_log_file_drain(); + slot.is_some() +} + fn log_drain_loop(log_dir: &str, max_size: u64, max_files: usize) { use std::io::Write; diff --git a/horus_core/src/core/mod.rs b/horus_core/src/core/mod.rs index 4cd42e396..f70c62cea 100644 --- a/horus_core/src/core/mod.rs +++ b/horus_core/src/core/mod.rs @@ -39,8 +39,8 @@ pub mod timer; // Log buffer types — pub for hlog! macro expansion and CLI log viewing (horus log) #[doc(hidden)] pub use log_buffer::{ - start_log_file_drain, LogEntry, LogType, SharedLogBuffer, GLOBAL_ERROR_BUFFER, - GLOBAL_LOG_BUFFER, GLOBAL_REMOTE_LOG_BUFFER, + start_log_file_drain, start_log_file_drain_once, LogEntry, LogType, SharedLogBuffer, + GLOBAL_ERROR_BUFFER, GLOBAL_LOG_BUFFER, GLOBAL_REMOTE_LOG_BUFFER, }; pub use duration_ext::{DurationExt, Frequency}; diff --git a/horus_core/src/drivers/exec_driver.rs b/horus_core/src/drivers/exec_driver.rs index 4ef2e9424..ab0b28eef 100644 --- a/horus_core/src/drivers/exec_driver.rs +++ b/horus_core/src/drivers/exec_driver.rs @@ -73,8 +73,19 @@ impl ExecDriver { } } - /// Create from a path string, args, and NodeParams (parsed from horus.toml). + /// Create from a manifest entry name, path, args, and NodeParams. + /// + /// `name` is the `[hardware.]` key. It used to be discarded and the + /// driver's name derived from the binary's file stem, so two `[hardware]` + /// entries pointing at one binary were indistinguishable in the logs while + /// the scheduler knew them by their distinct manifest keys. It is also what + /// `HORUS_DRIVER_NAME` is supposed to carry: only `from_params` ever set + /// that variable, and its sole caller is a unit test, so an exec driver + /// reading it on the live path got nothing — while the environment-surface + /// contract asserts "`HORUS_DRIVER_NAME` is injected into every user-written + /// exec driver". pub fn from_config( + name: &str, path: &str, args: Vec, params: &NodeParams, @@ -108,14 +119,11 @@ impl ExecDriver { } } - let name = std::path::Path::new(path) - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or("exec_driver") - .to_string(); + // The name the operator wrote, not the binary's file stem. + env.insert("HORUS_DRIVER_NAME".to_string(), name.to_string()); Ok(Self::new( - &name, + name, ExecDriverConfig { path: PathBuf::from(path), args, @@ -358,6 +366,43 @@ mod tests { assert!(driver.config.env.contains_key("HORUS_PARAM_PORT")); } + /// The live path must name the driver after the `[hardware.]` key + /// and inject `HORUS_DRIVER_NAME`. + /// + /// `from_config` is what a horus.toml exec entry actually takes; + /// `from_params` is called only by the test above. `from_config` used to + /// discard the entry name and derive the driver's name from the binary's + /// FILE STEM, so two `[hardware]` entries pointing at one binary were + /// indistinguishable in the logs while the scheduler knew them apart. It + /// injected `HORUS_PARAM_*` and no name at all, so an exec driver reading + /// `HORUS_DRIVER_NAME` on the live path got nothing — while the + /// environment-surface contract asserts it "is injected into every + /// user-written exec driver". + #[test] + fn from_config_names_the_driver_after_the_manifest_entry() { + let params = NodeParams::new(HashMap::new()); + + let driver = ExecDriver::from_config("front_lidar", "./rplidar_driver", vec![], ¶ms) + .expect("from_config"); + + assert_eq!( + driver.name(), + "front_lidar", + "the driver must be named after the [hardware.] key, not the \ + binary's file stem — two entries can share one binary" + ); + assert_eq!( + driver + .config + .env + .get("HORUS_DRIVER_NAME") + .map(String::as_str), + Some("front_lidar"), + "HORUS_DRIVER_NAME must reach the child on the path horus.toml \ + actually takes" + ); + } + #[test] fn test_exec_driver_health_check_no_child() { let config = ExecDriverConfig::default(); diff --git a/horus_core/src/drivers/mod.rs b/horus_core/src/drivers/mod.rs index 1605b4be0..4e654a330 100644 --- a/horus_core/src/drivers/mod.rs +++ b/horus_core/src/drivers/mod.rs @@ -72,6 +72,76 @@ pub fn load() -> HorusResult)>> { load_from(&path) } +/// Whether a `[hardware.]` table asks to be substituted by a stub under +/// `horus run --sim`. +/// +/// `simulated` is an ALIAS of `sim`, not a second key. `horus add +/// --driver --source sim` — and `--source simulated`, and `--source sim3d` — +/// all write `simulated = true` into horus.toml, so HORUS generated a key its +/// own loader then ignored: the entry constructed the REAL driver under +/// `horus run --sim`, which is the exact failure `--sim` exists to prevent. +/// `horus check` said nothing either, because `simulated` is a known field +/// rather than an unknown one, and the loader's `RESERVED` list kept it away +/// from the driver as well. +/// +/// A free function so the truth table can be tested without a config file or +/// the process-global environment — the same reason `sim_mode_enabled` is one. +fn asks_for_simulation(config: &toml::value::Table) -> bool { + config + .get("sim") + .or_else(|| config.get("simulated")) + .and_then(|v| v.as_bool()) + .unwrap_or(false) +} + +/// Whether a `HORUS_SIM_MODE` value asks for simulation. +/// +/// Presence alone used to mean "on", so the two most natural ways to say *off* +/// both turned simulation on. A deploy script setting `HORUS_SIM_MODE=0` to +/// force real hardware silently got inert stubs: actuators never commanded, +/// sensors never read, nothing logged. Parse the value, the way +/// `HORUS_NET_ENABLED` already does. +/// +/// A free function taking the value so the truth table can be tested without +/// racing on the process-global environment. It used to be a *copy* of this +/// predicate living inside `#[cfg(test)] mod sim_mode_tests`, which is a thing +/// that can silently stop matching what ships. +fn sim_mode_enabled(value: Option<&str>) -> bool { + value + .map(|v| !(v.is_empty() || v == "0" || v.eq_ignore_ascii_case("false"))) + .unwrap_or(false) +} + +/// Whether this process is running under `horus run --sim`. +fn sim_mode_active() -> bool { + sim_mode_enabled(std::env::var("HORUS_SIM_MODE").ok().as_deref()) +} + +/// The `HORUS_SIM_TARGETS` filter, or `None` for "every entry that asks". +fn sim_targets() -> Option> { + std::env::var("HORUS_SIM_TARGETS") + .ok() + .map(|s| s.split(',').map(String::from).collect()) +} + +/// Whether this named entry gets a stub in this process. +/// +/// The one predicate behind both loaders. `load_from` substitutes a +/// `SimStubNode`; `load_config_entries` reports it to its caller. They used to +/// disagree: `load_config_entries` had no simulation handling at all, so the +/// Python path built the REAL driver under `horus run --sim` -- the exact +/// failure `--sim` exists to prevent, and the one `asks_for_simulation` +/// already documents. +fn entry_is_simulated(name: &str, config: &toml::value::Table) -> bool { + if !asks_for_simulation(config) || !sim_mode_active() { + return false; + } + match sim_targets() { + Some(targets) => targets.iter().any(|t| t == name), + None => true, // no filter = all sim targets + } +} + /// Load hardware nodes from a specific config file. /// /// Useful for testing with alternate configs or multi-robot setups. @@ -91,23 +161,6 @@ pub fn load_from>(path: P) -> HorusResult> = if sim_mode { - std::env::var("HORUS_SIM_TARGETS") - .ok() - .map(|s| s.split(',').map(String::from).collect()) - } else { - None - }; - // Reserved keys that are NOT passed as NodeParams const RESERVED: &[&str] = &[ "use", @@ -135,24 +188,15 @@ pub fn load_from>(path: P) -> HorusResult targets.iter().any(|t| t == name), - None => true, // no filter = all sim targets - }; - if should_sim { - log::info!("hardware.{name}: simulation mode — using stub"); - nodes.push(( - name.clone(), - Box::new(SimStubNode { - node_name: format!("{name}_sim_stub"), - }), - )); - continue; - } + if entry_is_simulated(name, config) { + log::info!("hardware.{name}: simulation mode — using stub"); + nodes.push(( + name.clone(), + Box::new(SimStubNode { + node_name: format!("{name}_sim_stub"), + }), + )); + continue; } // Determine the node type name from 'use' field (new) or legacy source keys @@ -226,7 +270,7 @@ pub fn load_from>(path: P) -> HorusResult HorusResult> { robot_name_from(find_manifest()?) } +/// Parse `[hardware]` into `(name, use_name, params, simulated)` tuples. +/// +/// The non-instantiating half of [`load_from`]: it resolves the same key +/// fallback chain and the same simulation predicate, but hands the decision +/// back instead of constructing a node. The Python binding uses it, which is +/// why `simulated` is reported rather than acted on -- a Python driver class +/// is not a `Box` this crate can substitute a stub for. +/// +/// `simulated` is true when the entry asks for it (`sim` or its alias +/// `simulated`) AND this process runs under `horus run --sim`. It used to be +/// absent entirely, so `hardware.load()` in Python built the REAL driver under +/// `--sim` while the same manifest under Rust got a stub. pub fn load_config_entries>( path: P, -) -> HorusResult> { +) -> HorusResult> { let path = path.as_ref(); let content = std::fs::read_to_string(path) .map_err(|e| ConfigError::Other(format!("failed to read {}: {}", path.display(), e)))?; @@ -374,7 +430,12 @@ pub fn load_config_entries>( .map(|(k, v)| (k.clone(), v.clone())) .collect(); - entries.push((name.clone(), use_name, NodeParams::new(param_map))); + entries.push(( + name.clone(), + use_name, + NodeParams::new(param_map), + entry_is_simulated(name, config), + )); } Ok(entries) @@ -407,18 +468,61 @@ pub fn find_manifest() -> HorusResult { } #[cfg(test)] -mod sim_mode_tests { - /// Mirrors the `HORUS_SIM_MODE` predicate in `load_from`. +mod sim_target_tests { + use super::asks_for_simulation; + + fn table(toml_src: &str) -> toml::value::Table { + toml::from_str(toml_src).expect("test table") + } + + /// `simulated` must substitute the stub, because HORUS writes that key. /// - /// Kept as a free function so the truth table can be tested without racing - /// on the process-global environment, which every other test in this crate - /// also reads. - fn sim_mode_enabled(value: Option<&str>) -> bool { - value - .map(|v| !(v.is_empty() || v == "0" || v.eq_ignore_ascii_case("false"))) - .unwrap_or(false) + /// `horus add --driver --source sim` (and `--source simulated`, and + /// `--source sim3d`) all emit `simulated = true`. The loader read only + /// `sim`, so an entry HORUS generated itself constructed the REAL driver + /// under `horus run --sim` — the exact failure `--sim` exists to prevent — + /// and `horus check` stayed quiet, because `simulated` is a known field + /// rather than an unknown one. + #[test] + fn simulated_is_an_alias_of_sim() { + assert!( + asks_for_simulation(&table("use = \"rplidar\"\nsimulated = true\n")), + "`simulated = true` is what `horus add --source sim` writes; the \ + loader must honour it" + ); + assert!(asks_for_simulation(&table( + "use = \"rplidar\"\nsim = true\n" + ))); } + /// Neither key, or either set false, means the real driver. + #[test] + fn the_real_driver_is_the_default() { + assert!(!asks_for_simulation(&table("use = \"rplidar\"\n"))); + assert!(!asks_for_simulation(&table( + "use = \"rplidar\"\nsim = false\n" + ))); + assert!(!asks_for_simulation(&table( + "use = \"rplidar\"\nsimulated = false\n" + ))); + } + + /// An explicit `sim` wins over the alias — it is the documented key. + #[test] + fn sim_takes_precedence_over_the_alias() { + assert!(!asks_for_simulation(&table( + "sim = false\nsimulated = true\n" + ))); + } +} + +#[cfg(test)] +mod sim_mode_tests { + // The predicate under test is the one that ships. It used to be a private + // copy declared right here, which could stop matching `load_from` without + // a single test going red. + use super::sim_mode_enabled; + /// Presence alone used to mean "on", so the two most natural ways to say /// *off* both turned simulation on. A deploy script setting /// `HORUS_SIM_MODE=0` to force real hardware silently got inert stubs: diff --git a/horus_core/src/drivers/tests.rs b/horus_core/src/drivers/tests.rs index ee2d7a3c0..78393e339 100644 --- a/horus_core/src/drivers/tests.rs +++ b/horus_core/src/drivers/tests.rs @@ -319,6 +319,128 @@ sim = true std::fs::remove_file(&path).ok(); } + /// The two loaders must agree about what `--sim` covers. + /// + /// `load_from` substituted a stub while `load_config_entries` -- the whole + /// of the Python path -- reported nothing, so `hardware.load()` in Python + /// constructed the REAL driver under `horus run --sim`. Constructing a + /// driver is where the port is opened and the motor enabled, so `--sim` + /// was inert on that path. Load one manifest through both and require the + /// same verdict per entry. + #[test] + fn both_loaders_agree_on_what_sim_mode_covers() { + let _guard = SIM_ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + // SAFETY: guarded by SIM_ENV_LOCK to prevent parallel env var mutation + unsafe { + std::env::set_var("HORUS_SIM_MODE", "1"); + } + + registry::register("AgreementReal", |p| Ok(Box::new(StubNode::from_params(p)?))); + + let content = r#" +[hardware.simmed] +use = "AgreementReal" +sim = true + +[hardware.aliased] +use = "AgreementReal" +simulated = true + +[hardware.real] +use = "AgreementReal" +"#; + let path = write_temp_toml(content); + + let nodes = crate::drivers::load_from(&path).unwrap(); + let entries = crate::drivers::load_config_entries(&path).unwrap(); + + for (name, node) in &nodes { + let stubbed_by_load = node.name().contains("sim_stub"); + let (_, _, _, simulated) = entries + .iter() + .find(|(n, _, _, _)| n == name) + .unwrap_or_else(|| panic!("load_config_entries dropped {name}")); + assert_eq!( + stubbed_by_load, *simulated, + "the two loaders disagree about {name}: load_from stubbed={stubbed_by_load}, \ + load_config_entries simulated={simulated}" + ); + } + + // And the verdicts themselves are the right ones. + let simulated_names: Vec<&str> = entries + .iter() + .filter(|(_, _, _, sim)| *sim) + .map(|(n, _, _, _)| n.as_str()) + .collect(); + assert_eq!(simulated_names, vec!["aliased", "simmed"]); + + unsafe { + std::env::remove_var("HORUS_SIM_MODE"); + } + std::fs::remove_file(&path).ok(); + } + + /// Outside `--sim`, `sim = true` changes nothing on either path. + #[test] + fn config_entries_report_no_simulation_without_sim_mode() { + let _guard = SIM_ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + unsafe { + std::env::remove_var("HORUS_SIM_MODE"); + } + + let content = r#" +[hardware.sensor] +use = "AnythingAtAll" +sim = true +"#; + let path = write_temp_toml(content); + let entries = crate::drivers::load_config_entries(&path).unwrap(); + assert_eq!(entries.len(), 1); + assert!( + !entries[0].3, + "`sim = true` alone must not simulate - only `--sim` does" + ); + std::fs::remove_file(&path).ok(); + } + + /// HORUS_SIM_TARGETS must narrow both loaders the same way. + #[test] + fn config_entries_honour_the_selective_target_filter() { + let _guard = SIM_ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + unsafe { + std::env::set_var("HORUS_SIM_MODE", "1"); + std::env::set_var("HORUS_SIM_TARGETS", "lidar"); + } + + let content = r#" +[hardware.lidar] +use = "SomeType" +sim = true + +[hardware.imu] +use = "SomeType" +sim = true +"#; + let path = write_temp_toml(content); + let entries = crate::drivers::load_config_entries(&path).unwrap(); + let sim_of = |want: &str| { + entries + .iter() + .find(|(n, _, _, _)| n == want) + .map(|(_, _, _, s)| *s) + .unwrap() + }; + assert!(sim_of("lidar"), "lidar is in HORUS_SIM_TARGETS"); + assert!(!sim_of("imu"), "imu is not in HORUS_SIM_TARGETS"); + + unsafe { + std::env::remove_var("HORUS_SIM_MODE"); + std::env::remove_var("HORUS_SIM_TARGETS"); + } + std::fs::remove_file(&path).ok(); + } + #[test] fn load_exec_prefix() { let content = r#" @@ -329,8 +451,12 @@ use = "exec:/bin/echo" let nodes = crate::drivers::load_from(&path).unwrap(); assert_eq!(nodes.len(), 1); assert_eq!(nodes[0].0, "camera"); - // ExecDriver node name is derived from binary name - assert_eq!(nodes[0].1.name(), "echo"); + // The driver is named after the [hardware.] key, matching the + // name the scheduler knows it by. This used to assert "echo" — the + // binary's file stem — which is the defect, not the contract: two + // [hardware] entries pointing at one binary were indistinguishable in + // the driver's own logs while the scheduler knew them apart. + assert_eq!(nodes[0].1.name(), "camera"); std::fs::remove_file(&path).ok(); } diff --git a/horus_core/src/scheduling/async_executor.rs b/horus_core/src/scheduling/async_executor.rs index 6aacdd5aa..3f36df911 100644 --- a/horus_core/src/scheduling/async_executor.rs +++ b/horus_core/src/scheduling/async_executor.rs @@ -54,6 +54,15 @@ impl AsyncExecutor { ) -> Self { nodes.sort_by_key(|n| n.priority); + // `honor_safe_state_request` reports through `rt_diag`, so this + // executor needs the drain thread as much as the RT one does — without + // it, a safe-state panic on a async I/O node is queued into a ring nothing + // reads. Idempotent, and on the caller's thread. + super::rt_executor::start_rt_diag_drain(); + + // `spawn_best_effort`, not a bare `Builder`: this executor runs + // non-real-time node classes, so it must not inherit the tick thread's + // policy or its reserved-core mask. let handle = crate::scheduling::rt::spawn_best_effort("horus-async-io", 0, move || { Self::async_thread_main(nodes, running, tick_period, monitors) }) @@ -87,12 +96,16 @@ impl AsyncExecutor { // node blocked inside `tick()`, because this loop only re-checks // `running` between ticks — and `run_with_filter` calls this before it // shuts down or safes any other node. - super::primitives::join_with_timeout( + let nodes = super::primitives::join_with_timeout( handle, "Async I/O", super::primitives::SHUTDOWN_TIMEOUT_PER_THREAD, ) - .unwrap_or_default() + .unwrap_or_default(); + // The last diagnostics a torn-down executor queued are the ones worth + // having; the background drainer only wakes every RT_DIAG_POLL. + super::rt_executor::flush_rt_diag(); + nodes } /// Main function for the async I/O thread. @@ -140,12 +153,38 @@ impl AsyncExecutor { // is precisely one that is not in `ready_indices`. for node in nodes.iter_mut() { super::primitives::honor_safe_state_request(node, &monitors); + super::primitives::honor_restart_request(node, &monitors); + } + + // Operator commands reach every executor through the shared + // node_controls map: `horus node kill` sets stopped, `horus node + // pause` sets paused. Only the RT executor used to read them, so + // pausing or killing a async I/O node from the CLI printed success and + // then did nothing at all -- while `should_tick_node` says it is + // "honored the same way the executor threads do" and the control + // handler says the pause holds "for ALL node classes". + // + // A pause does not latch -- it is re-read every pass, so ResumeNode + // takes effect on the next one. That is the half the CLI actually + // drives: ControlCommand::PauseNode/ResumeNode set this flag. + // + // The `stopped` half latches into `is_stopped` the way the RT + // executor does. Nothing in the CLI sets it today (`set_stopped` is + // `#[allow(dead_code)]`, and `horus node kill` goes through the + // launch supervisor instead), so this is parity with the RT + // executor rather than a second user-facing fix -- but a flag two + // of five tick gates honour is exactly how the pause bug started. + for node in nodes.iter_mut() { + if monitors.node_controls.is_stopped(node.name.as_ref()) { + node.is_stopped = true; + } } for (i, node) in nodes.iter().enumerate() { if !node.initialized || node.is_stopped || node.is_paused + || monitors.node_controls.is_paused(node.name.as_ref()) || !node.failure_policy_allows_tick() { continue; @@ -474,6 +513,70 @@ mod tests { } } + /// `horus node pause` must reach a async I/O node. + /// + /// Operator commands travel through the shared `node_controls` map. Only + /// the RT executor read it, so pausing or killing a async I/O node from the + /// CLI reported success and changed nothing -- while `should_tick_node` + /// says the flag is "honored the same way the executor threads do" and the + /// control handler says an operator pause holds "for ALL node classes". + /// + /// Counts, not flags: the question is whether the node stopped running. + /// The `stopped` half is checked for parity with the RT executor; no CLI + /// path sets it today. + #[test] + fn an_operator_pause_and_kill_reach_an_async_node() { + let count = Arc::new(AtomicU64::new(0)); + let monitors = test_monitors(); + monitors.node_controls.register("paused_async"); + let running = Arc::new(AtomicBool::new(true)); + + let executor = AsyncExecutor::start( + vec![make_async_node("paused_async", count.clone())], + running.clone(), + 1_u64.ms(), + monitors.clone(), + ); + + let ticked = |from: u64| { + for _ in 0..200 { + if count.load(Ordering::Relaxed) > from { + return true; + } + std::thread::sleep(5_u64.ms()); + } + false + }; + + assert!(ticked(0), "the node never ticked at all"); + + monitors.node_controls.set_paused("paused_async", true); + std::thread::sleep(30_u64.ms()); + let frozen = count.load(Ordering::Relaxed); + std::thread::sleep(60_u64.ms()); + assert_eq!( + count.load(Ordering::Relaxed), + frozen, + "the node kept ticking through an operator pause" + ); + + monitors.node_controls.set_paused("paused_async", false); + assert!(ticked(frozen), "the node never resumed after ResumeNode"); + + monitors.node_controls.set_stopped("paused_async"); + std::thread::sleep(30_u64.ms()); + let killed_at = count.load(Ordering::Relaxed); + std::thread::sleep(60_u64.ms()); + assert_eq!( + count.load(Ordering::Relaxed), + killed_at, + "the node kept ticking after an operator kill" + ); + + running.store(false, Ordering::SeqCst); + let _ = executor.stop(); + } + fn make_async_node(name: &str, count: Arc) -> RegisteredNode { let node = CounterNode { name: name.to_string(), diff --git a/horus_core/src/scheduling/compute_executor.rs b/horus_core/src/scheduling/compute_executor.rs index 6b9a38f56..e8838007d 100644 --- a/horus_core/src/scheduling/compute_executor.rs +++ b/horus_core/src/scheduling/compute_executor.rs @@ -509,6 +509,15 @@ impl ComputeExecutor { ) -> Self { nodes.sort_by_key(|n| n.priority); + // `honor_safe_state_request` reports through `rt_diag`, so this + // executor needs the drain thread as much as the RT one does — without + // it, a safe-state panic on a compute node is queued into a ring nothing + // reads. Idempotent, and on the caller's thread. + super::rt_executor::start_rt_diag_drain(); + + // `spawn_best_effort`, not a bare `Builder`: this executor runs + // non-real-time node classes, so it must not inherit the tick thread's + // policy or its reserved-core mask. let handle = crate::scheduling::rt::spawn_best_effort("horus-compute", 0, move || { Self::compute_thread_main(nodes, running, tick_period, monitors) }) @@ -542,12 +551,16 @@ impl ComputeExecutor { // node blocked inside `tick()`, because this loop only re-checks // `running` between ticks — and `run_with_filter` calls this before it // shuts down or safes any other node. - super::primitives::join_with_timeout( + let nodes = super::primitives::join_with_timeout( handle, "Compute", super::primitives::SHUTDOWN_TIMEOUT_PER_THREAD, ) - .unwrap_or_default() + .unwrap_or_default(); + // The last diagnostics a torn-down executor queued are the ones worth + // having; the background drainer only wakes every RT_DIAG_POLL. + super::rt_executor::flush_rt_diag(); + nodes } /// Main function for the compute coordinator thread. @@ -585,12 +598,37 @@ impl ComputeExecutor { while running.load(Ordering::Relaxed) { let loop_start = Instant::now(); + // Operator commands reach every executor through the shared + // node_controls map: `horus node kill` sets stopped, `horus node + // pause` sets paused. Only the RT executor used to read them, so + // pausing or killing a compute node from the CLI printed success and + // then did nothing at all -- while `should_tick_node` says it is + // "honored the same way the executor threads do" and the control + // handler says the pause holds "for ALL node classes". + // + // A pause does not latch -- it is re-read every pass, so ResumeNode + // takes effect on the next one. That is the half the CLI actually + // drives: ControlCommand::PauseNode/ResumeNode set this flag. + // + // The `stopped` half latches into `is_stopped` the way the RT + // executor does. Nothing in the CLI sets it today (`set_stopped` is + // `#[allow(dead_code)]`, and `horus node kill` goes through the + // launch supervisor instead), so this is parity with the RT + // executor rather than a second user-facing fix -- but a flag two + // of five tick gates honour is exactly how the pause bug started. + for node in nodes.iter_mut() { + if monitors.node_controls.is_stopped(node.name.as_ref()) { + node.is_stopped = true; + } + } + // Classify which nodes should tick this cycle ready_indices.clear(); for (i, node) in nodes.iter().enumerate() { if !node.initialized || node.is_stopped || node.is_paused + || monitors.node_controls.is_paused(node.name.as_ref()) || !node.failure_policy_allows_tick() { continue; @@ -614,6 +652,23 @@ impl ComputeExecutor { ready_indices.push(i); } + // Safing requested by the main thread's watchdog ladder. Applied + // to EVERY node this executor owns, not just the ones ticking this + // pass — an Isolated node is precisely one that is not ticking. + // + // ABOVE the `ready_indices.is_empty()` early-continue, not below + // it. Below, this loop was skipped on exactly the pass the comment + // describes: when nothing is ready, which is the state a set of + // isolated or rate-limited nodes is permanently in. A link-loss + // `request_safe_state_all()` under `safety.on_link_lost = + // "safe_state"` — the path that deliberately does not latch an + // e-stop and leaves the scheduler running — would then never be + // consumed by this executor's nodes at all. + for node in nodes.iter_mut() { + super::primitives::honor_safe_state_request(node, &monitors); + super::primitives::honor_restart_request(node, &monitors); + } + if ready_indices.is_empty() { // Nothing to do — sleep and check again let elapsed = loop_start.elapsed(); @@ -623,13 +678,6 @@ impl ComputeExecutor { continue; } - // Safing requested by the main thread's watchdog ladder. Applied - // to EVERY node this executor owns, not just the ones ticking this - // pass — an Isolated node is precisely one that is not ticking. - for node in nodes.iter_mut() { - super::primitives::honor_safe_state_request(node, &monitors); - } - // Update last_tick for rate-limited nodes let now = Instant::now(); for &i in &ready_indices { @@ -911,6 +959,151 @@ mod tests { } } + /// `horus node pause` must reach a compute node. + /// + /// Operator commands travel through the shared `node_controls` map. Only + /// the RT executor read it, so pausing or killing a compute node from the + /// CLI reported success and changed nothing -- while `should_tick_node` + /// says the flag is "honored the same way the executor threads do" and the + /// control handler says an operator pause holds "for ALL node classes". + /// + /// Counts, not flags: the question is whether the node stopped running. + /// The `stopped` half is checked for parity with the RT executor; no CLI + /// path sets it today. + #[test] + fn an_operator_pause_and_kill_reach_a_compute_node() { + let count = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let monitors = test_monitors(); + monitors.node_controls.register("paused_compute"); + let running = Arc::new(AtomicBool::new(true)); + + let executor = ComputeExecutor::start( + vec![make_compute_node("paused_compute", count.clone())], + running.clone(), + 1_u64.ms(), + monitors.clone(), + ); + + let ticked = |from: u64| { + for _ in 0..200 { + if count.load(Ordering::Relaxed) > from { + return true; + } + std::thread::sleep(5_u64.ms()); + } + false + }; + + assert!(ticked(0), "the node never ticked at all"); + + // Pause: the count must stop moving. + monitors.node_controls.set_paused("paused_compute", true); + std::thread::sleep(30_u64.ms()); // let any in-flight pass finish + let frozen = count.load(Ordering::Relaxed); + std::thread::sleep(60_u64.ms()); + assert_eq!( + count.load(Ordering::Relaxed), + frozen, + "the node kept ticking through an operator pause" + ); + + // Resume: it must move again. + monitors.node_controls.set_paused("paused_compute", false); + assert!(ticked(frozen), "the node never resumed after ResumeNode"); + + // Kill: the count must stop, and stay stopped. + monitors.node_controls.set_stopped("paused_compute"); + std::thread::sleep(30_u64.ms()); + let killed_at = count.load(Ordering::Relaxed); + std::thread::sleep(60_u64.ms()); + assert_eq!( + count.load(Ordering::Relaxed), + killed_at, + "the node kept ticking after an operator kill" + ); + + running.store(false, Ordering::SeqCst); + let _ = executor.stop(); + } + + /// A safing diagnostic from a non-RT executor must actually be printed. + /// + /// `honor_safe_state_request` reports through `rt_diag`, which only queues + /// into a static ring — a separate drain thread does the printing, and it + /// used to be started by `RtExecutor::start_pool` alone. A program whose + /// nodes are all Compute (or Async, or Event) therefore started no drainer, + /// so every safing line it queued sat in the ring until a later line + /// overwrote it. Nothing was printed and nothing said so. + /// + /// The drain writes to the process's real stdout, which libtest cannot + /// capture in-process, so this re-runs itself as a child and reads what the + /// child printed. + #[test] + fn a_compute_only_program_still_prints_its_safing_diagnostics() { + const CHILD: &str = "HORUS_COMPUTE_DIAG_CHILD"; + const MARKER: &str = "compute-drain-marker"; + + if std::env::var_os(CHILD).is_some() { + // In the child: start a compute executor, which is the only thing + // that should be needed for a queued diagnostic to reach stdout. + let running = Arc::new(AtomicBool::new(true)); + let count = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let exec = ComputeExecutor::start( + vec![make_compute_node("diag_node", count)], + running.clone(), + Duration::from_millis(5), + test_monitors(), + ); + + super::super::rt_executor::rt_diag(format_args!("{MARKER}")); + + // No sleep. This used to wait out an RT_DIAG_POLL so the background + // drainer would have emitted the line, which made the test depend + // on that thread being scheduled inside a 200 ms window -- and on a + // loaded CI runner it was not, so the run failed with "never + // reached stdout" against code that was working. `stop()` now + // flushes the ring itself, so the line is emitted synchronously + // before this returns and no timing window is involved. + running.store(false, Ordering::SeqCst); + let _ = exec.stop(); + return; + } + + let exe = std::env::current_exe().expect("test binary path"); + let out = std::process::Command::new(exe) + .args([ + "--exact", + "scheduling::compute_executor::tests::a_compute_only_program_still_prints_its_safing_diagnostics", + "--nocapture", + "--test-threads", + "1", + ]) + .env(CHILD, "1") + .output() + .expect("re-running the test binary should work"); + + let stdout = String::from_utf8_lossy(&out.stdout); + assert!( + stdout.contains("1 passed"), + "the child ran no test — was this test renamed?\n{stdout}" + ); + // Asserts only that the line reaches stdout, which is what this test is + // named for. It deliberately does NOT try to prove the line came from + // the flush in `stop()` rather than from the background drainer: the + // drainer's FIRST poll happens when its thread starts, which is inside + // the same few microseconds as the `rt_diag` above, so an ordering + // assertion here passes whether or not `stop()` flushes. Isolating the + // flush would need a way to keep that thread from running at all, and a + // guard that cannot fail for the reason it states is worse than no + // guard. What the flush buys is covered by the argument at + // `flush_rt_diag`, not by this assertion. + assert!( + stdout.contains(MARKER), + "a diagnostic queued by a compute-only program never reached \ + stdout.\n{stdout}" + ); + } + fn make_compute_node(name: &str, count: Arc) -> RegisteredNode { let node = CounterNode { name: name.to_string(), diff --git a/horus_core/src/scheduling/control.rs b/horus_core/src/scheduling/control.rs index 42e41ce37..2dce507dd 100644 --- a/horus_core/src/scheduling/control.rs +++ b/horus_core/src/scheduling/control.rs @@ -34,6 +34,21 @@ pub enum ControlCommand { GracefulShutdown, /// Force a metrics flush to the SHM registry (useful for debugging). RequestMetricsFlush, + /// Re-run a node's `init()` on the next tick without stopping the scheduler. + /// + /// `horus node restart` used to send `PauseNode` followed immediately by + /// `ResumeNode` and claim that re-initialised the node. It did not: both + /// arms only toggle a shared atomic, and `reinit_pending_nodes` — the code + /// that actually re-runs `init()` — keys off `registered.initialized`, + /// which nothing outside registration ever set back to false. The command + /// printed "The scheduler will re-initialize this node on next tick" and + /// nothing happened. + /// + /// Added last so a command serialised by an older CLI still decodes. + RestartNode { + /// Name of the node to re-initialise. + name: String, + }, } impl ControlCommand { @@ -47,6 +62,7 @@ impl ControlCommand { } Self::GracefulShutdown => "Graceful shutdown".to_string(), Self::RequestMetricsFlush => "Flush metrics".to_string(), + Self::RestartNode { name } => format!("Restart node '{}'", name), } } } diff --git a/horus_core/src/scheduling/event_executor.rs b/horus_core/src/scheduling/event_executor.rs index b6663f6dc..173e6633f 100644 --- a/horus_core/src/scheduling/event_executor.rs +++ b/horus_core/src/scheduling/event_executor.rs @@ -164,6 +164,12 @@ impl EventExecutor { running: Arc, monitors: SharedMonitors, ) -> Self { + // `honor_safe_state_request` reports through `rt_diag`, so this + // executor needs the drain thread as much as the RT one does — without + // it, a safe-state panic on an event node is queued into a ring nothing + // reads. Idempotent, and on the caller's thread. + super::rt_executor::start_rt_diag_drain(); + let mut handles = Vec::with_capacity(nodes.len()); let mut notifiers = Vec::with_capacity(nodes.len()); @@ -254,6 +260,9 @@ impl EventExecutor { "[Event] Stopped ({} nodes returning to scheduler)", nodes.len() )); + // The last diagnostics a torn-down executor queued are the ones worth + // having; the background drainer only wakes every RT_DIAG_POLL. + super::rt_executor::flush_rt_diag(); nodes } @@ -330,10 +339,34 @@ impl EventExecutor { // before the gates below — an Isolated node is precisely one that // is not ticking, so gating this on tickability would never run it. super::primitives::honor_safe_state_request(&mut node, &monitors); + super::primitives::honor_restart_request(&mut node, &monitors); + + // Operator commands reach every executor through the shared + // node_controls map: `horus node kill` sets stopped, `horus node + // pause` sets paused. Only the RT executor used to read them, so + // pausing or killing a event node from the CLI printed success and + // then did nothing at all -- while `should_tick_node` says it is + // "honored the same way the executor threads do" and the control + // handler says the pause holds "for ALL node classes". + // + // A pause does not latch -- it is re-read every pass, so ResumeNode + // takes effect on the next one. That is the half the CLI actually + // drives: ControlCommand::PauseNode/ResumeNode set this flag. + // + // The `stopped` half latches into `is_stopped` the way the RT + // executor does. Nothing in the CLI sets it today (`set_stopped` is + // `#[allow(dead_code)]`, and `horus node kill` goes through the + // launch supervisor instead), so this is parity with the RT + // executor rather than a second user-facing fix -- but a flag two + // of five tick gates honour is exactly how the pause bug started. + if monitors.node_controls.is_stopped(node.name.as_ref()) { + node.is_stopped = true; + } if !node.initialized || node.is_stopped || node.is_paused + || monitors.node_controls.is_paused(node.name.as_ref()) || !node.failure_policy_allows_tick() { // Not tickable. Deliberately a timed sleep and not a wait on @@ -638,6 +671,98 @@ mod tests { } } + /// `horus node pause` must reach a event node. + /// + /// Operator commands travel through the shared `node_controls` map. Only + /// the RT executor read it, so pausing or killing a event node from the + /// CLI reported success and changed nothing -- while `should_tick_node` + /// says the flag is "honored the same way the executor threads do" and the + /// control handler says an operator pause holds "for ALL node classes". + /// + /// Counts, not flags: the question is whether the node stopped running. + /// The `stopped` half is checked for parity with the RT executor; no CLI + /// path sets it today. + #[test] + fn an_operator_pause_and_kill_reach_an_event_node() { + let count = Arc::new(AtomicU64::new(0)); + let monitors = test_monitors(); + monitors.node_controls.register("paused_event"); + let running = Arc::new(AtomicBool::new(true)); + + let executor = EventExecutor::start( + vec![make_event_node( + "paused_event", + "pause_topic", + count.clone(), + )], + running.clone(), + monitors.clone(), + ); + + // Wait for the watcher to register its notifier. + let mut registered = false; + for _ in 0..100 { + if crate::core::NodeInfo::notify_event("paused_event") { + registered = true; + break; + } + std::thread::sleep(5_u64.ms()); + } + assert!( + registered, + "the event watcher never registered its notifier" + ); + + let poke_until = |from: u64| { + for _ in 0..200 { + crate::core::NodeInfo::notify_event("paused_event"); + if count.load(Ordering::Relaxed) > from { + return true; + } + std::thread::sleep(5_u64.ms()); + } + false + }; + + assert!(poke_until(0), "the node never ticked at all"); + + // Paused: poking it must not tick it. + monitors.node_controls.set_paused("paused_event", true); + std::thread::sleep(30_u64.ms()); + let frozen = count.load(Ordering::Relaxed); + for _ in 0..12 { + crate::core::NodeInfo::notify_event("paused_event"); + std::thread::sleep(5_u64.ms()); + } + assert_eq!( + count.load(Ordering::Relaxed), + frozen, + "the node kept ticking through an operator pause" + ); + + monitors.node_controls.set_paused("paused_event", false); + assert!( + poke_until(frozen), + "the node never resumed after ResumeNode" + ); + + monitors.node_controls.set_stopped("paused_event"); + std::thread::sleep(30_u64.ms()); + let killed_at = count.load(Ordering::Relaxed); + for _ in 0..12 { + crate::core::NodeInfo::notify_event("paused_event"); + std::thread::sleep(5_u64.ms()); + } + assert_eq!( + count.load(Ordering::Relaxed), + killed_at, + "the node kept ticking after an operator kill" + ); + + running.store(false, Ordering::SeqCst); + let _ = executor.stop(); + } + #[test] fn test_event_executor_starts_and_stops() { let count = Arc::new(AtomicU64::new(0)); diff --git a/horus_core/src/scheduling/node_builder.rs b/horus_core/src/scheduling/node_builder.rs index 33951db1c..7743163aa 100644 --- a/horus_core/src/scheduling/node_builder.rs +++ b/horus_core/src/scheduling/node_builder.rs @@ -685,6 +685,23 @@ impl NodeRegistration { .into()); } + // The OTHER way `.no_alloc()` becomes false assurance: the flag is + // right and the execution class is right, but this binary never + // installed `RtAwareAllocator`, so nothing checks anything. The check + // above catches "the executor will not look"; this catches "there is + // nothing to look with". + // + // `warn_if_unenforced` was written for exactly this and says so — + // "Intended call site: node registration, once per `.no_alloc()` node" + // — and had no caller anywhere outside its own tests, so the user was + // never told. It warns rather than fails on purpose: refusing to build + // would force allocator installation on every library consumer, which + // its doc explains is what a library must not do. Process-wide-once, and + // a single atomic load after the first call. + if self.no_alloc { + crate::memory::rt_allocator::warn_if_unenforced(node_name); + } + // Validate execution class vs is_rt flag consistency if self.is_rt && !matches!( diff --git a/horus_core/src/scheduling/primitives.rs b/horus_core/src/scheduling/primitives.rs index a20cd737c..d2aa1016a 100644 --- a/horus_core/src/scheduling/primitives.rs +++ b/horus_core/src/scheduling/primitives.rs @@ -3,14 +3,20 @@ //! These are the building blocks that the scheduler composes for different //! execution strategies (sequential, parallel, future RT-thread, etc.). //! -//! # Everything above `run_tick` runs on an RT thread +//! # Everything above `run_tick` can run on an RT thread //! -//! `honor_safe_state_request` and `apply_degradation_action` are reached only -//! from `rt_executor` -- the scheduler keeps its own main-thread copy of the -//! degradation dispatch -- so they execute on a SCHED_FIFO thread, inside the -//! tick. They therefore report through `rt_diag`, which formats into a -//! statically allocated ring and never allocates, blocks or enters the kernel, -//! rather than through `print_line`. +//! `apply_degradation_action` is reached only from `rt_executor` -- the +//! scheduler keeps its own main-thread copy of the degradation dispatch -- so +//! it executes on a SCHED_FIFO thread, inside the tick. +//! +//! `honor_safe_state_request` is reached from ALL FOUR executors: its own +//! doc-comment says "every executor must call this at the top of its per-node +//! pass", and rt, compute, async and event all do. Only the RT one is +//! SCHED_FIFO, but the function is shared, so it reports the way the strictest +//! caller requires. +//! +//! That means `rt_diag`, which formats into a statically allocated ring and +//! never allocates, blocks or enters the kernel, rather than `print_line`. //! //! `print_line` does `isatty` + `tcgetattr`, takes the process-global stdout //! lock, then `write` and `flush`. Pointed at a slow consumer -- a serial @@ -20,8 +26,11 @@ //! or shut down the node in the same breath: the worst possible place to wait //! on a terminal. //! -//! The drain half is started by `RtExecutor::start` on the caller's thread -//! before any RT thread exists, so a line queued from here always has a drainer. +//! The drain half is started by every executor's `start`, on the caller's +//! thread before any executor thread exists, so a line queued from here always +//! has a drainer. It used to be started by `RtExecutor::start` alone, which +//! made that last sentence false for any program without RT nodes: safing +//! diagnostics went into the ring and nothing ever read them out. use std::time::{Duration, Instant}; @@ -62,6 +71,141 @@ pub(crate) fn guard_fault_callback(f: impl FnOnce()) -> bool { std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).is_err() } +/// A node's safing callback panicked on an executor, so the node did NOT reach +/// a safe state. Stop ticking it, and — because a node that cannot safe itself +/// is a safety failure — escalate to a system emergency stop for critical +/// nodes. +/// +/// The executor twin of `Scheduler::note_safing_failure`, which states the +/// intent as a property of the system rather than of one dispatch path. The +/// isolation half of that was ported to the executors (see +/// [`guard_fault_callback`]); the ESCALATION half was not, so the identical +/// event halted the robot on a BestEffort node and merely stopped an RT one — +/// and every RT node under a scheduler `.watchdog()` is a critical node, so the +/// class most likely to be critical was the one the escalation omitted. +pub(crate) fn note_safing_failure( + node: &mut RegisteredNode, + monitors: &super::types::SharedMonitors, + action: &str, +) { + node.is_stopped = true; + let critical = monitors + .safety + .as_ref() + .is_some_and(|m| m.is_critical_node(&node.name)); + if critical { + super::rt_executor::rt_diag(format_args!( + "[RT-thread] '{}' PANICKED in {} and could NOT reach a safe state — EMERGENCY STOP", + node.name, action + )); + if let Some(ref estop) = monitors.estop { + // `trigger_fmt`/`format_args!`, not `trigger`/`format!`. This is + // the RT panic path — the `[RT-thread]` line directly above says + // so — and `format!` allocates on it. The `String` wrapper this + // used to call no longer exists for that reason. + estop.trigger_fmt(format_args!( + "critical node '{}' panicked in {} and could not reach a safe state", + node.name, action + )); + } + } else { + super::rt_executor::rt_diag(format_args!( + "[RT-thread] '{}' panicked in {} and could NOT reach a safe state — node stopped", + node.name, action + )); + } +} + +/// Honour a pending restart request raised by `horus node restart`, if this +/// node has one. +/// +/// Same shape and same reason as [`honor_safe_state_request`]: `init()` needs +/// `&mut dyn Node`, which the executor owns, so the main thread raises a flag +/// and the owning executor consumes it here, once per raise. +/// +/// Every executor must call this at the top of its per-node pass. Skipping it +/// in one executor means `horus node restart` silently does nothing for that +/// whole class of node — which is what the control-command handler alone did +/// for all four executor classes, because after the class partition the +/// scheduler's `nodes` vector holds only the main-thread group. +/// +/// A restart also lifts an operator pause: a wedged node is usually paused +/// before it is restarted, and leaving it paused would make the restart a +/// no-op at the very next tick gate. +pub(crate) fn honor_restart_request( + node: &mut RegisteredNode, + monitors: &super::types::SharedMonitors, +) { + let requested = monitors + .node_controls + .take_restart_request(node.name.as_ref()); + if requested { + monitors.node_controls.set_paused(node.name.as_ref(), false); + } + honor_restart_request_with(node, requested, monitors); +} + +/// The same, for a caller that already holds the node's control block. +/// +/// The by-name form above takes an `RwLock` read guard and hashes the node name +/// on EVERY call. That is fine for the executors that poll it once per loop +/// iteration, but the RT executor calls it per node per tick, from the top of +/// `tick_node` -- so at 1 kHz with 8 nodes the by-name form would put 8000 +/// lock acquisitions a second back into the hot path that +/// `NodeControlMap::handle` exists to keep out. `safe_state_requested`, +/// `stopped` and `paused` are already read through the cached handle there; +/// this is the fourth flag and had no reason to be the exception. +pub(crate) fn honor_restart_request_via( + node: &mut RegisteredNode, + control: Option<&super::types::NodeControl>, + monitors: &super::types::SharedMonitors, +) { + let requested = control.is_some_and(|c| { + c.restart_requested + .swap(false, std::sync::atomic::Ordering::AcqRel) + }); + if requested { + if let Some(c) = control { + // A restart lifts an operator pause; see the note above. + c.paused.store(false, std::sync::atomic::Ordering::Release); + } + } + honor_restart_request_with(node, requested, monitors); +} + +fn honor_restart_request_with( + node: &mut RegisteredNode, + requested: bool, + monitors: &super::types::SharedMonitors, +) { + let _ = monitors; + if !requested { + return; + } + + node.is_paused = false; + + let target = &mut node.node; + let panicked = guard_fault_callback(|| { + let _ = target.init(); + }); + + if panicked { + // A node that cannot initialise must not be ticked. + node.is_stopped = true; + super::rt_executor::rt_diag(format_args!( + " Restart: '{}' PANICKED in init() — node stopped", + node.name + )); + } else { + node.initialized = true; + super::rt_executor::rt_diag(format_args!( + " Restart: '{}' re-initialised on its executor", + node.name + )); + } +} + /// Honour a pending safe-state request raised by the main thread's watchdog /// ladder, if this node has one. /// @@ -200,13 +344,7 @@ pub(crate) fn apply_degradation_action( DegradationAction::Isolate(ref name) => { set_health(node, NodeHealthState::Isolated); let target = &mut node.node; - let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - target.enter_safe_state() - })) - .is_err(); - if panicked { - node.is_stopped = true; - } + let panicked = guard_fault_callback(|| target.enter_safe_state()); super::rt_executor::rt_diag(format_args!( " Degradation: '{name}' — isolated, entered safe state{}", if panicked { @@ -215,6 +353,9 @@ pub(crate) fn apply_degradation_action( "" } )); + if panicked { + note_safing_failure(node, monitors, "enter_safe_state"); + } if let Some(ref monitor) = monitors.safety { monitor.record_degrade_activation(); } @@ -222,15 +363,17 @@ pub(crate) fn apply_degradation_action( DegradationAction::Kill(ref name) => { set_health(node, NodeHealthState::Isolated); let target = &mut node.node; - let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let panicked = guard_fault_callback(|| { let _ = target.shutdown(); - })) - .is_err(); + }); node.is_stopped = true; super::rt_executor::rt_diag(format_args!( " KILL: '{name}' — permanently removed from execution after shutdown(){}", if panicked { " (shutdown panicked)" } else { "" } )); + if panicked { + note_safing_failure(node, monitors, "shutdown"); + } if let Some(ref monitor) = monitors.safety { monitor.record_degrade_activation(); } diff --git a/horus_core/src/scheduling/rt_executor.rs b/horus_core/src/scheduling/rt_executor.rs index 562e22ff5..4aadeec07 100644 --- a/horus_core/src/scheduling/rt_executor.rs +++ b/horus_core/src/scheduling/rt_executor.rs @@ -178,6 +178,9 @@ static RT_DIAG_TRUNCATED: AtomicU64 = AtomicU64::new(0); /// diagnostic can never wait on a drainer, which is the priority inversion the /// blocking `print_line` had. static RT_DIAG_TAIL: Mutex = Mutex::new(0); +/// The index a drain last stopped on because its producer had not finished +/// writing. Guarded by `RT_DIAG_TAIL`, so plain `Relaxed` is enough. +static RT_DIAG_STALLED_AT: AtomicU64 = AtomicU64::new(u64::MAX); static RT_DIAG_DRAIN_STARTED: Once = Once::new(); @@ -265,11 +268,36 @@ pub(crate) fn rt_diag(args: fmt::Arguments<'_>) { /// The seqlock check is the whole safety argument: `idx` names a *generation*, /// not just a slot, so a reader can always tell "the line I asked for" from /// "some newer line that happens to live here now". -fn read_slot(idx: u64, out: &mut [u8; RT_DIAG_LINE_CAP]) -> Option { +/// What a reader found in a slot it asked for. +/// +/// The two failure cases used to be one `None`, and the drain treated both the +/// same way: skip the index and advance the tail past it. They are opposites. +/// A LAPPED slot is genuinely gone. An IN-FLIGHT slot belongs to a producer +/// that has claimed the index and is still writing it -- it will be complete +/// microseconds later, and skipping it throws away a line that was never lost. +enum SlotRead { + Ready(usize), + /// Claimed, not finished. `seq` is behind the value this index will carry. + InFlight, + /// A later producer has already recycled this slot; the line is gone. + Lapped, +} + +fn read_slot(idx: u64, out: &mut [u8; RT_DIAG_LINE_CAP]) -> SlotRead { let slot = &RT_DIAG_RING[(idx & RT_DIAG_SLOT_MASK) as usize]; let want = idx.wrapping_mul(2).wrapping_add(2); - if slot.seq.load(Ordering::Acquire) != want { - return None; + let classify = |seq: u64| { + // `seq` is derived from a monotonic claim index, so "behind" is a plain + // comparison; u64 does not wrap in any reachable lifetime. + if seq < want { + SlotRead::InFlight + } else { + SlotRead::Lapped + } + }; + let seq = slot.seq.load(Ordering::Acquire); + if seq != want { + return classify(seq); } let len = (slot.len.load(Ordering::Relaxed) as usize).min(RT_DIAG_LINE_CAP); for (i, dst) in out[..len].iter_mut().enumerate() { @@ -280,10 +308,11 @@ fn read_slot(idx: u64, out: &mut [u8; RT_DIAG_LINE_CAP]) -> Option { // being reordered past this load — an `Acquire` *load* would only order // what comes after it. std::sync::atomic::fence(Ordering::Acquire); - if slot.seq.load(Ordering::Relaxed) != want { - return None; + let seq = slot.seq.load(Ordering::Relaxed); + if seq != want { + return classify(seq); } - Some(len) + SlotRead::Ready(len) } /// Print every queued diagnostic through `emit`, then report what was lost. @@ -306,16 +335,52 @@ fn rt_diag_drain(emit: impl Fn(&str)) -> usize { let mut line = [0u8; RT_DIAG_LINE_CAP]; let mut printed = 0usize; + let mut next = *tail; for idx in *tail..head { - let Some(len) = read_slot(idx, &mut line) else { - continue; - }; - if let Ok(text) = std::str::from_utf8(&line[..len]) { - emit(text); - printed += 1; + match read_slot(idx, &mut line) { + SlotRead::Ready(len) => { + if let Ok(text) = std::str::from_utf8(&line[..len]) { + emit(text); + printed += 1; + } + next = idx + 1; + } + SlotRead::Lapped => { + RT_DIAG_DROPPED.fetch_add(1, Ordering::Relaxed); + next = idx + 1; + } + SlotRead::InFlight => { + // The producer bumped `head` and has not finished its store + // yet. Stopping here and retrying leaves the line to the next + // drain, 25 ms away; advancing past it -- which is what this + // loop used to do for every unreadable slot -- discarded a line + // that was in the middle of being written. That is a race the + // producer always loses on a contended machine, and it is how a + // one-line diagnostic followed by an immediate shutdown went + // missing on CI while passing everywhere else. + // + // Bounded, because a producer that panics between claiming the + // index and completing the store would otherwise wedge the ring + // for the life of the process and cost every LATER line too. + // Stalling twice on the same index is taken as a dead producer: + // give up that one line and keep going. + if RT_DIAG_STALLED_AT.swap(idx, Ordering::Relaxed) == idx { + RT_DIAG_DROPPED.fetch_add(1, Ordering::Relaxed); + next = idx + 1; + continue; + } + break; + } } } - *tail = head; + // Only a drain that consumed the whole range clears the marker. `next > + // *tail` would also be true of a drain that emitted three lines and then + // stalled on the fourth -- clearing it there would reset the counter every + // round and the bounded skip could never fire. + if next == head { + RT_DIAG_STALLED_AT.store(u64::MAX, Ordering::Relaxed); + } + *tail = next; drop(tail); let dropped = RT_DIAG_DROPPED.swap(0, Ordering::Relaxed); @@ -336,15 +401,44 @@ fn rt_diag_drain(emit: impl Fn(&str)) -> usize { /// Start the diagnostic drain thread, once per process. /// -/// Called from `start_pool`, on the CALLER's thread. Never from an RT thread: -/// spawning is a `clone(2)` plus a stack mapping, which is exactly the class of -/// cost this whole mechanism exists to keep off the tick loop. +/// Called from every executor's `start`, on the CALLER's thread. Never from an +/// RT thread: spawning is a `clone(2)` plus a stack mapping, which is exactly +/// the class of cost this whole mechanism exists to keep off the tick loop. +/// +/// Not just `start_pool`. `honor_safe_state_request` reports through `rt_diag` +/// and every executor calls it, so a program with no RT nodes at all queued +/// safing diagnostics -- including "PANICKED in enter_safe_state" -- into a +/// ring that nothing ever read. They were overwritten after `RT_DIAG_SLOTS` +/// lines and never printed. /// /// The thread runs for the life of the process. It is one wakeup every /// `RT_DIAG_POLL` and it must outlive any individual executor, because an /// executor that is being torn down is precisely when its last diagnostics /// matter. -fn start_rt_diag_drain() { +/// Drain whatever is still in the diagnostic ring, now, on the calling thread. +/// +/// Every executor's `stop()` owes its nodes this. The background drainer wakes +/// only every `RT_DIAG_POLL`, so a program that queues a diagnostic and then +/// shuts down inside that window exits with the line still in the ring — and +/// the lines that arrive in the last milliseconds before a shutdown are the +/// ones worth having: "PANICKED in enter_safe_state", a final deadline miss. +/// +/// `RtExecutor::stop` has done this since the ring existed. Compute, Event and +/// AsyncIo did not, though all three call `start_rt_diag_drain` and all three +/// queue through `honor_safe_state_request` — so for a compute-only, event-only +/// or async-only program the guarantee simply did not hold. Same class +/// partition that keeps catching this code: fixed on the RT path, left on the +/// other three. +/// +/// It also makes the spawn-failure warning in `start_rt_diag_drain` true. That +/// warning tells the operator diagnostics "will only be flushed at shutdown"; +/// with no shutdown flush on three of the four executors, they were not +/// flushed at all. +pub(crate) fn flush_rt_diag() { + rt_diag_drain(print_line); +} + +pub(crate) fn start_rt_diag_drain() { RT_DIAG_DRAIN_STARTED.call_once(|| { let spawned = super::rt::spawn_best_effort("horus-rt-diag", 5, || loop { rt_diag_drain(print_line); @@ -1309,7 +1403,7 @@ impl RtExecutor { // run shorter than `RT_DIAG_POLL` — or one whose last deadline miss came // in the final milliseconds, which is the interesting case — would exit // with its diagnostics still sitting in the ring. - rt_diag_drain(print_line); + flush_rt_diag(); // Same reason: a run shorter than one drain poll must not exit with its // RT log lines still sitting in the ring. crate::core::hlog_rt::flush(); @@ -1330,7 +1424,14 @@ impl RtExecutor { name_hash: u64, timing: Option<&super::safety_monitor::NodeTimingState>, watchdog: Option<&super::safety_monitor::Watchdog>, + control: Option<&super::types::NodeControl>, ) { + // A pending `horus node restart` for a node this executor owns. At the + // top of the single per-node entry point, so every caller honours it + // and a suppressed node is still restartable — the whole point of + // restarting one is that it is not currently running well. + super::primitives::honor_restart_request_via(node, control, monitors); + // Failure-policy backoff (Restart) / cooldown (Skip): skip this tick // while the node is suppressed. if !node.failure_policy_allows_tick() { @@ -1465,8 +1566,18 @@ impl RtExecutor { "[RT-thread] BUDGET ENFORCE: '{}' exceeded 2x budget ({:?} > {:?}) — node stopped", node.name, tr.duration, tick_budget * 2 )); - let _ = node.node.shutdown(); - node.is_stopped = true; + // Guarded: an unwind out of a node's own shutdown + // would otherwise escape into the loop's outer + // catch_unwind and skip everything after it. + let target = &mut node.node; + let panicked = super::primitives::guard_fault_callback(|| { + let _ = target.shutdown(); + }); + if panicked { + super::primitives::note_safing_failure(node, monitors, "shutdown"); + } else { + node.is_stopped = true; + } } } BudgetPolicy::EmergencyStop => { @@ -1475,15 +1586,21 @@ impl RtExecutor { "[RT-thread] BUDGET E-STOP: '{}' budget violation ({:?} > {:?})", node.name, tr.duration, tick_budget )); - let _ = node.node.shutdown(); - node.is_stopped = true; - // Latch the SafetyMonitor before signalling the thread to - // exit. `running.store(false)` alone is a plain shutdown - // flag: it leaves get_state() reporting Normal, writes no + // Latch the SafetyMonitor BEFORE calling into the node. + // `running.store(false)` alone is a plain shutdown flag: + // it leaves get_state() reporting Normal, writes no // blackbox EmergencyStop, and never populates // PENDING_LOCAL_ESTOP — so horus_net had nothing to // broadcast and peer robots were never told this one // emergency-stopped. + // + // Ordering matters as much as the latch. `shutdown()` is + // user code on a node that is already failing; it used to + // run first and unguarded, so a panic in it unwound past + // the latch, the running flag and everything else into + // the loop's outer catch_unwind — whose only action is to + // stop the node. The e-stop this policy exists to fire + // was silently cancelled by the callback it fired. if let Some(ref estop) = monitors.estop { estop.trigger_fmt(format_args!( "RT node '{}' budget violation ({:?} > {:?}) with BudgetPolicy::EmergencyStop", @@ -1492,6 +1609,11 @@ impl RtExecutor { } // Signal stop via running flag — RT thread will exit running.store(false, Ordering::SeqCst); + node.is_stopped = true; + let target = &mut node.node; + let _ = super::primitives::guard_fault_callback(|| { + let _ = target.shutdown(); + }); } } } @@ -1510,6 +1632,15 @@ impl RtExecutor { deadline, node.miss_policy, ); + if miss.is_none() { + // The tick met its deadline: end the consecutive-miss run. + // `check_tick_budget` above must not do this — it is measuring + // a different thing, and a node released late misses its + // deadline while sitting comfortably inside its budget. + if let Some(ref monitor) = monitors.safety { + monitor.record_deadline_met(&node.name); + } + } if miss.is_none() && node.in_safe_mode { // Met the deadline again: clear the latch so a node that // recovers can be safed once more if it degrades later. @@ -1625,7 +1756,18 @@ impl RtExecutor { SuppressedSuffix(hidden) )); } - node.node.enter_safe_state(); + let target = &mut node.node; + if super::primitives::guard_fault_callback(|| target.enter_safe_state()) + { + // A critical node that cannot reach a safe state + // is a system safety failure, not a node-local + // one — the same escalation the main loop makes. + super::primitives::note_safing_failure( + node, + monitors, + "enter_safe_state", + ); + } } } DeadlineAction::EmergencyStop => { @@ -2135,7 +2277,9 @@ impl RtExecutor { let loop_start = Instant::now(); for (idx, node) in nodes.iter_mut().enumerate() { - if !node.initialized || node.is_stopped { + // An uninitialized node has never run, so it has nothing to + // safe and no state to leave behind. + if !node.initialized { continue; } @@ -2144,8 +2288,22 @@ impl RtExecutor { // Safing requested by the main thread's watchdog ladder. Runs // before the pause/stop gates: an Isolated node must reach its // safe state even if it is also about to be stopped. + // + // `is_stopped` used to be part of the guard at the top of this + // loop, which made that sentence false for the case it was + // written for: a node already stopped — by `BudgetPolicy:: + // Enforce`, by the kill rung, by `horus node kill` — was + // skipped before ever reaching this call, so it kept holding + // whatever setpoint it last commanded. Stopping a node halts + // its ticks; it does not put its actuator anywhere. The sibling + // executors safe every node they own regardless of stop state, + // and so does the main loop. super::primitives::honor_safe_state_request_via(node, control, &monitors); + if node.is_stopped { + continue; + } + // Check shared control flags (set by CLI: horus node pause/kill). // Unregistered node (`None`) reads as neither stopped nor // paused, matching what the by-name accessors returned on a miss. @@ -2209,6 +2367,7 @@ impl RtExecutor { name_hashes[idx], timings[idx].as_deref(), watchdogs[idx].as_deref(), + control, ) })); warmed[idx] = true; @@ -2268,6 +2427,95 @@ impl Drop for RtExecutor { } } +#[cfg(test)] +mod diag_ring_inflight { + use super::*; + + /// Both tests below drive the one process-global ring by hand, and the + /// second one's repeated drains perform the bounded skip -- which steps + /// past the first one's deliberately in-flight slot. Run in parallel they + /// fail each other; the lock is what keeps each in sole possession. + static RING_TESTS: Mutex<()> = Mutex::new(()); + + /// A drain must not step over a line that is still being written. + /// + /// `rt_diag` bumps `RT_DIAG_HEAD` first and only then marks the slot + /// complete. A drain landing in that window used to get `None` from + /// `read_slot`, `continue`, and then set `tail = head` -- stepping over the + /// line permanently. The producer had done nothing wrong and the line was + /// never recoverable, so neither the 25 ms drainer nor the flush in + /// `stop()` could ever print it. + /// + /// Nanoseconds wide on an idle machine, which is why it showed up as a CI + /// failure on a contended runner and nowhere else. + /// + /// Asserts on the tail rather than on what got emitted: the background + /// drainer may be running in this process and would race a + /// "the next drain emits it" assertion. + #[test] + fn a_drain_does_not_step_over_a_line_still_being_written() { + let _guard = RING_TESTS.lock().unwrap_or_else(|e| e.into_inner()); + const MARKER: &str = "inflight-slot-marker"; + + // Claim an index and stop where a preempted producer stops: head bumped, + // sequence odd, payload not yet written. + let idx = RT_DIAG_HEAD.fetch_add(1, Ordering::Relaxed); + let slot = &RT_DIAG_RING[(idx & RT_DIAG_SLOT_MASK) as usize]; + slot.seq + .store(idx.wrapping_mul(2).wrapping_add(1), Ordering::Relaxed); + std::sync::atomic::fence(Ordering::Release); + + rt_diag_drain(|_| {}); + + let tail_now = *RT_DIAG_TAIL.lock().unwrap_or_else(|e| e.into_inner()); + assert!( + tail_now <= idx, + "a drain advanced the tail to {tail_now}, past index {idx}, whose \ + producer had not finished writing. That line can never be emitted \ + by anyone: the slot is skipped now and the tail never goes back." + ); + + // Finish the write and drain, so the ring is not left blocked for the + // rest of this test binary. + for (i, b) in MARKER.as_bytes().iter().enumerate() { + slot.bytes[i].store(*b, Ordering::Relaxed); + } + slot.len.store(MARKER.len() as u32, Ordering::Relaxed); + slot.seq + .store(idx.wrapping_mul(2).wrapping_add(2), Ordering::Release); + rt_diag_drain(|_| {}); + } + + /// ...but a producer that dies mid-write must not wedge the ring forever. + /// + /// Retrying an in-flight slot is only safe if it is bounded. A panic + /// between the head bump and the completing store leaves a slot odd for + /// good; without a bound every later diagnostic in the process would queue + /// behind it. The second stall on the same index gives up that one line. + #[test] + fn a_producer_that_dies_mid_write_does_not_wedge_the_ring() { + let _guard = RING_TESTS.lock().unwrap_or_else(|e| e.into_inner()); + let idx = RT_DIAG_HEAD.fetch_add(1, Ordering::Relaxed); + let slot = &RT_DIAG_RING[(idx & RT_DIAG_SLOT_MASK) as usize]; + slot.seq + .store(idx.wrapping_mul(2).wrapping_add(1), Ordering::Relaxed); + std::sync::atomic::fence(Ordering::Release); + + // First drain stalls on it, second gives up on it. + rt_diag_drain(|_| {}); + rt_diag_drain(|_| {}); + rt_diag_drain(|_| {}); + + let tail_now = *RT_DIAG_TAIL.lock().unwrap_or_else(|e| e.into_inner()); + assert!( + tail_now > idx, + "the ring is still stuck at {idx} after repeated drains: a producer \ + that never completes its write would block every later diagnostic \ + for the life of the process." + ); + } +} + #[cfg(test)] mod tests { /// The idle-exit budget must track the guard spin it has to fit inside, @@ -2358,6 +2606,167 @@ mod tests { } } + /// A node whose init counts, so a restart is observable. + struct ReinitNode { + name: String, + inits: Arc, + } + + impl Node for ReinitNode { + fn name(&self) -> &str { + &self.name + } + fn init(&mut self) -> crate::error::HorusResult<()> { + self.inits + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + Ok(()) + } + fn tick(&mut self) {} + } + + /// `horus node restart` must reach a node living on an executor. + /// + /// The control-command handler searches `Scheduler::nodes`, and the class + /// partition leaves only the main-thread group in there. So restart reached + /// BestEffort nodes and reported every RT, Compute, Event and AsyncIo node + /// as unknown — that is, every node with a `.rate()`, which is exactly what + /// an operator restarts. `init()` needs `&mut dyn Node`, owned by the + /// executor, so the request crosses through the shared control map the same + /// way the watchdog ladder's safing request does. + #[test] + fn a_restart_request_reaches_a_node_on_an_executor() { + use crate::core::NodeInfo; + use std::sync::atomic::Ordering as AOrd; + + let inits = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let mut node = make_rt_registered( + "executor_node", + Arc::new(std::sync::atomic::AtomicU64::new(0)), + ); + node.node = super::super::types::NodeKind::new(Box::new(ReinitNode { + name: "executor_node".to_string(), + inits: inits.clone(), + })); + node.context = Some(NodeInfo::new("executor_node".to_string())); + + let monitors = test_monitors(); + monitors.node_controls.register("executor_node"); + let running = Arc::new(AtomicBool::new(true)); + // The cached control block, the way the executor loop resolves it. + // Passing `None` here would leave the restart flag unread and the + // assertions below trivially satisfied. + let control = monitors.node_controls.handle("executor_node"); + + // No request pending: ticking must not re-initialise anything. + RtExecutor::tick_node( + &mut node, + &monitors, + &running, + false, + 0, + None, + 0, + None, + None, + control.as_deref(), + ); + assert_eq!( + inits.load(AOrd::Relaxed), + 0, + "a tick with no restart pending must not re-init" + ); + + // The main thread raises the request for a node it does not own. + assert!( + monitors.node_controls.request_restart("executor_node"), + "the control map is registered for all five groups, so the node is known" + ); + RtExecutor::tick_node( + &mut node, + &monitors, + &running, + false, + 0, + None, + 0, + None, + None, + control.as_deref(), + ); + assert_eq!( + inits.load(AOrd::Relaxed), + 1, + "the executor that owns the node must honour the restart — searching \ + the scheduler's own `nodes` cannot reach it after the class partition" + ); + + // Honoured exactly once per raise. + RtExecutor::tick_node( + &mut node, + &monitors, + &running, + false, + 0, + None, + 0, + None, + None, + control.as_deref(), + ); + assert_eq!( + inits.load(AOrd::Relaxed), + 1, + "the flag is consumed, so a restart happens once per request" + ); + } + + /// A restart lifts an operator pause. + /// + /// A wedged node is usually paused before it is restarted; leaving it + /// paused would make the restart a no-op at the very next tick gate. + #[test] + fn a_restart_clears_the_pause_that_preceded_it() { + let inits = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let mut node = make_rt_registered( + "paused_node", + Arc::new(std::sync::atomic::AtomicU64::new(0)), + ); + node.node = super::super::types::NodeKind::new(Box::new(ReinitNode { + name: "paused_node".to_string(), + inits: inits.clone(), + })); + node.is_paused = true; + + let monitors = test_monitors(); + monitors.node_controls.register("paused_node"); + monitors.node_controls.set_paused("paused_node", true); + let running = Arc::new(AtomicBool::new(true)); + // The cached control block, the way the executor loop resolves it. + // Passing `None` here would leave the restart flag unread and the + // assertions below trivially satisfied. + let control = monitors.node_controls.handle("paused_node"); + + monitors.node_controls.request_restart("paused_node"); + RtExecutor::tick_node( + &mut node, + &monitors, + &running, + false, + 0, + None, + 0, + None, + None, + control.as_deref(), + ); + + assert!(!node.is_paused, "the restart must lift the pause"); + assert!( + !monitors.node_controls.is_paused("paused_node"), + "and clear it in the shared map the executors gate on" + ); + } + /// The executor half of the graduated watchdog ladder. /// /// The ladder runs on the main thread (the only one a hung node cannot @@ -2453,7 +2862,9 @@ mod tests { // The RT loop wraps `tick_node` exactly like this. let outcome = catch_unwind(AssertUnwindSafe(|| { - RtExecutor::tick_node(&mut node, &monitors, &running, true, 0, None, 0, None, None) + RtExecutor::tick_node( + &mut node, &monitors, &running, true, 0, None, 0, None, None, None, + ) })); assert!( @@ -2630,6 +3041,7 @@ mod tests { 0, None, None, + None, ); assert_eq!( @@ -2640,6 +3052,163 @@ mod tests { ); } + /// The RT executor's half of the consecutive-miss contract, on the failure + /// mode `check_deadline_from_release` exists to catch: a node woken late + /// that then executes in microseconds. Its deadline is blown from release; + /// its budget is untouched. + /// + /// `tick_node` calls `check_tick_budget` unconditionally, before the + /// deadline check. While the budget check owned the reset, that in-budget + /// tick cleared the run the deadline miss had just started, so the counter + /// was pinned at 1 no matter how long the node had been failing — neither + /// `warn_after: 3` nor the `max_deadline_misses` ceiling could ever fire + /// for it. A met deadline, and only that, ends the run. + #[test] + fn late_release_misses_accumulate_and_a_met_deadline_clears_them() { + use std::sync::atomic::AtomicU64; + + let count = Arc::new(AtomicU64::new(0)); + let mut node = make_rt_registered("arm_controller", count.clone()); + node.deadline = Some(Duration::from_millis(1)); + // A budget it comfortably meets — the tick itself is a counter bump. + node.tick_budget = Some(Duration::from_millis(1)); + + let monitor = Arc::new(SafetyMonitor::new(1000)); + let mut monitors = test_monitors(); + monitors.safety = Some(monitor.clone()); + let running = Arc::new(AtomicBool::new(true)); + + // Released 5 ms late, five times over. Every tick is inside its budget + // and every tick has missed its 1 ms deadline before it even began. + for _ in 0..5 { + RtExecutor::tick_node( + &mut node, &monitors, &running, false, 5_000_000, None, 0, None, None, None, + ); + } + assert_eq!( + monitor.consecutive_misses("arm_controller"), + 5, + "five late releases in a row are five consecutive deadline misses; \ + the node staying inside its BUDGET says nothing about its DEADLINE" + ); + + // Released on time: the deadline is met, and the run ends. + RtExecutor::tick_node( + &mut node, &monitors, &running, false, 0, None, 0, None, None, None, + ); + assert_eq!( + monitor.consecutive_misses("arm_controller"), + 0, + "a tick that met its deadline must clear the run" + ); + } + + /// A node whose safing callbacks panic — the failure both tests are about. + struct UnsafeableNode { + name: String, + } + + impl Node for UnsafeableNode { + fn name(&self) -> &str { + &self.name + } + fn tick(&mut self) {} + fn enter_safe_state(&mut self) { + panic!("cannot reach a safe state"); + } + fn shutdown(&mut self) -> crate::error::HorusResult<()> { + panic!("cannot shut down"); + } + } + + fn make_unsafeable(name: &str) -> RegisteredNode { + use crate::core::NodeInfo; + let mut node = make_rt_registered(name, Arc::new(std::sync::atomic::AtomicU64::new(0))); + node.node = super::super::types::NodeKind::new(Box::new(UnsafeableNode { + name: name.to_string(), + })); + node.context = Some(NodeInfo::new(name.to_string())); + node + } + + /// A panic in `shutdown()` must not cancel the emergency stop it was + /// called by. + /// + /// `BudgetPolicy::EmergencyStop` used to call `shutdown()` bare and BEFORE + /// latching. `shutdown()` is user code on a node that is already failing, + /// so an unwind out of it escaped past the latch, past `running.store`, and + /// into the loop's outer `catch_unwind` — whose only action is to stop the + /// node. `get_state()` kept reporting Normal, no blackbox EmergencyStop was + /// written, `PENDING_LOCAL_ESTOP` stayed empty so horus_net broadcast + /// nothing, and the other RT nodes kept ticking. The documented "e-stop on + /// any budget violation" was silently cancelled by the callback it fired. + #[test] + fn a_panicking_shutdown_does_not_cancel_the_budget_emergency_stop() { + let _estop_serial = super::super::safety_monitor::estop_queue_guard(); + let mut node = make_unsafeable("budget_estop_node"); + node.tick_budget = Some(Duration::from_nanos(1)); + node.budget_policy = super::super::safety_monitor::BudgetPolicy::EmergencyStop; + + let monitor = Arc::new(SafetyMonitor::new(1000)); + let mut monitors = test_monitors(); + monitors.estop = Some(monitor.estop_trigger()); + monitors.safety = Some(monitor.clone()); + let running = Arc::new(AtomicBool::new(true)); + + RtExecutor::tick_node( + &mut node, &monitors, &running, false, 0, None, 0, None, None, None, + ); + + assert!( + monitor.is_emergency_stop(), + "the emergency stop must be latched even though shutdown() panicked" + ); + assert!( + !running.load(Ordering::SeqCst), + "the RT thread must still be told to exit" + ); + } + + /// A critical node that cannot reach a safe state must halt the robot. + /// + /// The docs say so without qualification, and `note_safing_failure` states + /// it as a property of the system: "a node that cannot safe itself is a + /// safety failure — escalate to a system emergency stop for critical + /// nodes". Only the main loop did it. The executors got the ISOLATION half + /// of that (`guard_fault_callback`, added "so both halves of the framework + /// isolate recovery callbacks identically") and not the escalation, so the + /// identical event halted the robot on a BestEffort node and merely stopped + /// an RT one — and every RT node under a scheduler `.watchdog()` is + /// critical, so the class most likely to be critical was the one that was + /// omitted. + #[test] + fn a_critical_rt_node_that_cannot_safe_itself_triggers_an_emergency_stop() { + let _estop_serial = super::super::safety_monitor::estop_queue_guard(); + let mut node = make_unsafeable("critical_safer"); + node.deadline = Some(Duration::from_millis(1)); + node.miss_policy = Miss::SafeMode; + + let monitor = Arc::new(SafetyMonitor::new(1000)); + monitor.add_critical_node("critical_safer".to_string(), Duration::from_millis(100)); + let mut monitors = test_monitors(); + monitors.estop = Some(monitor.estop_trigger()); + monitors.safety = Some(monitor.clone()); + let running = Arc::new(AtomicBool::new(true)); + + // Released 5 ms late against a 1 ms deadline: a miss, so SafeMode + // dispatches `enter_safe_state()` — which panics. + RtExecutor::tick_node( + &mut node, &monitors, &running, false, 5_000_000, None, 0, None, None, None, + ); + + assert!( + monitor.is_emergency_stop(), + "a CRITICAL node that panicked trying to reach a safe state has not \ + reached one; stopping just that node leaves the robot running with \ + an unsafe actuator" + ); + } + struct CounterNode { name: String, count: Arc, @@ -4991,7 +5560,7 @@ mod tests { fn find_queued(range: std::ops::Range, pred: impl Fn(&str) -> bool) -> Option { let mut buf = [0u8; RT_DIAG_LINE_CAP]; for idx in range { - let Some(len) = read_slot(idx, &mut buf) else { + let SlotRead::Ready(len) = read_slot(idx, &mut buf) else { continue; }; if let Ok(text) = std::str::from_utf8(&buf[..len]) { @@ -5080,7 +5649,7 @@ mod tests { let mut buf = [0u8; RT_DIAG_LINE_CAP]; assert!( - read_slot(start, &mut buf).is_none(), + matches!(read_slot(start, &mut buf), SlotRead::Lapped), "after {flood} emits into {RT_DIAG_SLOTS} slots the first line must have \ been recycled — an unbounded queue is a memory leak on the RT path" ); diff --git a/horus_core/src/scheduling/safety_monitor.rs b/horus_core/src/scheduling/safety_monitor.rs index 7314bccaf..e5d1d69a5 100644 --- a/horus_core/src/scheduling/safety_monitor.rs +++ b/horus_core/src/scheduling/safety_monitor.rs @@ -116,7 +116,7 @@ static PENDING_LOCAL_ESTOP: std::sync::Mutex> = std::sync::Mutex: static ESTOP_QUEUE_SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); #[cfg(test)] -fn estop_queue_guard() -> std::sync::MutexGuard<'static, ()> { +pub(crate) fn estop_queue_guard() -> std::sync::MutexGuard<'static, ()> { ESTOP_QUEUE_SERIAL.lock().unwrap_or_else(|e| e.into_inner()) } @@ -532,9 +532,11 @@ pub(crate) struct NodeTimingReport { /// /// Correctness here rests on one invariant: a node is owned by exactly one /// executor, so exactly one thread ever WRITES its state. That is what makes -/// `record_tick`'s `store(0)` on `consecutive_misses` safe against +/// `record_deadline_met`'s `store(0)` on `consecutive_misses` safe against /// `record_miss`'s increment — they are the same thread, one tick apart, and -/// cannot interleave. +/// cannot interleave. (The store used to live in `record_tick`, which reset the +/// run on any tick that made its BUDGET; a node can make its budget and still +/// miss its deadline, so a met deadline is what ends the run.) /// /// The counters use `fetch_add`/`fetch_max` anyway, so a second writer would /// not lose a count; what it would break is the reset, which could be clobbered @@ -634,16 +636,22 @@ impl NodeTimingState { } } - // Reset the consecutive-miss run only on a tick that did NOT violate. + // The consecutive-miss run is NOT cleared here. // - // This used to run unconditionally at the TOP of the function, before - // the budget check. For a node that overruns its budget and misses its - // deadline on the same tick — the common case, since an overrunning - // tick is exactly what blows the deadline — the sequence was: clear to - // 0 here, then `record_miss` increments to 1. Pinned at 1 forever, so - // `evaluate_degradation`, whose ladder starts at 3 consecutive misses, - // could never fire for the nodes in the worst trouble. - self.consecutive_misses.store(0, Ordering::Release); + // It counts missed DEADLINES; this function knows only about the + // BUDGET, and the two are independent -- a node released late executes + // well inside its budget and still misses. Clearing from the budget + // check therefore zeroed the run on the very ticks it needed to count: + // such a node was reset to 0 here, incremented to 1 by `record_miss`, + // and pinned at 1 forever, so the ladder starting at 3 could never fire + // for exactly the node in the worst trouble. + // + // Moving the reset below the budget check (rather than removing it) + // only narrows that to the ticks that also overran, which is not the + // dominant case. A met deadline is the only thing that ends a run of + // missed deadlines, so `SafetyMonitor::record_deadline_met` owns it -- + // called from rt_executor.rs and scheduler/mod.rs on every tick that + // met one. See `check_deadline_from_release`. None } @@ -675,7 +683,9 @@ pub(crate) struct DegradationPolicy { pub(crate) reduce_after: u64, /// Consecutive misses before isolating node (default: 10) pub(crate) isolate_after: u64, - /// Consecutive misses before killing node — permanently removing from execution (default: 20) + /// Consecutive misses before killing node — permanently removing from + /// execution (default: 20, raised to `max_deadline_misses` when that + /// ceiling is higher, so the ceiling stays reachable — see `reaching`) pub(crate) kill_after: u64, /// Successful ticks at reduced rate before restoring original (default: 100) pub(crate) recovery_ticks: u64, @@ -693,6 +703,32 @@ impl Default for DegradationPolicy { } } +impl DegradationPolicy { + /// The default ladder, with its terminal rung moved out of the way of a + /// configured `max_deadline_misses` ceiling. + /// + /// `Kill` is permanent — it sets `is_stopped`, and a stopped node is + /// skipped forever — so a node killed at 20 consecutive misses can never + /// tick again, and its consecutive count can never climb any higher. With + /// `kill_after` fixed at 20 and `max_deadline_misses` defaulting to 100, + /// the documented emergency stop was unreachable in every shipped + /// configuration: the ladder always retired the node first. Nothing warned, + /// because the only setter for the ladder is `#[cfg(test)]`, so no user + /// build could raise the rung either. + /// + /// A hardcoded constant must not silently cap a documented safety + /// threshold, so the rung moves instead. Where the two now coincide the + /// e-stop wins, because both dispatch paths record the miss — which + /// evaluates the ceiling — before evaluating the ladder. + fn reaching(max_deadline_misses: u64) -> Self { + let base = Self::default(); + Self { + kill_after: base.kill_after.max(max_deadline_misses), + ..base + } + } +} + /// Current degradation stage for a node. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum DegradationStage { @@ -837,6 +873,17 @@ impl BudgetEnforcer { self.check_budget_of(node, &state, actual) } + /// Record that a node MET its deadline, clearing its consecutive-miss run. + /// + /// No allocation and no insert on the miss of the map: a node with no entry + /// has no run to clear, and this runs on the RT thread on every healthy + /// tick — by far the hottest of these paths. + pub(crate) fn record_deadline_met(&self, node: &str) { + if let Some(state) = self.node_timing.read().get(node) { + state.consecutive_misses.store(0, Ordering::Release); + } + } + /// Get timing stats for a specific node. #[cfg(test)] pub(crate) fn node_stats(&self, node: &str) -> Option { @@ -1019,7 +1066,7 @@ impl SafetyMonitor { deadline_misses: AtomicU64::new(0), max_deadline_misses, degrade_activations: AtomicU64::new(0), - degradation_policy: DegradationPolicy::default(), + degradation_policy: DegradationPolicy::reaching(max_deadline_misses), degradation_states: Mutex::new(HashMap::new()), degraded_nodes: AtomicUsize::new(0), external_safe_state_pending: AtomicBool::new(false), @@ -1303,11 +1350,36 @@ impl SafetyMonitor { result } + /// Record that a node met its deadline, clearing its consecutive-miss run. + /// + /// The counterpart to `record_deadline_miss`, and the ONLY thing that + /// clears the run. Call it on every tick whose deadline check came back + /// clean, from both dispatch paths. + /// + /// This used to be a side effect of `check_budget`, which is the wrong + /// contract twice over. A budget overrun and a deadline miss are different + /// failures — a node woken late can blow its deadline having executed in a + /// tenth of its budget — so an in-budget tick is no evidence the deadline + /// was met. And neither production caller ran it on a healthy tick anyway: + /// the main loop calls `check_tick_budget` only from inside the + /// budget-violation branch, and a node with a deadline and no budget skips + /// that block entirely, so on those two paths nothing ever cleared the + /// count and `max_deadline_misses` was a lifetime total wearing the name of + /// a consecutive one. + pub(crate) fn record_deadline_met(&self, node_name: &str) { + self.budget_enforcer.record_deadline_met(node_name); + } + /// Record a deadline miss with severity tracking. /// /// Tracks per-node: total misses, consecutive misses, worst miss duration. - /// Triggers emergency stop only when the process-wide total crosses - /// `max_deadline_misses`. + /// Triggers an emergency stop only when THIS node's CONSECUTIVE run reaches + /// `max_deadline_misses`; the run is cleared by `record_deadline_met` on + /// any tick that met its deadline. The process-wide `deadline_misses` total + /// is kept for the timing report and triggers nothing — it used to be the + /// e-stop trigger, and as a lifetime count across every node that was + /// indefensible on a real robot. See the note in + /// `record_deadline_miss_with_severity`. pub(crate) fn record_deadline_miss(&self, node_name: &str) { self.record_deadline_miss_with_severity(node_name, 0); } @@ -1338,7 +1410,8 @@ impl SafetyMonitor { // is not keeping up right now. A run of misses is a node in trouble; a // scattered few that each recover are jitter, and jitter on a stock // kernel is not a reason to stop a robot. `consecutive_misses` is reset - // by `check_budget` on any tick that did not violate, so recovery + // by `record_deadline_met` on any tick that MET its deadline, on both + // dispatch paths and whether or not the node has a budget, so recovery // genuinely clears the count. let consecutive = { let state = self.budget_enforcer.handle(node_name); @@ -1414,6 +1487,15 @@ impl SafetyMonitor { .get_mut(node_name) .expect("inserted immediately above"); + // Every caller reaches this only after recording a deadline miss, so + // the run of recovery ticks is over. `record_successful_tick` means + // "the tick did not panic" and both callers reach it on a tick already + // recorded as a miss in the same pass; without this, a node that missed + // EVERY deadline still accumulated `recovery_ticks` of "success" and + // was restored to the rate it had just proved it could not hold. The + // field's own comment already claimed to be a consecutive run. + state.recovery_counter = 0; + let action = if consecutive_misses >= policy.kill_after && state.stage != DegradationStage::Killed { @@ -2322,9 +2404,17 @@ mod tests { assert_eq!(state.consecutive_misses(), 3); assert!(state.is_chronic()); - // Successful tick resets consecutive counter + // A tick inside budget is NOT a reset: this state knows only about the + // budget, and a node released late misses its deadline having executed + // well inside it. Clearing here pinned the run at 1 for exactly that + // node. Only a met DEADLINE ends the run, via `record_deadline_met`. state.record_tick(10_u64.us()); - assert_eq!(state.consecutive_misses(), 0); + assert_eq!(state.consecutive_misses(), 3); + assert!(state.is_chronic()); + + // What `record_deadline_met` does, which is the only thing that ends + // the run. + state.consecutive_misses.store(0, Ordering::Release); assert!(!state.is_chronic()); // But total misses persist assert_eq!(state.total_deadline_misses(), 3); @@ -2873,6 +2963,81 @@ mod tests { /// there eventually whatever its health. That is a timer wearing a safety /// threshold's name. /// + /// The configured `max_deadline_misses` ceiling must be reachable. + /// + /// `DegradationAction::Kill` sets `is_stopped`, and a stopped node is + /// skipped forever on both dispatch paths — so a node the ladder kills at + /// 20 consecutive misses can never tick again, and its consecutive count + /// can never climb any higher. With `kill_after` hardcoded at 20 and the + /// ceiling defaulting to 100, the emergency stop this knob names could not + /// fire in ANY shipped configuration, and the ladder's only setter is + /// `#[cfg(test)]`, so no user build could raise the rung either. + /// + /// The assertion is on the ORDER of the two terminal events, not on the + /// counter: at this level nothing removes the node, so the count keeps + /// climbing and the e-stop would fire on its own. What made the ceiling + /// dead is the `Kill` that lands first. + #[test] + fn the_ladder_does_not_retire_a_node_before_its_configured_ceiling() { + let _estop_serial = super::estop_queue_guard(); + // Above the ladder's default terminal rung of 20 — as the shipped + // default of 100 also is. + let ceiling = 25u64; + let monitor = SafetyMonitor::new(ceiling); + + for i in 1..=ceiling { + monitor.record_deadline_miss("arm_controller"); + let consecutive = monitor.consecutive_misses("arm_controller"); + let action = monitor.evaluate_degradation("arm_controller", consecutive, Some(100.0)); + + if !monitor.is_emergency_stop() { + assert!( + !matches!(action, DegradationAction::Kill(_)), + "the ladder killed the node at {} consecutive misses. It can \ + never tick again, so it can never reach the configured \ + ceiling of {} and the emergency stop that number names is \ + unreachable", + i, + ceiling + ); + } + } + + assert!( + monitor.is_emergency_stop(), + "{} consecutive misses is the configured ceiling — the emergency \ + stop must fire", + ceiling + ); + } + + /// The same thing at the value users actually ship: the documented default. + #[test] + fn the_default_ceiling_is_reachable() { + let _estop_serial = super::estop_queue_guard(); + let default_ceiling = super::super::config::SchedulerConfig::default() + .realtime + .max_deadline_misses; + let monitor = SafetyMonitor::new(default_ceiling); + + for i in 1..=default_ceiling { + monitor.record_deadline_miss("arm_controller"); + let consecutive = monitor.consecutive_misses("arm_controller"); + let action = monitor.evaluate_degradation("arm_controller", consecutive, Some(100.0)); + if !monitor.is_emergency_stop() { + assert!( + !matches!(action, DegradationAction::Kill(_)), + "the default ladder retired the node at {} of the default \ + ceiling's {} consecutive misses", + i, + default_ceiling + ); + } + } + + assert!(monitor.is_emergency_stop()); + } + /// Here one node misses ten times the ceiling, recovering between each. It /// must not stop the robot. #[test] @@ -2884,8 +3049,12 @@ mod tests { for _ in 0..(ceiling * 10) { monitor.record_deadline_miss("wheel_controller"); - // A tick inside budget: `check_budget` clears the consecutive run. - let _ = monitor.check_tick_budget("wheel_controller", Duration::from_micros(100)); + // The next tick MET its deadline, which is what ends the run. This + // used to be spelled `check_tick_budget(.., in_budget_duration)`, + // asserting a contract the runtime does not hold: an in-budget tick + // is not a met deadline, and neither production caller reached that + // call on a healthy tick anyway. + monitor.record_deadline_met("wheel_controller"); } assert!( @@ -2932,6 +3101,124 @@ mod tests { assert!(monitor.is_emergency_stop()); } + /// The consecutive-miss run must count DEADLINE misses, not budget overruns. + /// + /// This is the RT executor's exact call order for the failure mode + /// `check_deadline_from_release` was written to catch: a node woken late + /// that then executes well inside its budget. `rt_executor.rs` calls + /// `check_tick_budget` unconditionally (:1273) and the deadline check + /// afterwards (:1356), so every one of these ticks is in budget and every + /// one misses its deadline. + /// + /// The reset used to live in `check_budget`, so the in-budget tick cleared + /// the run the deadline miss had just started: the counter was pinned at 1 + /// forever. Neither `warn_after: 3` nor the `max_deadline_misses` ceiling + /// could ever fire for the failure mode that, by `primitives.rs`'s own + /// account, "actually dominates". + #[test] + fn an_in_budget_tick_does_not_clear_a_deadline_miss_run() { + let monitor = SafetyMonitor::new(1000); + monitor.set_tick_budget("arm_controller".to_string(), Duration::from_millis(1)); + + for _ in 0..10 { + // Well inside the 1 ms budget — the budget check sees nothing wrong. + let _ = monitor.check_tick_budget("arm_controller", Duration::from_micros(100)); + // ...and yet the deadline, measured from the scheduled release, was missed. + monitor.record_deadline_miss("arm_controller"); + } + + assert_eq!( + monitor.consecutive_misses("arm_controller"), + 10, + "ten consecutive deadline misses must read as ten; a tick that stayed \ + inside its BUDGET says nothing about whether it met its DEADLINE" + ); + } + + /// The other half of the contract: a tick that meets its deadline clears + /// the run, for a node that has a deadline and no budget at all. + /// + /// `node_builder.rs` derives a deadline from a budget and never the + /// reverse, so this node's `tick_budget` is `None` and the main loop's + /// whole `if let Some(tick_budget)` block — the only place that used to + /// clear the counter — is skipped. Before `record_deadline_met` existed + /// there was no path that cleared it, and the count was a lifetime total: + /// a node missing once an hour reached any ceiling eventually. + #[test] + fn a_tick_that_meets_its_deadline_clears_the_run_without_any_budget() { + let monitor = SafetyMonitor::new(1000); + // Deliberately no `set_tick_budget`. + + for _ in 0..10 { + monitor.record_deadline_miss("wheel_controller"); + monitor.record_deadline_met("wheel_controller"); + } + + assert_eq!( + monitor.consecutive_misses("wheel_controller"), + 0, + "a node that misses and immediately recovers, ten times over, is \ + jitter — its consecutive run is zero, not ten" + ); + } + + /// A tick that missed its deadline is not evidence of recovery. + /// + /// `record_successful_tick` means "did not panic", and both callers reach + /// it on a tick already recorded as a deadline miss in the same pass. With + /// nothing clearing `recovery_counter` on a miss, a node at `RateReduced` + /// that missed EVERY deadline still accumulated `recovery_ticks` of + /// "success" and was restored to the rate it had just demonstrated it + /// could not hold. + #[test] + fn a_node_that_never_meets_a_deadline_is_never_declared_recovered() { + let mut monitor = SafetyMonitor::new(10_000); + // isolate_after is out of reach so the node stays at RateReduced and + // the test observes recovery, not a stage transition. + monitor.set_degradation_policy(DegradationPolicy { + warn_after: 3, + reduce_after: 5, + isolate_after: 10_000, + kill_after: 20_000, + recovery_ticks: 10, + }); + + let miss_tick = |m: &SafetyMonitor| { + m.record_deadline_miss("arm_controller"); + let consecutive = m.consecutive_misses("arm_controller"); + m.evaluate_degradation("arm_controller", consecutive, Some(100.0)); + // The RT executor reaches this on any tick whose closure returned + // `Ok` — including the one just recorded as a miss. + m.record_successful_tick("arm_controller") + }; + + for _ in 0..5 { + miss_tick(&monitor); + } + assert_eq!( + monitor.degradation_stage("arm_controller"), + DegradationStage::RateReduced, + "five consecutive misses must reduce the rate" + ); + + // Keep missing, for twice the recovery window. + for i in 0..(10 * 2) { + let action = miss_tick(&monitor); + assert!( + !matches!(action, DegradationAction::RestoreRate { .. }), + "tick {} missed its deadline like every tick before it — \ + restoring its full rate says it recovered when it never did", + i + ); + } + + assert_eq!( + monitor.degradation_stage("arm_controller"), + DegradationStage::RateReduced, + "a node that has not met a deadline yet must still be degraded" + ); + } + // ======================================================================== // Stress and multi-node tests // ======================================================================== @@ -4241,7 +4528,7 @@ mod budget_atomics_correctness { /// counter, so a dropped sample leaves it stale-high and the NEXT miss can /// cross a ceiling it had no business crossing. #[test] - fn a_clean_tick_clears_the_miss_run_and_prevents_a_spurious_stop() { + fn a_met_deadline_clears_the_miss_run_and_prevents_a_spurious_stop() { let m = SafetyMonitor::new(3); m.set_tick_budget("servo".to_string(), Duration::from_micros(800)); let state = m.tick_budget_handle("servo"); @@ -4252,13 +4539,26 @@ mod budget_atomics_correctness { } assert_eq!(state.consecutive_misses(), 2); - // A tick well inside budget, taken the way the RT loop takes it. + // A tick well inside BUDGET does not end a run of missed DEADLINES. + // The two are independent: a node released late executes well inside + // its budget and still misses. Clearing here is what pinned such a + // node's run at 1 and kept the ladder, which starts at 3, from ever + // firing for exactly the node in the worst trouble. m.check_tick_budget_of("servo", &state, Duration::from_micros(10)) .expect("10us is inside an 800us budget"); + assert_eq!( + state.consecutive_misses(), + 2, + "a tick inside budget is not evidence the deadline was met" + ); + + // Met deadline, the way both loops report one (rt_executor.rs:1462, + // scheduler/mod.rs:5715). This is the only thing that ends the run. + m.record_deadline_met("servo"); assert_eq!( state.consecutive_misses(), 0, - "a clean tick must clear the run" + "a met deadline must clear the run" ); // So the next two misses must NOT reach the ceiling. diff --git a/horus_core/src/scheduling/scheduler/mod.rs b/horus_core/src/scheduling/scheduler/mod.rs index 568decbdb..686967af0 100644 --- a/horus_core/src/scheduling/scheduler/mod.rs +++ b/horus_core/src/scheduling/scheduler/mod.rs @@ -1,4 +1,5 @@ use crate::core::hlog::{clear_node_context, set_node_context}; +use crate::core::log_buffer::start_log_file_drain_once; use crate::core::tick_context::{clear_tick_context, set_tick_context}; use crate::core::{DurationExt, NodeInfo, NodePresence}; use crate::error::{HorusContext, HorusResult}; @@ -709,6 +710,21 @@ impl Scheduler { s = s.with_recording(); } + // Drain the log ring buffer to disk if HORUS_LOG_FILE is set. + // + // `start_log_file_drain` reads the variable itself and returns `None` + // when it is unset, so this call is unconditional and users who do not + // opt in pay nothing. That guard lives INSIDE the function precisely so + // a call site could be unconditional — but there was no call site. The + // symbol appeared four times in the tree: the definition, its + // re-export, and two lines of one integration test that invoked it by + // hand. So `HORUS_LOG_FILE=1 horus run` wrote no `.horus/logs/horus.log` + // and `HORUS_LOG_DIR`/`_MAX_SIZE`/`_MAX_FILES` were dead with it, while + // the test that proves the drain works kept it green. An operator who + // sets a documented variable to get a post-incident log and finds an + // empty disk is the exact failure that variable exists to prevent. + start_log_file_drain_once(); + s } @@ -873,18 +889,27 @@ impl Scheduler { // feature, and no watchdog is not. Say so, because silently changing a // safety timeout is its own problem — the operator needs to know the // value actually in force. - let ms = timeout.as_millis() as u64; + // + // That rule held for the sub-1ms case and was inverted for every other + // one: `as_millis()` truncates, so `.watchdog(1500_u64.us())` stored 1ms + // and its 3x critical rung latched a fleet-wide e-stop at 3ms instead of + // the 4.5ms asked for, with no diagnostic. Tightening is the direction + // that produces spurious halts on a healthy node. Round the whole range + // up, and report any value that had to change. + let ms = timeout.as_nanos().div_ceil(1_000_000) as u64; self.pending_config.realtime.watchdog_timeout_ms = if timeout.is_zero() { 0 // explicit disable — honour it - } else if ms == 0 { + } else if !timeout.subsec_nanos().is_multiple_of(1_000_000) || ms == 0 { + let effective = Duration::from_millis(ms.max(1)); crate::hlog!( warn, - "watchdog({:?}) is below the 1ms resolution the scheduler config stores; \ - using 1ms. Sub-millisecond watchdogs are not supported — previously this \ - truncated to 0, which DISABLED the watchdog entirely.", - timeout + "watchdog({:?}) is not a whole number of milliseconds, which is all the \ + scheduler config stores; using {:?}. Rounded UP on purpose: a looser \ + watchdog is a safety feature, a tighter one halts a healthy robot.", + timeout, + effective ); - 1 + ms.max(1) } else { ms }; @@ -907,17 +932,54 @@ impl Scheduler { self.pending_config.monitoring.black_box_size_mb = size_mb; let bb_dir = self.monitor.working_dir.join(".horus").join("blackbox"); let bb = super::blackbox::BlackBox::new(size_mb).with_path(bb_dir); - self.monitor.blackbox = Some(Arc::new(Mutex::new(bb))); + let shared = Arc::new(Mutex::new(bb)); + + // Wire the external-event hook. `set_blackbox_hook` says "Called by the + // Scheduler at startup to wire the blackbox" and nothing called it, so + // `record_external_event` -- the entry point horus_net's replicator uses + // for peer-lost, replication-failure and desync events -- looked up an + // unset OnceLock and dropped every event. A distributed robot's flight + // recorder therefore held nothing about the network, which is the part + // of a distributed incident you most want a timeline of. + // + // OnceLock: the first scheduler to enable a blackbox owns the hook for + // the life of the process. A second scheduler's `.blackbox()` still gets + // its own recorder for local events; only external ones stay with the + // first. That is the existing contract of a process-global hook, not a + // new limitation. + let hook_target = Arc::clone(&shared); + super::blackbox::set_blackbox_hook(move |event| { + if let Ok(mut bb) = hook_target.lock() { + bb.record(event); + } + }); + + self.monitor.blackbox = Some(shared); // The producer side, handed to the executors. Created with the recorder // and never separately: an executor holding a ring the drain does not // know about would queue events nothing ever reads. + // + // The hook above and this ring reach the same recorder by different + // routes and that is deliberate: the hook is for events raised off the + // tick path (horus_net's replicator), the ring is for events raised ON + // it, which must not touch a mutex or serde where they are produced. self.monitor.blackbox_ring = Some(Arc::new(super::blackbox_ring::BbRing::new())); self } /// Set the maximum number of deadline misses before emergency stop. /// - /// Default: 100. Only meaningful when nodes have `.rate()` set. + /// Default: 100. Per node and consecutive: a tick that meets its deadline + /// clears the run, so a node that misses intermittently and recovers never + /// reaches the ceiling. + /// + /// The graduated degradation ladder (warn at 3 consecutive misses, half + /// rate at 5, isolate at 10) runs below this. Its terminal rung — killing + /// the node outright, normally at 20 — is moved above `n` when `n` is + /// higher, so this ceiling is always reachable; killing the node is + /// permanent, and a killed node cannot go on to miss anything. + /// + /// Only meaningful when nodes have `.rate()` set. /// /// # Example /// ```rust,ignore @@ -5072,6 +5134,11 @@ impl Scheduler { /// Process control commands — delegates to execution.rs topic-based implementation. fn process_control_commands(&mut self) { + // Names asked to restart. Collected rather than applied in place because + // the loop below holds a mutable borrow of `self.control_topic`, and + // clearing `initialized` needs `self.nodes`. + let mut restart_requests: Vec = Vec::new(); + // Poll the control topic for commands from CLI (horus node kill/pause/resume) if let Some(ref mut ctl) = self.control_topic { while let Some(cmd) = ctl.recv() { @@ -5098,10 +5165,72 @@ impl Scheduler { self.running .store(false, std::sync::atomic::Ordering::SeqCst); } + ControlCommand::RestartNode { ref name } => { + restart_requests.push(name.clone()); + } _ => {} // Other commands (SetNodeRate, etc.) — not yet implemented } } } + + // Clearing `initialized` is the whole mechanism: `reinit_pending_nodes` + // runs on the tick loop and re-runs `init()` for any node that is not + // stopped, not transiently skipped, and not initialised. + // + // Two different pauses are cleared below and they are not the same + // thing, which this comment used to blur. `registered.is_paused` is the + // one-tick `Miss::Skip` flag and IS what `reinit_pending_nodes` reads; + // the operator pause lives in `node_controls` and is not read there at + // all. So lifting the operator pause is not what lets the re-init + // happen — it is what lets the node TICK once it has been re-inited, + // which matters because a wedged node is usually paused before it is + // restarted and would otherwise come back initialised and still frozen. + for name in restart_requests { + if let Some(ref controls) = self.node_controls { + controls.set_paused(&name, false); + } + // The scheduler's own `nodes` holds ONLY the main-thread group + // after the class partition, so searching it alone reached + // BestEffort nodes and reported every RT, Compute, Event and + // AsyncIo node as unknown — that is, every node with a `.rate()`, + // which is exactly what an operator restarts. + // + // `init()` needs `&mut dyn Node`, owned by the executor, so a node + // this scheduler does not hold is asked through the shared control + // map and the owning executor honours it — the same handoff the + // watchdog ladder uses for `enter_safe_state()`. + match self + .nodes + .iter_mut() + .find(|n| n.name.as_ref() == name.as_str()) + { + Some(registered) => { + registered.initialized = false; + registered.is_paused = false; + print_line(&format!( + "[CONTROL] Node '{}' will re-initialize on the next tick", + name + )); + } + None => { + let asked = self + .node_controls + .as_ref() + .is_some_and(|c| c.request_restart(&name)); + if asked { + print_line(&format!( + "[CONTROL] Node '{}' will re-initialize on its executor", + name + )); + } else { + print_line(&format!( + "[CONTROL] Restart requested for unknown node '{}'", + name + )); + } + } + } + } } /// Clean up session directory (no-op with flat namespace) @@ -5862,6 +5991,15 @@ impl Scheduler { if let Some(deadline) = self.nodes[i].deadline { let miss = TimingEnforcer::check_deadline(tick_start, deadline, self.nodes[i].miss_policy); + if miss.is_none() { + // The tick met its deadline: end the consecutive-miss run. + // This is the only path that clears it, and it has to run + // for every RT node with a deadline — including one with no + // tick budget, which never reaches the budget block above. + if let Some(ref monitor) = self.monitor.safety { + monitor.record_deadline_met(&node_name); + } + } if miss.is_none() && self.nodes[i].in_safe_mode { // See the RT executor: a latch that never clears turns // Miss::SafeMode into a one-shot for the process lifetime. diff --git a/horus_core/src/scheduling/scheduler/recording.rs b/horus_core/src/scheduling/scheduler/recording.rs index e516b4759..d8077100b 100644 --- a/horus_core/src/scheduling/scheduler/recording.rs +++ b/horus_core/src/scheduling/scheduler/recording.rs @@ -193,6 +193,16 @@ impl Scheduler { }) .unwrap_or_default(); + // How many nodes we could not place. The lookup below is keyed by NODE + // NAME, and it used to be handed `node_id` — a generated hex string + // (`format!("{:x}{:x}", unix_nanos, node_count)`) that can never equal + // a name. Every lookup missed, `unwrap_or(0)` gave every node priority + // 0, and `add_replay`'s stable sort then left the replay in + // `node_recordings` HashMap order: neither the recorded order nor the + // same order twice. That is the exact opposite of the "tick-perfect + // determinism" this module is for, and nothing said a word. + let mut unplaced: Vec<&str> = Vec::new(); + for (node_id, relative_path) in &scheduler_recording.node_recordings { // `relative_path` comes verbatim out of the recording file. Joined // unchecked, `../../../..` escapes the session directory entirely @@ -203,7 +213,23 @@ impl Scheduler { crate::scheduling::record_replay::sanitize_path_component(relative_path), ); if node_path.exists() { - let priority = priority_map.get(node_id.as_str()).copied().unwrap_or(0); + // The name is the part of `@.horus` before the LAST + // `@`, which is how `Scheduler::add` writes it — `rsplit_once` + // so a node whose own name contains `@` still resolves. + let node_name = relative_path + .strip_suffix(".horus") + .unwrap_or(relative_path) + .rsplit_once('@') + .map(|(name, _id)| name) + .unwrap_or(relative_path); + + let priority = match priority_map.get(node_name) { + Some(p) => *p, + None => { + unplaced.push(node_name); + 0 + } + }; if let Err(e) = scheduler.add_replay(node_path, priority) { print_line(&format!( "Warning: Failed to load node '{}': {}", @@ -213,6 +239,18 @@ impl Scheduler { } } + // Silence is what let the by-id lookup survive: every node missing is + // indistinguishable from every node placed, because both end at + // priority 0. Say so, once, with the names. + if !unplaced.is_empty() && !priority_map.is_empty() { + print_line(&format!( + "[REPLAY] Warning: {} node(s) are not in the recorded execution \ + order and will replay in an arbitrary position: {}", + unplaced.len(), + unplaced.join(", ") + )); + } + Ok(scheduler) } diff --git a/horus_core/src/scheduling/scheduler/tests.rs b/horus_core/src/scheduling/scheduler/tests.rs index faf038908..177a91e78 100644 --- a/horus_core/src/scheduling/scheduler/tests.rs +++ b/horus_core/src/scheduling/scheduler/tests.rs @@ -3075,6 +3075,53 @@ fn test_builder_order_independence() { ); } +/// A watchdog timeout is rounded UP, never tightened. +/// +/// The config stores whole milliseconds. `as_millis()` truncates, so +/// `.watchdog(1500_u64.us())` used to store 1 ms — and the graduated ladder +/// latches a system-wide emergency stop past 3x the window, so it fired at 3 ms +/// instead of the 4.5 ms the operator asked for, with no diagnostic. +/// +/// Tightening is the dangerous direction: it halts a healthy robot. The builder +/// already argued exactly this for the sub-1 ms case — "a slightly-looser +/// watchdog is a safety feature, and no watchdog is not" — and then truncated +/// every other value. +#[test] +fn a_watchdog_timeout_is_never_silently_tightened() { + let _guard = lock_scheduler(); + + // The case the old code got right. + let sub_ms = Scheduler::new().watchdog(500_u64.us()); + assert_eq!( + sub_ms.pending_config.realtime.watchdog_timeout_ms, 1, + "a sub-millisecond watchdog must not round to 0, which disables it" + ); + + // The case it got backwards. + let fractional = Scheduler::new().watchdog(1500_u64.us()); + assert_eq!( + fractional.pending_config.realtime.watchdog_timeout_ms, 2, + "1.5 ms must round UP to 2 ms; truncating to 1 ms makes the 3x \ + emergency-stop rung fire at 3 ms instead of 4.5 ms" + ); + + let just_over = Scheduler::new().watchdog(Duration::from_micros(2_001)); + assert_eq!( + just_over.pending_config.realtime.watchdog_timeout_ms, 3, + "any remainder rounds up — the stored window must never be shorter \ + than the one that was asked for" + ); + + // Whole milliseconds are untouched, and an explicit zero still disables. + let whole = Scheduler::new().watchdog(500_u64.ms()); + assert_eq!(whole.pending_config.realtime.watchdog_timeout_ms, 500); + let disabled = Scheduler::new().watchdog(Duration::ZERO); + assert_eq!( + disabled.pending_config.realtime.watchdog_timeout_ms, 0, + "an explicit zero is a disable and must be honoured" + ); +} + // ============================================================================ // Phase 6: Graduated Safety Monitor Response // ============================================================================ @@ -5709,6 +5756,87 @@ fn shared_monitors_carry_a_safety_handle_for_executors() { ); } +/// The main loop must clear a node's consecutive-miss run on a tick that MET +/// its deadline — and this node has a deadline and no tick budget, so it never +/// enters the budget block that used to nominally own the reset. +/// +/// `node_builder` derives a deadline from a budget and never the reverse, so +/// this shape is ordinary: `.deadline(..)` with no `.tick_budget(..)`. For it, +/// `check_timing_violations` skipped the whole `if let Some(tick_budget)` +/// block, nothing ever called `check_budget`, and `consecutive_misses` was a +/// lifetime total. A node missing once an hour and recovering instantly +/// reached any ceiling eventually — the exact "timer wearing a safety +/// threshold's name" the per-node ceiling was introduced to end. +struct AlternatingNode { + name: String, + ticks: Arc, +} + +impl Node for AlternatingNode { + fn name(&self) -> &str { + &self.name + } + + fn tick(&mut self) { + // Miss the deadline on every other tick, comfortably meeting it in + // between. Never two misses in a row. + if self.ticks.fetch_add(1, Ordering::SeqCst).is_multiple_of(2) { + std::thread::sleep(Duration::from_millis(5)); + } + } +} + +#[test] +fn a_met_deadline_clears_the_miss_run_for_a_node_with_no_budget() { + let _guard = lock_scheduler(); + let ticks = Arc::new(AtomicUsize::new(0)); + + let mut scheduler = Scheduler::new() + .tick_rate(100_u64.hz()) + // Keeps the RT node on the main thread, so this exercises + // `check_timing_violations` rather than the RT executor. + .deterministic(true); + scheduler + .add(AlternatingNode { + name: "flaky".to_string(), + ticks: ticks.clone(), + }) + .deadline(1_u64.ms()) // deliberately no .tick_budget() + .build(); + + let _ = scheduler.run_for(400_u64.ms()); + + let monitor = scheduler + .monitor + .safety + .as_ref() + .expect("a node with a deadline produces a safety monitor"); + + // Anti-vacuity: the node must actually have missed, repeatedly, or the + // assertion below passes for the wrong reason. + let total_misses = monitor + .all_node_timing() + .into_iter() + .find(|r| r.name == "flaky") + .map(|r| r.deadline_misses) + .unwrap_or(0); + assert!( + total_misses >= 3, + "precondition: the node must miss its deadline repeatedly for this \ + test to mean anything (saw {} misses in {} ticks)", + total_misses, + ticks.load(Ordering::SeqCst) + ); + + assert!( + monitor.consecutive_misses("flaky") <= 1, + "the node met its deadline between every miss, so its CONSECUTIVE run \ + is at most 1 — it read {} because nothing on this path ever cleared \ + it, making a per-node consecutive ceiling a lifetime total", + monitor.consecutive_misses("flaky") + ); +} + #[cfg(test)] mod watchdog_message_honesty { use super::super::watchdog_critical_message; diff --git a/horus_core/src/scheduling/types.rs b/horus_core/src/scheduling/types.rs index 0da46daf6..a82754b82 100644 --- a/horus_core/src/scheduling/types.rs +++ b/horus_core/src/scheduling/types.rs @@ -848,6 +848,17 @@ pub struct NodeControl { /// would honour the request, so it will not be safed by its own executor. /// That case is what the e-stop escalation exists for. pub safe_state_requested: std::sync::atomic::AtomicBool, + /// Set by the main thread when `horus node restart` names a node the + /// scheduler does not own; cleared by the executor that does, which then + /// re-runs `init()` on it. + /// + /// Same reason as `safe_state_requested`: `init()` needs `&mut dyn Node`, + /// and after the class partition the scheduler's own `nodes` holds only the + /// main-thread group. Restart handled solely against that vector reached + /// BestEffort nodes and silently reported every RT, Compute, Event and + /// AsyncIo node as unknown — which is every node with a `.rate()`, i.e. + /// exactly the ones an operator restarts. + pub restart_requested: std::sync::atomic::AtomicBool, } impl NodeControlMap { @@ -859,6 +870,7 @@ impl NodeControlMap { stopped: std::sync::atomic::AtomicBool::new(false), health: AtomicHealthState::default(), safe_state_requested: std::sync::atomic::AtomicBool::new(false), + restart_requested: std::sync::atomic::AtomicBool::new(false), }) }); } @@ -951,6 +963,40 @@ impl NodeControlMap { } } + /// Ask the executor that owns `name` to re-run its `init()`. + /// + /// Returns whether the node is known to the control map at all, so the + /// caller can tell "asked" from "no such node" — the control map is + /// registered for all five execution groups, so an unknown name here really + /// is unknown. + pub fn request_restart(&self, name: &str) -> bool { + self.inner + .read() + .unwrap_or_else(|e| e.into_inner()) + .get(name) + .map(|c| { + c.restart_requested + .store(true, std::sync::atomic::Ordering::Release); + true + }) + .unwrap_or(false) + } + + /// Consume a pending restart request, returning whether one was set. + /// + /// Clears the flag, so a request is honoured exactly once per raise. + pub fn take_restart_request(&self, name: &str) -> bool { + self.inner + .read() + .unwrap_or_else(|e| e.into_inner()) + .get(name) + .map(|c| { + c.restart_requested + .swap(false, std::sync::atomic::Ordering::AcqRel) + }) + .unwrap_or(false) + } + /// Consume a pending safing request, returning whether one was set. /// /// Clears the flag, so a request is honoured exactly once per raise. diff --git a/horus_core/src/testing/mock_topic.rs b/horus_core/src/testing/mock_topic.rs index cf0b5b1fd..db2e14efb 100644 --- a/horus_core/src/testing/mock_topic.rs +++ b/horus_core/src/testing/mock_topic.rs @@ -51,8 +51,20 @@ pub struct MockTopic { name: String, buffer: Mutex>, config: MockTopicConfig, + /// Send ATTEMPTS. Drives the fault-injection schedules, and matches what + /// the real transport reports as `messages_sent` — which counts every + /// `send()` before dispatch, so a message dropped on a full ring counts. send_count: AtomicU64, + /// Receive ATTEMPTS. Drives `recv_fail_after` / `recv_fail_every_n`. recv_count: AtomicU64, + /// Messages actually handed back by `recv`/`try_recv`. + /// + /// This is what `messages_received` reports, because that is what the real + /// transport counts: it increments only when `recv()` returns `Some`. The + /// mock used to report its ATTEMPT count there, so a double whose whole + /// purpose is to behave like the transport disagreed with it the moment a + /// recv came back empty — which is most polls on an idle topic. + recv_delivered: AtomicU64, send_failures: AtomicU64, recv_failures: AtomicU64, } @@ -66,6 +78,7 @@ impl MockTopic { config, send_count: AtomicU64::new(0), recv_count: AtomicU64::new(0), + recv_delivered: AtomicU64::new(0), send_failures: AtomicU64::new(0), recv_failures: AtomicU64::new(0), } @@ -161,7 +174,11 @@ impl MockTopic { } let mut buf = self.buffer.lock().unwrap(); - buf.pop_front() + let got = buf.pop_front(); + if got.is_some() { + self.recv_delivered.fetch_add(1, Ordering::Relaxed); + } + got } /// Read the most recent message without consuming it. @@ -186,7 +203,7 @@ impl MockTopic { pub fn metrics(&self) -> TopicMetrics { TopicMetrics::new( self.send_count.load(Ordering::Relaxed), - self.recv_count.load(Ordering::Relaxed), + self.recv_delivered.load(Ordering::Relaxed), self.send_failures.load(Ordering::Relaxed), self.recv_failures.load(Ordering::Relaxed), // A mock never enters the retry cold path, so it can never overrun @@ -230,6 +247,37 @@ mod tests { // ── Basic operations ── + /// The double must report the same quantities the transport does. + /// + /// `messages_received` used to be the mock's recv ATTEMPT count, while the + /// real transport increments only when `recv()` hands a message back. So a + /// test written against the mock and then run against a real topic + /// disagreed the moment a poll came back empty — which is most polls on an + /// idle topic. A double that reports what the real thing does not is the + /// one defect a double cannot have. + #[test] + fn metrics_count_deliveries_not_poll_attempts() { + let t = MockTopic::simple("parity"); + t.send(1u64); + t.send(2u64); + + // Two deliveries, then three empty polls. + assert_eq!(t.try_recv(), Some(1)); + assert_eq!(t.try_recv(), Some(2)); + for _ in 0..3 { + assert_eq!(t.try_recv(), None); + } + + let m = t.metrics(); + assert_eq!(m.messages_sent(), 2, "sends are counted as attempts"); + assert_eq!( + m.messages_received(), + 2, + "only the two DELIVERED messages count — the three empty polls are \ + not receipts, and the real transport does not count them" + ); + } + #[test] fn mock_topic_send_recv_roundtrip() { let topic: MockTopic = MockTopic::simple("test"); diff --git a/horus_core/tests/blackbox_intent.rs b/horus_core/tests/blackbox_intent.rs index d45eb74b5..08303b50d 100644 --- a/horus_core/tests/blackbox_intent.rs +++ b/horus_core/tests/blackbox_intent.rs @@ -224,3 +224,54 @@ fn test_blackbox_intent_anomaly_detection() { ); } } + +// ============================================================================ +// The external-event hook +// ============================================================================ + +/// INTENT: "An event recorded from outside the scheduler reaches the recorder." +/// +/// `record_external_event` is how a subsystem that does not own the scheduler +/// -- horus_net's replicator, which reports peer loss, replication failure and +/// desync -- gets into the flight recorder. It forwards to a process-global +/// hook, and `set_blackbox_hook` documents itself as "Called by the Scheduler +/// at startup to wire the blackbox". +/// +/// Nothing called it. The hook stayed unset for the life of every process, so +/// `record_external_event` looked up an empty `OnceLock` and dropped the event +/// on the floor. A distributed robot's recorder held a complete local timeline +/// and nothing at all about the network, which is the half of a distributed +/// incident you most need. +/// +/// Enabling `.blackbox()` is the user's request for a flight recorder, so that +/// is where the wiring belongs. This asserts through the public surface: build +/// a scheduler with a recorder, emit an external event, read it back. +#[cfg(feature = "blackbox")] +#[test] +fn an_external_event_reaches_a_scheduler_that_enabled_its_blackbox() { + use horus_core::core::DurationExt; + use horus_core::scheduling::{record_external_event, Scheduler}; + + let scheduler = Scheduler::new().tick_rate(100_u64.hz()).blackbox(1); + + record_external_event(BlackBoxEvent::NodeError { + name: "peer_link".to_string(), + error: "replication peer lost".to_string(), + severity: horus_core::error::Severity::Permanent, + }); + + let recorder = scheduler + .get_blackbox() + .expect(".blackbox(1) must give the scheduler a recorder"); + let events = recorder.lock().unwrap_or_else(|p| p.into_inner()).events(); + + assert!( + events.iter().any(|r| matches!( + &r.event, + BlackBoxEvent::NodeError { error, .. } if error == "replication peer lost" + )), + "an external event never reached the flight recorder: the hook \ + set_blackbox_hook exists for is not wired. Recorded: {} event(s)", + events.len() + ); +} diff --git a/horus_core/tests/control_restart.rs b/horus_core/tests/control_restart.rs new file mode 100644 index 000000000..7e5950956 --- /dev/null +++ b/horus_core/tests/control_restart.rs @@ -0,0 +1,152 @@ +//! `horus node restart` must actually re-initialise the node. +//! +//! The CLI sent `PauseNode` immediately followed by `ResumeNode` and printed +//! "The scheduler will re-initialize this node on next tick". It did not. +//! Both arms only toggle a shared atomic; the code that re-runs `init()` — +//! `reinit_pending_nodes` — keys off `RegisteredNode::initialized`, and outside +//! registration nothing in production ever set that back to false. A grep of +//! the workspace found the only two `initialized = false` assignments in +//! `#[cfg(test)]` code. +//! +//! Four places in the runtime asserted the behaviour anyway: the clap help +//! ("re-initialize without killing scheduler"), `restart_node`'s doc comment, +//! an inline comment reading "Pause then resume triggers re-init in the +//! scheduler", and `reinit_pending_nodes`'s own comment, "set by control +//! commands". On a robot the operator restarts a wedged node, is told it +//! worked, and moves on. +//! +//! Run: `cargo test -p horus_core --test control_restart` + +use horus_core::core::duration_ext::DurationExt; +use horus_core::core::Node; +use horus_core::scheduling::control::ControlCommand; +use horus_core::scheduling::Scheduler; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::Arc; + +struct CountsInit { + inits: Arc, +} + +impl Node for CountsInit { + fn name(&self) -> &str { + "restartable" + } + fn init(&mut self) -> horus_core::error::Result<()> { + self.inits.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + fn tick(&mut self) {} +} + +/// A `RestartNode` command re-runs `init()` while the scheduler is running. +/// +/// Driven through `run_for` rather than `tick_once`: the control topic is +/// created in `run_with_filter`, so under `tick_once` `self.control_topic` is +/// None and no control command is ever polled. That is worth knowing on its +/// own — `horus node pause/resume/restart` reach a scheduler only while it is +/// in `run()`. +#[test] +fn restart_command_reinitialises_the_node() { + let inits = Arc::new(AtomicU32::new(0)); + let sched_name = format!("restart_test_{}", std::process::id()); + let ctl_name = format!("horus.ctl.{}", sched_name); + + // Open the control topic before the run so the send cannot race setup. + let ctl: horus_core::communication::Topic = + horus_core::communication::Topic::new_with_kind( + &ctl_name, + horus_core::communication::TopicKind::System as u8, + ) + .expect("control topic opens"); + + let inits_for_thread = inits.clone(); + let name_for_thread = sched_name.clone(); + let handle = std::thread::spawn(move || { + let mut scheduler = Scheduler::new() + .name(&name_for_thread) + .tick_rate(200_u64.hz()); + scheduler + .add(CountsInit { + inits: inits_for_thread, + }) + .build() + .expect("node registers"); + scheduler.run_for(1200_u64.ms()).expect("run_for"); + }); + + // Poll for the startup init() rather than sleeping a fixed 400 ms. + // + // A fixed wait is a bet that the scheduler thread got scheduled inside it. + // On a loaded runner — which is the normal state of CI, and of this repo's + // own workstation under concurrent builds — it does not, and the test fails + // reporting 0 inits as though `RestartNode` were broken. Polling turns the + // same wall clock into an upper bound instead of an assumption: it returns + // as soon as the init lands, and only the diagnostic changes if it never + // does. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while inits.load(Ordering::SeqCst) < 1 && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert_eq!( + inits.load(Ordering::SeqCst), + 1, + "the node should have been initialised once by startup" + ); + + ctl.send(ControlCommand::RestartNode { + name: "restartable".to_string(), + }); + + handle.join().expect("scheduler thread joins"); + assert_eq!( + inits.load(Ordering::SeqCst), + 2, + "RestartNode must clear `initialized` so reinit_pending_nodes re-runs init(); \ + Pause+Resume left it at 1 while the CLI reported success" + ); +} + +/// A restart naming an unregistered node must not disturb a registered one. +#[test] +fn restart_for_an_unknown_node_leaves_the_others_alone() { + let inits = Arc::new(AtomicU32::new(0)); + let sched_name = format!("restart_unknown_{}", std::process::id()); + let ctl_name = format!("horus.ctl.{}", sched_name); + + let ctl: horus_core::communication::Topic = + horus_core::communication::Topic::new_with_kind( + &ctl_name, + horus_core::communication::TopicKind::System as u8, + ) + .expect("control topic opens"); + + let inits_for_thread = inits.clone(); + let name_for_thread = sched_name.clone(); + let handle = std::thread::spawn(move || { + let mut scheduler = Scheduler::new() + .name(&name_for_thread) + .tick_rate(200_u64.hz()); + scheduler + .add(CountsInit { + inits: inits_for_thread, + }) + .build() + .expect("node registers"); + scheduler.run_for(1200_u64.ms()).expect("run_for"); + }); + + std::thread::sleep(std::time::Duration::from_millis(400)); + assert_eq!(inits.load(Ordering::SeqCst), 1); + + ctl.send(ControlCommand::RestartNode { + name: "no_such_node".to_string(), + }); + + handle.join().expect("scheduler thread joins"); + assert_eq!( + inits.load(Ordering::SeqCst), + 1, + "a restart naming an unregistered node must not re-init a different one" + ); +} diff --git a/horus_core/tests/hlog_default_level_doc.rs b/horus_core/tests/hlog_default_level_doc.rs new file mode 100644 index 000000000..c3cd89652 --- /dev/null +++ b/horus_core/tests/hlog_default_level_doc.rs @@ -0,0 +1,66 @@ +//! The `hlog!` doc must not name a default level the constant contradicts. +//! +//! `hlog!`'s doc-comment claimed "The default level is `DEFAULT_LEVEL` (`info`, +//! i.e. `debug` is off)" while `DEFAULT_LEVEL` in the same file was — and still +//! is — `LEVEL_DEBUG`. Three other places in that file say so outright, +//! including the constant's own doc and the test `nothing_is_suppressed_by_default`. +//! +//! That is the doc a node author reads before deciding whether a +//! `hlog!(debug, ...)` in a tick loop is free. It is not: in an unconfigured +//! process every one of them runs `format!`, `chrono::Local::now()` and a +//! bincode serialise into the shared buffer. +//! +//! This guard lives in its own file, not in `hlog.rs`'s test module, so it +//! cannot match its own text. + +use std::path::PathBuf; + +fn hlog_source() -> String { + let path: PathBuf = [env!("CARGO_MANIFEST_DIR"), "src", "core", "hlog.rs"] + .iter() + .collect(); + std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("reading {}: {e}", path.display())) +} + +/// The macro doc and the constant must agree about the shipped default. +#[test] +fn the_hlog_macro_doc_agrees_with_default_level() { + let src = hlog_source(); + + let debug_default = src.contains("pub const DEFAULT_LEVEL: u8 = LEVEL_DEBUG;"); + let info_default = src.contains("pub const DEFAULT_LEVEL: u8 = LEVEL_INFO;"); + assert!( + debug_default ^ info_default, + "could not find the DEFAULT_LEVEL declaration — has it been renamed or \ + given a different form? This guard reads it literally." + ); + + // Only the `hlog!` macro's own doc block, so the constant's doc (which + // legitimately discusses `Info` as the conventional-but-rejected default) + // is not scanned. + let start = src + .find("/// # Filtering") + .expect("the hlog! doc's Filtering section is the anchor for this guard"); + let end = src[start..] + .find("/// # Example") + .map(|i| start + i) + .expect("the Filtering section should be followed by Example"); + let filtering = &src[start..end]; + + if debug_default { + assert!( + filtering.contains("it is `debug`"), + "DEFAULT_LEVEL is LEVEL_DEBUG but the hlog! doc does not say so:\n{filtering}" + ); + assert!( + !filtering.contains("(`info`, i.e. `debug`\n/// is off)"), + "DEFAULT_LEVEL is LEVEL_DEBUG and the hlog! doc still claims `info`, \ + debug off — the exact claim this guard exists for:\n{filtering}" + ); + } else { + assert!( + !filtering.contains("it is `debug`"), + "DEFAULT_LEVEL is LEVEL_INFO but the hlog! doc still says `debug`:\n{filtering}" + ); + } +} diff --git a/horus_core/tests/log_cross_layer_tests.rs b/horus_core/tests/log_cross_layer_tests.rs index db02ec919..234c3fac2 100644 --- a/horus_core/tests/log_cross_layer_tests.rs +++ b/horus_core/tests/log_cross_layer_tests.rs @@ -597,6 +597,13 @@ fn measure_in_isolated_child(test_name: &str) -> bool { .stderr(std::process::Stdio::null()) .output() .expect("spawn isolated child"); + + // The child created /dev/shm/horus_/ and nothing unlinks it when a + // process exits, so each isolated child otherwise leaves a namespace + // directory behind for good. Removed here because the child is the process + // that dies. + let _ = std::fs::remove_dir_all(horus_sys::shm::shm_base_dir_for(&ns)); + let stdout = String::from_utf8_lossy(&out.stdout); assert!( out.status.success(), diff --git a/horus_core/tests/log_error_buffer_tests.rs b/horus_core/tests/log_error_buffer_tests.rs index 4cda7554b..affc86ad1 100644 --- a/horus_core/tests/log_error_buffer_tests.rs +++ b/horus_core/tests/log_error_buffer_tests.rs @@ -72,6 +72,14 @@ fn count_in_isolated_child(test_name: &str) -> bool { .stderr(std::process::Stdio::null()) .output() .expect("spawn isolated child"); + // The child materialised /dev/shm/horus_/ and nothing unlinks it when a + // process exits, so without this every isolated child left a namespace + // directory behind — one per test, per run, forever. They accumulate across + // CI runs and local runs alike, and a full /dev/shm is the documented cause + // of the rotating, unattributable failures this suite otherwise shows. + // Removed here rather than in the child: the child is the process that dies. + let _ = std::fs::remove_dir_all(horus_sys::shm::shm_base_dir_for(&ns)); + let stdout = String::from_utf8_lossy(&out.stdout); assert!( out.status.success(), diff --git a/horus_core/tests/log_file_drain_wiring.rs b/horus_core/tests/log_file_drain_wiring.rs new file mode 100644 index 000000000..d3e1a248e --- /dev/null +++ b/horus_core/tests/log_file_drain_wiring.rs @@ -0,0 +1,85 @@ +//! `HORUS_LOG_FILE=1` has to actually start something. +//! +//! `start_log_file_drain` was called by no shipped code path. A tree-wide grep +//! for the symbol returned four hits: the definition, its re-export, and two +//! lines of one integration test that invoked it by hand. So the drain worked, +//! its test passed, and `HORUS_LOG_FILE=1 horus run` still wrote no +//! `.horus/logs/horus.log` — with `HORUS_LOG_DIR`, `_MAX_SIZE` and +//! `_MAX_FILES` dead alongside it. An operator who sets a documented variable +//! to get a post-incident log and finds an empty disk is the exact failure the +//! variable exists to prevent. +//! +//! This test therefore does NOT call the drain. It sets the variable, builds a +//! `Scheduler` the way any program does, and asserts a log file appears. It is +//! alone in its binary because it writes process-global environment and the +//! drain starts once per process. + +use horus_core::core::log_buffer::{LogEntry, LogType, GLOBAL_LOG_BUFFER}; +use horus_core::scheduling::Scheduler; +use std::time::Duration; + +#[test] +fn setting_horus_log_file_makes_a_scheduler_write_logs_to_disk() { + let dir = std::env::temp_dir().join(format!("horus-log-drain-wiring-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("temp log dir"); + + // SAFETY: this test is alone in its binary, so nothing else is reading or + // writing the environment concurrently. + unsafe { + std::env::set_var("HORUS_LOG_FILE", "1"); + std::env::set_var("HORUS_LOG_DIR", dir.to_str().unwrap()); + } + + // The whole point: building a scheduler is all a program does. Nothing + // here calls `start_log_file_drain`. + let _scheduler = Scheduler::new(); + + // Keep pushing while polling, rather than pushing once up front. + // + // `log_drain_loop` snapshots `GLOBAL_LOG_BUFFER.write_idx()` when its + // thread starts and only ever writes entries newer than that. On a loaded + // machine the thread can reach that snapshot after a one-shot burst has + // already been pushed, and then there is nothing newer to write and the + // test waits out its whole window for a drain that is working correctly. + // (CI caught exactly that.) Pushing every round guarantees entries after + // the snapshot whenever the thread got going. + let log_file = dir.join("horus.log"); + let mut content = String::new(); + for round in 0..40 { + for i in 0..5 { + GLOBAL_LOG_BUFFER.push(LogEntry { + timestamp: format!("2026-09-04T00:00:{:02}.000Z", i), + tick_number: GLOBAL_LOG_BUFFER.write_idx() + 1, + node_name: "wiring_probe".to_string(), + log_type: LogType::Info, + topic: None, + message: format!("round {} entry {}", round, i), + tick_us: 0, + ipc_ns: 0, + }); + } + // The drain polls every 500 ms. + std::thread::sleep(Duration::from_millis(250)); + if let Ok(c) = std::fs::read_to_string(&log_file) { + if c.contains("wiring_probe") { + content = c; + break; + } + } + } + + let found = content.contains("wiring_probe"); + let _ = std::fs::remove_dir_all(&dir); + unsafe { + std::env::remove_var("HORUS_LOG_FILE"); + std::env::remove_var("HORUS_LOG_DIR"); + } + + assert!( + found, + "HORUS_LOG_FILE=1 was set and a Scheduler was built, so {} should \ + contain the log entries. Nothing started the drain.", + log_file.display() + ); +} diff --git a/horus_core/tests/participant_count_leak.rs b/horus_core/tests/participant_count_leak.rs index becbe5eb3..1c9edd6c3 100644 --- a/horus_core/tests/participant_count_leak.rs +++ b/horus_core/tests/participant_count_leak.rs @@ -1,13 +1,10 @@ -//! OPEN: participant counts only ever grow within a process, and the backend is -//! chosen from them. +//! A registration is given back when its handle is dropped. //! -//! `Drop for RingTopic` (`communication/topic/mod.rs`) releases FanoutShm -//! endpoint bits and spill keep-alives and leaves the participant table -//! untouched. The only `decrement_to_floor` calls on `publisher_count` / -//! `subscriber_count` are in `reap_dead_participants_now`, which refuses to reap -//! on an expired lease alone (an idle subscriber spends much of its life looking -//! expired) and otherwise wants a dead pid. Within one living process, nothing -//! ever gives a registration back. +//! It was not, and nothing else inside a living process did it either. The only +//! `decrement_to_floor` calls on `publisher_count`/`subscriber_count` are in +//! `reap_dead_participants_now`, which wants an expired lease AND a dead pid, +//! and in `register_role`'s last-resort pass, which only runs once all 16 slots +//! are occupied. So within one process the counts only ever grew. //! //! That is not a bookkeeping wart, because `detect_optimal_backend` //! (`topic/header.rs`) selects the backend from those counts: @@ -16,28 +13,25 @@ //! subs > 1 && is_pod -> PodShm (broadcast; overwrites the oldest) //! //! So a transient subscriber on a fresh thread -- a worker pool, a scoped task, -//! a one-shot diagnostic read -- permanently converts a command topic from +//! a one-shot diagnostic read -- permanently converted a command topic from //! backpressured to overwrite-oldest, with nothing alive to justify it and no -//! path back. The test below observes exactly that: `sub_count()` climbs 1, 2, 3 -//! across three threads that are each fully joined before the next starts, and -//! the backend flips at the second one. +//! path back for the life of the shared-memory segment. It silently voided +//! `try_send`'s full-ring `Err` and `send_blocking`'s backpressure on an e-stop +//! path. //! -//! # Why this is `#[ignore]` rather than fixed +//! # Why the obvious fix was not enough //! -//! The obvious fix -- deregister in `Drop` -- was implemented and measured, and -//! it is NOT safe as written. A handle records its participant slot index, but a -//! slot is not owned for the handle's lifetime: `register_role`'s later passes -//! reclaim lease-expired slots, and a lease looks expired whenever its owner has -//! been quiet, so a live handle's slot can be taken by another thread. Releasing +//! Deregistering in `Drop` against the recorded slot INDEX was implemented and +//! measured, and it was not safe: a slot is not owned for a handle's lifetime. +//! `register_role`'s pass 3 reclaims slots whose thread has ended, so releasing //! against a stale index charges the drop to somebody else's registration. -//! Guarding it with the entry's (pid, thread-id hash) narrows the window but does -//! not close it: thread ids are reused after a thread exits, and the hash is a -//! u32 that can collide. +//! Guarding with (pid, thread-id hash) narrows the window without closing it -- +//! thread ids are reused, and the hash is a u32 that can collide. //! -//! Measured, interleaved against unmodified main on the same machine and load, -//! three pairs of the full `horus_core` lib suite: +//! Measured then, interleaved against unmodified main on the same machine and +//! load, three pairs of the full `horus_core` lib suite: //! -//! main: 0, 0, 0 failures with the Drop fix: 0, 3, 2 failures +//! main: 0, 0, 0 failures with the naive Drop fix: 0, 3, 2 failures //! //! The failures were `multithread_nonpod_subscribers_each_get_a_contiguous_accounted_stream` //! and `multithread_nonpod_lapped_stream_stays_ordered` -- a subscriber whose @@ -49,17 +43,22 @@ //! registration becomes silent message loss on a subscriber that is still //! reading -- strictly worse than the over-count being fixed. //! -//! A correct fix needs unambiguous slot ownership: a generation counter on -//! `ParticipantEntry`, bumped on every claim and recorded by the handle, so a -//! release can tell its own registration from a later one in the same slot. The -//! entry has 3 bytes of padding left after `source_host`, which is room for a -//! `u16` generation. That is the shape; it needs building and testing properly, -//! against a machine that is not also running the rest of this suite. +//! # What makes it safe now +//! +//! `ParticipantEntry` carries a `generation`, bumped on every claim in +//! `finish_claim` and recorded by the handle beside the slot index, so a +//! release can tell its own registration from a later one in the same slot. It +//! also carries a `handles` count, because an entry is keyed on (pid, thread) +//! and several `Topic`s on one thread deliberately share one: the first to drop +//! must not clear an entry the others are still using. `release_participant` +//! refuses -- rather than guesses -- on every mismatch. +//! +//! Both fields came out of the entry's existing padding, so it is still 24 +//! bytes and the 640-byte header is unchanged. use horus_core::communication::Topic; #[test] -#[ignore = "documents an open defect: see the module comment. Run with --ignored."] fn transient_subscribers_permanently_flip_a_topic_to_a_lossy_backend() { let name = "leak_transient_subs"; let producer: Topic = Topic::new(name).expect("producer"); @@ -79,8 +78,8 @@ fn transient_subscribers_permanently_flip_a_topic_to_a_lossy_backend() { observed.push((producer.sub_count(), producer.backend_name().to_string())); } - // What SHOULD hold, and does not: no subscriber is alive at any of these - // points, so the count should be 0 or 1 and the backend should stay SpscShm. + // No subscriber is alive at any of these points, so the count is 0 or 1 + // and the backend stays SpscShm. assert!( observed.iter().all(|(c, _)| *c <= 1), "sub_count climbed to {:?} with no live subscriber at any sample point", @@ -91,3 +90,204 @@ fn transient_subscribers_permanently_flip_a_topic_to_a_lossy_backend() { "a backpressured command topic became lossy with no live subscriber: {observed:?}" ); } + +/// Several handles on ONE thread share a single registration, and the first to +/// go must not take it away from the others. +/// +/// `register_role` keys an entry on (pid, thread), not on the handle: a second +/// `Topic` on the same thread finds the existing entry, ORs in its role bit and +/// deliberately does NOT increment `subscriber_count` again. So a release that +/// matched only on (slot, generation) would clear an entry two other handles +/// were still using — and under-counting is the dangerous direction, because +/// `nothing_is_draining` lets a producer retire unread slots once `sub_count()` +/// reads 0. That is silent message loss on a subscriber that is still reading, +/// strictly worse than the over-count being fixed. +#[test] +fn one_handle_leaving_does_not_deregister_its_siblings_on_the_same_thread() { + let name = "leak_shared_entry"; + let producer: Topic = Topic::new(name).expect("producer"); + producer.send(1u64); + + let sub_a: Topic = Topic::new(name).expect("sub a"); + let sub_b: Topic = Topic::new(name).expect("sub b"); + let _ = sub_a.try_recv(); + let _ = sub_b.try_recv(); + + assert_eq!( + producer.sub_count(), + 1, + "precondition: two handles on one thread share one registration" + ); + + drop(sub_a); + assert_eq!( + producer.sub_count(), + 1, + "sub_b is still reading, so the registration must survive its \ + sibling's departure" + ); + + // And it really can still read. + producer.send(7u64); + let mut seen = None; + for _ in 0..1000 { + if let Some(v) = sub_b.try_recv() { + seen = Some(v); + if seen == Some(7) { + break; + } + } + } + assert_eq!( + seen, + Some(7), + "the surviving handle must still receive after its sibling dropped" + ); + + drop(sub_b); + assert_eq!( + producer.sub_count(), + 0, + "the last handle to go gives the registration back" + ); +} + +/// Concurrent claim and release must leave the counters exactly where they +/// started. +/// +/// The release path guards on (slot, generation) and per-role handle counts, +/// and those are read-modify-write sequences on shared memory that several +/// threads reach at once. Reading the code cannot rule out an interleaving that +/// double-decrements or strands a slot, and both directions are bad: +/// over-counting pins a command topic on a lossy broadcast backend, while +/// under-counting lets `nothing_is_draining` retire unread slots beneath a +/// subscriber that is still reading. +/// +/// Threads churn handles against one topic; when they are all joined, nothing +/// is alive, so both counts must be back to zero. +#[test] +fn concurrent_churn_returns_every_registration() { + let name = "leak_concurrent_churn"; + let producer: Topic = Topic::new(name).expect("producer"); + producer.send(1u64); + + const THREADS: usize = 8; + const ROUNDS: usize = 40; + + let mut handles = Vec::new(); + for _ in 0..THREADS { + let n = name.to_string(); + handles.push(std::thread::spawn(move || { + for _ in 0..ROUNDS { + // A subscriber and a publisher on the same thread share ONE + // participant entry, so this exercises the per-role handle + // counts as well as the generation guard. + let sub: Topic = Topic::new(&n).expect("subscriber"); + let pubr: Topic = Topic::new(&n).expect("publisher"); + let _ = sub.try_recv(); + pubr.send(2u64); + drop(sub); + drop(pubr); + } + })); + } + for h in handles { + h.join().expect("churn thread"); + } + + // The producer above is this thread's own handle and is still alive, so it + // still holds a publisher registration. Everything the churn threads + // claimed must be gone. + assert_eq!( + producer.sub_count(), + 0, + "every churn subscriber was dropped, so no subscriber registration may \ + remain — a leak here pins the topic on a broadcast backend for the \ + life of the segment" + ); + assert_eq!( + producer.pub_count(), + 1, + "only this thread's producer is still alive; the churn publishers must \ + have given their registrations back, and none may have taken this \ + one's with it" + ); + + // And the topic still works, which under-counting would have broken. + producer.send(3u64); + let reader: Topic = Topic::new(name).expect("reader"); + let mut seen = false; + for _ in 0..1000 { + if reader.try_recv().is_some() { + seen = true; + break; + } + } + assert!(seen, "the topic must still deliver after the churn"); +} + +/// A handle taking a role a SIBLING already holds must still be counted. +/// +/// An entry is keyed on (pid, thread) and its role bits are shared by every +/// handle on that thread, so "the entry already has the producer bit" says +/// nothing about whether THIS handle holds it. The per-role handle count was +/// incremented only when the bit was newly set on the entry, or when the handle +/// had no slot yet — so the second handle to take an already-set role was not +/// counted, and the first `Drop` handed back a role another live handle was +/// still using. +/// +/// That is the under-count direction, which is the dangerous one: +/// `nothing_is_draining` lets a producer retire unread slots once `sub_count()` +/// reads 0, so a lost subscriber registration becomes silent message loss on a +/// subscriber that is still reading. +#[test] +fn a_second_handle_taking_a_shared_role_is_counted() { + let name = "leak_shared_roles"; + + // Two handles on THIS thread, each publisher and subscriber, so both take + // both roles on one shared participant entry. + let a: Topic = Topic::new(name).expect("handle a"); + let b: Topic = Topic::new(name).expect("handle b"); + a.send(1u64); + let _ = a.try_recv(); + b.send(2u64); + let _ = b.try_recv(); + + assert_eq!(a.pub_count(), 1, "precondition: one thread, one entry"); + assert_eq!(a.sub_count(), 1); + + drop(a); + + assert_eq!( + b.pub_count(), + 1, + "b is still publishing, so the producer registration must survive a's \ + departure — releasing it lets a peer's detect_optimal_backend see a \ + topic with no publisher" + ); + assert_eq!( + b.sub_count(), + 1, + "b is still reading, so the subscriber registration must survive. At 0, \ + `nothing_is_draining` lets a producer retire slots b has not read" + ); + + // And b really can still both send and receive. + b.send(3u64); + let mut seen = false; + for _ in 0..1000 { + if b.try_recv().is_some() { + seen = true; + break; + } + } + assert!(seen, "the surviving handle must still receive"); + + drop(b); + let after: Topic = Topic::new(name).expect("observer"); + assert_eq!( + after.pub_count(), + 0, + "the last handle to go gives both roles back" + ); +} diff --git a/horus_core/tests/participant_exhaustion_is_reported.rs b/horus_core/tests/participant_exhaustion_is_reported.rs new file mode 100644 index 000000000..f7e274b11 --- /dev/null +++ b/horus_core/tests/participant_exhaustion_is_reported.rs @@ -0,0 +1,158 @@ +//! The 17th participant must be told why it has no comms. +//! +//! A topic holds at most 16 participants. `register_role` builds a precise +//! error for the one that does not fit — it names the limit and says what to do +//! about it — and all three call sites threw it away: +//! `recv_uninitialized` and `read_latest` were `if ....is_err() { return None }` +//! and `send_uninitialized` the same with `Err(msg)`. Those are the only +//! `ensure_consumer`/`ensure_producer` call sites in the tree, so the message +//! reached nothing. +//! +//! Because the role stays `Unregistered`, every later `recv()` re-enters the +//! same cold path and fails identically: `None` forever, from a subscriber that +//! is indistinguishable from an idle one. The identical 16-slot wall one layer +//! down is reported deliberately and loudly, under a comment reading "never let +//! 'all 16 slots live' become a SILENT no-comms endpoint" — with the rate +//! limiter for it sitting in the same file, unused by this path. + +use horus_core::communication::Topic; +use std::sync::{Arc, Mutex}; + +/// The participant table's size. Mirrors `MAX_PARTICIPANTS` in +/// `topic/header.rs`, which is `pub(crate)` and so not importable here. +const MAX_PARTICIPANTS: u32 = 16; +/// Live subscribers needed to fill it, given the producer holds one slot. +const HOLDERS: usize = MAX_PARTICIPANTS as usize - 1; + +/// A `tracing::Subscriber` that keeps the formatted fields of every event. +/// +/// Hand-rolled against `tracing` core so this test needs no new dependency. +struct Capture(Arc>>); + +impl tracing::Subscriber for Capture { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::Id { + tracing::Id::from_u64(1) + } + fn record(&self, _: &tracing::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::Id, _: &tracing::Id) {} + fn event(&self, event: &tracing::Event<'_>) { + struct Fields(String); + impl tracing::field::Visit for Fields { + fn record_debug(&mut self, _: &tracing::field::Field, value: &dyn std::fmt::Debug) { + self.0.push_str(&format!("{:?} ", value)); + } + } + let mut fields = Fields(String::new()); + event.record(&mut fields); + self.0 + .lock() + .unwrap_or_else(|p| p.into_inner()) + .push(fields.0); + } + fn enter(&self, _: &tracing::Id) {} + fn exit(&self, _: &tracing::Id) {} +} + +#[test] +fn a_participant_that_does_not_fit_is_told_so() { + let name = format!("participant_exhaustion_{}", std::process::id()); + + // Process-wide, not `set_default`: the registration that gets refused + // happens on a spawned thread, and `set_default` is thread-local. A thread + // is required — the main thread already holds a participant entry for its + // publisher, and `ensure_consumer` upgrades that entry's role in place + // rather than claiming a second slot, so it is never refused. + // This test is alone in its binary, so the global default is free. + let captured = Arc::new(Mutex::new(Vec::new())); + tracing::subscriber::set_global_default(Capture(Arc::clone(&captured))) + .expect("no other subscriber in this test binary"); + + let Ok(producer) = Topic::::new(&name) else { + eprintln!("skipping: no shared memory available"); + return; + }; + producer.send(1u64); + + // Fill the participant table with LIVE holders. + // + // This used to spawn threads and let them exit, relying on the leak + // documented in participant_count_leak.rs to keep their slots occupied. + // That leak is fixed — a dropped handle gives its registration back — so + // the table stopped filling and this test skipped itself unconditionally: + // "the participant table did not fill (1/16)". The anti-vacuity guard + // reported it rather than passing quietly, which is what such a guard is + // for, but the test proved nothing until it held its registrations open. + let (ready_tx, ready_rx) = std::sync::mpsc::channel::<()>(); + let release = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let mut holders = Vec::new(); + for _ in 0..HOLDERS { + let n = name.clone(); + let ready = ready_tx.clone(); + let release = Arc::clone(&release); + holders.push(std::thread::spawn(move || { + let sub = Topic::::new(&n).expect("holder subscriber"); + let _ = sub.try_recv(); // claims this thread's participant slot + let _ = ready.send(()); + // Hold the registration open until the assertions are done. + while !release.load(std::sync::atomic::Ordering::Acquire) { + std::thread::sleep(std::time::Duration::from_millis(2)); + } + drop(sub); + })); + } + drop(ready_tx); + for _ in 0..HOLDERS { + ready_rx + .recv_timeout(std::time::Duration::from_secs(30)) + .expect("every holder must claim its slot"); + } + + // Anti-vacuity: assert the table really is full BEFORE asking whether the + // overflow was reported. Without this the test passes on a platform where a + // slot was always available and nothing was ever refused. + let occupied = producer.sub_count() + producer.pub_count(); + + // One more subscriber, on a thread of its own so it needs a slot of its + // own. There is none left to give it. + let n = name.clone(); + let received = std::thread::spawn(move || { + let sub = Topic::::new(&n).expect("the handle itself still constructs"); + sub.try_recv() + }) + .join() + .expect("late subscriber thread"); + + release.store(true, std::sync::atomic::Ordering::Release); + for h in holders { + h.join().expect("holder thread"); + } + + let messages = captured.lock().unwrap_or_else(|p| p.into_inner()).clone(); + + assert!( + occupied >= MAX_PARTICIPANTS, + "precondition: the table must be full for anything to be refused \ + ({occupied}/{MAX_PARTICIPANTS} occupied)" + ); + assert!( + received.is_none(), + "precondition: with the table full this subscriber holds no \ + registration, so it must receive nothing" + ); + + assert!( + messages.iter().any(|m| m.contains("participant slots")), + "a subscriber was refused a registration and will receive nothing for \ + the life of its handle, and was told nothing. Warnings seen: {:?}", + messages + ); + assert!( + messages.iter().any(|m| m.contains(&name)), + "the warning must name the topic — an operator needs to know WHICH \ + subsystem went quiet. Warnings seen: {:?}", + messages + ); +} diff --git a/horus_core/tests/record_replay_e2e.rs b/horus_core/tests/record_replay_e2e.rs index 4c134f0b0..bea8d773f 100644 --- a/horus_core/tests/record_replay_e2e.rs +++ b/horus_core/tests/record_replay_e2e.rs @@ -1839,3 +1839,81 @@ fn test_recording_with_no_publisher_has_empty_inputs() { ); } } + +// ============================================================================ +// Test: replay restores the RECORDED execution order +// ============================================================================ + +/// `replay_from` must give each node the priority its recorded position had. +/// +/// It looked the priority up by `node_id` — the key of `node_recordings`, a +/// generated hex string (`format!("{:x}{:x}", unix_nanos, node_count)`) — while +/// the map is keyed by node NAME. A node id can never equal a name, so every +/// lookup missed, `unwrap_or(0)` gave every node priority 0, and `add_replay`'s +/// stable sort left the replay in `node_recordings` HashMap order: neither the +/// recorded order nor the same order twice. +/// +/// This asserts the priorities, not the tick counts, because a replay whose +/// node ordering is arbitrary still ticks every node the right number of times +/// — which is why the existing tests passed throughout. +#[test] +fn replay_restores_the_recorded_execution_order() { + use horus_core::scheduling::SchedulerRecording; + + let tmp = TestTempDir::new("replay_order"); + let session = tmp.path().to_path_buf(); + + // Three nodes, recorded in a known order that is NOT alphabetical and not + // the order they are registered in below. + let order = ["zulu", "alpha", "mike"]; + + let mut scheduler_recording = SchedulerRecording::new("sched_id", "order_session"); + for (i, name) in order.iter().enumerate() { + // The id shape `Scheduler::add` generates: a hex string, never a name. + let node_id = format!("{:x}{:x}", 1_700_000_000_000_000_000u64 + i as u64, i); + let relative_path = format!("{name}@{node_id}.horus"); + + let mut rec = NodeRecording::new(name, &node_id, "order_session"); + for tick in 0..3u64 { + rec.add_snapshot(NodeTickSnapshot::new(tick).with_duration(100)); + } + rec.finish(); + rec.save(&session.join(&relative_path)) + .expect("write node recording"); + + scheduler_recording.add_node_recording(&node_id, &relative_path); + } + scheduler_recording.record_execution_order(order.iter().map(|s| s.to_string()).collect()); + + let scheduler_path = session.join("scheduler@sched_id.horus"); + scheduler_recording + .save(&scheduler_path) + .expect("write scheduler recording"); + + let scheduler = Scheduler::replay_from(scheduler_path).expect("replay_from"); + + let mut by_name: Vec<(String, u32)> = scheduler + .metrics() + .into_iter() + .map(|m| (m.name().to_string(), m.order())) + .collect(); + by_name.sort_by_key(|(_, order)| *order); + + let replayed: Vec<&str> = by_name.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!( + replayed, + order.to_vec(), + "replay must restore the recorded execution order, got priorities {by_name:?}" + ); + + // And the priorities must be distinct — all-zero is the failure mode, and + // it looks like a valid order until you notice the sort has nothing to do. + let distinct: std::collections::BTreeSet = + by_name.iter().map(|(_, order)| *order).collect(); + assert_eq!( + distinct.len(), + order.len(), + "every replayed node got the same priority, so the ordering is whatever \ + the recordings HashMap iterated: {by_name:?}" + ); +} diff --git a/horus_core/tests/rt_no_alloc.rs b/horus_core/tests/rt_no_alloc.rs index 07f2c9b2b..c3afcc524 100644 --- a/horus_core/tests/rt_no_alloc.rs +++ b/horus_core/tests/rt_no_alloc.rs @@ -92,6 +92,74 @@ fn this_binary_has_no_rt_allocator_and_reports_that_honestly() { ); } +/// Registering a `.no_alloc()` node must actually print the warning. +/// +/// `warn_if_unenforced` names its own call site — "Intended call site: node +/// registration, once per `.no_alloc()` node" — and had no caller anywhere +/// outside the two tests that invoke it directly. So the function worked +/// perfectly and no user ever saw it: `.no_alloc()` in a binary without the +/// allocator was silently inert, which is the precise failure the warning text +/// exists to prevent. +/// +/// Calling the function proves the function. This builds a node and requires +/// the warning to come out of a real process's stderr, which is the part that +/// was missing. The child is this same test binary — the one that deliberately +/// does not install the allocator. +#[test] +fn building_a_no_alloc_node_warns_that_nothing_is_enforcing_it() { + const CHILD: &str = "HORUS_NO_ALLOC_WARN_CHILD"; + + struct Inert; + impl Node for Inert { + fn name(&self) -> &str { + "unenforced_controller" + } + fn tick(&mut self) {} + } + + if std::env::var_os(CHILD).is_some() { + let mut scheduler = Scheduler::new().tick_rate(1000_u64.hz()); + scheduler + .add(Inert) + .order(0) + .rate(1000_u64.hz()) + .no_alloc() + .build() + .expect("a .no_alloc() RT node must still build — this warns, it does not fail"); + return; + } + + let exe = std::env::current_exe().expect("test binary path"); + let out = std::process::Command::new(exe) + .args([ + "--exact", + "building_a_no_alloc_node_warns_that_nothing_is_enforcing_it", + "--nocapture", + "--test-threads", + "1", + ]) + .env(CHILD, "1") + .output() + .expect("re-running the test binary should work"); + + let stdout = String::from_utf8_lossy(&out.stdout); + assert!( + stdout.contains("1 passed"), + "the child ran no test — was this test renamed?\n{stdout}" + ); + + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains(".no_alloc() IS NOT BEING ENFORCED"), + "registering a .no_alloc() node in a binary with no RtAwareAllocator \ + printed no warning, so the user is told nothing.\nstderr:\n{stderr}" + ); + assert!( + stderr.contains("unenforced_controller"), + "the warning must name the node that asked for the guarantee.\nstderr:\n{stderr}" + ); +} + /// Without the allocator, `.no_alloc()` is inert — allocating inside the bracket /// is not detected. Asserting the no-op keeps the documentation honest and makes /// the warning above load-bearing rather than decorative. diff --git a/horus_core/tests/safing_precedes_the_stop_gates.rs b/horus_core/tests/safing_precedes_the_stop_gates.rs new file mode 100644 index 000000000..aef5234c2 --- /dev/null +++ b/horus_core/tests/safing_precedes_the_stop_gates.rs @@ -0,0 +1,120 @@ +//! A pending safe-state request must be consumed before a node is skipped. +//! +//! Every executor polls `honor_safe_state_request` once per pass. The property +//! that matters is not that the call exists — it is WHERE it sits relative to +//! the guards that `continue` past a node. Put it after them and the poll is +//! skipped for exactly the nodes the ladder is trying to safe: an isolated node +//! is one that is not ticking, and a stopped node is one holding whatever +//! setpoint it last commanded. +//! +//! Two orderings regressed that way and are pinned here: +//! +//! * the RT executor skipped `is_stopped` nodes before the poll, while its own +//! comment said the poll "runs before the pause/stop gates: an Isolated node +//! must reach its safe state even if it is also about to be stopped"; +//! * the compute coordinator put the poll below its `ready_indices.is_empty()` +//! early-`continue`, so on a pass with nothing ready — the steady state for +//! a set of isolated or rate-limited nodes — no node was safed at all. +//! +//! These loops run on their own threads with a live `SharedMonitors` and cannot +//! be entered from a test, so this reads the source. That is the same shape as +//! `horus_manager/tests/monitor_plugin_lookup_contract.rs`, and it carries the +//! same lesson: comments are stripped first, because both bodies *describe* +//! the ordering in prose and a guard that matched the prose would be satisfied +//! by the very comment the code contradicts. + +use std::fs; +use std::path::PathBuf; + +fn repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("horus_core has a parent") + .to_path_buf() +} + +/// A file's source with every `//` comment line removed. +fn code_only(rel: &str) -> String { + let src = fs::read_to_string(repo_root().join(rel)).unwrap_or_else(|e| panic!("{rel}: {e}")); + src.lines() + .filter(|l| !l.trim_start().starts_with("//")) + .collect::>() + .join("\n") +} + +/// The slice of `code` between the first occurrence of `from` and of `to`. +fn between(code: &str, from: &str, to: &str, what: &str) -> String { + let a = code.find(from).unwrap_or_else(|| { + panic!("{what}: anchor {from:?} is gone; this guard would cover nothing") + }); + let b = code[a..] + .find(to) + .unwrap_or_else(|| panic!("{what}: anchor {to:?} is gone; this guard would cover nothing")); + code[a..a + b].to_string() +} + +#[test] +fn the_rt_executor_safes_a_node_before_it_skips_a_stopped_one() { + let code = code_only("horus_core/src/scheduling/rt_executor.rs"); + + // The per-node preamble of the tick loop: from the loop header to the first + // budget check, which is well past both the safing poll and the stop gates. + let body = between( + &code, + "for (idx, node) in nodes.iter_mut().enumerate() {", + "tick_node(", + "rt executor tick loop", + ); + + let safing = body + .find("honor_safe_state_request") + .expect("the RT executor must poll honor_safe_state_request in its tick loop"); + let stop_gate = body + .find("node.is_stopped") + .expect("the RT executor must still gate on is_stopped somewhere in the loop"); + + assert!( + safing < stop_gate, + "the RT executor skips stopped nodes at offset {stop_gate} before polling \ + honor_safe_state_request at {safing}. Stopping a node halts its ticks; it \ + does not put its actuator anywhere, so a node stopped by BudgetPolicy::Enforce \ + or `horus node kill` keeps holding its last commanded setpoint through a \ + safe-state request. The loop's own comment says this poll runs BEFORE the \ + pause/stop gates." + ); +} + +#[test] +fn the_compute_executor_safes_its_nodes_even_when_none_is_ready() { + let code = code_only("horus_core/src/scheduling/compute_executor.rs"); + + let body = between( + &code, + "if ready_indices.is_empty() {", + "let now = Instant::now();", + "compute coordinator loop", + ); + + assert!( + !body.contains("honor_safe_state_request"), + "the compute coordinator polls honor_safe_state_request AFTER its \ + `ready_indices.is_empty()` early-continue, so on any pass with nothing \ + ready no node is safed — and a set of isolated or rate-limited nodes is \ + permanently in that state. Under `safety.on_link_lost = \"safe_state\"`, \ + which deliberately does not latch an e-stop and leaves the scheduler \ + running, the request would never be consumed at all." + ); + + // ...and it is polled before that point. + let preamble = between( + &code, + "ready_indices.push(i);", + "if ready_indices.is_empty() {", + "compute coordinator preamble", + ); + assert!( + preamble.contains("honor_safe_state_request"), + "the compute coordinator must poll honor_safe_state_request before the \ + `ready_indices.is_empty()` early-continue" + ); +} diff --git a/horus_cpp/include/horus/horus_c.h b/horus_cpp/include/horus/horus_c.h index 5e34b24ac..3c81172a8 100644 --- a/horus_cpp/include/horus/horus_c.h +++ b/horus_cpp/include/horus/horus_c.h @@ -545,6 +545,10 @@ int horus_params_get_i64(const HorusParams* params, const char* key, in int horus_params_set_i64(const HorusParams* params, const char* key, int64_t value); int horus_params_get_bool(const HorusParams* params, const char* key, bool* out); int horus_params_set_bool(const HorusParams* params, const char* key, bool value); +/* Returns the value's FULL length in bytes; writes at most buf_len-1 of them + * plus a NUL. A return >= buf_len means the value was truncated and names the + * buffer size needed to get it whole. Returns -1 if the key is absent, or if + * buf is NULL or buf_len is 0. Never splits a UTF-8 sequence. */ int horus_params_get_string(const HorusParams* params, const char* key, char* buf, size_t buf_len); int horus_params_set_string(const HorusParams* params, const char* key, const char* value); bool horus_params_has(const HorusParams* params, const char* key); diff --git a/horus_cpp/include/horus/params.hpp b/horus_cpp/include/horus/params.hpp index a1f4968e3..fe308dcbe 100644 --- a/horus_cpp/include/horus/params.hpp +++ b/horus_cpp/include/horus/params.hpp @@ -59,11 +59,30 @@ class Params { return horus_params_get_bool(handle_, key, &v) ? std::optional(v) : std::nullopt; } + /// The string value for `key`, complete, or nullopt if absent. + /// + /// This used to hand back whatever fitted in a 1024-byte stack buffer, with + /// no way for a caller to tell a clipped value from a whole one — the same + /// defect this class's constructor refuses to ship for a bad file, because + /// "`get(key, default)` gives the caller no way to tell those apart from + /// the values in the file". Rust and Python always returned the whole + /// string; only C++ clipped. The shim reports the required length now, so a + /// value that does not fit is fetched again into a buffer that holds it. std::optional get_string(const char* key) const { char buf[1024]; int len = horus_params_get_string(handle_, key, reinterpret_cast(buf), sizeof(buf)); if (len < 0) return std::nullopt; - return std::string(buf, static_cast(len)); + const size_t needed = static_cast(len); + if (needed < sizeof(buf)) { + return std::string(buf, needed); + } + // Did not fit: re-call with a buffer sized from the reported length. + std::string out(needed + 1, '\0'); + int again = horus_params_get_string(handle_, key, reinterpret_cast(&out[0]), + out.size()); + if (again < 0) return std::nullopt; + out.resize(static_cast(again)); + return out; } // ── Typed setters ────────────────────────────────────────────────── diff --git a/horus_cpp/src/c_api.rs b/horus_cpp/src/c_api.rs index 7b625f473..d0e164ffc 100644 --- a/horus_cpp/src/c_api.rs +++ b/horus_cpp/src/c_api.rs @@ -489,6 +489,23 @@ unsafe fn topic_name_from_c<'a>(name: *const c_char) -> Option<&'a str> { } } +/// Borrow a service or action endpoint name out of C, or give back `None`. +/// +/// These are not topic constructors, so they do not touch the topic last-error +/// slot — but `CStr::from_ptr` is undefined behaviour on null just the same, +/// and `horus_c.h` is a public ABI that any C caller can pass null to. Same +/// hazard as the one `topic_name_from_c` closes, minus the error channel. +/// +/// # Safety +/// `name` must be null or a pointer to a NUL-terminated string that stays +/// valid and unwritten for the duration of the call. +unsafe fn endpoint_name_from_c<'a>(name: *const c_char) -> Option<&'a str> { + if name.is_null() { + return None; + } + CStr::from_ptr(name).to_str().ok() +} + /// Why the last publisher/subscriber constructor on THIS thread returned null. /// /// Writes at most `buf_len - 1` bytes plus a NUL and returns the bytes written, @@ -899,16 +916,20 @@ pub struct HorusJsonWireMsg { pub _padding: [u8; 11], } -/// Create JsonWireMessage publisher. +/// Create JsonWireMessage publisher. Returns null on error; +/// `horus_topic_last_error` then holds the reason. #[no_mangle] pub unsafe extern "C" fn horus_publisher_json_wire_new(name: *const c_char) -> *mut HorusPublisher { - let name = match CStr::from_ptr(name).to_str() { - Ok(n) => n, - Err(_) => return std::ptr::null_mut(), + let name = match topic_name_from_c(name) { + Some(n) => n, + None => return std::ptr::null_mut(), }; match topic_ffi::publisher_json_wire_new(name) { Ok(pub_) => Box::into_raw(pub_) as *mut HorusPublisher, - Err(_) => std::ptr::null_mut(), + Err(e) => { + set_topic_error(e); + std::ptr::null_mut() + } } } @@ -938,18 +959,22 @@ pub unsafe extern "C" fn horus_publisher_json_wire_send( topic_ffi::publisher_json_wire_send(pub_, wire); } -/// Create JsonWireMessage subscriber. +/// Create JsonWireMessage subscriber. Returns null on error; +/// `horus_topic_last_error` then holds the reason. #[no_mangle] pub unsafe extern "C" fn horus_subscriber_json_wire_new( name: *const c_char, ) -> *mut HorusSubscriber { - let name = match CStr::from_ptr(name).to_str() { - Ok(n) => n, - Err(_) => return std::ptr::null_mut(), + let name = match topic_name_from_c(name) { + Some(n) => n, + None => return std::ptr::null_mut(), }; match topic_ffi::subscriber_json_wire_new(name) { Ok(sub) => Box::into_raw(sub) as *mut HorusSubscriber, - Err(_) => std::ptr::null_mut(), + Err(e) => { + set_topic_error(e); + std::ptr::null_mut() + } } } @@ -1381,7 +1406,25 @@ pub unsafe extern "C" fn horus_params_set_bool( } } -/// Get param as string. Writes to buf, returns length. -1 if not found. +/// Get param as string. +/// +/// Returns the value's FULL length in bytes, and writes at most `buf_len - 1` +/// of them plus a NUL. A return `>= buf_len` therefore means the value was +/// truncated and tells the caller exactly how large a buffer to re-call with. +/// Returns -1 if the key is absent, or if `buf` is null or `buf_len` is 0 — +/// the same convention as `horus_topic_last_error`. +/// +/// This used to return the number of bytes WRITTEN, which gave a caller no way +/// to tell a complete value from a clipped one, and no way to size around it: +/// the clipped string came back as an ordinary `std::optional` +/// with a value in it. It also returned that positive count for a null `buf`, +/// and split UTF-8 sequences. `horus_topic_last_error` in this file already +/// does none of those things, over inputs that are strictly less arbitrary — +/// a topic name is length-checked before it gets there, while a param value +/// holds paths, URLs and calibration blobs. Nothing anywhere states a length +/// limit for a param; 1024 was an artefact of a stack buffer, which is why +/// Rust, Python and C++ disagreed about what `get("key")` returned for one +/// file. #[no_mangle] pub unsafe extern "C" fn horus_params_get_string( params: *const HorusParams, @@ -1389,7 +1432,7 @@ pub unsafe extern "C" fn horus_params_get_string( buf: *mut u8, buf_len: usize, ) -> i32 { - if params.is_null() || key.is_null() { + if params.is_null() || key.is_null() || buf.is_null() || buf_len == 0 { return -1; } let params = &*(params as *const params_ffi::FfiParams); @@ -1400,12 +1443,23 @@ pub unsafe extern "C" fn horus_params_get_string( match params_ffi::params_get_string(params, key) { Some(v) => { let bytes = v.as_bytes(); - let copy_len = bytes.len().min(buf_len.saturating_sub(1)); - if !buf.is_null() && buf_len > 0 { - std::ptr::copy_nonoverlapping(bytes.as_ptr(), buf, copy_len); - *buf.add(copy_len) = 0; + let mut copy_len = bytes.len().min(buf_len - 1); + // Back off to a char boundary: half a UTF-8 sequence is not + // something to hand a caller that may re-encode it. + while copy_len > 0 && copy_len < bytes.len() && (bytes[copy_len] & 0xC0) == 0x80 { + copy_len -= 1; } - copy_len as i32 + std::ptr::copy_nonoverlapping(bytes.as_ptr(), buf, copy_len); + *buf.add(copy_len) = 0; + // The REQUIRED length, not the written one. + // + // Saturating rather than `as i32`: the plain cast wraps, so a value + // longer than `i32::MAX` came back negative and a caller following + // the documented convention would read it as the `-1` that means + // "no such param". Saturating keeps the sign honest — the answer is + // still "your buffer is too small", which is the only thing a + // caller can act on, and no buffer that size can be satisfied. + i32::try_from(bytes.len()).unwrap_or(i32::MAX) } None => -1, } @@ -1466,9 +1520,9 @@ pub struct HorusServiceServer { /// Create a service client. #[no_mangle] pub unsafe extern "C" fn horus_service_client_new(name: *const c_char) -> *mut HorusServiceClient { - let name = match CStr::from_ptr(name).to_str() { - Ok(n) => n, - Err(_) => return std::ptr::null_mut(), + let name = match endpoint_name_from_c(name) { + Some(n) => n, + None => return std::ptr::null_mut(), }; Box::into_raw(service_ffi::service_client_new(name)) as *mut HorusServiceClient } @@ -1516,9 +1570,9 @@ pub unsafe extern "C" fn horus_service_client_call( /// Create a service server. #[no_mangle] pub unsafe extern "C" fn horus_service_server_new(name: *const c_char) -> *mut HorusServiceServer { - let name = match CStr::from_ptr(name).to_str() { - Ok(n) => n, - Err(_) => return std::ptr::null_mut(), + let name = match endpoint_name_from_c(name) { + Some(n) => n, + None => return std::ptr::null_mut(), }; Box::into_raw(service_ffi::service_server_new(name)) as *mut HorusServiceServer } @@ -1596,9 +1650,9 @@ pub struct HorusActionGoalHandle { /// Create an action client. #[no_mangle] pub unsafe extern "C" fn horus_action_client_new(name: *const c_char) -> *mut HorusActionClient { - let name = match CStr::from_ptr(name).to_str() { - Ok(n) => n, - Err(_) => return std::ptr::null_mut(), + let name = match endpoint_name_from_c(name) { + Some(n) => n, + None => return std::ptr::null_mut(), }; Box::into_raw(action_ffi::action_client_new(name)) as *mut HorusActionClient } @@ -1736,9 +1790,9 @@ pub unsafe extern "C" fn horus_goal_handle_destroy(handle: *mut HorusGoalHandle) /// Create an action server. #[no_mangle] pub unsafe extern "C" fn horus_action_server_new(name: *const c_char) -> *mut HorusActionServer { - let name = match CStr::from_ptr(name).to_str() { - Ok(n) => n, - Err(_) => return std::ptr::null_mut(), + let name = match endpoint_name_from_c(name) { + Some(n) => n, + None => return std::ptr::null_mut(), }; Box::into_raw(action_ffi::action_server_new(name)) as *mut HorusActionServer } @@ -2060,7 +2114,7 @@ pub unsafe extern "C" fn horus_blackbox_record(category: *const c_char, message: let text = format!("[blackbox] {}", msg); let log_type = horus_core::core::log_buffer::LogType::Warning; let entry = horus_core::core::log_buffer::LogEntry { - timestamp: String::new(), + timestamp: String::new(), // publish_log stamps it tick_number: 0, node_name: cat.to_string(), log_type: log_type.clone(), @@ -2108,7 +2162,7 @@ pub unsafe extern "C" fn horus_log(level: u8, node_name: *const c_char, message: _ => horus_core::core::log_buffer::LogType::Info, }; let entry = horus_core::core::log_buffer::LogEntry { - timestamp: String::new(), // filled by log_buffer if empty + timestamp: String::new(), // publish_log stamps it tick_number: 0, node_name: node.to_string(), log_type: log_type.clone(), @@ -2149,6 +2203,210 @@ mod tests { std::env::var_os("HORUS_CPP_FFI_EMIT_CHILD").is_some() } + /// Every topic constructor must refuse a null name and report why. + /// + /// `topic_name_from_c`'s doc says "Every topic constructor starts here, + /// which is what keeps the two invariants the last-error channel rests on". + /// The two JsonWireMessage constructors did not: they called + /// `CStr::from_ptr(name)` directly, which is undefined behaviour on null — + /// the exact "segfault at best" case the guard was written to close — and + /// never cleared or set the error slot, so `horus_topic_last_error` after + /// one of them answered with whatever the previous topic had failed for. + /// + /// json_wire is the transport under every C++ service and action call, so + /// it is the least obscure entry point in the file, not the most. + /// + /// Enumerated by hand rather than by grep: a list that scans the source can + /// only find the constructors that already look like the others. + #[test] + fn every_topic_constructor_refuses_a_null_name_with_a_reason() { + // (label, constructor returns non-null?) + let attempts: Vec<(&str, bool)> = unsafe { + vec![ + ( + "publisher_cmd_vel", + !horus_publisher_cmd_vel_new(std::ptr::null()).is_null(), + ), + ( + "subscriber_cmd_vel", + !horus_subscriber_cmd_vel_new(std::ptr::null()).is_null(), + ), + ( + "publisher_json_wire", + !horus_publisher_json_wire_new(std::ptr::null()).is_null(), + ), + ( + "subscriber_json_wire", + !horus_subscriber_json_wire_new(std::ptr::null()).is_null(), + ), + ( + "publisher_laser_scan", + !horus_publisher_laser_scan_new(std::ptr::null()).is_null(), + ), + ( + "subscriber_laser_scan", + !horus_subscriber_laser_scan_new(std::ptr::null()).is_null(), + ), + ( + "publisher_imu", + !horus_publisher_imu_new(std::ptr::null()).is_null(), + ), + ( + "subscriber_imu", + !horus_subscriber_imu_new(std::ptr::null()).is_null(), + ), + ] + }; + + for (label, returned_non_null) in &attempts { + assert!(!returned_non_null, "{label} accepted a null topic name"); + } + + // And each one must have left its own reason behind. Checked after the + // loop above only for the last call; do it per-call so a constructor + // that forgets `set_topic_error` is caught rather than reading the + // previous one's reason. + unsafe fn reason() -> String { + let mut buf = [0u8; 128]; + let n = horus_topic_last_error(buf.as_mut_ptr(), buf.len()); + assert!(n >= 0, "horus_topic_last_error should not fail here"); + String::from_utf8_lossy(&buf[..n as usize]).into_owned() + } + + unsafe { + let ctors: Vec<(&str, Box)> = vec![ + ( + "publisher_cmd_vel", + Box::new(|| { + horus_publisher_cmd_vel_new(std::ptr::null()); + }), + ), + ( + "subscriber_cmd_vel", + Box::new(|| { + horus_subscriber_cmd_vel_new(std::ptr::null()); + }), + ), + ( + "publisher_json_wire", + Box::new(|| { + horus_publisher_json_wire_new(std::ptr::null()); + }), + ), + ( + "subscriber_json_wire", + Box::new(|| { + horus_subscriber_json_wire_new(std::ptr::null()); + }), + ), + ( + "publisher_laser_scan", + Box::new(|| { + horus_publisher_laser_scan_new(std::ptr::null()); + }), + ), + ( + "subscriber_laser_scan", + Box::new(|| { + horus_subscriber_laser_scan_new(std::ptr::null()); + }), + ), + ( + "publisher_imu", + Box::new(|| { + horus_publisher_imu_new(std::ptr::null()); + }), + ), + ( + "subscriber_imu", + Box::new(|| { + horus_subscriber_imu_new(std::ptr::null()); + }), + ), + ]; + + for (label, call) in ctors { + // Arm a reason that names a DIFFERENT topic, so a constructor + // that neither clears nor sets the slot reads as this one. + set_topic_error("stale reason from some other topic".to_string()); + call(); + let got = reason(); + assert!( + got.contains("null"), + "{label} returned null but left `{got}` in the error slot — \ + it neither cleared nor set its own reason" + ); + } + } + } + + /// A service or action endpoint name must not be dereferenced when null. + /// + /// Not covered by the topic invariant above (these do not touch the topic + /// error slot), but `CStr::from_ptr` is undefined behaviour on null in + /// exactly the same way, and `horus_c.h` is a public ABI. + #[test] + fn service_and_action_constructors_refuse_a_null_name() { + unsafe { + assert!(horus_service_client_new(std::ptr::null()).is_null()); + assert!(horus_service_server_new(std::ptr::null()).is_null()); + assert!(horus_action_client_new(std::ptr::null()).is_null()); + assert!(horus_action_server_new(std::ptr::null()).is_null()); + } + } + + /// A C++ log line must carry a time, like every other language's does. + /// + /// Both C++ entry points built their `LogEntry` with + /// `timestamp: String::new()` and the comment "filled by log_buffer if + /// empty". `publish_log` filled nothing, so `horus log` showed a blank time + /// column for every line a C++ node emitted while the identical `hlog!` + /// call from Rust carried one. Assert against a real Rust entry rather than + /// a hardcoded pattern, so the two cannot drift into different shapes. + #[test] + fn a_cpp_log_line_is_stamped_like_a_rust_one() { + use horus_core::core::log_buffer::{LogType, GLOBAL_LOG_BUFFER}; + + let node = format!("cpp_stamp_{}", std::process::id()); + let name = CString::new(node.clone()).unwrap(); + let msg = CString::new("from a C++ node").unwrap(); + unsafe { horus_log(0, name.as_ptr(), msg.as_ptr()) }; + + let cat = CString::new(node.clone()).unwrap(); + let event = CString::new("estop").unwrap(); + unsafe { horus_blackbox_record(cat.as_ptr(), event.as_ptr()) }; + + // The Rust side of the comparison, through the real Rust path. + let rust_node = format!("rust_stamp_{}", std::process::id()); + horus_core::core::hlog::log_as_node(LogType::Info, &rust_node, "from a Rust node"); + + let cpp = GLOBAL_LOG_BUFFER.for_node(&node); + assert_eq!(cpp.len(), 2, "both C++ entry points should have logged"); + let rust = GLOBAL_LOG_BUFFER.for_node(&rust_node); + assert_eq!(rust.len(), 1); + let reference = &rust[0].timestamp; + + for entry in &cpp { + assert!( + !entry.timestamp.is_empty(), + "a C++ log entry reached the buffer with no timestamp: {entry:?}" + ); + assert_eq!( + entry.timestamp.len(), + reference.len(), + "C++ stamp {:?} is not the shape Rust writes ({reference:?})", + entry.timestamp + ); + // HH:MM:SS.mmm — same separators in the same places. + let shape = |t: &str| { + t.chars() + .map(|c| if c.is_ascii_digit() { 'd' } else { c }) + .collect::() + }; + assert_eq!(shape(&entry.timestamp), shape(reference)); + } + } + /// Re-run `test_name` in a child process and hand back its stderr. fn child_stderr(test_name: &str) -> String { let exe = std::env::current_exe().expect("test binary path"); @@ -2405,6 +2663,103 @@ mod tests { } } + /// A param value must not come back silently clipped. + /// + /// `horus_params_get_string` returned the bytes WRITTEN, so a caller could + /// not tell a complete value from a truncated one and had no way to size a + /// buffer around it: `Params::get_string` handed back an ordinary + /// `std::optional` with a clipped value in it. Rust and Python + /// always returned the whole string; only C++ clipped, over inputs — paths, + /// URLs, calibration blobs — that are strictly more arbitrary than the + /// topic names `horus_topic_last_error` already handles correctly in this + /// same file. + #[test] + fn params_get_string_reports_the_length_it_needed() { + unsafe { + let params = horus_params_new(); + let key = CString::new("calibration").unwrap(); + let value = "x".repeat(2000); + let cvalue = CString::new(value.as_str()).unwrap(); + assert_eq!( + horus_params_set_string(params, key.as_ptr(), cvalue.as_ptr()), + 0 + ); + + let mut buf = [0xAAu8; 16]; + let n = horus_params_get_string(params, key.as_ptr(), buf.as_mut_ptr(), buf.len()); + assert_eq!( + n, 2000, + "the REQUIRED length, so a caller can size a buffer that holds \ + the whole value" + ); + assert_eq!(buf[15], 0, "the terminator must still fit"); + + // Sized from the reported length, the value arrives whole. + let mut big = vec![0u8; n as usize + 1]; + let again = horus_params_get_string(params, key.as_ptr(), big.as_mut_ptr(), big.len()); + assert_eq!(again, 2000); + assert_eq!( + std::str::from_utf8(&big[..2000]).unwrap(), + value, + "the second call must return the complete value" + ); + + horus_params_destroy(params); + } + } + + /// A null buffer is an error, not a positive byte count. + #[test] + fn params_get_string_refuses_a_null_buffer() { + unsafe { + let params = horus_params_new(); + let key = CString::new("name").unwrap(); + let v = CString::new("lidar").unwrap(); + assert_eq!(horus_params_set_string(params, key.as_ptr(), v.as_ptr()), 0); + + assert_eq!( + horus_params_get_string(params, key.as_ptr(), std::ptr::null_mut(), 64), + -1, + "a null buffer used to return a positive 'bytes written'" + ); + let mut buf = [0u8; 8]; + assert_eq!( + horus_params_get_string(params, key.as_ptr(), buf.as_mut_ptr(), 0), + -1 + ); + + horus_params_destroy(params); + } + } + + /// Truncation must not split a UTF-8 sequence. + #[test] + fn params_get_string_truncates_on_a_char_boundary() { + unsafe { + let params = horus_params_new(); + let key = CString::new("label").unwrap(); + // Three-byte characters, so a naive byte cut lands mid-sequence. + let value = "\u{4e2d}".repeat(10); + let cvalue = CString::new(value.as_str()).unwrap(); + assert_eq!( + horus_params_set_string(params, key.as_ptr(), cvalue.as_ptr()), + 0 + ); + + let mut buf = [0xAAu8; 8]; + let n = horus_params_get_string(params, key.as_ptr(), buf.as_mut_ptr(), buf.len()); + assert_eq!(n, 30, "the full length is still reported"); + + let nul = buf.iter().position(|&b| b == 0).expect("NUL written"); + assert!( + std::str::from_utf8(&buf[..nul]).is_ok(), + "no split UTF-8 sequence — the caller may re-encode this" + ); + + horus_params_destroy(params); + } + } + #[test] fn c_api_params_all_types() { unsafe { diff --git a/horus_cpp/tests/cpp_user_api_test.cpp b/horus_cpp/tests/cpp_user_api_test.cpp index dbf2f215c..37727d5ec 100644 --- a/horus_cpp/tests/cpp_user_api_test.cpp +++ b/horus_cpp/tests/cpp_user_api_test.cpp @@ -177,6 +177,46 @@ TEST(UserApi, RuntimeParams) { EXPECT_FALSE(params.has("nope")); } +// A value longer than the internal stack buffer must come back WHOLE. +// +// `Params::get_string` reads into a 1024-byte stack buffer. It used to return +// whatever fitted, as an ordinary std::optional with no way for a +// caller to tell a clipped value from a complete one — and it clipped on a raw +// byte boundary, so a UTF-8 value could come back invalid. Rust and Python +// always returned the whole string; only C++ clipped, over values that hold +// paths, URLs and calibration blobs. +// +// The shim reports the REQUIRED length now and this wrapper re-fetches when the +// value does not fit. That re-call is the part with an off-by-one to get wrong, +// and nothing exercised it. +TEST(UserApi, RuntimeParamsLongStringRoundTrips) { + horus::Params params; + + // Either side of the 1024-byte stack buffer, and exactly on it. + for (size_t len : {1023u, 1024u, 1025u, 4096u}) { + const std::string value(len, 'x'); + params.set("blob", value.c_str()); + auto got = params.get_string("blob"); + ASSERT_TRUE(got.has_value()) << "length " << len; + EXPECT_EQ(got->size(), len) << "truncated at length " << len; + EXPECT_EQ(*got, value) << "content differs at length " << len; + } +} + +// Truncation, when a caller drives the C shim directly, must not split UTF-8. +TEST(UserApi, RuntimeParamsTruncationKeepsValidUtf8) { + horus::Params params; + // Three-byte characters, so a naive byte cut lands mid-sequence. + std::string value; + for (int i = 0; i < 10; ++i) value += "\xe4\xb8\xad"; + params.set("label", value.c_str()); + + // The wrapper always returns the whole value. + auto whole = params.get_string("label"); + ASSERT_TRUE(whole.has_value()); + EXPECT_EQ(*whole, value); +} + // ═══════════════════════════════════════════════════════════════════════════ // 5. TransformFrame // ═══════════════════════════════════════════════════════════════════════════ diff --git a/horus_manager/src/cargo_gen.rs b/horus_manager/src/cargo_gen.rs index d29503fd9..2fc81ebbf 100644 --- a/horus_manager/src/cargo_gen.rs +++ b/horus_manager/src/cargo_gen.rs @@ -120,8 +120,8 @@ pub fn generate( // Add user deps from horus.toml write_deps_section(&mut cargo, &manifest.dependencies, project_dir, &horus_dir)?; - // Add driver dependencies from [drivers] config tables - write_driver_deps(&mut cargo, &manifest.drivers); + // Add driver dependencies from the [hardware]/[drivers] config tables + write_driver_deps(&mut cargo, &manifest.hardware_entries()); // ── Dev dependencies ───────────────────────────────────────────────── if include_dev && !manifest.dev_dependencies.is_empty() { @@ -871,7 +871,7 @@ fn writable_by_horus(config_path: &Path) -> bool { /// - `camera = "opencv"` / `gps = true` → no dependency (legacy, handled by feature flags) fn write_driver_deps( cargo: &mut String, - drivers: &std::collections::BTreeMap, + drivers: &std::collections::BTreeMap<&str, &crate::manifest::DriverValue>, ) { use crate::manifest::DriverValue; @@ -882,7 +882,7 @@ fn write_driver_deps( let mut added_crates: BTreeMap, Vec)> = BTreeMap::new(); for value in drivers.values() { - match value { + match *value { DriverValue::Config(cfg) => { // Terra drivers: no longer auto-resolved — user adds terra-horus // as a normal dependency in [dependencies]. Skip terra entries. @@ -916,7 +916,16 @@ fn write_driver_deps( } if !added_crates.is_empty() { - writeln!(cargo, "\n# Driver dependencies (from [drivers])").unwrap(); + // "[hardware] or [drivers]", not just the former: these entries come + // from `Manifest::hardware_entries()`, which merges `[hardware]` with + // the legacy `[drivers]` table. A user still on `[drivers]` would read + // the old header, look at a `[hardware]` table they do not have, and + // conclude the generator had invented the dependency. + writeln!( + cargo, + "\n# Driver dependencies (from [hardware], or legacy [drivers])" + ) + .unwrap(); for (crate_name, (version, features)) in &added_crates { let ver = version.as_deref().unwrap_or("*"); if features.is_empty() { @@ -987,6 +996,28 @@ fn write_implicit_deps(cargo: &mut String, manifest: &HorusManifest) { } } +/// Whether the `net` capability was asked for, by either route that works. +/// +/// This is the condition that becomes `cargo --features net`, and therefore the +/// only thing that decides whether the built binary contains horus_net at all. +/// +/// Extracted so `horus doctor` can ask the same question the build asks. +/// `doctor` used to answer it from `HORUS_NET`, which nothing else in the tree +/// reads — so it told operators to set a variable that changed nothing except +/// what `doctor` then said about it. +pub fn net_capability_requested(manifest: &HorusManifest) -> bool { + std::env::var("HORUS_ENABLE") + .map(|v| { + v.split(',') + .any(|c| matches!(c.trim().to_lowercase().as_str(), "net" | "network")) + }) + .unwrap_or(false) + || manifest + .enable + .iter() + .any(|c| matches!(c.to_lowercase().as_str(), "net" | "network")) +} + /// Features to enable on the `horus` dependency. /// /// `horus run --net` used to set `HORUS_NET=1` and stop there. That variable is @@ -1022,16 +1053,7 @@ fn horus_dep_features(manifest: &HorusManifest) -> Vec { // driver needs a feature on `horus`. let mut features: Vec = Vec::new(); - let requested_net = std::env::var("HORUS_ENABLE") - .map(|v| { - v.split(',') - .any(|c| matches!(c.trim().to_lowercase().as_str(), "net" | "network")) - }) - .unwrap_or(false) - || manifest - .enable - .iter() - .any(|c| matches!(c.to_lowercase().as_str(), "net" | "network")); + let requested_net = net_capability_requested(manifest); if requested_net && !features.iter().any(|f| f == "net") { features.push("net".to_string()); @@ -2274,6 +2296,54 @@ mod horus_dep_feature_tests { assert!(with_enable(Some("NET"), || horus_dep_features(&m)).contains(&"net".to_string())); } + /// `horus doctor` must answer from the same condition the build uses. + /// + /// It used to answer from `HORUS_NET`, which nothing in the tree reads, and + /// print "use --net or HORUS_NET=1 to enable" — advice whose second half + /// changed nothing except that `doctor` then reported networking as + /// enabled, confirming a state that did not exist. Worse than inert. + #[test] + fn horus_net_does_not_make_a_build_networked() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + let m = HorusManifest::default(); + + let prev = std::env::var("HORUS_NET").ok(); + // SAFETY: ENV_LOCK serialises the tests that touch process-global env. + unsafe { std::env::set_var("HORUS_NET", "1") }; + let answered = net_capability_requested(&m); + match prev { + Some(v) => unsafe { std::env::set_var("HORUS_NET", v) }, + None => unsafe { std::env::remove_var("HORUS_NET") }, + } + + assert!( + !answered, + "HORUS_NET=1 must not report a networked build — it reaches no \ + build step, so reporting it as enabled tells the operator a \ + falsehood they cannot act on" + ); + } + + /// ...and the routes that DO work must all answer yes. + #[test] + fn the_documented_routes_all_request_the_net_capability() { + let m = HorusManifest::default(); + // `horus run --net` appends "net" to the capability list, which becomes + // HORUS_ENABLE. + assert!(with_enable(Some("net"), || net_capability_requested(&m))); + assert!(with_enable(Some("network"), || net_capability_requested( + &m + ))); + + // `enable = ["net"]` in horus.toml. + let mut manifest = HorusManifest::default(); + manifest.enable = vec!["net".to_string()]; + assert!(with_enable(None, || net_capability_requested(&manifest))); + + // And nothing at all means no. + assert!(!with_enable(None, || net_capability_requested(&m))); + } + /// It must be opt-in: an ordinary build must not link the network stack. #[test] fn net_is_absent_by_default() { @@ -2817,6 +2887,71 @@ mod tests { ); } + /// `[hardware]` is the documented preferred spelling and `[drivers]` the + /// legacy one, yet only `[drivers]` used to reach the generator — so a + /// crates.io driver declared the preferred way produced no Cargo + /// dependency at all, and the project failed to build with an + /// unresolved-crate error naming something the user had already declared. + #[test] + fn a_driver_under_hardware_generates_the_same_dep_as_under_drivers() { + let cfg = DriverTableConfig { + crate_name: Some("rplidar-driver".into()), + ..Default::default() + }; + + let mut under_hardware = test_manifest(BTreeMap::new()); + under_hardware + .hardware + .insert("lidar".into(), DriverValue::Config(cfg.clone())); + + let mut under_drivers = test_manifest(BTreeMap::new()); + under_drivers + .drivers + .insert("lidar".into(), DriverValue::Config(cfg)); + + let mut from_hardware = String::new(); + write_driver_deps(&mut from_hardware, &under_hardware.hardware_entries()); + let mut from_drivers = String::new(); + write_driver_deps(&mut from_drivers, &under_drivers.hardware_entries()); + + assert!( + from_hardware.contains("rplidar-driver"), + "a [hardware] driver must generate its crate dep, got: {from_hardware:?}" + ); + assert_eq!( + from_hardware, from_drivers, + "the preferred spelling must not do less than the legacy one" + ); + } + + /// A name in both tables is one piece of hardware, not two deps. + #[test] + fn hardware_wins_over_a_same_named_legacy_driver() { + let mut manifest = test_manifest(BTreeMap::new()); + manifest.hardware.insert( + "arm".into(), + DriverValue::Config(DriverTableConfig { + crate_name: Some("new-arm".into()), + ..Default::default() + }), + ); + manifest.drivers.insert( + "arm".into(), + DriverValue::Config(DriverTableConfig { + crate_name: Some("old-arm".into()), + ..Default::default() + }), + ); + + let mut out = String::new(); + write_driver_deps(&mut out, &manifest.hardware_entries()); + assert!(out.contains("new-arm"), "got: {out:?}"); + assert!( + !out.contains("old-arm"), + "the legacy entry must not shadow-add a second dep, got: {out:?}" + ); + } + #[test] fn battle_driver_features_propagation_camera_lidar() { // Drivers with camera + lidar should propagate features to horus/horus_library path deps. diff --git a/horus_manager/src/commands/doctor.rs b/horus_manager/src/commands/doctor.rs index 44898e313..8180033a4 100644 --- a/horus_manager/src/commands/doctor.rs +++ b/horus_manager/src/commands/doctor.rs @@ -1224,15 +1224,7 @@ fn check_drivers(ctx: &dispatch::ProjectContext) -> CheckResult { } }; - // Merge [hardware] and legacy [drivers] entries - let mut all_entries: std::collections::BTreeMap = - std::collections::BTreeMap::new(); - for (k, v) in &manifest.hardware { - all_entries.insert(k.clone(), v); - } - for (k, v) in &manifest.drivers { - all_entries.entry(k.clone()).or_insert(v); - } + let all_entries = manifest.hardware_entries(); if all_entries.is_empty() { return CheckResult { @@ -1280,7 +1272,7 @@ fn check_drivers(ctx: &dispatch::ProjectContext) -> CheckResult { let urdf_path = std::path::Path::new(desc); let sensors = crate::urdf::extract_sensors_from_urdf(urdf_path); if !sensors.is_empty() { - let hw_keys: Vec<&str> = all_entries.keys().map(|s| s.as_str()).collect(); + let hw_keys: Vec<&str> = all_entries.keys().copied().collect(); let warnings = crate::urdf::validate_driver_keys(&hw_keys, &sensors); for warning in warnings { details.push(format!(" ! {}", warning)); @@ -1874,9 +1866,18 @@ fn check_network() -> CheckResult { details.push(format!("Namespace: {}", namespace)); // Check 3: horus_net enabled check - let net_enabled = std::env::var("HORUS_NET") - .map(|v| v == "1" || v == "true") - .unwrap_or(false); + // + // Asked the way the BUILD asks it. This used to read `HORUS_NET`, which + // nothing else in the tree reads — so the advice it printed + // ("use --net or HORUS_NET=1 to enable") handed the operator a remedy that + // changed nothing except that `doctor` then reported networking as enabled, + // confirming a state that did not exist. What actually puts horus_net in + // the binary is the `net` capability reaching cargo_gen, via `enable` in + // horus.toml or `HORUS_ENABLE`; `horus run --net` works because it appends + // to that list. + let manifest = crate::manifest::HorusManifest::load_from(std::path::Path::new("horus.toml")) + .unwrap_or_default(); + let net_enabled = crate::cargo_gen::net_capability_requested(&manifest); let no_net = std::env::var("HORUS_NO_NETWORK") .map(|v| v == "1" || v == "true") .unwrap_or(false); @@ -1886,7 +1887,10 @@ fn check_network() -> CheckResult { } else if net_enabled { details.push("horus_net: enabled".to_string()); } else { - details.push("horus_net: available (use --net or HORUS_NET=1 to enable)".to_string()); + details.push( + "horus_net: available (enable it with `horus run --net`, enable = [\"net\"] in horus.toml, or HORUS_ENABLE=net)" + .to_string(), + ); } // Check 4: multicast reachability (basic socket test) diff --git a/horus_manager/src/commands/launch.rs b/horus_manager/src/commands/launch.rs index 196f09ccc..27e3e50d5 100644 --- a/horus_manager/src/commands/launch.rs +++ b/horus_manager/src/commands/launch.rs @@ -72,7 +72,17 @@ pub struct LaunchNode { #[serde(default)] pub args: Vec, - /// Namespace prefix for topics + /// Shared-memory tree for THIS node, overriding the file-level one. + /// + /// Not a topic prefix. There is no prefix or remap mechanism in horus_core; + /// `HORUS_NAMESPACE` names the shared-memory tree, which is isolation. This + /// is exported after the file-level value so the more specific one wins. + /// + /// **A node with its own namespace cannot see its siblings' topics, and + /// they cannot see its.** That is what a separate SHM tree means, and it is + /// the whole effect of this key — so set it when you want that node + /// isolated, and leave it unset otherwise. If you wanted a topic prefix, + /// this is not it, and HORUS does not have one. #[serde(default)] pub namespace: Option, @@ -690,10 +700,25 @@ pub fn run_launch( // Check if any process has exited let mut stopped_indices: Vec = Vec::new(); + // What to record for nodes that leave `processes` this iteration. + // + // The session manifest is what `horus launch --status` reads, and its + // STATUS column has "crashed" and "stopped" arms that nothing could + // reach: `status` was assigned exactly once, `"running"` at + // construction, and never mutated. So a node that died was still listed + // as running — and this very loop already knows it died, because it + // logs `NodeCrashed` two lines down. + let mut departed: Vec<(String, &'static str)> = Vec::new(); for (i, entry) in processes.iter_mut().enumerate() { match entry.child.try_wait() { Ok(Some(status)) => { let should_restart = should_restart_node(&entry.node.restart, &status); + // A non-zero exit is a crash; a clean one is an ordinary stop. + let final_status = if status.success() { + "stopped" + } else { + "crashed" + }; if !status.success() { println!( @@ -760,6 +785,7 @@ pub fn run_launch( e ); stopped_indices.push(i); + departed.push((entry.name.clone(), final_status)); } } } else if should_restart { @@ -770,8 +796,10 @@ pub fn run_launch( MAX_RESTARTS, ); stopped_indices.push(i); + departed.push((entry.name.clone(), final_status)); } else { stopped_indices.push(i); + departed.push((entry.name.clone(), final_status)); } } Ok(None) => {} // Still running @@ -786,13 +814,18 @@ pub fn run_launch( processes.remove(i); } - // Update session manifest with current process states - for (i, entry) in processes.iter().enumerate() { - if let Some(se) = session.processes.get_mut(i) { - se.pid = entry.child.id(); - se.restart_count = entry.restart_count; - } - } + // Update session manifest with current process states. + // + // Keyed by NAME, not by position. The positional form went wrong the + // moment anything was removed from `processes`: the indices shifted, so + // a surviving node's PID was written into a dead node's row, and an + // operator asking whether the perception node crashed was told + // "running" beside a PID belonging to a different, healthy process. + let live: Vec<(String, u32, u32)> = processes + .iter() + .map(|e| (e.name.clone(), e.child.id(), e.restart_count)) + .collect(); + refresh_session_manifest(&mut session, &live, &departed); // Re-discover Schedulers every ~2s (every 20 iterations) if session.processes.iter().any(|e| e.scheduler_name.is_none()) { session.discover_schedulers(); @@ -885,6 +918,43 @@ pub fn run_launch( Ok(()) } +/// Write the supervisor's view of each node back into the session manifest. +/// +/// `live` is `(name, pid, restart_count)` for every node still supervised; +/// `departed` names the nodes that left this iteration and why. +/// +/// Keyed by NAME, not by position. The positional form this replaces went wrong +/// the moment anything was removed from the live vector: the indices shifted, +/// so a surviving node's PID was written into a dead node's row, and an +/// operator asking whether the perception node crashed was told "running" +/// beside a PID belonging to a different, healthy process. +/// +/// It also sets `status`, which nothing used to mutate. `show_launch_status` +/// colours "running", "crashed" and "stopped", but the field was assigned +/// exactly once — `"running"` at construction — so the other two arms could not +/// be reached and every node read as running forever. The monitor loop already +/// knows the outcome: it logs `NodeCrashed` in the same iteration. +fn refresh_session_manifest( + session: &mut LaunchSession, + live: &[(String, u32, u32)], + departed: &[(String, &'static str)], +) { + for (name, status) in departed { + if let Some(se) = session.processes.iter_mut().find(|se| &se.name == name) { + se.status = (*status).to_string(); + } + } + for (name, pid, restart_count) in live { + if let Some(se) = session.processes.iter_mut().find(|se| &se.name == name) { + se.pid = *pid; + se.restart_count = *restart_count; + // Still supervised — including a node that crashed a moment ago and + // was successfully respawned. + se.status = "running".to_string(); + } + } +} + /// Show active launch sessions read from SHM manifests. pub fn show_launch_status() -> HorusResult<()> { let sessions = LaunchSession::read_all(); @@ -1008,16 +1078,43 @@ fn should_restart_node(policy: &str, status: &std::process::ExitStatus) -> bool } } +/// The shared-memory tree a node is launched into: its own, else the file's. +/// +/// One function so `run_launch` and `print_launch_plan` cannot disagree about +/// which value wins — `--dry-run` describing a launch that does not happen is +/// the defect this file has already had twice. +fn effective_namespace<'a>( + file_level: Option<&'a str>, + node_level: Option<&'a str>, +) -> Option<&'a str> { + node_level.or(file_level) +} + /// Print the launch plan (dry run) fn print_launch_plan(config: &LaunchConfig, global_namespace: &Option) { for (i, node) in config.nodes.iter().enumerate() { - let full_name = match (global_namespace, &node.namespace) { - (Some(global), Some(local)) => format!("{}/{}/{}", global, local, node.name), - (Some(ns), None) | (None, Some(ns)) => format!("{}/{}", ns, node.name), - (None, None) => node.name.clone(), - }; - - println!(" {}. {}", i + 1, full_name.white().bold()); + // The name the child is actually given (`HORUS_NODE_NAME`), which is + // the bare one. This used to compose `global/local/name` — an identity + // no process ever receives, so `--dry-run` described a launch that does + // not happen. Both namespaces are shown separately below, because they + // do something: each names an SHM tree, and a node-level one overrides + // the file-level value for that node alone. + println!(" {}. {}", i + 1, node.name.white().bold()); + match ( + effective_namespace(global_namespace.as_deref(), node.namespace.as_deref()), + node.namespace.is_some(), + ) { + (Some(ns), true) => println!( + " {} {} {}", + "Namespace:".dimmed(), + ns, + "(shared-memory tree — isolated from the other nodes)".yellow() + ), + (Some(ns), false) => { + println!(" {} {} (shared-memory tree)", "Namespace:".dimmed(), ns) + } + (None, _) => {} + } if let Some(ref pkg) = node.package { println!(" {} {}", "Package:".dimmed(), pkg); @@ -1186,10 +1283,7 @@ fn launch_node( // // Node-level last so it wins: a setting on the node is more specific than // the file's default, which is the only reason to write one. - if let Some(ref ns) = namespace { - cmd.env("HORUS_NAMESPACE", ns); - } - if let Some(ref ns) = node.namespace { + if let Some(ns) = effective_namespace(namespace.as_deref(), node.namespace.as_deref()) { cmd.env("HORUS_NAMESPACE", ns); } @@ -1531,23 +1625,22 @@ nodes: ); } + /// A node-level `namespace:` overrides the file-level one for that node. + /// + /// This test used to compute `global/local/node` from its own inputs and + /// assert the result equalled what it had just computed — it called nothing + /// from this module and would have passed against any implementation. The + /// composed identity it described is gone (no process was ever given it), + /// and what the key does now is select a shared-memory tree. #[test] - fn node_namespace_construction() { - // Mirrors the logic in run_launch/print_launch_plan - let cases = vec![ - (Some("global"), Some("local"), "node", "global/local/node"), - (Some("global"), None, "node", "global/node"), - (None, Some("local"), "node", "local/node"), - (None, None, "node", "node"), - ]; - for (global, local, name, expected) in cases { - let full = match (global, local) { - (Some(g), Some(l)) => format!("{}/{}/{}", g, l, name), - (Some(ns), None) | (None, Some(ns)) => format!("{}/{}", ns, name), - (None, None) => name.to_string(), - }; - assert_eq!(full, expected); - } + fn a_node_level_namespace_overrides_the_file_level_one() { + assert_eq!( + effective_namespace(Some("/robot"), Some("/arm")), + Some("/arm") + ); + assert_eq!(effective_namespace(Some("/robot"), None), Some("/robot")); + assert_eq!(effective_namespace(None, Some("/arm")), Some("/arm")); + assert_eq!(effective_namespace(None, None), None); } #[test] @@ -2500,3 +2593,95 @@ nodes: n } } + +#[cfg(test)] +mod session_manifest_tests { + use super::{refresh_session_manifest, LaunchProcessEntry, LaunchSession}; + + fn entry(name: &str, pid: u32) -> LaunchProcessEntry { + LaunchProcessEntry { + name: name.to_string(), + pid, + status: "running".to_string(), + scheduler_name: None, + control_topic: None, + node_names: Vec::new(), + restart_count: 0, + } + } + + fn session(names: &[(&str, u32)]) -> LaunchSession { + LaunchSession { + session: "s".to_string(), + launcher_pid: 1, + start_time: 0, + launch_file: "robot.launch.yaml".to_string(), + namespace: None, + processes: names.iter().map(|(n, p)| entry(n, *p)).collect(), + } + } + + /// A node that died must not be listed as running, beside somebody else's PID. + /// + /// The refresh used to walk the live vector and the manifest in parallel by + /// INDEX. The monitor loop removes a dead node from the live vector, so + /// from that point the two are misaligned: the survivor's PID was written + /// into the dead node's row. Combined with `status` never being mutated — + /// it was assigned `"running"` once at construction, and + /// `show_launch_status`'s "crashed" and "stopped" arms were unreachable — + /// `horus launch --status` told an operator the crashed perception node was + /// running, and showed a healthy process's PID next to it. + #[test] + fn a_departed_node_is_not_reported_as_running_with_a_live_pid() { + let mut s = session(&[("perception", 100), ("control", 200), ("logger", 300)]); + + // `perception` crashed and was dropped from the supervisor's vector; + // the other two are still up. + let live = vec![ + ("control".to_string(), 200u32, 0u32), + ("logger".to_string(), 300u32, 0u32), + ]; + let departed = vec![("perception".to_string(), "crashed")]; + + refresh_session_manifest(&mut s, &live, &departed); + + let perception = s.processes.iter().find(|e| e.name == "perception").unwrap(); + assert_eq!( + perception.status, "crashed", + "the loop that logs NodeCrashed already knows this node died" + ); + assert_eq!( + perception.pid, 100, + "a dead node must keep its own PID — the positional refresh wrote \ + the SURVIVOR's pid into this row" + ); + + let control = s.processes.iter().find(|e| e.name == "control").unwrap(); + assert_eq!(control.status, "running"); + assert_eq!(control.pid, 200, "a live node keeps its own PID"); + let logger = s.processes.iter().find(|e| e.name == "logger").unwrap(); + assert_eq!(logger.pid, 300); + } + + /// A clean exit is "stopped", not "crashed". + #[test] + fn a_node_that_exited_cleanly_reads_as_stopped() { + let mut s = session(&[("oneshot", 42)]); + refresh_session_manifest(&mut s, &[], &[("oneshot".to_string(), "stopped")]); + assert_eq!(s.processes[0].status, "stopped"); + } + + /// A node that crashed and was respawned is running again. + #[test] + fn a_respawned_node_reads_as_running_again() { + let mut s = session(&[("flaky", 7)]); + s.processes[0].status = "crashed".to_string(); + + // Back in the live set under a new PID, with a restart recorded. + refresh_session_manifest(&mut s, &[("flaky".to_string(), 99, 1)], &[]); + + assert_eq!(s.processes[0].status, "running"); + assert_eq!(s.processes[0].pid, 99); + assert_eq!(s.processes[0].restart_count, 1); + } +} diff --git a/horus_manager/src/commands/node.rs b/horus_manager/src/commands/node.rs index 64a50ad09..b02890722 100644 --- a/horus_manager/src/commands/node.rs +++ b/horus_manager/src/commands/node.rs @@ -580,7 +580,7 @@ pub fn kill_node(name: &str, force: bool) -> HorusResult<()> { Ok(()) } -/// Restart a node (pause + resume via control topic to trigger re-init) +/// Restart a node — asks the scheduler to re-run its `init()` on the next tick. pub fn restart_node(name: &str) -> HorusResult<()> { let nodes = discover_nodes()?; let node = find_node(&nodes, name)?; @@ -591,16 +591,13 @@ pub fn restart_node(name: &str) -> HorusResult<()> { node.name.white().bold() ); - // Pause then resume triggers re-init in the scheduler + // Pause+Resume was sent here and claimed to trigger re-init. It did not: + // both arms only toggle a shared atomic, and nothing cleared the flag that + // `reinit_pending_nodes` keys off, so the node kept running untouched while + // this printed success. send_control_command( node, - ControlCommand::PauseNode { - name: node.name.clone(), - }, - )?; - send_control_command( - node, - ControlCommand::ResumeNode { + ControlCommand::RestartNode { name: node.name.clone(), }, )?; diff --git a/horus_manager/src/commands/plugin.rs b/horus_manager/src/commands/plugin.rs index d1bb921f9..6ece15c1e 100644 --- a/horus_manager/src/commands/plugin.rs +++ b/horus_manager/src/commands/plugin.rs @@ -443,11 +443,17 @@ fn resolve_package_dir(name: &str, version: &str, global: bool) -> Option, local: bool) -> HorusResult<()> { - // Plugins default to global installation (CLI-wide) - let global = !local; - +/// Pass `-t ` to install into a registered workspace instead. +/// +/// `target` used to be collapsed to a `bool` at the call site — `let local = +/// target.is_some()` — and the NAME was thrown away. What was left then called +/// `detect_or_select_workspace`, which resolves the CURRENT DIRECTORY's +/// workspace or, outside any workspace, prints a menu and blocks on stdin. Both +/// of those are things `horus install` documents that it never does, and with +/// no tty the stdin read returns empty and the command fails with "Invalid +/// selection" rather than installing where it was told. The package half of the +/// same flag already resolved the name properly; this now does the same thing. +pub fn run_install(plugin: String, ver: Option, target: Option) -> HorusResult<()> { println!( "{} Installing plugin: {}{}", cli_output::ICON_INFO.magenta().bold(), @@ -456,21 +462,26 @@ pub fn run_install(plugin: String, ver: Option, local: bool) -> HorusRes ); // Determine install target - plugins default to global - let install_target = if global { - println!(" Scope: {}", "global (user cache)".dimmed()); - workspace::InstallTarget::Global - } else { - let target = workspace::detect_or_select_workspace(true) - .map_err(|e| HorusError::Config(ConfigError::Other(e.to_string())))?; - match &target { - workspace::InstallTarget::Local(p) => { - println!(" Scope: {}", format!("local ({})", p.display()).dimmed()); - } - workspace::InstallTarget::Global => { - println!(" Scope: {}", "global (user cache)".dimmed()); - } + let install_target = match target { + None => { + println!(" Scope: {}", "global (user cache)".dimmed()); + workspace::InstallTarget::Global + } + Some(target_name) => { + let registry = workspace::WorkspaceRegistry::load() + .map_err(|e| HorusError::Config(ConfigError::Other(e.to_string())))?; + let ws = registry.find_by_name(&target_name).ok_or_else(|| { + HorusError::Config(ConfigError::Other(format!( + "Workspace '{}' not found", + target_name + ))) + })?; + println!( + " Scope: {}", + format!("local ({})", ws.path.display()).dimmed() + ); + workspace::InstallTarget::Local(ws.path.clone()) } - target }; // Install via the existing package install flow. Without a registry there @@ -1423,4 +1434,38 @@ version = "2.0.0" ); } } + + /// `-t ` on a plugin install must name a REGISTERED WORKSPACE. + /// + /// It used to be collapsed to `let local = target.is_some()` at the call + /// site, throwing the name away, and what was left called + /// `detect_or_select_workspace`. That resolves the workspace of the + /// directory you happen to be standing in or, outside any workspace, prints + /// a menu and blocks on stdin — the two things `horus install` documents it + /// never does. With no tty the read comes back empty and the command dies + /// with "Invalid selection" instead of installing where it was asked. + /// + /// An unknown name must therefore fail by NAME, before any registry or + /// network work, exactly as the package half of the same flag does. + #[test] + fn a_plugin_target_names_a_workspace_not_the_current_directory() { + let err = super::run_install( + "some-plugin".to_string(), + None, + Some("no_such_workspace_xyzzy".to_string()), + ) + .expect_err("an unknown workspace name must be an error, not a prompt"); + + let msg = err.to_string(); + assert!( + msg.contains("no_such_workspace_xyzzy"), + "the error must name the workspace that was asked for, so the user \ + can see it was taken as a name rather than ignored: {msg}" + ); + assert!( + msg.contains("not found"), + "an unregistered workspace name is 'not found', not a selection \ + prompt or a silent fall back to the current directory: {msg}" + ); + } } diff --git a/horus_manager/src/commands/run/mod.rs b/horus_manager/src/commands/run/mod.rs index 74d8b9357..70267552e 100644 --- a/horus_manager/src/commands/run/mod.rs +++ b/horus_manager/src/commands/run/mod.rs @@ -73,24 +73,34 @@ enum ExecutionTarget { /// `AllowList` were unreachable in shipped code (the import guard was /// permanently `Auto`). The 2026-07-30 audit flagged that as a live safety gap. /// +/// That remedy was then wired into ONE of the five paths that spawn a run +/// child — `execute_multiple_files`, reached only when `horus run` is handed +/// more than one file. A bare `horus run`, a `horus.toml` run, a directory run +/// and a `[workspace]` run all resolve to a single executable, so on every +/// invocation a real single-binary robot makes, the safety gap those comments +/// describe was still open. Hence pairs rather than a `Command`: the same +/// translation now feeds `build_child_env`, which every single-executable path +/// already uses, so there is one answer and no path can quietly miss it. +/// /// An environment variable already set by the operator wins: an explicit /// `HORUS_NET_*=...` on the command line should not be silently overridden by /// a value in a project file. -fn apply_network_config(cmd: &mut std::process::Command) { +fn network_env() -> Vec<(String, String)> { + let mut out: Vec<(String, String)> = Vec::new(); let manifest_path = Path::new(HORUS_TOML); if !manifest_path.exists() { - return; + return out; } let Ok(manifest) = HorusManifest::load_from(manifest_path) else { - return; + return out; }; let Some(net) = manifest.network.as_ref() else { - return; + return out; }; let mut set = |key: &str, value: String| { if std::env::var_os(key).is_none() { - cmd.env(key, value); + out.push((key.to_string(), value)); } }; @@ -139,6 +149,17 @@ fn apply_network_config(cmd: &mut std::process::Command) { set("HORUS_NET_ON_LINK_LOST", action.clone()); } } + out +} + +/// Apply the `[network]` translation to a child `Command`. +/// +/// For the paths that build a `Command` directly rather than going through +/// `build_child_env`. +fn apply_network_config(cmd: &mut std::process::Command) { + for (key, value) in network_env() { + cmd.env(key, value); + } } pub fn execute_run( @@ -303,9 +324,21 @@ fn execute_workspace( let binary = crate::build_dirs::binary_path(&project_dir, profile, member); if binary.exists() { cli_output::success(&format!("Running {}", member)); - let status = std::process::Command::new(&binary).args(&args).status()?; + let mut cmd = std::process::Command::new(&binary); + cmd.args(&args); + // This path builds its child `Command` by hand instead of going + // through `build_child_env`, so it needs the `[network]` + // translation applied explicitly. + apply_network_config(&mut cmd); + let status = cmd.status()?; if !status.success() { - let code = status.code().unwrap_or(1); + // `shell_exit_code`, not `status.code().unwrap_or(1)`. On Unix + // a child killed by a signal has no exit code at all, and the + // `unwrap_or(1)` reported every one of them — SIGSEGV, SIGKILL, + // the Ctrl-C the operator just pressed — as a plain "exited + // with code 1". That is the defect `shell_exit_code` was + // introduced for; this call site was missed. + let code = crate::error_wrapper::shell_exit_code(&status); return Err(anyhow!("Process exited with code {}", code)); } } @@ -983,6 +1016,15 @@ pub(crate) fn build_child_env() -> Result> { // that does not work. One builder, one answer. env_vars.push(("PYTHONPATH".to_string(), run_python::build_python_path()?)); + // `[network]` from horus.toml. This belongs here, not at one spawn site: + // `safety.on_link_lost`, `import` and `deny_export` are safety controls, + // and they used to reach the runtime only when `horus run` was handed + // several files. Every path that runs a single executable — a bare + // `horus run`, a `horus.toml` run, a directory run — comes through this + // function, so wiring it here is what makes the table apply to the + // invocation a real robot actually uses. + env_vars.extend(network_env()); + Ok(env_vars) } @@ -1235,6 +1277,108 @@ mod tests { assert!(names.contains(&"PYTHONPATH"), "Must set PYTHONPATH"); } + /// `[network]` must reach the child on the path a single-binary robot runs. + /// + /// `safety.on_link_lost`, `import` and `deny_export` are safety controls. + /// The translation that arms them was wired into ONE of the five spawn + /// paths — `execute_multiple_files`, reached only when `horus run` is + /// handed more than one file. A bare `horus run`, a `horus.toml` run and a + /// directory run all resolve to a single executable and went through + /// `build_child_env`, which set `PATH`, `LD_LIBRARY_PATH` and `PYTHONPATH` + /// and nothing else. On those paths `NetConfig::default()` read an + /// environment nobody had populated: `import` stayed `Auto`, `deny_export` + /// stayed empty, and the documented comms-loss safe state never armed. + #[test] + fn build_child_env_carries_the_network_table_to_the_child() { + let tmp = tempfile::TempDir::new().unwrap(); + fs::create_dir_all(tmp.path().join(".horus/bin")).unwrap(); + fs::create_dir_all(tmp.path().join(".horus/lib")).unwrap(); + fs::write( + tmp.path().join(HORUS_TOML), + r#" +[project] +name = "link_loss_robot" +version = "0.1.0" + +[network] +import = "deny" +deny_export = ["/motor/cmd", "/estop"] + +[network.safety] +on_link_lost = "safe_state" +missed_threshold = 3 +"#, + ) + .unwrap(); + + let _guard = crate::CWD_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let original = std::env::current_dir().unwrap_or_else(|_| std::env::temp_dir()); + std::env::set_current_dir(tmp.path()).unwrap(); + // The operator's environment wins over the manifest, so a stray + // HORUS_NET_* here would mask the thing under test. + let preset: Vec<&str> = [ + "HORUS_NET_IMPORT", + "HORUS_NET_DENY_EXPORT", + "HORUS_NET_ON_LINK_LOST", + ] + .into_iter() + .filter(|k| std::env::var_os(k).is_some()) + .collect(); + let result = build_child_env(); + std::env::set_current_dir(original).unwrap(); + drop(_guard); + + assert!( + preset.is_empty(), + "precondition: {:?} set in this environment would override the manifest", + preset + ); + + let env_vars = result.unwrap(); + let get = |key: &str| { + env_vars + .iter() + .find(|(k, _)| k == key) + .map(|(_, v)| v.as_str()) + }; + + assert_eq!( + get("HORUS_NET_ON_LINK_LOST"), + Some("safe_state"), + "the comms-loss safe state must arm on the path a single-binary \ + robot actually runs, not only on a multi-file `horus run`" + ); + assert_eq!(get("HORUS_NET_IMPORT"), Some("deny")); + assert_eq!(get("HORUS_NET_DENY_EXPORT"), Some("/motor/cmd,/estop")); + assert_eq!(get("HORUS_NET_MISSED_THRESHOLD"), Some("3")); + } + + /// An explicit `HORUS_NET_*` in the operator's environment still wins. + #[test] + fn an_explicit_network_variable_beats_the_manifest() { + let tmp = tempfile::TempDir::new().unwrap(); + fs::write( + tmp.path().join(HORUS_TOML), + "[project]\nname = \"p\"\nversion = \"0.1.0\"\n\n[network]\nimport = \"deny\"\n", + ) + .unwrap(); + + let _guard = crate::CWD_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let original = std::env::current_dir().unwrap_or_else(|_| std::env::temp_dir()); + std::env::set_current_dir(tmp.path()).unwrap(); + // SAFETY: the CWD lock serialises the tests that touch process globals. + unsafe { std::env::set_var("HORUS_NET_IMPORT", "auto") }; + let pairs = network_env(); + unsafe { std::env::remove_var("HORUS_NET_IMPORT") }; + std::env::set_current_dir(original).unwrap(); + drop(_guard); + + assert!( + !pairs.iter().any(|(k, _)| k == "HORUS_NET_IMPORT"), + "the manifest must not override an explicit HORUS_NET_IMPORT" + ); + } + #[test] fn build_child_env_path_includes_horus_bin() { let tmp = tempfile::TempDir::new().unwrap(); diff --git a/horus_manager/src/commands/run/run_python.rs b/horus_manager/src/commands/run/run_python.rs index 22c7373d3..307882343 100644 --- a/horus_manager/src/commands/run/run_python.rs +++ b/horus_manager/src/commands/run/run_python.rs @@ -53,7 +53,7 @@ pub(super) fn execute_python_node(file: PathBuf, args: Vec, _release: bo // Propagate the child's exit code as an error if !status.success() { - let code = status.code().unwrap_or(1); + let code = crate::error_wrapper::shell_exit_code(&status); crate::error_wrapper::emit_diagnostics(&crate::error_wrapper::exit_code_hint( "python", code, )); @@ -77,7 +77,7 @@ pub(super) fn execute_python_node(file: PathBuf, args: Vec, _release: bo // Propagate the child's exit code as an error if !status.success() { - let code = status.code().unwrap_or(1); + let code = crate::error_wrapper::shell_exit_code(&status); crate::error_wrapper::emit_diagnostics(&crate::error_wrapper::exit_code_hint( "python", code, )); diff --git a/horus_manager/src/commands/run/run_rust.rs b/horus_manager/src/commands/run/run_rust.rs index 6dcc489ef..e05548d1b 100644 --- a/horus_manager/src/commands/run/run_rust.rs +++ b/horus_manager/src/commands/run/run_rust.rs @@ -509,7 +509,7 @@ pub(super) fn execute_from_cargo_toml( cmd.envs(child_env.iter().cloned()); let status = cmd.status()?; if !status.success() { - let code = status.code().unwrap_or(1); + let code = crate::error_wrapper::shell_exit_code(&status); crate::error_wrapper::emit_diagnostics(&crate::error_wrapper::exit_code_hint( "rust", code, )); @@ -862,7 +862,7 @@ pub(super) fn execute_with_scheduler( let status = cmd.status()?; if !status.success() { - let code = status.code().unwrap_or(1); + let code = crate::error_wrapper::shell_exit_code(&status); crate::error_wrapper::emit_diagnostics(&crate::error_wrapper::exit_code_hint( "cargo", code, )); @@ -888,7 +888,7 @@ pub(super) fn execute_with_scheduler( let status = cmd.status()?; if !status.success() { - let code = status.code().unwrap_or(1); + let code = crate::error_wrapper::shell_exit_code(&status); crate::error_wrapper::emit_diagnostics(&crate::error_wrapper::exit_code_hint( "rust", code, )); @@ -964,7 +964,7 @@ pub(super) fn execute_with_scheduler( let status = cmd.status()?; if !status.success() { - let code = status.code().unwrap_or(1); + let code = crate::error_wrapper::shell_exit_code(&status); crate::error_wrapper::emit_diagnostics(&crate::error_wrapper::exit_code_hint( "cargo", code, )); @@ -989,7 +989,7 @@ pub(super) fn execute_with_scheduler( let status = cmd.status()?; if !status.success() { - let code = status.code().unwrap_or(1); + let code = crate::error_wrapper::shell_exit_code(&status); crate::error_wrapper::emit_diagnostics(&crate::error_wrapper::exit_code_hint( "rust", code, )); diff --git a/horus_manager/src/error_wrapper.rs b/horus_manager/src/error_wrapper.rs index 1b157d57e..36ffb126d 100644 --- a/horus_manager/src/error_wrapper.rs +++ b/horus_manager/src/error_wrapper.rs @@ -840,6 +840,26 @@ pub fn python_error_hint(stderr: &str) -> Vec { // ── Exit code hints ───────────────────────────────────────────────────────── +/// The shell-convention exit code for a finished process. +/// +/// `ExitStatus::code()` returns `None` on Unix when a signal killed the child, +/// so `code().unwrap_or(1)` collapses every signal death into the one code +/// `exit_code_hint` deliberately says nothing about. That silently disabled the +/// 137/139/128+N arms below in every real run — including the OOM hint, which is +/// the most useful one this module has. Shells report a signal death as 128+N; +/// so do we, and the hints become reachable. +pub fn shell_exit_code(status: &std::process::ExitStatus) -> i32 { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + if let Some(signal) = status.signal() { + return 128 + signal; + } + } + // Windows has no signals: code() is always Some for a finished process. + status.code().unwrap_or(1) +} + /// Interpret a process exit code and return diagnostics for non-obvious codes. /// /// Common Unix exit codes and signal-based exits (128+N) are translated into @@ -1997,6 +2017,48 @@ mod tests { assert!(exit_code_hint("python", 0).is_empty()); } + /// The 137/139/128+N arms are worth nothing if no caller can ever produce + /// those numbers. Every one of them used to read `status.code().unwrap_or(1)`, + /// which on Unix is `None` -> 1 for a signal death, so a genuinely + /// OOM-killed child got the "too generic, no hint" branch. Kill a real + /// process and walk the actual production path: status -> code -> hint. + #[test] + #[cfg(unix)] + fn a_signal_killed_child_reaches_the_oom_hint() { + let status = std::process::Command::new("sh") + .arg("-c") + .arg("kill -9 $$") + .status() + .expect("sh should be available on any unix host"); + assert!(!status.success()); + assert_eq!( + status.code(), + None, + "a SIGKILLed child reports no exit code - that is the whole problem" + ); + + let code = shell_exit_code(&status); + assert_eq!(code, 137, "SIGKILL is signal 9, so the shell code is 128+9"); + + let diags = exit_code_hint("python", code); + assert!( + diags.iter().any(|d| d.code == "H053"), + "an OOM-killed child must get the OOM hint, got {:?}", + diags.iter().map(|d| &d.code).collect::>() + ); + } + + /// A plain non-zero exit must keep going through unchanged. + #[test] + fn shell_exit_code_passes_an_ordinary_failure_through() { + let status = std::process::Command::new("sh") + .arg("-c") + .arg("exit 3") + .status() + .expect("sh should be available"); + assert_eq!(shell_exit_code(&status), 3); + } + // ── CMake error hints ─────────────────────────────────────────────── #[test] diff --git a/horus_manager/src/main.rs b/horus_manager/src/main.rs index cc41fa866..5255f6686 100644 --- a/horus_manager/src/main.rs +++ b/horus_manager/src/main.rs @@ -2522,17 +2522,18 @@ fn run_command(command: Commands) -> HorusResult<()> { horus_manager::error_wrapper::set_json_diagnostics(true); } - // `--net` used to set this variable and stop there. HORUS_NET is - // read in exactly one place — `horus doctor` — so the flag never - // reached the build: the `net` Cargo feature is not in `default`, - // the generated manifest carried no `features = ["net"]`, and the - // resulting binary contained no horus_net symbols at all. + // `--net` used to set a `HORUS_NET` variable and stop there. + // Nothing read it but `horus doctor`, so the flag never reached the + // build: the `net` Cargo feature is not in `default`, the generated + // manifest carried no `features = ["net"]`, and the resulting + // binary contained no horus_net symbols at all. // - // Push it through the capability list as well, which is what - // actually turns into `cargo --features`. + // The capability list is what actually turns into + // `cargo --features`, so that is the only thing to push it through. + // `HORUS_NET` is gone: it never affected a build, and `doctor` now + // reads the same capability the build does. let mut enable = enable; if net { - std::env::set_var("HORUS_NET", "1"); let list = enable.get_or_insert_with(Vec::new); if !list.iter().any(|c| c.eq_ignore_ascii_case("net")) { list.push("net".to_string()); @@ -2911,16 +2912,20 @@ fn run_command(command: Commands) -> HorusResult<()> { ), } } - result.map_err(HorusError::from)?; - - if let Some(ref manifest) = hooks_manifest { - if let Err(e) = commands::hooks::run_hooks("post_test", manifest) { - eprintln!("Hook failed: {}", e); - return Err(HorusError::from(e)); + // Teardown, on both paths — the same treatment `post_run` and + // `post_build` get. `post_test` used to sit after an early `?`, so a + // failing `horus test` skipped it entirely. That is backwards for + // the phase the hooks contract itself calls "the only phase with + // teardown": a `post_test = ["release-bench"]` exists to release + // the bench, and it was skipped precisely when the tests aborted + // mid-run and the hardware was most likely left live. + let result = result.map_err(HorusError::from); + match hooks_manifest { + Some(ref manifest) => { + commands::hooks::run_teardown_hooks("post_test", manifest, result) } + None => result, } - - Ok(()) } Commands::Monitor { args } => { @@ -3189,8 +3194,7 @@ fn run_command(command: Commands) -> HorusResult<()> { let (pkg_name, pkg_ver) = split_name_version(name, ver); let result = if plugin { - let local = target.is_some(); - commands::plugin::run_install(pkg_name.clone(), pkg_ver.clone(), local) + commands::plugin::run_install(pkg_name.clone(), pkg_ver.clone(), target) } else { commands::pkg::run_install_standalone(&pkg_name, pkg_ver.as_deref(), target) }; @@ -3508,7 +3512,12 @@ fn run_command(command: Commands) -> HorusResult<()> { watch, extra_args, } => { - if extract || json || md || html || coverage || diff.is_some() { + // `watch` belongs in this guard. `--watch` is declared as "Watch + // for file changes and regenerate", and `run_extract` honours it + // first thing — but a bare `horus doc --watch` fell to the `else` + // arm, which opens rustdoc once and returns, so the flag did + // nothing unless an extract flag happened to be passed with it. + if extract || json || md || html || coverage || watch || diff.is_some() { let config = commands::doc_extract::ExtractConfig { json, md, diff --git a/horus_manager/src/manifest.rs b/horus_manager/src/manifest.rs index 6339b8f7f..697f0dfca 100644 --- a/horus_manager/src/manifest.rs +++ b/horus_manager/src/manifest.rs @@ -1267,6 +1267,28 @@ fn warn_unknown_keys_once(path: &Path, content: &str) { // ─── HorusManifest: loading ───────────────────────────────────────────────── impl HorusManifest { + /// Every hardware entry: `[hardware]` plus the legacy `[drivers]` behind it. + /// + /// `[hardware]` is the preferred spelling and `[drivers]` the compatibility + /// one, so a consumer that reads only `drivers` makes the *preferred* + /// spelling do strictly less than the legacy one. That is what happened: + /// `doctor` and `native_sync` merged the two, while `cargo_gen` and + /// `pyproject_gen` read `drivers` alone, so a driver declared under + /// `[hardware]` generated no Cargo dependency and no PyPI dependency. + /// Everything that wants "the hardware the project declares" goes through + /// here so the two spellings cannot drift apart again. + /// + /// A name present in both resolves to the `[hardware]` entry. + pub fn hardware_entries(&self) -> BTreeMap<&str, &DriverValue> { + let mut merged: BTreeMap<&str, &DriverValue> = BTreeMap::new(); + for (name, value) in &self.hardware { + merged.insert(name.as_str(), value); + } + for (name, value) in &self.drivers { + merged.entry(name.as_str()).or_insert(value); + } + merged + } /// Load manifest from a TOML file path. /// /// Warns on stderr about keys HORUS does not understand, once per file per diff --git a/horus_manager/src/native_sync.rs b/horus_manager/src/native_sync.rs index 9c32dfad7..d24d470c3 100644 --- a/horus_manager/src/native_sync.rs +++ b/horus_manager/src/native_sync.rs @@ -312,9 +312,7 @@ fn deps_equal(a: &DependencyValue, b: &DependencyValue) -> bool { /// normal dependency. Only package drivers generate Cargo deps. fn collect_driver_crates(manifest: &HorusManifest) -> Vec { let mut crates = Vec::new(); - // Check both [hardware] and legacy [drivers] - let all_drivers = manifest.hardware.values().chain(manifest.drivers.values()); - for driver_val in all_drivers { + for driver_val in manifest.hardware_entries().into_values() { if let crate::manifest::DriverValue::Config(cfg) = driver_val { if let Some(package) = &cfg.package { crates.push(package.clone()); diff --git a/horus_manager/src/pyproject_gen.rs b/horus_manager/src/pyproject_gen.rs index a4ebaf9f5..09987026f 100644 --- a/horus_manager/src/pyproject_gen.rs +++ b/horus_manager/src/pyproject_gen.rs @@ -73,8 +73,8 @@ pub fn generate( // ── Dependencies ───────────────────────────────────────────────────── let mut pypi_deps = collect_python_deps(&manifest.dependencies); - // Add PyPI driver dependencies from [drivers] - for value in manifest.drivers.values() { + // Add PyPI driver dependencies from [hardware]/[drivers] + for value in manifest.hardware_entries().into_values() { if let crate::manifest::DriverValue::Config(cfg) = value { if let Some(ref pip_name) = cfg.pip { let version = cfg diff --git a/horus_manager/tests/completion_install_contract.rs b/horus_manager/tests/completion_install_contract.rs index bfab9000d..a81a56007 100644 --- a/horus_manager/tests/completion_install_contract.rs +++ b/horus_manager/tests/completion_install_contract.rs @@ -191,7 +191,12 @@ impl Sandbox { fn uninstall(&self, xdg: &[(&str, &str)]) -> String { let script = format!( - "HORUS_DIR=\"$HOME/.horus\"\n\ + // `set -e`, because uninstall.sh sets it at line 31. Without it the + // harness is more forgiving than the shipped script: a command that + // legitimately exits non-zero (grep selecting no lines, say) reads + // as harmless here and aborts the real uninstall partway through. + "set -e\n\ + HORUS_DIR=\"$HOME/.horus\"\n\ GREEN=''; YELLOW=''; NC=''\n\ REMOVED=0; SKIPPED=0\n\ {paths}\n{removal}\n{profiles}\n\ @@ -523,3 +528,569 @@ fn the_uninstaller_lists_every_location_the_installer_writes() { "the marker install.sh writes into .zshrc and the one uninstall.sh deletes must match" ); } + +/// A prefix install must export `HORUS_PREFIX`, not just `PATH`. +/// +/// Both readers take the prefix from the environment of the running `horus` +/// process and from nowhere else — `version.rs` says so outright: "HORUS_PREFIX +/// is the whole interface ... there is no second name to read, and no way to +/// discover a prefix install from outside its own tree". `run_rust`'s cache +/// roots and the version gate's state root are both derived from it. +/// +/// The installer never put it in the user's environment: the rc file got a PATH +/// line only, `horus env --init` writes proxy functions and no exports, and the +/// prefix epilogue told the user to export `PATH` and `HORUS_SOURCE`. So for +/// every prefix user the cache root fell back to `~/.horus/cache` — deduped +/// away as the legacy root, meaning the prefix cache was never searched — and +/// the state root was `~/.horus`, which holds none of that install's files. The +/// Rust half of the change worked only for someone who guessed the variable. +#[test] +fn a_prefix_install_exports_horus_prefix_to_the_shell() { + let sb = Sandbox::new(); + let prefix = sb.path().join("opt-horus"); + let script = format!( + "INSTALL_DIR=\"{prefix}/bin\"\n\ + HORUS_PREFIX=\"{prefix}\"\n\ + BINARY_NAME=horus\n\ + RED=''; GREEN=''; YELLOW=''; CYAN=''; BOLD=''; NC=''\n\ + info(){{ echo \" -> $1\"; }}\n\ + ok(){{ echo \" ok $1\"; }}\n\ + warn(){{ echo \" ! $1\"; }}\n\ + fail(){{ echo \" x $1\"; }}\n\ + INSTALL_START=$(date +%s)\n\ + VERSION=0.0.0-test\n\ + set -e\n\ + {tail}", + prefix = prefix.display(), + tail = install_tail() + ); + sb.run(&script, "/bin/bash", false, &[]); + + let rc = sb + .read(".bashrc") + .expect("the installer must write to the rc file"); + assert!( + rc.contains("export HORUS_PREFIX="), + "a prefix install must export HORUS_PREFIX — without it `horus run` \ + never searches the prefix cache and the version gate reads a state \ + root that holds none of this install's files. rc was:\n{rc}" + ); + assert!( + rc.contains(&prefix.display().to_string()), + "the exported value must be the prefix that was installed to:\n{rc}" + ); +} + +/// ...including on a re-install, where PATH is already configured. +/// +/// This is the population the line exists for: a prefix install puts +/// `${HORUS_PREFIX}/bin` on PATH, so every upgrade and re-install takes the +/// "PATH already configured" arm. The export first landed inside the `else` arm +/// of that check, which made it dead for exactly those users — the same mistake +/// install.sh already records 60 lines above for the shell-integration block +/// that used to live there. +#[test] +fn a_prefix_install_exports_horus_prefix_on_a_reinstall_too() { + let sb = Sandbox::new(); + let prefix = sb.path().join("opt-horus"); + let script = format!( + "INSTALL_DIR=\"{prefix}/bin\"\n\ + HORUS_PREFIX=\"{prefix}\"\n\ + BINARY_NAME=horus\n\ + RED=''; GREEN=''; YELLOW=''; CYAN=''; BOLD=''; NC=''\n\ + info(){{ echo \" -> $1\"; }}\n\ + ok(){{ echo \" ok $1\"; }}\n\ + warn(){{ echo \" ! $1\"; }}\n\ + fail(){{ echo \" x $1\"; }}\n\ + INSTALL_START=$(date +%s)\n\ + VERSION=0.0.0-test\n\ + PATH=\"$INSTALL_DIR:$PATH\"\n\ + set -e\n\ + {tail}", + prefix = prefix.display(), + tail = install_tail() + ); + // INSTALL_DIR is on PATH above, which is what every re-install of a prefix + // install looks like. + let out = sb.run(&script, "/bin/bash", false, &[]); + assert!( + out.contains("PATH already configured"), + "precondition: this test must exercise the already-configured arm:\n{out}" + ); + + let rc = sb.read(".bashrc").unwrap_or_default(); + assert!( + rc.contains("export HORUS_PREFIX="), + "a re-install must still export HORUS_PREFIX — it is the population \ + that most needs it, since a prefix install always puts its bin on \ + PATH. rc was:\n{rc}" + ); +} + +/// A stale prefix line is replaced, not counted as already configured. +#[test] +fn a_changed_prefix_replaces_the_old_export() { + let sb = Sandbox::new(); + std::fs::write( + sb.path().join(".bashrc"), + "export HORUS_PREFIX=\"/old/location\"\n", + ) + .unwrap(); + + let prefix = sb.path().join("opt-horus"); + let script = format!( + "INSTALL_DIR=\"{prefix}/bin\"\n\ + HORUS_PREFIX=\"{prefix}\"\n\ + BINARY_NAME=horus\n\ + RED=''; GREEN=''; YELLOW=''; CYAN=''; BOLD=''; NC=''\n\ + info(){{ echo \" -> $1\"; }}\n\ + ok(){{ echo \" ok $1\"; }}\n\ + warn(){{ echo \" ! $1\"; }}\n\ + fail(){{ echo \" x $1\"; }}\n\ + INSTALL_START=$(date +%s)\n\ + VERSION=0.0.0-test\n\ + PATH=\"$INSTALL_DIR:$PATH\"\n\ + set -e\n\ + {tail}", + prefix = prefix.display(), + tail = install_tail() + ); + sb.run(&script, "/bin/bash", false, &[]); + + let rc = sb.read(".bashrc").unwrap_or_default(); + assert!( + rc.contains(&prefix.display().to_string()), + "the new prefix must be exported:\n{rc}" + ); + assert!( + !rc.contains("/old/location"), + "a stale prefix line must be removed, not left to win by being first \ + in the file:\n{rc}" + ); +} + +/// The `export HORUS_PREFIX` line must be fish-safe. +/// +/// fish has no `export` builtin, and a POSIX export line stops `config.fish` +/// loading at that point — the exact failure this file already records for the +/// PATH line, where a poisoned config stayed poisoned through every upgrade +/// because the guard found the broken line and skipped writing a working one. +#[test] +fn the_prefix_export_uses_fish_syntax_in_config_fish() { + let sb = Sandbox::new(); + let prefix = sb.path().join("opt-horus"); + let script = format!( + "INSTALL_DIR=\"{prefix}/bin\"\n\ + HORUS_PREFIX=\"{prefix}\"\n\ + BINARY_NAME=horus\n\ + RED=''; GREEN=''; YELLOW=''; CYAN=''; BOLD=''; NC=''\n\ + info(){{ echo \" -> $1\"; }}\n\ + ok(){{ echo \" ok $1\"; }}\n\ + warn(){{ echo \" ! $1\"; }}\n\ + fail(){{ echo \" x $1\"; }}\n\ + INSTALL_START=$(date +%s)\n\ + VERSION=0.0.0-test\n\ + set -e\n\ + {tail}", + prefix = prefix.display(), + tail = install_tail() + ); + sb.run(&script, "/usr/bin/fish", false, &[]); + + let rc = sb + .read(".config/fish/config.fish") + .expect("the installer must write config.fish for a fish shell"); + assert!( + rc.contains("set -gx HORUS_PREFIX"), + "config.fish needs `set -gx`, not `export`:\n{rc}" + ); + assert!( + !rc.contains("export HORUS_PREFIX"), + "a POSIX export line stops config.fish loading at that point:\n{rc}" + ); +} + +/// The `check_rust_version` gate, verbatim from `install.sh`. +fn rust_gate_slice() -> String { + let src = fs::read_to_string(repo_root().join("install.sh")).expect("install.sh must exist"); + let begin = "check_rust_version() {"; + let end = " ok \"Rust $found (>= $required required)\"\n}"; + let b = src + .find(begin) + .unwrap_or_else(|| panic!("install.sh lost the marker {begin:?}")); + let e = src + .find(end) + .unwrap_or_else(|| panic!("install.sh lost the marker {end:?}")); + assert!(b < e, "install.sh markers are out of order"); + let slice = src[b..e + end.len()].to_string(); + // Without these the harness would be exercising an empty shell function and + // every assertion below would hold vacuously. + for needed in [ + "rust-version", + "rustc --version", + "$HORUS_SRC_DIR/Cargo.toml", + ] { + assert!( + slice.contains(needed), + "the check_rust_version slice no longer contains {needed:?} — this test would pass vacuously" + ); + } + slice +} + +/// Run the gate against a stub `rustc` reporting `rustc_version`, with the +/// workspace floor set to `required`. `sort_has_dash_v` chooses whether the +/// `sort` on PATH understands `-V`. Returns the gate's output plus a +/// PROCEEDED/ABORTED verdict. +fn run_rust_gate(rustc_version: &str, required: &str, sort_has_dash_v: bool) -> String { + let sb = Sandbox::new(); + let stub = sb.path().join("stub"); + fs::create_dir_all(&stub).unwrap(); + fs::create_dir_all(sb.path().join("src")).unwrap(); + fs::write( + sb.path().join("src/Cargo.toml"), + format!("[workspace.package]\nrust-version = \"{required}\"\n"), + ) + .unwrap(); + + let rustc = stub.join("rustc"); + fs::write( + &rustc, + format!("#!/bin/bash\necho 'rustc {rustc_version} (deadbeef 2026-01-01)'\n"), + ) + .unwrap(); + // A `sort` that rejects -V, the way any sort predating the GNU/BSD version + // extension does. The point is not one named platform: it is that the gate + // must not convict a toolchain on a comparison it was unable to perform. + if !sort_has_dash_v { + let sort = stub.join("sort"); + fs::write( + &sort, + "#!/bin/bash\nfor a in \"$@\"; do case \"$a\" in -V|--version-sort)\n\ + echo 'sort: illegal option -- V' >&2; exit 2 ;; esac; done\nexec /usr/bin/sort \"$@\"\n", + ) + .unwrap(); + set_exec(&sort); + } + set_exec(&rustc); + + let script = format!( + "set -u\n\ + PATH=\"$HOME/stub:$PATH\"\n\ + HORUS_SRC_DIR=\"$HOME/src\"\n\ + fail() {{ echo \"GATE_FAIL: $*\"; }}\n\ + ok() {{ echo \"GATE_OK: $*\"; }}\n\ + {}\n\ + if ( check_rust_version ); then echo PROCEEDED; else echo ABORTED; fi\n", + rust_gate_slice() + ); + sb.run(&script, "/bin/bash", true, &[]) +} + +fn set_exec(p: &Path) { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(p, fs::Permissions::from_mode(0o755)).unwrap(); + } +} + +#[test] +fn the_rust_version_gate_does_not_abort_when_it_cannot_compare() { + let out = run_rust_gate("1.90.0", "1.85", false); + assert!( + out.contains("PROCEEDED"), + "a `sort` without -V made the version gate abort an install on a toolchain \ + that satisfies the floor (1.90.0 >= 1.85). A gate that cannot perform its \ + comparison must fail open, as the two guards above it already do:\n{out}" + ); + assert!( + !out.contains("or newer is required"), + "the installer accused a perfectly good toolchain of being too old:\n{out}" + ); +} + +#[test] +fn the_rust_version_gate_still_rejects_a_genuinely_old_toolchain() { + let out = run_rust_gate("1.70.0", "1.85", true); + assert!( + out.contains("ABORTED") && out.contains("or newer is required"), + "failing open must not mean failing blind: 1.70.0 is below the 1.85 floor \ + and the gate has to say so:\n{out}" + ); +} + +#[test] +fn the_rust_version_gate_compares_numerically_not_lexically() { + let out = run_rust_gate("1.100.0", "1.9", true); + assert!( + out.contains("PROCEEDED"), + "1.100 is newer than 1.9; a string compare says otherwise and would reject \ + every toolchain once the minor version passes 9:\n{out}" + ); +} + +/// Drive the installer's tail with `HORUS_PREFIX` set, the way `--prefix` does. +fn install_with_prefix(sb: &Sandbox, prefix: &Path, shell: &str) -> String { + let script = format!( + "INSTALL_DIR=\"{prefix}/bin\"\n\ + HORUS_PREFIX=\"{prefix}\"\n\ + BINARY_NAME=horus\n\ + RED=''; GREEN=''; YELLOW=''; CYAN=''; BOLD=''; NC=''\n\ + info(){{ echo \" -> $1\"; }}\n\ + ok(){{ echo \" ok $1\"; }}\n\ + warn(){{ echo \" ! $1\"; }}\n\ + fail(){{ echo \" x $1\"; }}\n\ + INSTALL_START=$(date +%s)\n\ + VERSION=0.0.0-test\n\ + set -e\n\ + {tail}", + prefix = prefix.display(), + tail = install_tail() + ); + sb.run(&script, shell, false, &[]) +} + +/// Uninstalling must take the `HORUS_PREFIX` export back out of the rc file. +/// +/// It is not merely untidy to leave it. `install.sh:245` reads the variable: +/// +/// if [ -n "${HORUS_PREFIX:-}" ]; then INSTALL_DIR="${HORUS_PREFIX}/bin" +/// +/// so the line the uninstaller leaves behind is still exported by every new +/// shell, and the *next* install silently lands in the deleted prefix instead +/// of the default location — with nothing on screen to say so. Uninstall, +/// open a terminal, reinstall, and horus is back in /opt/horus. +#[test] +fn uninstalling_removes_the_horus_prefix_export_from_the_rc() { + let sb = Sandbox::new(); + let prefix = sb.path().join("opt-horus"); + fs::write(sb.path().join(".bashrc"), "# bashrc\n").unwrap(); + install_with_prefix(&sb, &prefix, "/bin/bash"); + let before = sb.read(".bashrc").unwrap_or_default(); + assert!( + before.contains("HORUS_PREFIX="), + "precondition: the installer should have written the export:\n{before}" + ); + + sb.uninstall(&[]); + + let after = sb.read(".bashrc").unwrap_or_default(); + assert!( + !after.contains("HORUS_PREFIX="), + "the uninstaller left the HORUS_PREFIX export behind, pointing at a \ + prefix it just deleted. Every later shell exports it, and the next \ + install reads it and reinstalls into the removed directory:\n{after}" + ); +} + +/// The same line, in fish's syntax, in the file fish actually reads. +/// +/// `clean_shell_profiles` walks `.bashrc`/`.zshrc`/`.profile`/`.bash_profile`; +/// `config.fish` is not among them, and the `conf.d/horus.fish` it does delete +/// is a different file. The installer writes `set -gx HORUS_PREFIX` into +/// `config.fish`, so for fish users the export survived the uninstall +/// regardless of how the POSIX side behaved. +#[test] +fn uninstalling_removes_the_fish_horus_prefix_export() { + let sb = Sandbox::new(); + let prefix = sb.path().join("opt-horus"); + let fish_dir = sb.path().join(".config/fish"); + fs::create_dir_all(&fish_dir).unwrap(); + fs::write(fish_dir.join("config.fish"), "# config.fish\n").unwrap(); + install_with_prefix(&sb, &prefix, "/usr/bin/fish"); + let before = sb.read(".config/fish/config.fish").unwrap_or_default(); + assert!( + before.contains("HORUS_PREFIX"), + "precondition: the installer should have written the fish export:\n{before}" + ); + + sb.uninstall(&[]); + + let after = sb.read(".config/fish/config.fish").unwrap_or_default(); + assert!( + !after.contains("HORUS_PREFIX"), + "the uninstaller left the fish HORUS_PREFIX line behind:\n{after}" + ); +} + +/// The cleanup deletes horus's line and nothing that merely looks like it. +/// +/// A line-deleting uninstaller is one missing anchor away from eating a +/// stranger's config. `HORUS_PREFIX=` is anchored by the `=`; the fish form has +/// no `=`, so without an explicit end-of-name boundary +/// `set -gx HORUS_PREFIX_OTHER` matches the same pattern and is silently +/// removed from a file the user has to notice is wrong on their own. +#[test] +fn the_prefix_cleanup_leaves_similarly_named_variables_alone() { + let sb = Sandbox::new(); + fs::write( + sb.path().join(".bashrc"), + "# bashrc\n\ + export HORUS_PREFIX=\"/opt/horus\"\n\ + export HORUS_PREFIX_EXTRA=\"keep me\"\n\ + export MY_HORUS_PREFIX=\"keep me too\"\n", + ) + .unwrap(); + let fish_dir = sb.path().join(".config/fish"); + fs::create_dir_all(&fish_dir).unwrap(); + fs::write( + fish_dir.join("config.fish"), + "# config.fish\n\ + set -gx HORUS_PREFIX \"/opt/horus\"\n\ + set -gx HORUS_PREFIX_OTHER \"keep me\"\n", + ) + .unwrap(); + + sb.uninstall(&[]); + + let rc = sb.read(".bashrc").unwrap_or_default(); + assert!( + !rc.contains("export HORUS_PREFIX=\""), + "horus's own line survived:\n{rc}" + ); + assert!( + rc.contains("HORUS_PREFIX_EXTRA") && rc.contains("MY_HORUS_PREFIX"), + "the uninstaller deleted a variable that only shares a prefix with \ + horus's, out of a file it does not own:\n{rc}" + ); + + let fish = sb.read(".config/fish/config.fish").unwrap_or_default(); + assert!( + !fish.contains("set -gx HORUS_PREFIX \""), + "horus's own fish line survived:\n{fish}" + ); + assert!( + fish.contains("HORUS_PREFIX_OTHER"), + "the fish cleanup has no end-of-name boundary and ate a neighbouring \ + variable:\n{fish}" + ); +} + +/// A prefix install's PATH line goes; the shared `~/.local/bin` line stays. +/// +/// Both halves matter. The prefix line names a directory this uninstall just +/// deleted, so leaving it is dead weight the user has to find themselves. The +/// default line names `~/.local/bin` — which pip, pipx, cargo and rustup all +/// install into — so removing it because horus happened to write it would take +/// every one of those off PATH. That is why the removal matches an exact +/// string built from `$HORUS_PREFIX` rather than anything resembling +/// `export PATH=`. +#[test] +fn uninstalling_a_prefix_install_removes_only_its_own_path_line() { + let sb = Sandbox::new(); + let prefix = sb.path().join("opt-horus"); + let p = prefix.display().to_string(); + fs::write( + sb.path().join(".bashrc"), + format!( + "# bashrc\n\ + export PATH=\"{p}/bin:$PATH\"\n\ + export PATH=\"$HOME/.local/bin:$PATH\"\n\ + export PATH=\"/opt/somebody-else/bin:$PATH\"\n" + ), + ) + .unwrap(); + + sb.uninstall(&[("HORUS_PREFIX", &p)]); + + let rc = sb.read(".bashrc").unwrap_or_default(); + assert!( + !rc.contains(&format!("{p}/bin")), + "the prefix PATH line points at a directory the uninstaller just \ + deleted and must not survive it:\n{rc}" + ); + assert!( + rc.contains("$HOME/.local/bin"), + "the uninstaller removed the shared ~/.local/bin PATH line. pip, pipx, \ + cargo and rustup install there too; taking that line out over horus \ + breaks all of them:\n{rc}" + ); + assert!( + rc.contains("/opt/somebody-else/bin"), + "the uninstaller removed a PATH line that has nothing to do with \ + horus:\n{rc}" + ); +} + +/// The default install must not trip the prefix branch at all. +#[test] +fn uninstalling_a_default_install_touches_no_path_line() { + let sb = Sandbox::new(); + let original = "# bashrc\nexport PATH=\"$HOME/.local/bin:$PATH\"\n"; + fs::write(sb.path().join(".bashrc"), original).unwrap(); + + sb.uninstall(&[]); + + let rc = sb.read(".bashrc").unwrap_or_default(); + assert!( + rc.contains("$HOME/.local/bin"), + "a default uninstall (no HORUS_PREFIX) edited a PATH line it has no \ + claim on:\n{rc}" + ); +} + +/// An rc file whose only content is the prefix PATH line. +/// +/// Two ways this went wrong, both invisible in a multi-line fixture: +/// `grep -v` exits 1 when it selects nothing, so gating the rewrite on `&&` +/// left the line in the one file where removing it mattered most; and under +/// `set -e` — which uninstall.sh sets at line 31 — that same exit status +/// aborts the uninstaller partway through, leaving everything after this +/// block undone. +#[test] +fn an_rc_file_that_is_only_the_prefix_path_line_is_emptied_not_skipped() { + let sb = Sandbox::new(); + let prefix = sb.path().join("opt-horus"); + let p = prefix.display().to_string(); + fs::write( + sb.path().join(".bashrc"), + format!("export PATH=\"{p}/bin:$PATH\"\n"), + ) + .unwrap(); + + let out = sb.uninstall(&[("HORUS_PREFIX", &p)]); + + let rc = sb.read(".bashrc").unwrap_or_default(); + assert!( + !rc.contains(&format!("{p}/bin")), + "the only line in the file was the one to remove, and it survived:\n{rc}" + ); + assert!( + out.contains("Cleaned the prefix PATH line"), + "the uninstaller did not report cleaning the line:\n{out}" + ); +} + +/// A config.fish holding only a *similarly named* variable must report nothing. +/// +/// The guard grep and the sed that follows it have to agree. When the guard was +/// unanchored it matched `HORUS_PREFIX_OTHER`, the anchored sed then correctly +/// deleted nothing, and the block still printed "Cleaned HORUS_PREFIX from +/// config.fish" and counted a removal. uninstall.sh already carries a comment +/// about the last time it reported a cleanup that never happened (the declined +/// -sudo path in the RT block); this is the same failure. +#[test] +fn a_config_fish_without_horus_prefix_reports_no_cleanup() { + let sb = Sandbox::new(); + let fish_dir = sb.path().join(".config/fish"); + fs::create_dir_all(&fish_dir).unwrap(); + fs::write( + fish_dir.join("config.fish"), + "# config.fish\nset -gx HORUS_PREFIX_OTHER \"keep me\"\n", + ) + .unwrap(); + + let out = sb.uninstall(&[]); + + let fish = sb.read(".config/fish/config.fish").unwrap_or_default(); + assert!( + fish.contains("HORUS_PREFIX_OTHER"), + "the unrelated variable was deleted:\n{fish}" + ); + assert!( + !out.contains("Cleaned HORUS_PREFIX from config.fish"), + "the uninstaller reported cleaning a HORUS_PREFIX line that was never \ + there and never removed:\n{out}" + ); +} diff --git a/horus_manager/tests/docs_parity.rs b/horus_manager/tests/docs_parity.rs index bf0963982..c6b8430f2 100644 --- a/horus_manager/tests/docs_parity.rs +++ b/horus_manager/tests/docs_parity.rs @@ -179,6 +179,10 @@ const TEST_ONLY: &[&str] = &[ "HORUS_LIVE5_GHOST_CAPACITY", // horus_cpp/src/c_api.rs — the child half of the FFI emit test. "HORUS_CPP_FFI_EMIT_CHILD", + // scheduling/compute_executor.rs — the child half of the test that a + // compute-only program still drains its RT diagnostics to stdout. Read + // inside a `#[cfg(test)]` module; unreachable outside the test binary. + "HORUS_COMPUTE_DIAG_CHILD", // commands/fmt.rs — the switch CI uses to turn a skipped tool into a // failure, so a missing clang-format is not silently tolerated. "HORUS_TEST_REQUIRE_TOOLS", diff --git a/horus_manager/tests/hooks_contract.rs b/horus_manager/tests/hooks_contract.rs index 749322bfa..162569887 100644 --- a/horus_manager/tests/hooks_contract.rs +++ b/horus_manager/tests/hooks_contract.rs @@ -232,3 +232,62 @@ fn no_hooks_suppresses_the_post_hook_as_well() { "--no-hooks must disable post_run as well as pre_run:\n{combined}" ); } + +/// Symmetry of the key names is not the property that matters. +/// +/// `every_phase_has_both_a_pre_and_a_post_hook` pins the table, and the table +/// was symmetric while `post_test` still had success-only semantics: it sat +/// after an early `?` in the `Commands::Test` arm, so a failing `horus test` +/// skipped it entirely. That is backwards for the phase this file's own header +/// calls the one that had teardown — `post_test = ["release-bench"]` exists to +/// release the bench, and it was skipped precisely when the tests aborted +/// mid-run and the hardware was most likely left live. +/// +/// So this reads the dispatch itself. Every `post_` phase must go through +/// `run_teardown_hooks`, whose contract is "after a command finished, whatever +/// the outcome", and none may go through the plain `run_hooks`, which is the +/// setup form and only runs on the path that reached it. +#[test] +fn every_post_hook_is_dispatched_as_teardown() { + let main_rs = std::fs::read_to_string("src/main.rs").expect("horus_manager/src/main.rs"); + + let post_phases: Vec<&str> = KNOWN_HOOKS + .iter() + .copied() + .filter(|h| h.starts_with("post_")) + .collect(); + assert!( + !post_phases.is_empty(), + "precondition: there are post_ hooks to check" + ); + + let mut wrong = Vec::new(); + for phase in &post_phases { + let teardown = format!("run_teardown_hooks(\"{phase}\""); + let setup = format!("run_hooks(\"{phase}\""); + // These two patterns cannot overlap: the character before `hooks("` is + // `_` in `run_teardown_hooks(`, so `run_hooks("` is not a substring of + // it. Counting them independently is therefore exact. + let setup_calls = main_rs.matches(&setup).count(); + + if !main_rs.contains(&teardown) { + wrong.push(format!( + "`{phase}` is never dispatched through run_teardown_hooks, so it \ + does not run when the command failed" + )); + } + if setup_calls > 0 { + wrong.push(format!( + "`{phase}` is dispatched through the plain run_hooks in {setup_calls} \ + place(s); that form is skipped when the command failed" + )); + } + } + + assert!( + wrong.is_empty(), + "a post_ hook that does not run on the failure path is a hook that \ + skips teardown exactly when teardown matters:\n {}", + wrong.join("\n ") + ); +} diff --git a/horus_manager/tests/monitor_plugin_lookup_contract.rs b/horus_manager/tests/monitor_plugin_lookup_contract.rs new file mode 100644 index 000000000..b4b74bd10 --- /dev/null +++ b/horus_manager/tests/monitor_plugin_lookup_contract.rs @@ -0,0 +1,138 @@ +//! `horus monitor` must look where `horus install` writes. +//! +//! These were two hardcoded paths in two files about 800 lines apart, and they +//! disagreed: `horus install horus-monitor` symlinks into +//! `PluginRegistry::global_bin_dir()` — the platform CONFIG dir — while the +//! lookup checked `~/.horus/bin`, which nothing writes to (`~/.horus` holds +//! installer STATE: `installed_version`, `install_manifest.toml`, `cache/`). +//! So the documented install-then-run flow could not work on any platform, and +//! all the user got was "Monitor plugin is not installed." (#184) +//! +//! `monitor` is a declared clap subcommand, so it never reaches the generic +//! plugin dispatch that resolves this correctly — which is why the drift was +//! invisible to every other plugin command. +//! +//! The fix was to call the accessor instead of restating the path. Nothing +//! pinned that, so this reads the source of `which_monitor_binary` and asserts +//! it. `which_monitor_binary` lives in the binary crate, so an integration test +//! cannot call it; reading it is what is available, and it is enough to catch +//! the one thing that went wrong — a second hardcoded copy of a path. +//! +//! Run: `cargo test -p horus_manager --test monitor_plugin_lookup_contract` + +use std::path::PathBuf; + +fn repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf() +} + +/// The CODE of `which_monitor_binary`, with comment lines removed. +/// +/// Stripping comments is the whole point. The first version of this guard +/// matched the raw body, and the function's own comment explains what +/// `global_bin_dir()` is — so the assertion was satisfied by the prose +/// describing the fix rather than by the call performing it, and would have +/// stayed green if the call were replaced by a hardcoded path with the comment +/// left in place. That is the failure mode a source-reading guard has to be +/// built against. +fn which_monitor_binary_code() -> String { + which_monitor_binary_body() + .lines() + .filter(|l| !l.trim_start().starts_with("//")) + .collect::>() + .join("\n") +} + +/// The body of `which_monitor_binary`, from its signature to the matching brace. +fn which_monitor_binary_body() -> String { + let src = std::fs::read_to_string(repo_root().join("src/main.rs")).expect("main.rs must exist"); + let at = src + .find("fn which_monitor_binary()") + .expect("which_monitor_binary must exist; this test would otherwise cover nothing"); + let rest = &src[at..]; + let open = rest.find('{').expect("function body"); + let mut depth = 0usize; + for (i, ch) in rest[open..].char_indices() { + match ch { + '{' => depth += 1, + '}' => { + depth -= 1; + if depth == 0 { + return rest[open..open + i + 1].to_string(); + } + } + _ => {} + } + } + panic!("unbalanced braces in which_monitor_binary"); +} + +/// The lookup resolves the global plugin directory through the accessor. +#[test] +fn the_monitor_lookup_uses_the_accessor_the_installer_writes_through() { + let body = which_monitor_binary_code(); + assert!( + body.contains("global_bin_dir()"), + "which_monitor_binary must resolve the global plugin directory through \ + PluginRegistry::global_bin_dir() — the same accessor `horus install` \ + links into, `horus remove` unlinks from and `horus plugin trust` \ + resolves against. Restating the path is what let the two drift.\n\ + Body was:\n{body}" + ); +} + +/// And it does not restate a home-relative plugin path of its own. +/// +/// The project-local `.horus/bin/horus-monitor` check is a different thing and +/// stays: it is CWD-relative and gated behind `HORUS_ALLOW_LOCAL_PLUGINS`. What +/// must not come back is a HOME-relative `~/.horus/bin`, which is the copy that +/// disagreed with the installer. +#[test] +fn the_monitor_lookup_does_not_hardcode_a_home_relative_plugin_dir() { + let body = which_monitor_binary_body(); + let offenders: Vec<&str> = body + .lines() + .filter(|l| !l.trim_start().starts_with("//")) + .filter(|l| l.contains("home_dir") || l.contains("HOME")) + .collect(); + assert!( + offenders.is_empty(), + "which_monitor_binary must not build a plugin path from the home \ + directory — `~/.horus` holds installer state, not plugin binaries. \ + Offending lines:\n{}", + offenders.join("\n") + ); +} + +/// And the project-local path stays behind its opt-in. +/// +/// `.horus/bin/horus-monitor` is CWD-relative. It was once checked +/// unconditionally and BEFORE `PATH`, so `cd`-ing into a cloned repository that +/// ships that file and running `horus monitor` executed it directly — no trust +/// gate, no signature check, none of the sandbox `plugins::executor` applies — +/// and it SHADOWED a properly installed binary. Cloning a repository must not +/// be sufficient to run its code. +/// +/// The two tests above would both still pass with the gate deleted: one checks +/// the accessor is used, the other that no HOME-relative path is built, and +/// neither looks at the local branch. So the property is asserted here. +#[test] +fn the_project_local_lookup_stays_gated_behind_the_opt_in() { + let body = which_monitor_binary_body(); + + let mentions_local = body.contains(".horus/bin"); + if !mentions_local { + // The local branch was removed entirely — the shadowing risk is gone + // with it, which satisfies the property this test exists to protect. + return; + } + + assert!( + body.contains("HORUS_ALLOW_LOCAL_PLUGINS"), + "which_monitor_binary still consults the project-local .horus/bin, so \ + that branch must remain gated behind HORUS_ALLOW_LOCAL_PLUGINS. \ + Without the gate, cloning a repository that ships \ + .horus/bin/horus-monitor is enough to execute it, ahead of PATH and \ + ahead of a properly installed binary.\nBody was:\n{body}" + ); +} diff --git a/horus_net/src/replicator.rs b/horus_net/src/replicator.rs index aef9fe16f..5be07bf1b 100644 --- a/horus_net/src/replicator.rs +++ b/horus_net/src/replicator.rs @@ -1231,6 +1231,26 @@ impl Replicator { /// Send a system topic payload to all known peers + multicast. fn send_system_topic(&mut self, topic_name: &str, payload: &[u8]) { + // Ask the export guard, which is the only place that ever asks it about + // a system topic. + // + // `ImportExportGuard::allow_export` handles `_horus.*` explicitly and + // its comment says `deny_export = ["_horus.logs"]` "now actually + // works" — but its one production caller, `handle_export`, filters + // `self.readers`, and a reader only exists for a topic in the + // `TopicRegistry`. Nothing registers `_horus.logs` or `_horus.presence` + // there, so the guard was never asked about either, and they left the + // machine down this path instead: every Error and Warning line and the + // full node inventory, multicast once per timer tick, to any host that + // announced an interest. An operator who set `deny_export` got no + // change in behaviour and a green test telling them it was honoured. + // + // `_horus.estop` does not come through here (EstopBroadcaster owns it), + // so this cannot be used to silence a fleet e-stop. + if !self.guard.allow_export(topic_name) { + return; + } + // Share the framing (and therefore the monotonic sequence) with the // e-stop path. This used to build its own header with a hard-coded // `sequence: 0`, which the receiver's dedup — which runs ahead of the @@ -1389,6 +1409,59 @@ mod tests { // These drive the private receive path directly. Run with --test-threads=1: // the data-packet test installs the global external emergency-stop hook. + /// `deny_export` on a system topic must actually stop it leaving. + /// + /// `ImportExportGuard::allow_export` handles `_horus.*` and its comment says + /// `deny_export = ["_horus.logs"]` "now actually works" — with a green test + /// beside it asserting exactly that. Both were true of the guard function + /// and false of the process: `allow_export`'s only production caller filters + /// `self.readers`, which is populated from the `TopicRegistry`, and nothing + /// registers `_horus.logs` or `_horus.presence` there. They left the machine + /// through `send_system_topic` instead, which never asked. + /// + /// Asserted on `system_seq` rather than on packets: `frame_system_topic` + /// advances it exactly once per message that gets framed, so it counts what + /// was sent without needing a peer, a socket or a multicast group. + #[test] + fn deny_export_on_a_system_topic_stops_it_being_broadcast() { + use crate::registry::{SYSTEM_TOPIC_LOGS, SYSTEM_TOPIC_PRESENCE}; + + let registry = Arc::new(TopicRegistry::new()); + let mut config = NetConfig::test_config(0); + config.deny_export = vec![SYSTEM_TOPIC_LOGS.to_string()]; + let mut rep = Replicator::new(registry, config).unwrap(); + + let before = rep.system_seq; + rep.send_system_topic(SYSTEM_TOPIC_LOGS, b"an error line"); + assert_eq!( + rep.system_seq, before, + "a denied system topic must not be framed or sent" + ); + + // The one that was NOT denied still goes, so this is a filter and not + // an off switch. + rep.send_system_topic(SYSTEM_TOPIC_PRESENCE, b"node inventory"); + assert_eq!( + rep.system_seq, + before + 1, + "denying one system topic must not silence the others" + ); + } + + /// The default is unchanged: zero-config LAN replication still exports both. + #[test] + fn system_topics_are_still_exported_by_default() { + use crate::registry::{SYSTEM_TOPIC_LOGS, SYSTEM_TOPIC_PRESENCE}; + + let registry = Arc::new(TopicRegistry::new()); + let mut rep = Replicator::new(registry, NetConfig::test_config(0)).unwrap(); + + let before = rep.system_seq; + rep.send_system_topic(SYSTEM_TOPIC_LOGS, b"an error line"); + rep.send_system_topic(SYSTEM_TOPIC_PRESENCE, b"node inventory"); + assert_eq!(rep.system_seq, before + 2); + } + fn test_replicator() -> Replicator { let registry = Arc::new(TopicRegistry::new()); Replicator::new(registry, NetConfig::test_config(0)).unwrap() diff --git a/horus_py/horus/__init__.py b/horus_py/horus/__init__.py index a1250328d..9b3490fb4 100644 --- a/horus_py/horus/__init__.py +++ b/horus_py/horus/__init__.py @@ -11,7 +11,7 @@ img = horus.Image(480, 640, "rgb8") # Send over a topic (zero-copy) - topic = horus.Topic("camera.rgb") + topic = horus.Topic(horus.Image, endpoint="camera.rgb") topic.send(img) # Receive (in another process) @@ -34,10 +34,16 @@ 'budget'``. The ``us`` and ``ms`` constants are available: ``from horus import us, ms`` -2. **Topic type** - ``Topic(int)`` or ``Topic(42)`` raises TypeError. +2. **Topic type** - ``Topic(42)`` raises TypeError. ``Topic(int)`` does NOT: + an unrecognised class opens a generic MessagePack topic named after it + (here, ``"int"``), which is almost never what was meant. Wrong: ``topic = Topic(42)`` Right: ``topic = Topic(CmdVel)`` or ``Topic("my_topic")`` + A pool-backed type needs the type AND the name: + ``Topic(Image, endpoint="camera.rgb")``. Passing only the name gives an + untyped topic, and sending an ``Image`` over it raises. + 3. **Rate must be positive** - ``Node(rate=0)`` or ``Node(rate=-1)`` raises ValueError. 4. **Logging outside scheduler** - ``node.log_info("msg")`` called outside @@ -96,6 +102,9 @@ def __init__(self, topic: str, msg_type=None, capacity: int = 1024): try: from horus._horus import ( Topic, # Unified communication API + # Reads an existing ring's recorded type from its SHM header without + # attaching to it — see Node._detect_topic_type. + peek_topic_type as _peek_topic_type, # Internal: Rust bridge classes (not user-facing — use horus.Scheduler wrapper instead) Scheduler as _PyScheduler, SchedulerConfig as _SchedulerConfig, @@ -218,6 +227,7 @@ def __init__(self, topic: str, msg_type=None, capacity: int = 1024): HorusNotFoundError, HorusTransformError, HorusTimeoutError, + HorusBackpressureError, # Coordinate transforms TransformFrame, Transform, @@ -258,6 +268,8 @@ def minimal(): return _SchedulerConfig() def get_version(): return "0.1.0-mock" + def _peek_topic_type(name): return None + class Image: pass @@ -282,6 +294,14 @@ class HorusTransformError(Exception): class HorusTimeoutError(Exception): """Raised when a blocking operation times out.""" + class HorusBackpressureError(Exception): + """Raised when send_blocking() is refused for want of backpressure. + + The topic's backend overwrites unread messages and has no full ring to + wait on, so the call was refused before any write. Unlike a timeout this + is not transient: it lasts as long as the topology that caused it. + """ + # ── Python enums wrapping Rust constants ───────────────────────────────────── @@ -727,6 +747,25 @@ def _setup_topics(self): # try to read the type from the existing SHM header if msg_type is None and topic in self.sub_topics: msg_type = self._detect_topic_type(topic) + if msg_type is None: + # The ring exists and holds a type this build cannot name. + # Falling through would open a GenericMessage topic against + # it, and the runtime's size check would refuse with a + # message about byte counts that names neither `subs=`, nor + # the type the ring actually holds, nor the fix. + ring_type = self._peek_ring_type(topic) + if ring_type: + raise RuntimeError( + f"topic {topic!r} already holds {ring_type!r} messages, " + f"but it was declared as a bare string in subs=, which " + f"means an untyped MessagePack topic. Those cannot share " + f"a name. Declare the type: " + f"subs=[Sub({topic!r}, {ring_type})] " + f"-- {ring_type} is a class, so import it first " + f"(from horus import {ring_type}). If this build has " + f"no {ring_type}, the ring was written by a peer with " + f"a message type this install does not carry." + ) if msg_type is not None: # Typed topic — Pod zero-copy path (~1.5μs) @@ -735,24 +774,43 @@ def _setup_topics(self): # String topic — GenericMessage path (~5-50μs) self._topics[topic] = Topic(topic, capacity) + @staticmethod + def _peek_ring_type(topic_name: str): + """The type name an existing ring records, or None. Never raises.""" + try: + name = _peek_topic_type(topic_name) + except Exception: + return None + if not name or name.lower() == 'genericmessage': + return None + return name + def _detect_topic_type(self, topic_name: str): """Try to detect the topic's message type from an existing SHM header. Returns the message type class if found, None otherwise. + + This used to open ``Topic(topic_name, 64)`` and read a ``type_name`` + attribute off it. Neither half worked: ``Topic`` exposes no + ``type_name``, so the value was always ``None``; and the probe could + not have succeeded anyway, because ``Topic(name, capacity)`` builds a + ``GenericMessage`` topic and attaching that to an existing typed ring + is refused by the runtime's size check -- correctly, since a + name-based exemption once let a generic handle write past the end of a + typed mapping. So step 2 of the documented resolution order never ran + for any topic that has ever existed, and ``_TYPE_NAME_MAP`` had no + reachable reader. + + ``peek_topic_type`` reads the header without attaching, which is the + only way to ask this question without already knowing the answer. """ try: - # Create a temporary generic topic to probe the SHM header - probe = Topic(topic_name, 64) - # Check the topic's type name from the SHM header - type_name = getattr(probe, 'type_name', None) - if type_name and callable(type_name): - type_name = type_name() - if not type_name or type_name.lower() == 'genericmessage': - return None - # Look up the type by name in our known types - return _TYPE_NAME_MAP.get(type_name.lower()) + type_name = _peek_topic_type(topic_name) except Exception: return None + if not type_name or type_name.lower() == 'genericmessage': + return None + return _TYPE_NAME_MAP.get(type_name.lower()) def _ensure_topic(self, topic: str) -> None: """Ensure a Topic object exists for the given name (auto-create if needed).""" @@ -2002,6 +2060,7 @@ def run(*nodes: Node, "HorusNotFoundError", "HorusTransformError", "HorusTimeoutError", + "HorusBackpressureError", # Submodules "msggen", "perception", diff --git a/horus_py/horus/_horus.pyi b/horus_py/horus/_horus.pyi index ad6aa06ea..ace20493c 100644 --- a/horus_py/horus/_horus.pyi +++ b/horus_py/horus/_horus.pyi @@ -10,6 +10,12 @@ from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, Union def get_version() -> str: ... def get_timestamp_ns() -> int: ... +# The message type an existing topic's ring already holds, or None. Reads the +# SHM header only; it never constructs a Topic. `horus/__init__.py` imports it +# as `_peek_topic_type` and falls back to a None-returning stub when the +# extension was built without it. +def peek_topic_type(topic: str) -> Optional[str]: ... + # ── Framework clock ───────────────────────────────────────────────────────── def time_now() -> float: ... @@ -111,6 +117,7 @@ def time_rng_float() -> float: ... class HorusNotFoundError(Exception): ... class HorusTransformError(Exception): ... class HorusTimeoutError(Exception): ... +class HorusBackpressureError(Exception): ... # ── NodeState ──────────────────────────────────────────────────────────────── @@ -171,6 +178,8 @@ class Topic: def endpoint(self) -> Optional[str]: ... @property def backend_type(self) -> str: ... + @property + def provides_backpressure(self) -> bool: ... def send(self, message: Any, node: Optional[Any] = None) -> bool: ... def try_send(self, message: Any, node: Optional[Any] = None) -> bool: ... def send_blocking(self, message: Any, timeout_s: float, node: Optional[Any] = None) -> None: ... diff --git a/horus_py/horus/msggen/__init__.py b/horus_py/horus/msggen/__init__.py index 6c41b6c5e..e554d5f18 100644 --- a/horus_py/horus/msggen/__init__.py +++ b/horus_py/horus/msggen/__init__.py @@ -1,8 +1,12 @@ """ HORUS Message Generator — Generate custom Rust message types from Python. -Generates Rust PyO3 code and compiles via maturin. The generated messages -get full zero-copy IPC support (~3μs latency) just like built-in types. +Generates Rust PyO3 code and compiles via maturin. + +Generated messages travel over the MessagePack path, not the zero-copy POD +path the built-in types use: `Topic()` resolves a generated class through the +generic backend. They work — `send()` flattens the instance to its fields — +but do not assume built-in-type latency for them. Usage: from horus.msggen import register_message, build_messages @@ -16,8 +20,15 @@ build_messages() # Generates Rust code + runs maturin develop - from horus import RobotStatus, Topic - topic = Topic(RobotStatus) # Full zero-copy typed topic + from horus._horus import RobotStatus # NOT `from horus import` — see below + from horus import Topic + topic = Topic(RobotStatus) + topic.send(RobotStatus(battery_level=0.9, error_code=0, + is_active=True, timestamp=0)) + +A generated class is registered on the extension module, so it is imported +from ``horus._horus``; ``horus/__init__.py`` re-exports a fixed list that a +generated name is not on. """ from .generator import ( diff --git a/horus_py/horus/msggen/builder.py b/horus_py/horus/msggen/builder.py index 382252dc5..5c1b31fbb 100644 --- a/horus_py/horus/msggen/builder.py +++ b/horus_py/horus/msggen/builder.py @@ -192,8 +192,14 @@ def build_messages(force: bool = False, verbose: bool = True) -> bool: print("\nBuild successful! Custom messages are now available.") print("\nUsage:") for msg in messages: - print(f" from horus import {msg.name}, Topic") + # `from horus import ` raises ImportError: the class is + # registered on the extension module and `horus/__init__.py` + # re-exports a fixed hard-coded list, which a generated name is + # not on. Import from `horus._horus`, where it actually lives. + print(f" from horus._horus import {msg.name}") + print(f" from horus import Topic") print(f" topic = Topic({msg.name})") + print(f" topic.send({msg.name}(...))") return True diff --git a/horus_py/horus/msggen/generator.py b/horus_py/horus/msggen/generator.py index 3f667bb40..2233d654c 100644 --- a/horus_py/horus/msggen/generator.py +++ b/horus_py/horus/msggen/generator.py @@ -280,12 +280,20 @@ def _generate_rust_code(msg: MessageDef) -> str: def generate_messages_from_yaml(yaml_path: str, output_dir: Optional[Path] = None) -> List[str]: - if not _HAS_YAML: - raise ImportError("pyyaml is required for YAML message generation: pip install pyyaml") - """ Generate messages from a YAML definition file. + Emitter only: this calls ``generate_message`` for each entry and never + ``register_message``, so nothing reaches the build registry and + ``build_messages()`` will not compile the result. Omit ``output_dir`` and it + writes no files at all — it just returns the sources. + + Do not point ``output_dir`` at ``horus_py/src/custom_messages``. That + directory is generator-owned: ``build_messages()`` rewrites its ``mod.rs`` + to name only the registered messages, so a file dropped there is declared + by no ``mod`` statement, never compiled, and left behind by every later + build. + YAML format: ```yaml messages: @@ -301,6 +309,15 @@ def generate_messages_from_yaml(yaml_path: str, output_dir: Optional[Path] = Non type: bool ``` """ + # After the docstring, not before it. This guard used to sit above the + # triple-quoted block, which made that block a no-op expression rather than + # a docstring: `generate_messages_from_yaml.__doc__` was None and `help()` + # showed none of the schema below. `python_api_docs.rs` states the standard + # this fails — "its docstrings are the only API reference a user gets at the + # prompt". + if not _HAS_YAML: + raise ImportError("pyyaml is required for YAML message generation: pip install pyyaml") + with open(yaml_path) as f: data = yaml.safe_load(f) diff --git a/horus_py/src/drivers.rs b/horus_py/src/drivers.rs index 4f55da9ff..81998cf3d 100644 --- a/horus_py/src/drivers.rs +++ b/horus_py/src/drivers.rs @@ -16,11 +16,53 @@ use crate::driver_params::PyDriverParams; static PY_DRIVERS: LazyLock>>> = LazyLock::new(|| Mutex::new(HashMap::new())); +// ── Simulation stub ──────────────────────────────────────────────────── + +/// Stand-in for a driver that `horus run --sim` says must not touch hardware. +/// +/// The Python twin of `horus_core`'s `SimStubNode`. Under `--sim`, the Rust +/// loader substitutes a no-op node for every entry marked `sim = true`; the +/// Python loader used to instantiate the registered driver class anyway, +/// because `load_config_entries` reported no simulation state. Constructing a +/// driver is where the serial port gets opened and the motor gets enabled, so +/// `--sim` did nothing at all on the Python path. +/// +/// Keeps the entry's params so simulation code can still read the config. +#[pyclass(name = "SimStub")] +pub struct PySimStub { + /// The `[hardware.]` key this stands in for, suffixed `_sim_stub`. + #[pyo3(get)] + name: String, + /// The config the real driver would have received. + #[pyo3(get)] + params: Py, +} + +#[pymethods] +impl PySimStub { + /// No-op: a stub has no hardware to bring up. + fn init(&self) {} + + /// No-op: a stub has no hardware to drive. + fn tick(&self) {} + + /// Always true — lets user code branch on "am I simulated?". + #[getter] + fn simulated(&self) -> bool { + true + } + + fn __repr__(&self) -> String { + format!("SimStub(name={:?})", self.name) + } +} + // ── Module functions ─────────────────────────────────────────────────── /// Load hardware config from ``horus.toml`` ``[hardware]`` section. /// /// Returns a list of ``(name, obj)`` tuples where ``obj`` is either: +/// - A ``SimStub`` (if ``horus run --sim`` says this entry is simulated) /// - A Python object (if a matching class was registered via ``register()``) /// - A ``NodeParams`` dict-like (if no Python class was registered) /// @@ -67,7 +109,23 @@ pub fn load_from(py: Python<'_>, path: &str) -> PyResult) let mut result = Vec::new(); - for (name, use_name, params) in entries { + for (name, use_name, params, simulated) in entries { + // `--sim` says this entry must not touch hardware. Constructing the + // registered class is exactly where it would, so stop before that — + // the same substitution horus_core makes for a Rust node. + if simulated { + let py_params = Py::new(py, PyDriverParams { inner: params })?; + let stub = Py::new( + py, + PySimStub { + name: format!("{name}_sim_stub"), + params: py_params, + }, + )?; + result.push((name, stub.into_any())); + continue; + } + // Try to instantiate from Python registry let cls = PY_DRIVERS .lock() @@ -112,6 +170,7 @@ pub fn register_drivers_module(parent: &Bound<'_, PyModule>) -> PyResult<()> { hardware.add_function(wrap_pyfunction!(robot_name, &hardware)?)?; hardware.add_function(wrap_pyfunction!(robot_name_from, &hardware)?)?; hardware.add_class::()?; + hardware.add_class::()?; hardware.setattr( "__doc__", diff --git a/horus_py/src/errors.rs b/horus_py/src/errors.rs index c838e438e..3aff2ca4c 100644 --- a/horus_py/src/errors.rs +++ b/horus_py/src/errors.rs @@ -4,6 +4,7 @@ //! - `HorusNotFoundError` — missing topics, frames, nodes //! - `HorusTransformError` — transform extrapolation, stale data //! - `HorusTimeoutError` — blocking operation timeouts +//! - `HorusBackpressureError` — send_blocking() refused: no backpressure to wait on use horus_core::error::HorusError; use pyo3::create_exception; @@ -30,6 +31,14 @@ create_exception!( PyException, "Raised when a blocking operation times out." ); +create_exception!( + _horus, + HorusBackpressureError, + PyException, + "Raised when send_blocking() is refused because the topic's backend has no \ + backpressure. Unlike a timeout this is not transient: it lasts as long as \ + the topology that caused it, so retrying cannot succeed." +); /// Build the error message, appending actionable `.help()` text when available. /// diff --git a/horus_py/src/lib.rs b/horus_py/src/lib.rs index 98887339c..8dc51b352 100644 --- a/horus_py/src/lib.rs +++ b/horus_py/src/lib.rs @@ -114,8 +114,13 @@ fn _horus(m: &Bound<'_, PyModule>) -> PyResult<()> { "HorusTimeoutError", m.py().get_type::(), )?; + m.add( + "HorusBackpressureError", + m.py().get_type::(), + )?; m.add_function(wrap_pyfunction!(get_version, m)?)?; + m.add_function(wrap_pyfunction!(topic::peek_topic_type, m)?)?; // Time API — horus.now(), horus.dt(), etc. m.add_function(wrap_pyfunction!(time_now, m)?)?; diff --git a/horus_py/src/scheduler.rs b/horus_py/src/scheduler.rs index 69bbf5230..45037ff66 100644 --- a/horus_py/src/scheduler.rs +++ b/horus_py/src/scheduler.rs @@ -859,6 +859,25 @@ impl PyScheduler { } } +/// `self.inner` is `None` for exactly one reason: `run()` took the +/// scheduler out to drive it (`with_inner_run`), so it is running right now. +/// +/// Introspection used to answer that state with an invented value -- zero +/// nodes, zero ticks, `false` for `is_recording`, the literal string +/// `"PythonScheduler"` -- which is indistinguishable from a real +/// observation. A node-health check polling from inside a tick therefore +/// read "no nodes, tick 0, not recording" off a scheduler that had all +/// three, and `is_recording() == false` on a session that was recording is +/// the kind of answer someone deletes data over. +/// +/// `get_node_stats` in this same block already refuses ("Stats unavailable +/// while scheduler is running"), Invariant 3 in the module header says a +/// re-entrant call "will find `self.inner == None` and receive +/// `RuntimeError`", and every mutator here says so too. This is that error. +fn unavailable_while_running(what: &str) -> pyo3::PyErr { + PyRuntimeError::new_err(format!("{what} unavailable while scheduler is running")) +} + #[pymethods] impl PyScheduler { #[new] @@ -1103,7 +1122,7 @@ impl PyScheduler { let removed = self.removed_nodes.lock().map_err(lock_poisoned)?; Ok(sched.node_list().len() - removed.len()) } - None => Ok(0), + None => Err(unavailable_while_running("Node count")), } } @@ -1115,7 +1134,7 @@ impl PyScheduler { let removed = self.removed_nodes.lock().map_err(lock_poisoned)?; Ok(sched.node_list().contains(&name) && !removed.contains(&name)) } - None => Ok(false), + None => Err(unavailable_while_running("The node list")), } } @@ -1131,7 +1150,7 @@ impl PyScheduler { .filter(|n| !removed.contains(n)) .collect()) } - None => Ok(Vec::new()), + None => Err(unavailable_while_running("The node list")), } } @@ -1147,7 +1166,7 @@ impl PyScheduler { } Ok(None) } - None => Ok(None), + None => Err(unavailable_while_running("Node info")), } } @@ -1167,10 +1186,9 @@ impl PyScheduler { /// Get runtime capabilities as a dict. fn capabilities(&self, py: Python) -> PyResult>> { let guard = self.inner.lock().map_err(lock_poisoned)?; - let inner = match guard.as_ref() { - Some(s) => s, - None => return Ok(None), - }; + let inner = guard + .as_ref() + .ok_or_else(|| unavailable_while_running("Capabilities"))?; match inner.capabilities() { Some(caps) => { let dict = PyDict::new(py); @@ -1189,17 +1207,16 @@ impl PyScheduler { let guard = self.inner.lock().map_err(lock_poisoned)?; match guard.as_ref() { Some(sched) => Ok(sched.has_full_rt()), - None => Ok(false), + None => Err(unavailable_while_running("RT capability")), } } /// Get RT degradations as a list of dicts. fn degradations(&self, py: Python) -> PyResult> { let guard = self.inner.lock().map_err(lock_poisoned)?; - let inner = match guard.as_ref() { - Some(s) => s, - None => return Ok(PyList::empty(py).into()), - }; + let inner = guard + .as_ref() + .ok_or_else(|| unavailable_while_running("Degradations"))?; let result = PyList::empty(py); for deg in inner.degradations() { let dict = PyDict::new(py); @@ -1216,7 +1233,7 @@ impl PyScheduler { let guard = self.inner.lock().map_err(lock_poisoned)?; match guard.as_ref() { Some(sched) => Ok(sched.current_tick()), - None => Ok(0), + None => Err(unavailable_while_running("The tick count")), } } @@ -1225,7 +1242,7 @@ impl PyScheduler { let guard = self.inner.lock().map_err(lock_poisoned)?; match guard.as_ref() { Some(sched) => Ok(sched.scheduler_name().to_string()), - None => Ok("PythonScheduler".to_string()), + None => Err(unavailable_while_running("The scheduler name")), } } @@ -1236,10 +1253,9 @@ impl PyScheduler { /// Get safety statistics as a dict. fn safety_stats(&self, py: Python) -> PyResult>> { let guard = self.inner.lock().map_err(lock_poisoned)?; - let inner = match guard.as_ref() { - Some(s) => s, - None => return Ok(None), - }; + let inner = guard + .as_ref() + .ok_or_else(|| unavailable_while_running("Safety stats"))?; match inner.safety_stats() { Some(stats) => { let dict = PyDict::new(py); @@ -1263,7 +1279,7 @@ impl PyScheduler { let guard = self.inner.lock().map_err(lock_poisoned)?; match guard.as_ref() { Some(sched) => Ok(sched.is_recording()), - None => Ok(false), + None => Err(unavailable_while_running("Recording state")), } } @@ -1272,7 +1288,7 @@ impl PyScheduler { let guard = self.inner.lock().map_err(lock_poisoned)?; match guard.as_ref() { Some(sched) => Ok(sched.is_replaying()), - None => Ok(false), + None => Err(unavailable_while_running("Replay state")), } } diff --git a/horus_py/src/topic.rs b/horus_py/src/topic.rs index f245fd24d..d3e09af62 100644 --- a/horus_py/src/topic.rs +++ b/horus_py/src/topic.rs @@ -97,6 +97,56 @@ fn topic_lock(lock: &Mutex) -> PyResult> { .map_err(|e| PyRuntimeError::new_err(format!("Topic lock poisoned: {e}"))) } +/// A message object's public, non-callable attributes as a dict. +/// +/// The Rust twin of `horus._message_to_dict`, and used the same way: only +/// after a direct conversion has already been refused, so it needs no guess +/// about which types are messages. Returns `None` for anything with nothing to +/// extract, and for the built-in and array-like types a caller would never mean +/// to flatten. +fn message_object_as_dict<'py>( + py: Python<'py>, + obj: &pyo3::Bound<'py, PyAny>, +) -> Option> { + use pyo3::types::{PyAnyMethods, PyDict, PyDictMethods}; + + // Anything pythonize already handles, or would mangle. + if obj.is_none() + || obj.is_instance_of::() + || obj.is_instance_of::() + || obj.is_instance_of::() + || obj.is_instance_of::() + || obj.is_instance_of::() + || obj.is_instance_of::() + || obj.is_instance_of::() + || obj.is_instance_of::() + || obj.is_instance_of::() + || obj.hasattr("__array_interface__").unwrap_or(false) + || obj.hasattr("__array__").unwrap_or(false) + { + return None; + } + + let dict = PyDict::new(py); + let names = obj.dir().ok()?; + for name in names.iter() { + let Ok(key) = name.extract::() else { + continue; + }; + if key.starts_with('_') { + continue; + } + let Ok(value) = obj.getattr(key.as_str()) else { + continue; + }; + if value.is_callable() { + continue; + } + let _ = dict.set_item(key, value); + } + (!dict.is_empty()).then_some(dict) +} + /// Encode an arbitrary Python object as a `GenericMessage` (MessagePack payload). /// /// Shared by `send`, `try_send`, and `send_blocking` so an untyped @@ -104,9 +154,31 @@ fn topic_lock(lock: &Mutex) -> PyResult> { /// matter which of the three the caller reached for. fn generic_message_from_py(py: Python, message: &Py) -> PyResult { let bound = message.bind(py); - let value: serde_json::Value = pythonize::depythonize(bound).map_err(|e| { - pyo3::exceptions::PyTypeError::new_err(format!("Failed to convert Python object: {}", e)) - })?; + let value: serde_json::Value = match pythonize::depythonize(bound) { + Ok(v) => v, + // `depythonize` cannot read an opaque `#[pyclass]`, which is what every + // HORUS message class and every msggen-generated class is. So + // `Topic(RobotStatus).send(instance)` — the line `build_messages()` + // prints on success — raised `TypeError: Failed to convert Python + // object`, while `node.send("robot.status", instance)` worked, because + // `Node.send` catches that refusal and retries the object as a dict. + // Two spellings of one operation disagreeing is the defect; the + // conversion belongs here, where all three send paths share it. + Err(first) => { + let Some(as_dict) = message_object_as_dict(py, bound) else { + return Err(pyo3::exceptions::PyTypeError::new_err(format!( + "Failed to convert Python object: {}", + first + ))); + }; + pythonize::depythonize(&as_dict).map_err(|e| { + pyo3::exceptions::PyTypeError::new_err(format!( + "Failed to convert Python object: {}", + e + )) + })? + } + }; let msgpack_bytes = rmp_serde::to_vec(&value).map_err(|e| { pyo3::exceptions::PyRuntimeError::new_err(format!( "Failed to serialize to MessagePack: {}", @@ -149,6 +221,38 @@ fn send_timeout_err( crate::errors::to_py_err(send_timeout_error(topic, elapsed, deadline)) } +/// The message type an existing topic's ring already holds, or `None`. +/// +/// Reads the SHM header only; it never constructs a `Topic`. That is the whole +/// point. Asking "what type is this ring?" by opening it is circular — opening +/// requires choosing a type, and choosing the wrong one is refused by the size +/// check in `RingTopic::new`. The previous auto-detection did exactly that: +/// `probe = Topic(name, 64)` builds a `Topic`, which the header +/// check is guaranteed to refuse on any typed ring, so the probe always raised +/// into a bare `except Exception` and the step could never succeed. +#[pyfunction] +pub fn peek_topic_type(name: &str) -> Option { + horus_core::communication::peek_topic_type_name(name) +} + +/// `SendBlockingError::NoBackpressure` as a Python exception. +/// +/// NOT `HorusTimeoutError`. The core writes this variant specifically to be +/// un-retryable — "it does not clear on retry — it lasts as long as the +/// topology that caused it" — and a timeout is the one error a caller retries. +/// Collapsing the two handed Python the opposite instruction, on the path the +/// method is documented for: emergency stop and motor setpoints. It is also +/// permanent and easy to reach, since any POD topic with a second subscriber +/// (a `horus topic echo`, a monitor) resolves to `PodShm`. +/// +/// The core's own message names the fix, so it is passed through verbatim. +fn no_backpressure_err(topic: &str) -> PyErr { + crate::errors::HorusBackpressureError::new_err(format!( + "send_blocking on topic {topic}: {}", + horus_core::communication::topic::SendBlockingError::NoBackpressure + )) +} + /// The `HorusError` behind [`send_timeout_err`], split out so its wording can be /// asserted on without a Python interpreter to unwrap a `PyErr` with. fn send_timeout_error( @@ -359,12 +463,17 @@ macro_rules! pod_topic_types { /// Send a POD message, waiting up to `timeout` for ring space. /// - /// `Ok(Some(true))` sent, `Ok(Some(false))` the ring stayed full for - /// the whole timeout, `Ok(None)` not a POD type (caller handles it). + /// `Ok(Some(Ok(())))` sent, `Ok(Some(Err(e)))` the core refused and + /// said why, `Ok(None)` not a POD type (caller handles it). + /// + /// This used to be `.is_ok()`, which threw away WHICH + /// `SendBlockingError` came back, so the caller had nothing left to + /// distinguish "the ring stayed full" from "this backend has no + /// full ring at all" and reported both as a timeout. fn send_blocking_pod( &self, py: Python, message: &Py, node: &Option>, timeout: std::time::Duration, start: std::time::Instant, - ) -> PyResult> { + ) -> PyResult>> { match &self.topic_type { $( TopicType::$rust_ty(topic) => { @@ -379,10 +488,18 @@ macro_rules! pod_topic_types { // consumer drains, so the GIL cannot be held across it: // a 10 ms wait here would stop every other Python thread // in the process for 10 ms. + // + // Both halves matter: `topic_lock` turns a poisoned + // lock into a Python error instead of panicking, and + // the `Result` is carried out whole so the caller can + // still tell `NoBackpressure` from `Timeout`. + // Collapsing it with `.is_ok()` here is what made a + // deliberate, un-retryable refusal arrive as a + // timeout. let sent = py.detach(|| { - topic_lock(&topic_ref).map(|g| g.send_blocking(val, timeout).is_ok()) + topic_lock(&topic_ref).map(|g| g.send_blocking(val, timeout)) })?; - if sent { + if sent.is_ok() { if let Some(s) = summary { log_ipc_event(py, node, &self.name, s, start.elapsed().as_nanos() as u64, "log_pub"); @@ -960,13 +1077,7 @@ impl PyTopic { let img = py_img.inner().clone(); let log_msg = format!("Image({}x{})", img.height(), img.width()); let topic_ref = topic.clone(); - let sent = py.detach(|| { - topic_ref - .lock() - .expect("topic lock poisoned") - .try_send(img) - .is_ok() - }); + let sent = py.detach(|| topic_lock(&topic_ref).map(|g| g.try_send(img).is_ok()))?; if sent && node.is_some() { log_ipc_event( py, @@ -984,13 +1095,7 @@ impl PyTopic { let pc = py_pc.inner().clone(); let log_msg = format!("PointCloud({} pts)", pc.point_count()); let topic_ref = topic.clone(); - let sent = py.detach(|| { - topic_ref - .lock() - .expect("topic lock poisoned") - .try_send(pc) - .is_ok() - }); + let sent = py.detach(|| topic_lock(&topic_ref).map(|g| g.try_send(pc).is_ok()))?; if sent && node.is_some() { log_ipc_event( py, @@ -1008,13 +1113,8 @@ impl PyTopic { let depth = py_depth.inner().clone(); let log_msg = format!("DepthImage({}x{})", depth.height(), depth.width()); let topic_ref = topic.clone(); - let sent = py.detach(|| { - topic_ref - .lock() - .expect("topic lock poisoned") - .try_send(depth) - .is_ok() - }); + let sent = + py.detach(|| topic_lock(&topic_ref).map(|g| g.try_send(depth).is_ok()))?; if sent && node.is_some() { log_ipc_event( py, @@ -1032,13 +1132,7 @@ impl PyTopic { use horus::core::LogSummary; let log_summary = msg.log_summary(); let topic_ref = topic.clone(); - let sent = py.detach(|| { - topic_ref - .lock() - .expect("topic lock poisoned") - .try_send(msg) - .is_ok() - }); + let sent = py.detach(|| topic_lock(&topic_ref).map(|g| g.try_send(msg).is_ok()))?; if sent && node.is_some() { log_ipc_event( py, @@ -1077,7 +1171,12 @@ impl PyTopic { /// /// Raises: /// HorusTimeoutError: the buffer stayed full for the whole timeout; - /// the message was NOT published. + /// the message was NOT published. Transient — retrying can work. + /// HorusBackpressureError: this topic's backend overwrites unread + /// messages and has no full ring to wait on, so the call was + /// refused before any write. NOT transient: it lasts as long as + /// the topology that caused it, so retrying cannot help. Check + /// `provides_backpressure()` after the first send. /// ValueError: timeout_s is negative, NaN, or infinite. /// NotImplementedError: pool-backed topics (Image, PointCloud, /// DepthImage, Tensor) have no blocking send path. @@ -1086,7 +1185,10 @@ impl PyTopic { /// try: /// estop.send_blocking(EmergencyStop(True), 0.05) /// except horus.HorusTimeoutError: - /// node.log_error("e-stop not delivered") + /// node.log_error("e-stop not delivered — ring full") + /// except horus.HorusBackpressureError: + /// node.log_error("e-stop topic is broadcast-backed; delivery " + /// "was never guaranteed") #[pyo3(signature = (message, timeout_s, node=None))] fn send_blocking( &self, @@ -1101,10 +1203,13 @@ impl PyTopic { // Fast path: POD types (generated by pod_topic_types! macro) if let Some(sent) = self.send_blocking_pod(py, &message, &node, timeout, start)? { - return if sent { - Ok(()) - } else { - Err(send_timeout_err(&self.name, start.elapsed(), timeout)) + use horus_core::communication::topic::SendBlockingError; + return match sent { + Ok(()) => Ok(()), + Err(SendBlockingError::NoBackpressure) => Err(no_backpressure_err(&self.name)), + Err(SendBlockingError::Timeout) => { + Err(send_timeout_err(&self.name, start.elapsed(), timeout)) + } }; } @@ -1115,15 +1220,16 @@ impl PyTopic { use horus::core::LogSummary; let log_summary = msg.log_summary(); let topic_ref = topic.clone(); - let sent = py.detach(|| { - topic_ref - .lock() - .expect("topic lock poisoned") - .send_blocking(msg, timeout) - .is_ok() - }); - if !sent { - return Err(send_timeout_err(&self.name, start.elapsed(), timeout)); + let sent = + py.detach(|| topic_lock(&topic_ref).map(|g| g.send_blocking(msg, timeout)))?; + if let Err(e) = sent { + use horus_core::communication::topic::SendBlockingError; + return Err(match e { + SendBlockingError::NoBackpressure => no_backpressure_err(&self.name), + SendBlockingError::Timeout => { + send_timeout_err(&self.name, start.elapsed(), timeout) + } + }); } if node.is_some() { log_ipc_event( @@ -1348,6 +1454,32 @@ impl PyTopic { self.endpoint.clone() } + /// Whether a full ring can make this topic refuse a send. + /// + /// This is a property of the BACKEND, and the backend is chosen at runtime + /// from how many participants are attached — one subscriber gives you + /// `SpscShm`, which refuses when full; attach a second (a logger, a + /// recorder, a `horus topic echo`) and the same topic becomes `PodShm` or + /// `FanoutShm`, which overwrite instead. On those, `send_blocking()` raises + /// `HorusBackpressureError` and `send()` silently loses data. + /// + /// Assert it on any topic carrying commands rather than samples, AFTER the + /// first send — the backend is unresolved until a handle moves a message, + /// and an unresolved backend answers False. + /// + /// ```python + /// estop.send(EmergencyStop(True)) # resolves the backend + /// assert estop.provides_backpressure, estop.backend_type + /// ``` + /// + /// Answered from `backend_type` through the runtime's own + /// `backend_provides_backpressure`, so this cannot drift from what + /// `send_blocking()` will actually do. + #[getter] + fn provides_backpressure(&self) -> bool { + horus_core::communication::topic::backend_provides_backpressure(&self.backend_type()) + } + /// Get the backend type name #[getter] fn backend_type(&self) -> String { @@ -2164,6 +2296,97 @@ mod tests { ); } + /// Auto-detecting a subscription's type must not require attaching to it. + /// + /// `_setup_topics` documents a three-step resolution whose middle step is + /// "auto-detect from SHM header". It was implemented as `probe = + /// Topic(name, 64)` followed by `getattr(probe, "type_name", None)`, and + /// both halves were dead: `Topic` exposes no `type_name`, and the probe + /// builds a `GenericMessage` topic, which the runtime refuses to attach to + /// a typed ring. So the step could never succeed for any topic that has + /// ever existed, and the ~70-entry `_TYPE_NAME_MAP` built for it had no + /// reachable reader. + /// + /// The first assertion below is that refusal — it is correct and must + /// stay, since a name-based exemption once let a generic handle write + /// megabytes past the end of a typed mapping. The question just cannot be + /// answered by attaching, which is what `peek_topic_type` is for. + #[test] + fn the_ring_type_is_readable_without_attaching_to_it() { + use horus::communication::Topic as CoreTopic; + + let name = format!("peek_probe_{}", std::process::id()); + + let Ok(typed) = CoreTopic::::new(&name) else { + eprintln!("skipping: no shared memory available"); + return; + }; + typed.send(CmdVel::new(1.0, 0.0)); + + // The old probe. It cannot open the ring, so it could never report + // anything about it. + assert!( + CoreTopic::::new(&name).is_err(), + "a generic handle must still be refused on a typed ring" + ); + + // The peek reads the header instead of attaching, and answers. + let seen = peek_topic_type(&name); + assert_eq!( + seen.as_deref(), + Some("CmdVel"), + "the header records the ring's type and it must be readable" + ); + + assert_eq!( + peek_topic_type("peek_probe_no_such_topic_anywhere"), + None, + "a topic that does not exist has no type" + ); + + drop(typed); + } + + /// A refusal for want of backpressure must NOT arrive as a timeout. + /// + /// The core writes `SendBlockingError::NoBackpressure` specifically to be + /// un-retryable — "it does not clear on retry — it lasts as long as the + /// topology that caused it" — and a timeout is the one error a caller + /// retries. `send_blocking_pod` used to end in `.is_ok()`, throwing the + /// variant away, and both arms reported `HorusTimeoutError`: the opposite + /// instruction, on the path this method is documented for (emergency stop + /// and motor setpoints). It is easy to reach, too — any POD topic resolves + /// to `PodShm` the moment a second subscriber attaches, so a `horus topic + /// echo` against an e-stop topic starts it raising forever. + #[test] + fn a_backpressure_refusal_is_not_a_timeout() { + // The exception types are Python objects, so this test needs an + // interpreter. Safe to call repeatedly. + Python::initialize(); + Python::attach(|py| { + let err = no_backpressure_err("estop"); + + assert!( + err.is_instance_of::(py), + "must raise HorusBackpressureError" + ); + assert!( + !err.is_instance_of::(py), + "must NOT be catchable as a timeout — a caller who retries a \ + timeout would retry forever, because this condition is \ + topological and permanent" + ); + + let msg = err.value(py).to_string(); + assert!(msg.contains("estop"), "must name the topic: {msg}"); + // The core's message names the fix; it must survive the crossing. + assert!( + msg.contains("MpscShm") || msg.contains("backpressured backend"), + "must carry the runtime's remediation text: {msg}" + ); + }); + } + /// The pool-backed refusal must say which call and which type, and point at /// the counter that makes the loss visible instead. #[test] diff --git a/horus_py/src/transform_frame.rs b/horus_py/src/transform_frame.rs index 857631b5d..ff6e45d18 100644 --- a/horus_py/src/transform_frame.rs +++ b/horus_py/src/transform_frame.rs @@ -482,7 +482,11 @@ impl PyTransformFrame { /// Transform once available /// /// Raises: - /// ValueError: If timeout expires before transform is available + /// horus.HorusTimeoutError: If timeout expires before the transform is + /// available. It derives from ``Exception``, not ``ValueError`` — + /// this said ``ValueError`` for a while, so the ``except + /// ValueError`` recovery path a reader wrote from this docstring + /// never caught the timeout and the error left the node's tick(). #[pyo3(signature = (src, dst, timeout_sec=5.0))] fn wait_for_transform( &self, @@ -662,7 +666,8 @@ impl PyTransformFrame { /// Interpolated transform at the given timestamp /// /// Raises: - /// ValueError: If timestamp is outside buffered range + /// horus.HorusTransformError: If the timestamp is outside the buffered + /// range. It derives from ``Exception``, not ``ValueError``. fn tf_at_strict(&self, src: &str, dst: &str, timestamp_ns: u64) -> PyResult { self.inner .tf_at_strict(src, dst, timestamp_ns) diff --git a/horus_py/tests/test_generic_topic_message_objects.py b/horus_py/tests/test_generic_topic_message_objects.py new file mode 100644 index 000000000..36e3d00ad --- /dev/null +++ b/horus_py/tests/test_generic_topic_message_objects.py @@ -0,0 +1,141 @@ +"""An untyped topic must accept a message object, not just a dict. + +`Topic("name")` carries `GenericMessage` (MessagePack), and +`generic_message_from_py` ran `pythonize::depythonize` on the Python object — +which cannot read an opaque `#[pyclass]`, i.e. every HORUS message class and +every msggen-generated class. So `Topic(RobotStatus).send(instance)`, the line +`build_messages()` PRINTS on success, raised `TypeError: Failed to convert +Python object`, while `node.send("robot.status", instance)` worked, because +`Node.send` catches that refusal and retries the object as a dict. + +Two spellings of one operation disagreeing is the defect. The conversion now +lives where all three send paths share it. +""" +import uuid + +import pytest + +import horus +from horus import CmdVel, Pose2D, Topic + + +def _unique(prefix): + return f"{prefix}_{uuid.uuid4().hex[:8]}" + + +def test_a_message_class_survives_an_untyped_topic(): + t = Topic(_unique("generic_msgclass")) + t.send(CmdVel(1.0, 0.5)) + + got = t.recv() + assert got is not None, "the message must arrive" + assert got["linear"] == pytest.approx(1.0) + assert got["angular"] == pytest.approx(0.5) + + +def test_try_send_and_send_blocking_agree_with_send(): + """All three publish paths share the conversion, so all three accept it.""" + t = Topic(_unique("generic_all_paths")) + assert t.try_send(Pose2D(1.0, 2.0, 0.5)) is True + t.send_blocking(Pose2D(3.0, 4.0, 1.5), 0.2) + + seen = [] + while (m := t.recv()) is not None: + seen.append(m) + assert len(seen) == 2, f"both publishes must arrive: {seen}" + assert seen[0]["x"] == pytest.approx(1.0) + assert seen[1]["x"] == pytest.approx(3.0) + + +def test_a_plain_dict_still_works_unchanged(): + """The fallback must not disturb the ordinary MessagePack path.""" + t = Topic(_unique("generic_dict")) + t.send({"a": 1, "b": [1, 2, 3]}) + assert t.recv() == {"a": 1, "b": [1, 2, 3]} + + +def test_something_with_nothing_to_extract_still_raises(): + """The fallback is a retry, not a licence to encode anything.""" + t = Topic(_unique("generic_unencodable")) + with pytest.raises(TypeError): + t.send(lambda x: x) + + +def test_the_package_quick_start_runs(): + """`help(horus)`'s first example is the only API reference some users read. + + It published an `Image` over a bare-string topic and called it zero-copy; + the call raised and nothing after it ran. + """ + img = horus.Image(48, 64, "rgb8") + topic = horus.Topic(horus.Image, endpoint=_unique("camera.rgb")) + topic.send(img) + + received = topic.recv() + assert received is not None + arr = received.to_numpy() + assert arr.shape == (48, 64, 3) + + +def test_send_blocking_on_a_broadcast_topic_raises_backpressure_not_timeout(): + """The mapping must hold on the SEND PATH, not just in the constructor. + + The Rust-side test asserts `no_backpressure_err` builds the right exception. + That is the easy half: the defect was that `send_blocking_pod` ended in + `.is_ok()`, discarding WHICH `SendBlockingError` came back, so the send path + reported every refusal as a timeout no matter what the core returned. This + drives a real broadcast-backed topic instead. + + A second subscriber is what resolves a POD topic to `PodShm`, and it has to + be on another thread: an entry is keyed on (pid, thread), so two handles + here would share one registration and the topic would stay `SpscShm`. + """ + import threading + + name = _unique("estop_broadcast") + pub = Topic(CmdVel, endpoint=name) + + stop = threading.Event() + ready = threading.Barrier(3) + + def hold(): + sub = Topic(CmdVel, endpoint=name) + sub.try_recv() # claims this thread's participant slot + ready.wait() + while not stop.is_set(): + sub.try_recv() + + threads = [threading.Thread(target=hold, daemon=True) for _ in range(2)] + for t in threads: + t.start() + ready.wait() + + try: + # Resolve the backend, then confirm the topology we need. + pub.send(CmdVel(0.0, 0.0)) + if pub.provides_backpressure: + pytest.skip(f"topic did not reach a broadcast backend: {pub.backend_type}") + + with pytest.raises(horus.HorusBackpressureError) as excinfo: + pub.send_blocking(CmdVel(1.0, 0.0), 0.05) + + msg = str(excinfo.value) + assert name in msg, f"the error must name the topic: {msg}" + assert "backpressure" in msg.lower() or "MpscShm" in msg, ( + f"the core's remediation text must survive the crossing: {msg}" + ) + finally: + stop.set() + for t in threads: + t.join(timeout=5) + + +def test_provides_backpressure_reports_the_backend(): + """The property Python had no way to ask before.""" + name = _unique("bp_single") + t = Topic(CmdVel, endpoint=name) + assert t.provides_backpressure is False, "unresolved backend must fail closed" + t.send(CmdVel(0.0, 0.0)) + assert t.provides_backpressure is True, ( + "a single-subscriber POD topic resolves to SpscShm, which refuses a full ring" + ) diff --git a/horus_py/tests/test_introspection_never_invents.py b/horus_py/tests/test_introspection_never_invents.py new file mode 100644 index 000000000..8b8c06983 --- /dev/null +++ b/horus_py/tests/test_introspection_never_invents.py @@ -0,0 +1,115 @@ +"""Scheduler introspection must not invent an answer while the scheduler runs. + +`run()` takes the native scheduler out of the PyScheduler to drive it, so every +accessor sees `inner == None` for the whole run. Those accessors used to answer +that state with a fabricated value — zero nodes, tick 0, `False` for +`is_recording()`, the literal `"PythonScheduler"` — indistinguishable from a +real observation. Anything polling from inside a tick read a scheduler that +looked empty and idle while it was neither. + +`get_node_stats` in the same class always refused with "Stats unavailable while +scheduler is running", and Invariant 3 in `scheduler.rs` says a re-entrant call +gets a `RuntimeError`. These assert the rest now agree. +""" + +import horus + + +def _drive(probe): + """Run a one-node scheduler, calling `probe(sched)` from inside a tick.""" + seen = [] + + def tick(node): + if not seen: + seen.append(probe(sched._scheduler)) + + node = horus.Node(name="probe_node", tick=tick, rate=200, order=0) + sched = horus.Scheduler(tick_rate=200.0) + sched.add(node) + sched.run(duration=0.05) + assert seen, "the probe never ran — the tick callback did not fire" + return seen[0] + + +def _refused(call): + def probe(native): + try: + return ("value", call(native)) + except RuntimeError as exc: + return ("raised", str(exc)) + + kind, payload = _drive(probe) + assert kind == "raised", ( + f"introspection returned {payload!r} from inside a running tick " + "instead of refusing" + ) + assert "while scheduler is running" in payload, payload + return payload + + +def test_node_count_is_not_reported_as_zero_while_running(): + _refused(lambda s: s.get_node_count()) + + +def test_node_names_are_not_reported_as_empty_while_running(): + _refused(lambda s: s.get_node_names()) + + +def test_has_node_does_not_deny_a_node_that_exists(): + _refused(lambda s: s.has_node("probe_node")) + + +def test_current_tick_is_not_reported_as_zero_while_running(): + _refused(lambda s: s.current_tick()) + + +def test_scheduler_name_is_not_a_hardcoded_string_while_running(): + _refused(lambda s: s.scheduler_name()) + + +def test_recording_state_is_not_reported_as_false_while_running(): + # The dangerous one: "not recording" on a session that is recording. + _refused(lambda s: s.is_recording()) + + +def test_replay_state_is_not_reported_as_false_while_running(): + _refused(lambda s: s.is_replaying()) + + +def test_rt_capability_is_not_denied_while_running(): + _refused(lambda s: s.has_full_rt()) + + +def test_degradations_are_not_reported_as_none_while_running(): + _refused(lambda s: s.degradations()) + + +def test_safety_stats_do_not_read_as_absent_while_running(): + _refused(lambda s: s.safety_stats()) + + +def test_capabilities_do_not_read_as_absent_while_running(): + _refused(lambda s: s.capabilities()) + + +def test_status_still_answers_honestly_while_running(): + """`status()` always said "running" here — it is the one that was right.""" + kind, payload = _drive(lambda s: ("value", s.status())[1] and ("value", s.status())) + assert kind == "value" + assert payload == "running" + + +def test_everything_answers_normally_once_the_run_is_over(): + node = horus.Node(name="after_run", tick=lambda n: None, rate=200, order=0) + sched = horus.Scheduler(tick_rate=200.0) + sched.add(node) + sched.run(duration=0.02) + + native = sched._scheduler + assert native.get_node_count() == 1 + assert native.get_node_names() == ["after_run"] + assert native.has_node("after_run") is True + assert native.is_recording() is False + assert native.is_replaying() is False + assert isinstance(native.scheduler_name(), str) + assert native.current_tick() >= 0 diff --git a/horus_py/tests/test_sim_mode_blocks_real_drivers.py b/horus_py/tests/test_sim_mode_blocks_real_drivers.py new file mode 100644 index 000000000..af25eba54 --- /dev/null +++ b/horus_py/tests/test_sim_mode_blocks_real_drivers.py @@ -0,0 +1,109 @@ +"""`horus run --sim` must not construct a real driver on the Python path. + +The Rust loader substitutes a stub for every `[hardware]` entry marked +`sim = true`; the Python loader used to instantiate the registered driver +class regardless, because the shared parse reported no simulation state. +Constructing a driver is where the serial port is opened and the motor is +enabled, so `--sim` did nothing at all here. +""" + +import os + +import pytest + +from horus import drivers as hardware + + +CONSTRUCTED = [] + + +class RecordingDriver: + """Stands in for a driver whose __init__ touches hardware.""" + + def __init__(self, params): + CONSTRUCTED.append(params) + self.params = params + + +@pytest.fixture(autouse=True) +def _clean_env(): + CONSTRUCTED.clear() + saved = {k: os.environ.get(k) for k in ("HORUS_SIM_MODE", "HORUS_SIM_TARGETS")} + yield + for key, value in saved.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def _manifest(tmp_path, body): + path = tmp_path / "horus.toml" + path.write_text(body) + return str(path) + + +def test_sim_mode_returns_a_stub_instead_of_the_driver(tmp_path): + hardware.register_driver("RecordingDriver", RecordingDriver) + path = _manifest( + tmp_path, + '[hardware.arm]\nuse = "RecordingDriver"\nsim = true\nport = "/dev/ttyACM0"\n', + ) + + os.environ["HORUS_SIM_MODE"] = "1" + loaded = hardware.load_from(path) + + assert len(loaded) == 1 + name, node = loaded[0] + assert name == "arm" + assert CONSTRUCTED == [], "--sim must not construct the real driver" + assert type(node).__name__ == "SimStub" + assert node.simulated is True + assert node.name == "arm_sim_stub" + # The stub still carries the config, so simulation code can read it. + assert node.params.get("port") == "/dev/ttyACM0" + node.init() + node.tick() + + +def test_without_sim_mode_the_real_driver_is_constructed(tmp_path): + hardware.register_driver("RecordingDriver", RecordingDriver) + path = _manifest( + tmp_path, '[hardware.arm]\nuse = "RecordingDriver"\nsim = true\n' + ) + + os.environ.pop("HORUS_SIM_MODE", None) + loaded = hardware.load_from(path) + + assert len(CONSTRUCTED) == 1, "`sim = true` alone must not simulate" + assert isinstance(loaded[0][1], RecordingDriver) + + +def test_simulated_is_an_alias_of_sim_here_too(tmp_path): + hardware.register_driver("RecordingDriver", RecordingDriver) + path = _manifest( + tmp_path, + '[hardware.arm]\nuse = "RecordingDriver"\nsimulated = true\n', + ) + + os.environ["HORUS_SIM_MODE"] = "1" + loaded = hardware.load_from(path) + + assert CONSTRUCTED == [] + assert type(loaded[0][1]).__name__ == "SimStub" + + +def test_sim_targets_narrows_which_entries_are_stubbed(tmp_path): + hardware.register_driver("RecordingDriver", RecordingDriver) + path = _manifest( + tmp_path, + '[hardware.lidar]\nuse = "RecordingDriver"\nsim = true\n\n' + '[hardware.imu]\nuse = "RecordingDriver"\nsim = true\n', + ) + + os.environ["HORUS_SIM_MODE"] = "1" + os.environ["HORUS_SIM_TARGETS"] = "lidar" + by_name = dict(hardware.load_from(path)) + + assert type(by_name["lidar"]).__name__ == "SimStub" + assert isinstance(by_name["imu"], RecordingDriver) diff --git a/horus_py/tests/test_subs_type_detection.py b/horus_py/tests/test_subs_type_detection.py new file mode 100644 index 000000000..189c3a501 --- /dev/null +++ b/horus_py/tests/test_subs_type_detection.py @@ -0,0 +1,73 @@ +"""A bare-string `subs=` must resolve the ring's real type, not guess MessagePack. + +`Node._setup_topics` documents a three-step type resolution whose middle step is +"auto-detect from SHM header". It was implemented as ``probe = Topic(name, 64)`` +followed by ``getattr(probe, "type_name", None)``, and both halves were dead: +``Topic`` exposes no ``type_name``, and the probe builds a GenericMessage topic, +which the runtime refuses to attach to a typed ring. So the step never ran for +any topic that has ever existed, the ~70-entry ``_TYPE_NAME_MAP`` built for it +had no reachable reader, and a bare ``subs=["scan"]`` against a typed ring was +answered with a RuntimeError about byte counts that named neither `subs=`, nor +the type the ring holds, nor the fix. +""" +import uuid + +import pytest + +import horus +from horus import CmdVel, LaserScan, Topic + + +def _unique(name): + return f"{name}_{uuid.uuid4().hex[:8]}" + + +def test_peek_reads_the_ring_type_without_attaching(): + name = _unique("peek_typed") + typed = Topic(CmdVel, endpoint=name) + typed.send(CmdVel(1.0, 0.0)) + + assert horus._peek_topic_type(name) == "CmdVel" + assert horus._peek_topic_type(_unique("never_created")) is None + + +def test_a_bare_subs_string_resolves_to_the_rings_type(): + name = _unique("auto_typed") + # A publisher creates the ring as a typed LaserScan topic. + pub = Topic(LaserScan, endpoint=name) + pub.send(LaserScan()) + + # The subscriber says only the name — no type. Step 2 must now resolve it. + node = horus.Node(name="detector", subs=[name]) + node._setup_topics() + + resolved = node._topics[name] + assert not resolved.is_generic(), ( + "a bare subs= string on an existing typed ring must resolve to that " + "type, not open an untyped MessagePack topic that cannot attach" + ) + + +def test_an_unmappable_ring_type_says_what_to_do(): + """The residual case must name the topic, the type and the fix. + + Falling through to a GenericMessage topic gets the runtime's size-mismatch + RuntimeError, which talks about byte counts and names none of those three. + """ + name = _unique("unmappable") + pub = Topic(CmdVel, endpoint=name) + pub.send(CmdVel(0.0, 0.0)) + + node = horus.Node(name="detector", subs=[name]) + # Pretend this build has no Python class for the ring's type. + saved = horus._TYPE_NAME_MAP.pop("cmdvel") + try: + with pytest.raises(RuntimeError) as excinfo: + node._setup_topics() + finally: + horus._TYPE_NAME_MAP["cmdvel"] = saved + + msg = str(excinfo.value) + assert name in msg, f"must name the topic: {msg}" + assert "CmdVel" in msg, f"must name the type the ring holds: {msg}" + assert "subs=" in msg, f"must name the fix: {msg}" diff --git a/horus_py/tests/test_tf_raises_what_it_documents.py b/horus_py/tests/test_tf_raises_what_it_documents.py new file mode 100644 index 000000000..7b11e2602 --- /dev/null +++ b/horus_py/tests/test_tf_raises_what_it_documents.py @@ -0,0 +1,61 @@ +"""The TransformFrame docstrings must name the exception the code raises. + +`wait_for_transform` and `tf_at_strict` documented `ValueError`. They route +through `to_py_err`, which maps `HorusError::Timeout` to `HorusTimeoutError` +and `HorusError::Transform` to `HorusTransformError` — both built on +`PyException`, so neither is a subclass of `ValueError`. The recovery path the +docstring told you to write (`except ValueError`) never caught the timeout, and +it escaped the node's tick() on exactly the case the timeout parameter exists +to handle. +""" + +import inspect + +import pytest + +import horus + + +def _tf(): + return horus.TransformFrame() + + +def test_wait_for_transform_raises_the_documented_exception(): + tf = _tf() + with pytest.raises(horus.HorusTimeoutError): + tf.wait_for_transform("nowhere_src", "nowhere_dst", 0.01) + + +def test_tf_at_strict_raises_the_documented_exception(): + """Registered frames, a timestamp far outside the buffered range.""" + tf = _tf() + tf.register_frame("world", None) + tf.register_frame("sensor", "world") + at = 4_000_000_000_000 + tf.update_transform("sensor", horus.Transform.from_translation([1.0, 0.0, 0.0]), at) + + # An hour before anything was buffered: strict mode must refuse rather than + # extrapolate, which is the whole difference from tf_at(). + with pytest.raises(horus.HorusTransformError): + tf.tf_at_strict("sensor", "world", at - 3_600_000_000_000) + + +def test_neither_is_a_value_error(): + """Pin the premise: this is why the old docstring was wrong, not a nitpick.""" + assert not issubclass(horus.HorusTimeoutError, ValueError) + assert not issubclass(horus.HorusTransformError, ValueError) + + +@pytest.mark.parametrize( + "method,expected", + [ + ("wait_for_transform", "HorusTimeoutError"), + ("tf_at_strict", "HorusTransformError"), + ], +) +def test_the_docstring_names_that_exception(method, expected): + doc = inspect.getdoc(getattr(horus.TransformFrame, method)) or "" + assert expected in doc, f"{method} docstring does not name {expected}:\n{doc}" + assert "ValueError:" not in doc, ( + f"{method} docstring still promises ValueError, which it does not raise:\n{doc}" + ) diff --git a/horus_sys/src/shm/mod.rs b/horus_sys/src/shm/mod.rs index 54d8904ca..167e9f989 100644 --- a/horus_sys/src/shm/mod.rs +++ b/horus_sys/src/shm/mod.rs @@ -245,27 +245,19 @@ pub fn shm_namespace() -> String { /// - macOS: `/tmp/horus_/` /// - Windows: `%TEMP%\horus_\` pub fn shm_base_dir() -> PathBuf { - let dir_name = format!("horus_{}", shm_namespace()); - - #[cfg(target_os = "linux")] - { - PathBuf::from("/dev/shm").join(&dir_name) - } - - #[cfg(target_os = "macos")] - { - PathBuf::from("/tmp").join(&dir_name) - } - - #[cfg(target_os = "windows")] - { - std::env::temp_dir().join(&dir_name) - } + shm_base_dir_for(&shm_namespace()) +} - #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] - { - PathBuf::from("/tmp").join(&dir_name) - } +/// Base directory for a NAMED namespace, which need not be this process's. +/// +/// The namespace a process uses is fixed at startup from `HORUS_NAMESPACE`, so +/// `shm_base_dir()` can only ever name its own. A caller that knows another +/// namespace by name — a parent that spawned a child under a private one, or +/// `horus clean` sweeping the machine — had no way to name its directory except +/// by restating the platform mapping, which is how two copies of one path drift +/// apart. +pub fn shm_base_dir_for(namespace: &str) -> PathBuf { + shm_parent_dir().join(format!("horus_{}", sanitize_namespace(namespace))) } /// Topics directory for shared memory message passing. @@ -534,6 +526,60 @@ const REGION_NAME_MAX: usize = 512; /// A single embedded `/` remains legal: legacy hierarchical topic names rely on /// it, and `ShmRegion::new` creates the intermediate directory. Only traversal /// and absolute/rooted forms are rejected. +#[cfg(test)] +mod namespace_path_tests { + use super::*; + + /// The finding: `shm_base_dir_for` interpolated its argument raw, and its + /// callers hand the result to `remove_dir_all`. + /// + /// The fix is not a new sanitizer — `sanitize_namespace` was already in + /// this module and already strict enough — it is that this path was the one + /// place that skipped it. + #[test] + fn a_namespace_can_never_escape_the_shm_parent() { + let parent = shm_parent_dir(); + for hostile in [ + "../../../../etc", + "..", + ".", + "a/../../b", + "a/b", + "a\\b", + "/absolute", + ] { + let dir = shm_base_dir_for(hostile); + assert_eq!( + dir.parent(), + Some(parent.as_path()), + "{hostile:?} produced {dir:?}, which is not directly under the SHM parent" + ); + let leaf = dir + .file_name() + .expect("a leaf") + .to_string_lossy() + .to_string(); + assert!( + !leaf.contains("..") && !leaf.contains('/') && !leaf.contains('\\'), + "{hostile:?} left a traversable leaf {leaf:?}" + ); + } + } + + /// The default namespace's directory is a compatibility contract: rewriting + /// it would strand every running deployment's shared memory. + #[test] + fn the_ordinary_namespaces_keep_their_paths() { + for ok in ["default", "test_a1b2", "robot_01"] { + assert_eq!(sanitize_namespace(ok), ok, "{ok:?} must not be rewritten"); + } + assert_eq!( + shm_base_dir_for("default"), + shm_parent_dir().join("horus_default") + ); + } +} + pub fn validate_region_name(name: &str) -> anyhow::Result<()> { anyhow::ensure!(!name.is_empty(), "SHM region name must not be empty"); anyhow::ensure!( diff --git a/install.sh b/install.sh index 545322f23..3533ce2b0 100755 --- a/install.sh +++ b/install.sh @@ -559,7 +559,28 @@ check_rust_version() { [ -n "$found" ] || return 0 # Compare as version numbers, not as strings: 1.100 is newer than 1.9. - if [ "$(printf '%s\n%s\n' "$required" "$found" | sort -V | head -n1)" != "$required" ]; then + # + # Done in awk rather than with `sort -V`, because this gate has to fail + # OPEN. `sort -V` is not POSIX; where it is missing or errors, the command + # substitution is empty, `"" != "$required"` is true, and the installer + # aborts telling the user their toolchain is too old — an accusation it has + # not actually established, about a machine that may well be fine. The two + # guards directly above deliberately return 0 when they cannot determine an + # answer; this one used to be the odd one out. POSIX awk is everywhere + # rustc is, and it compares field by field, so 1.100 > 1.9 as intended. + if ! awk -v have="$found" -v want="$required" ' + BEGIN { + nh = split(have, H, ".") + nw = split(want, W, ".") + n = (nh > nw ? nh : nw) + for (i = 1; i <= n; i++) { + h = (i <= nh ? H[i] + 0 : 0) + w = (i <= nw ? W[i] + 0 : 0) + if (h > w) exit 0 + if (h < w) exit 1 + } + exit 0 + }' /dev/null; then fail "Rust $required or newer is required; found $found." echo " Run: rustup update stable" exit 1 @@ -914,6 +935,68 @@ else ok "Added to PATH" fi +# A prefix install hangs entirely on this variable being in the environment that +# runs `horus`. Both readers take it from there and from nowhere else -- +# version.rs says so outright: "HORUS_PREFIX is the whole interface ... there is +# no second name to read, and no way to discover a prefix install from outside +# its own tree". Without it `horus run`'s cache root falls back to +# ~/.horus/cache and the version gate reads a state root holding none of this +# install's files. +# +# OUTSIDE the PATH check, and deliberately. This first landed inside the else +# arm, which made it dead for the population it exists to repair: a prefix +# install puts ${HORUS_PREFIX}/bin on PATH, so every re-install and upgrade +# takes the "PATH already configured" arm and would never have been given the +# variable. That is the same mistake this file already records 60 lines above +# for the shell-integration block that used to live there. +# +# The rc file and its directory may therefore not exist yet, and SHELL_RC is +# only assigned in the else arm -- so re-detect nothing, just guard on it. +if [ -n "${HORUS_PREFIX:-}" ] && [ -n "$SHELL_RC" ] && [ -z "$TARGET_USER" ]; then + mkdir -p "$(dirname "$SHELL_RC")" 2>/dev/null || true + case "$SHELL_RC" in + */config.fish) + # Value-aware, like the POSIX arm below. Testing only for the + # PRESENCE of `set -gx HORUS_PREFIX` treated a line naming a + # DIFFERENT prefix as "already configured", so a fish user who + # reinstalled to a new location kept the old export and a shell + # that pointed at a tree the installer had just replaced. The + # POSIX arm was fixed for exactly this; fish was left behind. + if grep -qE "^[[:space:]]*set +(-[a-zA-Z]+ +)*HORUS_PREFIX([[:space:]]|\$)" "$SHELL_RC" 2>/dev/null; then + grep -qF "set -gx HORUS_PREFIX \"${HORUS_PREFIX}\"" "$SHELL_RC" 2>/dev/null || { + sed -E -i.horusbak "/^[[:space:]]*set +(-[a-zA-Z]+ +)*HORUS_PREFIX([[:space:]]|\$)/d" "$SHELL_RC" 2>/dev/null || \ + sed -E -i '' "/^[[:space:]]*set +(-[a-zA-Z]+ +)*HORUS_PREFIX([[:space:]]|\$)/d" "$SHELL_RC" 2>/dev/null + rm -f "${SHELL_RC}.horusbak" 2>/dev/null + echo "set -gx HORUS_PREFIX \"${HORUS_PREFIX}\"" >> "$SHELL_RC" + } + else + echo "set -gx HORUS_PREFIX \"${HORUS_PREFIX}\"" >> "$SHELL_RC" + fi + ;; + *) + # Value-aware: a stale line naming a DIFFERENT prefix must be + # replaced, not treated as "already configured". + # + # `grep -E` / `sed -E`, not a BRE with `\?`. That quantifier is a GNU + # extension: on the BSD tools macOS ships it matches nothing, so the + # stale line was never found and the installer appended a second + # HORUS_PREFIX line beneath it. The first one wins when the rc is + # sourced, so a moved prefix kept pointing at the old tree. CI's + # macOS parity job caught exactly that. + if grep -qE "^(export )?HORUS_PREFIX=" "$SHELL_RC" 2>/dev/null; then + grep -qF "HORUS_PREFIX=\"${HORUS_PREFIX}\"" "$SHELL_RC" 2>/dev/null || { + sed -E -i.horusbak "/^(export )?HORUS_PREFIX=/d" "$SHELL_RC" 2>/dev/null || \ + sed -E -i '' "/^(export )?HORUS_PREFIX=/d" "$SHELL_RC" 2>/dev/null + rm -f "${SHELL_RC}.horusbak" 2>/dev/null + echo "export HORUS_PREFIX=\"${HORUS_PREFIX}\"" >> "$SHELL_RC" + } + else + echo "export HORUS_PREFIX=\"${HORUS_PREFIX}\"" >> "$SHELL_RC" + fi + ;; + esac +fi + # --- Shell integration --- # # `horus env --init` writes ~/.horus/env.sh and appends a source line to @@ -1106,12 +1189,13 @@ if [ -n "$TARGET_USER" ]; then fi if [ -n "${HORUS_PREFIX:-}" ]; then - # find_horus_source_dir() (run_rust.rs) searches HORUS_SOURCE, a handful of - # fixed development paths, then the two cache roots — none of which is an - # arbitrary prefix. Say so rather than let the first `horus run` discover it. - warn "Installed under ${HORUS_PREFIX}, which 'horus run' does not search. Each user needs:" + # HORUS_PREFIX is what makes `horus run` search this prefix's cache and what + # points the version gate at this install's state root. Both read it from + # the environment of the running process, so it has to be exported -- + # exporting HORUS_SOURCE instead only papered over the first of the two. + warn "Installed under ${HORUS_PREFIX}. Each user needs:" echo " export PATH=\"${INSTALL_DIR}:\$PATH\"" - echo " export HORUS_SOURCE=\"${HORUS_SRC_DIR}\"" + echo " export HORUS_PREFIX=\"${HORUS_PREFIX}\"" echo "" # Nothing outside the prefix records where the prefix was, so the uninstaller # cannot find it either — uninstall.sh reads HORUS_PREFIX from its own @@ -1119,10 +1203,4 @@ if [ -n "${HORUS_PREFIX:-}" ]; then warn "To remove it later, pass the same prefix:" echo " curl -fsSL https://github.com/softmata/horus/raw/main/uninstall.sh | HORUS_PREFIX=${HORUS_PREFIX} bash -s -- --yes" echo "" - # version.rs reads ~/.horus/installed_version and ~/.horus/install_manifest.toml - # unconditionally, so the copies written under the prefix are state nothing - # reads back. The gate stays quiet rather than mis-firing, but it also cannot - # warn about drift on this install. - warn "The version gate is inactive for a prefix install: version.rs looks in ~/.horus, not ${HORUS_PREFIX}." - echo "" fi diff --git a/uninstall.sh b/uninstall.sh index aab9a96b1..6831aee60 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -964,7 +964,7 @@ fi # In a function, with markers, for the same reason as the completion # removal above: the regression test runs this exact text. clean_shell_profiles() { - local profile + local profile fish_config prefix_path_line prefix_fish_line # Shell profiles: remove horus completion eval lines and shell integration for profile in "$HOME/.bashrc" "$HOME/.zshrc" "$HOME/.profile" "$HOME/.bash_profile"; do # The block install.sh's add_zsh_fpath_block() writes. It is delimited by @@ -1000,8 +1000,98 @@ clean_shell_profiles() { echo -e " ${GREEN}[+]${NC} Cleaned horus shell integration from $(basename $profile)" REMOVED=$((REMOVED + 1)) fi + # The `export HORUS_PREFIX=` line install.sh writes for a --prefix + # install. Leaving it is not untidiness: install.sh:245 reads the + # variable to pick INSTALL_DIR, so a line that survives the uninstall is + # still exported by every new shell and the NEXT install silently lands + # in the prefix this run just deleted. + # + # ERE via -E, not a `\?` BRE: that quantifier is a GNU extension and + # matches nothing on the sed macOS ships, which would leave the line in + # place on exactly the platform this cleanup is hardest to notice on. + if [ -f "$profile" ] && grep -qE "^(export )?HORUS_PREFIX=" "$profile" 2>/dev/null; then + cp "$profile" "${profile}.horus-backup" 2>/dev/null + sed -E -i.bak "/^(export )?HORUS_PREFIX=/d" "$profile" 2>/dev/null || \ + sed -E -i '' "/^(export )?HORUS_PREFIX=/d" "$profile" 2>/dev/null + rm -f "${profile}.bak" 2>/dev/null + echo -e " ${GREEN}[+]${NC} Cleaned HORUS_PREFIX export from $(basename $profile)" + REMOVED=$((REMOVED + 1)) + fi done + # The PATH line install.sh writes for a --prefix install points at a + # directory this run has just deleted. + # + # Matched as an exact string, never as a pattern, and only for a prefix. + # A default install writes `export PATH="$HOME/.local/bin:$PATH"` for a + # directory pip, pipx, cargo and rustup also populate: deleting that line + # because horus happened to write it would take every one of those off the + # user's PATH. `$HORUS_PREFIX/bin` is a directory horus created and horus + # removed, so it is the only one safe to take back out. + if [ -n "${HORUS_PREFIX:-}" ] && [ "$HORUS_PREFIX" != "$HORUS_DIR" ]; then + prefix_path_line="export PATH=\"${HORUS_PREFIX}/bin:\$PATH\"" + prefix_fish_line="fish_add_path ${HORUS_PREFIX}/bin" + for profile in "$HOME/.bashrc" "$HOME/.zshrc" "$HOME/.profile" \ + "$HOME/.bash_profile" "$HOME/.config/fish/config.fish"; do + [ -f "$profile" ] || continue + if grep -qF "$prefix_path_line" "$profile" 2>/dev/null || + grep -qF "$prefix_fish_line" "$profile" 2>/dev/null; then + cp "$profile" "${profile}.horus-backup" 2>/dev/null + # grep exits 1 when it selects no lines, which is exactly + # what happens to an rc file whose only content was that PATH + # line -- the file where the removal matters most. Gating the + # rewrite on `&&` would skip it there, and `set -e` (line 31) + # would abort the whole uninstall on the same status. Hence + # the braces and the `|| true` rather than `... && mv`. + { grep -vF "$prefix_path_line" "$profile" 2>/dev/null \ + | grep -vF "$prefix_fish_line" > "${profile}.horus-new" 2>/dev/null; } || true + # ...but `|| true` swallows exit 2 as well, and exit 2 means + # grep could not READ the file. The redirect creates the target + # either way, so `[ -f ]` alone could not tell "this rc held + # nothing but our line" from "this rc could not be read", and in + # the second case the `mv` replaced a profile it had failed to + # read with an empty file. + # + # `-r` is the whole guard, and it is deliberately the ONLY extra + # condition. An earlier attempt here also required the result to + # be non-empty "unless the profile held nothing else", testing + # that with `[ "$(grep -cvF ... || echo 1)" = "0" ]` — which is + # wrong twice over: `grep -c` PRINTS "0" and EXITS 1 when the + # count is zero, so the `|| echo 1` fired on the legitimate case + # and the substitution became "0\n1", and the whole point of + # this block is the file whose only line is ours. That is the + # exact case the two failure modes in this function's test are + # about, and it made a third. + if [ -f "${profile}.horus-new" ] && [ -r "$profile" ]; then + mv "${profile}.horus-new" "$profile" + fi + rm -f "${profile}.horus-new" 2>/dev/null + echo -e " ${GREEN}[+]${NC} Cleaned the prefix PATH line from $(basename $profile)" + REMOVED=$((REMOVED + 1)) + fi + done + fi + + # fish keeps its own syntax in its own file, and the loop above does not + # walk it. install.sh writes `set -gx HORUS_PREFIX "..."` into config.fish; + # the conf.d/horus.fish removed below is a different file entirely, so for + # fish users the export survived no matter what the POSIX side did. + fish_config="$HOME/.config/fish/config.fish" + # Anchored the same way the sed below is. An unanchored "HORUS_PREFIX" + # also matches a user's HORUS_PREFIX_OTHER, and the sed would then delete + # nothing while this block still printed "Cleaned" and counted a removal -- + # the same false success this script already records for the declined-sudo + # path in the RT cleanup above. + if [ -f "$fish_config" ] && + grep -qE "^[[:space:]]*set +(-[a-zA-Z]+ +)*HORUS_PREFIX([[:space:]]|$)" "$fish_config" 2>/dev/null; then + cp "$fish_config" "${fish_config}.horus-backup" 2>/dev/null + sed -E -i.bak "/^[[:space:]]*set +(-[a-zA-Z]+ +)*HORUS_PREFIX([[:space:]]|$)/d" "$fish_config" 2>/dev/null || \ + sed -E -i '' "/^[[:space:]]*set +(-[a-zA-Z]+ +)*HORUS_PREFIX([[:space:]]|$)/d" "$fish_config" 2>/dev/null + rm -f "${fish_config}.bak" 2>/dev/null + echo -e " ${GREEN}[+]${NC} Cleaned HORUS_PREFIX from config.fish" + REMOVED=$((REMOVED + 1)) + fi + # Remove fish shell integration if [ -f "$HOME/.config/fish/conf.d/horus.fish" ]; then rm -f "$HOME/.config/fish/conf.d/horus.fish"