Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
65 commits
Select commit Hold shift + click to select a range
b20e9f7
fix(monitor): look for the plugin where horus install puts it
starswallow Sep 4, 2026
6640fda
fix(control): make horus node restart actually restart the node
starswallow Sep 4, 2026
3028b02
fix(safety): count consecutive deadline misses, not budget overruns
starswallow Sep 4, 2026
88688dc
fix(safety): make the max_deadline_misses emergency stop reachable
starswallow Sep 4, 2026
51ade03
fix(run): apply [network] on every path that spawns a run child
starswallow Sep 4, 2026
01d5939
fix(log): HORUS_LOG_FILE actually starts the file drain
starswallow Sep 4, 2026
ff8239f
fix(topic): try_send and send_blocking move the staleness counter
starswallow Sep 5, 2026
8c87bf3
fix(topic): tell the 17th participant why it has no comms
starswallow Sep 5, 2026
dbd4d94
fix(python): a refusal for want of backpressure is not a timeout
starswallow Sep 5, 2026
f557cdd
fix(python): make subs= type auto-detection actually run
starswallow Sep 5, 2026
5770e6e
fix(cli): three flags that did not do what they said
starswallow Sep 5, 2026
8de0220
test(log): fix a race in the drain wiring test that CI caught
starswallow Sep 5, 2026
6e1d7a0
fix(topic): give a participant registration back when its handle drops
starswallow Sep 5, 2026
d0fa1df
fix(topic): stop advertising a migration drain that is not performed
starswallow Sep 5, 2026
c4f981b
fix(rt): a safing callback that panics must not cancel the safety action
starswallow Sep 5, 2026
d5b507c
fix: a watchdog is never tightened, and topic metrics count real traffic
starswallow Sep 5, 2026
65f461a
fix(launch): --status told the truth about nothing, and namespace: li…
starswallow Sep 5, 2026
a92dc95
fix(python): an untyped topic accepts a message object, and the quick…
starswallow Sep 5, 2026
69a80b9
fix(drivers): honour `simulated`, and name an exec driver after its e…
starswallow Sep 5, 2026
c556dac
fix(cpp): a param value must not come back silently clipped
starswallow Sep 5, 2026
5f568ab
fix(install): a prefix install must export the variable it depends on
starswallow Sep 5, 2026
c9b7096
merge: origin/main into fix/docs-runtime-divergence
starswallow Sep 5, 2026
e09c9fa
merge: drop the duplicated monitor lookup, keep main's
starswallow Sep 5, 2026
fa4a59a
merge: origin/main (#192, #193) into fix/docs-runtime-divergence
starswallow Sep 5, 2026
cf6b35b
test(cpp): cover the get_string re-call, which nothing exercised
starswallow Sep 5, 2026
69a6b3b
test(topic): stress the participant release under concurrent churn
starswallow Sep 5, 2026
f97a5e2
fix(topic): count a handle taking a role its sibling already holds
starswallow Sep 5, 2026
84d2dec
test(monitor): pin the opt-in on the project-local plugin lookup
starswallow Sep 5, 2026
8264221
fix(tests): stop leaking a shared-memory namespace per isolated child
starswallow Sep 5, 2026
4fde12f
fix(topic): pool-backed try_send moves the staleness counter too
starswallow Sep 5, 2026
a9ca7d9
fix(control): horus node restart must reach a node on an executor
starswallow Sep 5, 2026
ca021a0
test: three of this branch's own guards did not guard
starswallow Sep 5, 2026
8fc089f
test: anchor the safing-order guard on code, not on a comment
starswallow Sep 5, 2026
316dc0f
fix: three more defects in this branch's own work
starswallow Sep 5, 2026
3162491
fix: an error that asked for the impossible, and a stale doc-comment
starswallow Sep 5, 2026
cf23bb6
fix: three more declared-but-not-delivered defects
starswallow Sep 5, 2026
e6e9681
revert: restore four files the previous commit truncated
starswallow Sep 5, 2026
0609bb1
fix: two hints that could never fire, and a preferred section nobody …
starswallow Sep 5, 2026
b9841bd
fix: --sim did nothing at all on the Python driver path
starswallow Sep 5, 2026
4982855
fix(python): a Python doc example was compiled as Rust
starswallow Sep 5, 2026
ba88c7e
fix(install): the stale-prefix removal used a GNU-only regex
starswallow Sep 5, 2026
a4ee330
fix: a C++ log line reached horus log with no time on it
starswallow Sep 5, 2026
4725035
fix(install): the Rust version gate failed closed when it could not c…
starswallow Sep 5, 2026
d6fdf1c
fix: scheduler introspection invented answers while the scheduler ran
starswallow Sep 5, 2026
d42d532
fix: safing diagnostics were queued with nothing to drain them
starswallow Sep 5, 2026
7867ad2
fix(uninstall): the HORUS_PREFIX export survived the uninstall
starswallow Sep 5, 2026
94e5560
fix(uninstall): a prefix install's PATH line outlived the uninstall
starswallow Sep 5, 2026
b3cc642
fix(uninstall): the fish prefix cleanup reported removals it never made
starswallow Sep 5, 2026
7de6fc5
Merge origin/main into fix/docs-runtime-divergence
starswallow Sep 5, 2026
3b1ffa3
fix: horus node pause did nothing to three of the four node classes
starswallow Sep 5, 2026
386e401
fix: nothing ever called the warning that .no_alloc() is unenforced
starswallow Sep 5, 2026
c2334ca
fix: the blackbox never recorded a single network event
starswallow Sep 5, 2026
9c7b4ac
test: exempt the compute-diag child switch from the env-var doc guard
starswallow Sep 5, 2026
9764e2b
fix(scheduling): three of the four executors dropped their last diagn…
starswallow Sep 5, 2026
8f25e79
fix: replay lost the recorded node order, and two docs contradicted t…
starswallow Sep 5, 2026
1a5af5e
fix: three more places the code did not do what its own file says
starswallow Sep 5, 2026
e09a635
fix(scheduling): a diagnostic being written when a drain ran was lost…
starswallow Sep 6, 2026
aacebb2
Merge commit '1a5af5ef15a74f1de0ae279920257928d117e486' into merge/ma…
starswallow Sep 6, 2026
31e292b
fix: nine findings from the review, one of which could delete a user'…
starswallow Sep 6, 2026
f725ec9
docs(scheduler): the restart path clears two different pauses, not one
starswallow Sep 6, 2026
1e5297b
Merge main (#203, the jitter budget) into fix/docs-runtime-divergence
starswallow Sep 6, 2026
fd22aea
Merge remote-tracking branch 'origin/main' into fix/189-review-followups
starswallow Sep 6, 2026
27a032f
ci: pin the documentation half of this audit alongside the code half
starswallow Sep 6, 2026
6fa2268
fix(uninstall): my own guard broke the one case the guard is about
starswallow Sep 6, 2026
f5499e6
fix(py): declare peek_topic_type in the stubs, and make the error run…
starswallow Sep 6, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion .github/workflows/docs-contract.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,14 @@ env:
# Verifying a different docs commit means bumping this SHA in a PR, so the
# change is reviewed here before it runs here.
#
# Bumped from 4a612b1 to 9a1a0c1 (horus-docs main, the merge of
# horus-docs#17). That PR is the documentation half of the same bidirectional
# audit this branch is the code half of, so the two belong at the same pin:
# against 4a612b1 the contract validates this branch's runtime corrections
# against pages that still describe what the runtime did BEFORE them, which
# is the drift the audit existed to find. Exactly one commit, and 4a612b1 is
# its parent. Verified from this tree: 14 passed, 0 failed.
#
# Bumped from 38368c8 to 4a612b1 (horus-docs main, the merge of
# horus-docs#18), for the same reason 38368c8 replaced af057d9 before it.
# This branch adds six operator-facing variables — HORUS_RECORD_MAX_MB,
Expand All @@ -107,7 +115,7 @@ env:
# it: 14 passed, 0 failed. The bump is 7 commits, not 1 — horus-docs#18 is
# only the newest of them, and that suite result is what says the other six,
# written by other people for other reasons, are compatible with this branch.
DOCS_PINNED_REF: 4a612b108d51308537cdd5c737bbc0181f8354ac
DOCS_PINNED_REF: 9a1a0c107bc2e04b75cbdd08a04462e242285bce

jobs:
# ── Tier 1 ────────────────────────────────────────────────────────────────
Expand Down
4 changes: 2 additions & 2 deletions benchmarks/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ the withdrawn tables dropped, and it is not comparable across kinds:
| `CrossThread-MPMC` | `PodShm` | `send` | Producer-side `send()` cost under contention |
| `CrossProc-1P1C` | `SpscShm` | `one-way` | Pub-to-sub delivery, RDTSC timestamp in payload |
| `CrossProc-2P1C` | `MpscShm` | `one-way` | Pub-to-sub delivery |
| `CrossProc-1PMC` | `SpmcShm` | `one-way` | Pub-to-sub delivery |
| `CrossProc-1PMC` | `PodShm` | `one-way` | Pub-to-sub delivery; one-to-many POD routes to broadcast |
| `CrossProc-PodShm` | `PodShm` | `broadcast` | Latest-value broadcast; readers may skip ahead |
| `RawAtomic` | — | `one-way` | Raw SHM atomic store/load — the hardware floor |

Expand Down Expand Up @@ -104,7 +104,7 @@ sudo cpupower frequency-set -g performance
echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo
```

> Cross-process paced benchmarks (SpscShm, MpscShm, PodShm) are heavily affected by OS scheduling under `powersave` governor. The SpmcShm path (seqlock broadcast) is least affected since readers don't block.
> Cross-process paced benchmarks (SpscShm, MpscShm) are heavily affected by OS scheduling under `powersave` governor. The `PodShm` broadcast path is least affected, since its readers do not block — a one-to-many POD topology resolves to `PodShm`, not `SpmcShm`, which is why no row above names the latter.

## License

Expand Down
5 changes: 3 additions & 2 deletions horus_core/src/communication/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,9 @@ pub use topic::{set_topic_verbose, TOPIC_VERBOSE_OFFSET};
// Ring-buffer inspector for CLI tools (`horus topic echo`)
#[doc(hidden)]
pub use topic::{
read_latest_slot_bytes, read_slots_since, read_topic_header_info, read_topic_messages_total,
read_topic_sequence, shm_map_count, TopicHeaderInfo, TopicReader, TopicSlotRead,
peek_topic_type_name, read_latest_slot_bytes, read_slots_since, read_topic_header_info,
read_topic_messages_total, read_topic_sequence, shm_map_count, TopicHeaderInfo, TopicReader,
TopicSlotRead,
};

// Topic lifecycle hook for horus_net network replication
Expand Down
61 changes: 53 additions & 8 deletions horus_core/src/communication/topic/dispatch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1199,7 +1199,7 @@ pub(super) fn send_fanout_shm<T: Clone + Send + Sync + Serialize + DeserializeOw
// COMM-H1: never let "all 16 slots live" become a SILENT no-comms
// endpoint (worse than the pre-fix loud panic). Warn LOUDLY, but
// rate-limited per topic so a hot send-loop can't flood the log.
if should_report_endpoint_exhaustion(topic.name()) {
if should_report_endpoint_exhaustion(topic.name(), Exhausted::FanoutEndpoints) {
tracing::warn!(
"FanoutShm topic '{}': all {} publisher endpoint slots are \
live — this publisher has NO comms until a slot frees \
Expand Down Expand Up @@ -1321,7 +1321,7 @@ pub(super) fn recv_fanout_shm<T: Clone + Send + Sync + Serialize + DeserializeOw
}
None => {
// COMM-H1: same as the send path — loud, rate-limited, never silent.
if should_report_endpoint_exhaustion(topic.name()) {
if should_report_endpoint_exhaustion(topic.name(), Exhausted::FanoutEndpoints) {
tracing::warn!(
"FanoutShm topic '{}': all {} subscriber endpoint slots are \
live — this subscriber has NO comms until a slot frees \
Expand Down Expand Up @@ -1887,7 +1887,16 @@ pub(super) fn send_uninitialized<
topic: &RingTopic<T>,
msg: T,
) -> Result<(), T> {
if topic.ensure_producer().is_err() {
if let Err(e) = topic.ensure_producer() {
// As `recv_uninitialized`: the caller gets its message back with no
// indication that the cause is permanent for this handle.
if should_report_endpoint_exhaustion(topic.name(), Exhausted::ParticipantTable) {
tracing::warn!(
"topic '{}': this publisher is NOT registered and will deliver nothing — {}",
topic.name(),
e
);
Comment thread
starswallow marked this conversation as resolved.
Resolved
}
return Err(msg);
}
// SAFETY: ensure_producer() → initialize_backend() → set_dispatch_fn_ptrs()
Expand Down Expand Up @@ -2612,7 +2621,24 @@ pub(super) fn recv_uninitialized<
>(
topic: &RingTopic<T>,
) -> Option<T> {
if topic.ensure_consumer().is_err() {
if let Err(e) = topic.ensure_consumer() {
// Registration failed — almost always the 16-slot participant table
// being full. The role stays `Unregistered`, so every later `recv()`
// re-enters this cold path and fails identically: `None` forever, from
// a subscriber that looks merely idle. Discarding the error here made
// the message `register_role` builds — which names the limit and says
// what to do about it — reach nothing at all.
//
// The same 16-slot wall one layer down is reported, deliberately, by
// this same rate limiter: "never let 'all 16 slots live' become a
// SILENT no-comms endpoint".
if should_report_endpoint_exhaustion(topic.name(), Exhausted::ParticipantTable) {
tracing::warn!(
"topic '{}': this subscriber is NOT registered and will receive nothing — {}",
topic.name(),
e
);
}
return None;
}
// SAFETY: ensure_consumer() → initialize_backend() → set_dispatch_fn_ptrs()
Expand All @@ -2632,29 +2658,48 @@ pub(super) fn recv_uninitialized<
/// Keyed by topic and rate-limited per topic instead, so a hot loop still emits
/// once a minute rather than once ever, and a second topic hitting the same
/// wall is never masked by the first.
fn should_report_endpoint_exhaustion(topic: &str) -> bool {
///
/// Keyed by `reason` as well as topic, for the same argument one level down: a
/// topic can hit two different walls — its FanoutShm endpoint bitmap and the
/// 16-slot participant table — and they have different causes and different
/// fixes. Sharing one bucket would let whichever fired first silence the other
/// for a minute, which is the masking this function exists to prevent.
pub(super) fn should_report_endpoint_exhaustion(topic: &str, reason: Exhausted) -> bool {
use std::collections::HashMap;
use std::sync::Mutex;
use std::time::{Duration, Instant};

const QUIET: Duration = Duration::from_secs(60);
static LAST: Mutex<Option<HashMap<String, Instant>>> = Mutex::new(None);
#[allow(clippy::type_complexity)]
static LAST: Mutex<Option<HashMap<(String, Exhausted), Instant>>> = Mutex::new(None);

let mut guard = match LAST.lock() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
let seen = guard.get_or_insert_with(HashMap::new);
let now = Instant::now();
match seen.get(topic) {
let key = (topic.to_string(), reason);
match seen.get(&key) {
Some(t) if now.duration_since(*t) < QUIET => false,
_ => {
seen.insert(topic.to_string(), now);
seen.insert(key, now);
true
}
}
}

/// Which wall a topic ran into. Rate-limited separately; see
/// `should_report_endpoint_exhaustion`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub(super) enum Exhausted {
/// The FanoutShm publisher or subscriber endpoint bitmap is full.
FanoutEndpoints,
/// The topic's 16-slot participant table is full, so this handle holds no
/// registration and will never send or receive anything.
ParticipantTable,
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
Loading
Loading