From 4c48d049b381ab3a10ce541fb6fc734c0f77b653 Mon Sep 17 00:00:00 2001 From: mfw78 Date: Tue, 18 Aug 2026 00:36:24 +0000 Subject: [PATCH] refactor(runtime)!: rename the dispatch latency and source reconnect metrics Rename nexum_runtime_event_latency_seconds to nexum_runtime_dispatch_latency_seconds and nexum_runtime_stream_reconnects_total to nexum_runtime_source_reconnects_total, and rename the reconnect counter label kind to source_kind. The label values stay block and chain-log: the value names the RPC transport, not a trigger. The chain-log reconnect task's tracing field carried event; it now carries chain-log, so the log line and the series agree for one source. The dropped and latency help strings now say triggers, and the reconnect help describes the emitted label set. docs/production.md section 6 and the section 7 alert rules move to the new names in the same change. The exporter now sets buckets for the latency histogram. Without them metrics-exporter-prometheus renders a histogram as a quantile summary, so the section 7 latency rule read a _bucket series that never existed and could never fire, and the section 6 type column was untrue. A test renders the recorder and holds the metric to its bucket series. BREAKING CHANGE: a Prometheus series is its full label set including __name__, so each renamed series ends at its cumulative value and a new series starts at zero across the deploy; rate() and increase() lose the delta at that boundary. There is no dual-emit window. Closes #243 AI Assistance: Claude Fable 5 used for the implementation. --- crates/nexum-runtime/src/addons.rs | 49 ++++++++++++++++--- crates/nexum-runtime/src/metrics.rs | 10 ++-- .../nexum-runtime/src/runtime/event_loop.rs | 10 ++-- .../nexum-runtime/src/supervisor/dispatch.rs | 2 +- docs/production.md | 8 +-- 5 files changed, 58 insertions(+), 21 deletions(-) diff --git a/crates/nexum-runtime/src/addons.rs b/crates/nexum-runtime/src/addons.rs index 51f3de23..92fab9ed 100644 --- a/crates/nexum-runtime/src/addons.rs +++ b/crates/nexum-runtime/src/addons.rs @@ -3,7 +3,7 @@ //! installs a facility from the resolved config and returns a handle the //! launcher keeps alive for the run. -use metrics_exporter_prometheus::BuildError; +use metrics_exporter_prometheus::{BuildError, Matcher, PrometheusBuilder}; use tracing::info; use crate::engine_config::MetricsSection; @@ -71,6 +71,21 @@ pub type AddOns = Vec>; /// recorder alone so `metrics::counter!` call sites stay live but no port opens. pub struct PrometheusAddOn; +/// Bucket bounds for the dispatch latency histogram, spanning the 5 s +/// alert threshold in `docs/production.md`. +const DISPATCH_LATENCY_BUCKETS: &[f64] = &[ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, +]; + +// Without explicit buckets the exporter renders a histogram as a quantile +// summary, and the `_bucket` series the latency alert reads never exists. +fn prometheus_builder() -> Result { + PrometheusBuilder::new().set_buckets_for_metric( + Matcher::Full("nexum_runtime_dispatch_latency_seconds".to_owned()), + DISPATCH_LATENCY_BUCKETS, + ) +} + impl RuntimeAddOn for PrometheusAddOn { fn install(&self, ctx: &AddOnsContext<'_>) -> anyhow::Result { if ctx.metrics.enabled { @@ -82,17 +97,16 @@ impl RuntimeAddOn for PrometheusAddOn { addr: ctx.metrics.bind_addr.clone(), cause, })?; - metrics_exporter_prometheus::PrometheusBuilder::new() - .with_http_listener(addr) - .install() + prometheus_builder() + .and_then(|builder| builder.with_http_listener(addr).install()) .map_err(|cause| PrometheusError::Exporter { addr, cause })?; crate::metrics::describe_all(); info!(addr = %addr, "metrics exporter listening at /metrics"); } else { // Recorder installed globally so metrics call sites stay live; // no HTTP port is opened. It accumulates samples in memory, unread. - metrics_exporter_prometheus::PrometheusBuilder::new() - .install_recorder() + prometheus_builder() + .and_then(|builder| builder.install_recorder().map(drop)) .map_err(|cause| PrometheusError::Recorder { cause })?; crate::metrics::describe_all(); } @@ -106,6 +120,29 @@ mod tests { use crate::engine_config::MetricsSection; use crate::test_utils::Refusal; + /// The `NexumDispatchLatency` alert reads `_bucket` series by `le`, so + /// the latency metric must render as a Prometheus histogram. + #[test] + fn the_latency_histogram_renders_bucket_series() { + const NAME: &str = "nexum_runtime_dispatch_latency_seconds"; + let recorder = prometheus_builder() + .expect("a non-empty bucket list builds") + .build_recorder(); + let handle = recorder.handle(); + metrics::with_local_recorder(&recorder, || { + metrics::histogram!(NAME, "module" => "m", "trigger_kind" => "block").record(0.5); + }); + let rendered = handle.render(); + assert!( + rendered.contains(&format!("# TYPE {NAME} histogram")), + "exposition:\n{rendered}", + ); + assert!( + rendered.contains(&format!("{NAME}_bucket{{")) && rendered.contains("le=\"5\""), + "exposition:\n{rendered}", + ); + } + /// An enabled exporter with an unparseable bind address fails at install. #[test] fn prometheus_add_on_rejects_an_invalid_bind_addr() { diff --git a/crates/nexum-runtime/src/metrics.rs b/crates/nexum-runtime/src/metrics.rs index 53e460fe..507611c6 100644 --- a/crates/nexum-runtime/src/metrics.rs +++ b/crates/nexum-runtime/src/metrics.rs @@ -47,12 +47,12 @@ pub const METRICS: &[Metric] = &[ Metric { name: "nexum_runtime_dispatch_dropped_total", kind: Kind::Counter, - help: "Events dropped before dispatch, by reason.", + help: "Triggers dropped before dispatch, by reason.", }, Metric { - name: "nexum_runtime_event_latency_seconds", + name: "nexum_runtime_dispatch_latency_seconds", kind: Kind::Histogram, - help: "Wall-clock seconds to dispatch one event.", + help: "Wall-clock seconds to dispatch one trigger.", }, Metric { name: "nexum_runtime_module_errors_total", @@ -70,9 +70,9 @@ pub const METRICS: &[Metric] = &[ help: "Module restarts after a trap.", }, Metric { - name: "nexum_runtime_stream_reconnects_total", + name: "nexum_runtime_source_reconnects_total", kind: Kind::Counter, - help: "Stream reconnects by kind and chain; kind \"chain-log\" also carries module.", + help: "Source reconnects by source_kind and chain; source_kind \"chain-log\" also carries module.", }, ]; diff --git a/crates/nexum-runtime/src/runtime/event_loop.rs b/crates/nexum-runtime/src/runtime/event_loop.rs index f2d711b9..6e612af7 100644 --- a/crates/nexum-runtime/src/runtime/event_loop.rs +++ b/crates/nexum-runtime/src/runtime/event_loop.rs @@ -137,8 +137,8 @@ async fn reconnecting_block_task( } else { info!(chain_id, attempt, "block source reopened"); metrics::counter!( - "nexum_runtime_stream_reconnects_total", - "kind" => "block", + "nexum_runtime_source_reconnects_total", + "source_kind" => "block", "chain_id" => chain_id.to_string(), ) .increment(1); @@ -339,8 +339,8 @@ async fn reconnecting_chain_log_task( "event source reopened" ); metrics::counter!( - "nexum_runtime_stream_reconnects_total", - "kind" => "chain-log", + "nexum_runtime_source_reconnects_total", + "source_kind" => "chain-log", "chain_id" => chain_id.to_string(), "module" => module.clone(), ) @@ -547,7 +547,7 @@ pub async fn run( Some((module, chain, log, cursor_key)) => { NextTrigger::Event(module, chain, Box::new(log), cursor_key) } - None => NextTrigger::StreamPanic("event"), + None => NextTrigger::StreamPanic("chain-log"), }, next = extension_deliveries.next() => match next { Some(delivery) => NextTrigger::Extension(delivery), diff --git a/crates/nexum-runtime/src/supervisor/dispatch.rs b/crates/nexum-runtime/src/supervisor/dispatch.rs index f5a1408d..30f4ff1a 100644 --- a/crates/nexum-runtime/src/supervisor/dispatch.rs +++ b/crates/nexum-runtime/src/supervisor/dispatch.rs @@ -280,7 +280,7 @@ impl Supervisor { "dispatch ok" ); metrics::histogram!( - "nexum_runtime_event_latency_seconds", + "nexum_runtime_dispatch_latency_seconds", "module" => module.name.to_string(), "trigger_kind" => trigger_kind, ) diff --git a/docs/production.md b/docs/production.md index 7b051697..02547169 100644 --- a/docs/production.md +++ b/docs/production.md @@ -176,14 +176,14 @@ With `enabled = false` the recorder is still installed, so call sites stay live, | Metric | Type | Labels | Meaning | |---|---|---|---| | `nexum_runtime_boot_refusals_total` | counter | `error_kind` | Boot refusals by error kind. | -| `nexum_runtime_event_latency_seconds` | histogram | `module`, `trigger_kind` | Wall-clock seconds to dispatch one trigger. | +| `nexum_runtime_dispatch_latency_seconds` | histogram | `module`, `trigger_kind` | Wall-clock seconds to dispatch one trigger. | | `nexum_runtime_dispatch_dropped_total` | counter | `module`, `trigger_kind`, `reason` | Triggers dropped before dispatch. `reason = "rate_limited"` is the per-component dispatch rate limit (`[limits.dispatch]`, default `burst = 256` and `refill_per_sec = 128`). `reason = "shutdown"` is a stop landing mid fan-out: the fan-out follows `[[modules]]` order, so the same trailing modules are skipped at every stop. A block is not replayed; an event is, from its cursor. | | `nexum_runtime_module_errors_total` | counter | `module`, `error_kind` | Module faults and traps. `error_kind = "trap"` is a wasmtime trap; other values are fault labels. | | `nexum_runtime_module_restarts_total` | counter | `module` | Module restart attempts. | | `nexum_runtime_module_poisoned` | gauge | `module` | `1` once a module crosses `[limits.poison]` (default 5 failures in 600 s). Stays `1` until the process restarts. | | `nexum_runtime_chain_request_total` | counter | `chain_id`, `method`, `outcome` | Every `chain::request`. A method outside the read surface is counted as `method=""` with `outcome="err"`. The `outcome="err"` rate is the RPC-degraded signal. | | `nexum_runtime_chain_response_capped_total` | counter | `chain_id`, `method` | Responses rejected for exceeding `[limits.chain] response_body_max_bytes` (default 1 MiB). | -| `nexum_runtime_stream_reconnects_total` | counter | `kind`, `chain_id`, `module` | Stream reconnects. `kind="block"` is per chain; `kind="chain-log"` also carries `module`. | +| `nexum_runtime_source_reconnects_total` | counter | `source_kind`, `chain_id`, `module` | Source reconnects. `source_kind="block"` is per chain; `source_kind="chain-log"` also carries `module`. | `crates/nexum-runtime/src/metrics.rs` is the single source of the name set, and a test refuses any emitted name the table does not carry. @@ -230,7 +230,7 @@ groups: summary: "Nexum RPC error rate above 5% on chain {{ $labels.chain_id }}" - alert: NexumReconnectStorm - expr: rate(nexum_runtime_stream_reconnects_total[5m]) > 0.1 + expr: rate(nexum_runtime_source_reconnects_total[5m]) > 0.1 for: 5m labels: { severity: ticket } annotations: @@ -239,7 +239,7 @@ groups: - alert: NexumDispatchLatency expr: | histogram_quantile(0.95, - sum by (module, le) (rate(nexum_runtime_event_latency_seconds_bucket[10m]))) > 5 + sum by (module, le) (rate(nexum_runtime_dispatch_latency_seconds_bucket[10m]))) > 5 for: 15m labels: { severity: ticket } annotations: