Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 43 additions & 6 deletions crates/nexum-runtime/src/addons.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
//! installs a facility from the resolved config and returns a handle the
//! launcher keeps alive for the run.

use metrics_exporter_prometheus::BuildError;
use metrics_exporter_prometheus::{BuildError, Matcher, PrometheusBuilder};
use tracing::info;

use crate::engine_config::MetricsSection;
Expand Down Expand Up @@ -71,6 +71,21 @@ pub type AddOns = Vec<Box<dyn RuntimeAddOn>>;
/// recorder alone so `metrics::counter!` call sites stay live but no port opens.
pub struct PrometheusAddOn;

/// Bucket bounds for the dispatch latency histogram, spanning the 5 s
/// alert threshold in `docs/production.md`.
const DISPATCH_LATENCY_BUCKETS: &[f64] = &[
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
];

// Without explicit buckets the exporter renders a histogram as a quantile
// summary, and the `_bucket` series the latency alert reads never exists.
fn prometheus_builder() -> Result<PrometheusBuilder, BuildError> {
PrometheusBuilder::new().set_buckets_for_metric(
Matcher::Full("nexum_runtime_dispatch_latency_seconds".to_owned()),
DISPATCH_LATENCY_BUCKETS,
)
}

impl RuntimeAddOn for PrometheusAddOn {
fn install(&self, ctx: &AddOnsContext<'_>) -> anyhow::Result<AddOnHandle> {
if ctx.metrics.enabled {
Expand All @@ -82,17 +97,16 @@ impl RuntimeAddOn for PrometheusAddOn {
addr: ctx.metrics.bind_addr.clone(),
cause,
})?;
metrics_exporter_prometheus::PrometheusBuilder::new()
.with_http_listener(addr)
.install()
prometheus_builder()
.and_then(|builder| builder.with_http_listener(addr).install())
.map_err(|cause| PrometheusError::Exporter { addr, cause })?;
crate::metrics::describe_all();
info!(addr = %addr, "metrics exporter listening at /metrics");
} else {
// Recorder installed globally so metrics call sites stay live;
// no HTTP port is opened. It accumulates samples in memory, unread.
metrics_exporter_prometheus::PrometheusBuilder::new()
.install_recorder()
prometheus_builder()
.and_then(|builder| builder.install_recorder().map(drop))
.map_err(|cause| PrometheusError::Recorder { cause })?;
crate::metrics::describe_all();
}
Expand All @@ -106,6 +120,29 @@ mod tests {
use crate::engine_config::MetricsSection;
use crate::test_utils::Refusal;

/// The `NexumDispatchLatency` alert reads `_bucket` series by `le`, so
/// the latency metric must render as a Prometheus histogram.
#[test]
fn the_latency_histogram_renders_bucket_series() {
const NAME: &str = "nexum_runtime_dispatch_latency_seconds";
let recorder = prometheus_builder()
.expect("a non-empty bucket list builds")
.build_recorder();
let handle = recorder.handle();
metrics::with_local_recorder(&recorder, || {
metrics::histogram!(NAME, "module" => "m", "trigger_kind" => "block").record(0.5);
});
let rendered = handle.render();
assert!(
rendered.contains(&format!("# TYPE {NAME} histogram")),
"exposition:\n{rendered}",
);
assert!(
rendered.contains(&format!("{NAME}_bucket{{")) && rendered.contains("le=\"5\""),
"exposition:\n{rendered}",
);
}

/// An enabled exporter with an unparseable bind address fails at install.
#[test]
fn prometheus_add_on_rejects_an_invalid_bind_addr() {
Expand Down
10 changes: 5 additions & 5 deletions crates/nexum-runtime/src/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,12 @@ pub const METRICS: &[Metric] = &[
Metric {
name: "nexum_runtime_dispatch_dropped_total",
kind: Kind::Counter,
help: "Events dropped before dispatch, by reason.",
help: "Triggers dropped before dispatch, by reason.",
},
Metric {
name: "nexum_runtime_event_latency_seconds",
name: "nexum_runtime_dispatch_latency_seconds",
kind: Kind::Histogram,
help: "Wall-clock seconds to dispatch one event.",
help: "Wall-clock seconds to dispatch one trigger.",
},
Metric {
name: "nexum_runtime_module_errors_total",
Expand All @@ -70,9 +70,9 @@ pub const METRICS: &[Metric] = &[
help: "Module restarts after a trap.",
},
Metric {
name: "nexum_runtime_stream_reconnects_total",
name: "nexum_runtime_source_reconnects_total",
kind: Kind::Counter,
help: "Stream reconnects by kind and chain; kind \"chain-log\" also carries module.",
help: "Source reconnects by source_kind and chain; source_kind \"chain-log\" also carries module.",
},
];

Expand Down
10 changes: 5 additions & 5 deletions crates/nexum-runtime/src/runtime/event_loop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,8 @@ async fn reconnecting_block_task(
} else {
info!(chain_id, attempt, "block source reopened");
metrics::counter!(
"nexum_runtime_stream_reconnects_total",
"kind" => "block",
"nexum_runtime_source_reconnects_total",
"source_kind" => "block",
"chain_id" => chain_id.to_string(),
)
.increment(1);
Expand Down Expand Up @@ -339,8 +339,8 @@ async fn reconnecting_chain_log_task(
"event source reopened"
);
metrics::counter!(
"nexum_runtime_stream_reconnects_total",
"kind" => "chain-log",
"nexum_runtime_source_reconnects_total",
"source_kind" => "chain-log",
"chain_id" => chain_id.to_string(),
"module" => module.clone(),
)
Expand Down Expand Up @@ -547,7 +547,7 @@ pub async fn run<T: RuntimeTypes, G>(
Some((module, chain, log, cursor_key)) => {
NextTrigger::Event(module, chain, Box::new(log), cursor_key)
}
None => NextTrigger::StreamPanic("event"),
None => NextTrigger::StreamPanic("chain-log"),
},
next = extension_deliveries.next() => match next {
Some(delivery) => NextTrigger::Extension(delivery),
Expand Down
2 changes: 1 addition & 1 deletion crates/nexum-runtime/src/supervisor/dispatch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,7 @@ impl<T: RuntimeTypes> Supervisor<T> {
"dispatch ok"
);
metrics::histogram!(
"nexum_runtime_event_latency_seconds",
"nexum_runtime_dispatch_latency_seconds",
"module" => module.name.to_string(),
"trigger_kind" => trigger_kind,
)
Expand Down
8 changes: 4 additions & 4 deletions docs/production.md
Original file line number Diff line number Diff line change
Expand Up @@ -176,14 +176,14 @@ With `enabled = false` the recorder is still installed, so call sites stay live,
| Metric | Type | Labels | Meaning |
|---|---|---|---|
| `nexum_runtime_boot_refusals_total` | counter | `error_kind` | Boot refusals by error kind. |
| `nexum_runtime_event_latency_seconds` | histogram | `module`, `trigger_kind` | Wall-clock seconds to dispatch one trigger. |
| `nexum_runtime_dispatch_latency_seconds` | histogram | `module`, `trigger_kind` | Wall-clock seconds to dispatch one trigger. |
| `nexum_runtime_dispatch_dropped_total` | counter | `module`, `trigger_kind`, `reason` | Triggers dropped before dispatch. `reason = "rate_limited"` is the per-component dispatch rate limit (`[limits.dispatch]`, default `burst = 256` and `refill_per_sec = 128`). `reason = "shutdown"` is a stop landing mid fan-out: the fan-out follows `[[modules]]` order, so the same trailing modules are skipped at every stop. A block is not replayed; an event is, from its cursor. |
| `nexum_runtime_module_errors_total` | counter | `module`, `error_kind` | Module faults and traps. `error_kind = "trap"` is a wasmtime trap; other values are fault labels. |
| `nexum_runtime_module_restarts_total` | counter | `module` | Module restart attempts. |
| `nexum_runtime_module_poisoned` | gauge | `module` | `1` once a module crosses `[limits.poison]` (default 5 failures in 600 s). Stays `1` until the process restarts. |
| `nexum_runtime_chain_request_total` | counter | `chain_id`, `method`, `outcome` | Every `chain::request`. A method outside the read surface is counted as `method="<denied>"` with `outcome="err"`. The `outcome="err"` rate is the RPC-degraded signal. |
| `nexum_runtime_chain_response_capped_total` | counter | `chain_id`, `method` | Responses rejected for exceeding `[limits.chain] response_body_max_bytes` (default 1 MiB). |
| `nexum_runtime_stream_reconnects_total` | counter | `kind`, `chain_id`, `module` | Stream reconnects. `kind="block"` is per chain; `kind="chain-log"` also carries `module`. |
| `nexum_runtime_source_reconnects_total` | counter | `source_kind`, `chain_id`, `module` | Source reconnects. `source_kind="block"` is per chain; `source_kind="chain-log"` also carries `module`. |

`crates/nexum-runtime/src/metrics.rs` is the single source of the name set, and a test refuses any emitted name the table does not carry.

Expand Down Expand Up @@ -230,7 +230,7 @@ groups:
summary: "Nexum RPC error rate above 5% on chain {{ $labels.chain_id }}"

- alert: NexumReconnectStorm
expr: rate(nexum_runtime_stream_reconnects_total[5m]) > 0.1
expr: rate(nexum_runtime_source_reconnects_total[5m]) > 0.1
for: 5m
labels: { severity: ticket }
annotations:
Expand All @@ -239,7 +239,7 @@ groups:
- alert: NexumDispatchLatency
expr: |
histogram_quantile(0.95,
sum by (module, le) (rate(nexum_runtime_event_latency_seconds_bucket[10m]))) > 5
sum by (module, le) (rate(nexum_runtime_dispatch_latency_seconds_bucket[10m]))) > 5
for: 15m
labels: { severity: ticket }
annotations:
Expand Down
Loading