From 4aadc7245732648a02defdcd1f439392a053905b Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Sat, 14 Nov 2026 20:10:00 -0300 Subject: [PATCH 01/11] feat: with `--metrics`, serve the tip, the step of the peer and the reconnects at `GET /metrics`, in the Prometheus text format (#79) `--metrics` takes a port on 127.0.0.1, or an address and a port. elo listens there before it dials, and logs the address in a `metrics` line. A failed bind is a fault of this host, so elo exits with 1. The HTTP code is our own and has no dependency: one request for each connection, a head of at most 8 KiB, one connection at a time, each in at most 10 s. #70 can use the same code. The metrics are the height and the work of the header tip, the step of the connection (`dial`, `sync`, `synced`, `wait`) and the reconnects. The work is in decimal, with all its digits, so `Work` now divides with a remainder. The values are in `Cell`s: elo runs on one thread, and the node and the server are two futures of one task. --- src/http.rs | 281 ++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 2 + src/logging.rs | 14 ++- src/main.rs | 170 ++++++++++++++++++++++++++-- src/metrics.rs | 260 ++++++++++++++++++++++++++++++++++++++++++ src/pow.rs | 63 ++++++++++- tests/bitcoind.rs | 79 ++++++++++++- 7 files changed, 855 insertions(+), 14 deletions(-) create mode 100644 src/http.rs create mode 100644 src/metrics.rs diff --git a/src/http.rs b/src/http.rs new file mode 100644 index 0000000..8a45e67 --- /dev/null +++ b/src/http.rs @@ -0,0 +1,281 @@ +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +pub(crate) const HEAD_LENGTH_MAX: usize = 8 * 1024; + +pub(crate) const EXCHANGE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +pub(crate) const ACCEPT_ERROR_WAIT: std::time::Duration = std::time::Duration::from_secs(1); + +const HEAD_END: &[u8] = b"\r\n\r\n"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Request<'a> { + pub(crate) method: &'a str, + pub(crate) target: &'a str, +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Response { + Ok { + content_type: &'static str, + body: String, + }, + BadRequest, + NotFound, + MethodNotAllowed { + allow: &'static str, + }, +} + +#[derive(Debug)] +enum HeadError { + Io(std::io::Error), + Closed, + TooLong, +} + +pub(crate) async fn serve( + listener: tokio::net::TcpListener, + answer: impl Fn(Request<'_>) -> Response, +) -> std::convert::Infallible { + loop { + match listener.accept().await { + Ok((mut stream, _)) => { + let exchange = exchange(&mut stream, &answer); + let _ = tokio::time::timeout(EXCHANGE_TIMEOUT, exchange).await; + } + Err(_) => tokio::time::sleep(ACCEPT_ERROR_WAIT).await, + } + } +} + +async fn exchange( + stream: &mut tokio::net::TcpStream, + answer: impl Fn(Request<'_>) -> Response, +) -> Result<(), std::io::Error> { + let response = match read_head(stream).await { + Ok(head) => match request(&head) { + Some(request) => answer(request), + None => Response::BadRequest, + }, + Err(HeadError::TooLong) => Response::BadRequest, + Err(HeadError::Closed) => return Ok(()), + Err(HeadError::Io(error)) => return Err(error), + }; + stream.write_all(&response.bytes()).await?; + stream.shutdown().await +} + +async fn read_head(stream: &mut (impl tokio::io::AsyncRead + Unpin)) -> Result, HeadError> { + let mut buffer = vec![0; HEAD_LENGTH_MAX]; + let mut filled: usize = 0; + loop { + let read = stream + .read(&mut buffer[filled..]) + .await + .map_err(HeadError::Io)?; + if read == 0 { + return Err(HeadError::Closed); + } + filled += read; + assert!(filled <= HEAD_LENGTH_MAX); + let found = buffer[..filled] + .windows(HEAD_END.len()) + .position(|window| window == HEAD_END); + if let Some(position) = found { + buffer.truncate(position + HEAD_END.len()); + return Ok(buffer); + } + if filled == HEAD_LENGTH_MAX { + return Err(HeadError::TooLong); + } + } +} + +fn request(head: &[u8]) -> Option> { + assert!(head.ends_with(HEAD_END)); + let line_end = head.windows(2).position(|window| window == b"\r\n")?; + let line = std::str::from_utf8(&head[..line_end]).ok()?; + let mut parts = line.split(' '); + let (Some(method), Some(target), Some(version), None) = + (parts.next(), parts.next(), parts.next(), parts.next()) + else { + return None; + }; + if method.is_empty() || !method.bytes().all(token_byte) { + return None; + } + if target.is_empty() || !target.bytes().all(|byte| byte.is_ascii_graphic()) { + return None; + } + if !matches!(version, "HTTP/1.1" | "HTTP/1.0") { + return None; + } + Some(Request { method, target }) +} + +fn token_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || b"!#$%&'*+-.^_`|~".contains(&byte) +} + +impl Response { + fn bytes(&self) -> Vec { + let (status, headers, body) = match self { + Response::Ok { content_type, body } => ( + "200 OK", + format!("Content-Type: {content_type}\r\n"), + body.as_str(), + ), + Response::BadRequest => ("400 Bad Request", String::new(), ""), + Response::NotFound => ("404 Not Found", String::new(), ""), + Response::MethodNotAllowed { allow } => { + ("405 Method Not Allowed", format!("Allow: {allow}\r\n"), "") + } + }; + format!( + "HTTP/1.1 {status}\r\n{headers}Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .into_bytes() + } +} + +#[cfg(test)] +mod tests { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + fn block_on(future: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .enable_io() + .enable_time() + .build() + .unwrap() + .block_on(future) + } + + fn answer(request: super::Request<'_>) -> super::Response { + super::Response::Ok { + content_type: "text/plain", + body: format!("{} {}", request.method, request.target), + } + } + + async fn exchange(listener: &tokio::net::TcpListener, sent: &[u8]) -> String { + let address = listener.local_addr().unwrap(); + let mut client = tokio::net::TcpStream::connect(address).await.unwrap(); + client.write_all(sent).await.unwrap(); + client.shutdown().await.unwrap(); + let (mut server, _) = listener.accept().await.unwrap(); + super::exchange(&mut server, answer).await.unwrap(); + drop(server); + let mut received = Vec::new(); + client.read_to_end(&mut received).await.unwrap(); + String::from_utf8(received).unwrap() + } + + #[test] + fn request_line_is_a_method_a_target_and_http_1() { + for (head, expected) in [ + ( + &b"GET /metrics HTTP/1.1\r\nHost: localhost\r\n\r\n"[..], + Some(("GET", "/metrics")), + ), + (b"GET /metrics HTTP/1.0\r\n\r\n", Some(("GET", "/metrics"))), + (b"POST /a?b=c HTTP/1.1\r\n\r\n", Some(("POST", "/a?b=c"))), + (b"M-1 * HTTP/1.1\r\n\r\n", Some(("M-1", "*"))), + (b"GET /metrics HTTP/2\r\n\r\n", None), + (b"GET /metrics http/1.1\r\n\r\n", None), + (b"GET /metrics\r\n\r\n", None), + (b"GET /metrics HTTP/1.1\r\n\r\n", None), + (b"GET /metrics HTTP/1.1 \r\n\r\n", None), + (b" GET /metrics HTTP/1.1\r\n\r\n", None), + (b"G(T /metrics HTTP/1.1\r\n\r\n", None), + (b"GET /m\x7fetrics HTTP/1.1\r\n\r\n", None), + (b"GET /\xc3\xa9 HTTP/1.1\r\n\r\n", None), + (b"GET /\xff HTTP/1.1\r\n\r\n", None), + (b"\r\n\r\n", None), + ] { + let request = + super::request(head).map(|super::Request { method, target }| (method, target)); + assert_eq!(request, expected, "{:?}", String::from_utf8_lossy(head)); + } + } + + #[test] + fn each_response_has_its_status_its_headers_and_a_length() { + for (response, expected) in [ + ( + super::Response::Ok { + content_type: "text/plain", + body: String::from("ab"), + }, + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\n\ + Connection: close\r\n\r\nab", + ), + ( + super::Response::BadRequest, + "HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ), + ( + super::Response::NotFound, + "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ), + ( + super::Response::MethodNotAllowed { allow: "GET" }, + "HTTP/1.1 405 Method Not Allowed\r\nAllow: GET\r\nContent-Length: 0\r\n\ + Connection: close\r\n\r\n", + ), + ] { + assert_eq!(String::from_utf8(response.bytes()).unwrap(), expected); + } + } + + #[test] + fn exchange_answers_a_head_and_refuses_a_bad_or_long_one() { + let fits = format!( + "GET / HTTP/1.1\r\nA: {}\r\n\r\n", + "a".repeat(super::HEAD_LENGTH_MAX - 23) + ); + assert_eq!(fits.len(), 8 * 1024); + let long = format!( + "GET / HTTP/1.1\r\nA: {}\r\n\r\n", + "a".repeat(super::HEAD_LENGTH_MAX - 22) + ); + let ok = |body: &str| { + format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\n\ + Connection: close\r\n\r\n{body}", + body.len() + ) + }; + let bad = "HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + block_on(async { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + for (sent, expected) in [ + (&b"GET /metrics HTTP/1.1\r\n\r\n"[..], ok("GET /metrics")), + (b"GET /metrics HTTP/1.1\r\n\r\nbody", ok("GET /metrics")), + (fits.as_bytes(), ok("GET /")), + (long.as_bytes(), String::from(bad)), + (b"GET /metrics\r\n\r\n", String::from(bad)), + (b"GET /metrics HTTP/1.1\r\n", String::new()), + (b"", String::new()), + ] { + let received = exchange(&listener, sent).await; + assert_eq!(received, expected, "{:?}", String::from_utf8_lossy(sent)); + } + }); + } + + #[test] + fn head_end_over_two_reads_is_found() { + let head = b"GET / HTTP/1.1\r\n\r\n"; + block_on(async { + for split in 1..super::HEAD_END.len() { + let (first, second) = head.split_at(head.len() - split); + let mut reader = AsyncReadExt::chain(first, second); + let read = super::read_head(&mut reader).await.unwrap(); + assert_eq!(read, head, "{split}"); + } + }); + } +} diff --git a/src/lib.rs b/src/lib.rs index 976e6c8..5aeb1c1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,9 @@ mod block; pub mod chain; pub mod chainparams; mod headerssync; +mod http; pub mod logging; +pub mod metrics; pub mod net; mod net_processing; mod pow; diff --git a/src/logging.rs b/src/logging.rs index 43492fb..55edcbf 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -29,6 +29,9 @@ pub enum Event<'a> { peer: std::net::SocketAddr, tip: &'a crate::chain::Tip, }, + Metrics { + address: std::net::SocketAddr, + }, End { destination: &'a crate::net::Destination, reason: &'a dyn std::fmt::Display, @@ -55,6 +58,7 @@ impl Event<'_> { match self { Event::Version { .. } => "version", Event::Tip { .. } => "tip", + Event::Metrics { .. } => "metrics", Event::End { .. } => "end", Event::Exit { .. } => "exit", Event::Stop { .. } => "stop", @@ -64,7 +68,10 @@ impl Event<'_> { fn level(&self) -> Level { match self { - Event::Version { .. } | Event::Tip { .. } | Event::Stop { .. } => Level::Info, + Event::Version { .. } + | Event::Tip { .. } + | Event::Metrics { .. } + | Event::Stop { .. } => Level::Info, Event::End { .. } => Level::Warn, Event::Exit { .. } | Event::Panic { .. } => Level::Error, } @@ -89,6 +96,7 @@ impl Event<'_> { line.field("hash", tip.hash_display()); line.field("work", tip.work()); } + Event::Metrics { address } => line.field("address", address), Event::End { destination, reason, @@ -264,6 +272,10 @@ mod tests { hash=0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206 \ work=0000000000000000000000000000000000000000000000000000000000000002", ), + ( + super::Event::Metrics { address: PEER }, + "level=info event=metrics address=127.0.0.1:18444", + ), ( super::Event::End { destination: &crate::net::Destination::Address( diff --git a/src/main.rs b/src/main.rs index 1eb53e7..45b84c7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ -const USAGE: &str = "usage: elo [[:port]] +const USAGE: &str = + "usage: elo [--metrics [:]] [[:port]] regtest has no seeds, so it needs a peer"; const EXIT_USAGE: u8 = 2; @@ -14,7 +15,7 @@ fn main() -> std::process::ExitCode { })); let args: Vec = std::env::args_os().skip(1).collect(); - let Some((network, source)) = parse(&args) else { + let Some(options) = parse(&args) else { eprintln!("{USAGE}"); return std::process::ExitCode::from(EXIT_USAGE); }; @@ -27,23 +28,83 @@ fn main() -> std::process::ExitCode { Ok(runtime) => runtime, Err(error) => return exit(&format_args!("runtime: {error}")), }; - let code = runtime.block_on(run(network, source)); + let code = runtime.block_on(run(options)); runtime.shutdown_background(); code } -async fn run( +#[derive(Debug, PartialEq, Eq)] +struct Options { network: elo::chainparams::Network, source: elo::net::Source, -) -> std::process::ExitCode { + metrics: Option, +} + +async fn run(options: Options) -> std::process::ExitCode { let mut signals = match Signals::new() { Ok(signals) => signals, Err(error) => return exit(&format_args!("signals: {error}")), }; - let mut chain = elo::chain::Chain::new(network); + let listener = match options.metrics { + None => None, + Some(address) => match listen(address).await { + Ok((listener, address)) => { + if let Err(error) = log(&elo::logging::Event::Metrics { address }) { + return exit_stdout(error); + } + Some(listener) + } + Err(error) => return exit(&format_args!("metrics: {error}")), + }, + }; + let mut chain = elo::chain::Chain::new(options.network); + let metrics = elo::metrics::Metrics::new(&chain); + let serve = async { + match listener { + Some(listener) => metrics.serve(listener).await, + None => std::future::pending().await, + } + }; + let node = node(&options.source, &mut chain, &metrics, &mut signals); + beside(serve, node).await +} + +async fn listen( + address: std::net::SocketAddr, +) -> Result<(tokio::net::TcpListener, std::net::SocketAddr), std::io::Error> { + let listener = tokio::net::TcpListener::bind(address).await?; + let address = listener.local_addr()?; + Ok((listener, address)) +} + +async fn beside( + background: impl Future, + work: impl Future, +) -> T { + let mut background = std::pin::pin!(background); + let mut work = std::pin::pin!(work); + std::future::poll_fn(|context| { + if let std::task::Poll::Ready(output) = work.as_mut().poll(context) { + return std::task::Poll::Ready(output); + } + match background.as_mut().poll(context) { + std::task::Poll::Ready(never) => match never {}, + std::task::Poll::Pending => std::task::Poll::Pending, + } + }) + .await +} + +async fn node( + source: &elo::net::Source, + chain: &mut elo::chain::Chain, + metrics: &elo::metrics::Metrics, + signals: &mut Signals, +) -> std::process::ExitCode { loop { let destination = source.destination(); - let end = match signals.unless(dial(&destination, &mut chain)).await { + metrics.peer(elo::metrics::PeerState::Dial); + let end = match signals.unless(dial(&destination, chain, metrics)).await { Ok(Ok(end)) => end, Ok(Err(error)) => return exit_stdout(error), Err(signal) => return stop(signal), @@ -51,6 +112,8 @@ async fn run( if !end.retries() { return exit(&end); } + metrics.reconnect(); + metrics.peer(elo::metrics::PeerState::Wait); let logged = log(&elo::logging::Event::End { destination: &destination, reason: &end, @@ -191,11 +254,13 @@ impl std::fmt::Display for End { async fn dial( destination: &elo::net::Destination, chain: &mut elo::chain::Chain, + metrics: &elo::metrics::Metrics, ) -> Result { let (connection, version) = match elo::net::handshake(destination, chain).await { Ok(handshake) => handshake, Err(error) => return Ok(End::Handshake(error)), }; + metrics.peer(elo::metrics::PeerState::Sync); let address = connection.address(); log(&elo::logging::Event::Version { peer: address, @@ -206,7 +271,9 @@ async fn dial( Ok(sync) => sync, Err(error) => return Ok(End::Run(error)), }; + metrics.peer(elo::metrics::PeerState::Synced); loop { + metrics.tip(&tip); log(&elo::logging::Event::Tip { peer: address, tip: &tip, @@ -218,7 +285,13 @@ async fn dial( } } -fn parse(args: &[std::ffi::OsString]) -> Option<(elo::chainparams::Network, elo::net::Source)> { +fn parse(args: &[std::ffi::OsString]) -> Option { + let (metrics, args) = match args { + [flag, address, args @ ..] if flag == "--metrics" => { + (Some(metrics_address(address.to_str()?)?), args) + } + args => (None, args), + }; let (network, peer) = match args { [network] => (network, None), [network, peer] => (network, Some(peer)), @@ -235,7 +308,21 @@ fn parse(args: &[std::ffi::OsString]) -> Option<(elo::chainparams::Network, elo: Some(peer) => elo::net::Source::Destination(destination(peer.to_str()?, network)?), None => elo::net::Source::Seeds(elo::net::Seeds::new(network)?), }; - Some((network, source)) + Some(Options { + network, + source, + metrics, + }) +} + +fn metrics_address(text: &str) -> Option { + if let Ok(port) = text.parse() { + return Some(std::net::SocketAddr::from(( + std::net::Ipv4Addr::LOCALHOST, + port, + ))); + } + text.parse().ok() } fn destination(peer: &str, network: elo::chainparams::Network) -> Option { @@ -268,6 +355,12 @@ fn address(peer: &str, default_port: std::num::NonZeroU16) -> Option Option<(elo::chainparams::Network, elo::net::Source)> { + let options = parse_options(args)?; + assert_eq!(options.metrics, None, "{args:?}"); + Some((options.network, options.source)) + } + + fn parse_options(args: &[&str]) -> Option { let args: Vec = args.iter().map(std::ffi::OsString::from).collect(); super::parse(&args) } @@ -415,6 +508,65 @@ mod tests { } } + #[test] + fn metrics_flag_before_the_network_takes_a_port_on_localhost_or_an_address() { + let localhost = |port| std::net::SocketAddr::from(([127, 0, 0, 1], port)); + for (address, expected) in [ + ("9332", Some(localhost(9332))), + ("0", Some(localhost(0))), + ("127.0.0.1:9332", Some(localhost(9332))), + ( + "0.0.0.0:9332", + Some(std::net::SocketAddr::from(([0, 0, 0, 0], 9332))), + ), + ( + "[::1]:9332", + Some(std::net::SocketAddr::from(( + std::net::Ipv6Addr::LOCALHOST, + 9332, + ))), + ), + ("65536", None), + ("-1", None), + ("127.0.0.1", None), + ("localhost:9332", None), + (":9332", None), + ("", None), + ] { + let options = parse_options(&["--metrics", address, "regtest", "127.0.0.1"]); + assert_eq!( + options.map(|options| options.metrics), + expected.map(Some), + "{address}" + ); + } + let options = parse_options(&["--metrics", "9332", "regtest", "127.0.0.1"]).unwrap(); + assert_eq!(options.network, elo::chainparams::Network::Regtest); + assert_eq!( + options.source, + elo::net::Source::Destination(elo::net::Destination::Address( + elo::net::PeerAddress::new(localhost(18444)).unwrap() + )) + ); + for args in [ + &["--metrics"][..], + &["--metrics", "9332"], + &["regtest", "127.0.0.1", "--metrics", "9332"], + &["regtest", "--metrics", "9332"], + &["--metric", "9332", "regtest", "127.0.0.1"], + &[ + "--metrics", + "9332", + "--metrics", + "9333", + "regtest", + "127.0.0.1", + ], + ] { + assert_eq!(parse_options(args), None, "{args:?}"); + } + } + #[test] fn no_peer_is_the_seeds_of_the_network() { for (name, network) in [ diff --git a/src/metrics.rs b/src/metrics.rs new file mode 100644 index 0000000..f126018 --- /dev/null +++ b/src/metrics.rs @@ -0,0 +1,260 @@ +use std::fmt::Write; + +const PATH: &str = "/metrics"; + +const CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8"; + +const BODY_LENGTH_MAX: usize = 1024; + +#[derive(Debug)] +pub struct Metrics { + tip: std::cell::Cell, + peer: std::cell::Cell, + reconnects: std::cell::Cell, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PeerState { + Dial, + Sync, + Synced, + Wait, +} + +impl PeerState { + const ALL: [PeerState; 4] = [ + PeerState::Dial, + PeerState::Sync, + PeerState::Synced, + PeerState::Wait, + ]; + + fn name(self) -> &'static str { + match self { + PeerState::Dial => "dial", + PeerState::Sync => "sync", + PeerState::Synced => "synced", + PeerState::Wait => "wait", + } + } +} + +impl Metrics { + pub fn new(chain: &crate::chain::Chain) -> Metrics { + Metrics { + tip: std::cell::Cell::new(chain.tip()), + peer: std::cell::Cell::new(PeerState::Dial), + reconnects: std::cell::Cell::new(0), + } + } + + pub fn tip(&self, tip: &crate::chain::Tip) { + self.tip.set(*tip); + } + + pub fn peer(&self, state: PeerState) { + self.peer.set(state); + } + + pub fn reconnect(&self) { + self.reconnects.set(self.reconnects.get().saturating_add(1)); + } + + pub async fn serve(&self, listener: tokio::net::TcpListener) -> std::convert::Infallible { + crate::http::serve(listener, |request| self.answer(request)).await + } + + fn answer(&self, request: crate::http::Request<'_>) -> crate::http::Response { + if request.target != PATH { + return crate::http::Response::NotFound; + } + if request.method != "GET" { + return crate::http::Response::MethodNotAllowed { allow: "GET" }; + } + crate::http::Response::Ok { + content_type: CONTENT_TYPE, + body: self.text(), + } + } + + fn text(&self) -> String { + let tip = self.tip.get(); + let mut text = Text::default(); + text.family( + "elo_header_tip_height", + "gauge", + "The height of the tip of the header chain.", + ); + text.sample("elo_header_tip_height", "", tip.height()); + text.family( + "elo_header_tip_work", + "gauge", + "The work of the header chain up to its tip: the expected number of hashes.", + ); + text.sample("elo_header_tip_work", "", tip.work().decimal()); + text.family( + "elo_peer_state", + "gauge", + "The step of the connection to the peer: 1 for the step it is in, 0 for each other.", + ); + let peer = self.peer.get(); + for state in PeerState::ALL { + text.sample( + "elo_peer_state", + format_args!("state=\"{}\"", state.name()), + u8::from(state == peer), + ); + } + text.family( + "elo_reconnects_total", + "counter", + "The connections that ended, and that elo dials again after a wait.", + ); + text.sample("elo_reconnects_total", "", self.reconnects.get()); + text.end() + } +} + +#[derive(Default)] +struct Text { + text: String, +} + +impl Text { + fn family(&mut self, name: &'static str, kind: &'static str, help: &'static str) { + assert!(name.starts_with("elo_"), "{name}"); + assert!( + name.bytes() + .all(|byte| byte.is_ascii_lowercase() || byte == b'_'), + "{name}" + ); + assert!(!help.contains(['\\', '\n']), "{name}"); + let written = write!(self.text, "# HELP {name} {help}\n# TYPE {name} {kind}\n"); + assert!(written.is_ok(), "{name}"); + } + + fn sample( + &mut self, + name: &'static str, + labels: impl std::fmt::Display, + value: impl std::fmt::Display, + ) { + let mut labels_text = labels.to_string(); + if !labels_text.is_empty() { + labels_text = format!("{{{labels_text}}}"); + } + let written = writeln!(self.text, "{name}{labels_text} {value}"); + assert!(written.is_ok(), "{name}"); + } + + fn end(self) -> String { + assert!(self.text.ends_with('\n')); + assert!(self.text.len() <= BODY_LENGTH_MAX, "{}", self.text.len()); + self.text + } +} + +#[cfg(test)] +mod tests { + const EXPECTED_START: &str = "\ +# HELP elo_header_tip_height The height of the tip of the header chain. +# TYPE elo_header_tip_height gauge +elo_header_tip_height 0 +# HELP elo_header_tip_work The work of the header chain up to its tip: the expected number of hashes. +# TYPE elo_header_tip_work gauge +elo_header_tip_work 2 +# HELP elo_peer_state The step of the connection to the peer: 1 for the step it is in, 0 for each other. +# TYPE elo_peer_state gauge +"; + + const EXPECTED_END: &str = "\ +# HELP elo_reconnects_total The connections that ended, and that elo dials again after a wait. +# TYPE elo_reconnects_total counter +"; + + fn metrics() -> super::Metrics { + super::Metrics::new(&crate::chain::Chain::new( + crate::chainparams::Network::Regtest, + )) + } + + fn states(dial: u8, sync: u8, synced: u8, wait: u8) -> String { + format!( + "elo_peer_state{{state=\"dial\"}} {dial}\nelo_peer_state{{state=\"sync\"}} {sync}\n\ + elo_peer_state{{state=\"synced\"}} {synced}\nelo_peer_state{{state=\"wait\"}} {wait}\n" + ) + } + + #[test] + fn text_has_the_tip_of_the_chain_the_dial_and_no_reconnect_at_start() { + let expected = format!( + "{EXPECTED_START}{}{EXPECTED_END}elo_reconnects_total 0\n", + states(1, 0, 0, 0) + ); + assert_eq!(metrics().text(), expected); + } + + #[test] + fn text_has_the_last_tip_the_state_and_each_reconnect() { + let metrics = metrics(); + let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let headers = crate::fixtures::headers_on_genesis(3); + let extended = chain.extend( + crate::fixtures::headers_batch(&headers), + crate::fixtures::CLOCK, + ); + assert_eq!(extended, Ok(())); + metrics.tip(&chain.tip()); + for (state, expected) in [ + (super::PeerState::Sync, states(0, 1, 0, 0)), + (super::PeerState::Synced, states(0, 0, 1, 0)), + (super::PeerState::Wait, states(0, 0, 0, 1)), + (super::PeerState::Dial, states(1, 0, 0, 0)), + ] { + metrics.peer(state); + metrics.reconnect(); + let text = metrics.text(); + assert!(text.contains("elo_header_tip_height 3\n"), "{text}"); + assert!(text.contains("elo_header_tip_work 8\n"), "{text}"); + assert!(text.contains(&expected), "{state:?}: {text}"); + } + assert!(metrics.text().ends_with("elo_reconnects_total 4\n")); + } + + #[test] + fn answer_is_the_text_for_a_get_of_the_path_only() { + let metrics = metrics(); + let request = |method, target| crate::http::Request { method, target }; + for (request, expected) in [ + ( + request("GET", "/metrics"), + crate::http::Response::Ok { + content_type: "text/plain; version=0.0.4; charset=utf-8", + body: metrics.text(), + }, + ), + ( + request("HEAD", "/metrics"), + crate::http::Response::MethodNotAllowed { allow: "GET" }, + ), + (request("GET", "/"), crate::http::Response::NotFound), + (request("GET", "/metrics/"), crate::http::Response::NotFound), + ( + request("GET", "/metrics?a=b"), + crate::http::Response::NotFound, + ), + (request("POST", "/"), crate::http::Response::NotFound), + ] { + assert_eq!(metrics.answer(request), expected, "{request:?}"); + } + } + + #[test] + fn longest_text_is_below_the_maximum() { + let text = metrics().text(); + let one_digit_values_length = 3; + let longest_values_length = usize::MAX.to_string().len() + 78 + u64::MAX.to_string().len(); + let longest = text.len() - one_digit_values_length + longest_values_length; + assert!(longest <= super::BODY_LENGTH_MAX, "{longest}"); + } +} diff --git a/src/pow.rs b/src/pow.rs index c77c9d6..b39b31f 100644 --- a/src/pow.rs +++ b/src/pow.rs @@ -54,6 +54,15 @@ impl Target { } } +const DECIMAL_CHUNK_BASE: u64 = 10_u64.pow(19); + +const DECIMAL_CHUNK_DIVISOR: Work = Work { + high: 0, + low: DECIMAL_CHUNK_BASE as u128, +}; + +const DECIMAL_CHUNKS_COUNT: usize = 5; + #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub(crate) struct Work { high: u128, @@ -107,9 +116,13 @@ impl Work { } fn divided_by(self, divisor: Work) -> Work { + self.divided_with_remainder(divisor).0 + } + + fn divided_with_remainder(self, divisor: Work) -> (Work, Work) { assert_ne!(divisor, Work::ZERO); let Some(shift) = divisor.leading_zeros().checked_sub(self.leading_zeros()) else { - return Work::ZERO; + return (Work::ZERO, self); }; let mut step_divisor = divisor.shifted_left(shift); let mut remainder = self; @@ -124,7 +137,30 @@ impl Work { } assert!(remainder < divisor); assert!(quotient <= self); - quotient + (quotient, remainder) + } + + pub(crate) fn decimal(self) -> impl std::fmt::Display { + std::fmt::from_fn(move |f| { + let mut chunks = [0_u64; DECIMAL_CHUNKS_COUNT]; + let mut rest = self; + for chunk in chunks.iter_mut().rev() { + let (quotient, remainder) = rest.divided_with_remainder(DECIMAL_CHUNK_DIVISOR); + assert_eq!(remainder.high, 0); + *chunk = u64::try_from(remainder.low).unwrap_or(u64::MAX); + assert!(*chunk < DECIMAL_CHUNK_BASE); + rest = quotient; + } + assert_eq!(rest, Work::ZERO); + let first_index = chunks + .iter() + .position(|chunk| *chunk != 0) + .unwrap_or(DECIMAL_CHUNKS_COUNT - 1); + write!(f, "{}", chunks[first_index])?; + chunks[first_index + 1..] + .iter() + .try_for_each(|chunk| write!(f, "{chunk:019}")) + }) } fn shifted_left(self, bits: u32) -> Work { @@ -969,6 +1005,29 @@ mod tests { } } + #[test] + fn decimal_of_work_is_its_number_in_base_ten() { + let work = |high, low| super::Work { high, low }; + for (number, expected) in [ + (super::Work::ZERO, "0"), + (work(0, 2), "2"), + (work(0, 9_999_999_999_999_999_999), "9999999999999999999"), + (work(0, 10_000_000_000_000_000_000), "10000000000000000000"), + (work(0, 10_000_000_000_000_000_001), "10000000000000000001"), + ( + work(0, u128::MAX), + "340282366920938463463374607431768211455", + ), + (work(1, 0), "340282366920938463463374607431768211456"), + ( + super::Work::MAX, + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + ), + ] { + assert_eq!(number.decimal().to_string(), expected, "{number}"); + } + } + #[test] fn work_adds_with_a_carry_and_stops_at_the_highest_number() { let work = |high, low| super::Work { high, low }; diff --git a/tests/bitcoind.rs b/tests/bitcoind.rs index de313a0..88be9d2 100644 --- a/tests/bitcoind.rs +++ b/tests/bitcoind.rs @@ -590,6 +590,78 @@ fn announcement_before_the_answer_to_our_getheaders_keeps_the_connection() { ); } +#[test] +fn metrics_of_elo_have_its_tip_its_step_and_each_reconnect() { + let bitcoind = Bitcoind::start(); + bitcoind.cli(&["generatetodescriptor", "2001", "raw(51)"]); + let best = bitcoind.cli(&["getbestblockhash"]); + let chain_work = bitcoind.cli(&["getblockheader", best.trim_end()]); + let chain_work = u128::from_str_radix(string_field(&chain_work, "chainwork"), 16).unwrap(); + let peer = bitcoind.peer.to_string(); + let elo = Elo::start_with(&["--metrics", "127.0.0.1:0", "regtest", &peer]); + let listen = elo.event("metrics", SYNC_TIMEOUT); + let address = listen + .strip_prefix("level=info event=metrics address=") + .unwrap(); + let address: std::net::SocketAddr = address.parse().unwrap(); + assert!(address.ip().is_loopback(), "{address}"); + assert_ne!(address.port(), 0); + elo.event("tip", SYNC_TIMEOUT); + + let metrics = scrape(address); + for line in [ + String::from("elo_header_tip_height 2001\n"), + format!("elo_header_tip_work {chain_work}\n"), + String::from("elo_peer_state{state=\"synced\"} 1\n"), + ] { + assert!(metrics.contains(&line), "{line}: {metrics}"); + } + assert!(metrics.ends_with("elo_reconnects_total 0\n"), "{metrics}"); + + bitcoind.cli(&["-named", "disconnectnode", "nodeid=0"]); + elo.event("end", END_TIMEOUT); + let metrics = scrape(address); + assert!( + metrics.contains("elo_peer_state{state=\"wait\"} 1\n"), + "{metrics}" + ); + assert!(metrics.ends_with("elo_reconnects_total 1\n"), "{metrics}"); +} + +#[test] +fn metrics_address_in_use_exits_with_1() { + let taken = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = taken.local_addr().unwrap().to_string(); + let mut elo = Elo::start_with(&["--metrics", &address, "regtest", "127.0.0.1"]); + let exit = elo.event("exit", END_TIMEOUT); + assert!( + exit.starts_with("level=error event=exit reason=\"metrics: "), + "{exit}" + ); + assert_eq!(elo.exit_code(), Some(1)); +} + +fn scrape(address: std::net::SocketAddr) -> String { + let mut stream = std::net::TcpStream::connect(address).unwrap(); + std::io::Write::write_all( + &mut stream, + b"GET /metrics HTTP/1.1\r\nHost: localhost\r\n\r\n", + ) + .unwrap(); + let mut response = String::new(); + std::io::Read::read_to_string(&mut stream, &mut response).unwrap(); + let (head, body) = response.split_once("\r\n\r\n").unwrap(); + assert_eq!( + head, + format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4; charset=utf-8\r\n\ + Content-Length: {}\r\nConnection: close", + body.len() + ) + ); + body.to_owned() +} + fn settle(bitcoind: &Bitcoind, elo: &Elo) { let start = std::time::Instant::now(); loop { @@ -672,9 +744,12 @@ impl Elo { } fn start_on(peer: &str) -> Self { + Self::start_with(&["regtest", peer]) + } + + fn start_with(args: &[&str]) -> Self { let mut child = std::process::Command::new(env!("CARGO_BIN_EXE_elo")) - .arg("regtest") - .arg(peer) + .args(args) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) .spawn() From 26bea6b4718e6142287fb26d4901062324b0f282 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Sat, 14 Nov 2026 20:10:00 -0300 Subject: [PATCH 02/11] docs: say how `--metrics` serves the metrics, and the `metrics` line (#79) --- README.md | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 62d6820..b9bf394 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ chain stays in memory, so the next connection syncs from its tip. A chain below the minimum work starts again from genesis. ```sh -elo [[:port]] # IPv6 as [::1]:8333 +elo [--metrics [:]] [[:port]] # IPv6 as [::1]:8333 ``` The peer is an IP address or a name. Before each dial, elo looks up the name @@ -82,8 +82,9 @@ peer. SIGTERM and SIGINT (Ctrl-C) stop elo at any step, also in the wait before a dial. It logs the signal and exits with 0. elo exits with 1 only on a fault of -this host: its async runtime or its signal handlers do not start, a write to -stdout fails, or a read of `/dev/urandom` fails. A bug panics: elo logs the +this host: its async runtime or its signal handlers do not start, it cannot +listen on the address of `--metrics`, a write to stdout fails, or a read of +`/dev/urandom` fails. A bug panics: elo logs the panic and exits with 101. Bad usage exits with 2. ## Log @@ -102,6 +103,7 @@ level=info event=stop signal=SIGTERM | --- | --- | --- | | `version` | `info` | `peer` `version` `services` `user_agent` `start_height` `relay` | | `tip` | `info` | `peer` `height` `hash` `work` | +| `metrics` | `info` | `address`: elo serves its metrics there | | `end` | `warn` | `destination` `reason` `wait_ms`: elo dials again after `wait_ms`. `destination` is what the dial looked up or dialed: the peer of the command line, also a name, or the seed, such as `seed.bitcoin.sipa.be.:8333`. `peer` is always the address of a connection | | `exit` | `error` | `reason`: a fault of this host, and elo exits with 1 | | `stop` | `info` | `signal`: elo exits with 0 | @@ -113,6 +115,25 @@ a `"` or a control character is in quotes, with escapes. A value is at most 260 at most 16 KiB, whatever the peer sends. When a write to stdout fails, elo writes the `exit` line to stderr only, because the write that failed can leave part of its line on stdout. +## Metrics + +With `--metrics`, elo serves its metrics at `GET /metrics`, in the Prometheus text format. A +scraper, such as Prometheus or Grafana Alloy, pulls them, and elo pushes nothing. The flag takes a +port on 127.0.0.1, or an address and a port, such as `0.0.0.0:9332` for a scraper in another +container. Port 0 takes a free port. elo logs the address in a `metrics` line before it dials. + +| Metric | Type | Value | +| --- | --- | --- | +| `elo_header_tip_height` | gauge | The height of the tip of the header chain | +| `elo_header_tip_work` | gauge | The work of the chain up to the tip, in decimal | +| `elo_peer_state` | gauge | 1 for the step of the connection, and 0 for each other. The `state` label is `dial`, `sync`, `synced` or `wait` | +| `elo_reconnects_total` | counter | The connections that ended, and that elo dials again after the wait | + +The tip moves when the `tip` line does: at the end of a sync, and then with each new tip. As for the +log, the names are an interface, and `src/metrics.rs` holds them. elo answers one connection at a +time, each in at most 10 s, and closes it after the answer. A request head is at most 8 KiB. Another +path gets 404, and another method gets 405. + ## Build Nix builds everything. `flake.lock` pins nixpkgs and the other inputs, and @@ -130,7 +151,8 @@ Each push to `main` also pushes an image of elo to GHCR, as `ghcr.io/lorenzolfm/ `:sha-`. CI runs one push at a time and drops a push that waits when a newer one comes, so a burst of merges gets an image for its last commit only. Before the push, CI runs the image against a regtest `bitcoind` until it prints the `tip`. The arguments of the image are the arguments of -elo. It runs as user 65534 and needs no volume, no port and no writable path: +elo. It runs as user 65534 and needs no volume, no port (unless you give `--metrics`) and no +writable path: ```sh nix build .#elo-image && docker load --input result From 9a498c4670a191d95a14846ceca95480924c06c5 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Sun, 15 Nov 2026 08:57:18 -0300 Subject: [PATCH 03/11] fix: search for the end of the head only in the bytes of the last read, so that a head in many small reads costs no quadratic time (#79) The search starts at the last 4 bytes of the previous fill. Before, each read searched the full buffer again, so 8 KiB in one-byte reads was 33 million window compares on the thread of the node. --- src/http.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/http.rs b/src/http.rs index 8a45e67..24ef7e2 100644 --- a/src/http.rs +++ b/src/http.rs @@ -77,13 +77,14 @@ async fn read_head(stream: &mut (impl tokio::io::AsyncRead + Unpin)) -> Result Date: Sun, 15 Nov 2026 08:58:07 -0300 Subject: [PATCH 04/11] fix: count a reconnect at each move into `wait`, and assert each move of the peer state and that the tip does not lose work (#79) `reconnect` and `peer(Wait)` were two calls that `main.rs` had to keep together. Now the move into `wait` is the reconnect, so the counter cannot drift from the state. `peer` asserts the steps of a connection: `dial` to `sync` to `synced`, each of them to `wait`, and `wait` to `dial`. So `main.rs` sets `dial` after the wait, and `Metrics::new` starts in `dial`. The chain asserts more work at each new tip, so `tip` asserts that the work does not go down. The height can go down on a reorganization, so `tip` does not assert it. --- src/main.rs | 3 +- src/metrics.rs | 91 +++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 80 insertions(+), 14 deletions(-) diff --git a/src/main.rs b/src/main.rs index 45b84c7..5ff0869 100644 --- a/src/main.rs +++ b/src/main.rs @@ -103,7 +103,6 @@ async fn node( ) -> std::process::ExitCode { loop { let destination = source.destination(); - metrics.peer(elo::metrics::PeerState::Dial); let end = match signals.unless(dial(&destination, chain, metrics)).await { Ok(Ok(end)) => end, Ok(Err(error)) => return exit_stdout(error), @@ -112,7 +111,6 @@ async fn node( if !end.retries() { return exit(&end); } - metrics.reconnect(); metrics.peer(elo::metrics::PeerState::Wait); let logged = log(&elo::logging::Event::End { destination: &destination, @@ -125,6 +123,7 @@ async fn node( if let Err(signal) = signals.unless(tokio::time::sleep(RECONNECT_WAIT)).await { return stop(signal); } + metrics.peer(elo::metrics::PeerState::Dial); } } diff --git a/src/metrics.rs b/src/metrics.rs index f126018..07fd40b 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -49,17 +49,24 @@ impl Metrics { } pub fn tip(&self, tip: &crate::chain::Tip) { + assert!(tip.work() >= self.tip.get().work()); self.tip.set(*tip); } pub fn peer(&self, state: PeerState) { + let before = self.peer.get(); + match state { + PeerState::Dial => assert_eq!(before, PeerState::Wait), + PeerState::Sync => assert_eq!(before, PeerState::Dial), + PeerState::Synced => assert_eq!(before, PeerState::Sync), + PeerState::Wait => { + assert_ne!(before, PeerState::Wait); + self.reconnects.set(self.reconnects.get().saturating_add(1)); + } + } self.peer.set(state); } - pub fn reconnect(&self) { - self.reconnects.set(self.reconnects.get().saturating_add(1)); - } - pub async fn serve(&self, listener: tokio::net::TcpListener) -> std::convert::Infallible { crate::http::serve(listener, |request| self.answer(request)).await } @@ -178,6 +185,21 @@ elo_header_tip_work 2 )) } + fn metrics_in(state: super::PeerState) -> super::Metrics { + let metrics = metrics(); + let path: &[super::PeerState] = match state { + super::PeerState::Dial => &[], + super::PeerState::Sync => &[super::PeerState::Sync], + super::PeerState::Synced => &[super::PeerState::Sync, super::PeerState::Synced], + super::PeerState::Wait => &[super::PeerState::Wait], + }; + for step in path { + metrics.peer(*step); + } + assert_eq!(metrics.peer.get(), state); + metrics + } + fn states(dial: u8, sync: u8, synced: u8, wait: u8) -> String { format!( "elo_peer_state{{state=\"dial\"}} {dial}\nelo_peer_state{{state=\"sync\"}} {sync}\n\ @@ -195,7 +217,7 @@ elo_header_tip_work 2 } #[test] - fn text_has_the_last_tip_the_state_and_each_reconnect() { + fn text_has_the_last_tip_the_state_and_each_wait_as_a_reconnect() { let metrics = metrics(); let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); let headers = crate::fixtures::headers_on_genesis(3); @@ -205,20 +227,65 @@ elo_header_tip_work 2 ); assert_eq!(extended, Ok(())); metrics.tip(&chain.tip()); - for (state, expected) in [ - (super::PeerState::Sync, states(0, 1, 0, 0)), - (super::PeerState::Synced, states(0, 0, 1, 0)), - (super::PeerState::Wait, states(0, 0, 0, 1)), - (super::PeerState::Dial, states(1, 0, 0, 0)), + for (state, expected, reconnects) in [ + (super::PeerState::Wait, states(0, 0, 0, 1), 1), + (super::PeerState::Dial, states(1, 0, 0, 0), 1), + (super::PeerState::Sync, states(0, 1, 0, 0), 1), + (super::PeerState::Wait, states(0, 0, 0, 1), 2), + (super::PeerState::Dial, states(1, 0, 0, 0), 2), + (super::PeerState::Sync, states(0, 1, 0, 0), 2), + (super::PeerState::Synced, states(0, 0, 1, 0), 2), + (super::PeerState::Wait, states(0, 0, 0, 1), 3), ] { metrics.peer(state); - metrics.reconnect(); let text = metrics.text(); assert!(text.contains("elo_header_tip_height 3\n"), "{text}"); assert!(text.contains("elo_header_tip_work 8\n"), "{text}"); assert!(text.contains(&expected), "{state:?}: {text}"); + let reconnects_line = format!("elo_reconnects_total {reconnects}\n"); + assert!(text.ends_with(&reconnects_line), "{state:?}: {text}"); + } + } + + #[test] + fn peer_state_moves_only_on_the_steps_of_a_connection() { + for before in super::PeerState::ALL { + for after in super::PeerState::ALL { + let allowed = matches!( + (before, after), + (super::PeerState::Dial, super::PeerState::Sync) + | (super::PeerState::Sync, super::PeerState::Synced) + | ( + super::PeerState::Dial + | super::PeerState::Sync + | super::PeerState::Synced, + super::PeerState::Wait + ) + | (super::PeerState::Wait, super::PeerState::Dial) + ); + let metrics = metrics_in(before); + let moved = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + metrics.peer(after); + })); + assert_eq!(moved.is_ok(), allowed, "{before:?} -> {after:?}"); + } } - assert!(metrics.text().ends_with("elo_reconnects_total 4\n")); + } + + #[test] + #[should_panic(expected = "tip.work() >= self.tip.get().work()")] + fn tip_with_less_work_than_the_last_is_a_bug() { + let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let genesis = chain.tip(); + let headers = crate::fixtures::headers_on_genesis(1); + let extended = chain.extend( + crate::fixtures::headers_batch(&headers), + crate::fixtures::CLOCK, + ); + assert_eq!(extended, Ok(())); + let metrics = super::Metrics::new(&chain); + metrics.tip(&chain.tip()); + metrics.tip(&genesis); } #[test] From a0c4f12a9fab971b02bbfcce6c0d6281b0060821 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Sun, 15 Nov 2026 08:58:52 -0300 Subject: [PATCH 05/11] refactor: write each sample through the family it belongs to, with its label checked, into a body of its maximum capacity (#79) `Text::family` returns a `Family` that has the name, so `text()` writes each name once and a sample cannot get the name of another family. `sample_labelled` takes the label and its value apart, not an empty string for no label. It asserts the label bytes, and that the value has no quote, backslash or newline, which the text format must escape. The body starts at `BODY_LENGTH_MAX`, so one scrape allocates once. --- src/metrics.rs | 114 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 80 insertions(+), 34 deletions(-) diff --git a/src/metrics.rs b/src/metrics.rs index 07fd40b..0e71f52 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -86,72 +86,64 @@ impl Metrics { fn text(&self) -> String { let tip = self.tip.get(); - let mut text = Text::default(); + let mut text = Text::new(); text.family( "elo_header_tip_height", "gauge", "The height of the tip of the header chain.", - ); - text.sample("elo_header_tip_height", "", tip.height()); + ) + .sample(tip.height()); text.family( "elo_header_tip_work", "gauge", "The work of the header chain up to its tip: the expected number of hashes.", - ); - text.sample("elo_header_tip_work", "", tip.work().decimal()); - text.family( + ) + .sample(tip.work().decimal()); + let mut family = text.family( "elo_peer_state", "gauge", "The step of the connection to the peer: 1 for the step it is in, 0 for each other.", ); let peer = self.peer.get(); for state in PeerState::ALL { - text.sample( - "elo_peer_state", - format_args!("state=\"{}\"", state.name()), - u8::from(state == peer), - ); + family.sample_labelled("state", state.name(), u8::from(state == peer)); } text.family( "elo_reconnects_total", "counter", "The connections that ended, and that elo dials again after a wait.", - ); - text.sample("elo_reconnects_total", "", self.reconnects.get()); + ) + .sample(self.reconnects.get()); text.end() } } -#[derive(Default)] struct Text { text: String, } +struct Family<'a> { + text: &'a mut String, + name: &'static str, +} + impl Text { - fn family(&mut self, name: &'static str, kind: &'static str, help: &'static str) { + fn new() -> Text { + Text { + text: String::with_capacity(BODY_LENGTH_MAX), + } + } + + fn family(&mut self, name: &'static str, kind: &'static str, help: &'static str) -> Family<'_> { assert!(name.starts_with("elo_"), "{name}"); - assert!( - name.bytes() - .all(|byte| byte.is_ascii_lowercase() || byte == b'_'), - "{name}" - ); + assert!(name.bytes().all(name_byte), "{name}"); assert!(!help.contains(['\\', '\n']), "{name}"); let written = write!(self.text, "# HELP {name} {help}\n# TYPE {name} {kind}\n"); assert!(written.is_ok(), "{name}"); - } - - fn sample( - &mut self, - name: &'static str, - labels: impl std::fmt::Display, - value: impl std::fmt::Display, - ) { - let mut labels_text = labels.to_string(); - if !labels_text.is_empty() { - labels_text = format!("{{{labels_text}}}"); + Family { + text: &mut self.text, + name, } - let written = writeln!(self.text, "{name}{labels_text} {value}"); - assert!(written.is_ok(), "{name}"); } fn end(self) -> String { @@ -161,6 +153,32 @@ impl Text { } } +impl Family<'_> { + fn sample(&mut self, value: impl std::fmt::Display) { + let name = self.name; + let written = writeln!(self.text, "{name} {value}"); + assert!(written.is_ok(), "{name}"); + } + + fn sample_labelled( + &mut self, + label: &'static str, + label_value: &'static str, + value: impl std::fmt::Display, + ) { + let name = self.name; + assert!(!label.is_empty(), "{name}"); + assert!(label.bytes().all(name_byte), "{name}"); + assert!(!label_value.contains(['"', '\\', '\n']), "{name}"); + let written = writeln!(self.text, "{name}{{{label}=\"{label_value}\"}} {value}"); + assert!(written.is_ok(), "{name}"); + } +} + +fn name_byte(byte: u8) -> bool { + byte.is_ascii_lowercase() || byte == b'_' +} + #[cfg(test)] mod tests { const EXPECTED_START: &str = "\ @@ -320,8 +338,36 @@ elo_header_tip_work 2 fn longest_text_is_below_the_maximum() { let text = metrics().text(); let one_digit_values_length = 3; - let longest_values_length = usize::MAX.to_string().len() + 78 + u64::MAX.to_string().len(); + let work_max_decimal_length = 78; + let longest_values_length = + usize::MAX.to_string().len() + work_max_decimal_length + u64::MAX.to_string().len(); let longest = text.len() - one_digit_values_length + longest_values_length; assert!(longest <= super::BODY_LENGTH_MAX, "{longest}"); } + + #[test] + fn label_of_other_bytes_or_a_label_value_with_a_quote_a_backslash_or_a_newline_is_a_bug() { + for (label, label_value) in [ + ("b", "a\"b"), + ("b", "a\\b"), + ("b", "a\nb"), + ("", "c"), + ("B", "c"), + ("b-c", "c"), + ] { + let sampled = std::panic::catch_unwind(|| { + let mut text = super::Text::new(); + text.family("elo_a", "gauge", "A.") + .sample_labelled(label, label_value, 0); + }); + assert!(sampled.is_err(), "{label:?} {label_value:?}"); + } + let mut text = super::Text::new(); + text.family("elo_a", "gauge", "A.") + .sample_labelled("b", "c_d", 0); + assert_eq!( + text.end(), + "# HELP elo_a A.\n# TYPE elo_a gauge\nelo_a{b=\"c_d\"} 0\n" + ); + } } From 29cb247137fdacabe38e9a045c3b8e57d25fe749 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Sun, 15 Nov 2026 08:59:18 -0300 Subject: [PATCH 06/11] fix: assert that a header value of a response has no line end, and check each part of the request line in its own condition (#79) `content_type` and `allow` come from our code, so a CR or LF in one is a bug that would put a header of its own in the response. --- src/http.rs | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/src/http.rs b/src/http.rs index 24ef7e2..ba39251 100644 --- a/src/http.rs +++ b/src/http.rs @@ -103,10 +103,16 @@ fn request(head: &[u8]) -> Option> { else { return None; }; - if method.is_empty() || !method.bytes().all(token_byte) { + if method.is_empty() { return None; } - if target.is_empty() || !target.bytes().all(|byte| byte.is_ascii_graphic()) { + if !method.bytes().all(token_byte) { + return None; + } + if target.is_empty() { + return None; + } + if !target.bytes().all(|byte| byte.is_ascii_graphic()) { return None; } if !matches!(version, "HTTP/1.1" | "HTTP/1.0") { @@ -122,14 +128,18 @@ fn token_byte(byte: u8) -> bool { impl Response { fn bytes(&self) -> Vec { let (status, headers, body) = match self { - Response::Ok { content_type, body } => ( - "200 OK", - format!("Content-Type: {content_type}\r\n"), - body.as_str(), - ), + Response::Ok { content_type, body } => { + assert!(!content_type.contains(['\r', '\n']), "{content_type:?}"); + ( + "200 OK", + format!("Content-Type: {content_type}\r\n"), + body.as_str(), + ) + } Response::BadRequest => ("400 Bad Request", String::new(), ""), Response::NotFound => ("404 Not Found", String::new(), ""), Response::MethodNotAllowed { allow } => { + assert!(!allow.contains(['\r', '\n']), "{allow:?}"); ("405 Method Not Allowed", format!("Allow: {allow}\r\n"), "") } }; @@ -231,6 +241,22 @@ mod tests { } } + #[test] + fn header_value_with_a_line_end_is_a_bug() { + for value in ["a\rb", "a\nb", "a\r\nB: c"] { + for response in [ + super::Response::Ok { + content_type: value, + body: String::new(), + }, + super::Response::MethodNotAllowed { allow: value }, + ] { + let written = std::panic::catch_unwind(|| response.bytes()); + assert!(written.is_err(), "{value:?}"); + } + } + } + #[test] fn exchange_answers_a_head_and_refuses_a_bad_or_long_one() { let fits = format!( From 35ad69165991d3bb90da3c92343c12f0a50317c5 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Sun, 15 Nov 2026 09:00:11 -0300 Subject: [PATCH 07/11] fix: answer in at most 1 s, from a listen backlog of 4, so that slow clients in the queue cannot hold a scrape past its timeout (#79) With 10 s for each connection and the default backlog of 1024, k slow clients before a scrape held it for k * 10 s. One idle connection each 10 s was enough to fail each scrape, as the default scrape timeout of Prometheus is 10 s too. A head is at most 8 KiB, so 1 s is enough. Linux queues one connection more than the backlog, so a scrape waits for at most 5 connections and its own: 6 s. A const assert keeps this below the 10 s. When the queue is full, the kernel drops the connect, and the client tries again. `http::listen` binds with the backlog in our code. It sets `SO_REUSEADDR`, as `TcpListener::bind` did. --- README.md | 4 +++- src/http.rs | 18 ++++++++++++++++-- src/main.rs | 6 +++--- src/metrics.rs | 15 +++++++++++++++ 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b9bf394..1c5ca7f 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,9 @@ container. Port 0 takes a free port. elo logs the address in a `metrics` line be The tip moves when the `tip` line does: at the end of a sync, and then with each new tip. As for the log, the names are an interface, and `src/metrics.rs` holds them. elo answers one connection at a -time, each in at most 10 s, and closes it after the answer. A request head is at most 8 KiB. Another +time, each in at most 1 s, and closes it after the answer. The listen backlog is 4, so at most 5 +connections come before a scrape, and it gets its answer in at most 6 s. The scrape timeout of +Prometheus is 10 s by default. A request head is at most 8 KiB. Another path gets 404, and another method gets 405. ## Build diff --git a/src/http.rs b/src/http.rs index ba39251..e2ffc3a 100644 --- a/src/http.rs +++ b/src/http.rs @@ -2,7 +2,9 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; pub(crate) const HEAD_LENGTH_MAX: usize = 8 * 1024; -pub(crate) const EXCHANGE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +pub(crate) const EXCHANGE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1); + +pub(crate) const LISTEN_BACKLOG: u32 = 4; pub(crate) const ACCEPT_ERROR_WAIT: std::time::Duration = std::time::Duration::from_secs(1); @@ -34,6 +36,18 @@ enum HeadError { TooLong, } +pub(crate) fn listen( + address: std::net::SocketAddr, +) -> Result { + let socket = match address { + std::net::SocketAddr::V4(_) => tokio::net::TcpSocket::new_v4()?, + std::net::SocketAddr::V6(_) => tokio::net::TcpSocket::new_v6()?, + }; + socket.set_reuseaddr(true)?; + socket.bind(address)?; + socket.listen(LISTEN_BACKLOG) +} + pub(crate) async fn serve( listener: tokio::net::TcpListener, answer: impl Fn(Request<'_>) -> Response, @@ -277,7 +291,7 @@ mod tests { }; let bad = "HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; block_on(async { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let listener = super::listen("127.0.0.1:0".parse().unwrap()).unwrap(); for (sent, expected) in [ (&b"GET /metrics HTTP/1.1\r\n\r\n"[..], ok("GET /metrics")), (b"GET /metrics HTTP/1.1\r\n\r\nbody", ok("GET /metrics")), diff --git a/src/main.rs b/src/main.rs index 5ff0869..6a154b8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -47,7 +47,7 @@ async fn run(options: Options) -> std::process::ExitCode { }; let listener = match options.metrics { None => None, - Some(address) => match listen(address).await { + Some(address) => match listen(address) { Ok((listener, address)) => { if let Err(error) = log(&elo::logging::Event::Metrics { address }) { return exit_stdout(error); @@ -69,10 +69,10 @@ async fn run(options: Options) -> std::process::ExitCode { beside(serve, node).await } -async fn listen( +fn listen( address: std::net::SocketAddr, ) -> Result<(tokio::net::TcpListener, std::net::SocketAddr), std::io::Error> { - let listener = tokio::net::TcpListener::bind(address).await?; + let listener = elo::metrics::Metrics::listen(address)?; let address = listener.local_addr()?; Ok((listener, address)) } diff --git a/src/metrics.rs b/src/metrics.rs index 0e71f52..2d562af 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -6,6 +6,15 @@ const CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8"; const BODY_LENGTH_MAX: usize = 1024; +const SCRAPE_TIMEOUT_DEFAULT: std::time::Duration = std::time::Duration::from_secs(10); + +const _: () = assert!( + crate::http::EXCHANGE_TIMEOUT + .saturating_mul(crate::http::LISTEN_BACKLOG + 2) + .as_millis() + < SCRAPE_TIMEOUT_DEFAULT.as_millis() +); + #[derive(Debug)] pub struct Metrics { tip: std::cell::Cell, @@ -67,6 +76,12 @@ impl Metrics { self.peer.set(state); } + pub fn listen( + address: std::net::SocketAddr, + ) -> Result { + crate::http::listen(address) + } + pub async fn serve(&self, listener: tokio::net::TcpListener) -> std::convert::Infallible { crate::http::serve(listener, |request| self.answer(request)).await } From 4f844b68e90bf342d89843ad0493daa8d1baa7af Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Sun, 15 Nov 2026 09:02:19 -0300 Subject: [PATCH 08/11] feat: count each failed connection to the metrics by its kind, in `elo_metrics_failures_total` (#79) `serve` dropped the error of each connection, so a limit of file descriptors or a run of slow clients left no trace. Now `serve` gives each failure to a function of the caller: `accept` for an accept error, `io` for a read or write error, and `timeout` for no answer in 1 s. A client that closes before a head, or that sends a bad head, is not a failure. The new family makes the longest body 1184 bytes, so its maximum is now 2 KiB. --- README.md | 1 + src/http.rs | 54 ++++++++++++++++++++++++++++++++-- src/metrics.rs | 75 ++++++++++++++++++++++++++++++++++++++++++----- tests/bitcoind.rs | 4 +-- 4 files changed, 123 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 1c5ca7f..ea77703 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,7 @@ container. Port 0 takes a free port. elo logs the address in a `metrics` line be | `elo_header_tip_work` | gauge | The work of the chain up to the tip, in decimal | | `elo_peer_state` | gauge | 1 for the step of the connection, and 0 for each other. The `state` label is `dial`, `sync`, `synced` or `wait` | | `elo_reconnects_total` | counter | The connections that ended, and that elo dials again after the wait | +| `elo_metrics_failures_total` | counter | The connections to the metrics that failed. The `kind` label is `accept` for an accept error, `io` for a read or write error, or `timeout` for no answer in 1 s | The tip moves when the `tip` line does: at the end of a sync, and then with each new tip. As for the log, the names are an interface, and `src/metrics.rs` holds them. elo answers one connection at a diff --git a/src/http.rs b/src/http.rs index e2ffc3a..d03e4e2 100644 --- a/src/http.rs +++ b/src/http.rs @@ -29,6 +29,13 @@ pub(crate) enum Response { }, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Failure { + Accept, + Io, + Timeout, +} + #[derive(Debug)] enum HeadError { Io(std::io::Error), @@ -51,14 +58,22 @@ pub(crate) fn listen( pub(crate) async fn serve( listener: tokio::net::TcpListener, answer: impl Fn(Request<'_>) -> Response, + failed: impl Fn(Failure), ) -> std::convert::Infallible { loop { match listener.accept().await { Ok((mut stream, _)) => { let exchange = exchange(&mut stream, &answer); - let _ = tokio::time::timeout(EXCHANGE_TIMEOUT, exchange).await; + match tokio::time::timeout(EXCHANGE_TIMEOUT, exchange).await { + Ok(Ok(())) => {} + Ok(Err(_)) => failed(Failure::Io), + Err(_) => failed(Failure::Timeout), + } + } + Err(_) => { + failed(Failure::Accept); + tokio::time::sleep(ACCEPT_ERROR_WAIT).await; } - Err(_) => tokio::time::sleep(ACCEPT_ERROR_WAIT).await, } } } @@ -307,6 +322,41 @@ mod tests { }); } + #[test] + fn serve_counts_a_connection_with_no_head_in_time_and_one_reset_and_answers_the_next() { + tokio::runtime::Builder::new_current_thread() + .enable_io() + .enable_time() + .start_paused(true) + .build() + .unwrap() + .block_on(async { + let listener = super::listen("127.0.0.1:0".parse().unwrap()).unwrap(); + let address = listener.local_addr().unwrap(); + let idle = tokio::net::TcpStream::connect(address).await.unwrap(); + let reset = tokio::net::TcpStream::connect(address).await.unwrap(); + reset.set_zero_linger().unwrap(); + drop(reset); + let mut client = tokio::net::TcpStream::connect(address).await.unwrap(); + client.write_all(b"GET / HTTP/1.1\r\n\r\n").await.unwrap(); + let failures = std::cell::RefCell::new(Vec::new()); + let serve = super::serve(listener, answer, |failure| { + failures.borrow_mut().push(failure); + }); + let served = tokio::time::timeout(super::EXCHANGE_TIMEOUT * 3, serve).await; + assert!(served.is_err()); + assert_eq!( + *failures.borrow(), + [super::Failure::Timeout, super::Failure::Io] + ); + let mut received = String::new(); + client.read_to_string(&mut received).await.unwrap(); + assert!(received.starts_with("HTTP/1.1 200 OK\r\n"), "{received}"); + assert!(received.ends_with("\r\n\r\nGET /"), "{received}"); + drop(idle); + }); + } + #[test] fn head_end_over_two_reads_is_found() { let head = b"GET / HTTP/1.1\r\n\r\n"; diff --git a/src/metrics.rs b/src/metrics.rs index 2d562af..ae6b0dd 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -4,7 +4,7 @@ const PATH: &str = "/metrics"; const CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8"; -const BODY_LENGTH_MAX: usize = 1024; +const BODY_LENGTH_MAX: usize = 2 * 1024; const SCRAPE_TIMEOUT_DEFAULT: std::time::Duration = std::time::Duration::from_secs(10); @@ -20,6 +20,9 @@ pub struct Metrics { tip: std::cell::Cell, peer: std::cell::Cell, reconnects: std::cell::Cell, + accept_failures: std::cell::Cell, + io_failures: std::cell::Cell, + timeout_failures: std::cell::Cell, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -54,6 +57,9 @@ impl Metrics { tip: std::cell::Cell::new(chain.tip()), peer: std::cell::Cell::new(PeerState::Dial), reconnects: std::cell::Cell::new(0), + accept_failures: std::cell::Cell::new(0), + io_failures: std::cell::Cell::new(0), + timeout_failures: std::cell::Cell::new(0), } } @@ -83,7 +89,21 @@ impl Metrics { } pub async fn serve(&self, listener: tokio::net::TcpListener) -> std::convert::Infallible { - crate::http::serve(listener, |request| self.answer(request)).await + crate::http::serve( + listener, + |request| self.answer(request), + |failure| self.failed(failure), + ) + .await + } + + fn failed(&self, failure: crate::http::Failure) { + let failures = match failure { + crate::http::Failure::Accept => &self.accept_failures, + crate::http::Failure::Io => &self.io_failures, + crate::http::Failure::Timeout => &self.timeout_failures, + }; + failures.set(failures.get().saturating_add(1)); } fn answer(&self, request: crate::http::Request<'_>) -> crate::http::Response { @@ -129,6 +149,18 @@ impl Metrics { "The connections that ended, and that elo dials again after a wait.", ) .sample(self.reconnects.get()); + let mut family = text.family( + "elo_metrics_failures_total", + "counter", + "The connections to the metrics that failed: an accept error, a read or write error, or no answer in time.", + ); + for (kind, failures) in [ + ("accept", &self.accept_failures), + ("io", &self.io_failures), + ("timeout", &self.timeout_failures), + ] { + family.sample_labelled("kind", kind, failures.get()); + } text.end() } } @@ -210,6 +242,11 @@ elo_header_tip_work 2 const EXPECTED_END: &str = "\ # HELP elo_reconnects_total The connections that ended, and that elo dials again after a wait. # TYPE elo_reconnects_total counter +"; + + const EXPECTED_FAILURES_HEAD: &str = "\ +# HELP elo_metrics_failures_total The connections to the metrics that failed: an accept error, a read or write error, or no answer in time. +# TYPE elo_metrics_failures_total counter "; fn metrics() -> super::Metrics { @@ -233,6 +270,14 @@ elo_header_tip_work 2 metrics } + fn failures(accept: u8, io: u8, timeout: u8) -> String { + format!( + "elo_metrics_failures_total{{kind=\"accept\"}} {accept}\n\ + elo_metrics_failures_total{{kind=\"io\"}} {io}\n\ + elo_metrics_failures_total{{kind=\"timeout\"}} {timeout}\n" + ) + } + fn states(dial: u8, sync: u8, synced: u8, wait: u8) -> String { format!( "elo_peer_state{{state=\"dial\"}} {dial}\nelo_peer_state{{state=\"sync\"}} {sync}\n\ @@ -243,8 +288,9 @@ elo_header_tip_work 2 #[test] fn text_has_the_tip_of_the_chain_the_dial_and_no_reconnect_at_start() { let expected = format!( - "{EXPECTED_START}{}{EXPECTED_END}elo_reconnects_total 0\n", - states(1, 0, 0, 0) + "{EXPECTED_START}{}{EXPECTED_END}elo_reconnects_total 0\n{EXPECTED_FAILURES_HEAD}{}", + states(1, 0, 0, 0), + failures(0, 0, 0) ); assert_eq!(metrics().text(), expected); } @@ -276,7 +322,22 @@ elo_header_tip_work 2 assert!(text.contains("elo_header_tip_work 8\n"), "{text}"); assert!(text.contains(&expected), "{state:?}: {text}"); let reconnects_line = format!("elo_reconnects_total {reconnects}\n"); - assert!(text.ends_with(&reconnects_line), "{state:?}: {text}"); + assert!(text.contains(&reconnects_line), "{state:?}: {text}"); + } + } + + #[test] + fn text_has_each_failure_of_its_kind() { + let metrics = metrics(); + for (failure, expected) in [ + (crate::http::Failure::Timeout, failures(0, 0, 1)), + (crate::http::Failure::Io, failures(0, 1, 1)), + (crate::http::Failure::Accept, failures(1, 1, 1)), + (crate::http::Failure::Timeout, failures(1, 1, 2)), + ] { + metrics.failed(failure); + let text = metrics.text(); + assert!(text.ends_with(&expected), "{failure:?}: {text}"); } } @@ -352,10 +413,10 @@ elo_header_tip_work 2 #[test] fn longest_text_is_below_the_maximum() { let text = metrics().text(); - let one_digit_values_length = 3; + let one_digit_values_length = 6; let work_max_decimal_length = 78; let longest_values_length = - usize::MAX.to_string().len() + work_max_decimal_length + u64::MAX.to_string().len(); + usize::MAX.to_string().len() + work_max_decimal_length + 4 * u64::MAX.to_string().len(); let longest = text.len() - one_digit_values_length + longest_values_length; assert!(longest <= super::BODY_LENGTH_MAX, "{longest}"); } diff --git a/tests/bitcoind.rs b/tests/bitcoind.rs index 88be9d2..44031d0 100644 --- a/tests/bitcoind.rs +++ b/tests/bitcoind.rs @@ -616,7 +616,7 @@ fn metrics_of_elo_have_its_tip_its_step_and_each_reconnect() { ] { assert!(metrics.contains(&line), "{line}: {metrics}"); } - assert!(metrics.ends_with("elo_reconnects_total 0\n"), "{metrics}"); + assert!(metrics.contains("elo_reconnects_total 0\n"), "{metrics}"); bitcoind.cli(&["-named", "disconnectnode", "nodeid=0"]); elo.event("end", END_TIMEOUT); @@ -625,7 +625,7 @@ fn metrics_of_elo_have_its_tip_its_step_and_each_reconnect() { metrics.contains("elo_peer_state{state=\"wait\"} 1\n"), "{metrics}" ); - assert!(metrics.ends_with("elo_reconnects_total 1\n"), "{metrics}"); + assert!(metrics.contains("elo_reconnects_total 1\n"), "{metrics}"); } #[test] From 4d2d76313011829cbdaf21924dfa745ac1cb81a6 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Sun, 15 Nov 2026 11:05:14 -0300 Subject: [PATCH 09/11] refactor: give the type of each family as a `Kind`, and assert that only a counter has a name that ends with `_total` (#79) The type of a family was free text, so a family of type `guage` was possible. Prometheus refuses a scrape with an unknown type. The text of the tests did not find it, because the same hand writes the text and its test. Now a `Kind` is `Gauge` or `Counter`, and one match gives its name. A counter name ends with `_total`, and a gauge name does not. --- src/metrics.rs | 57 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/src/metrics.rs b/src/metrics.rs index ae6b0dd..713b9b5 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -124,19 +124,19 @@ impl Metrics { let mut text = Text::new(); text.family( "elo_header_tip_height", - "gauge", + Kind::Gauge, "The height of the tip of the header chain.", ) .sample(tip.height()); text.family( "elo_header_tip_work", - "gauge", + Kind::Gauge, "The work of the header chain up to its tip: the expected number of hashes.", ) .sample(tip.work().decimal()); let mut family = text.family( "elo_peer_state", - "gauge", + Kind::Gauge, "The step of the connection to the peer: 1 for the step it is in, 0 for each other.", ); let peer = self.peer.get(); @@ -145,13 +145,13 @@ impl Metrics { } text.family( "elo_reconnects_total", - "counter", + Kind::Counter, "The connections that ended, and that elo dials again after a wait.", ) .sample(self.reconnects.get()); let mut family = text.family( "elo_metrics_failures_total", - "counter", + Kind::Counter, "The connections to the metrics that failed: an accept error, a read or write error, or no answer in time.", ); for (kind, failures) in [ @@ -165,6 +165,21 @@ impl Metrics { } } +#[derive(Clone, Copy, Debug)] +enum Kind { + Gauge, + Counter, +} + +impl Kind { + fn name(self) -> &'static str { + match self { + Kind::Gauge => "gauge", + Kind::Counter => "counter", + } + } +} + struct Text { text: String, } @@ -181,10 +196,16 @@ impl Text { } } - fn family(&mut self, name: &'static str, kind: &'static str, help: &'static str) -> Family<'_> { + fn family(&mut self, name: &'static str, kind: Kind, help: &'static str) -> Family<'_> { assert!(name.starts_with("elo_"), "{name}"); assert!(name.bytes().all(name_byte), "{name}"); + assert_eq!( + name.ends_with("_total"), + matches!(kind, Kind::Counter), + "{name}" + ); assert!(!help.contains(['\\', '\n']), "{name}"); + let kind = kind.name(); let written = write!(self.text, "# HELP {name} {help}\n# TYPE {name} {kind}\n"); assert!(written.is_ok(), "{name}"); Family { @@ -433,17 +454,37 @@ elo_header_tip_work 2 ] { let sampled = std::panic::catch_unwind(|| { let mut text = super::Text::new(); - text.family("elo_a", "gauge", "A.") + text.family("elo_a", super::Kind::Gauge, "A.") .sample_labelled(label, label_value, 0); }); assert!(sampled.is_err(), "{label:?} {label_value:?}"); } let mut text = super::Text::new(); - text.family("elo_a", "gauge", "A.") + text.family("elo_a", super::Kind::Gauge, "A.") .sample_labelled("b", "c_d", 0); assert_eq!( text.end(), "# HELP elo_a A.\n# TYPE elo_a gauge\nelo_a{b=\"c_d\"} 0\n" ); } + + #[test] + fn counter_name_ends_with_total_and_a_gauge_name_does_not() { + for (name, kind) in [ + ("elo_a", super::Kind::Counter), + ("elo_a_total", super::Kind::Gauge), + ] { + let written = std::panic::catch_unwind(|| { + super::Text::new().family(name, kind, "A.").sample(0); + }); + assert!(written.is_err(), "{name} {kind:?}"); + } + let mut text = super::Text::new(); + text.family("elo_a_total", super::Kind::Counter, "A.") + .sample(0); + assert_eq!( + text.end(), + "# HELP elo_a_total A.\n# TYPE elo_a_total counter\nelo_a_total 0\n" + ); + } } From a516d0866f3915ce99fb28fcdc435ceb07e63985 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Sun, 15 Nov 2026 11:07:12 -0300 Subject: [PATCH 10/11] refactor: give each value of a sample its maximum length, and assert that the body fits when each value is at its maximum (#79) A test proved the maximum of the body with numbers by hand: 6 values of one digit, 78 for the work, and 4 counters. A new family that did not change that test kept the test green, and then the assert in `end` could stop elo at a scrape, when a counter got large. Now each type of a value has `LENGTH_MAX`, and `Text` counts the bytes that each value can still grow. `end` asserts that the body fits with each value at its maximum, so each test that writes the text proves the maximum. The longest body is now 1192 bytes, as a state of `u8` can have 3 digits. `Work::decimal` gives a `Decimal`, whose maximum is 78, and its test asserts the 78 against `Work::MAX`. --- src/metrics.rs | 89 +++++++++++++++++++++++++++++++++++++------------- src/pow.rs | 57 ++++++++++++++++++++------------ 2 files changed, 103 insertions(+), 43 deletions(-) diff --git a/src/metrics.rs b/src/metrics.rs index 713b9b5..c68ab37 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -180,12 +180,35 @@ impl Kind { } } +trait Value: std::fmt::Display { + const LENGTH_MAX: usize; +} + +impl Value for u64 { + const LENGTH_MAX: usize = 20; +} + +const _: () = assert!(usize::BITS <= u64::BITS); + +impl Value for usize { + const LENGTH_MAX: usize = ::LENGTH_MAX; +} + +impl Value for u8 { + const LENGTH_MAX: usize = 3; +} + +impl Value for crate::pow::Decimal { + const LENGTH_MAX: usize = crate::pow::Decimal::LENGTH_MAX; +} + struct Text { text: String, + growth: usize, } struct Family<'a> { - text: &'a mut String, + text: &'a mut Text, name: &'static str, } @@ -193,6 +216,7 @@ impl Text { fn new() -> Text { Text { text: String::with_capacity(BODY_LENGTH_MAX), + growth: 0, } } @@ -208,38 +232,47 @@ impl Text { let kind = kind.name(); let written = write!(self.text, "# HELP {name} {help}\n# TYPE {name} {kind}\n"); assert!(written.is_ok(), "{name}"); - Family { - text: &mut self.text, - name, - } + Family { text: self, name } + } + + fn value(&mut self, value: V) { + let start = self.text.len(); + let written = writeln!(self.text, "{value}"); + assert!(written.is_ok()); + let length = self.text.len() - start - 1; + assert!(length <= V::LENGTH_MAX, "{length}"); + self.growth += V::LENGTH_MAX - length; } fn end(self) -> String { assert!(self.text.ends_with('\n')); - assert!(self.text.len() <= BODY_LENGTH_MAX, "{}", self.text.len()); + let length_max = self.text.len() + self.growth; + assert!(length_max <= BODY_LENGTH_MAX, "{length_max}"); self.text } } impl Family<'_> { - fn sample(&mut self, value: impl std::fmt::Display) { + fn sample(&mut self, value: impl Value) { let name = self.name; - let written = writeln!(self.text, "{name} {value}"); + let written = write!(self.text.text, "{name} "); assert!(written.is_ok(), "{name}"); + self.text.value(value); } fn sample_labelled( &mut self, label: &'static str, label_value: &'static str, - value: impl std::fmt::Display, + value: impl Value, ) { let name = self.name; assert!(!label.is_empty(), "{name}"); assert!(label.bytes().all(name_byte), "{name}"); assert!(!label_value.contains(['"', '\\', '\n']), "{name}"); - let written = writeln!(self.text, "{name}{{{label}=\"{label_value}\"}} {value}"); + let written = write!(self.text.text, "{name}{{{label}=\"{label_value}\"}} "); assert!(written.is_ok(), "{name}"); + self.text.value(value); } } @@ -432,14 +465,26 @@ elo_header_tip_work 2 } #[test] - fn longest_text_is_below_the_maximum() { - let text = metrics().text(); - let one_digit_values_length = 6; - let work_max_decimal_length = 78; - let longest_values_length = - usize::MAX.to_string().len() + work_max_decimal_length + 4 * u64::MAX.to_string().len(); - let longest = text.len() - one_digit_values_length + longest_values_length; - assert!(longest <= super::BODY_LENGTH_MAX, "{longest}"); + fn longest_text_of_each_value_has_its_maximum_length() { + use super::Value; + assert_eq!(u64::MAX.to_string().len(), u64::LENGTH_MAX); + assert!(usize::MAX.to_string().len() <= usize::LENGTH_MAX); + assert_eq!(u8::MAX.to_string().len(), u8::LENGTH_MAX); + } + + #[test] + fn text_whose_values_can_grow_past_the_maximum_is_a_bug() { + let text = |samples| { + let mut text = super::Text::new(); + let mut family = text.family("elo_a", super::Kind::Gauge, "AAAAAAAAAAAAAAAAA"); + for _ in 0..samples { + family.sample(0_u64); + } + text.end() + }; + assert_eq!(text(74).len() + 74 * 19, super::BODY_LENGTH_MAX); + let ended = std::panic::catch_unwind(|| text(75)); + assert!(ended.is_err()); } #[test] @@ -455,13 +500,13 @@ elo_header_tip_work 2 let sampled = std::panic::catch_unwind(|| { let mut text = super::Text::new(); text.family("elo_a", super::Kind::Gauge, "A.") - .sample_labelled(label, label_value, 0); + .sample_labelled(label, label_value, 0_u8); }); assert!(sampled.is_err(), "{label:?} {label_value:?}"); } let mut text = super::Text::new(); text.family("elo_a", super::Kind::Gauge, "A.") - .sample_labelled("b", "c_d", 0); + .sample_labelled("b", "c_d", 0_u8); assert_eq!( text.end(), "# HELP elo_a A.\n# TYPE elo_a gauge\nelo_a{b=\"c_d\"} 0\n" @@ -475,13 +520,13 @@ elo_header_tip_work 2 ("elo_a_total", super::Kind::Gauge), ] { let written = std::panic::catch_unwind(|| { - super::Text::new().family(name, kind, "A.").sample(0); + super::Text::new().family(name, kind, "A.").sample(0_u8); }); assert!(written.is_err(), "{name} {kind:?}"); } let mut text = super::Text::new(); text.family("elo_a_total", super::Kind::Counter, "A.") - .sample(0); + .sample(0_u8); assert_eq!( text.end(), "# HELP elo_a_total A.\n# TYPE elo_a_total counter\nelo_a_total 0\n" diff --git a/src/pow.rs b/src/pow.rs index b39b31f..f730b7e 100644 --- a/src/pow.rs +++ b/src/pow.rs @@ -140,27 +140,8 @@ impl Work { (quotient, remainder) } - pub(crate) fn decimal(self) -> impl std::fmt::Display { - std::fmt::from_fn(move |f| { - let mut chunks = [0_u64; DECIMAL_CHUNKS_COUNT]; - let mut rest = self; - for chunk in chunks.iter_mut().rev() { - let (quotient, remainder) = rest.divided_with_remainder(DECIMAL_CHUNK_DIVISOR); - assert_eq!(remainder.high, 0); - *chunk = u64::try_from(remainder.low).unwrap_or(u64::MAX); - assert!(*chunk < DECIMAL_CHUNK_BASE); - rest = quotient; - } - assert_eq!(rest, Work::ZERO); - let first_index = chunks - .iter() - .position(|chunk| *chunk != 0) - .unwrap_or(DECIMAL_CHUNKS_COUNT - 1); - write!(f, "{}", chunks[first_index])?; - chunks[first_index + 1..] - .iter() - .try_for_each(|chunk| write!(f, "{chunk:019}")) - }) + pub(crate) fn decimal(self) -> Decimal { + Decimal(self) } fn shifted_left(self, bits: u32) -> Work { @@ -205,6 +186,36 @@ impl Work { } } +#[derive(Clone, Copy, Debug)] +pub(crate) struct Decimal(Work); + +impl Decimal { + pub(crate) const LENGTH_MAX: usize = 78; +} + +impl std::fmt::Display for Decimal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut chunks = [0_u64; DECIMAL_CHUNKS_COUNT]; + let mut rest = self.0; + for chunk in chunks.iter_mut().rev() { + let (quotient, remainder) = rest.divided_with_remainder(DECIMAL_CHUNK_DIVISOR); + assert_eq!(remainder.high, 0); + *chunk = u64::try_from(remainder.low).unwrap_or(u64::MAX); + assert!(*chunk < DECIMAL_CHUNK_BASE); + rest = quotient; + } + assert_eq!(rest, Work::ZERO); + let first_index = chunks + .iter() + .position(|chunk| *chunk != 0) + .unwrap_or(DECIMAL_CHUNKS_COUNT - 1); + write!(f, "{}", chunks[first_index])?; + chunks[first_index + 1..] + .iter() + .try_for_each(|chunk| write!(f, "{chunk:019}")) + } +} + impl std::fmt::Display for Work { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{:032x}{:032x}", self.high, self.low) @@ -1026,6 +1037,10 @@ mod tests { ] { assert_eq!(number.decimal().to_string(), expected, "{number}"); } + assert_eq!( + super::Work::MAX.decimal().to_string().len(), + super::Decimal::LENGTH_MAX + ); } #[test] From 4073553198f0fd8a1b578ab90a2c2d3dd7d744d9 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Sun, 15 Nov 2026 16:14:27 -0300 Subject: [PATCH 11/11] fix: answer at most 8 connections at the same time, and close one when 8 newer ones come before its answer, so that idle clients cannot hold a scrape (#79) elo answered one connection at a time. The listen backlog of 4 holds 5 connections, and when it is full, the kernel drops the SYN of a scrape, and the client sends it again after 1 s, then 3 s. So the bound of 6 s held only for a scrape in the queue. In a netns, 8 idle clients made a scrape take 7.8 s, and 16 made it take 10.8 s, past the scrape timeout of 10 s of Prometheus. Now `serve` accepts each connection when it comes, and keeps at most 8 exchanges, each with its own timeout of 1 s. A new connection goes into the next of 8 places in turn. When that place has an exchange with no answer, elo closes it and counts it as `evicted`. So a connection stays open until its answer, its timeout, or 8 newer connections. With 4, 8, 16 or 64 idle clients, a scrape now takes less than 1 ms. The exchanges are futures in an array, which one `poll_fn` polls in the task of `serve`, so nothing is spawned, as before. The const assert now only asserts that the timeout of an exchange is less than the scrape timeout, as no queue of connections waits before a scrape. --- README.md | 13 ++-- src/http.rs | 173 +++++++++++++++++++++++++++++++++++++------------ src/metrics.rs | 32 ++++----- 3 files changed, 155 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index ea77703..a3b93ee 100644 --- a/README.md +++ b/README.md @@ -128,14 +128,15 @@ container. Port 0 takes a free port. elo logs the address in a `metrics` line be | `elo_header_tip_work` | gauge | The work of the chain up to the tip, in decimal | | `elo_peer_state` | gauge | 1 for the step of the connection, and 0 for each other. The `state` label is `dial`, `sync`, `synced` or `wait` | | `elo_reconnects_total` | counter | The connections that ended, and that elo dials again after the wait | -| `elo_metrics_failures_total` | counter | The connections to the metrics that failed. The `kind` label is `accept` for an accept error, `io` for a read or write error, or `timeout` for no answer in 1 s | +| `elo_metrics_failures_total` | counter | The connections to the metrics that failed. The `kind` label is `accept` for an accept error, `io` for a read or write error, `timeout` for no answer in 1 s, or `evicted` for 8 newer connections before the answer | The tip moves when the `tip` line does: at the end of a sync, and then with each new tip. As for the -log, the names are an interface, and `src/metrics.rs` holds them. elo answers one connection at a -time, each in at most 1 s, and closes it after the answer. The listen backlog is 4, so at most 5 -connections come before a scrape, and it gets its answer in at most 6 s. The scrape timeout of -Prometheus is 10 s by default. A request head is at most 8 KiB. Another -path gets 404, and another method gets 405. +log, the names are an interface, and `src/metrics.rs` holds them. elo accepts each connection when +it comes and answers at most 8 at the same time, each in at most 1 s. It closes each connection after +its answer, and also closes a connection when 8 newer connections come before its answer, so idle +clients cannot hold a scrape. A scrape gets its answer in at most 1 s, unless 8 newer connections +come before its answer. The scrape timeout of Prometheus is 10 s by default. A request +head is at most 8 KiB. Another path gets 404, and another method gets 405. ## Build diff --git a/src/http.rs b/src/http.rs index d03e4e2..5954a02 100644 --- a/src/http.rs +++ b/src/http.rs @@ -4,7 +4,9 @@ pub(crate) const HEAD_LENGTH_MAX: usize = 8 * 1024; pub(crate) const EXCHANGE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1); -pub(crate) const LISTEN_BACKLOG: u32 = 4; +pub(crate) const EXCHANGES_MAX: usize = 8; + +const LISTEN_BACKLOG: u32 = 4; pub(crate) const ACCEPT_ERROR_WAIT: std::time::Duration = std::time::Duration::from_secs(1); @@ -34,6 +36,7 @@ pub(crate) enum Failure { Accept, Io, Timeout, + Evicted, } #[derive(Debug)] @@ -60,22 +63,56 @@ pub(crate) async fn serve( answer: impl Fn(Request<'_>) -> Response, failed: impl Fn(Failure), ) -> std::convert::Infallible { - loop { - match listener.accept().await { - Ok((mut stream, _)) => { - let exchange = exchange(&mut stream, &answer); - match tokio::time::timeout(EXCHANGE_TIMEOUT, exchange).await { - Ok(Ok(())) => {} - Ok(Err(_)) => failed(Failure::Io), - Err(_) => failed(Failure::Timeout), + let answer = &answer; + let mut exchanges: [Option<_>; EXCHANGES_MAX] = std::array::from_fn(|_| None); + let mut next: usize = 0; + let mut accept_wait = std::pin::pin!(tokio::time::sleep(std::time::Duration::ZERO)); + let mut accept_waits = false; + std::future::poll_fn(|context| { + loop { + if accept_waits { + if accept_wait.as_mut().poll(context).is_pending() { + break; + } + accept_waits = false; + } + match listener.poll_accept(context) { + std::task::Poll::Ready(Ok((mut stream, _))) => { + let slot = &mut exchanges[next]; + if slot.is_some() { + failed(Failure::Evicted); + } + *slot = Some(Box::pin(tokio::time::timeout( + EXCHANGE_TIMEOUT, + async move { exchange(&mut stream, answer).await }, + ))); + next = (next + 1) % EXCHANGES_MAX; } + std::task::Poll::Ready(Err(_)) => { + failed(Failure::Accept); + accept_wait.set(tokio::time::sleep(ACCEPT_ERROR_WAIT)); + accept_waits = true; + } + std::task::Poll::Pending => break, } - Err(_) => { - failed(Failure::Accept); - tokio::time::sleep(ACCEPT_ERROR_WAIT).await; + } + for slot in &mut exchanges { + let Some(exchange) = slot else { + continue; + }; + let std::task::Poll::Ready(exchanged) = exchange.as_mut().poll(context) else { + continue; + }; + match exchanged { + Ok(Ok(())) => {} + Ok(Err(_)) => failed(Failure::Io), + Err(_) => failed(Failure::Timeout), } + *slot = None; } - } + std::task::Poll::Pending + }) + .await } async fn exchange( @@ -322,39 +359,91 @@ mod tests { }); } - #[test] - fn serve_counts_a_connection_with_no_head_in_time_and_one_reset_and_answers_the_next() { - tokio::runtime::Builder::new_current_thread() - .enable_io() - .enable_time() - .start_paused(true) - .build() - .unwrap() - .block_on(async { - let listener = super::listen("127.0.0.1:0".parse().unwrap()).unwrap(); - let address = listener.local_addr().unwrap(); - let idle = tokio::net::TcpStream::connect(address).await.unwrap(); - let reset = tokio::net::TcpStream::connect(address).await.unwrap(); - reset.set_zero_linger().unwrap(); - drop(reset); - let mut client = tokio::net::TcpStream::connect(address).await.unwrap(); - client.write_all(b"GET / HTTP/1.1\r\n\r\n").await.unwrap(); - let failures = std::cell::RefCell::new(Vec::new()); + fn served( + clients: impl FnOnce(std::net::SocketAddr) -> F, + ) -> (F::Output, Vec) { + block_on(async { + let listener = super::listen("127.0.0.1:0".parse().unwrap()).unwrap(); + let address = listener.local_addr().unwrap(); + let failures = std::cell::RefCell::new(Vec::new()); + let output = { let serve = super::serve(listener, answer, |failure| { failures.borrow_mut().push(failure); }); - let served = tokio::time::timeout(super::EXCHANGE_TIMEOUT * 3, serve).await; - assert!(served.is_err()); - assert_eq!( - *failures.borrow(), - [super::Failure::Timeout, super::Failure::Io] - ); - let mut received = String::new(); - client.read_to_string(&mut received).await.unwrap(); - assert!(received.starts_with("HTTP/1.1 200 OK\r\n"), "{received}"); - assert!(received.ends_with("\r\n\r\nGET /"), "{received}"); - drop(idle); + let mut serve = std::pin::pin!(serve); + let mut clients = std::pin::pin!(clients(address)); + std::future::poll_fn(|context| { + if let std::task::Poll::Ready(output) = clients.as_mut().poll(context) { + return std::task::Poll::Ready(output); + } + match serve.as_mut().poll(context) { + std::task::Poll::Ready(never) => match never {}, + std::task::Poll::Pending => std::task::Poll::Pending, + } + }) + .await + }; + (output, failures.into_inner()) + }) + } + + async fn scrape(address: std::net::SocketAddr) -> String { + let mut client = tokio::net::TcpStream::connect(address).await.unwrap(); + client.write_all(b"GET / HTTP/1.1\r\n\r\n").await.unwrap(); + let mut received = String::new(); + client.read_to_string(&mut received).await.unwrap(); + received + } + + #[test] + fn serve_answers_a_head_while_a_connection_has_none_and_counts_its_timeout_and_a_reset() { + let ((received, answered, idle_end), failures) = served(async |address| { + let start = std::time::Instant::now(); + let mut idle = tokio::net::TcpStream::connect(address).await.unwrap(); + let reset = tokio::net::TcpStream::connect(address).await.unwrap(); + reset.set_zero_linger().unwrap(); + drop(reset); + let received = scrape(address).await; + let answered = start.elapsed(); + let read = idle.read(&mut [0; 1]).await.unwrap(); + assert_eq!(read, 0); + (received, answered, start.elapsed()) + }); + assert!(received.starts_with("HTTP/1.1 200 OK\r\n"), "{received}"); + assert!(received.ends_with("\r\n\r\nGET /"), "{received}"); + assert!(answered < super::EXCHANGE_TIMEOUT / 2, "{answered:?}"); + assert!(idle_end >= super::EXCHANGE_TIMEOUT, "{idle_end:?}"); + assert_eq!(failures, [super::Failure::Io, super::Failure::Timeout]); + } + + #[test] + fn connection_with_as_many_newer_ones_as_the_maximum_is_evicted_for_the_newest() { + for (idle_count, evicted) in [ + (super::EXCHANGES_MAX - 1, false), + (super::EXCHANGES_MAX, true), + ] { + let ((received, oldest_end), failures) = served(async |address| { + let mut idle = Vec::new(); + for _ in 0..idle_count { + idle.push(tokio::net::TcpStream::connect(address).await.unwrap()); + } + let received = scrape(address).await; + let wait = super::EXCHANGE_TIMEOUT / 10; + let oldest_end = tokio::time::timeout(wait, idle[0].read(&mut [0; 1])).await; + (received, oldest_end.map(Result::unwrap)) }); + assert!( + received.ends_with("\r\n\r\nGET /"), + "{idle_count}: {received}" + ); + assert_eq!(oldest_end.is_ok(), evicted, "{idle_count}: {oldest_end:?}"); + if evicted { + assert_eq!(oldest_end, Ok(0), "{idle_count}"); + assert_eq!(failures, [super::Failure::Evicted], "{idle_count}"); + } else { + assert_eq!(failures, [], "{idle_count}"); + } + } } #[test] diff --git a/src/metrics.rs b/src/metrics.rs index c68ab37..a2bd827 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -8,12 +8,8 @@ const BODY_LENGTH_MAX: usize = 2 * 1024; const SCRAPE_TIMEOUT_DEFAULT: std::time::Duration = std::time::Duration::from_secs(10); -const _: () = assert!( - crate::http::EXCHANGE_TIMEOUT - .saturating_mul(crate::http::LISTEN_BACKLOG + 2) - .as_millis() - < SCRAPE_TIMEOUT_DEFAULT.as_millis() -); +const _: () = + assert!(crate::http::EXCHANGE_TIMEOUT.as_millis() < SCRAPE_TIMEOUT_DEFAULT.as_millis()); #[derive(Debug)] pub struct Metrics { @@ -23,6 +19,7 @@ pub struct Metrics { accept_failures: std::cell::Cell, io_failures: std::cell::Cell, timeout_failures: std::cell::Cell, + evicted_failures: std::cell::Cell, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -60,6 +57,7 @@ impl Metrics { accept_failures: std::cell::Cell::new(0), io_failures: std::cell::Cell::new(0), timeout_failures: std::cell::Cell::new(0), + evicted_failures: std::cell::Cell::new(0), } } @@ -102,6 +100,7 @@ impl Metrics { crate::http::Failure::Accept => &self.accept_failures, crate::http::Failure::Io => &self.io_failures, crate::http::Failure::Timeout => &self.timeout_failures, + crate::http::Failure::Evicted => &self.evicted_failures, }; failures.set(failures.get().saturating_add(1)); } @@ -152,12 +151,13 @@ impl Metrics { let mut family = text.family( "elo_metrics_failures_total", Kind::Counter, - "The connections to the metrics that failed: an accept error, a read or write error, or no answer in time.", + "The connections to the metrics that failed: an accept error, a read or write error, no answer in time, or a newer connection in its place.", ); for (kind, failures) in [ ("accept", &self.accept_failures), ("io", &self.io_failures), ("timeout", &self.timeout_failures), + ("evicted", &self.evicted_failures), ] { family.sample_labelled("kind", kind, failures.get()); } @@ -299,7 +299,7 @@ elo_header_tip_work 2 "; const EXPECTED_FAILURES_HEAD: &str = "\ -# HELP elo_metrics_failures_total The connections to the metrics that failed: an accept error, a read or write error, or no answer in time. +# HELP elo_metrics_failures_total The connections to the metrics that failed: an accept error, a read or write error, no answer in time, or a newer connection in its place. # TYPE elo_metrics_failures_total counter "; @@ -324,11 +324,12 @@ elo_header_tip_work 2 metrics } - fn failures(accept: u8, io: u8, timeout: u8) -> String { + fn failures(accept: u8, io: u8, timeout: u8, evicted: u8) -> String { format!( "elo_metrics_failures_total{{kind=\"accept\"}} {accept}\n\ elo_metrics_failures_total{{kind=\"io\"}} {io}\n\ - elo_metrics_failures_total{{kind=\"timeout\"}} {timeout}\n" + elo_metrics_failures_total{{kind=\"timeout\"}} {timeout}\n\ + elo_metrics_failures_total{{kind=\"evicted\"}} {evicted}\n" ) } @@ -344,7 +345,7 @@ elo_header_tip_work 2 let expected = format!( "{EXPECTED_START}{}{EXPECTED_END}elo_reconnects_total 0\n{EXPECTED_FAILURES_HEAD}{}", states(1, 0, 0, 0), - failures(0, 0, 0) + failures(0, 0, 0, 0) ); assert_eq!(metrics().text(), expected); } @@ -384,10 +385,11 @@ elo_header_tip_work 2 fn text_has_each_failure_of_its_kind() { let metrics = metrics(); for (failure, expected) in [ - (crate::http::Failure::Timeout, failures(0, 0, 1)), - (crate::http::Failure::Io, failures(0, 1, 1)), - (crate::http::Failure::Accept, failures(1, 1, 1)), - (crate::http::Failure::Timeout, failures(1, 1, 2)), + (crate::http::Failure::Timeout, failures(0, 0, 1, 0)), + (crate::http::Failure::Io, failures(0, 1, 1, 0)), + (crate::http::Failure::Evicted, failures(0, 1, 1, 1)), + (crate::http::Failure::Accept, failures(1, 1, 1, 1)), + (crate::http::Failure::Timeout, failures(1, 1, 2, 1)), ] { metrics.failed(failure); let text = metrics.text();