From 65f4d7dbc7717e1abc1de100e374ba0bf8e2b63b Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Sun, 15 Nov 2026 16:34:55 -0300 Subject: [PATCH 1/3] feat: count the messages and the bytes in and out of the peer, for each command, in four counters of the metrics (#79) The socket of a connection has the metrics, so each message is counted at the one place where elo reads it and the one place where elo writes it. A received message counts when elo has its full frame and the frame is valid. A sent message counts when its write ends. The bytes include the 24 bytes of the header, as in the per-message bytes of the peer. The label `command` has a fixed set of values: each command that elo knows, and `other` for each other command. So a peer cannot add series. The sent and the received counters have the same values, so that one query can match the two directions. Some sent series stay at 0. The counters do not reset at a reconnect, as a counter must not go down. The longest body is now 4993 bytes, so `BODY_LENGTH_MAX` is 8 KiB. --- src/main.rs | 2 +- src/metrics.rs | 262 +++++++++++++++++++++++++++++++++++++++++++++- src/net.rs | 93 ++++++++++------ tests/bitcoind.rs | 59 ++++++++++- 4 files changed, 377 insertions(+), 39 deletions(-) diff --git a/src/main.rs b/src/main.rs index 6a154b8..76f470b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -255,7 +255,7 @@ async fn dial( chain: &mut elo::chain::Chain, metrics: &elo::metrics::Metrics, ) -> Result { - let (connection, version) = match elo::net::handshake(destination, chain).await { + let (connection, version) = match elo::net::handshake(destination, chain, metrics).await { Ok(handshake) => handshake, Err(error) => return Ok(End::Handshake(error)), }; diff --git a/src/metrics.rs b/src/metrics.rs index a2bd827..286fbc3 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -4,18 +4,37 @@ const PATH: &str = "/metrics"; const CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8"; -const BODY_LENGTH_MAX: usize = 2 * 1024; +const BODY_LENGTH_MAX: usize = 8 * 1024; const SCRAPE_TIMEOUT_DEFAULT: std::time::Duration = std::time::Duration::from_secs(10); const _: () = assert!(crate::http::EXCHANGE_TIMEOUT.as_millis() < SCRAPE_TIMEOUT_DEFAULT.as_millis()); +const COMMANDS: [(crate::protocol::MessageType, &str); 10] = [ + (crate::protocol::VERSION, "version"), + (crate::protocol::VERACK, "verack"), + (crate::protocol::PING, "ping"), + (crate::protocol::PONG, "pong"), + (crate::protocol::GETHEADERS, "getheaders"), + (crate::protocol::HEADERS, "headers"), + (crate::protocol::WTXIDRELAY, "wtxidrelay"), + (crate::protocol::SENDADDRV2, "sendaddrv2"), + (crate::protocol::SENDHEADERS, "sendheaders"), + (crate::protocol::INV, "inv"), +]; + +const COMMAND_OTHER: &str = "other"; + +const COMMANDS_COUNT: usize = COMMANDS.len() + 1; + #[derive(Debug)] pub struct Metrics { tip: std::cell::Cell, peer: std::cell::Cell, reconnects: std::cell::Cell, + received: Traffic, + sent: Traffic, accept_failures: std::cell::Cell, io_failures: std::cell::Cell, timeout_failures: std::cell::Cell, @@ -54,6 +73,8 @@ impl Metrics { tip: std::cell::Cell::new(chain.tip()), peer: std::cell::Cell::new(PeerState::Dial), reconnects: std::cell::Cell::new(0), + received: Traffic::new(), + sent: Traffic::new(), accept_failures: std::cell::Cell::new(0), io_failures: std::cell::Cell::new(0), timeout_failures: std::cell::Cell::new(0), @@ -80,6 +101,14 @@ impl Metrics { self.peer.set(state); } + pub(crate) fn received(&self, msg_type: crate::protocol::MessageType, payload_length: usize) { + self.received.count(msg_type, payload_length); + } + + pub(crate) fn sent(&self, msg_type: crate::protocol::MessageType, payload_length: usize) { + self.sent.count(msg_type, payload_length); + } + pub fn listen( address: std::net::SocketAddr, ) -> Result { @@ -148,6 +177,33 @@ impl Metrics { "The connections that ended, and that elo dials again after a wait.", ) .sample(self.reconnects.get()); + for (name, help, counts) in [ + ( + "elo_peer_received_messages_total", + "The messages from the peer that elo read whole and checked, by command: other for each command that elo does not know.", + &self.received.messages, + ), + ( + "elo_peer_received_bytes_total", + "The bytes of the messages from the peer, with their headers, by command.", + &self.received.bytes, + ), + ( + "elo_peer_sent_messages_total", + "The messages to the peer that elo wrote whole, by command.", + &self.sent.messages, + ), + ( + "elo_peer_sent_bytes_total", + "The bytes of the messages to the peer, with their headers, by command.", + &self.sent.bytes, + ), + ] { + let mut family = text.family(name, Kind::Counter, help); + for (command, count) in command_names().zip(counts) { + family.sample_labelled("command", command, count.get()); + } + } let mut family = text.family( "elo_metrics_failures_total", Kind::Counter, @@ -165,6 +221,41 @@ impl Metrics { } } +#[derive(Debug)] +struct Traffic { + messages: [std::cell::Cell; COMMANDS_COUNT], + bytes: [std::cell::Cell; COMMANDS_COUNT], +} + +impl Traffic { + fn new() -> Traffic { + Traffic { + messages: [const { std::cell::Cell::new(0) }; COMMANDS_COUNT], + bytes: [const { std::cell::Cell::new(0) }; COMMANDS_COUNT], + } + } + + fn count(&self, msg_type: crate::protocol::MessageType, payload_length: usize) { + let index = COMMANDS + .iter() + .position(|(known, _)| *known == msg_type) + .unwrap_or(COMMANDS.len()); + let length = crate::transport::MESSAGE_HEADER_SIZE.saturating_add(payload_length); + let length = u64::try_from(length).unwrap_or(u64::MAX); + let messages = &self.messages[index]; + messages.set(messages.get().saturating_add(1)); + let bytes = &self.bytes[index]; + bytes.set(bytes.get().saturating_add(length)); + } +} + +fn command_names() -> impl Iterator { + COMMANDS + .iter() + .map(|(_, name)| *name) + .chain([COMMAND_OTHER]) +} + #[derive(Clone, Copy, Debug)] enum Kind { Gauge, @@ -303,6 +394,39 @@ elo_header_tip_work 2 # TYPE elo_metrics_failures_total counter "; + const EXPECTED_TRAFFIC_HEADS: [&str; 4] = [ + "\ +# HELP elo_peer_received_messages_total The messages from the peer that elo read whole and checked, by command: other for each command that elo does not know. +# TYPE elo_peer_received_messages_total counter +", + "\ +# HELP elo_peer_received_bytes_total The bytes of the messages from the peer, with their headers, by command. +# TYPE elo_peer_received_bytes_total counter +", + "\ +# HELP elo_peer_sent_messages_total The messages to the peer that elo wrote whole, by command. +# TYPE elo_peer_sent_messages_total counter +", + "\ +# HELP elo_peer_sent_bytes_total The bytes of the messages to the peer, with their headers, by command. +# TYPE elo_peer_sent_bytes_total counter +", + ]; + + const COMMAND_NAMES: [&str; 11] = [ + "version", + "verack", + "ping", + "pong", + "getheaders", + "headers", + "wtxidrelay", + "sendaddrv2", + "sendheaders", + "inv", + "other", + ]; + fn metrics() -> super::Metrics { super::Metrics::new(&crate::chain::Chain::new( crate::chainparams::Network::Regtest, @@ -333,6 +457,26 @@ elo_header_tip_work 2 ) } + fn traffic(family: usize, counts: [u64; 11]) -> String { + let name = [ + "elo_peer_received_messages_total", + "elo_peer_received_bytes_total", + "elo_peer_sent_messages_total", + "elo_peer_sent_bytes_total", + ][family]; + let mut traffic = String::from(EXPECTED_TRAFFIC_HEADS[family]); + for (command, count) in COMMAND_NAMES.into_iter().zip(counts) { + traffic.push_str(&format!("{name}{{command=\"{command}\"}} {count}\n")); + } + traffic + } + + fn message_type(name: &str) -> crate::protocol::MessageType { + let mut bytes = [0; crate::protocol::MESSAGE_TYPE_SIZE]; + bytes[..name.len()].copy_from_slice(name.as_bytes()); + crate::protocol::MessageType::new(bytes).unwrap() + } + fn states(dial: u8, sync: u8, synced: u8, wait: u8) -> String { format!( "elo_peer_state{{state=\"dial\"}} {dial}\nelo_peer_state{{state=\"sync\"}} {sync}\n\ @@ -343,8 +487,12 @@ elo_header_tip_work 2 #[test] fn text_has_the_tip_of_the_chain_the_dial_and_no_reconnect_at_start() { let expected = format!( - "{EXPECTED_START}{}{EXPECTED_END}elo_reconnects_total 0\n{EXPECTED_FAILURES_HEAD}{}", + "{EXPECTED_START}{}{EXPECTED_END}elo_reconnects_total 0\n{}{}{}{}{EXPECTED_FAILURES_HEAD}{}", states(1, 0, 0, 0), + traffic(0, [0; 11]), + traffic(1, [0; 11]), + traffic(2, [0; 11]), + traffic(3, [0; 11]), failures(0, 0, 0, 0) ); assert_eq!(metrics().text(), expected); @@ -381,6 +529,106 @@ elo_header_tip_work 2 } } + #[test] + fn text_has_each_message_and_its_bytes_with_the_header_by_its_command_and_other_for_an_unknown_one() + { + let metrics = metrics(); + let none = [0; 11]; + for (step, received, expected) in [ + ( + (crate::protocol::VERSION, 102), + true, + [ + [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + none, + none, + ], + ), + ( + (message_type("sendcmpct"), 9), + true, + [ + [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], + [126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33], + none, + none, + ], + ), + ( + (message_type("feefilter"), 8), + true, + [ + [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], + [126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65], + none, + none, + ], + ), + ( + (crate::protocol::INV, 37), + true, + [ + [1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2], + [126, 0, 0, 0, 0, 0, 0, 0, 0, 61, 65], + none, + none, + ], + ), + ( + (crate::protocol::VERSION, 0), + false, + [ + [1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2], + [126, 0, 0, 0, 0, 0, 0, 0, 0, 61, 65], + [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ], + ), + ( + (crate::protocol::GETHEADERS, 1029), + false, + [ + [1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2], + [126, 0, 0, 0, 0, 0, 0, 0, 0, 61, 65], + [1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0], + [24, 0, 0, 0, 1053, 0, 0, 0, 0, 0, 0], + ], + ), + ( + (crate::protocol::VERSION, 102), + true, + [ + [2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2], + [252, 0, 0, 0, 0, 0, 0, 0, 0, 61, 65], + [1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0], + [24, 0, 0, 0, 1053, 0, 0, 0, 0, 0, 0], + ], + ), + ] { + let (msg_type, payload_length) = step; + if received { + metrics.received(msg_type, payload_length); + } else { + metrics.sent(msg_type, payload_length); + } + let text = metrics.text(); + let expected: String = (0..4) + .map(|family| traffic(family, expected[family])) + .collect(); + assert!(text.contains(&expected), "{msg_type:?}: {text}"); + } + } + + #[test] + fn name_of_each_known_command_is_its_message_type() { + let names: Vec<&str> = super::command_names().collect(); + assert_eq!(names, COMMAND_NAMES); + for (msg_type, name) in super::COMMANDS { + assert_eq!(message_type(name), msg_type, "{name}"); + } + } + #[test] fn text_has_each_failure_of_its_kind() { let metrics = metrics(); @@ -478,14 +726,18 @@ elo_header_tip_work 2 fn text_whose_values_can_grow_past_the_maximum_is_a_bug() { let text = |samples| { let mut text = super::Text::new(); - let mut family = text.family("elo_a", super::Kind::Gauge, "AAAAAAAAAAAAAAAAA"); + let mut family = text.family( + "elo_a", + super::Kind::Gauge, + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ); for _ in 0..samples { family.sample(0_u64); } text.end() }; - assert_eq!(text(74).len() + 74 * 19, super::BODY_LENGTH_MAX); - let ended = std::panic::catch_unwind(|| text(75)); + assert_eq!(text(301).len() + 301 * 19, super::BODY_LENGTH_MAX); + let ended = std::panic::catch_unwind(|| text(302)); assert!(ended.is_err()); } diff --git a/src/net.rs b/src/net.rs index eef9937..ebe0eb3 100644 --- a/src/net.rs +++ b/src/net.rs @@ -343,23 +343,24 @@ enum ReceiveError { #[derive(Debug)] pub struct Connection<'a> { - socket: Socket, + socket: Socket<'a>, peer: crate::net_processing::Peer<'a>, address: std::net::SocketAddr, } #[derive(Debug)] pub struct Synced<'a> { - socket: Socket, + socket: Socket<'a>, peer: crate::net_processing::Peer<'a>, tip: crate::chain::Tip, } #[derive(Debug)] -struct Socket { +struct Socket<'a> { stream: tokio::net::TcpStream, network: crate::chainparams::Network, frame: PartialFrame, + metrics: &'a crate::metrics::Metrics, } #[derive(Debug)] @@ -378,12 +379,13 @@ enum PartialFrame { pub async fn handshake<'a>( destination: &Destination, chain: &'a mut crate::chain::Chain, + metrics: &'a crate::metrics::Metrics, ) -> Result<(Connection<'a>, crate::protocol::Version), HandshakeError> { let addresses = addresses(destination) .await .map_err(HandshakeError::Lookup)?; let (address, stream) = connect_first(&addresses).await?; - handshake_stream(address, stream, chain).await + handshake_stream(address, stream, chain, metrics).await } async fn addresses(destination: &Destination) -> Result { @@ -489,25 +491,27 @@ async fn connect(address: PeerAddress) -> Result( address: std::net::SocketAddr, stream: tokio::net::TcpStream, - chain: &mut crate::chain::Chain, -) -> Result<(Connection<'_>, crate::protocol::Version), HandshakeError> { - let exchange = handshake_stream_exchange(address, stream, chain); + chain: &'a mut crate::chain::Chain, + metrics: &'a crate::metrics::Metrics, +) -> Result<(Connection<'a>, crate::protocol::Version), HandshakeError> { + let exchange = handshake_stream_exchange(address, stream, chain, metrics); match tokio::time::timeout(HANDSHAKE_TIMEOUT, exchange).await { Ok(result) => result, Err(_elapsed) => Err(HandshakeError::HandshakeTimeout), } } -async fn handshake_stream_exchange( +async fn handshake_stream_exchange<'a>( address: std::net::SocketAddr, stream: tokio::net::TcpStream, - chain: &mut crate::chain::Chain, -) -> Result<(Connection<'_>, crate::protocol::Version), HandshakeError> { + chain: &'a mut crate::chain::Chain, + metrics: &'a crate::metrics::Metrics, +) -> Result<(Connection<'a>, crate::protocol::Version), HandshakeError> { let network = chain.network(); - let mut socket = Socket::new(stream, network); + let mut socket = Socket::new(stream, network, metrics); let work_min = crate::pow::chain_work_min(network); let salt = salt().map_err(HandshakeError::Salt)?; let (mut handshake, outbound) = crate::net_processing::Handshake::start( @@ -589,7 +593,7 @@ impl<'a> Synced<'a> { } async fn run_turn<'a>( - socket: &mut Socket, + socket: &mut Socket<'_>, peer: crate::net_processing::Peer<'a>, ) -> Result<(crate::net_processing::Peer<'a>, Option), RunError> { let now = tokio::time::Instant::now().into_std(); @@ -605,7 +609,7 @@ async fn run_turn<'a>( } async fn run_exchange<'a>( - socket: &mut Socket, + socket: &mut Socket<'_>, peer: crate::net_processing::Peer<'a>, message: &crate::transport::Message, ) -> Result<(crate::net_processing::Peer<'a>, Option), RunError> { @@ -631,7 +635,7 @@ async fn run_exchange<'a>( } async fn run_send( - socket: &mut Socket, + socket: &mut Socket<'_>, outbound: &crate::net_processing::PeerOutbound, ) -> Result<(), RunError> { let send = socket.send_peer_outbound(outbound); @@ -641,12 +645,17 @@ async fn run_send( } } -impl Socket { - fn new(stream: tokio::net::TcpStream, network: crate::chainparams::Network) -> Socket { +impl<'a> Socket<'a> { + fn new( + stream: tokio::net::TcpStream, + network: crate::chainparams::Network, + metrics: &'a crate::metrics::Metrics, + ) -> Socket<'a> { Socket { stream, network, frame: PartialFrame::new(), + metrics, } } @@ -693,7 +702,9 @@ impl Socket { payload: &[u8], ) -> Result<(), std::io::Error> { let frame = crate::transport::encode(self.network, msg_type, payload); - self.stream.write_all(&frame).await + self.stream.write_all(&frame).await?; + self.metrics.sent(msg_type, payload.len()); + Ok(()) } async fn receive(&mut self) -> Result { @@ -704,6 +715,8 @@ impl Socket { .map_err(ReceiveError::Transport)? { assert!(matches!(self.frame, PartialFrame::Header { filled: 0, .. })); + self.metrics + .received(message.msg_type(), message.payload().len()); return Ok(message); } self.read().await?; @@ -1018,6 +1031,7 @@ mod tests { #[test] fn their_version_gets_our_verack() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1036,7 +1050,7 @@ mod tests { verack }); - let result = block_on(super::handshake(&destination(peer), &mut chain)); + let result = block_on(super::handshake(&destination(peer), &mut chain, &metrics)); let verack = peer_thread.join().unwrap(); assert_eq!(verack, crate::fixtures::hex(crate::fixtures::VERACK_FRAME)); assert!(result.is_ok(), "{result:?}"); @@ -1045,6 +1059,7 @@ mod tests { #[test] fn our_version_has_the_local_clock_in_seconds() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1062,7 +1077,7 @@ mod tests { i64::try_from(elapsed.as_secs()).unwrap() }; let before = seconds(); - let _closed = block_on(super::handshake(&destination(peer), &mut chain)); + let _closed = block_on(super::handshake(&destination(peer), &mut chain, &metrics)); let after = seconds(); let message = peer_thread.join().unwrap(); assert_eq!(message.msg_type(), crate::protocol::VERSION); @@ -1077,13 +1092,14 @@ mod tests { #[test] fn silent_peer_is_handshake_timeout() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let (result, elapsed) = block_on(async { let stream = tokio::net::TcpStream::connect(peer).await.unwrap(); tokio::time::pause(); let start = tokio::time::Instant::now(); - let result = super::handshake_stream(peer, stream, &mut chain).await; + let result = super::handshake_stream(peer, stream, &mut chain, &metrics).await; (result, start.elapsed()) }); assert!( @@ -1096,6 +1112,7 @@ mod tests { #[test] fn silent_peer_after_our_getheaders_is_headers_timeout() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1108,7 +1125,7 @@ mod tests { }); let (result, elapsed) = block_on(async { - let (connection, _theirs) = super::handshake(&destination(peer), &mut chain) + let (connection, _theirs) = super::handshake(&destination(peer), &mut chain, &metrics) .await .unwrap(); tokio::time::pause(); @@ -1136,6 +1153,7 @@ mod tests { #[test] fn header_far_after_the_local_clock_ends_the_connection() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1160,7 +1178,7 @@ mod tests { }); let result = block_on(async { - let (connection, _theirs) = super::handshake(&destination(peer), &mut chain) + let (connection, _theirs) = super::handshake(&destination(peer), &mut chain, &metrics) .await .unwrap(); connection.sync().await @@ -1181,6 +1199,7 @@ mod tests { fn sync_that_ends_below_the_minimum_of_the_network_ends_the_connection() { let network = crate::chainparams::Network::Signet; let mut chain = crate::chain::Chain::new(network); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1215,7 +1234,7 @@ mod tests { }); let result = block_on(async { - let (connection, _theirs) = super::handshake(&destination(peer), &mut chain) + let (connection, _theirs) = super::handshake(&destination(peer), &mut chain, &metrics) .await .unwrap(); connection.sync().await @@ -1242,6 +1261,7 @@ mod tests { #[test] fn silent_peer_after_the_sync_is_ping_timeout() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1256,7 +1276,7 @@ mod tests { }); let (error, elapsed) = block_on(async { - let (connection, _theirs) = super::handshake(&destination(peer), &mut chain) + let (connection, _theirs) = super::handshake(&destination(peer), &mut chain, &metrics) .await .unwrap(); let start = tokio::time::Instant::now(); @@ -1284,6 +1304,7 @@ mod tests { #[test] fn empty_announcement_after_the_sync_does_not_give_the_tip_again() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let headers = crate::fixtures::headers_on_genesis(1); let announcement = crate::transport::encode( crate::chainparams::Network::Regtest, @@ -1307,7 +1328,7 @@ mod tests { }); let (synced_tip, next_tip) = block_on(async { - let (connection, _theirs) = super::handshake(&destination(peer), &mut chain) + let (connection, _theirs) = super::handshake(&destination(peer), &mut chain, &metrics) .await .unwrap(); let (synced, synced_tip) = connection.sync().await.unwrap(); @@ -1324,6 +1345,7 @@ mod tests { #[test] fn pong_moves_the_next_ping_to_ping_interval_after_the_last() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1345,7 +1367,7 @@ mod tests { }); let (error, elapsed) = block_on(async { - let (connection, _theirs) = super::handshake(&destination(peer), &mut chain) + let (connection, _theirs) = super::handshake(&destination(peer), &mut chain, &metrics) .await .unwrap(); let start = tokio::time::Instant::now(); @@ -1375,12 +1397,13 @@ mod tests { #[test] fn peer_that_stops_reading_is_send_timeout() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || accept_handshake(&listener)); let (result, elapsed) = block_on(async { - let (connection, _theirs) = super::handshake(&destination(peer), &mut chain) + let (connection, _theirs) = super::handshake(&destination(peer), &mut chain, &metrics) .await .unwrap(); let _stream = peer_thread.join().unwrap(); @@ -1409,6 +1432,7 @@ mod tests { #[test] fn peer_that_closes_inside_a_frame_is_closed() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1418,7 +1442,7 @@ mod tests { stream.write_all(&frame[..frame.len() - 1]).unwrap(); }); - let result = block_on(super::handshake(&destination(peer), &mut chain)); + let result = block_on(super::handshake(&destination(peer), &mut chain, &metrics)); peer_thread.join().unwrap(); assert!( matches!(result, Err(super::HandshakeError::Closed)), @@ -1428,6 +1452,8 @@ mod tests { #[test] fn frame_split_by_a_timeout_is_received_whole() { + let chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let (first_sent, first_sent_receiver) = std::sync::mpsc::channel(); @@ -1444,7 +1470,8 @@ mod tests { let (timeout, message) = block_on(async { let stream = tokio::net::TcpStream::connect(peer).await.unwrap(); - let mut socket = super::Socket::new(stream, crate::chainparams::Network::Regtest); + let mut socket = + super::Socket::new(stream, crate::chainparams::Network::Regtest, &metrics); first_sent_receiver.recv().unwrap(); tokio::time::pause(); let receive = socket.receive(); @@ -1463,6 +1490,7 @@ mod tests { #[test] fn ping_in_one_write_with_their_verack_gets_a_pong() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1484,7 +1512,7 @@ mod tests { }); let result = block_on(async { - let (connection, _theirs) = super::handshake(&destination(peer), &mut chain) + let (connection, _theirs) = super::handshake(&destination(peer), &mut chain, &metrics) .await .unwrap(); connection.sync().await @@ -1502,11 +1530,12 @@ mod tests { #[test] fn refused_connect_is_connect_error() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let peer = std::net::TcpListener::bind("127.0.0.1:0") .unwrap() .local_addr() .unwrap(); - let result = block_on(super::handshake(&destination(peer), &mut chain)); + let result = block_on(super::handshake(&destination(peer), &mut chain, &metrics)); let Err(error) = result else { panic!("{result:?}"); }; diff --git a/tests/bitcoind.rs b/tests/bitcoind.rs index 44031d0..06aea8d 100644 --- a/tests/bitcoind.rs +++ b/tests/bitcoind.rs @@ -128,10 +128,12 @@ fn handshake_with_bitcoind_returns_its_version() { .build() .unwrap(); let mut chain = elo::chain::Chain::new(elo::chainparams::Network::Regtest); + let metrics = elo::metrics::Metrics::new(&chain); let (_connection, version) = runtime .block_on(elo::net::handshake( &elo::net::Destination::Address(elo::net::PeerAddress::new(bitcoind.peer).unwrap()), &mut chain, + &metrics, )) .unwrap(); @@ -591,7 +593,7 @@ fn announcement_before_the_answer_to_our_getheaders_keeps_the_connection() { } #[test] -fn metrics_of_elo_have_its_tip_its_step_and_each_reconnect() { +fn metrics_of_elo_have_its_tip_its_step_each_reconnect_and_the_bytes_of_each_command() { let bitcoind = Bitcoind::start(); bitcoind.cli(&["generatetodescriptor", "2001", "raw(51)"]); let best = bitcoind.cli(&["getbestblockhash"]); @@ -618,6 +620,36 @@ fn metrics_of_elo_have_its_tip_its_step_and_each_reconnect() { } assert!(metrics.contains("elo_reconnects_total 0\n"), "{metrics}"); + settle(&bitcoind, &elo); + let metrics = scrape(address); + for (direction, field) in [ + ("received", "bytessent_per_msg"), + ("sent", "bytesrecv_per_msg"), + ] { + let mut expected = [0; COMMANDS.len() + 1]; + for (command, bytes) in per_message_bytes(&bitcoind, field) { + let index = COMMANDS + .iter() + .position(|known| *known == command) + .unwrap_or(COMMANDS.len()); + expected[index] += bytes; + } + for (command, bytes) in COMMANDS.iter().chain(&["other"]).zip(expected) { + let line = + format!("elo_peer_{direction}_bytes_total{{command=\"{command}\"}} {bytes}\n"); + assert!(metrics.contains(&line), "{line}: {metrics}"); + } + } + for line in [ + "elo_peer_received_messages_total{command=\"version\"} 1\n", + "elo_peer_received_messages_total{command=\"verack\"} 1\n", + "elo_peer_sent_messages_total{command=\"version\"} 1\n", + "elo_peer_sent_messages_total{command=\"verack\"} 1\n", + "elo_peer_sent_messages_total{command=\"sendheaders\"} 1\n", + ] { + assert!(metrics.contains(line), "{line}: {metrics}"); + } + bitcoind.cli(&["-named", "disconnectnode", "nodeid=0"]); elo.event("end", END_TIMEOUT); let metrics = scrape(address); @@ -703,6 +735,31 @@ fn pong_from(bitcoind: &Bitcoind, elo: &Elo, ends: usize) { assert_eq!(all.len(), ends, "elo ended the connection: {all:?}"); } +const COMMANDS: [&str; 10] = [ + "version", + "verack", + "ping", + "pong", + "getheaders", + "headers", + "wtxidrelay", + "sendaddrv2", + "sendheaders", + "inv", +]; + +fn per_message_bytes(bitcoind: &Bitcoind, field: &str) -> Vec<(String, u64)> { + let peer_info = bitcoind.cli(&["getpeerinfo"]); + object_text(&peer_info, field) + .split(',') + .map(|pair| { + let (message, bytes) = pair.split_once(':').unwrap(); + let message = message.trim().trim_matches('"'); + (message.to_owned(), bytes.trim().parse().unwrap()) + }) + .collect() +} + fn message_bytes(bitcoind: &Bitcoind, field: &str, message: &str) -> u64 { let peer_info = bitcoind.cli(&["getpeerinfo"]); let key = format!("\"{message}\": "); From ff237c70835e2a36180abd77951966e108944571 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Sun, 15 Nov 2026 16:35:16 -0300 Subject: [PATCH 2/3] docs: say what the four counters of the messages count, and their `command` values (#79) --- README.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a3b93ee..08b4558 100644 --- a/README.md +++ b/README.md @@ -128,15 +128,22 @@ container. Port 0 takes a free port. elo logs the address in a `metrics` line be | `elo_header_tip_work` | gauge | The work of the chain up to the tip, in decimal | | `elo_peer_state` | gauge | 1 for the step of the connection, and 0 for each other. The `state` label is `dial`, `sync`, `synced` or `wait` | | `elo_reconnects_total` | counter | The connections that ended, and that elo dials again after the wait | +| `elo_peer_received_messages_total` | counter | The messages from the peer, for each command. The `command` label is `version`, `verack`, `ping`, `pong`, `getheaders`, `headers`, `wtxidrelay`, `sendaddrv2`, `sendheaders`, `inv`, or `other` for each other command | +| `elo_peer_received_bytes_total` | counter | The bytes of the messages from the peer, with their headers, for each command | +| `elo_peer_sent_messages_total` | counter | The messages to the peer, for each command | +| `elo_peer_sent_bytes_total` | counter | The bytes of the messages to the peer, with their headers, for each command | | `elo_metrics_failures_total` | counter | The connections to the metrics that failed. The `kind` label is `accept` for an accept error, `io` for a read or write error, `timeout` for no answer in 1 s, or `evicted` for 8 newer connections before the answer | -The tip moves when the `tip` line does: at the end of a sync, and then with each new tip. As for the -log, the names are an interface, and `src/metrics.rs` holds them. elo accepts each connection when -it comes and answers at most 8 at the same time, each in at most 1 s. It closes each connection after -its answer, and also closes a connection when 8 newer connections come before its answer, so idle -clients cannot hold a scrape. A scrape gets its answer in at most 1 s, unless 8 newer connections -come before its answer. The scrape timeout of Prometheus is 10 s by default. A request -head is at most 8 KiB. Another path gets 404, and another method gets 405. +The tip moves when the `tip` line does: at the end of a sync, and then with each new tip. A received +message counts when elo has all of its frame and the frame is valid, and a sent message counts when +its write ends. The four counters of the messages have the same `command` values, also for the +commands that elo does not send. They count each connection, and do not go back to 0 at a reconnect. +As for the log, the names are an interface, and `src/metrics.rs` holds them. elo accepts each +connection when it comes and answers at most 8 at the same time, each in at most 1 s. It closes each +connection after its answer, and also closes a connection when 8 newer connections come before its +answer, so idle clients cannot hold a scrape. A scrape gets its answer in at most 1 s, unless 8 +newer connections come before its answer. The scrape timeout of Prometheus is 10 s by default. A +request head is at most 8 KiB. Another path gets 404, and another method gets 405. ## Build From e8eea7d6bb7e2f274e61d1ace6719821d37334e7 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Sun, 15 Nov 2026 16:35:36 -0300 Subject: [PATCH 3/3] docs: say that a move into `wait` after a failed lookup, dial or handshake is a reconnect too, and that Prometheus rounds the work (#79) --- README.md | 4 ++-- src/metrics.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 08b4558..59ddd58 100644 --- a/README.md +++ b/README.md @@ -125,9 +125,9 @@ container. Port 0 takes a free port. elo logs the address in a `metrics` line be | Metric | Type | Value | | --- | --- | --- | | `elo_header_tip_height` | gauge | The height of the tip of the header chain | -| `elo_header_tip_work` | gauge | The work of the chain up to the tip, in decimal | +| `elo_header_tip_work` | gauge | The work of the chain up to the tip, in decimal with all its digits. Prometheus keeps each value as a 64-bit float, so it rounds a work above 2^53 | | `elo_peer_state` | gauge | 1 for the step of the connection, and 0 for each other. The `state` label is `dial`, `sync`, `synced` or `wait` | -| `elo_reconnects_total` | counter | The connections that ended, and that elo dials again after the wait | +| `elo_reconnects_total` | counter | Each failed lookup, dial or handshake and each connection that ended, after which elo waits and dials again | | `elo_peer_received_messages_total` | counter | The messages from the peer, for each command. The `command` label is `version`, `verack`, `ping`, `pong`, `getheaders`, `headers`, `wtxidrelay`, `sendaddrv2`, `sendheaders`, `inv`, or `other` for each other command | | `elo_peer_received_bytes_total` | counter | The bytes of the messages from the peer, with their headers, for each command | | `elo_peer_sent_messages_total` | counter | The messages to the peer, for each command | diff --git a/src/metrics.rs b/src/metrics.rs index 286fbc3..c5f7e7f 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -174,7 +174,7 @@ impl Metrics { text.family( "elo_reconnects_total", Kind::Counter, - "The connections that ended, and that elo dials again after a wait.", + "Each failed lookup, dial or handshake and each connection that ended, after which elo waits and dials again.", ) .sample(self.reconnects.get()); for (name, help, counts) in [ @@ -385,7 +385,7 @@ elo_header_tip_work 2 "; const EXPECTED_END: &str = "\ -# HELP elo_reconnects_total The connections that ended, and that elo dials again after a wait. +# HELP elo_reconnects_total Each failed lookup, dial or handshake and each connection that ended, after which elo waits and dials again. # TYPE elo_reconnects_total counter ";