diff --git a/README.md b/README.md index a3b93ee..59ddd58 100644 --- a/README.md +++ b/README.md @@ -125,18 +125,25 @@ container. Port 0 takes a free port. elo logs the address in a `metrics` line be | Metric | Type | Value | | --- | --- | --- | | `elo_header_tip_height` | gauge | The height of the tip of the header chain | -| `elo_header_tip_work` | gauge | The work of the chain up to the tip, in decimal | +| `elo_header_tip_work` | gauge | The work of the chain up to the tip, in decimal with all its digits. Prometheus keeps each value as a 64-bit float, so it rounds a work above 2^53 | | `elo_peer_state` | gauge | 1 for the step of the connection, and 0 for each other. The `state` label is `dial`, `sync`, `synced` or `wait` | -| `elo_reconnects_total` | counter | The connections that ended, and that elo dials again after the wait | +| `elo_reconnects_total` | counter | Each failed lookup, dial or handshake and each connection that ended, after which elo waits and dials again | +| `elo_peer_received_messages_total` | counter | The messages from the peer, for each command. The `command` label is `version`, `verack`, `ping`, `pong`, `getheaders`, `headers`, `wtxidrelay`, `sendaddrv2`, `sendheaders`, `inv`, or `other` for each other command | +| `elo_peer_received_bytes_total` | counter | The bytes of the messages from the peer, with their headers, for each command | +| `elo_peer_sent_messages_total` | counter | The messages to the peer, for each command | +| `elo_peer_sent_bytes_total` | counter | The bytes of the messages to the peer, with their headers, for each command | | `elo_metrics_failures_total` | counter | The connections to the metrics that failed. The `kind` label is `accept` for an accept error, `io` for a read or write error, `timeout` for no answer in 1 s, or `evicted` for 8 newer connections before the answer | -The tip moves when the `tip` line does: at the end of a sync, and then with each new tip. As for the -log, the names are an interface, and `src/metrics.rs` holds them. elo accepts each connection when -it comes and answers at most 8 at the same time, each in at most 1 s. It closes each connection after -its answer, and also closes a connection when 8 newer connections come before its answer, so idle -clients cannot hold a scrape. A scrape gets its answer in at most 1 s, unless 8 newer connections -come before its answer. The scrape timeout of Prometheus is 10 s by default. A request -head is at most 8 KiB. Another path gets 404, and another method gets 405. +The tip moves when the `tip` line does: at the end of a sync, and then with each new tip. A received +message counts when elo has all of its frame and the frame is valid, and a sent message counts when +its write ends. The four counters of the messages have the same `command` values, also for the +commands that elo does not send. They count each connection, and do not go back to 0 at a reconnect. +As for the log, the names are an interface, and `src/metrics.rs` holds them. elo accepts each +connection when it comes and answers at most 8 at the same time, each in at most 1 s. It closes each +connection after its answer, and also closes a connection when 8 newer connections come before its +answer, so idle clients cannot hold a scrape. A scrape gets its answer in at most 1 s, unless 8 +newer connections come before its answer. The scrape timeout of Prometheus is 10 s by default. A +request head is at most 8 KiB. Another path gets 404, and another method gets 405. ## Build diff --git a/src/main.rs b/src/main.rs index 6a154b8..76f470b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -255,7 +255,7 @@ async fn dial( chain: &mut elo::chain::Chain, metrics: &elo::metrics::Metrics, ) -> Result { - let (connection, version) = match elo::net::handshake(destination, chain).await { + let (connection, version) = match elo::net::handshake(destination, chain, metrics).await { Ok(handshake) => handshake, Err(error) => return Ok(End::Handshake(error)), }; diff --git a/src/metrics.rs b/src/metrics.rs index a2bd827..c5f7e7f 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -4,18 +4,37 @@ const PATH: &str = "/metrics"; const CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8"; -const BODY_LENGTH_MAX: usize = 2 * 1024; +const BODY_LENGTH_MAX: usize = 8 * 1024; const SCRAPE_TIMEOUT_DEFAULT: std::time::Duration = std::time::Duration::from_secs(10); const _: () = assert!(crate::http::EXCHANGE_TIMEOUT.as_millis() < SCRAPE_TIMEOUT_DEFAULT.as_millis()); +const COMMANDS: [(crate::protocol::MessageType, &str); 10] = [ + (crate::protocol::VERSION, "version"), + (crate::protocol::VERACK, "verack"), + (crate::protocol::PING, "ping"), + (crate::protocol::PONG, "pong"), + (crate::protocol::GETHEADERS, "getheaders"), + (crate::protocol::HEADERS, "headers"), + (crate::protocol::WTXIDRELAY, "wtxidrelay"), + (crate::protocol::SENDADDRV2, "sendaddrv2"), + (crate::protocol::SENDHEADERS, "sendheaders"), + (crate::protocol::INV, "inv"), +]; + +const COMMAND_OTHER: &str = "other"; + +const COMMANDS_COUNT: usize = COMMANDS.len() + 1; + #[derive(Debug)] pub struct Metrics { tip: std::cell::Cell, peer: std::cell::Cell, reconnects: std::cell::Cell, + received: Traffic, + sent: Traffic, accept_failures: std::cell::Cell, io_failures: std::cell::Cell, timeout_failures: std::cell::Cell, @@ -54,6 +73,8 @@ impl Metrics { tip: std::cell::Cell::new(chain.tip()), peer: std::cell::Cell::new(PeerState::Dial), reconnects: std::cell::Cell::new(0), + received: Traffic::new(), + sent: Traffic::new(), accept_failures: std::cell::Cell::new(0), io_failures: std::cell::Cell::new(0), timeout_failures: std::cell::Cell::new(0), @@ -80,6 +101,14 @@ impl Metrics { self.peer.set(state); } + pub(crate) fn received(&self, msg_type: crate::protocol::MessageType, payload_length: usize) { + self.received.count(msg_type, payload_length); + } + + pub(crate) fn sent(&self, msg_type: crate::protocol::MessageType, payload_length: usize) { + self.sent.count(msg_type, payload_length); + } + pub fn listen( address: std::net::SocketAddr, ) -> Result { @@ -145,9 +174,36 @@ impl Metrics { text.family( "elo_reconnects_total", Kind::Counter, - "The connections that ended, and that elo dials again after a wait.", + "Each failed lookup, dial or handshake and each connection that ended, after which elo waits and dials again.", ) .sample(self.reconnects.get()); + for (name, help, counts) in [ + ( + "elo_peer_received_messages_total", + "The messages from the peer that elo read whole and checked, by command: other for each command that elo does not know.", + &self.received.messages, + ), + ( + "elo_peer_received_bytes_total", + "The bytes of the messages from the peer, with their headers, by command.", + &self.received.bytes, + ), + ( + "elo_peer_sent_messages_total", + "The messages to the peer that elo wrote whole, by command.", + &self.sent.messages, + ), + ( + "elo_peer_sent_bytes_total", + "The bytes of the messages to the peer, with their headers, by command.", + &self.sent.bytes, + ), + ] { + let mut family = text.family(name, Kind::Counter, help); + for (command, count) in command_names().zip(counts) { + family.sample_labelled("command", command, count.get()); + } + } let mut family = text.family( "elo_metrics_failures_total", Kind::Counter, @@ -165,6 +221,41 @@ impl Metrics { } } +#[derive(Debug)] +struct Traffic { + messages: [std::cell::Cell; COMMANDS_COUNT], + bytes: [std::cell::Cell; COMMANDS_COUNT], +} + +impl Traffic { + fn new() -> Traffic { + Traffic { + messages: [const { std::cell::Cell::new(0) }; COMMANDS_COUNT], + bytes: [const { std::cell::Cell::new(0) }; COMMANDS_COUNT], + } + } + + fn count(&self, msg_type: crate::protocol::MessageType, payload_length: usize) { + let index = COMMANDS + .iter() + .position(|(known, _)| *known == msg_type) + .unwrap_or(COMMANDS.len()); + let length = crate::transport::MESSAGE_HEADER_SIZE.saturating_add(payload_length); + let length = u64::try_from(length).unwrap_or(u64::MAX); + let messages = &self.messages[index]; + messages.set(messages.get().saturating_add(1)); + let bytes = &self.bytes[index]; + bytes.set(bytes.get().saturating_add(length)); + } +} + +fn command_names() -> impl Iterator { + COMMANDS + .iter() + .map(|(_, name)| *name) + .chain([COMMAND_OTHER]) +} + #[derive(Clone, Copy, Debug)] enum Kind { Gauge, @@ -294,7 +385,7 @@ elo_header_tip_work 2 "; const EXPECTED_END: &str = "\ -# HELP elo_reconnects_total The connections that ended, and that elo dials again after a wait. +# HELP elo_reconnects_total Each failed lookup, dial or handshake and each connection that ended, after which elo waits and dials again. # TYPE elo_reconnects_total counter "; @@ -303,6 +394,39 @@ elo_header_tip_work 2 # TYPE elo_metrics_failures_total counter "; + const EXPECTED_TRAFFIC_HEADS: [&str; 4] = [ + "\ +# HELP elo_peer_received_messages_total The messages from the peer that elo read whole and checked, by command: other for each command that elo does not know. +# TYPE elo_peer_received_messages_total counter +", + "\ +# HELP elo_peer_received_bytes_total The bytes of the messages from the peer, with their headers, by command. +# TYPE elo_peer_received_bytes_total counter +", + "\ +# HELP elo_peer_sent_messages_total The messages to the peer that elo wrote whole, by command. +# TYPE elo_peer_sent_messages_total counter +", + "\ +# HELP elo_peer_sent_bytes_total The bytes of the messages to the peer, with their headers, by command. +# TYPE elo_peer_sent_bytes_total counter +", + ]; + + const COMMAND_NAMES: [&str; 11] = [ + "version", + "verack", + "ping", + "pong", + "getheaders", + "headers", + "wtxidrelay", + "sendaddrv2", + "sendheaders", + "inv", + "other", + ]; + fn metrics() -> super::Metrics { super::Metrics::new(&crate::chain::Chain::new( crate::chainparams::Network::Regtest, @@ -333,6 +457,26 @@ elo_header_tip_work 2 ) } + fn traffic(family: usize, counts: [u64; 11]) -> String { + let name = [ + "elo_peer_received_messages_total", + "elo_peer_received_bytes_total", + "elo_peer_sent_messages_total", + "elo_peer_sent_bytes_total", + ][family]; + let mut traffic = String::from(EXPECTED_TRAFFIC_HEADS[family]); + for (command, count) in COMMAND_NAMES.into_iter().zip(counts) { + traffic.push_str(&format!("{name}{{command=\"{command}\"}} {count}\n")); + } + traffic + } + + fn message_type(name: &str) -> crate::protocol::MessageType { + let mut bytes = [0; crate::protocol::MESSAGE_TYPE_SIZE]; + bytes[..name.len()].copy_from_slice(name.as_bytes()); + crate::protocol::MessageType::new(bytes).unwrap() + } + fn states(dial: u8, sync: u8, synced: u8, wait: u8) -> String { format!( "elo_peer_state{{state=\"dial\"}} {dial}\nelo_peer_state{{state=\"sync\"}} {sync}\n\ @@ -343,8 +487,12 @@ elo_header_tip_work 2 #[test] fn text_has_the_tip_of_the_chain_the_dial_and_no_reconnect_at_start() { let expected = format!( - "{EXPECTED_START}{}{EXPECTED_END}elo_reconnects_total 0\n{EXPECTED_FAILURES_HEAD}{}", + "{EXPECTED_START}{}{EXPECTED_END}elo_reconnects_total 0\n{}{}{}{}{EXPECTED_FAILURES_HEAD}{}", states(1, 0, 0, 0), + traffic(0, [0; 11]), + traffic(1, [0; 11]), + traffic(2, [0; 11]), + traffic(3, [0; 11]), failures(0, 0, 0, 0) ); assert_eq!(metrics().text(), expected); @@ -381,6 +529,106 @@ elo_header_tip_work 2 } } + #[test] + fn text_has_each_message_and_its_bytes_with_the_header_by_its_command_and_other_for_an_unknown_one() + { + let metrics = metrics(); + let none = [0; 11]; + for (step, received, expected) in [ + ( + (crate::protocol::VERSION, 102), + true, + [ + [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + none, + none, + ], + ), + ( + (message_type("sendcmpct"), 9), + true, + [ + [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], + [126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33], + none, + none, + ], + ), + ( + (message_type("feefilter"), 8), + true, + [ + [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], + [126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65], + none, + none, + ], + ), + ( + (crate::protocol::INV, 37), + true, + [ + [1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2], + [126, 0, 0, 0, 0, 0, 0, 0, 0, 61, 65], + none, + none, + ], + ), + ( + (crate::protocol::VERSION, 0), + false, + [ + [1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2], + [126, 0, 0, 0, 0, 0, 0, 0, 0, 61, 65], + [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ], + ), + ( + (crate::protocol::GETHEADERS, 1029), + false, + [ + [1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2], + [126, 0, 0, 0, 0, 0, 0, 0, 0, 61, 65], + [1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0], + [24, 0, 0, 0, 1053, 0, 0, 0, 0, 0, 0], + ], + ), + ( + (crate::protocol::VERSION, 102), + true, + [ + [2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2], + [252, 0, 0, 0, 0, 0, 0, 0, 0, 61, 65], + [1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0], + [24, 0, 0, 0, 1053, 0, 0, 0, 0, 0, 0], + ], + ), + ] { + let (msg_type, payload_length) = step; + if received { + metrics.received(msg_type, payload_length); + } else { + metrics.sent(msg_type, payload_length); + } + let text = metrics.text(); + let expected: String = (0..4) + .map(|family| traffic(family, expected[family])) + .collect(); + assert!(text.contains(&expected), "{msg_type:?}: {text}"); + } + } + + #[test] + fn name_of_each_known_command_is_its_message_type() { + let names: Vec<&str> = super::command_names().collect(); + assert_eq!(names, COMMAND_NAMES); + for (msg_type, name) in super::COMMANDS { + assert_eq!(message_type(name), msg_type, "{name}"); + } + } + #[test] fn text_has_each_failure_of_its_kind() { let metrics = metrics(); @@ -478,14 +726,18 @@ elo_header_tip_work 2 fn text_whose_values_can_grow_past_the_maximum_is_a_bug() { let text = |samples| { let mut text = super::Text::new(); - let mut family = text.family("elo_a", super::Kind::Gauge, "AAAAAAAAAAAAAAAAA"); + let mut family = text.family( + "elo_a", + super::Kind::Gauge, + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ); for _ in 0..samples { family.sample(0_u64); } text.end() }; - assert_eq!(text(74).len() + 74 * 19, super::BODY_LENGTH_MAX); - let ended = std::panic::catch_unwind(|| text(75)); + assert_eq!(text(301).len() + 301 * 19, super::BODY_LENGTH_MAX); + let ended = std::panic::catch_unwind(|| text(302)); assert!(ended.is_err()); } diff --git a/src/net.rs b/src/net.rs index eef9937..ebe0eb3 100644 --- a/src/net.rs +++ b/src/net.rs @@ -343,23 +343,24 @@ enum ReceiveError { #[derive(Debug)] pub struct Connection<'a> { - socket: Socket, + socket: Socket<'a>, peer: crate::net_processing::Peer<'a>, address: std::net::SocketAddr, } #[derive(Debug)] pub struct Synced<'a> { - socket: Socket, + socket: Socket<'a>, peer: crate::net_processing::Peer<'a>, tip: crate::chain::Tip, } #[derive(Debug)] -struct Socket { +struct Socket<'a> { stream: tokio::net::TcpStream, network: crate::chainparams::Network, frame: PartialFrame, + metrics: &'a crate::metrics::Metrics, } #[derive(Debug)] @@ -378,12 +379,13 @@ enum PartialFrame { pub async fn handshake<'a>( destination: &Destination, chain: &'a mut crate::chain::Chain, + metrics: &'a crate::metrics::Metrics, ) -> Result<(Connection<'a>, crate::protocol::Version), HandshakeError> { let addresses = addresses(destination) .await .map_err(HandshakeError::Lookup)?; let (address, stream) = connect_first(&addresses).await?; - handshake_stream(address, stream, chain).await + handshake_stream(address, stream, chain, metrics).await } async fn addresses(destination: &Destination) -> Result { @@ -489,25 +491,27 @@ async fn connect(address: PeerAddress) -> Result( address: std::net::SocketAddr, stream: tokio::net::TcpStream, - chain: &mut crate::chain::Chain, -) -> Result<(Connection<'_>, crate::protocol::Version), HandshakeError> { - let exchange = handshake_stream_exchange(address, stream, chain); + chain: &'a mut crate::chain::Chain, + metrics: &'a crate::metrics::Metrics, +) -> Result<(Connection<'a>, crate::protocol::Version), HandshakeError> { + let exchange = handshake_stream_exchange(address, stream, chain, metrics); match tokio::time::timeout(HANDSHAKE_TIMEOUT, exchange).await { Ok(result) => result, Err(_elapsed) => Err(HandshakeError::HandshakeTimeout), } } -async fn handshake_stream_exchange( +async fn handshake_stream_exchange<'a>( address: std::net::SocketAddr, stream: tokio::net::TcpStream, - chain: &mut crate::chain::Chain, -) -> Result<(Connection<'_>, crate::protocol::Version), HandshakeError> { + chain: &'a mut crate::chain::Chain, + metrics: &'a crate::metrics::Metrics, +) -> Result<(Connection<'a>, crate::protocol::Version), HandshakeError> { let network = chain.network(); - let mut socket = Socket::new(stream, network); + let mut socket = Socket::new(stream, network, metrics); let work_min = crate::pow::chain_work_min(network); let salt = salt().map_err(HandshakeError::Salt)?; let (mut handshake, outbound) = crate::net_processing::Handshake::start( @@ -589,7 +593,7 @@ impl<'a> Synced<'a> { } async fn run_turn<'a>( - socket: &mut Socket, + socket: &mut Socket<'_>, peer: crate::net_processing::Peer<'a>, ) -> Result<(crate::net_processing::Peer<'a>, Option), RunError> { let now = tokio::time::Instant::now().into_std(); @@ -605,7 +609,7 @@ async fn run_turn<'a>( } async fn run_exchange<'a>( - socket: &mut Socket, + socket: &mut Socket<'_>, peer: crate::net_processing::Peer<'a>, message: &crate::transport::Message, ) -> Result<(crate::net_processing::Peer<'a>, Option), RunError> { @@ -631,7 +635,7 @@ async fn run_exchange<'a>( } async fn run_send( - socket: &mut Socket, + socket: &mut Socket<'_>, outbound: &crate::net_processing::PeerOutbound, ) -> Result<(), RunError> { let send = socket.send_peer_outbound(outbound); @@ -641,12 +645,17 @@ async fn run_send( } } -impl Socket { - fn new(stream: tokio::net::TcpStream, network: crate::chainparams::Network) -> Socket { +impl<'a> Socket<'a> { + fn new( + stream: tokio::net::TcpStream, + network: crate::chainparams::Network, + metrics: &'a crate::metrics::Metrics, + ) -> Socket<'a> { Socket { stream, network, frame: PartialFrame::new(), + metrics, } } @@ -693,7 +702,9 @@ impl Socket { payload: &[u8], ) -> Result<(), std::io::Error> { let frame = crate::transport::encode(self.network, msg_type, payload); - self.stream.write_all(&frame).await + self.stream.write_all(&frame).await?; + self.metrics.sent(msg_type, payload.len()); + Ok(()) } async fn receive(&mut self) -> Result { @@ -704,6 +715,8 @@ impl Socket { .map_err(ReceiveError::Transport)? { assert!(matches!(self.frame, PartialFrame::Header { filled: 0, .. })); + self.metrics + .received(message.msg_type(), message.payload().len()); return Ok(message); } self.read().await?; @@ -1018,6 +1031,7 @@ mod tests { #[test] fn their_version_gets_our_verack() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1036,7 +1050,7 @@ mod tests { verack }); - let result = block_on(super::handshake(&destination(peer), &mut chain)); + let result = block_on(super::handshake(&destination(peer), &mut chain, &metrics)); let verack = peer_thread.join().unwrap(); assert_eq!(verack, crate::fixtures::hex(crate::fixtures::VERACK_FRAME)); assert!(result.is_ok(), "{result:?}"); @@ -1045,6 +1059,7 @@ mod tests { #[test] fn our_version_has_the_local_clock_in_seconds() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1062,7 +1077,7 @@ mod tests { i64::try_from(elapsed.as_secs()).unwrap() }; let before = seconds(); - let _closed = block_on(super::handshake(&destination(peer), &mut chain)); + let _closed = block_on(super::handshake(&destination(peer), &mut chain, &metrics)); let after = seconds(); let message = peer_thread.join().unwrap(); assert_eq!(message.msg_type(), crate::protocol::VERSION); @@ -1077,13 +1092,14 @@ mod tests { #[test] fn silent_peer_is_handshake_timeout() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let (result, elapsed) = block_on(async { let stream = tokio::net::TcpStream::connect(peer).await.unwrap(); tokio::time::pause(); let start = tokio::time::Instant::now(); - let result = super::handshake_stream(peer, stream, &mut chain).await; + let result = super::handshake_stream(peer, stream, &mut chain, &metrics).await; (result, start.elapsed()) }); assert!( @@ -1096,6 +1112,7 @@ mod tests { #[test] fn silent_peer_after_our_getheaders_is_headers_timeout() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1108,7 +1125,7 @@ mod tests { }); let (result, elapsed) = block_on(async { - let (connection, _theirs) = super::handshake(&destination(peer), &mut chain) + let (connection, _theirs) = super::handshake(&destination(peer), &mut chain, &metrics) .await .unwrap(); tokio::time::pause(); @@ -1136,6 +1153,7 @@ mod tests { #[test] fn header_far_after_the_local_clock_ends_the_connection() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1160,7 +1178,7 @@ mod tests { }); let result = block_on(async { - let (connection, _theirs) = super::handshake(&destination(peer), &mut chain) + let (connection, _theirs) = super::handshake(&destination(peer), &mut chain, &metrics) .await .unwrap(); connection.sync().await @@ -1181,6 +1199,7 @@ mod tests { fn sync_that_ends_below_the_minimum_of_the_network_ends_the_connection() { let network = crate::chainparams::Network::Signet; let mut chain = crate::chain::Chain::new(network); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1215,7 +1234,7 @@ mod tests { }); let result = block_on(async { - let (connection, _theirs) = super::handshake(&destination(peer), &mut chain) + let (connection, _theirs) = super::handshake(&destination(peer), &mut chain, &metrics) .await .unwrap(); connection.sync().await @@ -1242,6 +1261,7 @@ mod tests { #[test] fn silent_peer_after_the_sync_is_ping_timeout() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1256,7 +1276,7 @@ mod tests { }); let (error, elapsed) = block_on(async { - let (connection, _theirs) = super::handshake(&destination(peer), &mut chain) + let (connection, _theirs) = super::handshake(&destination(peer), &mut chain, &metrics) .await .unwrap(); let start = tokio::time::Instant::now(); @@ -1284,6 +1304,7 @@ mod tests { #[test] fn empty_announcement_after_the_sync_does_not_give_the_tip_again() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let headers = crate::fixtures::headers_on_genesis(1); let announcement = crate::transport::encode( crate::chainparams::Network::Regtest, @@ -1307,7 +1328,7 @@ mod tests { }); let (synced_tip, next_tip) = block_on(async { - let (connection, _theirs) = super::handshake(&destination(peer), &mut chain) + let (connection, _theirs) = super::handshake(&destination(peer), &mut chain, &metrics) .await .unwrap(); let (synced, synced_tip) = connection.sync().await.unwrap(); @@ -1324,6 +1345,7 @@ mod tests { #[test] fn pong_moves_the_next_ping_to_ping_interval_after_the_last() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1345,7 +1367,7 @@ mod tests { }); let (error, elapsed) = block_on(async { - let (connection, _theirs) = super::handshake(&destination(peer), &mut chain) + let (connection, _theirs) = super::handshake(&destination(peer), &mut chain, &metrics) .await .unwrap(); let start = tokio::time::Instant::now(); @@ -1375,12 +1397,13 @@ mod tests { #[test] fn peer_that_stops_reading_is_send_timeout() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || accept_handshake(&listener)); let (result, elapsed) = block_on(async { - let (connection, _theirs) = super::handshake(&destination(peer), &mut chain) + let (connection, _theirs) = super::handshake(&destination(peer), &mut chain, &metrics) .await .unwrap(); let _stream = peer_thread.join().unwrap(); @@ -1409,6 +1432,7 @@ mod tests { #[test] fn peer_that_closes_inside_a_frame_is_closed() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1418,7 +1442,7 @@ mod tests { stream.write_all(&frame[..frame.len() - 1]).unwrap(); }); - let result = block_on(super::handshake(&destination(peer), &mut chain)); + let result = block_on(super::handshake(&destination(peer), &mut chain, &metrics)); peer_thread.join().unwrap(); assert!( matches!(result, Err(super::HandshakeError::Closed)), @@ -1428,6 +1452,8 @@ mod tests { #[test] fn frame_split_by_a_timeout_is_received_whole() { + let chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let (first_sent, first_sent_receiver) = std::sync::mpsc::channel(); @@ -1444,7 +1470,8 @@ mod tests { let (timeout, message) = block_on(async { let stream = tokio::net::TcpStream::connect(peer).await.unwrap(); - let mut socket = super::Socket::new(stream, crate::chainparams::Network::Regtest); + let mut socket = + super::Socket::new(stream, crate::chainparams::Network::Regtest, &metrics); first_sent_receiver.recv().unwrap(); tokio::time::pause(); let receive = socket.receive(); @@ -1463,6 +1490,7 @@ mod tests { #[test] fn ping_in_one_write_with_their_verack_gets_a_pong() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let peer = listener.local_addr().unwrap(); let peer_thread = std::thread::spawn(move || { @@ -1484,7 +1512,7 @@ mod tests { }); let result = block_on(async { - let (connection, _theirs) = super::handshake(&destination(peer), &mut chain) + let (connection, _theirs) = super::handshake(&destination(peer), &mut chain, &metrics) .await .unwrap(); connection.sync().await @@ -1502,11 +1530,12 @@ mod tests { #[test] fn refused_connect_is_connect_error() { let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let metrics = crate::metrics::Metrics::new(&chain); let peer = std::net::TcpListener::bind("127.0.0.1:0") .unwrap() .local_addr() .unwrap(); - let result = block_on(super::handshake(&destination(peer), &mut chain)); + let result = block_on(super::handshake(&destination(peer), &mut chain, &metrics)); let Err(error) = result else { panic!("{result:?}"); }; diff --git a/tests/bitcoind.rs b/tests/bitcoind.rs index 44031d0..06aea8d 100644 --- a/tests/bitcoind.rs +++ b/tests/bitcoind.rs @@ -128,10 +128,12 @@ fn handshake_with_bitcoind_returns_its_version() { .build() .unwrap(); let mut chain = elo::chain::Chain::new(elo::chainparams::Network::Regtest); + let metrics = elo::metrics::Metrics::new(&chain); let (_connection, version) = runtime .block_on(elo::net::handshake( &elo::net::Destination::Address(elo::net::PeerAddress::new(bitcoind.peer).unwrap()), &mut chain, + &metrics, )) .unwrap(); @@ -591,7 +593,7 @@ fn announcement_before_the_answer_to_our_getheaders_keeps_the_connection() { } #[test] -fn metrics_of_elo_have_its_tip_its_step_and_each_reconnect() { +fn metrics_of_elo_have_its_tip_its_step_each_reconnect_and_the_bytes_of_each_command() { let bitcoind = Bitcoind::start(); bitcoind.cli(&["generatetodescriptor", "2001", "raw(51)"]); let best = bitcoind.cli(&["getbestblockhash"]); @@ -618,6 +620,36 @@ fn metrics_of_elo_have_its_tip_its_step_and_each_reconnect() { } assert!(metrics.contains("elo_reconnects_total 0\n"), "{metrics}"); + settle(&bitcoind, &elo); + let metrics = scrape(address); + for (direction, field) in [ + ("received", "bytessent_per_msg"), + ("sent", "bytesrecv_per_msg"), + ] { + let mut expected = [0; COMMANDS.len() + 1]; + for (command, bytes) in per_message_bytes(&bitcoind, field) { + let index = COMMANDS + .iter() + .position(|known| *known == command) + .unwrap_or(COMMANDS.len()); + expected[index] += bytes; + } + for (command, bytes) in COMMANDS.iter().chain(&["other"]).zip(expected) { + let line = + format!("elo_peer_{direction}_bytes_total{{command=\"{command}\"}} {bytes}\n"); + assert!(metrics.contains(&line), "{line}: {metrics}"); + } + } + for line in [ + "elo_peer_received_messages_total{command=\"version\"} 1\n", + "elo_peer_received_messages_total{command=\"verack\"} 1\n", + "elo_peer_sent_messages_total{command=\"version\"} 1\n", + "elo_peer_sent_messages_total{command=\"verack\"} 1\n", + "elo_peer_sent_messages_total{command=\"sendheaders\"} 1\n", + ] { + assert!(metrics.contains(line), "{line}: {metrics}"); + } + bitcoind.cli(&["-named", "disconnectnode", "nodeid=0"]); elo.event("end", END_TIMEOUT); let metrics = scrape(address); @@ -703,6 +735,31 @@ fn pong_from(bitcoind: &Bitcoind, elo: &Elo, ends: usize) { assert_eq!(all.len(), ends, "elo ended the connection: {all:?}"); } +const COMMANDS: [&str; 10] = [ + "version", + "verack", + "ping", + "pong", + "getheaders", + "headers", + "wtxidrelay", + "sendaddrv2", + "sendheaders", + "inv", +]; + +fn per_message_bytes(bitcoind: &Bitcoind, field: &str) -> Vec<(String, u64)> { + let peer_info = bitcoind.cli(&["getpeerinfo"]); + object_text(&peer_info, field) + .split(',') + .map(|pair| { + let (message, bytes) = pair.split_once(':').unwrap(); + let message = message.trim().trim_matches('"'); + (message.to_owned(), bytes.trim().parse().unwrap()) + }) + .collect() +} + fn message_bytes(bitcoind: &Bitcoind, field: &str, message: &str) -> u64 { let peer_info = bitcoind.cli(&["getpeerinfo"]); let key = format!("\"{message}\": ");