diff --git a/README.md b/README.md index 62d6820..a3b93ee 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ chain stays in memory, so the next connection syncs from its tip. A chain below the minimum work starts again from genesis. ```sh -elo [[:port]] # IPv6 as [::1]:8333 +elo [--metrics [:]] [[:port]] # IPv6 as [::1]:8333 ``` The peer is an IP address or a name. Before each dial, elo looks up the name @@ -82,8 +82,9 @@ peer. SIGTERM and SIGINT (Ctrl-C) stop elo at any step, also in the wait before a dial. It logs the signal and exits with 0. elo exits with 1 only on a fault of -this host: its async runtime or its signal handlers do not start, a write to -stdout fails, or a read of `/dev/urandom` fails. A bug panics: elo logs the +this host: its async runtime or its signal handlers do not start, it cannot +listen on the address of `--metrics`, a write to stdout fails, or a read of +`/dev/urandom` fails. A bug panics: elo logs the panic and exits with 101. Bad usage exits with 2. ## Log @@ -102,6 +103,7 @@ level=info event=stop signal=SIGTERM | --- | --- | --- | | `version` | `info` | `peer` `version` `services` `user_agent` `start_height` `relay` | | `tip` | `info` | `peer` `height` `hash` `work` | +| `metrics` | `info` | `address`: elo serves its metrics there | | `end` | `warn` | `destination` `reason` `wait_ms`: elo dials again after `wait_ms`. `destination` is what the dial looked up or dialed: the peer of the command line, also a name, or the seed, such as `seed.bitcoin.sipa.be.:8333`. `peer` is always the address of a connection | | `exit` | `error` | `reason`: a fault of this host, and elo exits with 1 | | `stop` | `info` | `signal`: elo exits with 0 | @@ -113,6 +115,29 @@ a `"` or a control character is in quotes, with escapes. A value is at most 260 at most 16 KiB, whatever the peer sends. When a write to stdout fails, elo writes the `exit` line to stderr only, because the write that failed can leave part of its line on stdout. +## Metrics + +With `--metrics`, elo serves its metrics at `GET /metrics`, in the Prometheus text format. A +scraper, such as Prometheus or Grafana Alloy, pulls them, and elo pushes nothing. The flag takes a +port on 127.0.0.1, or an address and a port, such as `0.0.0.0:9332` for a scraper in another +container. Port 0 takes a free port. elo logs the address in a `metrics` line before it dials. + +| Metric | Type | Value | +| --- | --- | --- | +| `elo_header_tip_height` | gauge | The height of the tip of the header chain | +| `elo_header_tip_work` | gauge | The work of the chain up to the tip, in decimal | +| `elo_peer_state` | gauge | 1 for the step of the connection, and 0 for each other. The `state` label is `dial`, `sync`, `synced` or `wait` | +| `elo_reconnects_total` | counter | The connections that ended, and that elo dials again after the wait | +| `elo_metrics_failures_total` | counter | The connections to the metrics that failed. The `kind` label is `accept` for an accept error, `io` for a read or write error, `timeout` for no answer in 1 s, or `evicted` for 8 newer connections before the answer | + +The tip moves when the `tip` line does: at the end of a sync, and then with each new tip. As for the +log, the names are an interface, and `src/metrics.rs` holds them. elo accepts each connection when +it comes and answers at most 8 at the same time, each in at most 1 s. It closes each connection after +its answer, and also closes a connection when 8 newer connections come before its answer, so idle +clients cannot hold a scrape. A scrape gets its answer in at most 1 s, unless 8 newer connections +come before its answer. The scrape timeout of Prometheus is 10 s by default. A request +head is at most 8 KiB. Another path gets 404, and another method gets 405. + ## Build Nix builds everything. `flake.lock` pins nixpkgs and the other inputs, and @@ -130,7 +155,8 @@ Each push to `main` also pushes an image of elo to GHCR, as `ghcr.io/lorenzolfm/ `:sha-`. CI runs one push at a time and drops a push that waits when a newer one comes, so a burst of merges gets an image for its last commit only. Before the push, CI runs the image against a regtest `bitcoind` until it prints the `tip`. The arguments of the image are the arguments of -elo. It runs as user 65534 and needs no volume, no port and no writable path: +elo. It runs as user 65534 and needs no volume, no port (unless you give `--metrics`) and no +writable path: ```sh nix build .#elo-image && docker load --input result diff --git a/src/http.rs b/src/http.rs new file mode 100644 index 0000000..5954a02 --- /dev/null +++ b/src/http.rs @@ -0,0 +1,461 @@ +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +pub(crate) const HEAD_LENGTH_MAX: usize = 8 * 1024; + +pub(crate) const EXCHANGE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1); + +pub(crate) const EXCHANGES_MAX: usize = 8; + +const LISTEN_BACKLOG: u32 = 4; + +pub(crate) const ACCEPT_ERROR_WAIT: std::time::Duration = std::time::Duration::from_secs(1); + +const HEAD_END: &[u8] = b"\r\n\r\n"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Request<'a> { + pub(crate) method: &'a str, + pub(crate) target: &'a str, +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Response { + Ok { + content_type: &'static str, + body: String, + }, + BadRequest, + NotFound, + MethodNotAllowed { + allow: &'static str, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Failure { + Accept, + Io, + Timeout, + Evicted, +} + +#[derive(Debug)] +enum HeadError { + Io(std::io::Error), + Closed, + TooLong, +} + +pub(crate) fn listen( + address: std::net::SocketAddr, +) -> Result { + let socket = match address { + std::net::SocketAddr::V4(_) => tokio::net::TcpSocket::new_v4()?, + std::net::SocketAddr::V6(_) => tokio::net::TcpSocket::new_v6()?, + }; + socket.set_reuseaddr(true)?; + socket.bind(address)?; + socket.listen(LISTEN_BACKLOG) +} + +pub(crate) async fn serve( + listener: tokio::net::TcpListener, + answer: impl Fn(Request<'_>) -> Response, + failed: impl Fn(Failure), +) -> std::convert::Infallible { + let answer = &answer; + let mut exchanges: [Option<_>; EXCHANGES_MAX] = std::array::from_fn(|_| None); + let mut next: usize = 0; + let mut accept_wait = std::pin::pin!(tokio::time::sleep(std::time::Duration::ZERO)); + let mut accept_waits = false; + std::future::poll_fn(|context| { + loop { + if accept_waits { + if accept_wait.as_mut().poll(context).is_pending() { + break; + } + accept_waits = false; + } + match listener.poll_accept(context) { + std::task::Poll::Ready(Ok((mut stream, _))) => { + let slot = &mut exchanges[next]; + if slot.is_some() { + failed(Failure::Evicted); + } + *slot = Some(Box::pin(tokio::time::timeout( + EXCHANGE_TIMEOUT, + async move { exchange(&mut stream, answer).await }, + ))); + next = (next + 1) % EXCHANGES_MAX; + } + std::task::Poll::Ready(Err(_)) => { + failed(Failure::Accept); + accept_wait.set(tokio::time::sleep(ACCEPT_ERROR_WAIT)); + accept_waits = true; + } + std::task::Poll::Pending => break, + } + } + for slot in &mut exchanges { + let Some(exchange) = slot else { + continue; + }; + let std::task::Poll::Ready(exchanged) = exchange.as_mut().poll(context) else { + continue; + }; + match exchanged { + Ok(Ok(())) => {} + Ok(Err(_)) => failed(Failure::Io), + Err(_) => failed(Failure::Timeout), + } + *slot = None; + } + std::task::Poll::Pending + }) + .await +} + +async fn exchange( + stream: &mut tokio::net::TcpStream, + answer: impl Fn(Request<'_>) -> Response, +) -> Result<(), std::io::Error> { + let response = match read_head(stream).await { + Ok(head) => match request(&head) { + Some(request) => answer(request), + None => Response::BadRequest, + }, + Err(HeadError::TooLong) => Response::BadRequest, + Err(HeadError::Closed) => return Ok(()), + Err(HeadError::Io(error)) => return Err(error), + }; + stream.write_all(&response.bytes()).await?; + stream.shutdown().await +} + +async fn read_head(stream: &mut (impl tokio::io::AsyncRead + Unpin)) -> Result, HeadError> { + let mut buffer = vec![0; HEAD_LENGTH_MAX]; + let mut filled: usize = 0; + loop { + let read = stream + .read(&mut buffer[filled..]) + .await + .map_err(HeadError::Io)?; + if read == 0 { + return Err(HeadError::Closed); + } + let search_start = filled.saturating_sub(HEAD_END.len()); + filled += read; + assert!(filled <= HEAD_LENGTH_MAX); + let found = buffer[search_start..filled] + .windows(HEAD_END.len()) + .position(|window| window == HEAD_END); + if let Some(position) = found { + buffer.truncate(search_start + position + HEAD_END.len()); + return Ok(buffer); + } + if filled == HEAD_LENGTH_MAX { + return Err(HeadError::TooLong); + } + } +} + +fn request(head: &[u8]) -> Option> { + assert!(head.ends_with(HEAD_END)); + let line_end = head.windows(2).position(|window| window == b"\r\n")?; + let line = std::str::from_utf8(&head[..line_end]).ok()?; + let mut parts = line.split(' '); + let (Some(method), Some(target), Some(version), None) = + (parts.next(), parts.next(), parts.next(), parts.next()) + else { + return None; + }; + if method.is_empty() { + return None; + } + if !method.bytes().all(token_byte) { + return None; + } + if target.is_empty() { + return None; + } + if !target.bytes().all(|byte| byte.is_ascii_graphic()) { + return None; + } + if !matches!(version, "HTTP/1.1" | "HTTP/1.0") { + return None; + } + Some(Request { method, target }) +} + +fn token_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || b"!#$%&'*+-.^_`|~".contains(&byte) +} + +impl Response { + fn bytes(&self) -> Vec { + let (status, headers, body) = match self { + Response::Ok { content_type, body } => { + assert!(!content_type.contains(['\r', '\n']), "{content_type:?}"); + ( + "200 OK", + format!("Content-Type: {content_type}\r\n"), + body.as_str(), + ) + } + Response::BadRequest => ("400 Bad Request", String::new(), ""), + Response::NotFound => ("404 Not Found", String::new(), ""), + Response::MethodNotAllowed { allow } => { + assert!(!allow.contains(['\r', '\n']), "{allow:?}"); + ("405 Method Not Allowed", format!("Allow: {allow}\r\n"), "") + } + }; + format!( + "HTTP/1.1 {status}\r\n{headers}Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .into_bytes() + } +} + +#[cfg(test)] +mod tests { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + fn block_on(future: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .enable_io() + .enable_time() + .build() + .unwrap() + .block_on(future) + } + + fn answer(request: super::Request<'_>) -> super::Response { + super::Response::Ok { + content_type: "text/plain", + body: format!("{} {}", request.method, request.target), + } + } + + async fn exchange(listener: &tokio::net::TcpListener, sent: &[u8]) -> String { + let address = listener.local_addr().unwrap(); + let mut client = tokio::net::TcpStream::connect(address).await.unwrap(); + client.write_all(sent).await.unwrap(); + client.shutdown().await.unwrap(); + let (mut server, _) = listener.accept().await.unwrap(); + super::exchange(&mut server, answer).await.unwrap(); + drop(server); + let mut received = Vec::new(); + client.read_to_end(&mut received).await.unwrap(); + String::from_utf8(received).unwrap() + } + + #[test] + fn request_line_is_a_method_a_target_and_http_1() { + for (head, expected) in [ + ( + &b"GET /metrics HTTP/1.1\r\nHost: localhost\r\n\r\n"[..], + Some(("GET", "/metrics")), + ), + (b"GET /metrics HTTP/1.0\r\n\r\n", Some(("GET", "/metrics"))), + (b"POST /a?b=c HTTP/1.1\r\n\r\n", Some(("POST", "/a?b=c"))), + (b"M-1 * HTTP/1.1\r\n\r\n", Some(("M-1", "*"))), + (b"GET /metrics HTTP/2\r\n\r\n", None), + (b"GET /metrics http/1.1\r\n\r\n", None), + (b"GET /metrics\r\n\r\n", None), + (b"GET /metrics HTTP/1.1\r\n\r\n", None), + (b"GET /metrics HTTP/1.1 \r\n\r\n", None), + (b" GET /metrics HTTP/1.1\r\n\r\n", None), + (b"G(T /metrics HTTP/1.1\r\n\r\n", None), + (b"GET /m\x7fetrics HTTP/1.1\r\n\r\n", None), + (b"GET /\xc3\xa9 HTTP/1.1\r\n\r\n", None), + (b"GET /\xff HTTP/1.1\r\n\r\n", None), + (b"\r\n\r\n", None), + ] { + let request = + super::request(head).map(|super::Request { method, target }| (method, target)); + assert_eq!(request, expected, "{:?}", String::from_utf8_lossy(head)); + } + } + + #[test] + fn each_response_has_its_status_its_headers_and_a_length() { + for (response, expected) in [ + ( + super::Response::Ok { + content_type: "text/plain", + body: String::from("ab"), + }, + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\n\ + Connection: close\r\n\r\nab", + ), + ( + super::Response::BadRequest, + "HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ), + ( + super::Response::NotFound, + "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ), + ( + super::Response::MethodNotAllowed { allow: "GET" }, + "HTTP/1.1 405 Method Not Allowed\r\nAllow: GET\r\nContent-Length: 0\r\n\ + Connection: close\r\n\r\n", + ), + ] { + assert_eq!(String::from_utf8(response.bytes()).unwrap(), expected); + } + } + + #[test] + fn header_value_with_a_line_end_is_a_bug() { + for value in ["a\rb", "a\nb", "a\r\nB: c"] { + for response in [ + super::Response::Ok { + content_type: value, + body: String::new(), + }, + super::Response::MethodNotAllowed { allow: value }, + ] { + let written = std::panic::catch_unwind(|| response.bytes()); + assert!(written.is_err(), "{value:?}"); + } + } + } + + #[test] + fn exchange_answers_a_head_and_refuses_a_bad_or_long_one() { + let fits = format!( + "GET / HTTP/1.1\r\nA: {}\r\n\r\n", + "a".repeat(super::HEAD_LENGTH_MAX - 23) + ); + assert_eq!(fits.len(), 8 * 1024); + let long = format!( + "GET / HTTP/1.1\r\nA: {}\r\n\r\n", + "a".repeat(super::HEAD_LENGTH_MAX - 22) + ); + let ok = |body: &str| { + format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\n\ + Connection: close\r\n\r\n{body}", + body.len() + ) + }; + let bad = "HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + block_on(async { + let listener = super::listen("127.0.0.1:0".parse().unwrap()).unwrap(); + for (sent, expected) in [ + (&b"GET /metrics HTTP/1.1\r\n\r\n"[..], ok("GET /metrics")), + (b"GET /metrics HTTP/1.1\r\n\r\nbody", ok("GET /metrics")), + (fits.as_bytes(), ok("GET /")), + (long.as_bytes(), String::from(bad)), + (b"GET /metrics\r\n\r\n", String::from(bad)), + (b"GET /metrics HTTP/1.1\r\n", String::new()), + (b"", String::new()), + ] { + let received = exchange(&listener, sent).await; + assert_eq!(received, expected, "{:?}", String::from_utf8_lossy(sent)); + } + }); + } + + fn served( + clients: impl FnOnce(std::net::SocketAddr) -> F, + ) -> (F::Output, Vec) { + block_on(async { + let listener = super::listen("127.0.0.1:0".parse().unwrap()).unwrap(); + let address = listener.local_addr().unwrap(); + let failures = std::cell::RefCell::new(Vec::new()); + let output = { + let serve = super::serve(listener, answer, |failure| { + failures.borrow_mut().push(failure); + }); + let mut serve = std::pin::pin!(serve); + let mut clients = std::pin::pin!(clients(address)); + std::future::poll_fn(|context| { + if let std::task::Poll::Ready(output) = clients.as_mut().poll(context) { + return std::task::Poll::Ready(output); + } + match serve.as_mut().poll(context) { + std::task::Poll::Ready(never) => match never {}, + std::task::Poll::Pending => std::task::Poll::Pending, + } + }) + .await + }; + (output, failures.into_inner()) + }) + } + + async fn scrape(address: std::net::SocketAddr) -> String { + let mut client = tokio::net::TcpStream::connect(address).await.unwrap(); + client.write_all(b"GET / HTTP/1.1\r\n\r\n").await.unwrap(); + let mut received = String::new(); + client.read_to_string(&mut received).await.unwrap(); + received + } + + #[test] + fn serve_answers_a_head_while_a_connection_has_none_and_counts_its_timeout_and_a_reset() { + let ((received, answered, idle_end), failures) = served(async |address| { + let start = std::time::Instant::now(); + let mut idle = tokio::net::TcpStream::connect(address).await.unwrap(); + let reset = tokio::net::TcpStream::connect(address).await.unwrap(); + reset.set_zero_linger().unwrap(); + drop(reset); + let received = scrape(address).await; + let answered = start.elapsed(); + let read = idle.read(&mut [0; 1]).await.unwrap(); + assert_eq!(read, 0); + (received, answered, start.elapsed()) + }); + assert!(received.starts_with("HTTP/1.1 200 OK\r\n"), "{received}"); + assert!(received.ends_with("\r\n\r\nGET /"), "{received}"); + assert!(answered < super::EXCHANGE_TIMEOUT / 2, "{answered:?}"); + assert!(idle_end >= super::EXCHANGE_TIMEOUT, "{idle_end:?}"); + assert_eq!(failures, [super::Failure::Io, super::Failure::Timeout]); + } + + #[test] + fn connection_with_as_many_newer_ones_as_the_maximum_is_evicted_for_the_newest() { + for (idle_count, evicted) in [ + (super::EXCHANGES_MAX - 1, false), + (super::EXCHANGES_MAX, true), + ] { + let ((received, oldest_end), failures) = served(async |address| { + let mut idle = Vec::new(); + for _ in 0..idle_count { + idle.push(tokio::net::TcpStream::connect(address).await.unwrap()); + } + let received = scrape(address).await; + let wait = super::EXCHANGE_TIMEOUT / 10; + let oldest_end = tokio::time::timeout(wait, idle[0].read(&mut [0; 1])).await; + (received, oldest_end.map(Result::unwrap)) + }); + assert!( + received.ends_with("\r\n\r\nGET /"), + "{idle_count}: {received}" + ); + assert_eq!(oldest_end.is_ok(), evicted, "{idle_count}: {oldest_end:?}"); + if evicted { + assert_eq!(oldest_end, Ok(0), "{idle_count}"); + assert_eq!(failures, [super::Failure::Evicted], "{idle_count}"); + } else { + assert_eq!(failures, [], "{idle_count}"); + } + } + } + + #[test] + fn head_end_over_two_reads_is_found() { + let head = b"GET / HTTP/1.1\r\n\r\n"; + block_on(async { + for split in 1..super::HEAD_END.len() { + let (first, second) = head.split_at(head.len() - split); + let mut reader = AsyncReadExt::chain(first, second); + let read = super::read_head(&mut reader).await.unwrap(); + assert_eq!(read, head, "{split}"); + } + }); + } +} diff --git a/src/lib.rs b/src/lib.rs index 976e6c8..5aeb1c1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,9 @@ mod block; pub mod chain; pub mod chainparams; mod headerssync; +mod http; pub mod logging; +pub mod metrics; pub mod net; mod net_processing; mod pow; diff --git a/src/logging.rs b/src/logging.rs index 43492fb..55edcbf 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -29,6 +29,9 @@ pub enum Event<'a> { peer: std::net::SocketAddr, tip: &'a crate::chain::Tip, }, + Metrics { + address: std::net::SocketAddr, + }, End { destination: &'a crate::net::Destination, reason: &'a dyn std::fmt::Display, @@ -55,6 +58,7 @@ impl Event<'_> { match self { Event::Version { .. } => "version", Event::Tip { .. } => "tip", + Event::Metrics { .. } => "metrics", Event::End { .. } => "end", Event::Exit { .. } => "exit", Event::Stop { .. } => "stop", @@ -64,7 +68,10 @@ impl Event<'_> { fn level(&self) -> Level { match self { - Event::Version { .. } | Event::Tip { .. } | Event::Stop { .. } => Level::Info, + Event::Version { .. } + | Event::Tip { .. } + | Event::Metrics { .. } + | Event::Stop { .. } => Level::Info, Event::End { .. } => Level::Warn, Event::Exit { .. } | Event::Panic { .. } => Level::Error, } @@ -89,6 +96,7 @@ impl Event<'_> { line.field("hash", tip.hash_display()); line.field("work", tip.work()); } + Event::Metrics { address } => line.field("address", address), Event::End { destination, reason, @@ -264,6 +272,10 @@ mod tests { hash=0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206 \ work=0000000000000000000000000000000000000000000000000000000000000002", ), + ( + super::Event::Metrics { address: PEER }, + "level=info event=metrics address=127.0.0.1:18444", + ), ( super::Event::End { destination: &crate::net::Destination::Address( diff --git a/src/main.rs b/src/main.rs index 1eb53e7..6a154b8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ -const USAGE: &str = "usage: elo [[:port]] +const USAGE: &str = + "usage: elo [--metrics [:]] [[:port]] regtest has no seeds, so it needs a peer"; const EXIT_USAGE: u8 = 2; @@ -14,7 +15,7 @@ fn main() -> std::process::ExitCode { })); let args: Vec = std::env::args_os().skip(1).collect(); - let Some((network, source)) = parse(&args) else { + let Some(options) = parse(&args) else { eprintln!("{USAGE}"); return std::process::ExitCode::from(EXIT_USAGE); }; @@ -27,23 +28,82 @@ fn main() -> std::process::ExitCode { Ok(runtime) => runtime, Err(error) => return exit(&format_args!("runtime: {error}")), }; - let code = runtime.block_on(run(network, source)); + let code = runtime.block_on(run(options)); runtime.shutdown_background(); code } -async fn run( +#[derive(Debug, PartialEq, Eq)] +struct Options { network: elo::chainparams::Network, source: elo::net::Source, -) -> std::process::ExitCode { + metrics: Option, +} + +async fn run(options: Options) -> std::process::ExitCode { let mut signals = match Signals::new() { Ok(signals) => signals, Err(error) => return exit(&format_args!("signals: {error}")), }; - let mut chain = elo::chain::Chain::new(network); + let listener = match options.metrics { + None => None, + Some(address) => match listen(address) { + Ok((listener, address)) => { + if let Err(error) = log(&elo::logging::Event::Metrics { address }) { + return exit_stdout(error); + } + Some(listener) + } + Err(error) => return exit(&format_args!("metrics: {error}")), + }, + }; + let mut chain = elo::chain::Chain::new(options.network); + let metrics = elo::metrics::Metrics::new(&chain); + let serve = async { + match listener { + Some(listener) => metrics.serve(listener).await, + None => std::future::pending().await, + } + }; + let node = node(&options.source, &mut chain, &metrics, &mut signals); + beside(serve, node).await +} + +fn listen( + address: std::net::SocketAddr, +) -> Result<(tokio::net::TcpListener, std::net::SocketAddr), std::io::Error> { + let listener = elo::metrics::Metrics::listen(address)?; + let address = listener.local_addr()?; + Ok((listener, address)) +} + +async fn beside( + background: impl Future, + work: impl Future, +) -> T { + let mut background = std::pin::pin!(background); + let mut work = std::pin::pin!(work); + std::future::poll_fn(|context| { + if let std::task::Poll::Ready(output) = work.as_mut().poll(context) { + return std::task::Poll::Ready(output); + } + match background.as_mut().poll(context) { + std::task::Poll::Ready(never) => match never {}, + std::task::Poll::Pending => std::task::Poll::Pending, + } + }) + .await +} + +async fn node( + source: &elo::net::Source, + chain: &mut elo::chain::Chain, + metrics: &elo::metrics::Metrics, + signals: &mut Signals, +) -> std::process::ExitCode { loop { let destination = source.destination(); - let end = match signals.unless(dial(&destination, &mut chain)).await { + let end = match signals.unless(dial(&destination, chain, metrics)).await { Ok(Ok(end)) => end, Ok(Err(error)) => return exit_stdout(error), Err(signal) => return stop(signal), @@ -51,6 +111,7 @@ async fn run( if !end.retries() { return exit(&end); } + metrics.peer(elo::metrics::PeerState::Wait); let logged = log(&elo::logging::Event::End { destination: &destination, reason: &end, @@ -62,6 +123,7 @@ async fn run( if let Err(signal) = signals.unless(tokio::time::sleep(RECONNECT_WAIT)).await { return stop(signal); } + metrics.peer(elo::metrics::PeerState::Dial); } } @@ -191,11 +253,13 @@ impl std::fmt::Display for End { async fn dial( destination: &elo::net::Destination, chain: &mut elo::chain::Chain, + metrics: &elo::metrics::Metrics, ) -> Result { let (connection, version) = match elo::net::handshake(destination, chain).await { Ok(handshake) => handshake, Err(error) => return Ok(End::Handshake(error)), }; + metrics.peer(elo::metrics::PeerState::Sync); let address = connection.address(); log(&elo::logging::Event::Version { peer: address, @@ -206,7 +270,9 @@ async fn dial( Ok(sync) => sync, Err(error) => return Ok(End::Run(error)), }; + metrics.peer(elo::metrics::PeerState::Synced); loop { + metrics.tip(&tip); log(&elo::logging::Event::Tip { peer: address, tip: &tip, @@ -218,7 +284,13 @@ async fn dial( } } -fn parse(args: &[std::ffi::OsString]) -> Option<(elo::chainparams::Network, elo::net::Source)> { +fn parse(args: &[std::ffi::OsString]) -> Option { + let (metrics, args) = match args { + [flag, address, args @ ..] if flag == "--metrics" => { + (Some(metrics_address(address.to_str()?)?), args) + } + args => (None, args), + }; let (network, peer) = match args { [network] => (network, None), [network, peer] => (network, Some(peer)), @@ -235,7 +307,21 @@ fn parse(args: &[std::ffi::OsString]) -> Option<(elo::chainparams::Network, elo: Some(peer) => elo::net::Source::Destination(destination(peer.to_str()?, network)?), None => elo::net::Source::Seeds(elo::net::Seeds::new(network)?), }; - Some((network, source)) + Some(Options { + network, + source, + metrics, + }) +} + +fn metrics_address(text: &str) -> Option { + if let Ok(port) = text.parse() { + return Some(std::net::SocketAddr::from(( + std::net::Ipv4Addr::LOCALHOST, + port, + ))); + } + text.parse().ok() } fn destination(peer: &str, network: elo::chainparams::Network) -> Option { @@ -268,6 +354,12 @@ fn address(peer: &str, default_port: std::num::NonZeroU16) -> Option Option<(elo::chainparams::Network, elo::net::Source)> { + let options = parse_options(args)?; + assert_eq!(options.metrics, None, "{args:?}"); + Some((options.network, options.source)) + } + + fn parse_options(args: &[&str]) -> Option { let args: Vec = args.iter().map(std::ffi::OsString::from).collect(); super::parse(&args) } @@ -415,6 +507,65 @@ mod tests { } } + #[test] + fn metrics_flag_before_the_network_takes_a_port_on_localhost_or_an_address() { + let localhost = |port| std::net::SocketAddr::from(([127, 0, 0, 1], port)); + for (address, expected) in [ + ("9332", Some(localhost(9332))), + ("0", Some(localhost(0))), + ("127.0.0.1:9332", Some(localhost(9332))), + ( + "0.0.0.0:9332", + Some(std::net::SocketAddr::from(([0, 0, 0, 0], 9332))), + ), + ( + "[::1]:9332", + Some(std::net::SocketAddr::from(( + std::net::Ipv6Addr::LOCALHOST, + 9332, + ))), + ), + ("65536", None), + ("-1", None), + ("127.0.0.1", None), + ("localhost:9332", None), + (":9332", None), + ("", None), + ] { + let options = parse_options(&["--metrics", address, "regtest", "127.0.0.1"]); + assert_eq!( + options.map(|options| options.metrics), + expected.map(Some), + "{address}" + ); + } + let options = parse_options(&["--metrics", "9332", "regtest", "127.0.0.1"]).unwrap(); + assert_eq!(options.network, elo::chainparams::Network::Regtest); + assert_eq!( + options.source, + elo::net::Source::Destination(elo::net::Destination::Address( + elo::net::PeerAddress::new(localhost(18444)).unwrap() + )) + ); + for args in [ + &["--metrics"][..], + &["--metrics", "9332"], + &["regtest", "127.0.0.1", "--metrics", "9332"], + &["regtest", "--metrics", "9332"], + &["--metric", "9332", "regtest", "127.0.0.1"], + &[ + "--metrics", + "9332", + "--metrics", + "9333", + "regtest", + "127.0.0.1", + ], + ] { + assert_eq!(parse_options(args), None, "{args:?}"); + } + } + #[test] fn no_peer_is_the_seeds_of_the_network() { for (name, network) in [ diff --git a/src/metrics.rs b/src/metrics.rs new file mode 100644 index 0000000..a2bd827 --- /dev/null +++ b/src/metrics.rs @@ -0,0 +1,537 @@ +use std::fmt::Write; + +const PATH: &str = "/metrics"; + +const CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8"; + +const BODY_LENGTH_MAX: usize = 2 * 1024; + +const SCRAPE_TIMEOUT_DEFAULT: std::time::Duration = std::time::Duration::from_secs(10); + +const _: () = + assert!(crate::http::EXCHANGE_TIMEOUT.as_millis() < SCRAPE_TIMEOUT_DEFAULT.as_millis()); + +#[derive(Debug)] +pub struct Metrics { + tip: std::cell::Cell, + peer: std::cell::Cell, + reconnects: std::cell::Cell, + accept_failures: std::cell::Cell, + io_failures: std::cell::Cell, + timeout_failures: std::cell::Cell, + evicted_failures: std::cell::Cell, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PeerState { + Dial, + Sync, + Synced, + Wait, +} + +impl PeerState { + const ALL: [PeerState; 4] = [ + PeerState::Dial, + PeerState::Sync, + PeerState::Synced, + PeerState::Wait, + ]; + + fn name(self) -> &'static str { + match self { + PeerState::Dial => "dial", + PeerState::Sync => "sync", + PeerState::Synced => "synced", + PeerState::Wait => "wait", + } + } +} + +impl Metrics { + pub fn new(chain: &crate::chain::Chain) -> Metrics { + Metrics { + tip: std::cell::Cell::new(chain.tip()), + peer: std::cell::Cell::new(PeerState::Dial), + reconnects: std::cell::Cell::new(0), + accept_failures: std::cell::Cell::new(0), + io_failures: std::cell::Cell::new(0), + timeout_failures: std::cell::Cell::new(0), + evicted_failures: std::cell::Cell::new(0), + } + } + + pub fn tip(&self, tip: &crate::chain::Tip) { + assert!(tip.work() >= self.tip.get().work()); + self.tip.set(*tip); + } + + pub fn peer(&self, state: PeerState) { + let before = self.peer.get(); + match state { + PeerState::Dial => assert_eq!(before, PeerState::Wait), + PeerState::Sync => assert_eq!(before, PeerState::Dial), + PeerState::Synced => assert_eq!(before, PeerState::Sync), + PeerState::Wait => { + assert_ne!(before, PeerState::Wait); + self.reconnects.set(self.reconnects.get().saturating_add(1)); + } + } + self.peer.set(state); + } + + pub fn listen( + address: std::net::SocketAddr, + ) -> Result { + crate::http::listen(address) + } + + pub async fn serve(&self, listener: tokio::net::TcpListener) -> std::convert::Infallible { + crate::http::serve( + listener, + |request| self.answer(request), + |failure| self.failed(failure), + ) + .await + } + + fn failed(&self, failure: crate::http::Failure) { + let failures = match failure { + crate::http::Failure::Accept => &self.accept_failures, + crate::http::Failure::Io => &self.io_failures, + crate::http::Failure::Timeout => &self.timeout_failures, + crate::http::Failure::Evicted => &self.evicted_failures, + }; + failures.set(failures.get().saturating_add(1)); + } + + fn answer(&self, request: crate::http::Request<'_>) -> crate::http::Response { + if request.target != PATH { + return crate::http::Response::NotFound; + } + if request.method != "GET" { + return crate::http::Response::MethodNotAllowed { allow: "GET" }; + } + crate::http::Response::Ok { + content_type: CONTENT_TYPE, + body: self.text(), + } + } + + fn text(&self) -> String { + let tip = self.tip.get(); + let mut text = Text::new(); + text.family( + "elo_header_tip_height", + Kind::Gauge, + "The height of the tip of the header chain.", + ) + .sample(tip.height()); + text.family( + "elo_header_tip_work", + Kind::Gauge, + "The work of the header chain up to its tip: the expected number of hashes.", + ) + .sample(tip.work().decimal()); + let mut family = text.family( + "elo_peer_state", + Kind::Gauge, + "The step of the connection to the peer: 1 for the step it is in, 0 for each other.", + ); + let peer = self.peer.get(); + for state in PeerState::ALL { + family.sample_labelled("state", state.name(), u8::from(state == peer)); + } + text.family( + "elo_reconnects_total", + Kind::Counter, + "The connections that ended, and that elo dials again after a wait.", + ) + .sample(self.reconnects.get()); + let mut family = text.family( + "elo_metrics_failures_total", + Kind::Counter, + "The connections to the metrics that failed: an accept error, a read or write error, no answer in time, or a newer connection in its place.", + ); + for (kind, failures) in [ + ("accept", &self.accept_failures), + ("io", &self.io_failures), + ("timeout", &self.timeout_failures), + ("evicted", &self.evicted_failures), + ] { + family.sample_labelled("kind", kind, failures.get()); + } + text.end() + } +} + +#[derive(Clone, Copy, Debug)] +enum Kind { + Gauge, + Counter, +} + +impl Kind { + fn name(self) -> &'static str { + match self { + Kind::Gauge => "gauge", + Kind::Counter => "counter", + } + } +} + +trait Value: std::fmt::Display { + const LENGTH_MAX: usize; +} + +impl Value for u64 { + const LENGTH_MAX: usize = 20; +} + +const _: () = assert!(usize::BITS <= u64::BITS); + +impl Value for usize { + const LENGTH_MAX: usize = ::LENGTH_MAX; +} + +impl Value for u8 { + const LENGTH_MAX: usize = 3; +} + +impl Value for crate::pow::Decimal { + const LENGTH_MAX: usize = crate::pow::Decimal::LENGTH_MAX; +} + +struct Text { + text: String, + growth: usize, +} + +struct Family<'a> { + text: &'a mut Text, + name: &'static str, +} + +impl Text { + fn new() -> Text { + Text { + text: String::with_capacity(BODY_LENGTH_MAX), + growth: 0, + } + } + + fn family(&mut self, name: &'static str, kind: Kind, help: &'static str) -> Family<'_> { + assert!(name.starts_with("elo_"), "{name}"); + assert!(name.bytes().all(name_byte), "{name}"); + assert_eq!( + name.ends_with("_total"), + matches!(kind, Kind::Counter), + "{name}" + ); + assert!(!help.contains(['\\', '\n']), "{name}"); + let kind = kind.name(); + let written = write!(self.text, "# HELP {name} {help}\n# TYPE {name} {kind}\n"); + assert!(written.is_ok(), "{name}"); + Family { text: self, name } + } + + fn value(&mut self, value: V) { + let start = self.text.len(); + let written = writeln!(self.text, "{value}"); + assert!(written.is_ok()); + let length = self.text.len() - start - 1; + assert!(length <= V::LENGTH_MAX, "{length}"); + self.growth += V::LENGTH_MAX - length; + } + + fn end(self) -> String { + assert!(self.text.ends_with('\n')); + let length_max = self.text.len() + self.growth; + assert!(length_max <= BODY_LENGTH_MAX, "{length_max}"); + self.text + } +} + +impl Family<'_> { + fn sample(&mut self, value: impl Value) { + let name = self.name; + let written = write!(self.text.text, "{name} "); + assert!(written.is_ok(), "{name}"); + self.text.value(value); + } + + fn sample_labelled( + &mut self, + label: &'static str, + label_value: &'static str, + value: impl Value, + ) { + let name = self.name; + assert!(!label.is_empty(), "{name}"); + assert!(label.bytes().all(name_byte), "{name}"); + assert!(!label_value.contains(['"', '\\', '\n']), "{name}"); + let written = write!(self.text.text, "{name}{{{label}=\"{label_value}\"}} "); + assert!(written.is_ok(), "{name}"); + self.text.value(value); + } +} + +fn name_byte(byte: u8) -> bool { + byte.is_ascii_lowercase() || byte == b'_' +} + +#[cfg(test)] +mod tests { + const EXPECTED_START: &str = "\ +# HELP elo_header_tip_height The height of the tip of the header chain. +# TYPE elo_header_tip_height gauge +elo_header_tip_height 0 +# HELP elo_header_tip_work The work of the header chain up to its tip: the expected number of hashes. +# TYPE elo_header_tip_work gauge +elo_header_tip_work 2 +# HELP elo_peer_state The step of the connection to the peer: 1 for the step it is in, 0 for each other. +# TYPE elo_peer_state gauge +"; + + const EXPECTED_END: &str = "\ +# HELP elo_reconnects_total The connections that ended, and that elo dials again after a wait. +# TYPE elo_reconnects_total counter +"; + + const EXPECTED_FAILURES_HEAD: &str = "\ +# HELP elo_metrics_failures_total The connections to the metrics that failed: an accept error, a read or write error, no answer in time, or a newer connection in its place. +# TYPE elo_metrics_failures_total counter +"; + + fn metrics() -> super::Metrics { + super::Metrics::new(&crate::chain::Chain::new( + crate::chainparams::Network::Regtest, + )) + } + + fn metrics_in(state: super::PeerState) -> super::Metrics { + let metrics = metrics(); + let path: &[super::PeerState] = match state { + super::PeerState::Dial => &[], + super::PeerState::Sync => &[super::PeerState::Sync], + super::PeerState::Synced => &[super::PeerState::Sync, super::PeerState::Synced], + super::PeerState::Wait => &[super::PeerState::Wait], + }; + for step in path { + metrics.peer(*step); + } + assert_eq!(metrics.peer.get(), state); + metrics + } + + fn failures(accept: u8, io: u8, timeout: u8, evicted: u8) -> String { + format!( + "elo_metrics_failures_total{{kind=\"accept\"}} {accept}\n\ + elo_metrics_failures_total{{kind=\"io\"}} {io}\n\ + elo_metrics_failures_total{{kind=\"timeout\"}} {timeout}\n\ + elo_metrics_failures_total{{kind=\"evicted\"}} {evicted}\n" + ) + } + + fn states(dial: u8, sync: u8, synced: u8, wait: u8) -> String { + format!( + "elo_peer_state{{state=\"dial\"}} {dial}\nelo_peer_state{{state=\"sync\"}} {sync}\n\ + elo_peer_state{{state=\"synced\"}} {synced}\nelo_peer_state{{state=\"wait\"}} {wait}\n" + ) + } + + #[test] + fn text_has_the_tip_of_the_chain_the_dial_and_no_reconnect_at_start() { + let expected = format!( + "{EXPECTED_START}{}{EXPECTED_END}elo_reconnects_total 0\n{EXPECTED_FAILURES_HEAD}{}", + states(1, 0, 0, 0), + failures(0, 0, 0, 0) + ); + assert_eq!(metrics().text(), expected); + } + + #[test] + fn text_has_the_last_tip_the_state_and_each_wait_as_a_reconnect() { + let metrics = metrics(); + let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let headers = crate::fixtures::headers_on_genesis(3); + let extended = chain.extend( + crate::fixtures::headers_batch(&headers), + crate::fixtures::CLOCK, + ); + assert_eq!(extended, Ok(())); + metrics.tip(&chain.tip()); + for (state, expected, reconnects) in [ + (super::PeerState::Wait, states(0, 0, 0, 1), 1), + (super::PeerState::Dial, states(1, 0, 0, 0), 1), + (super::PeerState::Sync, states(0, 1, 0, 0), 1), + (super::PeerState::Wait, states(0, 0, 0, 1), 2), + (super::PeerState::Dial, states(1, 0, 0, 0), 2), + (super::PeerState::Sync, states(0, 1, 0, 0), 2), + (super::PeerState::Synced, states(0, 0, 1, 0), 2), + (super::PeerState::Wait, states(0, 0, 0, 1), 3), + ] { + metrics.peer(state); + let text = metrics.text(); + assert!(text.contains("elo_header_tip_height 3\n"), "{text}"); + assert!(text.contains("elo_header_tip_work 8\n"), "{text}"); + assert!(text.contains(&expected), "{state:?}: {text}"); + let reconnects_line = format!("elo_reconnects_total {reconnects}\n"); + assert!(text.contains(&reconnects_line), "{state:?}: {text}"); + } + } + + #[test] + fn text_has_each_failure_of_its_kind() { + let metrics = metrics(); + for (failure, expected) in [ + (crate::http::Failure::Timeout, failures(0, 0, 1, 0)), + (crate::http::Failure::Io, failures(0, 1, 1, 0)), + (crate::http::Failure::Evicted, failures(0, 1, 1, 1)), + (crate::http::Failure::Accept, failures(1, 1, 1, 1)), + (crate::http::Failure::Timeout, failures(1, 1, 2, 1)), + ] { + metrics.failed(failure); + let text = metrics.text(); + assert!(text.ends_with(&expected), "{failure:?}: {text}"); + } + } + + #[test] + fn peer_state_moves_only_on_the_steps_of_a_connection() { + for before in super::PeerState::ALL { + for after in super::PeerState::ALL { + let allowed = matches!( + (before, after), + (super::PeerState::Dial, super::PeerState::Sync) + | (super::PeerState::Sync, super::PeerState::Synced) + | ( + super::PeerState::Dial + | super::PeerState::Sync + | super::PeerState::Synced, + super::PeerState::Wait + ) + | (super::PeerState::Wait, super::PeerState::Dial) + ); + let metrics = metrics_in(before); + let moved = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + metrics.peer(after); + })); + assert_eq!(moved.is_ok(), allowed, "{before:?} -> {after:?}"); + } + } + } + + #[test] + #[should_panic(expected = "tip.work() >= self.tip.get().work()")] + fn tip_with_less_work_than_the_last_is_a_bug() { + let mut chain = crate::chain::Chain::new(crate::chainparams::Network::Regtest); + let genesis = chain.tip(); + let headers = crate::fixtures::headers_on_genesis(1); + let extended = chain.extend( + crate::fixtures::headers_batch(&headers), + crate::fixtures::CLOCK, + ); + assert_eq!(extended, Ok(())); + let metrics = super::Metrics::new(&chain); + metrics.tip(&chain.tip()); + metrics.tip(&genesis); + } + + #[test] + fn answer_is_the_text_for_a_get_of_the_path_only() { + let metrics = metrics(); + let request = |method, target| crate::http::Request { method, target }; + for (request, expected) in [ + ( + request("GET", "/metrics"), + crate::http::Response::Ok { + content_type: "text/plain; version=0.0.4; charset=utf-8", + body: metrics.text(), + }, + ), + ( + request("HEAD", "/metrics"), + crate::http::Response::MethodNotAllowed { allow: "GET" }, + ), + (request("GET", "/"), crate::http::Response::NotFound), + (request("GET", "/metrics/"), crate::http::Response::NotFound), + ( + request("GET", "/metrics?a=b"), + crate::http::Response::NotFound, + ), + (request("POST", "/"), crate::http::Response::NotFound), + ] { + assert_eq!(metrics.answer(request), expected, "{request:?}"); + } + } + + #[test] + fn longest_text_of_each_value_has_its_maximum_length() { + use super::Value; + assert_eq!(u64::MAX.to_string().len(), u64::LENGTH_MAX); + assert!(usize::MAX.to_string().len() <= usize::LENGTH_MAX); + assert_eq!(u8::MAX.to_string().len(), u8::LENGTH_MAX); + } + + #[test] + fn text_whose_values_can_grow_past_the_maximum_is_a_bug() { + let text = |samples| { + let mut text = super::Text::new(); + let mut family = text.family("elo_a", super::Kind::Gauge, "AAAAAAAAAAAAAAAAA"); + for _ in 0..samples { + family.sample(0_u64); + } + text.end() + }; + assert_eq!(text(74).len() + 74 * 19, super::BODY_LENGTH_MAX); + let ended = std::panic::catch_unwind(|| text(75)); + assert!(ended.is_err()); + } + + #[test] + fn label_of_other_bytes_or_a_label_value_with_a_quote_a_backslash_or_a_newline_is_a_bug() { + for (label, label_value) in [ + ("b", "a\"b"), + ("b", "a\\b"), + ("b", "a\nb"), + ("", "c"), + ("B", "c"), + ("b-c", "c"), + ] { + let sampled = std::panic::catch_unwind(|| { + let mut text = super::Text::new(); + text.family("elo_a", super::Kind::Gauge, "A.") + .sample_labelled(label, label_value, 0_u8); + }); + assert!(sampled.is_err(), "{label:?} {label_value:?}"); + } + let mut text = super::Text::new(); + text.family("elo_a", super::Kind::Gauge, "A.") + .sample_labelled("b", "c_d", 0_u8); + assert_eq!( + text.end(), + "# HELP elo_a A.\n# TYPE elo_a gauge\nelo_a{b=\"c_d\"} 0\n" + ); + } + + #[test] + fn counter_name_ends_with_total_and_a_gauge_name_does_not() { + for (name, kind) in [ + ("elo_a", super::Kind::Counter), + ("elo_a_total", super::Kind::Gauge), + ] { + let written = std::panic::catch_unwind(|| { + super::Text::new().family(name, kind, "A.").sample(0_u8); + }); + assert!(written.is_err(), "{name} {kind:?}"); + } + let mut text = super::Text::new(); + text.family("elo_a_total", super::Kind::Counter, "A.") + .sample(0_u8); + assert_eq!( + text.end(), + "# HELP elo_a_total A.\n# TYPE elo_a_total counter\nelo_a_total 0\n" + ); + } +} diff --git a/src/pow.rs b/src/pow.rs index c77c9d6..f730b7e 100644 --- a/src/pow.rs +++ b/src/pow.rs @@ -54,6 +54,15 @@ impl Target { } } +const DECIMAL_CHUNK_BASE: u64 = 10_u64.pow(19); + +const DECIMAL_CHUNK_DIVISOR: Work = Work { + high: 0, + low: DECIMAL_CHUNK_BASE as u128, +}; + +const DECIMAL_CHUNKS_COUNT: usize = 5; + #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub(crate) struct Work { high: u128, @@ -107,9 +116,13 @@ impl Work { } fn divided_by(self, divisor: Work) -> Work { + self.divided_with_remainder(divisor).0 + } + + fn divided_with_remainder(self, divisor: Work) -> (Work, Work) { assert_ne!(divisor, Work::ZERO); let Some(shift) = divisor.leading_zeros().checked_sub(self.leading_zeros()) else { - return Work::ZERO; + return (Work::ZERO, self); }; let mut step_divisor = divisor.shifted_left(shift); let mut remainder = self; @@ -124,7 +137,11 @@ impl Work { } assert!(remainder < divisor); assert!(quotient <= self); - quotient + (quotient, remainder) + } + + pub(crate) fn decimal(self) -> Decimal { + Decimal(self) } fn shifted_left(self, bits: u32) -> Work { @@ -169,6 +186,36 @@ impl Work { } } +#[derive(Clone, Copy, Debug)] +pub(crate) struct Decimal(Work); + +impl Decimal { + pub(crate) const LENGTH_MAX: usize = 78; +} + +impl std::fmt::Display for Decimal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut chunks = [0_u64; DECIMAL_CHUNKS_COUNT]; + let mut rest = self.0; + for chunk in chunks.iter_mut().rev() { + let (quotient, remainder) = rest.divided_with_remainder(DECIMAL_CHUNK_DIVISOR); + assert_eq!(remainder.high, 0); + *chunk = u64::try_from(remainder.low).unwrap_or(u64::MAX); + assert!(*chunk < DECIMAL_CHUNK_BASE); + rest = quotient; + } + assert_eq!(rest, Work::ZERO); + let first_index = chunks + .iter() + .position(|chunk| *chunk != 0) + .unwrap_or(DECIMAL_CHUNKS_COUNT - 1); + write!(f, "{}", chunks[first_index])?; + chunks[first_index + 1..] + .iter() + .try_for_each(|chunk| write!(f, "{chunk:019}")) + } +} + impl std::fmt::Display for Work { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{:032x}{:032x}", self.high, self.low) @@ -969,6 +1016,33 @@ mod tests { } } + #[test] + fn decimal_of_work_is_its_number_in_base_ten() { + let work = |high, low| super::Work { high, low }; + for (number, expected) in [ + (super::Work::ZERO, "0"), + (work(0, 2), "2"), + (work(0, 9_999_999_999_999_999_999), "9999999999999999999"), + (work(0, 10_000_000_000_000_000_000), "10000000000000000000"), + (work(0, 10_000_000_000_000_000_001), "10000000000000000001"), + ( + work(0, u128::MAX), + "340282366920938463463374607431768211455", + ), + (work(1, 0), "340282366920938463463374607431768211456"), + ( + super::Work::MAX, + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + ), + ] { + assert_eq!(number.decimal().to_string(), expected, "{number}"); + } + assert_eq!( + super::Work::MAX.decimal().to_string().len(), + super::Decimal::LENGTH_MAX + ); + } + #[test] fn work_adds_with_a_carry_and_stops_at_the_highest_number() { let work = |high, low| super::Work { high, low }; diff --git a/tests/bitcoind.rs b/tests/bitcoind.rs index de313a0..44031d0 100644 --- a/tests/bitcoind.rs +++ b/tests/bitcoind.rs @@ -590,6 +590,78 @@ fn announcement_before_the_answer_to_our_getheaders_keeps_the_connection() { ); } +#[test] +fn metrics_of_elo_have_its_tip_its_step_and_each_reconnect() { + let bitcoind = Bitcoind::start(); + bitcoind.cli(&["generatetodescriptor", "2001", "raw(51)"]); + let best = bitcoind.cli(&["getbestblockhash"]); + let chain_work = bitcoind.cli(&["getblockheader", best.trim_end()]); + let chain_work = u128::from_str_radix(string_field(&chain_work, "chainwork"), 16).unwrap(); + let peer = bitcoind.peer.to_string(); + let elo = Elo::start_with(&["--metrics", "127.0.0.1:0", "regtest", &peer]); + let listen = elo.event("metrics", SYNC_TIMEOUT); + let address = listen + .strip_prefix("level=info event=metrics address=") + .unwrap(); + let address: std::net::SocketAddr = address.parse().unwrap(); + assert!(address.ip().is_loopback(), "{address}"); + assert_ne!(address.port(), 0); + elo.event("tip", SYNC_TIMEOUT); + + let metrics = scrape(address); + for line in [ + String::from("elo_header_tip_height 2001\n"), + format!("elo_header_tip_work {chain_work}\n"), + String::from("elo_peer_state{state=\"synced\"} 1\n"), + ] { + assert!(metrics.contains(&line), "{line}: {metrics}"); + } + assert!(metrics.contains("elo_reconnects_total 0\n"), "{metrics}"); + + bitcoind.cli(&["-named", "disconnectnode", "nodeid=0"]); + elo.event("end", END_TIMEOUT); + let metrics = scrape(address); + assert!( + metrics.contains("elo_peer_state{state=\"wait\"} 1\n"), + "{metrics}" + ); + assert!(metrics.contains("elo_reconnects_total 1\n"), "{metrics}"); +} + +#[test] +fn metrics_address_in_use_exits_with_1() { + let taken = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = taken.local_addr().unwrap().to_string(); + let mut elo = Elo::start_with(&["--metrics", &address, "regtest", "127.0.0.1"]); + let exit = elo.event("exit", END_TIMEOUT); + assert!( + exit.starts_with("level=error event=exit reason=\"metrics: "), + "{exit}" + ); + assert_eq!(elo.exit_code(), Some(1)); +} + +fn scrape(address: std::net::SocketAddr) -> String { + let mut stream = std::net::TcpStream::connect(address).unwrap(); + std::io::Write::write_all( + &mut stream, + b"GET /metrics HTTP/1.1\r\nHost: localhost\r\n\r\n", + ) + .unwrap(); + let mut response = String::new(); + std::io::Read::read_to_string(&mut stream, &mut response).unwrap(); + let (head, body) = response.split_once("\r\n\r\n").unwrap(); + assert_eq!( + head, + format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4; charset=utf-8\r\n\ + Content-Length: {}\r\nConnection: close", + body.len() + ) + ); + body.to_owned() +} + fn settle(bitcoind: &Bitcoind, elo: &Elo) { let start = std::time::Instant::now(); loop { @@ -672,9 +744,12 @@ impl Elo { } fn start_on(peer: &str) -> Self { + Self::start_with(&["regtest", peer]) + } + + fn start_with(args: &[&str]) -> Self { let mut child = std::process::Command::new(env!("CARGO_BIN_EXE_elo")) - .arg("regtest") - .arg(peer) + .args(args) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) .spawn()