From e8bb7e23a4ffa9006c17d2f5e59fbcc67d5b662d Mon Sep 17 00:00:00 2001 From: Igor Novgorodov Date: Mon, 22 Jun 2026 12:59:31 +0000 Subject: [PATCH 1/6] Add metrics to route provider & routing table manager --- src/core.rs | 3 +- src/routing/ic/route_provider/fetcher.rs | 85 ++++++++++++-- src/routing/ic/route_provider/health.rs | 95 +++++++++++++-- src/routing/ic/route_provider/mod.rs | 5 + src/routing/ic/route_provider/provider.rs | 20 +++- src/routing/ic/route_provider/routes.rs | 45 ++++++- src/routing/ic/routing_table_manager.rs | 137 +++++++++++++++++++--- 7 files changed, 344 insertions(+), 46 deletions(-) diff --git a/src/core.rs b/src/core.rs index 49e76fc..2dcdce9 100644 --- a/src/core.rs +++ b/src/core.rs @@ -143,7 +143,7 @@ pub async fn main( // Create route provider let (route_provider, dynamic_route_provider) = - setup_route_provider(cli, http_client_hyper, http_service.clone()).await?; + setup_route_provider(cli, http_client_hyper, http_service.clone(), ®istry).await?; health_manager.add(Arc::new(RouteProviderWrapper::new(route_provider.clone()))); // Create a separate Agent to use solely with Resolver. @@ -276,6 +276,7 @@ pub async fn main( ic_agent.clone(), MAINNET_ROOT_SUBNET_ID, cli.ic.ic_routing_table_poll_interval, + ®istry, )); health_manager.add(routing_table_manager.clone()); tasks.add("subnets_info_fetcher", routing_table_manager.clone()); diff --git a/src/routing/ic/route_provider/fetcher.rs b/src/routing/ic/route_provider/fetcher.rs index 2deda94..7988723 100644 --- a/src/routing/ic/route_provider/fetcher.rs +++ b/src/routing/ic/route_provider/fetcher.rs @@ -2,25 +2,72 @@ use std::{ fmt::{Debug, Display}, str::FromStr, sync::Arc, - time::Duration, + time::{Duration, Instant}, }; use anyhow::Context; use async_trait::async_trait; use fqdn::FQDN; -use ic_bn_lib::ic_agent::{ - Agent, - agent::{HttpService, route_provider::RouteProvider}, +use ic_bn_lib::{ + BoolYesNo, + ic_agent::{ + Agent, + agent::{HttpService, route_provider::RouteProvider}, + }, +}; +use prometheus::{ + HistogramVec, IntCounterVec, IntGauge, Registry, register_histogram_vec_with_registry, + register_int_counter_vec_with_registry, register_int_gauge_with_registry, }; use tokio::{select, sync::watch}; use tokio_util::sync::CancellationToken; use tracing::{info, warn}; -use crate::routing::ic::{ - MAINNET_ROOT_SUBNET_ID, - route_provider::{FetchesNodes, NodeList, RouteError}, +use crate::{ + metrics::HTTP_DURATION_BUCKETS, + routing::ic::{ + MAINNET_ROOT_SUBNET_ID, + route_provider::{FetchesNodes, NodeList, RouteError}, + }, }; +#[derive(Clone)] +pub struct Metrics { + nodes: IntGauge, + fetches: IntCounterVec, + duration: HistogramVec, +} + +impl Metrics { + pub fn new(registry: &Registry) -> Self { + Self { + nodes: register_int_gauge_with_registry!( + format!("route_provider_fetcher_nodes"), + format!("How many nodes were fetched"), + registry + ) + .unwrap(), + + fetches: register_int_counter_vec_with_registry!( + format!("route_provider_fetcher_count"), + format!("Counts number of fetches and their outcome"), + &["success"], + registry + ) + .unwrap(), + + duration: register_histogram_vec_with_registry!( + format!("route_provider_fetcher_duration"), + format!("Records the duration of node fetching in seconds"), + &["success"], + HTTP_DURATION_BUCKETS.to_vec(), + registry + ) + .unwrap(), + } + } +} + /// Fetches a list of API BN nodes using an IC Agent #[derive(Debug)] pub struct AgentFetcher { @@ -64,6 +111,7 @@ pub struct FetcherManager { fetcher: Arc, tx: watch::Sender, snapshot: NodeList, + metrics: Metrics, } impl Display for FetcherManager { @@ -79,16 +127,33 @@ impl Debug for FetcherManager { } impl FetcherManager { - pub fn new(fetcher: Arc, tx: watch::Sender) -> Self { + pub fn new( + fetcher: Arc, + tx: watch::Sender, + registry: &Registry, + ) -> Self { Self { fetcher, tx, snapshot: NodeList::default(), + metrics: Metrics::new(registry), } } + #[allow(clippy::cast_possible_wrap)] async fn refresh(&mut self) -> Result<(), RouteError> { - let nodes = self.fetcher.fetch_nodes().await?; + let start = Instant::now(); + let res = self.fetcher.fetch_nodes().await; + let success = res.is_ok().yesno(); + + self.metrics + .duration + .with_label_values(&[success]) + .observe(start.elapsed().as_secs_f64()); + self.metrics.fetches.with_label_values(&[success]).inc(); + + let nodes = res?; + self.metrics.nodes.set(nodes.len() as i64); // Safeguard against a case when (for whatever reason) an empty node list is fetched. // If we remove all nodes, then we'll end up in a deadlock situation: we can't fetch a new (correct) @@ -185,7 +250,7 @@ mod test { async fn test_fetcher() { let fetcher = TestFetcher::default(); let (tx, mut rx) = watch::channel(NodeList::new(vec![])); - let mut manager = FetcherManager::new(Arc::new(fetcher), tx); + let mut manager = FetcherManager::new(Arc::new(fetcher), tx, &Registry::new()); // Consume the initial value assert!(rx.borrow_and_update().is_empty()); diff --git a/src/routing/ic/route_provider/health.rs b/src/routing/ic/route_provider/health.rs index c642395..13c616b 100644 --- a/src/routing/ic/route_provider/health.rs +++ b/src/routing/ic/route_provider/health.rs @@ -12,8 +12,12 @@ use fqdn::FQDN; use futures::future::join_all; use http::Method; use http_body_util::Full; -use ic_bn_lib::http::shed::ewma::EWMA; +use ic_bn_lib::{BoolYesNo, http::shed::ewma::EWMA}; use ic_bn_lib_common::traits::http::ClientHttp; +use prometheus::{ + HistogramVec, IntCounterVec, IntGauge, Registry, register_histogram_vec_with_registry, + register_int_counter_vec_with_registry, register_int_gauge_with_registry, +}; use tokio::{ select, sync::{mpsc, watch}, @@ -23,10 +27,56 @@ use tokio::{ use tokio_util::{sync::CancellationToken, time::FutureExt}; use tracing::{info, warn}; -use crate::routing::ic::route_provider::{ - ChecksHealth, HealthCheckResult, HealthyNode, Node, NodeList, +use crate::{ + metrics::HTTP_DURATION_BUCKETS, + routing::ic::route_provider::{ChecksHealth, HealthCheckResult, HealthyNode, Node, NodeList}, }; +#[derive(Clone)] +pub struct Metrics { + nodes: IntGauge, + healthy_nodes: IntGauge, + checks: IntCounterVec, + check_duration: HistogramVec, +} + +impl Metrics { + pub fn new(registry: &Registry) -> Self { + Self { + nodes: register_int_gauge_with_registry!( + format!("route_provider_health_nodes"), + format!("How many nodes are there"), + registry + ) + .unwrap(), + + healthy_nodes: register_int_gauge_with_registry!( + format!("route_provider_health_healthy_nodes"), + format!("How many healthy nodes are there"), + registry + ) + .unwrap(), + + checks: register_int_counter_vec_with_registry!( + format!("route_provider_health_checks"), + format!("Counts number of health checks per node and their outcome"), + &["node", "success"], + registry + ) + .unwrap(), + + check_duration: register_histogram_vec_with_registry!( + format!("route_provider_health_check_duration"), + format!("Records the duration of node health checks in seconds"), + &["node", "success"], + HTTP_DURATION_BUCKETS.to_vec(), + registry + ) + .unwrap(), + } + } +} + #[derive(new)] pub struct HttpHealthChecker { client: Arc>>, @@ -93,6 +143,7 @@ pub struct HealthCheckActor { node: Arc, checker: Arc, tx: mpsc::Sender<(Arc, HealthCheckResult)>, + metrics: Metrics, } impl Display for HealthCheckActor { @@ -108,6 +159,21 @@ impl Debug for HealthCheckActor { } impl HealthCheckActor { + async fn health_check(&self) { + let start = Instant::now(); + let res = self.checker.health_check(&self.node).await; + let success = res.healthy.yesno(); + self.metrics + .checks + .with_label_values(&[&self.node.name, success]) + .inc(); + self.metrics + .check_duration + .with_label_values(&[&self.node.name, success]) + .observe(start.elapsed().as_secs_f64()); + self.tx.send((self.node.clone(), res)).await.ok(); + } + async fn run(self, interval: Duration, token: CancellationToken) { let mut int = tokio::time::interval(interval); int.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -115,9 +181,7 @@ impl HealthCheckActor { info!("{self}: Started"); loop { select! { - res = self.checker.health_check(&self.node) => { - self.tx.send((self.node.clone(), res)).await.ok(); - } + () = self.health_check() => {} () = token.cancelled() => { break; } @@ -162,6 +226,7 @@ pub struct HealthCheckManager { healthy_nodes_tx: watch::Sender>, idle_interval: Interval, ewma_alpha: f64, + metrics: Metrics, } impl Display for HealthCheckManager { @@ -184,6 +249,7 @@ impl HealthCheckManager { node_list_rx: watch::Receiver, healthy_nodes_tx: watch::Sender>, ewma_alpha: f64, + registry: &Registry, ) -> Self { let (tx, rx) = mpsc::channel(1024); let mut idle_interval = tokio::time::interval(idle_period); @@ -199,12 +265,18 @@ impl HealthCheckManager { healthy_nodes_tx, idle_interval, ewma_alpha, + metrics: Metrics::new(registry), } } fn start_actor(&mut self, node: FQDN) { let node = Arc::new(Node::new(node)); - let actor = HealthCheckActor::new(node.clone(), self.checker.clone(), self.tx.clone()); + let actor = HealthCheckActor::new( + node.clone(), + self.checker.clone(), + self.tx.clone(), + self.metrics.clone(), + ); let token = CancellationToken::new(); let handle = tokio::spawn(actor.run(self.check_interval, token.child_token())); @@ -221,7 +293,10 @@ impl HealthCheckManager { } /// Starts & stops actors to match the new list of nodes + #[allow(clippy::cast_possible_wrap)] async fn update_node_list(&mut self, node_list: NodeList) { + self.metrics.nodes.set(node_list.len() as i64); + let start = Instant::now(); // First extract the nodes that are gone and we need to stop the actors let to_stop = self @@ -262,6 +337,7 @@ impl HealthCheckManager { } /// Sends an updated list of healthy nodes to the receiver + #[allow(clippy::cast_possible_wrap)] fn send_healthy_nodes(&mut self) { // Do not send a list if we haven't yet got initial health check results // from all nodes @@ -279,7 +355,9 @@ impl HealthCheckManager { latency_us: x.latency_us.get().unwrap_or(f64::MAX), }) }) - .collect(); + .collect::>(); + + self.metrics.healthy_nodes.set(healthy_nodes.len() as i64); // Reset the idle timer so that we don't update too soon again self.idle_interval.reset(); @@ -410,6 +488,7 @@ mod test { node_list_rx, healthy_nodes_tx, 0.5, + &Registry::new(), ); let token = CancellationToken::new(); let handle = tokio::spawn(manager.run(token.child_token())); diff --git a/src/routing/ic/route_provider/mod.rs b/src/routing/ic/route_provider/mod.rs index dc3f60a..c251da8 100644 --- a/src/routing/ic/route_provider/mod.rs +++ b/src/routing/ic/route_provider/mod.rs @@ -19,6 +19,7 @@ use ic_bn_lib::ic_agent::agent::{ route_provider::{RoundRobinRouteProvider, RouteProvider}, }; use ic_bn_lib_common::traits::{Healthy, http::ClientHttp}; +use prometheus::Registry; use tokio::fs; use url::Url; @@ -126,6 +127,7 @@ impl IntoIterator for NodeList { #[derive(Clone, PartialEq, Eq)] pub struct Node { + name: String, hostname: FQDN, url: Url, uri_health: Uri, @@ -139,6 +141,7 @@ impl Node { let uri_health = format!("https://{hostname}/health").parse().unwrap(); Self { + name: hostname.to_string(), hostname, url, uri_health, @@ -182,6 +185,7 @@ pub async fn setup_route_provider( cli: &Cli, http_client: Arc>>, http_service: Arc, + registry: &Registry, ) -> anyhow::Result<(Arc, Option>)> { let health_checker = Arc::new(HttpHealthChecker::new( http_client.clone(), @@ -213,6 +217,7 @@ pub async fn setup_route_provider( cli.ic.ic_discovery_node_fetch_interval, cli.ic.ic_discovery_health_check_interval, cli.ic.ic_discovery_idle_interval, + registry, )?; Ok((rp.clone() as Arc, Some(rp))) diff --git a/src/routing/ic/route_provider/provider.rs b/src/routing/ic/route_provider/provider.rs index bfa860f..e271fd4 100644 --- a/src/routing/ic/route_provider/provider.rs +++ b/src/routing/ic/route_provider/provider.rs @@ -12,6 +12,7 @@ use ic_bn_lib::ic_agent::{ AgentError, agent::route_provider::{RouteProvider, RoutesStats}, }; +use prometheus::Registry; use tokio::{select, sync::watch}; use tokio_util::{sync::CancellationToken, task::TaskTracker}; use tracing::{info, warn}; @@ -99,6 +100,7 @@ impl DynamicRouteProvider { node_fetch_interval: Duration, health_check_interval: Duration, idle_period: Duration, + registry: &Registry, ) -> Result, RouteError> { if seed_list.is_empty() { return Err(RouteError::Other(anyhow!("Seed list should not be empty"))); @@ -133,8 +135,11 @@ impl DynamicRouteProvider { node_list_rx.mark_changed(); // Start node fetcher - let fetcher_manager = - FetcherManager::new(fetcher_factory(route_provider.clone())?, node_list_tx); + let fetcher_manager = FetcherManager::new( + fetcher_factory(route_provider.clone())?, + node_list_tx, + registry, + ); tracker.spawn(fetcher_manager.run(node_fetch_interval, token.child_token())); // Start route provider manager @@ -150,12 +155,18 @@ impl DynamicRouteProvider { node_list_rx, healthy_nodes_tx, ewma_alpha, + registry, ); tracker.spawn(health_check_manager.run(token.child_token())); // Start route manager - let routes_manager = - RoutesManager::new(healthy_nodes_rx, routes, k_top, reliability_weight); + let routes_manager = RoutesManager::new( + healthy_nodes_rx, + routes, + k_top, + reliability_weight, + registry, + ); tracker.spawn(routes_manager.run(token.child_token())); Ok(route_provider) @@ -291,6 +302,7 @@ mod test { Duration::from_millis(1), Duration::from_millis(1), Duration::from_secs(5), + &Registry::new(), ) .unwrap(); diff --git a/src/routing/ic/route_provider/routes.rs b/src/routing/ic/route_provider/routes.rs index 421ab8e..efc5f99 100644 --- a/src/routing/ic/route_provider/routes.rs +++ b/src/routing/ic/route_provider/routes.rs @@ -5,7 +5,7 @@ use std::{ }; use arc_swap::ArcSwapOption; -use derive_new::new; +use prometheus::{IntGauge, Registry, register_int_gauge_with_registry}; use tokio::{select, sync::watch}; use tokio_util::sync::CancellationToken; use tracing::{info, warn}; @@ -13,6 +13,24 @@ use url::Url; use crate::routing::ic::route_provider::{HealthyNode, wrr::Wrr}; +#[derive(Clone)] +pub struct Metrics { + active_nodes: IntGauge, +} + +impl Metrics { + pub fn new(registry: &Registry) -> Self { + Self { + active_nodes: register_int_gauge_with_registry!( + format!("route_provider_active_nodes"), + format!("How many nodes are in the active list"), + registry + ) + .unwrap(), + } + } +} + /// Snapshot of the routes pub struct RouteSnapshot { pub urls: Vec, @@ -48,12 +66,12 @@ impl Debug for Route { } /// Manages healthy routes & sorts them according to their usability -#[derive(new)] pub struct RoutesManager { healthy_nodes_rx: watch::Receiver>, routes: Arc>, k_top: Option, reliability_weight: f64, + metrics: Metrics, } impl Display for RoutesManager { @@ -90,9 +108,26 @@ fn calc_stddev(data: impl ExactSizeIterator + Clone) -> Option } impl RoutesManager { + pub fn new( + healthy_nodes_rx: watch::Receiver>, + routes: Arc>, + k_top: Option, + reliability_weight: f64, + registry: &Registry, + ) -> Self { + Self { + healthy_nodes_rx, + routes, + k_top, + reliability_weight, + metrics: Metrics::new(registry), + } + } + #[allow(clippy::cast_sign_loss)] #[allow(clippy::cast_possible_truncation)] #[allow(clippy::cast_precision_loss)] + #[allow(clippy::cast_possible_wrap)] fn update_routes(&self, mut list: Vec) { if list.is_empty() { self.routes.store(None); @@ -173,6 +208,8 @@ impl RoutesManager { routes.truncate(k); } + self.metrics.active_nodes.set(routes.len() as i64); + // Create & store the snapshot info!("{self}: New route snapshot stored: {routes:?}"); let urls_weights = routes.into_iter().map(|x| (x.weight, x.url)).collect(); @@ -247,7 +284,7 @@ mod test { let routes = Arc::new(ArcSwapOption::empty()); let (tx, rx) = watch::channel(vec![]); let token = CancellationToken::new(); - let manager = RoutesManager::new(rx, routes.clone(), Some(3), 0.9); + let manager = RoutesManager::new(rx, routes.clone(), Some(3), 0.9, &Registry::new()); let handle = tokio::spawn(manager.run(token.child_token())); // Send nodes @@ -279,7 +316,7 @@ mod test { // Test w/o k_top - publishes all nodes let (tx, rx) = watch::channel(vec![]); let token = CancellationToken::new(); - let manager = RoutesManager::new(rx, routes.clone(), None, 0.9); + let manager = RoutesManager::new(rx, routes.clone(), None, 0.9, &Registry::new()); let handle = tokio::spawn(manager.run(token.child_token())); // Send nodes diff --git a/src/routing/ic/routing_table_manager.rs b/src/routing/ic/routing_table_manager.rs index 63b7055..0ef040b 100644 --- a/src/routing/ic/routing_table_manager.rs +++ b/src/routing/ic/routing_table_manager.rs @@ -12,17 +12,24 @@ use arc_swap::ArcSwapOption; use async_trait::async_trait; use candid::Principal; use futures::future::join_all; -use ic_bn_lib::ic_agent::{ - Agent, agent::SubnetType as AgentSubnetType, hash_tree::SubtreeLookupResult, +use ic_bn_lib::{ + BoolYesNo, + ic_agent::{Agent, agent::SubnetType as AgentSubnetType, hash_tree::SubtreeLookupResult}, }; use ic_bn_lib_common::traits::{Healthy, Run}; +use prometheus::{ + HistogramVec, IntCounterVec, IntGauge, Registry, register_histogram_vec_with_registry, + register_int_counter_vec_with_registry, register_int_gauge_with_registry, +}; use tokio::time::interval; use tokio_util::sync::CancellationToken; use tracing::{info, warn}; +use crate::metrics::HTTP_DURATION_BUCKETS; + /// Retry interval used when no snapshot has been fetched yet and we are in the /// aggressive boot-strap loop. -const AGGRESSIVE_RETRY_INTERVAL: Duration = Duration::from_secs(5); +const AGGRESSIVE_RETRY_INTERVAL: Duration = Duration::from_secs(1); /// Minimal fraction of the total subnets that need to succeed to consider /// the whole fetch a success. @@ -38,6 +45,70 @@ pub enum SubnetType { Unknown, } +#[derive(Clone)] +pub struct Metrics { + subnets: IntGauge, + ranges: IntGauge, + id_fetches: IntCounterVec, + data_fetches: IntCounterVec, + id_fetches_duration: HistogramVec, + data_fetches_duration: HistogramVec, +} + +impl Metrics { + pub fn new(registry: &Registry) -> Self { + Self { + subnets: register_int_gauge_with_registry!( + format!("routing_table_manager_subnets"), + format!("How many subnets are there"), + registry + ) + .unwrap(), + + ranges: register_int_gauge_with_registry!( + format!("routing_table_manager_ranges"), + format!("How many canister ranges are there"), + registry + ) + .unwrap(), + + id_fetches: register_int_counter_vec_with_registry!( + format!("routing_table_manager_id_fetches"), + format!("Counts number of subnet-id fetches and their outcome"), + &["success"], + registry + ) + .unwrap(), + + data_fetches: register_int_counter_vec_with_registry!( + format!("routing_table_manager_data_fetches"), + format!("Counts number of per-subnet data fetches"), + &["subnet_id", "success"], + registry + ) + .unwrap(), + + id_fetches_duration: register_histogram_vec_with_registry!( + format!("routing_table_manager_id_fetches_duration"), + format!("Records the duration of subnet_ids fetching in seconds"), + &["success"], + HTTP_DURATION_BUCKETS.to_vec(), + registry + ) + .unwrap(), + + data_fetches_duration: register_histogram_vec_with_registry!( + format!("routing_table_manager_data_fetches_duration"), + format!("Records the duration of per-subnet data fetching in seconds"), + &["subnet_id", "success"], + HTTP_DURATION_BUCKETS.to_vec(), + registry + ) + .unwrap(), + } + } +} + pub trait LooksUpSubnetType: Send + Sync { /// Looks up the type of the subnet that `canister_id` belongs to. fn lookup_subnet_type(&self, canister_id: &Principal) -> Option; @@ -213,6 +284,7 @@ pub struct RoutingTableManager { /// `None` until the first successful fetch routing_table: Arc>, interval: Duration, + metrics: Metrics, } impl fmt::Debug for RoutingTableManager { @@ -242,14 +314,20 @@ impl LooksUpSubnetType for RoutingTableManager { } impl RoutingTableManager { - pub fn new(agent: Agent, root_subnet_id: Principal, interval: Duration) -> Self { - Self::new_with_fetcher(Arc::new(agent), root_subnet_id, interval) + pub fn new( + agent: Agent, + root_subnet_id: Principal, + interval: Duration, + registry: &Registry, + ) -> Self { + Self::new_with_fetcher(Arc::new(agent), root_subnet_id, interval, registry) } fn new_with_fetcher( fetcher: Arc, root_subnet_id: Principal, interval: Duration, + registry: &Registry, ) -> Self { assert!( interval > Duration::ZERO, @@ -262,6 +340,7 @@ impl RoutingTableManager { snapshot: Mutex::new(AHashMap::with_capacity(128)), routing_table: Arc::new(ArcSwapOption::empty()), interval, + metrics: Metrics::new(registry), } } @@ -270,11 +349,9 @@ impl RoutingTableManager { // Fetch the per-subnet data concurrently let start = Instant::now(); let futures = subnet_ids.iter().map(|subnet_id| async move { - self.subnet_info_fetcher - .fetch_subnet_data(subnet_id) - .await - .map(|x| (*subnet_id, x)) - .map_err(|e| (*subnet_id, e)) + let start = Instant::now(); + let res = self.subnet_info_fetcher.fetch_subnet_data(subnet_id).await; + (subnet_id, res, start.elapsed()) }); let results = join_all(futures).await; let dur = start.elapsed(); @@ -286,14 +363,23 @@ impl RoutingTableManager { // Insert/update the new data that was fetched. // If there was an error - the older data will stay in the snapshot. let mut ok = 0; - for r in results { - match r { - Ok((subnet_id, data)) => { + for (subnet_id, res, duration) in results { + let subnet_id_str = subnet_id.to_string(); + self.metrics + .data_fetches_duration + .with_label_values(&[&subnet_id_str, res.is_ok().yesno()]) + .observe(duration.as_secs_f64()); + self.metrics + .data_fetches + .with_label_values(&[&subnet_id_str, res.is_ok().yesno()]) + .inc(); + + match res { + Ok(data) => { ok += 1; - snapshot.insert(subnet_id, data); + snapshot.insert(*subnet_id, data); } - - Err((subnet_id, e)) => { + Err(e) => { warn!("{self}: {subnet_id}: error fetching data: {e:#}"); } } @@ -308,14 +394,25 @@ impl RoutingTableManager { /// Tries to update the routing table by refreshing the per-subnet data #[allow(clippy::cast_precision_loss)] + #[allow(clippy::cast_possible_wrap)] async fn update_routing_table(&self) -> Result<(), Error> { // Get the list of all subnet's IDs let start = Instant::now(); - let subnet_ids = self + let res = self .subnet_info_fetcher .fetch_subnet_ids(self.root_subnet_id) - .await - .context("unable to fetch subnet IDs")?; + .await; + self.metrics + .id_fetches + .with_label_values(&[res.is_ok().yesno()]) + .inc(); + self.metrics + .id_fetches_duration + .with_label_values(&[res.is_ok().yesno()]) + .observe(start.elapsed().as_secs_f64()); + + let subnet_ids = res.context("unable to fetch subnet IDs")?; + self.metrics.subnets.set(subnet_ids.len() as i64); info!( "{self}: Got a list of {} subnets in {}s", @@ -360,6 +457,7 @@ impl RoutingTableManager { routing_table.ranges.len() ); + self.metrics.ranges.set(routing_table.ranges.len() as i64); self.routing_table.store(Some(routing_table)); Ok(()) } @@ -576,6 +674,7 @@ mod tests { Arc::new(fetcher), MAINNET_ROOT_SUBNET_ID, Duration::MAX, + &Registry::new(), ); // 1st update should fail due to failed fetch_subnet_ids From 246d811d6a617a81ae841d68514979a6ca3e83a4 Mon Sep 17 00:00:00 2001 From: Igor Novgorodov Date: Mon, 22 Jun 2026 16:52:33 +0000 Subject: [PATCH 2/6] Add nodes picked metrics --- src/routing/ic/route_provider/provider.rs | 32 +++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/routing/ic/route_provider/provider.rs b/src/routing/ic/route_provider/provider.rs index e271fd4..5461e09 100644 --- a/src/routing/ic/route_provider/provider.rs +++ b/src/routing/ic/route_provider/provider.rs @@ -12,7 +12,7 @@ use ic_bn_lib::ic_agent::{ AgentError, agent::route_provider::{RouteProvider, RoutesStats}, }; -use prometheus::Registry; +use prometheus::{IntCounterVec, Registry, register_int_counter_vec_with_registry}; use tokio::{select, sync::watch}; use tokio_util::{sync::CancellationToken, task::TaskTracker}; use tracing::{info, warn}; @@ -25,6 +25,25 @@ use crate::routing::ic::route_provider::{ routes::{RouteSnapshot, RoutesManager}, }; +#[derive(Clone)] +pub struct Metrics { + nodes_picked: IntCounterVec, +} + +impl Metrics { + pub fn new(registry: &Registry) -> Self { + Self { + nodes_picked: register_int_counter_vec_with_registry!( + format!("route_provider_nodes_picked"), + format!("Counts the number of times each node was picked as a route"), + &["node"], + registry + ) + .unwrap(), + } + } +} + /// Handles incoming updates of the node list on behalf of [`DynamicRouteProvider`] #[derive(new)] #[allow(clippy::struct_field_names)] @@ -73,6 +92,7 @@ pub struct DynamicRouteProvider { routes: Arc>, token: CancellationToken, tracker: TaskTracker, + metrics: Metrics, } impl Display for DynamicRouteProvider { @@ -128,6 +148,7 @@ impl DynamicRouteProvider { routes: routes.clone(), token: token.clone(), tracker: tracker.clone(), + metrics: Metrics::new(registry), }); // [`NodeList`] distribution channels - initialize with a seed list & mark it as changed to trigger updates @@ -204,7 +225,14 @@ impl RouteProvider for DynamicRouteProvider { )); }; - Ok(snapshot.wrr.next().clone()) + let url = snapshot.wrr.next().clone(); + let hostname = url.authority(); + self.metrics + .nodes_picked + .with_label_values(&[hostname]) + .inc(); + + Ok(url) } fn routes_stats(&self) -> RoutesStats { From 45076d2f6ab0831829ff55eb3c69237a7308815a Mon Sep 17 00:00:00 2001 From: Igor Novgorodov Date: Tue, 23 Jun 2026 09:21:37 +0000 Subject: [PATCH 3/6] Fix gauge, add nit tests --- src/routing/ic/route_provider/provider.rs | 2 ++ src/routing/ic/route_provider/routes.rs | 1 + 2 files changed, 3 insertions(+) diff --git a/src/routing/ic/route_provider/provider.rs b/src/routing/ic/route_provider/provider.rs index 5461e09..ae10127 100644 --- a/src/routing/ic/route_provider/provider.rs +++ b/src/routing/ic/route_provider/provider.rs @@ -345,6 +345,7 @@ mod test { // Initially there should only be a single alive seed node, // while the fetcher is blocked by a semaphore. assert_eq!(urls, vec!["https://seed_node1/".parse().unwrap()]); + assert_eq!(rp.routes_stats(), RoutesStats::new(2, Some(1))); // Reset the state rp.node_list.store(None); @@ -371,6 +372,7 @@ mod test { "https://node1/".parse().unwrap() ] ); + assert_eq!(rp.routes_stats(), RoutesStats::new(4, Some(3))); // Make sure we get requested number of routes assert_eq!( diff --git a/src/routing/ic/route_provider/routes.rs b/src/routing/ic/route_provider/routes.rs index efc5f99..5513ae5 100644 --- a/src/routing/ic/route_provider/routes.rs +++ b/src/routing/ic/route_provider/routes.rs @@ -131,6 +131,7 @@ impl RoutesManager { fn update_routes(&self, mut list: Vec) { if list.is_empty() { self.routes.store(None); + self.metrics.active_nodes.set(0); return; } From 6c5940d3c3de8f49eaec5e11c0e6419d2b60609d Mon Sep 17 00:00:00 2001 From: Igor Novgorodov Date: Tue, 23 Jun 2026 10:32:36 +0000 Subject: [PATCH 4/6] nit: add reset interval --- src/routing/ic/route_provider/fetcher.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/routing/ic/route_provider/fetcher.rs b/src/routing/ic/route_provider/fetcher.rs index 7988723..aa51870 100644 --- a/src/routing/ic/route_provider/fetcher.rs +++ b/src/routing/ic/route_provider/fetcher.rs @@ -202,6 +202,7 @@ impl FetcherManager { // We've got our first successful fetch, use normal interval now int = tokio::time::interval(interval); int.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + int.reset(); } select! { From 5590955b9df6120bf682b2e1dfe4db2ea2c5e21e Mon Sep 17 00:00:00 2001 From: Igor Novgorodov Date: Tue, 23 Jun 2026 11:11:39 +0000 Subject: [PATCH 5/6] Add per-node health & latency gauges --- src/routing/ic/route_provider/health.rs | 50 ++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/src/routing/ic/route_provider/health.rs b/src/routing/ic/route_provider/health.rs index 13c616b..bf9b045 100644 --- a/src/routing/ic/route_provider/health.rs +++ b/src/routing/ic/route_provider/health.rs @@ -15,8 +15,9 @@ use http_body_util::Full; use ic_bn_lib::{BoolYesNo, http::shed::ewma::EWMA}; use ic_bn_lib_common::traits::http::ClientHttp; use prometheus::{ - HistogramVec, IntCounterVec, IntGauge, Registry, register_histogram_vec_with_registry, - register_int_counter_vec_with_registry, register_int_gauge_with_registry, + HistogramVec, IntCounterVec, IntGauge, IntGaugeVec, Registry, + register_histogram_vec_with_registry, register_int_counter_vec_with_registry, + register_int_gauge_vec_with_registry, register_int_gauge_with_registry, }; use tokio::{ select, @@ -36,6 +37,8 @@ use crate::{ pub struct Metrics { nodes: IntGauge, healthy_nodes: IntGauge, + node_states: IntGaugeVec, + node_latencies: IntGaugeVec, checks: IntCounterVec, check_duration: HistogramVec, } @@ -57,6 +60,22 @@ impl Metrics { ) .unwrap(), + node_states: register_int_gauge_vec_with_registry!( + format!("route_provider_health_node_states"), + format!("Per-node health states (1 - healthy, 0 - not)"), + &["node"], + registry + ) + .unwrap(), + + node_latencies: register_int_gauge_vec_with_registry!( + format!("route_provider_health_node_latencies"), + format!("Per-node latencies in msec"), + &["node"], + registry + ) + .unwrap(), + checks: register_int_counter_vec_with_registry!( format!("route_provider_health_checks"), format!("Counts number of health checks per node and their outcome"), @@ -324,6 +343,19 @@ impl HealthCheckManager { self.send_healthy_nodes(); } + // Set the metrics to -1 to indicate that the node is gone. + // TODO: Create a process to clean the timeseries from the Registry. + for state in &to_stop { + self.metrics + .node_states + .with_label_values(&[&state.node.name]) + .set(-1); + self.metrics + .node_latencies + .with_label_values(&[&state.node.name]) + .set(-1); + } + // Start & stop actors join_all(to_stop.into_iter().map(|x| x.stop_actor())).await; for node in to_start { @@ -368,6 +400,7 @@ impl HealthCheckManager { /// Update the state of the given node. /// Returns whether the state has changed. #[allow(clippy::cast_precision_loss)] + #[allow(clippy::cast_possible_truncation)] fn update_node_state(&mut self, node: &Arc, status: &HealthCheckResult) -> bool { // Ignore messages from missing nodes (might be buffered from actors that are stopped already) let Some(state) = self.nodes.get_mut(&node.hostname) else { @@ -379,11 +412,24 @@ impl HealthCheckManager { state.healthy = Some(status.healthy); state.reliability.add(f64::from(status.healthy)); + self.metrics + .node_states + .with_label_values(&[&node.name]) + .set(status.healthy.into()); + // Update the latency only if the node is healthy. // Otherwise e.g. the request timeout that leads to a failed health check would // impact the latency calculations in EWMA. if status.healthy { state.latency_us.add(status.latency.as_micros() as f64); + + // Use EWMA value as a latency in the metric to show the actual latencies + // used in the route weight calculation, not the raw ones. + // SAFETY: since we already added a value above - this is guaranteed to unwrap + self.metrics + .node_latencies + .with_label_values(&[&node.name]) + .set(state.latency_us.get().unwrap() as i64 / 1000); } state_changed From c1cbbda89b91779e07e057af0a1deaf8262d52e8 Mon Sep 17 00:00:00 2001 From: Igor Novgorodov Date: Tue, 23 Jun 2026 12:49:42 +0000 Subject: [PATCH 6/6] Update crates --- Cargo.lock | 390 +++++++++++++++++++---------------------------------- Cargo.toml | 6 +- 2 files changed, 138 insertions(+), 258 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6f2f0ed..97aca13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -159,9 +159,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "arrayvec" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe" [[package]] name = "ascii-canvas" @@ -212,7 +212,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure", ] @@ -224,7 +224,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure", ] @@ -236,7 +236,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -341,7 +341,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -489,7 +489,7 @@ checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -614,7 +614,7 @@ checksum = "f48d6ace212fdf1b45fd6b566bb40808415344642b76c3224c07c8df9da81e97" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -739,9 +739,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" [[package]] name = "bytesize" @@ -813,7 +813,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -854,7 +854,7 @@ dependencies = [ "lazy_static", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -865,9 +865,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.64" +version = "1.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" dependencies = [ "find-msvc-tools", "jobserver", @@ -1005,7 +1005,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1344,7 +1344,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1415,7 +1415,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1428,7 +1428,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1450,7 +1450,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1461,7 +1461,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1533,7 +1533,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1553,7 +1553,7 @@ checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1629,7 +1629,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2045,7 +2045,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2144,16 +2144,14 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", "rand_core 0.10.1", - "wasip2", - "wasip3", ] [[package]] @@ -2279,7 +2277,7 @@ dependencies = [ "indexmap 2.14.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2355,7 +2353,7 @@ dependencies = [ "tokio-rustls", "tracing", "url", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] @@ -2408,7 +2406,7 @@ dependencies = [ "tokio", "tokio-rustls", "tracing", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] @@ -2573,7 +2571,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] @@ -2698,7 +2696,7 @@ dependencies = [ "ahash", "anyhow", "arc-swap", - "arrayvec 0.7.6", + "arrayvec 0.7.7", "async-channel 2.5.0", "async-trait", "axum", @@ -2988,16 +2986,16 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-util", - "tower-http", + "tower-http 0.6.11", "tracing", "tracing-subscriber", ] [[package]] name = "ic-custom-domains-base" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78a38ca829cb19a17ba4041a30299db3473c3d43fb1079bf5115047d33032f80" +checksum = "c0ce0d7b9b102d7680752d59016af984d223e6855bb185aefe3297bde5347854" dependencies = [ "anyhow", "async-trait", @@ -3026,9 +3024,9 @@ dependencies = [ [[package]] name = "ic-custom-domains-canister-api" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bf7c8f8bcf2b8661db6ec0e4c4d834343c54d3245c9677be8da1e333a509eac" +checksum = "98ce91ac62ff60b8031d2a6e9c8e3d1ea2460d0868fc6e2f45e97322a49378dc" dependencies = [ "candid", "derive-new", @@ -3039,9 +3037,9 @@ dependencies = [ [[package]] name = "ic-custom-domains-canister-client" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d025de4e47c44c0462057c9b71f1b1a8110536f0ef479d413376ff022ebcdf77" +checksum = "8c30f0ddaea8a7550f11c5e72a9dda8abd3742e08efd8f80ec95e41b33b17587" dependencies = [ "anyhow", "arc-swap", @@ -3111,7 +3109,7 @@ dependencies = [ "ic-http-gateway-protocol", "ic-transport-types 0.47.3", "isbot", - "itertools 0.14.0", + "itertools 0.15.0", "lazy_static", "maxminddb", "mockall", @@ -3137,7 +3135,7 @@ dependencies = [ "tokio", "tokio-util", "tower", - "tower-http", + "tower-http 0.7.0", "tracing", "tracing-core", "tracing-serde", @@ -3476,12 +3474,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -3663,6 +3655,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -3696,7 +3697,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3715,7 +3716,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3846,9 +3847,9 @@ dependencies = [ [[package]] name = "leb128" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545" +checksum = "c83bff1d572d6b9aeef67ddfc8448e4a3737909cb28e81f97c791b9018703e52" [[package]] name = "leb128fmt" @@ -3906,9 +3907,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lru-cache" @@ -3942,9 +3943,9 @@ checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" [[package]] name = "mail-auth" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c99f0db3f1292f72149bd50fcca0354d243defda0d26532ef7281ca716c77592" +checksum = "62a9d0c96cc84030705fec4bc4789fa918c89caec0d95ce7ee96b6f922d4a664" dependencies = [ "aws-lc-rs", "flate2", @@ -3970,9 +3971,9 @@ dependencies = [ [[package]] name = "mail-parser" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2420e9ce11c2b0583ca97ddff7ab2398c8a613154e9b72e3bafdbf767f1d7" +checksum = "f2c0e7e0704500930be5b6c629f30d23fd1dde4d1800e138e04b3fa302e64d51" dependencies = [ "encoding_rs", "hashify", @@ -4096,7 +4097,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4539,7 +4540,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4765,16 +4766,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn 2.0.117", -] - [[package]] name = "primeorder" version = "0.13.6" @@ -4828,7 +4819,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4893,18 +4884,18 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.39.4" +version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" dependencies = [ "memchr", ] [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", @@ -4922,9 +4913,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "aws-lc-rs", "bytes", @@ -4958,9 +4949,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.45" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] @@ -5005,7 +4996,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20 0.10.0", - "getrandom 0.4.2", + "getrandom 0.4.3", "rand_core 0.10.1", ] @@ -5162,7 +5153,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5256,14 +5247,14 @@ dependencies = [ "tokio-rustls", "tokio-util", "tower", - "tower-http", + "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams 0.4.2", "web-sys", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] @@ -5302,7 +5293,7 @@ dependencies = [ "tokio-rustls", "tokio-util", "tower", - "tower-http", + "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", @@ -5400,9 +5391,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ "aws-lc-rs", "brotli", @@ -5438,7 +5429,7 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.18", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", "x509-parser 0.16.0", ] @@ -5587,7 +5578,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5690,7 +5681,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5701,7 +5692,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5736,7 +5727,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5791,7 +5782,7 @@ dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6019,7 +6010,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6146,7 +6137,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6158,7 +6149,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6192,9 +6183,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -6218,7 +6209,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6282,7 +6273,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -6329,7 +6320,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6340,7 +6331,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6385,9 +6376,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.49" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" dependencies = [ "deranged", "num-conv", @@ -6405,9 +6396,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.29" +version = "0.2.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" +checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" dependencies = [ "num-conv", "time-core", @@ -6466,7 +6457,7 @@ checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6505,7 +6496,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6629,6 +6620,26 @@ dependencies = [ "url", ] +[[package]] +name = "tower-http" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "http 1.4.2", + "http-body", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -6691,7 +6702,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6978,16 +6989,7 @@ version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", + "wit-bindgen", ] [[package]] @@ -7032,7 +7034,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -7045,16 +7047,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser 0.244.0", -] - [[package]] name = "wasm-encoder" version = "0.252.0" @@ -7062,19 +7054,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8185ae345fa5687c054626ff9a50e7089797a343d9904d1dc9820eb4c4d3196f" dependencies = [ "leb128fmt", - "wasmparser 0.252.0", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap 2.14.0", - "wasm-encoder 0.244.0", - "wasmparser 0.244.0", + "wasmparser", ] [[package]] @@ -7103,18 +7083,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap 2.14.0", - "semver", -] - [[package]] name = "wasmparser" version = "0.252.0" @@ -7136,7 +7104,7 @@ dependencies = [ "leb128fmt", "memchr", "unicode-width", - "wasm-encoder 0.252.0", + "wasm-encoder", ] [[package]] @@ -7170,9 +7138,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" dependencies = [ "rustls-pki-types", ] @@ -7183,14 +7151,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] name = "webpki-roots" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" dependencies = [ "rustls-pki-types", ] @@ -7253,7 +7221,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7264,7 +7232,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7458,100 +7426,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap 2.14.0", - "prettyplease", - "syn 2.0.117", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn 2.0.117", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap 2.14.0", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder 0.244.0", - "wasm-metadata", - "wasmparser 0.244.0", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap 2.14.0", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser 0.244.0", -] - [[package]] name = "woothee" version = "0.13.0" @@ -7669,7 +7549,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure", ] @@ -7690,7 +7570,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7710,7 +7590,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure", ] @@ -7731,7 +7611,7 @@ checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7764,7 +7644,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6aa179e..8f95162 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,12 +46,12 @@ ic-bn-lib = { version = "0.2.1", features = [ ] } ic-bn-lib-common = "0.2.1" ic-custom-domains-backend = "0.2" -ic-custom-domains-base = "0.2" +ic-custom-domains-base = "0.2.3" ic-http-certification = { version = "3.1.0", optional = true } ic-transport-types = "0.47" ic-http-gateway-protocol = { package = "ic-http-gateway-protocol", git = "https://github.com/dfinity/ic-http-gateway-protocol", tag = "v0.5.1" } isbot = "0.1" -itertools = "0.14.0" +itertools = "0.15.0" lazy_static = "1.5.0" maxminddb = "0.28.1" moka = { version = "0.12.8", features = ["sync", "future"] } @@ -83,7 +83,7 @@ time = { version = "0.3.47", features = ["macros", "serde"] } tokio = { version = "1.45.0", features = ["full", "tracing"] } tokio-util = { version = "0.7.12", features = ["full"] } tower = { version = "0.5.1", features = ["limit"] } -tower-http = { version = "0.6.1", features = ["cors", "compression-full"] } +tower-http = { version = "0.7", features = ["cors", "compression-full"] } tracing = "0.1.40" tracing-core = "0.1.32" tracing-serde = "0.2.0"