feat(NODE-1965): Add metrics to route provider & routing table manager

Cargo.toml

@@ -46,12 +46,12 @@ ic-bn-lib = { version = "0.2.1", features = [
 ] }
 ic-bn-lib-common = "0.2.1"
 ic-custom-domains-backend = "0.2"
-ic-custom-domains-base = "0.2"
+ic-custom-domains-base = "0.2.3"
 ic-http-certification = { version = "3.1.0", optional = true }
 ic-transport-types = "0.47"
 ic-http-gateway-protocol = { package = "ic-http-gateway-protocol", git = "https://github.com/dfinity/ic-http-gateway-protocol", tag = "v0.5.1" }
 isbot = "0.1"
-itertools = "0.14.0"
+itertools = "0.15.0"
 lazy_static = "1.5.0"
 maxminddb = "0.28.1"
 moka = { version = "0.12.8", features = ["sync", "future"] }
@@ -83,7 +83,7 @@ time = { version = "0.3.47", features = ["macros", "serde"] }
 tokio = { version = "1.45.0", features = ["full", "tracing"] }
 tokio-util = { version = "0.7.12", features = ["full"] }
 tower = { version = "0.5.1", features = ["limit"] }
-tower-http = { version = "0.6.1", features = ["cors", "compression-full"] }
+tower-http = { version = "0.7", features = ["cors", "compression-full"] }
 tracing = "0.1.40"
 tracing-core = "0.1.32"
 tracing-serde = "0.2.0"

src/core.rs

@@ -143,7 +143,7 @@ pub async fn main(
 
     // Create route provider
     let (route_provider, dynamic_route_provider) =
-        setup_route_provider(cli, http_client_hyper, http_service.clone()).await?;
+        setup_route_provider(cli, http_client_hyper, http_service.clone(), &registry).await?;
     health_manager.add(Arc::new(RouteProviderWrapper::new(route_provider.clone())));
 
     // Create a separate Agent to use solely with Resolver.
@@ -276,6 +276,7 @@ pub async fn main(
         ic_agent.clone(),
         MAINNET_ROOT_SUBNET_ID,
         cli.ic.ic_routing_table_poll_interval,
+        &registry,
     ));
     health_manager.add(routing_table_manager.clone());
     tasks.add("subnets_info_fetcher", routing_table_manager.clone());

src/routing/ic/route_provider/fetcher.rs

@@ -2,25 +2,72 @@ use std::{
     fmt::{Debug, Display},
     str::FromStr,
     sync::Arc,
-    time::Duration,
+    time::{Duration, Instant},
 };
 
 use anyhow::Context;
 use async_trait::async_trait;
 use fqdn::FQDN;
-use ic_bn_lib::ic_agent::{
-    Agent,
-    agent::{HttpService, route_provider::RouteProvider},
+use ic_bn_lib::{
+    BoolYesNo,
+    ic_agent::{
+        Agent,
+        agent::{HttpService, route_provider::RouteProvider},
+    },
+};
+use prometheus::{
+    HistogramVec, IntCounterVec, IntGauge, Registry, register_histogram_vec_with_registry,
+    register_int_counter_vec_with_registry, register_int_gauge_with_registry,
 };
 use tokio::{select, sync::watch};
 use tokio_util::sync::CancellationToken;
 use tracing::{info, warn};
 
-use crate::routing::ic::{
-    MAINNET_ROOT_SUBNET_ID,
-    route_provider::{FetchesNodes, NodeList, RouteError},
+use crate::{
+    metrics::HTTP_DURATION_BUCKETS,
+    routing::ic::{
+        MAINNET_ROOT_SUBNET_ID,
+        route_provider::{FetchesNodes, NodeList, RouteError},
+    },
 };
 
+#[derive(Clone)]
+pub struct Metrics {
+    nodes: IntGauge,
+    fetches: IntCounterVec,
+    duration: HistogramVec,
+}
+
+impl Metrics {
+    pub fn new(registry: &Registry) -> Self {
+        Self {
+            nodes: register_int_gauge_with_registry!(
+                format!("route_provider_fetcher_nodes"),
+                format!("How many nodes were fetched"),
+                registry
+            )
+            .unwrap(),
+
+            fetches: register_int_counter_vec_with_registry!(
+                format!("route_provider_fetcher_count"),
+                format!("Counts number of fetches and their outcome"),
+                &["success"],
+                registry
+            )
+            .unwrap(),
+
+            duration: register_histogram_vec_with_registry!(
+                format!("route_provider_fetcher_duration"),
+                format!("Records the duration of node fetching in seconds"),
+                &["success"],
+                HTTP_DURATION_BUCKETS.to_vec(),
+                registry
+            )
+            .unwrap(),
+        }
+    }
+}
+
 /// Fetches a list of API BN nodes using an IC Agent
 #[derive(Debug)]
 pub struct AgentFetcher {
@@ -64,6 +111,7 @@ pub struct FetcherManager {
     fetcher: Arc<dyn FetchesNodes>,
     tx: watch::Sender<NodeList>,
     snapshot: NodeList,
+    metrics: Metrics,
 }
 
 impl Display for FetcherManager {
@@ -79,16 +127,33 @@ impl Debug for FetcherManager {
 }
 
 impl FetcherManager {
-    pub fn new(fetcher: Arc<dyn FetchesNodes>, tx: watch::Sender<NodeList>) -> Self {
+    pub fn new(
+        fetcher: Arc<dyn FetchesNodes>,
+        tx: watch::Sender<NodeList>,
+        registry: &Registry,
+    ) -> Self {
         Self {
             fetcher,
             tx,
             snapshot: NodeList::default(),
+            metrics: Metrics::new(registry),
         }
     }
 
+    #[allow(clippy::cast_possible_wrap)]
     async fn refresh(&mut self) -> Result<(), RouteError> {
-        let nodes = self.fetcher.fetch_nodes().await?;
+        let start = Instant::now();
+        let res = self.fetcher.fetch_nodes().await;
+        let success = res.is_ok().yesno();
+
+        self.metrics
+            .duration
+            .with_label_values(&[success])
+            .observe(start.elapsed().as_secs_f64());
+        self.metrics.fetches.with_label_values(&[success]).inc();
+
+        let nodes = res?;
+        self.metrics.nodes.set(nodes.len() as i64);
 
         // Safeguard against a case when (for whatever reason) an empty node list is fetched.
         // If we remove all nodes, then we'll end up in a deadlock situation: we can't fetch a new (correct)
@@ -137,6 +202,7 @@ impl FetcherManager {
                 // We've got our first successful fetch, use normal interval now
                 int = tokio::time::interval(interval);
                 int.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+                int.reset();
             }
 
             select! {
@@ -185,7 +251,7 @@ mod test {
     async fn test_fetcher() {
         let fetcher = TestFetcher::default();
         let (tx, mut rx) = watch::channel(NodeList::new(vec![]));
-        let mut manager = FetcherManager::new(Arc::new(fetcher), tx);
+        let mut manager = FetcherManager::new(Arc::new(fetcher), tx, &Registry::new());
 
         // Consume the initial value
         assert!(rx.borrow_and_update().is_empty());