feat(libsy): expose algorithm metrics through driver

crates/libsy/src/algorithms/stage.rs

@@ -232,7 +232,9 @@ mod tests {
 
     use super::*;
     use crate::algorithms::util::stage::DECISION_SOURCE_KEY;
-    use crate::core::algorithm::{Algorithm, LlmTarget};
+    use crate::core::algorithm::{
+        Algorithm, AlgorithmMetricValue, LlmTarget, RunObservation, RunObserver,
+    };
     use crate::core::classifier::Score;
     use crate::core::state::StateValue;
     use switchyard_protocol::{
@@ -509,9 +511,14 @@ mod tests {
         let client = Arc::new(RecordingClient::default());
         let router = recording_router(client.clone(), config_with_notes())?;
         let ctx = Context::default();
+        let observations = Arc::new(Mutex::new(Vec::new()));
+        let observed = observations.clone();
+        let observer: RunObserver = Arc::new(move |observation| observed.lock().push(observation));
 
         router.clone().run(ctx.clone(), turn_request(false)).await?;
-        router.run(ctx, turn_request(true)).await?;
+        router
+            .run_observed(ctx, turn_request(true), Some(observer))
+            .await?;
 
         let calls = client.routed();
         assert_eq!(calls[0].target, "weak");
@@ -529,6 +536,49 @@ mod tests {
             "escalating turn should carry the note last: {:?}",
             calls[1].messages
         );
+
+        let metrics = observations
+            .lock()
+            .iter()
+            .filter_map(|observation| match observation {
+                RunObservation::AlgorithmMetric(metric) => Some(metric.clone()),
+                _ => None,
+            })
+            .collect::<Vec<_>>();
+        assert_eq!(metrics.len(), 6);
+        assert!(metrics.iter().all(|metric| {
+            metric.algorithm == STAGE_ROUTER
+                && matches!(metric.value, AlgorithmMetricValue::Histogram(_))
+        }));
+        assert_eq!(
+            metrics
+                .iter()
+                .filter(|metric| metric.name == "switchyard.stage_router.score")
+                .count(),
+            1
+        );
+        assert_eq!(
+            metrics
+                .iter()
+                .filter(|metric| metric.name == "switchyard.stage_router.confidence")
+                .count(),
+            1
+        );
+        let mut dimensions = metrics
+            .iter()
+            .filter(|metric| metric.name == "switchyard.stage_router.dimension")
+            .filter_map(|metric| {
+                metric
+                    .attributes
+                    .first()
+                    .map(|attribute| attribute.value.clone())
+            })
+            .collect::<Vec<_>>();
+        dimensions.sort();
+        assert_eq!(
+            dimensions,
+            ["exploring", "production_intensity", "severity", "spinning"]
+        );
         Ok(())
     }
 

crates/libsy/src/algorithms/util/llm_judge.rs

@@ -18,7 +18,7 @@ use switchyard_protocol::{
 };
 
 use super::classifier_contract::ClassifierContract;
-use crate::core::algorithm::{Driver, LlmTarget};
+use crate::core::algorithm::{Driver, LlmTarget, MetricAttribute};
 use crate::core::classifier::{Classification, Classifier};
 use crate::core::state::State;
 use crate::{LibsyError, Result};
@@ -235,31 +235,47 @@ where
                 }),
             )
             .await
-            .inspect_err(|error| report_fail_open(judge_model, error, libsy_error_reason(error)))
+            .inspect_err(|error| {
+                report_fail_open(driver, judge_model, error, libsy_error_reason(error))
+            })
             .ok()?;
         let aggregate = response
             .llm_response
             .into_agg()
             .await
-            .inspect_err(|error| report_fail_open(judge_model, error, client_error_reason(error)))
+            .inspect_err(|error| {
+                report_fail_open(driver, judge_model, error, client_error_reason(error))
+            })
             .ok()?;
         self.judge
             .parse(&aggregate)
-            .inspect_err(|error| report_fail_open(judge_model, error, "parse_error"))
+            .inspect_err(|error| report_fail_open(driver, judge_model, error, "parse_error"))
             .ok()
     }
 }
 
 /// Logs and counts a judge failure with a bounded label that excludes message content.
-fn report_fail_open(judge_model: &str, error: &dyn std::fmt::Display, reason: &'static str) {
+fn report_fail_open(
+    driver: &Driver,
+    judge_model: &str,
+    error: &dyn std::fmt::Display,
+    reason: &'static str,
+) {
     tracing::warn!(
         target: "libsy",
         judge_model,
         reason,
         error = %error,
         "judge verdict unavailable; routing without one"
     );
-    crate::observability::record_classifier_fail_open(judge_model, reason);
+    driver.record_counter(
+        "switchyard.classifier_fail_open",
+        1,
+        [
+            MetricAttribute::new("judge_model", judge_model),
+            MetricAttribute::new("reason", reason),
+        ],
+    );
 }
 
 /// Returns a bounded reason for a judge call that failed at the libsy layer.

crates/libsy/src/algorithms/util/stage.rs

@@ -23,7 +23,7 @@ use serde::Deserialize;
 use super::prompts;
 use super::tool_signals::ToolSignals;
 use crate::Result;
-use crate::core::algorithm::Driver;
+use crate::core::algorithm::{Driver, MetricAttribute};
 use crate::core::classifier::{Classification, Classifier, Score};
 use crate::core::state::{State, StateValue};
 use switchyard_protocol::Request;
@@ -43,6 +43,9 @@ const HARD_SEVERITY: f64 = 0.7;
 const SIGNAL_UNIT: f64 = 0.10;
 /// Critical severity forces the capable tier regardless of the scorer.
 const SEVERITY_CRITICAL: f32 = 1.0;
+const SCORE_METRIC: &str = "switchyard.stage_router.score";
+const CONFIDENCE_METRIC: &str = "switchyard.stage_router.confidence";
+const DIMENSION_METRIC: &str = "switchyard.stage_router.dimension";
 
 /// The two tiers a turn can route to.
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -340,6 +343,37 @@ pub fn pick_tier(signal: &ToolSignals, mode: PickerMode, confidence_threshold: f
     }
 }
 
+/// Records the router's scorer output and four bounded input dimensions.
+fn record_metrics(driver: Option<&Driver>, signal: &ToolSignals, outcome: &PickOutcome) {
+    let Some(driver) = driver else {
+        return;
+    };
+    let (score, confidence) = match outcome {
+        PickOutcome::Resolved {
+            score, confidence, ..
+        } => (*score, confidence.unwrap_or(score.abs())),
+        PickOutcome::ConsultClassifier {
+            score, confidence, ..
+        } => (*score, *confidence),
+    };
+    driver.record_histogram(SCORE_METRIC, score, []);
+    driver.record_histogram(CONFIDENCE_METRIC, confidence, []);
+
+    let dimensions = dimensions_from_signal(signal);
+    for (name, value) in [
+        ("severity", dimensions.severity),
+        ("spinning", dimensions.spinning),
+        ("exploring", dimensions.exploring),
+        ("production_intensity", dimensions.production_intensity),
+    ] {
+        driver.record_histogram(
+            DIMENSION_METRIC,
+            value,
+            [MetricAttribute::new("dimension", name)],
+        );
+    }
+}
+
 /// Build a resolved outcome (a decision made without the classifier).
 fn resolved(
     tier: Tier,
@@ -484,7 +518,7 @@ impl Classifier<State> for StageClassifier {
         &self,
         state: &mut State,
         request: &mut Request,
-        _driver: Option<&Driver>,
+        driver: Option<&Driver>,
     ) -> Result<(Classification, Option<switchyard_protocol::Response>)> {
         let tool_signals = &state.tool_signals;
         let Some(signal) = tool_signals else {
@@ -494,6 +528,7 @@ impl Classifier<State> for StageClassifier {
         };
 
         let outcome = pick_tier(signal, self.mode, self.confidence_threshold);
+        record_metrics(driver, signal, &outcome);
         match outcome {
             PickOutcome::Resolved {
                 tier,

crates/libsy/src/core/algorithm.rs

@@ -53,13 +53,56 @@ pub struct LlmCallObservation {
     pub usage: Option<Usage>,
 }
 
+/// One bounded attribute attached to an algorithm-defined metric.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct MetricAttribute {
+    /// Stable, low-cardinality attribute name.
+    pub key: &'static str,
+    /// Stable, low-cardinality attribute value.
+    pub value: String,
+}
+
+impl MetricAttribute {
+    /// Build an attribute from a static key and owned or borrowed value.
+    pub fn new(key: &'static str, value: impl Into<String>) -> Self {
+        Self {
+            key,
+            value: value.into(),
+        }
+    }
+}
+
+/// The value and aggregation kind of an algorithm-defined metric.
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum AlgorithmMetricValue {
+    /// A non-negative delta added to a cumulative counter.
+    Counter(u64),
+    /// One sample recorded in a histogram.
+    Histogram(f64),
+}
+
+/// One algorithm-defined metric emitted to OpenTelemetry and the run observer.
+#[derive(Clone, Debug, PartialEq)]
+pub struct AlgorithmMetricObservation {
+    /// Algorithm that emitted the metric.
+    pub algorithm: String,
+    /// Stable OpenTelemetry metric name. One name must always use the same value kind.
+    pub name: &'static str,
+    /// Counter delta or histogram sample.
+    pub value: AlgorithmMetricValue,
+    /// Bounded dimensions for grouping the metric.
+    pub attributes: Vec<MetricAttribute>,
+}
+
 /// One request-scoped observation emitted by the algorithm runner.
 #[derive(Clone, Debug)]
 pub enum RunObservation {
     /// A completed model call.
     LlmCall(LlmCallObservation),
     /// Routing time recorded by the `switchyard.routing_overhead_ms` metric.
     RoutingOverhead(Duration),
+    /// An algorithm-defined counter delta or histogram sample.
+    AlgorithmMetric(AlgorithmMetricObservation),
 }
 
 /// Request-scoped callback for algorithm-run observations.
@@ -152,6 +195,7 @@ impl CallLlmRequest {
 #[derive(Clone)]
 pub struct Driver {
     driver: TypeErasedDriver,
+    algorithm: Arc<str>,
     // How long the call that served this run took. We need this to calculate routing overhead.
     routed_call: Arc<Mutex<Option<Duration>>>,
     observer: Option<RunObserver>,
@@ -161,12 +205,13 @@ impl Driver {
     /// Build an empty driver with its step channel ready. Created per call by
     /// [`run_stream`](Algorithm::run_stream).
     pub(crate) fn new() -> Self {
-        Self::with_observer(None)
+        Self::with_observer("", None)
     }
 
-    fn with_observer(observer: Option<RunObserver>) -> Self {
+    fn with_observer(algorithm: &str, observer: Option<RunObserver>) -> Self {
         Self {
             driver: TypeErasedDriver::new(),
+            algorithm: Arc::from(algorithm),
             routed_call: Arc::new(Mutex::new(None)),
             observer,
         }
@@ -185,6 +230,64 @@ impl Driver {
         }
     }
 
+    /// Add a delta to an algorithm-defined OpenTelemetry counter and report the
+    /// same event to this run's observer. Names and attributes must be stable and
+    /// low-cardinality; never include request or session data.
+    pub fn record_counter(
+        &self,
+        name: &'static str,
+        delta: u64,
+        attributes: impl IntoIterator<Item = MetricAttribute>,
+    ) {
+        self.record_algorithm_metric(
+            name,
+            AlgorithmMetricValue::Counter(delta),
+            attributes.into_iter().collect(),
+        );
+    }
+
+    /// Record one sample in an algorithm-defined OpenTelemetry histogram and
+    /// report the same event to this run's observer. Non-finite samples are dropped.
+    pub fn record_histogram(
+        &self,
+        name: &'static str,
+        sample: f64,
+        attributes: impl IntoIterator<Item = MetricAttribute>,
+    ) {
+        if !sample.is_finite() {
+            tracing::debug!(
+                metric = name,
+                sample,
+                "dropping non-finite histogram sample"
+            );
+            return;
+        }
+        self.record_algorithm_metric(
+            name,
+            AlgorithmMetricValue::Histogram(sample),
+            attributes.into_iter().collect(),
+        );
+    }
+
+    fn record_algorithm_metric(
+        &self,
+        name: &'static str,
+        value: AlgorithmMetricValue,
+        mut attributes: Vec<MetricAttribute>,
+    ) {
+        attributes.retain(|attribute| attribute.key != "algorithm");
+        let observation = AlgorithmMetricObservation {
+            algorithm: self.algorithm.to_string(),
+            name,
+            value,
+            attributes,
+        };
+        observability::record_algorithm_metric(&observation);
+        if let Some(observer) = &self.observer {
+            observer(RunObservation::AlgorithmMetric(observation));
+        }
+    }
+
     /// Offload a model call: publish `routed` as a [`Step::CallLlm`] and await the
     /// consumer's [`Response`]. The call's context travels inside
     /// [`routed.ctx`](RoutedRequest::ctx). Errors if the stream is closed or the call failed.
@@ -626,7 +729,8 @@ pub trait Algorithm: Send + Sync + 'static {
     /// emitted as an `Err` item. Dropping the stream aborts the spawned algorithm task.
     ///
     /// Every invocation owns a separate [`Driver`]. `observer`, when present, receives
-    /// each completed model call and, after a successful routed run, its routing overhead.
+    /// completed model calls, algorithm-defined metrics, and, after a successful routed
+    /// run, its routing overhead.
     fn run_stream(
         self: Arc<Self>,
         ctx: Context,
@@ -640,7 +744,7 @@ pub trait Algorithm: Send + Sync + 'static {
             observability::ALGORITHM_KEY.to_string(),
             self.name().to_string(),
         );
-        let driver = Driver::with_observer(observer);
+        let driver = Driver::with_observer(self.name(), observer);
         let task_driver = driver.clone();
         let task_ctx = ctx.clone();
         let stream = task_driver.stream();
@@ -695,7 +799,7 @@ pub trait Algorithm: Send + Sync + 'static {
         self.run_observed(ctx, request, None).await
     }
 
-    /// Process a request to completion while reporting each model call to `observer`.
+    /// Process a request to completion while reporting run observations to `observer`.
     async fn run_observed(
         self: Arc<Self>,
         ctx: Context,

crates/libsy/src/lib.rs

@@ -6,8 +6,9 @@
 
 mod core;
 pub use core::algorithm::{
-    Algorithm, CallLlmRequest, Driver, LlmCallObservation, LlmTarget, LlmTargetSet, RoutedRequest,
-    RunObservation, RunObserver, Step, StepStream,
+    Algorithm, AlgorithmMetricObservation, AlgorithmMetricValue, CallLlmRequest, Driver,
+    LlmCallObservation, LlmTarget, LlmTargetSet, MetricAttribute, RoutedRequest, RunObservation,
+    RunObserver, Step, StepStream,
 };
 pub use core::classifier::{Classification, Classifier, Score};
 pub use core::processor::{Event, Processor};