hummingbird: router metrics

acceptance/router_priority/test.py

@@ -51,6 +51,10 @@ def measure_br(url: str):
             "total": 0,
             "interface": defaultdict(int),
         },
+        "router_priority_forwarded_pkts":{
+            "total": 0,
+            "interface": defaultdict(int),
+        },
     }
     text = requests.get(url).text
     for family in text_string_to_metric_families(text):
@@ -135,6 +139,19 @@ def _run(self):
         if busy_fwd == 0:
             print(f"Insufficient load: no packet drop occurred.")
             sys.exit(1)
+        bfd_sent_delta = metrics_after["router_bfd_sent_packets"]["total"] -\
+            metrics_before["router_bfd_sent_packets"]["total"]
+        print(f"BFD sent packets delta: {bfd_sent_delta}")
+        prio_fwd = metrics_after["router_priority_forwarded_pkts"]["total"] -\
+            metrics_before["router_priority_forwarded_pkts"]["total"]
+        print(f"Priority-forwarded packets delta: {prio_fwd}")
+        if prio_fwd <= 0:
+            print("Expected priority-forwarded packets to increase, but it did not.")
+            sys.exit(1)
+        if prio_fwd < bfd_sent_delta:
+            print("Priority-forwarded packets delta is lower than BFD sent packets delta.")
+            print(f"prio_fwd={prio_fwd}, bfd_sent_delta={bfd_sent_delta}")
+            sys.exit(1)
         print(f"router metrics follow.\n"
             f"Before:\n-----8<-----\n{metrics_before}\n-----8<-----\n"
             f"After: \n-----8<-----\n{metrics_after}\n-----8<-----")

doc/manuals/router/metrics.rst

@@ -57,6 +57,7 @@ local system (if any) are not counted in this number.
 
 **Labels**: ``interface``, ``isd_as`` and ``neighbor_isd_as``.
 
+
 Dropped packets total
 ---------------------
 
@@ -69,6 +70,19 @@ This metric reports the number of packets that were dropped because of errors.
 
 **Labels**: ``interface``, ``isd_as`` and ``neighbor_isd_as``.
 
+Priority forwarded packets total
+--------------------------------
+
+**Name**: ``router_priority_forwarded_pkts_total``
+
+**Type**: Counter
+
+**Description**: Total number of priority packets successfully forwarded by the
+router.
+
+**Labels**: ``interface``, ``isd_as``, ``neighbor_isd_as`` and ``sizeclass``.
+
+
 BFD state changes (inter-AS)
 ----------------------------
 
@@ -117,6 +131,66 @@ router in the local AS.
 
 **Labels**: ``sibling`` and ``isd_as``.
 
+Hummingbird packets processed total
+-----------------------------------
+
+**Name**: ``router_humm_processed_pkts_total``
+
+**Type**: Counter
+
+**Description**: Total number of Hummingbird packets received by the router
+processor.
+
+**Labels**: ``interface``, ``isd_as``, ``neighbor_isd_as`` and ``sizeclass``.
+
+Hummingbird flyover packets total
+---------------------------------
+
+**Name**: ``router_humm_flyover_pkts_total``
+
+**Type**: Counter
+
+**Description**: Total number of parsed Hummingbird packets with flyover hop
+fields.
+
+**Labels**: ``interface``, ``isd_as``, ``neighbor_isd_as`` and ``sizeclass``.
+
+Hummingbird freshness demotions total
+-------------------------------------
+
+**Name**: ``router_humm_demoted_freshness_total``
+
+**Type**: Counter
+
+**Description**: Total number of Hummingbird packets demoted to best-effort due
+to freshness checks.
+
+**Labels**: ``interface``, ``isd_as``, ``neighbor_isd_as`` and ``sizeclass``.
+
+Hummingbird expiration demotions total
+--------------------------------------
+
+**Name**: ``router_humm_demoted_expired_total``
+
+**Type**: Counter
+
+**Description**: Total number of Hummingbird packets demoted to best-effort due
+to expired reservations.
+
+**Labels**: ``interface``, ``isd_as``, ``neighbor_isd_as`` and ``sizeclass``.
+
+Hummingbird token bucket demotions total
+----------------------------------------
+
+**Name**: ``router_humm_demoted_tokenbucket_total``
+
+**Type**: Counter
+
+**Description**: Total number of Hummingbird packets demoted to best-effort due
+to token bucket checks.
+
+**Labels**: ``interface``, ``isd_as``, ``neighbor_isd_as`` and ``sizeclass``.
+
 Service instance count
 ----------------------
 

pkg/snet/squic/hummingbird_live_test.go

@@ -15,10 +15,15 @@
 package squic_test
 
 import (
+	"bufio"
 	"context"
+	"fmt"
+	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -35,6 +40,13 @@ const (
 	tinyServerRemoteAddr = "1-ff00:0:111,127.0.0.20:12345"
 )
 
+var tinyBRMetricsEndpoints = []string{
+	"http://127.0.0.9:30442/metrics",
+	"http://127.0.0.10:30442/metrics",
+	"http://127.0.0.17:30442/metrics",
+	"http://[fd00:f00d:cafe::7f00:9]:30442/metrics",
+}
+
 // TestQUICOverHummingbirdTinyTopology verifies that a QUIC handshake plus a
 // stream exchange succeeds when the client sends over a Hummingbird reservation
 // path in the running tiny topology.
@@ -83,6 +95,73 @@ func TestQUICOverHummingbirdTinyTopology(t *testing.T) {
 	require.NoError(t, <-serverErr)
 }
 
+// TestQUICOverHummingbirdTinyTopologyTokenBucketDemotion verifies that QUIC
+// still succeeds when the reservation bandwidth is intentionally tiny, causing
+// routers to demote packets to best-effort via token bucket checks.
+func TestQUICOverHummingbirdTinyTopologyTokenBucketDemotion(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping live tiny-topology test in short mode")
+	}
+	if os.Getenv("SCION_RUN_LIVE_TESTS") == "" {
+		t.Skip("set SCION_RUN_LIVE_TESTS=1 to run live tiny-topology tests")
+	}
+
+	before, err := scrapeHummCounterTotals(tinyBRMetricsEndpoints)
+	require.NoError(t, err)
+
+	keysRoot := requireTinyTopologyAssets(t)
+	serverLocal, err := hummingbirdtest.MustParseUDPAddr(tinyServerListenAddr)
+	require.NoError(t, err)
+	clientLocal, err := hummingbirdtest.MustParseUDPAddr(tinyClientListenAddr)
+	require.NoError(t, err)
+	serverRemote, err := hummingbirdtest.MustParseUDPAddr(tinyServerRemoteAddr)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+
+	serverErr := make(chan error, 1)
+	go func() {
+		serverErr <- hummingbirdtest.RunServer(
+			ctx,
+			tinyServerDaemonAddr,
+			serverLocal,
+			clientLocal.IA,
+			t.Logf,
+		)
+	}()
+
+	params := hummingbirdtest.DefaultReservationParams()
+	// Intentionally tiny so payload quickly exceeds token bucket and gets demoted.
+	params.Bandwidth = 1
+	err = hummingbirdtest.RunClientWithParams(
+		ctx,
+		tinyClientDaemonAddr,
+		clientLocal,
+		serverRemote,
+		keysRoot,
+		params,
+		t.Logf,
+	)
+	require.NoErrorf(t, err,
+		"QUIC dial timed out or failed; demoted packets should still complete over best-effort")
+
+	require.NoError(t, <-serverErr)
+
+	after, err := scrapeHummCounterTotals(tinyBRMetricsEndpoints)
+	require.NoError(t, err)
+
+	hummCounter := deltaCounter(after, before, "router_humm_processed_pkts_total")
+	flyoverCounter := deltaCounter(after, before, "router_humm_flyover_pkts_total")
+	demotedCounter := deltaCounter(after, before, "router_humm_demoted_tokenbucket_total")
+	t.Logf("total hummingbird packets: %f", hummCounter)
+	t.Logf("total flyover packets: %f", flyoverCounter)
+	t.Logf("total demotions: %f", demotedCounter)
+	require.Greater(t, hummCounter, float64(0))
+	require.Greater(t, flyoverCounter, float64(0))
+	require.Greater(t, demotedCounter, float64(0))
+}
+
 func requireTinyTopologyAssets(t *testing.T) string {
 	t.Helper()
 
@@ -100,3 +179,60 @@ func requireRepoRoot(t *testing.T) string {
 	require.True(t, ok, "resolve current file path")
 	return filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
 }
+
+func scrapeHummCounterTotals(endpoints []string) (map[string]float64, error) {
+	totals := map[string]float64{
+		"router_humm_processed_pkts_total":      0,
+		"router_humm_flyover_pkts_total":        0,
+		"router_humm_demoted_freshness_total":   0,
+		"router_humm_demoted_expired_total":     0,
+		"router_humm_demoted_tokenbucket_total": 0,
+	}
+	client := &http.Client{Timeout: 3 * time.Second}
+
+	for _, endpoint := range endpoints {
+		resp, err := client.Get(endpoint)
+		if err != nil {
+			return nil, fmt.Errorf("fetching %s: %w", endpoint, err)
+		}
+		if resp.StatusCode != http.StatusOK {
+			_ = resp.Body.Close()
+			return nil, fmt.Errorf("fetching %s: status %s", endpoint, resp.Status)
+		}
+
+		scanner := bufio.NewScanner(resp.Body)
+		for scanner.Scan() {
+			line := scanner.Text()
+			if !strings.HasPrefix(line, "router_humm_") || strings.HasPrefix(line, "#") {
+				continue
+			}
+			space := strings.LastIndexByte(line, ' ')
+			if space <= 0 || space == len(line)-1 {
+				continue
+			}
+			namePart := line[:space]
+			metricName := namePart
+			if brace := strings.IndexByte(namePart, '{'); brace >= 0 {
+				metricName = namePart[:brace]
+			}
+			if _, ok := totals[metricName]; !ok {
+				continue
+			}
+			v, err := strconv.ParseFloat(strings.TrimSpace(line[space+1:]), 64)
+			if err != nil {
+				continue
+			}
+			totals[metricName] += v
+		}
+		if err := scanner.Err(); err != nil {
+			_ = resp.Body.Close()
+			return nil, fmt.Errorf("reading %s: %w", endpoint, err)
+		}
+		_ = resp.Body.Close()
+	}
+	return totals, nil
+}
+
+func deltaCounter(after, before map[string]float64, metric string) float64 {
+	return after[metric] - before[metric]
+}