fix(runner/jaeger): fan out SPM metrics to match the backend composer

[mayankpande88]

jaeger_query_metrics required a metric_type param and hit a single
/api/metrics/{type} endpoint, but the backend composer sends
services/spanKinds (plural) and NO metric_type, and its parser expects
the combined four-metric blob — so every call errored.

Rewrite Metrics to replicate the legacy get_metrics: remap
services->service and spanKinds->spanKind, drop metric_type, and fan out
to /api/metrics/{calls,errors,latencies?quantile=0.95,
latencies?quantile=0.99}, assembling {calls, errors, latencies_p95,
latencies_p99}. A 404 (SPM storage not configured) surfaces the friendly
legacy message instead of a raw HTTP 404.

Note: the response key names and remapping mirror the legacy agent; the
jaeger_query_metrics composer/parser contract should be confirmed against
the backend repo before release.

Co-Authored-By: Claude Opus 4.8 (1M context) noreply@anthropic.com
Claude-Session: https://claude.ai/code/session_01L2HAXQH1gEtX7h4sUJsi9j

[gemini-code-assist]

Code Review

This pull request updates the Jaeger client's Metrics implementation to query Jaeger SPM (Service Performance Monitoring) metrics by fanning out requests to four distinct endpoints (calls, errors, and two latency quantiles) and aggregating the results. It also remaps plural query parameters to singular ones and gracefully handles 404 errors when SPM is not configured. The review feedback suggests executing these four requests concurrently using goroutines to improve performance, and consequently updating the test suite with proper synchronization (using sync.Mutex) to prevent concurrent map write panics.

[gemini-code-assist]

The current implementation of Metrics queries the four SPM endpoints sequentially. Since each query is an independent HTTP request, executing them sequentially can significantly increase the total latency of the Metrics call (up to the sum of all four requests). Fanning out these requests concurrently using goroutines and a buffered channel will reduce the total latency to the maximum latency of a single request, greatly improving performance.

	type result struct {
		key string
		raw json.RawMessage
		err error
	}
	ch := make(chan result, len(subs))
	for _, s := range subs {
		go func(s struct {
			key      string
			metric   string
			quantile string
		}) {
			q := cloneValues(base)
			if s.quantile != "" {
				q.Set("quantile", s.quantile)
			}
			raw, status, err := c.getRaw(ctx, "/api/metrics/"+s.metric, q)
			if err != nil {
				ch <- result{err: err}
				return
			}
			if status == http.StatusNotFound {
				ch <- result{err: fmt.Errorf("jaeger: SPM metrics not available (monitoring storage not configured)")}
				return
			}
			if status >= 400 {
				ch <- result{err: fmt.Errorf("jaeger metrics %s: HTTP %d: %s", s.metric, status, string(raw))}
				return
			}
			ch <- result{key: s.key, raw: raw}
		}(s)
	}

	out := make(map[string]json.RawMessage, len(subs))
	for i := 0; i < len(subs); i++ {
		select {
		case res := <-ch:
			if res.err != nil {
				return nil, res.err
			}
			out[res.key] = res.raw
		case <-ctx.Done():
			return nil, ctx.Err()
		}
	}
	return json.Marshal(out)

[gemini-code-assist]

Import the sync package to support thread-safe operations in the test when Metrics is executed concurrently.

Suggested change

	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strings"
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strings"
	"sync"

[gemini-code-assist]

The test TestMetrics_FansOutAndRemaps uses a shared map hits without synchronization. If Metrics is executed concurrently, multiple goroutines will write to hits concurrently, causing a panic due to concurrent map writes. Additionally, since both latency queries write to the same /api/metrics/latencies key, they overwrite each other, making it impossible to reliably verify both queries.

Protecting the map with a sync.Mutex and storing the queries in a slice solves both the concurrency panic and the overwriting issue.

func TestMetrics_FansOutAndRemaps(t *testing.T) {
	var mu sync.Mutex
	hits := map[string][]string{} // metric path -> raw queries
	c, srv := newTestClient(func(w http.ResponseWriter, r *http.Request) {
		mu.Lock()
		hits[r.URL.Path] = append(hits[r.URL.Path], r.URL.RawQuery)
		mu.Unlock()
		// Echo the metric name so we can assert the assembly mapping.
		_, _ = w.Write([]byte("{\"path\":\"" + r.URL.Path + "\"}"))
	})
	defer srv.Close()

	raw, err := c.Metrics(context.Background(), map[string]any{
		"services":    "frontend",
		"spanKinds":   "SPAN_KIND_SERVER",
		"metric_type": "should-be-dropped",
		"endTs":       1700000000,
	})
	if err != nil {
		t.Fatal(err)
	}

	// All four endpoints hit.
	mu.Lock()
	defer mu.Unlock()
	for _, p := range []string{"/api/metrics/calls", "/api/metrics/errors", "/api/metrics/latencies"} {
		if _, ok := hits[p]; !ok {
			t.Errorf("missing request to %s (hits: %v)", p, hits)
		}
	}
	// Remap: singular service/spanKind present, plurals + metric_type gone.
	callsQueries := hits["/api/metrics/calls"]
	if len(callsQueries) == 0 {
		t.Fatal("missing calls query")
	}
	q := callsQueries[0]
	if !strings.Contains(q, "service=frontend") || !strings.Contains(q, "spanKind=SPAN_KIND_SERVER") {
		t.Errorf("calls query = %q; want remapped service/spanKind", q)
	}
	if strings.Contains(q, "services=") || strings.Contains(q, "spanKinds=") || strings.Contains(q, "metric_type=") {
		t.Errorf("calls query = %q; plural/metric_type should be dropped", q)
	}
	if !strings.Contains(q, "endTs=1700000000") {
		t.Errorf("calls query = %q; want endTs passed through", q)
	}

	// Assembled shape.
	var out map[string]json.RawMessage
	if err := json.Unmarshal(raw, &out); err != nil {
		t.Fatalf("assembled result not JSON object: %v", err)
	}
	for _, k := range []string{\"calls\", \"errors\", \"latencies_p95\", \"latencies_p99\"} {
		if _, ok := out[k]; !ok {
			t.Errorf("assembled result missing key %q (got %v)", k, out)
		}
	}
	// The two latency sub-queries carry the right quantiles.
	latencies := hits["/api/metrics/latencies"]
	if len(latencies) != 2 {
		t.Errorf("expected 2 latency queries, got %d", len(latencies))
	}
	for _, lat := range latencies {
		if !strings.Contains(lat, "quantile=0.9") {
			t.Errorf("latencies query = %q; want a quantile", lat)
		}
	}
}