List recent failures under each failure count on /ui/analytics

README.md

@@ -1330,6 +1330,14 @@ Set `ui_request_sources: true` and both appear on `/ui/analytics` under **Runtim
 
 **Why addresses are not Prometheus labels.** Client tool names are exported as `proxy_client_requests_total{client}` because they come from a closed set. Addresses are not exported at all: the caller set is unbounded and outside the proxy's control, and every new address would create a time series that lives forever in your TSDB. The same goes for anything job-scoped -- a pipeline ID must never become a label. Per-address and per-request detail belongs in the access log, which records `remote_ip`, `user_agent`, `client`, `ecosystem` and `bytes` on every line as JSONL, ready for `jq`, Loki or whatever you ship logs to.
 
+### Failure details
+
+The failure counts under **Runtime** on `/ui/analytics` -- upstream errors, storage errors, integrity failures, health probe failures and scan errors -- say how often something failed, not what. Set `ui_failure_details: true` and each count gets a **Latest** list underneath it, newest first: the package URL and filename, the upstream URL or storage path, the error, and how long ago it happened.
+
+The list keeps the last 20 failures per row, in memory only, so it starts empty when the proxy starts, like the counts above it. Credentials, query strings and fragments are stripped from every URL before it is kept, including URLs inside error messages, since that is where registry tokens and signed-URL signatures live.
+
+The flag defaults to off for the same reason as `ui_request_sources`: package names and upstream hosts say what your users depend on, and `/ui` is not authenticated. Package names and URLs are never Prometheus labels; the same detail is in the proxy's log.
+
 ### Grafana dashboard
 
 A ready-made dashboard lives at [`deploy/grafana/git-pkgs-proxy.json`](deploy/grafana/git-pkgs-proxy.json). Import it via **Dashboards -> New -> Import** and pick your Prometheus data source when prompted; it has no hardcoded data source UID.

config.example.yaml

@@ -42,6 +42,13 @@ base_url: "http://localhost:8080"
 # otherwise read the addresses of your build fleet.
 # ui_request_sources: false
 
+# List the latest failures behind each failure count on /ui/analytics (upstream,
+# storage, integrity, health probe and scan errors): the package, its upstream
+# URL or storage path, and the error. URLs lose credentials and query strings.
+#
+# Off by default, for the same reason as ui_request_sources.
+# ui_failure_details: false
+
 # Artifact storage configuration
 storage:
   # Storage backend URL

docs/configuration.md

@@ -21,6 +21,7 @@ See `config.example.yaml` in the repository root for a complete example.
 | `ui_base_url` | `PROXY_UI_URL` | - | (defaults to `base_url`) | Public URL where the web UI is reached. Set separately when the UI lives behind a different hostname than package endpoints (e.g. public domain vs Docker network alias). Used for canonical/og:url tags and the install guide banner. The proxy still serves package endpoints on the same listener, so any reverse proxy fronting the UI publicly should restrict the public route to `PathPrefix(/ui)` to avoid exposing package endpoints. |
 | `trust_forwarded_for` | `PROXY_TRUST_FORWARDED_FOR` | - | `false` | Attribute requests to the leftmost `X-Forwarded-For` entry instead of the TCP peer address, in the structured log, the access log and the request-source table on `/ui/analytics`. |
 | `ui_request_sources` | `PROXY_UI_REQUEST_SOURCES` | - | `false` | Show the request-source table on `/ui/analytics`, which reports caller addresses, the tool each ran and how much each pulled. |
+| `ui_failure_details` | `PROXY_UI_FAILURE_DETAILS` | - | `false` | List the latest failures behind each failure count on `/ui/analytics`: the package, its upstream URL or storage path, and the error. URLs are shown without credentials or query strings. |
 
 ## Storage
 

internal/config/config.go

@@ -119,6 +119,14 @@ type Config struct {
 	// is enabled /ui exposes what is cached rather than who called.
 	UIRequestSources bool `json:"ui_request_sources" yaml:"ui_request_sources"`
 
+	// UIFailureDetails lists the latest failures behind each failure counter on
+	// /ui/analytics: the package, its upstream URL or storage path, and the
+	// error. URLs are shown without credentials or query strings.
+	//
+	// Off by default, for the same reason as UIRequestSources: package names and
+	// upstream hosts say what the proxy's users depend on.
+	UIFailureDetails bool `json:"ui_failure_details" yaml:"ui_failure_details"`
+
 	// Storage configures artifact storage.
 	Storage StorageConfig `json:"storage" yaml:"storage"`
 
@@ -1102,6 +1110,7 @@ func (c *Config) LoadFromEnv() {
 	setEnvString(&c.UIBaseURL, "PROXY_UI_URL")
 	setEnvBool(&c.TrustForwardedFor, "PROXY_TRUST_FORWARDED_FOR")
 	setEnvBool(&c.UIRequestSources, "PROXY_UI_REQUEST_SOURCES")
+	setEnvBool(&c.UIFailureDetails, "PROXY_UI_FAILURE_DETAILS")
 	setEnvString(&c.Storage.URL, "PROXY_STORAGE_URL")
 	setEnvString(&c.Storage.Path, "PROXY_STORAGE_PATH")
 	setEnvString(&c.Storage.MaxSize, "PROXY_STORAGE_MAX_SIZE")

internal/config/config_test.go

@@ -463,6 +463,7 @@ func TestLoadFromEnv(t *testing.T) {
 	t.Setenv("PROXY_ACCESS_LOG_PATH", "/tmp/proxy-access.jsonl")
 	t.Setenv("PROXY_TRUST_FORWARDED_FOR", "true")
 	t.Setenv("PROXY_UI_REQUEST_SOURCES", "true")
+	t.Setenv("PROXY_UI_FAILURE_DETAILS", "true")
 	t.Setenv("PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS", "registry.internal, 10.0.0.12")
 	t.Setenv("PROXY_UPSTREAM_ALLOW_LOOPBACK", "true")
 	t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "true")
@@ -500,6 +501,9 @@ func TestLoadFromEnv(t *testing.T) {
 	if !cfg.UIRequestSources {
 		t.Error("UIRequestSources = false, want true from the environment")
 	}
+	if !cfg.UIFailureDetails {
+		t.Error("UIFailureDetails = false, want true from the environment")
+	}
 	if got := strings.Join(cfg.Upstream.AllowPrivateHosts, ","); got != "registry.internal,10.0.0.12" {
 		t.Errorf("Upstream.AllowPrivateHosts = %q, want %q", got, "registry.internal,10.0.0.12")
 	}

internal/handler/gradle.go

@@ -100,7 +100,7 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http
 		exists, err := h.proxy.Storage.Exists(r.Context(), storagePath)
 		metrics.RecordStorageOperation("read", time.Since(existsStart))
 		if err != nil {
-			metrics.RecordStorageError("read")
+			metrics.RecordStorageError("read", metrics.FailureDetail{Path: storagePath, Err: err})
 			h.proxy.Logger.Error("failed to check gradle build cache entry", "key", key, "error", err)
 			http.Error(w, "failed to read cache entry", http.StatusInternalServerError)
 			return
@@ -116,7 +116,7 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http
 		size, err := h.proxy.Storage.Size(r.Context(), storagePath)
 		metrics.RecordStorageOperation("read", time.Since(sizeStart))
 		if err != nil {
-			metrics.RecordStorageError("read")
+			metrics.RecordStorageError("read", metrics.FailureDetail{Path: storagePath, Err: err})
 		} else if size >= 0 {
 			w.Header().Set(headerContentLength, strconv.FormatInt(size, 10))
 		}
@@ -134,7 +134,7 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http
 			http.NotFound(w, r)
 			return
 		}
-		metrics.RecordStorageError("read")
+		metrics.RecordStorageError("read", metrics.FailureDetail{Path: storagePath, Err: err})
 		h.proxy.Logger.Error("failed to open gradle build cache entry", "key", key, "error", err)
 		http.Error(w, "failed to read cache entry", http.StatusInternalServerError)
 		return
@@ -165,7 +165,7 @@ func (h *GradleBuildCacheHandler) handlePut(w http.ResponseWriter, r *http.Reque
 			return
 		}
 
-		metrics.RecordStorageError("write")
+		metrics.RecordStorageError("write", metrics.FailureDetail{Path: storagePath, Err: err})
 		h.proxy.Logger.Error("failed to store gradle build cache entry", "key", key, "error", err)
 		http.Error(w, "failed to write cache entry", http.StatusInternalServerError)
 		return

internal/handler/handler.go

@@ -367,7 +367,8 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s
 	reader, err := p.Storage.Open(ctx, artifact.StoragePath)
 	metrics.RecordStorageOperation("read", time.Since(start))
 	if err != nil {
-		metrics.RecordStorageError("read")
+		metrics.RecordStorageError("read", metrics.FailureDetail{
+			PURL: versionPURL, Filename: filename, Path: artifact.StoragePath, Err: err})
 		p.Logger.Warn("cached artifact missing from storage, will refetch",
 			"path", artifact.StoragePath, "error", err)
 		return nil, nil
@@ -378,7 +379,8 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s
 			p.Logger.Error("cached artifact failed integrity check",
 				"purl", versionPURL, "filename", filename,
 				"path", artifact.StoragePath, "reason", reason)
-			metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem))
+			metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem), metrics.FailureDetail{
+				PURL: versionPURL, Filename: filename, Path: artifact.StoragePath, Err: errors.New(reason)})
 			if err := p.DB.DiscardArtifact(versionPURL, filename, artifact.StoragePath); err != nil {
 				p.Logger.Warn("failed to clear corrupt artifact from cache", "error", err)
 			}
@@ -421,7 +423,8 @@ func (p *Proxy) rejectUnusableCacheRecord(artifact *database.CachedArtifact, ver
 	p.Logger.Warn("cached artifact has unusable integrity metadata",
 		"purl", versionPURL, "filename", filename,
 		"path", artifact.StoragePath, "error", cause)
-	metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem))
+	metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem), metrics.FailureDetail{
+		PURL: versionPURL, Filename: filename, Path: artifact.StoragePath, Err: cause})
 	if err := p.DB.DiscardArtifact(versionPURL, filename, artifact.StoragePath); err != nil {
 		p.Logger.Warn("failed to clear unusable artifact from cache", "error", err)
 	}
@@ -459,7 +462,8 @@ func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, fil
 
 	if err != nil {
 		metrics.RecordUpstreamFetch(ecosystem, fetchDuration)
-		metrics.RecordUpstreamError(ecosystem, "fetch_failed")
+		metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{
+			PURL: versionPURL, Filename: filename, URL: info.URL, Err: err})
 		if errors.Is(err, fetch.ErrNotFound) {
 			return artifacts.Artifact{}, "", ErrUpstreamNotFound
 		}
@@ -491,7 +495,8 @@ func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, fil
 	_ = artifact.Body.Close()
 	metrics.RecordStorageOperation("write", time.Since(storeStart))
 	if err != nil {
-		metrics.RecordStorageError("write")
+		metrics.RecordStorageError("write", metrics.FailureDetail{
+			PURL: versionPURL, Filename: filename, URL: upstreamURL, Path: storagePath, Err: err})
 		return artifacts.Artifact{}, "", fmt.Errorf("storing artifact: %w", err)
 	}
 
@@ -546,7 +551,8 @@ func (p *Proxy) openStoredArtifact(ctx context.Context, artifact artifacts.Artif
 	metrics.RecordStorageOperation("read", time.Since(readStart))
 
 	if err != nil {
-		metrics.RecordStorageError("read")
+		metrics.RecordStorageError("read", metrics.FailureDetail{
+			PURL: artifact.PURL, Filename: artifact.Filename, Path: storagePath, Err: err})
 		return nil, fmt.Errorf("opening cached artifact: %w", err)
 	}
 
@@ -1711,7 +1717,8 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi
 	artifact, err := p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers)
 	metrics.RecordUpstreamFetch(ecosystem, time.Since(fetchStart))
 	if err != nil {
-		metrics.RecordUpstreamError(ecosystem, "fetch_failed")
+		metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{
+			PURL: versionPURL, Filename: filename, URL: downloadURL, Err: err})
 		if errors.Is(err, fetch.ErrNotFound) {
 			return artifacts.Artifact{}, "", ErrUpstreamNotFound
 		}
@@ -1736,7 +1743,8 @@ func (p *Proxy) streamFromUpstream(ctx context.Context, ecosystem, name, version
 	artifact, err := fetchArtifact(ctx)
 	metrics.RecordUpstreamFetch(ecosystem, time.Since(fetchStart))
 	if err != nil {
-		metrics.RecordUpstreamError(ecosystem, "fetch_failed")
+		metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{
+			PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: err})
 		if errors.Is(err, fetch.ErrNotFound) {
 			return nil, ErrUpstreamNotFound
 		}
@@ -1748,7 +1756,8 @@ func (p *Proxy) streamFromUpstream(ctx context.Context, ecosystem, name, version
 		onError: func(read int64, err error) {
 			p.Logger.Warn("streaming artifact from upstream failed",
 				"purl", versionPURL, "filename", filename, "url", upstreamURL, "bytes", read, "error", err)
-			metrics.RecordUpstreamError(ecosystem, "stream_failed")
+			metrics.RecordUpstreamError(ecosystem, "stream_failed", metrics.FailureDetail{
+				PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: err})
 		},
 	}
 	result := &CacheResult{
@@ -1773,7 +1782,8 @@ func (p *Proxy) streamFromUpstream(ctx context.Context, ecosystem, name, version
 	result.Reader, err = checks.wrapFailOnMismatch(body, func(reason string) {
 		p.Logger.Error("streamed artifact failed integrity check",
 			"purl", versionPURL, "filename", filename, "url", upstreamURL, "reason", reason)
-		metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(ecosystem))
+		metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(ecosystem), metrics.FailureDetail{
+			PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: errors.New(reason)})
 	})
 	if err != nil {
 		_ = artifact.Body.Close()

internal/handler/handler_test.go

@@ -990,7 +990,17 @@ func TestGetOrFetchArtifactFromURL_CacheMiss(t *testing.T) {
 	}
 }
 
+// isolateRecentFailures empties the process-wide failure history for the
+// test and again when it ends, so neither earlier tests nor repeated runs
+// reach its assertions.
+func isolateRecentFailures(t *testing.T) {
+	t.Helper()
+	metrics.ResetRecentFailures()
+	t.Cleanup(metrics.ResetRecentFailures)
+}
+
 func TestGetOrFetchArtifactFromURL_FetchError(t *testing.T) {
+	isolateRecentFailures(t)
 	proxy, _, _, fetcher := setupTestProxy(t)
 	fetcher.fetchErr = errors.New("connection refused")
 	errorsBefore := testutil.ToFloat64(metrics.UpstreamErrors.WithLabelValues("pypi", "fetch_failed"))
@@ -1005,9 +1015,15 @@ func TestGetOrFetchArtifactFromURL_FetchError(t *testing.T) {
 	if diff := testutil.ToFloat64(metrics.UpstreamErrors.WithLabelValues("pypi", "fetch_failed")) - errorsBefore; diff != 1 {
 		t.Errorf("upstream errors delta = %.0f, want 1", diff)
 	}
+	recent := metrics.RecentFailures("proxy_upstream_errors_total", map[string]string{"ecosystem": "pypi", "error_type": "fetch_failed"})
+	if len(recent) != 1 || recent[0].PURL != "pkg:pypi/fail@1.0.0" ||
+		recent[0].URL != "https://pypi.org/files/fail-1.0.0.tar.gz" || recent[0].Error != "connection refused" {
+		t.Errorf("upstream failures = %+v, want one: the failed package, its URL and the error", recent)
+	}
 }
 
 func TestGetOrFetchArtifactFromURL_StoreError(t *testing.T) {
+	isolateRecentFailures(t)
 	proxy, _, store, fetcher := setupTestProxy(t)
 	store.storeErr = errors.New("disk full")
 	errorsBefore := testutil.ToFloat64(metrics.StorageErrors.WithLabelValues("write"))
@@ -1026,6 +1042,11 @@ func TestGetOrFetchArtifactFromURL_StoreError(t *testing.T) {
 	if diff := testutil.ToFloat64(metrics.StorageErrors.WithLabelValues("write")) - errorsBefore; diff != 1 {
 		t.Errorf("storage errors delta = %.0f, want 1", diff)
 	}
+	recent := metrics.RecentFailures("proxy_storage_errors_total", map[string]string{"operation": "write"})
+	if len(recent) != 1 || recent[0].PURL != "pkg:pypi/fail@1.0.0" || recent[0].Filename != "fail-1.0.0.tar.gz" ||
+		recent[0].Path == "" || recent[0].Error != "disk full" {
+		t.Errorf("storage failures = %+v, want one: the failed package, its storage path and the error", recent)
+	}
 }
 
 func TestServeArtifact(t *testing.T) {

internal/metrics/failures.go

@@ -0,0 +1,122 @@
+package metrics
+
+import (
+	"net/url"
+	"regexp"
+	"sync"
+	"time"
+)
+
+// recentFailuresPerSeries bounds how many failures are kept for each series of
+// a failure counter. The series set is already bounded by the counter's
+// labels, so this caps the whole store.
+const recentFailuresPerSeries = 20
+
+// FailureDetail is what a failure counter's call site knows beyond the
+// counter's labels: which artifact failed, where it came from or was going,
+// and why. Every field is optional.
+//
+// None of it becomes a metric label. Package names and URLs are unbounded, so
+// they are kept in a small in-memory store for /ui/analytics instead.
+type FailureDetail struct {
+	PURL     string
+	Filename string
+	URL      string
+	Path     string
+	Err      error
+}
+
+// Failure is one recorded failure, with credentials already stripped from the
+// URL and the error text.
+type Failure struct {
+	At       time.Time
+	PURL     string
+	Filename string
+	URL      string
+	Path     string
+	Error    string
+}
+
+// recentFailures holds the latest failures per counter series. It lives in
+// process memory only, so like the counters themselves it starts empty when
+// the proxy starts.
+var recentFailures = struct {
+	sync.Mutex
+	series map[string][]Failure
+}{series: make(map[string][]Failure)}
+
+// RecentFailures returns the latest failures recorded for one series of a
+// failure counter, newest first, or nil when the series has none.
+func RecentFailures(name string, labels map[string]string) []Failure {
+	recentFailures.Lock()
+	defer recentFailures.Unlock()
+
+	stored := recentFailures.series[failureKey(name, labels)]
+	if len(stored) == 0 {
+		return nil
+	}
+	return append([]Failure(nil), stored...)
+}
+
+func recordFailure(name string, labels map[string]string, detail FailureDetail) {
+	failure := Failure{
+		At:       time.Now(),
+		PURL:     detail.PURL,
+		Filename: detail.Filename,
+		URL:      redactURL(detail.URL),
+		Path:     detail.Path,
+	}
+	if detail.Err != nil {
+		failure.Error = redactURLsIn(detail.Err.Error())
+	}
+
+	key := failureKey(name, labels)
+
+	recentFailures.Lock()
+	defer recentFailures.Unlock()
+
+	stored := append([]Failure{failure}, recentFailures.series[key]...)
+	if len(stored) > recentFailuresPerSeries {
+		stored = stored[:recentFailuresPerSeries]
+	}
+	recentFailures.series[key] = stored
+}
+
+// ResetRecentFailures empties the recent failure history. The history is
+// process-wide, so tests that record or read failures call it to start from a
+// known state and to leave none behind.
+func ResetRecentFailures() {
+	recentFailures.Lock()
+	defer recentFailures.Unlock()
+	recentFailures.series = make(map[string][]Failure)
+}
+
+func failureKey(name string, labels map[string]string) string {
+	return name + "{" + labelKey(labels) + "}"
+}
+
+var urlInText = regexp.MustCompile(`[a-zA-Z][a-zA-Z0-9+.-]*://[^\s"'<>]+`)
+
+// redactURLsIn strips credentials from every URL embedded in text, such as the
+// one net/http quotes in a failed request's error.
+func redactURLsIn(text string) string {
+	return urlInText.ReplaceAllStringFunc(text, redactURL)
+}
+
+// redactURL drops the userinfo, query and fragment, which is where registry
+// tokens and signed-URL signatures live.
+func redactURL(raw string) string {
+	if raw == "" {
+		return ""
+	}
+	u, err := url.Parse(raw)
+	if err != nil {
+		return "(unparseable URL)"
+	}
+	u.User = nil
+	u.RawQuery = ""
+	u.ForceQuery = false
+	u.Fragment = ""
+	u.RawFragment = ""
+	return u.String()
+}