diff --git a/README.md b/README.md index 020916cc..e7c90b9e 100644 --- a/README.md +++ b/README.md @@ -1330,6 +1330,14 @@ Set `ui_request_sources: true` and both appear on `/ui/analytics` under **Runtim **Why addresses are not Prometheus labels.** Client tool names are exported as `proxy_client_requests_total{client}` because they come from a closed set. Addresses are not exported at all: the caller set is unbounded and outside the proxy's control, and every new address would create a time series that lives forever in your TSDB. The same goes for anything job-scoped -- a pipeline ID must never become a label. Per-address and per-request detail belongs in the access log, which records `remote_ip`, `user_agent`, `client`, `ecosystem` and `bytes` on every line as JSONL, ready for `jq`, Loki or whatever you ship logs to. +### Failure details + +The failure counts under **Runtime** on `/ui/analytics` -- upstream errors, storage errors, integrity failures, health probe failures and scan errors -- say how often something failed, not what. Set `ui_failure_details: true` and each count gets a **Latest** list underneath it, newest first: the package URL and filename, the upstream URL or storage path, the error, and how long ago it happened. + +The list keeps the last 20 failures per row, in memory only, so it starts empty when the proxy starts, like the counts above it. Credentials, query strings and fragments are stripped from every URL before it is kept, including URLs inside error messages, since that is where registry tokens and signed-URL signatures live. + +The flag defaults to off for the same reason as `ui_request_sources`: package names and upstream hosts say what your users depend on, and `/ui` is not authenticated. Package names and URLs are never Prometheus labels; the same detail is in the proxy's log. + ### Grafana dashboard A ready-made dashboard lives at [`deploy/grafana/git-pkgs-proxy.json`](deploy/grafana/git-pkgs-proxy.json). Import it via **Dashboards -> New -> Import** and pick your Prometheus data source when prompted; it has no hardcoded data source UID. diff --git a/config.example.yaml b/config.example.yaml index f91b7d37..d7ea9351 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -42,6 +42,13 @@ base_url: "http://localhost:8080" # otherwise read the addresses of your build fleet. # ui_request_sources: false +# List the latest failures behind each failure count on /ui/analytics (upstream, +# storage, integrity, health probe and scan errors): the package, its upstream +# URL or storage path, and the error. URLs lose credentials and query strings. +# +# Off by default, for the same reason as ui_request_sources. +# ui_failure_details: false + # Artifact storage configuration storage: # Storage backend URL diff --git a/docs/configuration.md b/docs/configuration.md index 6b174521..51afb8a0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -21,6 +21,7 @@ See `config.example.yaml` in the repository root for a complete example. | `ui_base_url` | `PROXY_UI_URL` | - | (defaults to `base_url`) | Public URL where the web UI is reached. Set separately when the UI lives behind a different hostname than package endpoints (e.g. public domain vs Docker network alias). Used for canonical/og:url tags and the install guide banner. The proxy still serves package endpoints on the same listener, so any reverse proxy fronting the UI publicly should restrict the public route to `PathPrefix(/ui)` to avoid exposing package endpoints. | | `trust_forwarded_for` | `PROXY_TRUST_FORWARDED_FOR` | - | `false` | Attribute requests to the leftmost `X-Forwarded-For` entry instead of the TCP peer address, in the structured log, the access log and the request-source table on `/ui/analytics`. | | `ui_request_sources` | `PROXY_UI_REQUEST_SOURCES` | - | `false` | Show the request-source table on `/ui/analytics`, which reports caller addresses, the tool each ran and how much each pulled. | +| `ui_failure_details` | `PROXY_UI_FAILURE_DETAILS` | - | `false` | List the latest failures behind each failure count on `/ui/analytics`: the package, its upstream URL or storage path, and the error. URLs are shown without credentials or query strings. | ## Storage diff --git a/internal/config/config.go b/internal/config/config.go index 25ee4d7e..b3e1ac90 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -119,6 +119,14 @@ type Config struct { // is enabled /ui exposes what is cached rather than who called. UIRequestSources bool `json:"ui_request_sources" yaml:"ui_request_sources"` + // UIFailureDetails lists the latest failures behind each failure counter on + // /ui/analytics: the package, its upstream URL or storage path, and the + // error. URLs are shown without credentials or query strings. + // + // Off by default, for the same reason as UIRequestSources: package names and + // upstream hosts say what the proxy's users depend on. + UIFailureDetails bool `json:"ui_failure_details" yaml:"ui_failure_details"` + // Storage configures artifact storage. Storage StorageConfig `json:"storage" yaml:"storage"` @@ -1102,6 +1110,7 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.UIBaseURL, "PROXY_UI_URL") setEnvBool(&c.TrustForwardedFor, "PROXY_TRUST_FORWARDED_FOR") setEnvBool(&c.UIRequestSources, "PROXY_UI_REQUEST_SOURCES") + setEnvBool(&c.UIFailureDetails, "PROXY_UI_FAILURE_DETAILS") setEnvString(&c.Storage.URL, "PROXY_STORAGE_URL") setEnvString(&c.Storage.Path, "PROXY_STORAGE_PATH") setEnvString(&c.Storage.MaxSize, "PROXY_STORAGE_MAX_SIZE") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 0f7a78fb..491514e1 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -463,6 +463,7 @@ func TestLoadFromEnv(t *testing.T) { t.Setenv("PROXY_ACCESS_LOG_PATH", "/tmp/proxy-access.jsonl") t.Setenv("PROXY_TRUST_FORWARDED_FOR", "true") t.Setenv("PROXY_UI_REQUEST_SOURCES", "true") + t.Setenv("PROXY_UI_FAILURE_DETAILS", "true") t.Setenv("PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS", "registry.internal, 10.0.0.12") t.Setenv("PROXY_UPSTREAM_ALLOW_LOOPBACK", "true") t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "true") @@ -500,6 +501,9 @@ func TestLoadFromEnv(t *testing.T) { if !cfg.UIRequestSources { t.Error("UIRequestSources = false, want true from the environment") } + if !cfg.UIFailureDetails { + t.Error("UIFailureDetails = false, want true from the environment") + } if got := strings.Join(cfg.Upstream.AllowPrivateHosts, ","); got != "registry.internal,10.0.0.12" { t.Errorf("Upstream.AllowPrivateHosts = %q, want %q", got, "registry.internal,10.0.0.12") } diff --git a/internal/handler/gradle.go b/internal/handler/gradle.go index 9c74a6ac..e2512092 100644 --- a/internal/handler/gradle.go +++ b/internal/handler/gradle.go @@ -100,7 +100,7 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http exists, err := h.proxy.Storage.Exists(r.Context(), storagePath) metrics.RecordStorageOperation("read", time.Since(existsStart)) if err != nil { - metrics.RecordStorageError("read") + metrics.RecordStorageError("read", metrics.FailureDetail{Path: storagePath, Err: err}) h.proxy.Logger.Error("failed to check gradle build cache entry", "key", key, "error", err) http.Error(w, "failed to read cache entry", http.StatusInternalServerError) return @@ -116,7 +116,7 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http size, err := h.proxy.Storage.Size(r.Context(), storagePath) metrics.RecordStorageOperation("read", time.Since(sizeStart)) if err != nil { - metrics.RecordStorageError("read") + metrics.RecordStorageError("read", metrics.FailureDetail{Path: storagePath, Err: err}) } else if size >= 0 { w.Header().Set(headerContentLength, strconv.FormatInt(size, 10)) } @@ -134,7 +134,7 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http http.NotFound(w, r) return } - metrics.RecordStorageError("read") + metrics.RecordStorageError("read", metrics.FailureDetail{Path: storagePath, Err: err}) h.proxy.Logger.Error("failed to open gradle build cache entry", "key", key, "error", err) http.Error(w, "failed to read cache entry", http.StatusInternalServerError) return @@ -165,7 +165,7 @@ func (h *GradleBuildCacheHandler) handlePut(w http.ResponseWriter, r *http.Reque return } - metrics.RecordStorageError("write") + metrics.RecordStorageError("write", metrics.FailureDetail{Path: storagePath, Err: err}) h.proxy.Logger.Error("failed to store gradle build cache entry", "key", key, "error", err) http.Error(w, "failed to write cache entry", http.StatusInternalServerError) return diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 449fcef7..0c395327 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -367,7 +367,8 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s reader, err := p.Storage.Open(ctx, artifact.StoragePath) metrics.RecordStorageOperation("read", time.Since(start)) if err != nil { - metrics.RecordStorageError("read") + metrics.RecordStorageError("read", metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, Path: artifact.StoragePath, Err: err}) p.Logger.Warn("cached artifact missing from storage, will refetch", "path", artifact.StoragePath, "error", err) return nil, nil @@ -378,7 +379,8 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s p.Logger.Error("cached artifact failed integrity check", "purl", versionPURL, "filename", filename, "path", artifact.StoragePath, "reason", reason) - metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem)) + metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem), metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, Path: artifact.StoragePath, Err: errors.New(reason)}) if err := p.DB.DiscardArtifact(versionPURL, filename, artifact.StoragePath); err != nil { p.Logger.Warn("failed to clear corrupt artifact from cache", "error", err) } @@ -421,7 +423,8 @@ func (p *Proxy) rejectUnusableCacheRecord(artifact *database.CachedArtifact, ver p.Logger.Warn("cached artifact has unusable integrity metadata", "purl", versionPURL, "filename", filename, "path", artifact.StoragePath, "error", cause) - metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem)) + metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem), metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, Path: artifact.StoragePath, Err: cause}) if err := p.DB.DiscardArtifact(versionPURL, filename, artifact.StoragePath); err != nil { p.Logger.Warn("failed to clear unusable artifact from cache", "error", err) } @@ -459,7 +462,8 @@ func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, fil if err != nil { metrics.RecordUpstreamFetch(ecosystem, fetchDuration) - metrics.RecordUpstreamError(ecosystem, "fetch_failed") + metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, URL: info.URL, Err: err}) if errors.Is(err, fetch.ErrNotFound) { return artifacts.Artifact{}, "", ErrUpstreamNotFound } @@ -491,7 +495,8 @@ func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, fil _ = artifact.Body.Close() metrics.RecordStorageOperation("write", time.Since(storeStart)) if err != nil { - metrics.RecordStorageError("write") + metrics.RecordStorageError("write", metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, URL: upstreamURL, Path: storagePath, Err: err}) return artifacts.Artifact{}, "", fmt.Errorf("storing artifact: %w", err) } @@ -546,7 +551,8 @@ func (p *Proxy) openStoredArtifact(ctx context.Context, artifact artifacts.Artif metrics.RecordStorageOperation("read", time.Since(readStart)) if err != nil { - metrics.RecordStorageError("read") + metrics.RecordStorageError("read", metrics.FailureDetail{ + PURL: artifact.PURL, Filename: artifact.Filename, Path: storagePath, Err: err}) return nil, fmt.Errorf("opening cached artifact: %w", err) } @@ -1711,7 +1717,8 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi artifact, err := p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers) metrics.RecordUpstreamFetch(ecosystem, time.Since(fetchStart)) if err != nil { - metrics.RecordUpstreamError(ecosystem, "fetch_failed") + metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, URL: downloadURL, Err: err}) if errors.Is(err, fetch.ErrNotFound) { return artifacts.Artifact{}, "", ErrUpstreamNotFound } @@ -1736,7 +1743,8 @@ func (p *Proxy) streamFromUpstream(ctx context.Context, ecosystem, name, version artifact, err := fetchArtifact(ctx) metrics.RecordUpstreamFetch(ecosystem, time.Since(fetchStart)) if err != nil { - metrics.RecordUpstreamError(ecosystem, "fetch_failed") + metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: err}) if errors.Is(err, fetch.ErrNotFound) { return nil, ErrUpstreamNotFound } @@ -1748,7 +1756,8 @@ func (p *Proxy) streamFromUpstream(ctx context.Context, ecosystem, name, version onError: func(read int64, err error) { p.Logger.Warn("streaming artifact from upstream failed", "purl", versionPURL, "filename", filename, "url", upstreamURL, "bytes", read, "error", err) - metrics.RecordUpstreamError(ecosystem, "stream_failed") + metrics.RecordUpstreamError(ecosystem, "stream_failed", metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: err}) }, } result := &CacheResult{ @@ -1773,7 +1782,8 @@ func (p *Proxy) streamFromUpstream(ctx context.Context, ecosystem, name, version result.Reader, err = checks.wrapFailOnMismatch(body, func(reason string) { p.Logger.Error("streamed artifact failed integrity check", "purl", versionPURL, "filename", filename, "url", upstreamURL, "reason", reason) - metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(ecosystem)) + metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(ecosystem), metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: errors.New(reason)}) }) if err != nil { _ = artifact.Body.Close() diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 4c05cbd9..ffc0d5f2 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -990,7 +990,17 @@ func TestGetOrFetchArtifactFromURL_CacheMiss(t *testing.T) { } } +// isolateRecentFailures empties the process-wide failure history for the +// test and again when it ends, so neither earlier tests nor repeated runs +// reach its assertions. +func isolateRecentFailures(t *testing.T) { + t.Helper() + metrics.ResetRecentFailures() + t.Cleanup(metrics.ResetRecentFailures) +} + func TestGetOrFetchArtifactFromURL_FetchError(t *testing.T) { + isolateRecentFailures(t) proxy, _, _, fetcher := setupTestProxy(t) fetcher.fetchErr = errors.New("connection refused") errorsBefore := testutil.ToFloat64(metrics.UpstreamErrors.WithLabelValues("pypi", "fetch_failed")) @@ -1005,9 +1015,15 @@ func TestGetOrFetchArtifactFromURL_FetchError(t *testing.T) { if diff := testutil.ToFloat64(metrics.UpstreamErrors.WithLabelValues("pypi", "fetch_failed")) - errorsBefore; diff != 1 { t.Errorf("upstream errors delta = %.0f, want 1", diff) } + recent := metrics.RecentFailures("proxy_upstream_errors_total", map[string]string{"ecosystem": "pypi", "error_type": "fetch_failed"}) + if len(recent) != 1 || recent[0].PURL != "pkg:pypi/fail@1.0.0" || + recent[0].URL != "https://pypi.org/files/fail-1.0.0.tar.gz" || recent[0].Error != "connection refused" { + t.Errorf("upstream failures = %+v, want one: the failed package, its URL and the error", recent) + } } func TestGetOrFetchArtifactFromURL_StoreError(t *testing.T) { + isolateRecentFailures(t) proxy, _, store, fetcher := setupTestProxy(t) store.storeErr = errors.New("disk full") errorsBefore := testutil.ToFloat64(metrics.StorageErrors.WithLabelValues("write")) @@ -1026,6 +1042,11 @@ func TestGetOrFetchArtifactFromURL_StoreError(t *testing.T) { if diff := testutil.ToFloat64(metrics.StorageErrors.WithLabelValues("write")) - errorsBefore; diff != 1 { t.Errorf("storage errors delta = %.0f, want 1", diff) } + recent := metrics.RecentFailures("proxy_storage_errors_total", map[string]string{"operation": "write"}) + if len(recent) != 1 || recent[0].PURL != "pkg:pypi/fail@1.0.0" || recent[0].Filename != "fail-1.0.0.tar.gz" || + recent[0].Path == "" || recent[0].Error != "disk full" { + t.Errorf("storage failures = %+v, want one: the failed package, its storage path and the error", recent) + } } func TestServeArtifact(t *testing.T) { diff --git a/internal/metrics/failures.go b/internal/metrics/failures.go new file mode 100644 index 00000000..425ba8e2 --- /dev/null +++ b/internal/metrics/failures.go @@ -0,0 +1,122 @@ +package metrics + +import ( + "net/url" + "regexp" + "sync" + "time" +) + +// recentFailuresPerSeries bounds how many failures are kept for each series of +// a failure counter. The series set is already bounded by the counter's +// labels, so this caps the whole store. +const recentFailuresPerSeries = 20 + +// FailureDetail is what a failure counter's call site knows beyond the +// counter's labels: which artifact failed, where it came from or was going, +// and why. Every field is optional. +// +// None of it becomes a metric label. Package names and URLs are unbounded, so +// they are kept in a small in-memory store for /ui/analytics instead. +type FailureDetail struct { + PURL string + Filename string + URL string + Path string + Err error +} + +// Failure is one recorded failure, with credentials already stripped from the +// URL and the error text. +type Failure struct { + At time.Time + PURL string + Filename string + URL string + Path string + Error string +} + +// recentFailures holds the latest failures per counter series. It lives in +// process memory only, so like the counters themselves it starts empty when +// the proxy starts. +var recentFailures = struct { + sync.Mutex + series map[string][]Failure +}{series: make(map[string][]Failure)} + +// RecentFailures returns the latest failures recorded for one series of a +// failure counter, newest first, or nil when the series has none. +func RecentFailures(name string, labels map[string]string) []Failure { + recentFailures.Lock() + defer recentFailures.Unlock() + + stored := recentFailures.series[failureKey(name, labels)] + if len(stored) == 0 { + return nil + } + return append([]Failure(nil), stored...) +} + +func recordFailure(name string, labels map[string]string, detail FailureDetail) { + failure := Failure{ + At: time.Now(), + PURL: detail.PURL, + Filename: detail.Filename, + URL: redactURL(detail.URL), + Path: detail.Path, + } + if detail.Err != nil { + failure.Error = redactURLsIn(detail.Err.Error()) + } + + key := failureKey(name, labels) + + recentFailures.Lock() + defer recentFailures.Unlock() + + stored := append([]Failure{failure}, recentFailures.series[key]...) + if len(stored) > recentFailuresPerSeries { + stored = stored[:recentFailuresPerSeries] + } + recentFailures.series[key] = stored +} + +// ResetRecentFailures empties the recent failure history. The history is +// process-wide, so tests that record or read failures call it to start from a +// known state and to leave none behind. +func ResetRecentFailures() { + recentFailures.Lock() + defer recentFailures.Unlock() + recentFailures.series = make(map[string][]Failure) +} + +func failureKey(name string, labels map[string]string) string { + return name + "{" + labelKey(labels) + "}" +} + +var urlInText = regexp.MustCompile(`[a-zA-Z][a-zA-Z0-9+.-]*://[^\s"'<>]+`) + +// redactURLsIn strips credentials from every URL embedded in text, such as the +// one net/http quotes in a failed request's error. +func redactURLsIn(text string) string { + return urlInText.ReplaceAllStringFunc(text, redactURL) +} + +// redactURL drops the userinfo, query and fragment, which is where registry +// tokens and signed-URL signatures live. +func redactURL(raw string) string { + if raw == "" { + return "" + } + u, err := url.Parse(raw) + if err != nil { + return "(unparseable URL)" + } + u.User = nil + u.RawQuery = "" + u.ForceQuery = false + u.Fragment = "" + u.RawFragment = "" + return u.String() +} diff --git a/internal/metrics/failures_test.go b/internal/metrics/failures_test.go new file mode 100644 index 00000000..3d6ab887 --- /dev/null +++ b/internal/metrics/failures_test.go @@ -0,0 +1,134 @@ +package metrics + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +// isolateRecentFailures empties the process-wide failure history for the +// test and again when it ends, so neither earlier tests nor repeated runs +// reach its assertions. +func isolateRecentFailures(t *testing.T) { + t.Helper() + ResetRecentFailures() + t.Cleanup(ResetRecentFailures) +} + +func TestRecentFailuresNewestFirst(t *testing.T) { + isolateRecentFailures(t) + + RecordUpstreamError("npm", "fetch_failed", FailureDetail{PURL: "pkg:npm/left-pad@1.0.0"}) + RecordUpstreamError("npm", "fetch_failed", FailureDetail{PURL: "pkg:npm/right-pad@2.0.0"}) + + got := RecentFailures("proxy_upstream_errors_total", map[string]string{"ecosystem": "npm", "error_type": "fetch_failed"}) + if len(got) != 2 { + t.Fatalf("got %d failures, want 2", len(got)) + } + if got[0].PURL != "pkg:npm/right-pad@2.0.0" || got[1].PURL != "pkg:npm/left-pad@1.0.0" { + t.Errorf("failures not newest first: %q, %q", got[0].PURL, got[1].PURL) + } + if got[0].At.IsZero() { + t.Error("failure has no timestamp") + } +} + +func TestRecentFailuresAreBoundedPerSeries(t *testing.T) { + isolateRecentFailures(t) + + for i := range recentFailuresPerSeries + 5 { + RecordStorageError("write", FailureDetail{Path: fmt.Sprintf("npm/pkg-%d.tgz", i)}) + } + + got := RecentFailures("proxy_storage_errors_total", map[string]string{"operation": "write"}) + if len(got) != recentFailuresPerSeries { + t.Fatalf("got %d failures, want the cap of %d", len(got), recentFailuresPerSeries) + } + if want := fmt.Sprintf("npm/pkg-%d.tgz", recentFailuresPerSeries+4); got[0].Path != want { + t.Errorf("newest failure = %q, want %q", got[0].Path, want) + } +} + +func TestRecentFailuresAreKeptPerSeries(t *testing.T) { + isolateRecentFailures(t) + + RecordIntegrityFailure("npm", FailureDetail{PURL: "pkg:npm/a@1"}) + RecordIntegrityFailure("pypi", FailureDetail{PURL: "pkg:pypi/b@1"}) + + npm := RecentFailures("proxy_integrity_failures_total", map[string]string{"ecosystem": "npm"}) + if len(npm) != 1 || npm[0].PURL != "pkg:npm/a@1" { + t.Errorf("npm series = %+v, want only pkg:npm/a@1", npm) + } + if got := RecentFailures("proxy_integrity_failures_total", map[string]string{"ecosystem": "deb"}); got != nil { + t.Errorf("unseen series = %+v, want nil", got) + } +} + +func TestRecordFailureStillCountsTheMetric(t *testing.T) { + isolateRecentFailures(t) + + before := testutil.ToFloat64(HealthProbeFailures.WithLabelValues("read")) + RecordHealthProbeFailure("read", FailureDetail{Err: errors.New("disk gone")}) + if diff := testutil.ToFloat64(HealthProbeFailures.WithLabelValues("read")) - before; diff != 1 { + t.Errorf("counter delta = %.0f, want 1", diff) + } + + got := RecentFailures("proxy_health_probe_failures_total", map[string]string{"step": "read"}) + if len(got) != 1 || got[0].Error != "disk gone" { + t.Errorf("failures = %+v, want one carrying the error text", got) + } +} + +func TestScanErrorRecordsDetail(t *testing.T) { + isolateRecentFailures(t) + + RecordScanError("NPM", "clamav", "timeout", FailureDetail{PURL: "pkg:npm/a@1", Filename: "a-1.tgz"}) + + got := RecentFailures("proxy_scan_errors_total", + map[string]string{"ecosystem": "npm", "scanner": "clamav", "error_type": "timeout"}) + if len(got) != 1 || got[0].Filename != "a-1.tgz" { + t.Errorf("failures = %+v, want one keyed by the normalised ecosystem", got) + } +} + +// The analytics page is unauthenticated, and private registry URLs carry +// credentials in the userinfo or the query string. Neither may be kept. +func TestRecentFailuresStripCredentials(t *testing.T) { + isolateRecentFailures(t) + + RecordUpstreamError("npm", "fetch_failed", FailureDetail{ + URL: "https://user:s3cret@registry.example.com/pkg/-/pkg-1.0.0.tgz?token=abc#frag", + Err: errors.New(`Get "https://user:s3cret@registry.example.com/pkg?sig=xyz": connection refused`), + }) + + got := RecentFailures("proxy_upstream_errors_total", map[string]string{"ecosystem": "npm", "error_type": "fetch_failed"}) + if len(got) != 1 { + t.Fatalf("got %d failures, want 1", len(got)) + } + if want := "https://registry.example.com/pkg/-/pkg-1.0.0.tgz"; got[0].URL != want { + t.Errorf("URL = %q, want %q", got[0].URL, want) + } + if want := `Get "https://registry.example.com/pkg": connection refused`; got[0].Error != want { + t.Errorf("Error = %q, want %q", got[0].Error, want) + } + for _, secret := range []string{"s3cret", "token", "sig=", "user:"} { + if strings.Contains(got[0].URL+got[0].Error, secret) { + t.Errorf("recorded failure still contains %q", secret) + } + } +} + +func TestRecentFailuresReturnsACopy(t *testing.T) { + isolateRecentFailures(t) + + RecordStorageError("read", FailureDetail{Path: "a"}) + labels := map[string]string{"operation": "read"} + RecentFailures("proxy_storage_errors_total", labels)[0].Path = "mutated" + + if got := RecentFailures("proxy_storage_errors_total", labels)[0].Path; got != "a" { + t.Errorf("stored failure was mutated through the returned slice: %q", got) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 23dca2cd..7567ff8e 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -338,9 +338,12 @@ func RecordUpstreamFetch(ecosystem string, duration time.Duration) { UpstreamFetchDuration.WithLabelValues(ecosystem).Observe(duration.Seconds()) } -// RecordUpstreamError increments upstream error counter. -func RecordUpstreamError(ecosystem, errorType string) { +// RecordUpstreamError increments upstream error counter and keeps the detail +// for /ui/analytics. +func RecordUpstreamError(ecosystem, errorType string, detail FailureDetail) { UpstreamErrors.WithLabelValues(ecosystem, errorType).Inc() + recordFailure("proxy_upstream_errors_total", + map[string]string{"ecosystem": ecosystem, "error_type": errorType}, detail) } // RecordStorageOperation tracks storage operation duration. @@ -348,20 +351,26 @@ func RecordStorageOperation(operation string, duration time.Duration) { StorageOperationDuration.WithLabelValues(operation).Observe(duration.Seconds()) } -// RecordIntegrityFailure increments the integrity failure counter. -func RecordIntegrityFailure(ecosystem string) { +// RecordIntegrityFailure increments the integrity failure counter and keeps +// the detail for /ui/analytics. +func RecordIntegrityFailure(ecosystem string, detail FailureDetail) { IntegrityFailures.WithLabelValues(ecosystem).Inc() + recordFailure("proxy_integrity_failures_total", map[string]string{"ecosystem": ecosystem}, detail) } -// RecordHealthProbeFailure increments the health probe failure counter. +// RecordHealthProbeFailure increments the health probe failure counter and +// keeps the detail for /ui/analytics. // step is one of: "write", "size", "read", "verify", "delete". -func RecordHealthProbeFailure(step string) { +func RecordHealthProbeFailure(step string, detail FailureDetail) { HealthProbeFailures.WithLabelValues(step).Inc() + recordFailure("proxy_health_probe_failures_total", map[string]string{"step": step}, detail) } -// RecordStorageError increments storage error counter. -func RecordStorageError(operation string) { +// RecordStorageError increments storage error counter and keeps the detail +// for /ui/analytics. +func RecordStorageError(operation string, detail FailureDetail) { StorageErrors.WithLabelValues(operation).Inc() + recordFailure("proxy_storage_errors_total", map[string]string{"operation": operation}, detail) } // RecordScanResult tracks a completed pre-cache scan call. @@ -375,8 +384,12 @@ func RecordScanResult(ecosystem, scannerName string, allowed bool, duration time // RecordScanError increments the scan error counter. // errorType is one of: "error" (scanner call failed), "timeout", "cancelled". -func RecordScanError(ecosystem, scannerName, errorType string) { - ScanErrors.WithLabelValues(purl.NormalizeEcosystem(ecosystem), scannerName, errorType).Inc() +// The detail is kept for /ui/analytics. +func RecordScanError(ecosystem, scannerName, errorType string, detail FailureDetail) { + ecosystem = purl.NormalizeEcosystem(ecosystem) + ScanErrors.WithLabelValues(ecosystem, scannerName, errorType).Inc() + recordFailure("proxy_scan_errors_total", + map[string]string{"ecosystem": ecosystem, "scanner": scannerName, "error_type": errorType}, detail) } // UpdateCacheStats updates cache size and artifact count gauges. diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index a4454676..cca39c89 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -30,19 +30,21 @@ func TestRecordCacheOperations(t *testing.T) { } func TestRecordUpstreamOperations(t *testing.T) { + isolateRecentFailures(t) RecordUpstreamFetch("npm", 500*time.Millisecond) RecordUpstreamFetch("pypi", 1*time.Second) - RecordUpstreamError("npm", "fetch_failed") - RecordUpstreamError("pypi", "not_found") + RecordUpstreamError("npm", "fetch_failed", FailureDetail{}) + RecordUpstreamError("pypi", "not_found", FailureDetail{}) // No panics = success } func TestRecordStorageOperations(t *testing.T) { + isolateRecentFailures(t) RecordStorageOperation("read", 10*time.Millisecond) RecordStorageOperation("write", 50*time.Millisecond) - RecordStorageError("read") - RecordStorageError("write") + RecordStorageError("read", FailureDetail{}) + RecordStorageError("write", FailureDetail{}) // No panics = success } diff --git a/internal/metrics/snapshot_test.go b/internal/metrics/snapshot_test.go index dd695f72..1a20906d 100644 --- a/internal/metrics/snapshot_test.go +++ b/internal/metrics/snapshot_test.go @@ -124,8 +124,9 @@ func TestSnapshotMissingMetric(t *testing.T) { // Gather reads the real registry, so every metric this package registers must // come back. This is the check that a newly added metric is reachable by the UI. func TestGatherSeesRegisteredMetrics(t *testing.T) { + isolateRecentFailures(t) RecordCacheHit("npm") - RecordScanError("npm", "clamav", "timeout") + RecordScanError("npm", "clamav", "timeout", FailureDetail{}) snap, err := Gather() if err != nil { diff --git a/internal/scanner/group.go b/internal/scanner/group.go index ed0be571..a31d3249 100644 --- a/internal/scanner/group.go +++ b/internal/scanner/group.go @@ -184,7 +184,8 @@ func (g *Group) evaluate(scanCtx context.Context, req Request, e entry) (finding // timed out or failed on its own. errType = "cancelled" } - metrics.RecordScanError(req.Ecosystem, e.scanner.Name(), errType) + metrics.RecordScanError(req.Ecosystem, e.scanner.Name(), errType, metrics.FailureDetail{ + PURL: req.PURL, Filename: req.Filename, Err: err}) if errType == "cancelled" { // Another scanner already decided the verdict and cancelled diff --git a/internal/server/analytics.go b/internal/server/analytics.go index eaf19484..b7b4a837 100644 --- a/internal/server/analytics.go +++ b/internal/server/analytics.go @@ -171,7 +171,7 @@ func (s *Server) handleAnalytics(w http.ResponseWriter, r *http.Request) { if err != nil { s.logger.Error("failed to gather runtime metrics", "error", err) } else { - data.Runtime = runtimeView(snap) + data.Runtime = runtimeView(snap, s.cfg != nil && s.cfg.UIFailureDetails) if s.cfg != nil && s.cfg.UIRequestSources { data.Runtime.SourcesOn = true data.Runtime.Sources = s.sources.Top(analyticsTopSources) diff --git a/internal/server/analytics_coverage_test.go b/internal/server/analytics_coverage_test.go index 2c4ac63a..12cc8959 100644 --- a/internal/server/analytics_coverage_test.go +++ b/internal/server/analytics_coverage_test.go @@ -54,6 +54,7 @@ var metricSurface = map[string]string{ // on the analytics page. The page is meant to be a complete view of what // /metrics exposes, so a silently unsurfaced metric is a gap, not a detail. func TestEveryMetricIsSurfaced(t *testing.T) { + isolateRecentFailures(t) // Touch every metric family so it is present in the registry output, since // a vector with no observed label values gathers as nothing at all. metrics.RecordRequest("npm", 200, 0) @@ -61,15 +62,15 @@ func TestEveryMetricIsSurfaced(t *testing.T) { metrics.RecordCacheHit("npm") metrics.RecordCacheMiss("npm") metrics.RecordUpstreamFetch("npm", 0) - metrics.RecordUpstreamError("npm", "fetch_failed") + metrics.RecordUpstreamError("npm", "fetch_failed", metrics.FailureDetail{}) metrics.RecordStorageOperation("get", 0) - metrics.RecordStorageError("get") - metrics.RecordIntegrityFailure("npm") - metrics.RecordHealthProbeFailure("write") + metrics.RecordStorageError("get", metrics.FailureDetail{}) + metrics.RecordIntegrityFailure("npm", metrics.FailureDetail{}) + metrics.RecordHealthProbeFailure("write", metrics.FailureDetail{}) metrics.UpdateCircuitBreakerState("example.test", 0) metrics.RecordCircuitBreakerTrip("example.test") metrics.RecordScanResult("npm", "clamav", false, 0) - metrics.RecordScanError("npm", "clamav", "timeout") + metrics.RecordScanError("npm", "clamav", "timeout", metrics.FailureDetail{}) metrics.UpdateCacheStats(1, 1) metrics.RecordArtifactEvicted("retention", "npm") metrics.UpdateEcosystemStats([]metrics.EcosystemStats{{Ecosystem: "npm", DownloadedBytes: 1}}) diff --git a/internal/server/analytics_failures_test.go b/internal/server/analytics_failures_test.go new file mode 100644 index 00000000..6f7856e7 --- /dev/null +++ b/internal/server/analytics_failures_test.go @@ -0,0 +1,118 @@ +package server + +import ( + "errors" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +// isolateRecentFailures empties the process-wide failure history for the +// test and again when it ends, so neither earlier tests nor repeated runs +// reach its assertions. +func isolateRecentFailures(t *testing.T) { + t.Helper() + metrics.ResetRecentFailures() + t.Cleanup(metrics.ResetRecentFailures) +} + +func storageErrorSnapshot(t *testing.T, operation string) *metrics.Snapshot { + t.Helper() + reg := prometheus.NewRegistry() + errs := prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "proxy_storage_errors_total", Help: "t"}, []string{"operation"}) + reg.MustRegister(errs) + errs.WithLabelValues(operation).Inc() + + snap, err := metrics.GatherFrom(reg) + if err != nil { + t.Fatalf("GatherFrom: %v", err) + } + return snap +} + +func TestRuntimeViewAttachesRecentFailuresWhenEnabled(t *testing.T) { + isolateRecentFailures(t) + const operation = "test-attach" + metrics.RecordStorageError(operation, metrics.FailureDetail{ + PURL: "pkg:npm/left-pad@1.0.0", Filename: "left-pad-1.0.0.tgz", Path: "npm/left-pad", Err: errors.New("disk full")}) + + v := runtimeView(storageErrorSnapshot(t, operation), true) + + if !v.FailureDetailsOn { + t.Error("FailureDetailsOn = false, want true") + } + if len(v.StorageErrors) != 1 || len(v.StorageErrors[0].Recent) != 1 { + t.Fatalf("StorageErrors = %+v, want one row with one recent failure", v.StorageErrors) + } + got := v.StorageErrors[0].Recent[0] + if got.PURL != "pkg:npm/left-pad@1.0.0" || got.Filename != "left-pad-1.0.0.tgz" || + got.Path != "npm/left-pad" || got.Error != "disk full" || got.Age == "" { + t.Errorf("recent failure = %+v", got) + } +} + +func TestRuntimeViewOmitsRecentFailuresWhenDisabled(t *testing.T) { + isolateRecentFailures(t) + const operation = "test-omit" + metrics.RecordStorageError(operation, metrics.FailureDetail{PURL: "pkg:npm/left-pad@1.0.0"}) + + v := runtimeView(storageErrorSnapshot(t, operation), false) + + if v.FailureDetailsOn { + t.Error("FailureDetailsOn = true, want false") + } + if len(v.StorageErrors) != 1 || v.StorageErrors[0].Recent != nil { + t.Errorf("StorageErrors = %+v, want the count without recent failures", v.StorageErrors) + } +} + +// Package names and upstream URLs say what the proxy's users depend on, so like +// the caller table they are published only when asked for. +func TestAnalyticsPageHidesFailureDetailsByDefault(t *testing.T) { + isolateRecentFailures(t) + ts := newTestServer(t) + defer ts.close() + + if ts.server.cfg.UIFailureDetails { + t.Fatal("UIFailureDetails defaults to true; it must default off") + } + metrics.RecordUpstreamError("npm", "fetch_failed", metrics.FailureDetail{PURL: "pkg:npm/hidden-by-default@1.0.0"}) + + body := ts.getOK(t, "/ui/analytics") + if strings.Contains(body, "hidden-by-default") { + t.Error("a failed package rendered with ui_failure_details off") + } + if !strings.Contains(body, "ui_failure_details") { + t.Error("the off-state note naming the config key did not render") + } +} + +func TestAnalyticsPageShowsFailureDetailsWhenEnabled(t *testing.T) { + isolateRecentFailures(t) + ts := newTestServer(t) + defer ts.close() + + ts.server.cfg.UIFailureDetails = true + metrics.RecordUpstreamError("npm", "fetch_failed", metrics.FailureDetail{ + PURL: "pkg:npm/shown-when-enabled@1.0.0", + URL: "https://registry.example.com/shown-when-enabled.tgz?token=secret", + Err: errors.New("upstream returned 503"), + }) + + body := ts.getOK(t, "/ui/analytics") + for _, want := range []string{ + "pkg:npm/shown-when-enabled@1.0.0", + "https://registry.example.com/shown-when-enabled.tgz", + "upstream returned 503", + } { + if !strings.Contains(body, want) { + t.Errorf("rendered page missing %q", want) + } + } + if strings.Contains(body, "token=secret") { + t.Error("rendered page leaks the URL's query string") + } +} diff --git a/internal/server/analytics_runtime.go b/internal/server/analytics_runtime.go index dad3d192..8e5d0d17 100644 --- a/internal/server/analytics_runtime.go +++ b/internal/server/analytics_runtime.go @@ -50,6 +50,12 @@ type RuntimeView struct { // SourcesOn gates the caller table. The page is unauthenticated, so the // addresses behind it are published only when asked for. SourcesOn bool + // FailureDetailsOn gates the recent failures listed under each failure + // row, for the same reason as SourcesOn. + FailureDetailsOn bool + // HasFailures is set when any failure counter has fired, which is when the + // note about turning failure details on is worth showing. + HasFailures bool } // ClientRow is one client tool's share of requests and bytes. @@ -65,6 +71,21 @@ type LabelledCount struct { Count string // Bad marks a failure row, which the template tints. Bad bool + // Recent lists the latest failures behind the count, newest first. + Recent []FailureRow + // series is the counter series the row was rendered from, for looking up + // what was recorded against it. + series map[string]string +} + +// FailureRow is one recorded failure behind a failure counter. +type FailureRow struct { + Age string + PURL string + Filename string + URL string + Path string + Error string } // LabelledStat is a histogram series rendered as a row: how many observations, @@ -82,13 +103,17 @@ type BreakerRow struct { Open bool } -// runtimeView shapes a registry snapshot for the analytics page. -func runtimeView(snap *metrics.Snapshot) RuntimeView { +// runtimeView shapes a registry snapshot for the analytics page. With +// failureDetails set, each failure row carries the latest failures behind it. +func runtimeView(snap *metrics.Snapshot, failureDetails bool) RuntimeView { if snap == nil { return RuntimeView{} } - v := RuntimeView{Available: true} + v := RuntimeView{Available: true, FailureDetailsOn: failureDetails} + failures := func(name string, labels ...string) []LabelledCount { + return failureRows(snap, name, failureDetails, labels...) + } requests := snap.Sum("proxy_requests_total") v.Requests = formatCount(int64(requests)) @@ -107,21 +132,23 @@ func runtimeView(snap *metrics.Snapshot) RuntimeView { v.UpstreamFetches = formatCount(int64(snap.Count("proxy_upstream_fetch_duration_seconds"))) v.UpstreamFetchMean = formatDuration(snap.Mean("proxy_upstream_fetch_duration_seconds")) - v.UpstreamErrors = failureRows(snap, "proxy_upstream_errors_total", "ecosystem", "error_type") + v.UpstreamErrors = failures("proxy_upstream_errors_total", "ecosystem", "error_type") v.StorageOps = histogramRows(snap, "proxy_storage_operation_duration_seconds", "operation") - v.StorageErrors = failureRows(snap, "proxy_storage_errors_total", "operation") - v.IntegrityFailures = failureRows(snap, "proxy_integrity_failures_total", "ecosystem") - v.ProbeFailures = failureRows(snap, "proxy_health_probe_failures_total", "step") + v.StorageErrors = failures("proxy_storage_errors_total", "operation") + v.IntegrityFailures = failures("proxy_integrity_failures_total", "ecosystem") + v.ProbeFailures = failures("proxy_health_probe_failures_total", "step") v.Evictions = countRows(snap, "proxy_artifacts_evicted_total", "reason", "ecosystem") v.Breakers = breakerRows(snap) - v.BreakerTrips = failureRows(snap, "proxy_circuit_breaker_trips_total", "registry") + v.BreakerTrips = failures("proxy_circuit_breaker_trips_total", "registry") v.Scans = histogramRows(snap, "proxy_scan_duration_seconds", "ecosystem", "scanner") - v.ScansBlocked = failureRows(snap, "proxy_scan_blocked_total", "ecosystem", "scanner") - v.ScanErrors = failureRows(snap, "proxy_scan_errors_total", "ecosystem", "scanner", "error_type") + v.ScansBlocked = failures("proxy_scan_blocked_total", "ecosystem", "scanner") + v.ScanErrors = failures("proxy_scan_errors_total", "ecosystem", "scanner", "error_type") v.ScanningOn = len(v.Scans) > 0 || len(v.ScansBlocked) > 0 || len(v.ScanErrors) > 0 + v.HasFailures = len(v.UpstreamErrors) > 0 || len(v.StorageErrors) > 0 || len(v.IntegrityFailures) > 0 || + len(v.ProbeFailures) > 0 || len(v.ScanErrors) > 0 v.ResponseBytes = formatSize(int64(snap.Sum("proxy_response_bytes_total"))) v.Clients = clientRows(snap) @@ -202,11 +229,15 @@ func statusClasses(snap *metrics.Snapshot) []LabelledCount { // failureRows renders every non-zero series of a failure counter, keyed by the // joined values of the given labels. Zero-valued series are dropped: a counter // that has never fired carries no information, and a wall of zeroes buries the -// rows that matter. -func failureRows(snap *metrics.Snapshot, name string, labels ...string) []LabelledCount { +// rows that matter. With withRecent set, each row carries the latest failures +// recorded for its series. +func failureRows(snap *metrics.Snapshot, name string, withRecent bool, labels ...string) []LabelledCount { rows := countRows(snap, name, labels...) for i := range rows { rows[i].Bad = true + if withRecent { + rows[i].Recent = recentFailureRows(metrics.RecentFailures(name, rows[i].series)) + } } return rows } @@ -219,8 +250,27 @@ func countRows(snap *metrics.Snapshot, name string, labels ...string) []Labelled continue } rows = append(rows, LabelledCount{ - Label: joinLabels(s, labels...), - Count: formatCount(int64(s.Value)), + Label: joinLabels(s, labels...), + Count: formatCount(int64(s.Value)), + series: s.Labels, + }) + } + return rows +} + +func recentFailureRows(failures []metrics.Failure) []FailureRow { + if len(failures) == 0 { + return nil + } + rows := make([]FailureRow, 0, len(failures)) + for _, f := range failures { + rows = append(rows, FailureRow{ + Age: formatTimeAgo(f.At), + PURL: f.PURL, + Filename: f.Filename, + URL: f.URL, + Path: f.Path, + Error: f.Error, }) } return rows diff --git a/internal/server/analytics_runtime_test.go b/internal/server/analytics_runtime_test.go index 422c02dd..5bff0f36 100644 --- a/internal/server/analytics_runtime_test.go +++ b/internal/server/analytics_runtime_test.go @@ -27,7 +27,7 @@ func TestFormatDuration(t *testing.T) { } func TestRuntimeViewNilSnapshot(t *testing.T) { - if v := runtimeView(nil); v.Available { + if v := runtimeView(nil, false); v.Available { t.Error("a nil snapshot must not report itself as available") } } @@ -63,7 +63,7 @@ func TestRuntimeViewShapesRegistry(t *testing.T) { if err != nil { t.Fatalf("GatherFrom: %v", err) } - v := runtimeView(snap) + v := runtimeView(snap, false) if !v.Available { t.Fatal("expected the view to be available") @@ -142,7 +142,7 @@ func TestRuntimeViewShowsEvictions(t *testing.T) { if err != nil { t.Fatalf("GatherFrom: %v", err) } - v := runtimeView(snap) + v := runtimeView(snap, false) if len(v.Evictions) != 1 { t.Fatalf("Evictions = %+v, want only the non-zero row", v.Evictions) @@ -168,7 +168,7 @@ func TestRuntimeViewDropsZeroCounters(t *testing.T) { if err != nil { t.Fatalf("GatherFrom: %v", err) } - v := runtimeView(snap) + v := runtimeView(snap, false) if len(v.StorageErrors) != 1 { t.Fatalf("expected only the non-zero row, got %+v", v.StorageErrors) @@ -185,7 +185,7 @@ func TestRuntimeViewEmptyRegistry(t *testing.T) { if err != nil { t.Fatalf("GatherFrom: %v", err) } - v := runtimeView(snap) + v := runtimeView(snap, false) if !v.Available { t.Error("an empty registry is still a valid snapshot") diff --git a/internal/server/health.go b/internal/server/health.go index 9533bd50..fd49cf13 100644 --- a/internal/server/health.go +++ b/internal/server/health.go @@ -169,9 +169,9 @@ func (c *healthCache) Check() error { if err != nil { var pe *probeError if errors.As(err, &pe) { - metrics.RecordHealthProbeFailure(pe.step) + metrics.RecordHealthProbeFailure(pe.step, metrics.FailureDetail{Err: pe.err}) } else { - metrics.RecordHealthProbeFailure("unknown") + metrics.RecordHealthProbeFailure("unknown", metrics.FailureDetail{Err: err}) } } diff --git a/internal/server/templates/components/runtime_metrics.html b/internal/server/templates/components/runtime_metrics.html index ff0f67cb..9804b475 100644 --- a/internal/server/templates/components/runtime_metrics.html +++ b/internal/server/templates/components/runtime_metrics.html @@ -92,6 +92,14 @@

Circuit breakers

+ {{if and .HasFailures (not .FailureDetailsOn)}} +

+ Per-failure detail is off. Set ui_failure_details: true + to list the packages, URLs and errors behind each failure count here, once + /ui is behind authentication; the same detail is in the log. +

+ {{end}} +
@@ -191,9 +199,14 @@

{{.Title}}

{{if .Rows}}
{{range .Rows}} -
+
{{.Label}}
{{.Count}}
+ {{if .Recent}} +
+ {{template "recent_failures" .Recent}} +
+ {{end}}
{{end}}
@@ -203,6 +216,29 @@

{{.Title}}

{{end}} +{{/* recent_failures lists the latest failures behind a failure count, newest first. */}} +{{define "recent_failures"}} +
+ + Latest {{len .}} + + +
+{{end}} + {{/* runtime_stats renders a histogram list: how many observations, and their mean. */}} {{define "runtime_stats"}}