From a0be33777c637b7571ffc7b646b9eb847db46a3d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 12:58:25 +0000 Subject: [PATCH 1/2] promtop: no rising/falling notes for the runtime's allocation and GC counts go_memstats_mallocs_total got a "falling" note whenever a burst of work ended, a series of /metrics renders for one: as an event counter it was held to Poisson noise, and with millions of allocations any change of 50% passed. The gcHeap names covered heap gauges, not these counters. Counts of the runtime's own work follow the service's load in bursts, so they are no news: allocations, frees, promotions and collections per second (go_memstats_{mallocs,frees,alloc_bytes}_total, go_memory_{allocations,allocated_bytes}_total, go_gc_cycles_*, jvm_memory_pool_allocated_bytes_total, jvm_gc_memory_{allocated,promoted}_bytes_total, and the collection rate that jvm_gc_collection_seconds and jvm_gc_pause_seconds plot without quantiles). They stay in the summary and on the panels with their trend; only the note goes. GC pause times (go_gc_duration_seconds quantiles, jvm_gc_pause_seconds_max) keep their notes, as do a growing RSS and 5xx. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01US3HyG56Ex8D8KAVo4hjax --- README.md | 3 +- internal/summary/summary.go | 17 ++- internal/summary/summary_test.go | 137 ++++++++++++++++++ .../analyzing-metrics-with-promtop/SKILL.md | 4 +- .../reference.md | 7 +- 5 files changed, 164 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7f67943..32617bd 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,8 @@ trends (three samples or more), and `-state` with `-offline` reads the history ` collected, so a cron job can ask a model about the last day. The detection is deterministic and small — health flags, usage at 90%, 5xx counters, deadlines under 7 days, timestamps that moved, and changes of 50% between the first and last third of the window that stand out of the noise -(Poisson for event counters, the spread of the samples otherwise; growing GC heaps are left out) — +(Poisson for event counters, the spread of the samples otherwise; growing GC heaps and the +runtime's allocation and GC counts are left out) — the model does the interpreting. [`skills/analyzing-metrics-with-promtop`](skills/analyzing-metrics-with-promtop) is an diff --git a/internal/summary/summary.go b/internal/summary/summary.go index c5b0921..c757d33 100644 --- a/internal/summary/summary.go +++ b/internal/summary/summary.go @@ -411,7 +411,7 @@ func notable(p *view.Panel, o Options) []Note { out = append(out, nt) case p.Kind == view.KindRate && serverError(s): // Counted once per metric below. - case !gcHeap.MatchString(p.Name()) && moved(p, s, st, o.Interval): + case !gcHeap.MatchString(p.Name()) && !runtimeWork(p) && moved(p, s, st, o.Interval): nt := note(ReasonRising) if st.Trend == ReasonFalling { nt.Reason = ReasonFalling @@ -471,6 +471,21 @@ func errorRate(live []*store.Series) (errs, all float64) { // leak shows in the resident memory, which keeps its trend notes. var gcHeap = regexp.MustCompile(`(?i)heap|^go_memstats_alloc_bytes$|^go_memory_used|^jvm_memory_(bytes_)?used`) +// runtimeCounts matches what a Go or JVM runtime counts as it works: +// allocations, frees, promotions and collections. +var runtimeCounts = regexp.MustCompile(`^go_memstats_(mallocs|frees|alloc_bytes)_total$|^go_memory_(allocations|allocated_bytes)_total$|` + + `^go_gc_cycles_|^go_gc_duration_seconds$|^jvm_memory_pool_allocated_bytes_total$|^jvm_gc_memory_(allocated|promoted)_bytes_total$|` + + `^jvm_gc_(collection|pause)_seconds$`) + +// runtimeWork reports whether p plots one of those counts per second. They +// follow the service's own work, in bursts, so a change is no news (and an +// event count in the millions makes any change significant); a leak shows in +// the resident memory. A GC summary plotting its pause times, not how often +// it collects, keeps its notes: pauses are short whatever the allocation rate. +func runtimeWork(p *view.Panel) bool { + return valueName(p) == "rate" && runtimeCounts.MatchString(p.Name()) +} + const ( nearLimit = 0.9 bigChange = 0.5 // relative change between the window's thirds that is notable diff --git a/internal/summary/summary_test.go b/internal/summary/summary_test.go index 9c9d3ee..e1e8f4d 100644 --- a/internal/summary/summary_test.go +++ b/internal/summary/summary_test.go @@ -113,6 +113,143 @@ func TestGaugeTrends(t *testing.T) { } } +// runtimeExposition is a Go and a JVM service over a window: allocations at +// alloc(i) per second, a GC per 100k of them whose pauses last pause(i) +// seconds, a resident memory that keeps climbing and a steady share of 5xx. +func runtimeExposition(alloc, pause func(i int) float64) func(i int) string { + var mallocs, frees, allocBytes, gcs, gcSum float64 + return func(i int) string { + if i > 0 { + r := alloc(i) * 10 // per scrape, 10s apart + mallocs += r + frees += 0.97 * r + allocBytes += 64 * r + gcs += r / 1e5 + gcSum += r / 1e5 * pause(i) + } + p := pause(i) + return fmt.Sprintf(`# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total %g +# TYPE go_memstats_frees_total counter +go_memstats_frees_total %g +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total %g +# TYPE go_memory_allocations_total counter +go_memory_allocations_total %g +# TYPE go_memory_allocated_bytes_total counter +go_memory_allocated_bytes_total %g +# TYPE go_gc_cycles_automatic_gc_cycles_total counter +go_gc_cycles_automatic_gc_cycles_total %g +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0"} %g +go_gc_duration_seconds{quantile="0.5"} %g +go_gc_duration_seconds{quantile="1"} %g +go_gc_duration_seconds_sum %g +go_gc_duration_seconds_count %g +# TYPE jvm_memory_pool_allocated_bytes_total counter +jvm_memory_pool_allocated_bytes_total{pool="G1 Eden Space"} %g +# TYPE jvm_gc_memory_allocated_bytes_total counter +jvm_gc_memory_allocated_bytes_total %g +# TYPE jvm_gc_memory_promoted_bytes_total counter +jvm_gc_memory_promoted_bytes_total %g +# TYPE jvm_gc_collection_seconds summary +jvm_gc_collection_seconds_sum{gc="G1 Young Generation"} %g +jvm_gc_collection_seconds_count{gc="G1 Young Generation"} %g +# TYPE jvm_gc_pause_seconds summary +jvm_gc_pause_seconds_sum{action="end of minor GC"} %g +jvm_gc_pause_seconds_count{action="end of minor GC"} %g +# TYPE jvm_gc_pause_seconds_max gauge +jvm_gc_pause_seconds_max{action="end of minor GC"} %g +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes %d +# TYPE http_requests_total counter +http_requests_total{code="200"} %d +http_requests_total{code="500"} %d +`, mallocs, frees, allocBytes, mallocs, allocBytes, gcs, + p/4, p/2, p, gcSum, gcs, + allocBytes, allocBytes, allocBytes/50, + gcSum, gcs, gcSum, gcs, p, + 100e6+i*20e6, 900*i, 100*i) + } +} + +func TestRuntimeChurn(t *testing.T) { + // A burst of work (renders of /metrics, a batch) changes the allocation + // rate tenfold. How much the runtime allocates, frees, promotes and + // collects follows the work, in bursts; that is no news, and a leak shows + // in the resident memory instead. A JVM summary without quantiles plots + // its collections per second, so it is such a count too. How long a GC + // pauses is another matter: pauses are short by design whatever the + // allocation rate, so pauses that grow keep their notes. + steady := func(int) float64 { return 1e6 } + burst := func(i int) float64 { + if i >= 5 { + return 1e5 + } + return 1e6 + } + quiet := func(i int) float64 { + if i >= 5 { + return 1e6 + } + return 1e5 + } + shortPauses := func(i int) float64 { return 100e-6 + float64(i%2)*5e-6 } + longerPauses := func(i int) float64 { + if i >= 5 { + return 10e-3 + } + return 1e-3 + } + churn := []string{ + "go_memstats_mallocs_total", "go_memstats_frees_total", "go_memstats_alloc_bytes_total", + "go_memory_allocations_total", "go_memory_allocated_bytes_total", "go_gc_cycles_automatic_gc_cycles_total", + "jvm_memory_pool_allocated_bytes_total", "jvm_gc_memory_allocated_bytes_total", + "jvm_gc_memory_promoted_bytes_total", "jvm_gc_collection_seconds", "jvm_gc_pause_seconds", + } + pauses := []string{"go_gc_duration_seconds", "jvm_gc_pause_seconds_max"} + for _, tc := range []struct { + name string + alloc, pause func(int) float64 + trend string // of the counts + pausesNoted bool + }{ + {"allocations fall", burst, shortPauses, ReasonFalling, false}, + {"allocations rise", quiet, shortPauses, ReasonRising, false}, + {"pauses grow", steady, longerPauses, "flat", true}, + } { + t.Run(tc.name, func(t *testing.T) { + st := ingest(t, 9, runtimeExposition(tc.alloc, tc.pause)) + o := Options{Interval: 10 * time.Second, From: t0, To: t0.Add(80 * time.Second), Now: t0.Add(80 * time.Second)} + d := Build(st, view.Build(st, view.Options{ShowAll: true}), o) // some are hidden as noise + for _, name := range churn { + if n := notes(d, name); len(n) != 0 { + t.Errorf("%s follows the work, no news: %+v", name, n) + } + m := find(d, name) + if m == nil || len(m.Series) != 1 || m.Series[0].Trend != tc.trend { + t.Errorf("%s stays in the summary with its trend %s: %+v", name, tc.trend, m) + } + } + for _, name := range pauses { + n := notes(d, name) + if tc.pausesNoted && (len(n) != 1 || n[0].Reason != ReasonRising) { + t.Errorf("%s: pauses ten times longer are news: %+v", name, n) + } + if !tc.pausesNoted && len(n) != 0 { + t.Errorf("%s: more GCs of the same short pause are no news: %+v", name, n) + } + } + if n := notes(d, "process_resident_memory_bytes"); len(n) != 1 || n[0].Reason != ReasonRising { + t.Errorf("a growing RSS is news: %+v", n) + } + if n := notes(d, "http_requests_total"); len(n) != 1 || n[0].Reason != ReasonErrors { + t.Errorf("5xx are news: %+v", n) + } + }) + } +} + func TestErrorShareUnfiltered(t *testing.T) { all := ingest(t, 3, func(i int) string { return fmt.Sprintf("# TYPE req_total counter\nreq_total{code=\"200\"} %d\nreq_total{code=\"500\"} %d\n", 300*i, 100*i) diff --git a/skills/analyzing-metrics-with-promtop/SKILL.md b/skills/analyzing-metrics-with-promtop/SKILL.md index 8ca2dd8..476bb35 100644 --- a/skills/analyzing-metrics-with-promtop/SKILL.md +++ b/skills/analyzing-metrics-with-promtop/SKILL.md @@ -60,7 +60,9 @@ Full field list: `reference.md` next to this file. - **Reporting "no trend" from a 2-scrape run.** Without `-for` there is no `trend`; sample a minute. - **Treating `notable` as a verdict.** It is a deterministic shortlist; check each item against the question. Garbage-collected heaps (`*heap*`, `jvm_memory_used`) never get trend notes: they - climb until each GC. Look for leaks in `process_resident_memory_bytes`. + climb until each GC. Nor do the runtime's allocation, free and GC counts (`go_memstats_mallocs_total`, + `jvm_gc_collection_seconds`...): they follow the work. GC pause times keep theirs. Look for leaks + in `process_resident_memory_bytes`. - **Calling a load swing a fault.** When traffic, CPU, latency and pools all rise or fall by a similar amount together, the load changed; a short window may catch one phase of a cycle. Faults show up as `down`, `errors`, `near_limit`, or one series moving against the rest. diff --git a/skills/analyzing-metrics-with-promtop/reference.md b/skills/analyzing-metrics-with-promtop/reference.md index b2e0edb..9d89e19 100644 --- a/skills/analyzing-metrics-with-promtop/reference.md +++ b/skills/analyzing-metrics-with-promtop/reference.md @@ -57,7 +57,12 @@ Poisson noise (3σ of the counts in a third). Gauges, quantiles and other rates they wobble within the thirds (Welch, 3σ), which needs 6 samples (`-for 12s` at the default `-i 2s`); quantiles also need 20 observations per third. Garbage-collected heap gauges (names with `heap`, `go_memstats_alloc_bytes`, `go_memory_used*`, `jvm_memory_used*`) get no trend notes: they -climb until each collection. +climb until each collection. Neither do the runtime's counts of its own work, which follow the +service's load in bursts: allocations, frees and collections per second +(`go_memstats_{mallocs,frees,alloc_bytes}_total`, `go_memory_{allocations,allocated_bytes}_total`, +`go_gc_cycles_*`, `jvm_memory_pool_allocated_bytes_total`, `jvm_gc_memory_{allocated,promoted}_bytes_total`, +the collection rate of `jvm_gc_collection_seconds` and `jvm_gc_pause_seconds`). Their `trend` stays; +GC pause times (`go_gc_duration_seconds` quantiles, `jvm_gc_pause_seconds_max`) keep their notes. ## Metrics From 1a05e101d241575c4389658e8a6adb8459f32770 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 13:07:02 +0000 Subject: [PATCH 2/2] promtop/dash: Render during the first scrape waits for it instead of failing Render with no history yet reads the metrics itself. When the scrape loop's first read was still in the handler, the in-process source refused a second call at once and Render returned "metrics handler still busy with an earlier scrape" (TestRenderSummaryNotable failed on macOS CI that way). The loop and Draw now take turns reading the source, and Draw uses the loop's scrape when it lands first. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01US3HyG56Ex8D8KAVo4hjax --- dash/render_test.go | 29 ++++++++++++++++++++++++++ internal/web/server.go | 46 +++++++++++++++++++++++++++++++++--------- 2 files changed, 66 insertions(+), 9 deletions(-) diff --git a/dash/render_test.go b/dash/render_test.go index 4eceade..4fd01d5 100644 --- a/dash/render_test.go +++ b/dash/render_test.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "strings" + "sync" "sync/atomic" "testing" "time" @@ -39,6 +40,34 @@ func TestRenderSummaryNotable(t *testing.T) { } } +// Render called while the loop's first scrape is still in the handler waits +// for that scrape instead of failing on a handler that is busy. +func TestRenderDuringFirstScrape(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + var first sync.Once + h := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + first.Do(func() { close(entered); <-release }) + _, _ = io.WriteString(w, "# TYPE app_up gauge\napp_up 1\n") + }) + d := New(h, Interval(time.Hour)) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + defer func() { unblock(); _ = d.Close() }() + <-entered + + errc := make(chan error, 1) + go func() { errc <- d.Render(t.Context(), io.Discard, RenderOptions{Format: FormatSummary}) }() + select { + case err := <-errc: + t.Fatalf("Render returned during the loop's first scrape: %v", err) + case <-time.After(50 * time.Millisecond): + } + unblock() + if err := <-errc; err != nil { + t.Fatal(err) + } +} + // Render needs no scrape loop to have run: with no history it reads the // metrics once itself. func TestRenderBeforeFirstScrape(t *testing.T) { diff --git a/internal/web/server.go b/internal/web/server.go index ed7f780..20e0c65 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -52,6 +52,10 @@ type Config struct { type Server struct { cfg Config + // scraping is held while the source is read: the loop and Draw take + // turns, since a metrics handler answers one scrape at a time. + scraping chan struct{} + mu sync.Mutex st *store.Store lastErr error @@ -81,7 +85,7 @@ func New(cfg Config) *Server { } return &Server{ cfg: cfg, st: store.New(cfg.History), subs: map[chan struct{}]struct{}{}, - cache: map[string][]byte{}, done: make(chan struct{}), + cache: map[string][]byte{}, done: make(chan struct{}), scraping: make(chan struct{}, 1), } } @@ -189,7 +193,13 @@ func (s *Server) scrapeLoop() { t := time.NewTicker(s.cfg.Interval) defer t.Stop() for { + select { + case s.scraping <- struct{}{}: + case <-s.done: + return + } s.ingest(s.cfg.Source.Scrape(ctx)) + <-s.scraping select { case <-s.done: return @@ -208,23 +218,41 @@ type History struct { // Draw calls fn with the history, holding the lock so no scrape changes // it meanwhile; fn must not keep the store. With no scrape yet it reads -// the source once first, and returns that scrape's error if it fails. +// the source once first, and returns that scrape's error if it fails; a +// scrape of the loop in progress is waited for instead. func (s *Server) Draw(ctx context.Context, fn func(History) error) error { - s.mu.Lock() - empty := s.st.Scrapes == 0 - s.mu.Unlock() - if empty { - r := s.cfg.Source.Scrape(ctx) - if r.Err != nil { + if s.empty() { + select { + case s.scraping <- struct{}{}: + case <-ctx.Done(): + return ctx.Err() + } + err := func() error { + defer func() { <-s.scraping }() + if !s.empty() { // the loop's scrape just landed + return nil + } + r := s.cfg.Source.Scrape(ctx) + if r.Err == nil { + s.ingest(r) + } return r.Err + }() + if err != nil { + return err } - s.ingest(r) } s.mu.Lock() defer s.mu.Unlock() return fn(History{Store: s.st, From: s.firstOK, To: s.lastOK, Interval: s.cfg.Interval}) } +func (s *Server) empty() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.st.Scrapes == 0 +} + // Ingest adds a scrape made elsewhere; it is how a Fed server gets data. // The families are only read, so the caller may keep using them. func (s *Server) Ingest(r scrape.Result) { s.ingest(r) }