diff --git a/README.md b/README.md index 7f67943..32617bd 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,8 @@ trends (three samples or more), and `-state` with `-offline` reads the history ` collected, so a cron job can ask a model about the last day. The detection is deterministic and small — health flags, usage at 90%, 5xx counters, deadlines under 7 days, timestamps that moved, and changes of 50% between the first and last third of the window that stand out of the noise -(Poisson for event counters, the spread of the samples otherwise; growing GC heaps are left out) — +(Poisson for event counters, the spread of the samples otherwise; growing GC heaps and the +runtime's allocation and GC counts are left out) — the model does the interpreting. [`skills/analyzing-metrics-with-promtop`](skills/analyzing-metrics-with-promtop) is an diff --git a/dash/render_test.go b/dash/render_test.go index 4eceade..4fd01d5 100644 --- a/dash/render_test.go +++ b/dash/render_test.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "strings" + "sync" "sync/atomic" "testing" "time" @@ -39,6 +40,34 @@ func TestRenderSummaryNotable(t *testing.T) { } } +// Render called while the loop's first scrape is still in the handler waits +// for that scrape instead of failing on a handler that is busy. +func TestRenderDuringFirstScrape(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + var first sync.Once + h := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + first.Do(func() { close(entered); <-release }) + _, _ = io.WriteString(w, "# TYPE app_up gauge\napp_up 1\n") + }) + d := New(h, Interval(time.Hour)) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + defer func() { unblock(); _ = d.Close() }() + <-entered + + errc := make(chan error, 1) + go func() { errc <- d.Render(t.Context(), io.Discard, RenderOptions{Format: FormatSummary}) }() + select { + case err := <-errc: + t.Fatalf("Render returned during the loop's first scrape: %v", err) + case <-time.After(50 * time.Millisecond): + } + unblock() + if err := <-errc; err != nil { + t.Fatal(err) + } +} + // Render needs no scrape loop to have run: with no history it reads the // metrics once itself. func TestRenderBeforeFirstScrape(t *testing.T) { diff --git a/internal/summary/summary.go b/internal/summary/summary.go index c5b0921..c757d33 100644 --- a/internal/summary/summary.go +++ b/internal/summary/summary.go @@ -411,7 +411,7 @@ func notable(p *view.Panel, o Options) []Note { out = append(out, nt) case p.Kind == view.KindRate && serverError(s): // Counted once per metric below. - case !gcHeap.MatchString(p.Name()) && moved(p, s, st, o.Interval): + case !gcHeap.MatchString(p.Name()) && !runtimeWork(p) && moved(p, s, st, o.Interval): nt := note(ReasonRising) if st.Trend == ReasonFalling { nt.Reason = ReasonFalling @@ -471,6 +471,21 @@ func errorRate(live []*store.Series) (errs, all float64) { // leak shows in the resident memory, which keeps its trend notes. var gcHeap = regexp.MustCompile(`(?i)heap|^go_memstats_alloc_bytes$|^go_memory_used|^jvm_memory_(bytes_)?used`) +// runtimeCounts matches what a Go or JVM runtime counts as it works: +// allocations, frees, promotions and collections. +var runtimeCounts = regexp.MustCompile(`^go_memstats_(mallocs|frees|alloc_bytes)_total$|^go_memory_(allocations|allocated_bytes)_total$|` + + `^go_gc_cycles_|^go_gc_duration_seconds$|^jvm_memory_pool_allocated_bytes_total$|^jvm_gc_memory_(allocated|promoted)_bytes_total$|` + + `^jvm_gc_(collection|pause)_seconds$`) + +// runtimeWork reports whether p plots one of those counts per second. They +// follow the service's own work, in bursts, so a change is no news (and an +// event count in the millions makes any change significant); a leak shows in +// the resident memory. A GC summary plotting its pause times, not how often +// it collects, keeps its notes: pauses are short whatever the allocation rate. +func runtimeWork(p *view.Panel) bool { + return valueName(p) == "rate" && runtimeCounts.MatchString(p.Name()) +} + const ( nearLimit = 0.9 bigChange = 0.5 // relative change between the window's thirds that is notable diff --git a/internal/summary/summary_test.go b/internal/summary/summary_test.go index 9c9d3ee..e1e8f4d 100644 --- a/internal/summary/summary_test.go +++ b/internal/summary/summary_test.go @@ -113,6 +113,143 @@ func TestGaugeTrends(t *testing.T) { } } +// runtimeExposition is a Go and a JVM service over a window: allocations at +// alloc(i) per second, a GC per 100k of them whose pauses last pause(i) +// seconds, a resident memory that keeps climbing and a steady share of 5xx. +func runtimeExposition(alloc, pause func(i int) float64) func(i int) string { + var mallocs, frees, allocBytes, gcs, gcSum float64 + return func(i int) string { + if i > 0 { + r := alloc(i) * 10 // per scrape, 10s apart + mallocs += r + frees += 0.97 * r + allocBytes += 64 * r + gcs += r / 1e5 + gcSum += r / 1e5 * pause(i) + } + p := pause(i) + return fmt.Sprintf(`# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total %g +# TYPE go_memstats_frees_total counter +go_memstats_frees_total %g +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total %g +# TYPE go_memory_allocations_total counter +go_memory_allocations_total %g +# TYPE go_memory_allocated_bytes_total counter +go_memory_allocated_bytes_total %g +# TYPE go_gc_cycles_automatic_gc_cycles_total counter +go_gc_cycles_automatic_gc_cycles_total %g +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0"} %g +go_gc_duration_seconds{quantile="0.5"} %g +go_gc_duration_seconds{quantile="1"} %g +go_gc_duration_seconds_sum %g +go_gc_duration_seconds_count %g +# TYPE jvm_memory_pool_allocated_bytes_total counter +jvm_memory_pool_allocated_bytes_total{pool="G1 Eden Space"} %g +# TYPE jvm_gc_memory_allocated_bytes_total counter +jvm_gc_memory_allocated_bytes_total %g +# TYPE jvm_gc_memory_promoted_bytes_total counter +jvm_gc_memory_promoted_bytes_total %g +# TYPE jvm_gc_collection_seconds summary +jvm_gc_collection_seconds_sum{gc="G1 Young Generation"} %g +jvm_gc_collection_seconds_count{gc="G1 Young Generation"} %g +# TYPE jvm_gc_pause_seconds summary +jvm_gc_pause_seconds_sum{action="end of minor GC"} %g +jvm_gc_pause_seconds_count{action="end of minor GC"} %g +# TYPE jvm_gc_pause_seconds_max gauge +jvm_gc_pause_seconds_max{action="end of minor GC"} %g +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes %d +# TYPE http_requests_total counter +http_requests_total{code="200"} %d +http_requests_total{code="500"} %d +`, mallocs, frees, allocBytes, mallocs, allocBytes, gcs, + p/4, p/2, p, gcSum, gcs, + allocBytes, allocBytes, allocBytes/50, + gcSum, gcs, gcSum, gcs, p, + 100e6+i*20e6, 900*i, 100*i) + } +} + +func TestRuntimeChurn(t *testing.T) { + // A burst of work (renders of /metrics, a batch) changes the allocation + // rate tenfold. How much the runtime allocates, frees, promotes and + // collects follows the work, in bursts; that is no news, and a leak shows + // in the resident memory instead. A JVM summary without quantiles plots + // its collections per second, so it is such a count too. How long a GC + // pauses is another matter: pauses are short by design whatever the + // allocation rate, so pauses that grow keep their notes. + steady := func(int) float64 { return 1e6 } + burst := func(i int) float64 { + if i >= 5 { + return 1e5 + } + return 1e6 + } + quiet := func(i int) float64 { + if i >= 5 { + return 1e6 + } + return 1e5 + } + shortPauses := func(i int) float64 { return 100e-6 + float64(i%2)*5e-6 } + longerPauses := func(i int) float64 { + if i >= 5 { + return 10e-3 + } + return 1e-3 + } + churn := []string{ + "go_memstats_mallocs_total", "go_memstats_frees_total", "go_memstats_alloc_bytes_total", + "go_memory_allocations_total", "go_memory_allocated_bytes_total", "go_gc_cycles_automatic_gc_cycles_total", + "jvm_memory_pool_allocated_bytes_total", "jvm_gc_memory_allocated_bytes_total", + "jvm_gc_memory_promoted_bytes_total", "jvm_gc_collection_seconds", "jvm_gc_pause_seconds", + } + pauses := []string{"go_gc_duration_seconds", "jvm_gc_pause_seconds_max"} + for _, tc := range []struct { + name string + alloc, pause func(int) float64 + trend string // of the counts + pausesNoted bool + }{ + {"allocations fall", burst, shortPauses, ReasonFalling, false}, + {"allocations rise", quiet, shortPauses, ReasonRising, false}, + {"pauses grow", steady, longerPauses, "flat", true}, + } { + t.Run(tc.name, func(t *testing.T) { + st := ingest(t, 9, runtimeExposition(tc.alloc, tc.pause)) + o := Options{Interval: 10 * time.Second, From: t0, To: t0.Add(80 * time.Second), Now: t0.Add(80 * time.Second)} + d := Build(st, view.Build(st, view.Options{ShowAll: true}), o) // some are hidden as noise + for _, name := range churn { + if n := notes(d, name); len(n) != 0 { + t.Errorf("%s follows the work, no news: %+v", name, n) + } + m := find(d, name) + if m == nil || len(m.Series) != 1 || m.Series[0].Trend != tc.trend { + t.Errorf("%s stays in the summary with its trend %s: %+v", name, tc.trend, m) + } + } + for _, name := range pauses { + n := notes(d, name) + if tc.pausesNoted && (len(n) != 1 || n[0].Reason != ReasonRising) { + t.Errorf("%s: pauses ten times longer are news: %+v", name, n) + } + if !tc.pausesNoted && len(n) != 0 { + t.Errorf("%s: more GCs of the same short pause are no news: %+v", name, n) + } + } + if n := notes(d, "process_resident_memory_bytes"); len(n) != 1 || n[0].Reason != ReasonRising { + t.Errorf("a growing RSS is news: %+v", n) + } + if n := notes(d, "http_requests_total"); len(n) != 1 || n[0].Reason != ReasonErrors { + t.Errorf("5xx are news: %+v", n) + } + }) + } +} + func TestErrorShareUnfiltered(t *testing.T) { all := ingest(t, 3, func(i int) string { return fmt.Sprintf("# TYPE req_total counter\nreq_total{code=\"200\"} %d\nreq_total{code=\"500\"} %d\n", 300*i, 100*i) diff --git a/internal/web/server.go b/internal/web/server.go index ed7f780..20e0c65 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -52,6 +52,10 @@ type Config struct { type Server struct { cfg Config + // scraping is held while the source is read: the loop and Draw take + // turns, since a metrics handler answers one scrape at a time. + scraping chan struct{} + mu sync.Mutex st *store.Store lastErr error @@ -81,7 +85,7 @@ func New(cfg Config) *Server { } return &Server{ cfg: cfg, st: store.New(cfg.History), subs: map[chan struct{}]struct{}{}, - cache: map[string][]byte{}, done: make(chan struct{}), + cache: map[string][]byte{}, done: make(chan struct{}), scraping: make(chan struct{}, 1), } } @@ -189,7 +193,13 @@ func (s *Server) scrapeLoop() { t := time.NewTicker(s.cfg.Interval) defer t.Stop() for { + select { + case s.scraping <- struct{}{}: + case <-s.done: + return + } s.ingest(s.cfg.Source.Scrape(ctx)) + <-s.scraping select { case <-s.done: return @@ -208,23 +218,41 @@ type History struct { // Draw calls fn with the history, holding the lock so no scrape changes // it meanwhile; fn must not keep the store. With no scrape yet it reads -// the source once first, and returns that scrape's error if it fails. +// the source once first, and returns that scrape's error if it fails; a +// scrape of the loop in progress is waited for instead. func (s *Server) Draw(ctx context.Context, fn func(History) error) error { - s.mu.Lock() - empty := s.st.Scrapes == 0 - s.mu.Unlock() - if empty { - r := s.cfg.Source.Scrape(ctx) - if r.Err != nil { + if s.empty() { + select { + case s.scraping <- struct{}{}: + case <-ctx.Done(): + return ctx.Err() + } + err := func() error { + defer func() { <-s.scraping }() + if !s.empty() { // the loop's scrape just landed + return nil + } + r := s.cfg.Source.Scrape(ctx) + if r.Err == nil { + s.ingest(r) + } return r.Err + }() + if err != nil { + return err } - s.ingest(r) } s.mu.Lock() defer s.mu.Unlock() return fn(History{Store: s.st, From: s.firstOK, To: s.lastOK, Interval: s.cfg.Interval}) } +func (s *Server) empty() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.st.Scrapes == 0 +} + // Ingest adds a scrape made elsewhere; it is how a Fed server gets data. // The families are only read, so the caller may keep using them. func (s *Server) Ingest(r scrape.Result) { s.ingest(r) } diff --git a/skills/analyzing-metrics-with-promtop/SKILL.md b/skills/analyzing-metrics-with-promtop/SKILL.md index 8ca2dd8..476bb35 100644 --- a/skills/analyzing-metrics-with-promtop/SKILL.md +++ b/skills/analyzing-metrics-with-promtop/SKILL.md @@ -60,7 +60,9 @@ Full field list: `reference.md` next to this file. - **Reporting "no trend" from a 2-scrape run.** Without `-for` there is no `trend`; sample a minute. - **Treating `notable` as a verdict.** It is a deterministic shortlist; check each item against the question. Garbage-collected heaps (`*heap*`, `jvm_memory_used`) never get trend notes: they - climb until each GC. Look for leaks in `process_resident_memory_bytes`. + climb until each GC. Nor do the runtime's allocation, free and GC counts (`go_memstats_mallocs_total`, + `jvm_gc_collection_seconds`...): they follow the work. GC pause times keep theirs. Look for leaks + in `process_resident_memory_bytes`. - **Calling a load swing a fault.** When traffic, CPU, latency and pools all rise or fall by a similar amount together, the load changed; a short window may catch one phase of a cycle. Faults show up as `down`, `errors`, `near_limit`, or one series moving against the rest. diff --git a/skills/analyzing-metrics-with-promtop/reference.md b/skills/analyzing-metrics-with-promtop/reference.md index b2e0edb..9d89e19 100644 --- a/skills/analyzing-metrics-with-promtop/reference.md +++ b/skills/analyzing-metrics-with-promtop/reference.md @@ -57,7 +57,12 @@ Poisson noise (3σ of the counts in a third). Gauges, quantiles and other rates they wobble within the thirds (Welch, 3σ), which needs 6 samples (`-for 12s` at the default `-i 2s`); quantiles also need 20 observations per third. Garbage-collected heap gauges (names with `heap`, `go_memstats_alloc_bytes`, `go_memory_used*`, `jvm_memory_used*`) get no trend notes: they -climb until each collection. +climb until each collection. Neither do the runtime's counts of its own work, which follow the +service's load in bursts: allocations, frees and collections per second +(`go_memstats_{mallocs,frees,alloc_bytes}_total`, `go_memory_{allocations,allocated_bytes}_total`, +`go_gc_cycles_*`, `jvm_memory_pool_allocated_bytes_total`, `jvm_gc_memory_{allocated,promoted}_bytes_total`, +the collection rate of `jvm_gc_collection_seconds` and `jvm_gc_pause_seconds`). Their `trend` stays; +GC pause times (`go_gc_duration_seconds` quantiles, `jvm_gc_pause_seconds_max`) keep their notes. ## Metrics