From 4e73c5636238f6f788b7c9089947cd3ec86d2351 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 15:36:57 -0400 Subject: [PATCH 1/6] =?UTF-8?q?=F0=9F=8C=B1=20runtime:=20loopback=20pprof,?= =?UTF-8?q?=20periodic=20memstats=20and=20cgroup-aware=20GOMEMLIMIT?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds pkg/runtimeobs: a loopback-only net/http/pprof listener (debug.pprof_addr, default 127.0.0.1:6060, 'off' disables), a 60 s 'mem: runtime snapshot' INFO line (heap in-use/live/objects, next GC, goroutines, OS threads, GOMEMLIMIT) read via runtime/metrics, the same snapshot under 'runtime' in GET /api/health, and a GOMEMLIMIT default of 85% of the cgroup memory limit when the env var is unset. Refs #14681 Signed-off-by: Andy Anderson --- changelog.d/added-runtime-pprof-memstats.md | 1 + src/cmd/hive/main.go | 5 + src/cmd/hive/runtime_observability.go | 40 +++ src/docs/operator-reference.md | 21 ++ src/pkg/config/config.go | 36 +++ src/pkg/dashboard/server.go | 3 +- src/pkg/runtimeobs/runtimeobs.go | 283 ++++++++++++++++++++ src/pkg/runtimeobs/runtimeobs_test.go | 172 ++++++++++++ 8 files changed, 560 insertions(+), 1 deletion(-) create mode 100644 changelog.d/added-runtime-pprof-memstats.md create mode 100644 src/cmd/hive/runtime_observability.go create mode 100644 src/pkg/runtimeobs/runtimeobs.go create mode 100644 src/pkg/runtimeobs/runtimeobs_test.go diff --git a/changelog.d/added-runtime-pprof-memstats.md b/changelog.d/added-runtime-pprof-memstats.md new file mode 100644 index 0000000000..1e25c322fc --- /dev/null +++ b/changelog.d/added-runtime-pprof-memstats.md @@ -0,0 +1 @@ +Loopback-only `net/http/pprof` listener (`debug.pprof_addr`, default `127.0.0.1:6060`, `off` to disable), a `mem: runtime snapshot` INFO line every 60 s (heap in-use/live/objects, next GC, goroutines, OS threads, GOMEMLIMIT), the same counters under `runtime` in `GET /api/health`, and a cgroup-aware GOMEMLIMIT default (85% of the container memory limit when the env var is unset). Refs #14681. diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 96fcfec70f..2c34a3bb90 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1168,6 +1168,11 @@ func (b *boot) bootConfigWith(deps bootConfigDeps) bool { b.ctx, cancel = context.WithCancel(context.Background()) b.cleanup.push(func() { cancel() }) + // Runtime observability (#14681): GOMEMLIMIT from the cgroup limit, the + // loopback pprof listener and the 60 s `mem:` line. Before tracing so the + // first mem line lands even when an exporter stalls. + b.startRuntimeObservability() + // Initialize OpenTelemetry tracing. Off by default: with no otel block // (or otel.enabled=false) this installs a no-op provider with zero export // overhead. Never fatal — a tracing setup error must not stop hive. diff --git a/src/cmd/hive/runtime_observability.go b/src/cmd/hive/runtime_observability.go new file mode 100644 index 0000000000..8d71019b8a --- /dev/null +++ b/src/cmd/hive/runtime_observability.go @@ -0,0 +1,40 @@ +package main + +import ( + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/runtimeobs" +) + +// startRuntimeObservability wires pkg/runtimeobs into the boot: the soft +// memory limit, the loopback-only pprof listener (debug.pprof_addr) and the +// periodic `mem:` log line (hivecommons/hive#14681). Nothing here is fatal — +// a bind failure only costs the profiles, never the hive. +func (b *boot) startRuntimeObservability() { + out := runtimeobs.ApplyMemoryLimit(nil, nil) + b.logger.Info("runtime: memory limit", + "source", out.Source, + "gomemlimit", out.Limit, + "cgroup_limit", out.CgroupLimit, + "percent", runtimeobs.MemoryLimitPercent, + ) + + addr := b.cfg.Debug.EffectivePprofAddr() + if addr != "" && !runtimeobs.IsLoopbackAddr(addr) { + b.logger.Warn("runtime: debug.pprof_addr is not a loopback address; using the default", + "configured", addr, "default", config.DefaultPprofAddr) + addr = config.DefaultPprofAddr + } + bound, stop, err := runtimeobs.StartPprof(addr, b.logger) + switch { + case err != nil: + b.logger.Warn("runtime: pprof listener not started", "addr", addr, "error", err) + case bound == "": + b.logger.Info("runtime: pprof listener disabled", "key", "debug.pprof_addr") + default: + b.cleanup.push(stop) + b.logger.Info("runtime: pprof listening (loopback only)", "addr", bound, "path", "/debug/pprof/") + } + + runtimeobs.StartLogger(b.ctx, b.logger, nil) + b.logger.Info("mem: runtime snapshot", runtimeobs.Take().LogAttrs()...) +} diff --git a/src/docs/operator-reference.md b/src/docs/operator-reference.md index f5da073492..7e5753bcd1 100644 --- a/src/docs/operator-reference.md +++ b/src/docs/operator-reference.md @@ -1466,6 +1466,27 @@ means the App/key/installation are healthy, but the hive still names the old repository owner after an org transfer; update the configured project org rather than trying to add the moved repository back to the old installation. +## Runtime diagnostics (`debug`, `GOMEMLIMIT`, `mem:` log line) + +Added for [#14681](https://github.com/hivecommons/hive/issues/14681) (hive +process RSS growing hundreds of MiB/min with no pprof to name the culprit). + +| Name | Source | Default | Purpose | +|---|---|---|---| +| `debug.pprof_addr` | `hive.yaml` | `127.0.0.1:6060` | `net/http/pprof` listener. **Loopback only**: a non-loopback host (`0.0.0.0`, pod IP) is rejected at startup with a warning and the default is used instead. `off` disables the listener. Profiles are under `/debug/pprof/`; `/debug/runtime` prints the same counters as the `mem:` log line. | +| `GOMEMLIMIT` | env | 85% of the cgroup memory limit | Go soft memory limit. When the env var is unset the hive reads `/sys/fs/cgroup/memory.max` (v2) or `memory/memory.limit_in_bytes` (v1) and sets `MemoryLimitPercent` (85%) of it via `debug.SetMemoryLimit`; an unlimited cgroup leaves the runtime default. Set the env var to override. Logged at startup as `runtime: memory limit` with `source=env|cgroup|none`. | +| `mem: runtime snapshot` | log (INFO, every 60 s) | always on | `heap_inuse`, `heap_live`, `heap_objects`, `total_sys`, `next_gc`, `gc_cycles`, `goroutines`, `threads` (OS threads from `/proc/self/status`), `gomemlimit`. Read from `runtime/metrics`, so no stop-the-world. | + +The same snapshot is embedded in `GET /api/health` as `runtime` so the +dashboard health probe and `kubectl exec … curl` see goroutine count and heap +without the pprof port. Taking profiles on a pod: + +```bash +kubectl -n port-forward deploy/hive 6060:6060 & +go tool pprof -top http://127.0.0.1:6060/debug/pprof/heap # inuse_space +curl -s 'http://127.0.0.1:6060/debug/pprof/goroutine?debug=1' | head # parked goroutines +``` + ## `cmd/hive` flags and environment ### Startup diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 050b06a3fe..be4dfde5c1 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -98,6 +98,9 @@ type Config struct { // operator-facing block; Tracing is retained as a legacy alias. OTel OTelConfig `yaml:"otel,omitempty" json:"otel,omitempty"` Tracing OTelConfig `yaml:"tracing,omitempty"` + // Debug holds process-level diagnostics knobs (pprof listener). See + // docs/operator-reference.md "Runtime diagnostics". + Debug DebugConfig `yaml:"debug,omitempty" json:"debug,omitempty"` // Triggers is an additive list of CEL-based declarative agent triggers. // Default empty → existing label/governor triggering is unchanged. Triggers []TriggerRule `yaml:"triggers,omitempty" json:"triggers,omitempty"` @@ -350,6 +353,39 @@ func (o OTelConfig) ServiceNameOrDefault() string { return DefaultOTelServiceName } +const ( + // DefaultPprofAddr is where the net/http/pprof listener binds when + // debug.pprof_addr is unset. Loopback only: the profiles leak source paths + // and heap contents, so they are reached via `kubectl port-forward` or an + // in-container curl, never over the pod network. + DefaultPprofAddr = "127.0.0.1:6060" + // PprofAddrOff is the debug.pprof_addr sentinel that disables the listener. + PprofAddrOff = "off" +) + +// DebugConfig is the optional `debug:` block for process-level diagnostics +// (hivecommons/hive#14681). +type DebugConfig struct { + // PprofAddr is the host:port the net/http/pprof listener binds to. Empty + // means DefaultPprofAddr; "off" disables it. The host must be a loopback + // address — any other host is rejected at startup and falls back to the + // default with a logged warning, so a typo can never expose profiles. + PprofAddr string `yaml:"pprof_addr,omitempty" json:"pprof_addr,omitempty"` +} + +// EffectivePprofAddr resolves PprofAddr to the address to bind, or "" when +// the listener is disabled. +func (d DebugConfig) EffectivePprofAddr() string { + addr := strings.TrimSpace(d.PprofAddr) + switch { + case addr == "": + return DefaultPprofAddr + case strings.EqualFold(addr, PprofAddrOff): + return "" + } + return addr +} + // IsZero reports whether the block contains no operator-provided settings. func (o OTelConfig) IsZero() bool { return !o.Enabled && o.Endpoint == "" && len(o.Headers) == 0 && o.ServiceName == "" && !o.Insecure && o.SampleRatio == 0 diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index d25d2b77e6..d6a6a23008 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -8,6 +8,7 @@ import ( "embed" "encoding/json" "fmt" + "github.com/hivecommons/hive/pkg/runtimeobs" "html" "io" "io/fs" @@ -3085,7 +3086,7 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { jsonResponse(w, map[string]string{"status": "starting"}) return } - body := map[string]any{"status": "ok"} + body := map[string]any{"status": "ok", "runtime": runtimeobs.Take()} if s.deps != nil && s.deps.CruxRuntimeHealth != nil { rt := s.deps.CruxRuntimeHealth() body["crux_runtime"] = rt diff --git a/src/pkg/runtimeobs/runtimeobs.go b/src/pkg/runtimeobs/runtimeobs.go new file mode 100644 index 0000000000..d3e8b0550b --- /dev/null +++ b/src/pkg/runtimeobs/runtimeobs.go @@ -0,0 +1,283 @@ +// Package runtimeobs gives the hive process the three things it lacked when +// hivecommons/hive#14681 (RSS +360 MiB/min, OOM at 24Gi) had to be diagnosed +// from /proc alone: a loopback net/http/pprof listener, a periodic memory / +// goroutine / thread log line, and a cgroup-aware GOMEMLIMIT default so the +// GC pushes back before the kernel does. +package runtimeobs + +import ( + "bytes" + "context" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "net/http/pprof" + "os" + "runtime" + "runtime/debug" + "runtime/metrics" + "strconv" + "strings" + "time" +) + +const ( + // LogInterval is how often StartLogger emits the `mem:` line. + LogInterval = 60 * time.Second + + // MemoryLimitPercent is the share of the cgroup memory limit handed to + // the Go runtime as GOMEMLIMIT when the env var is unset. The remaining + // 15% covers the non-Go RSS in the same cgroup (agent CLIs run in the + // same container) and runtime overhead outside the heap. + MemoryLimitPercent = 85 + + // cgroupV2MemoryMax and cgroupV1MemoryLimit are the files the container + // memory limit is read from, in order of preference. "max" (v2) or the + // kernel's "no limit" sentinel (v1) leave GOMEMLIMIT untouched. + cgroupV2MemoryMax = "/sys/fs/cgroup/memory.max" + cgroupV1MemoryLimit = "/sys/fs/cgroup/memory/memory.limit_in_bytes" + + // cgroupV1NoLimitFloor: v1 reports an unlimited cgroup as a page-rounded + // value just under 2^63; anything at or above this is "no limit". + cgroupV1NoLimitFloor = 1 << 62 + + // memoryLimitEnv is the Go runtime's own variable; when set the runtime + // has already applied it and this package only reports it. + memoryLimitEnv = "GOMEMLIMIT" + + // pprofReadHeaderTimeout bounds a slow client on the debug listener; the + // handlers themselves stream for as long as the profile asks (seconds=). + pprofReadHeaderTimeout = 10 * time.Second + + // procSelfStatus is where the OS thread count comes from on Linux. + procSelfStatus = "/proc/self/status" +) + +// Snapshot is one reading of the process-level runtime counters. Every field +// comes from runtime/metrics (no stop-the-world) except Threads, which is +// read from /proc on Linux and is 0 where that is unavailable. +type Snapshot struct { + HeapInuse uint64 `json:"heap_inuse_bytes"` + HeapLive uint64 `json:"heap_live_bytes"` + HeapObjects uint64 `json:"heap_objects"` + NextGC uint64 `json:"next_gc_bytes"` + TotalSys uint64 `json:"total_sys_bytes"` + NumGC uint64 `json:"gc_cycles"` + Goroutines int `json:"goroutines"` + Threads int `json:"threads"` + MemLimit int64 `json:"gomemlimit_bytes"` +} + +var sampleNames = []string{ + "/memory/classes/heap/objects:bytes", + "/gc/heap/live:bytes", + "/gc/heap/objects:objects", + "/gc/heap/goal:bytes", + "/memory/classes/total:bytes", + "/gc/cycles/total:gc-cycles", +} + +// Take reads a Snapshot. +func Take() Snapshot { + samples := make([]metrics.Sample, len(sampleNames)) + for i, n := range sampleNames { + samples[i].Name = n + } + metrics.Read(samples) + u := func(i int) uint64 { + if samples[i].Value.Kind() == metrics.KindUint64 { + return samples[i].Value.Uint64() + } + return 0 + } + return Snapshot{ + HeapInuse: u(0), + HeapLive: u(1), + HeapObjects: u(2), + NextGC: u(3), + TotalSys: u(4), + NumGC: u(5), + Goroutines: runtime.NumGoroutine(), + Threads: threadCount(), + MemLimit: debug.SetMemoryLimit(-1), + } +} + +// LogAttrs renders the snapshot as slog attributes. +func (s Snapshot) LogAttrs() []any { + return []any{ + "heap_inuse", s.HeapInuse, + "heap_live", s.HeapLive, + "heap_objects", s.HeapObjects, + "total_sys", s.TotalSys, + "next_gc", s.NextGC, + "gc_cycles", s.NumGC, + "goroutines", s.Goroutines, + "threads", s.Threads, + "gomemlimit", s.MemLimit, + } +} + +// threadCount parses Threads: from /proc/self/status; 0 when unavailable. +func threadCount() int { + data, err := os.ReadFile(procSelfStatus) + if err != nil { + return 0 + } + return ParseThreads(data) +} + +// ParseThreads extracts the Threads: field from a /proc//status body. +func ParseThreads(status []byte) int { + for _, line := range bytes.Split(status, []byte("\n")) { + if !bytes.HasPrefix(line, []byte("Threads:")) { + continue + } + n, err := strconv.Atoi(strings.TrimSpace(string(line[len("Threads:"):]))) + if err != nil { + return 0 + } + return n + } + return 0 +} + +// StartLogger emits one `mem:` INFO line per tick until ctx is done. tick +// is injectable so tests drive it without sleeping; nil means a real ticker +// at LogInterval. +func StartLogger(ctx context.Context, logger *slog.Logger, tick <-chan time.Time) { + if logger == nil { + logger = slog.Default() + } + var stop func() + if tick == nil { + t := time.NewTicker(LogInterval) + tick, stop = t.C, t.Stop + } + go func() { + if stop != nil { + defer stop() + } + for { + select { + case <-ctx.Done(): + return + case <-tick: + logger.Info("mem: runtime snapshot", Take().LogAttrs()...) + } + } + }() +} + +// ErrNotLoopback is returned by StartPprof for a non-loopback bind host. +var ErrNotLoopback = errors.New("pprof listener address must be loopback (127.0.0.0/8, ::1, or localhost)") + +// IsLoopbackAddr reports whether host:port names a loopback host. +func IsLoopbackAddr(addr string) bool { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return false + } + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +// StartPprof serves net/http/pprof on addr, which must be a loopback address. +// It returns the bound address and a stop func. An empty addr is a no-op. +func StartPprof(addr string, logger *slog.Logger) (bound string, stop func(), err error) { + if addr == "" { + return "", func() {}, nil + } + if !IsLoopbackAddr(addr) { + return "", nil, fmt.Errorf("%w: %q", ErrNotLoopback, addr) + } + ln, err := net.Listen("tcp", addr) + if err != nil { + return "", nil, err + } + mux := http.NewServeMux() + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + mux.HandleFunc("/debug/runtime", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + attrs := Take().LogAttrs() + for i := 0; i+1 < len(attrs); i += 2 { + fmt.Fprintf(w, "%v=%v\n", attrs[i], attrs[i+1]) + } + }) + srv := &http.Server{Handler: mux, ReadHeaderTimeout: pprofReadHeaderTimeout} + go func() { + if serveErr := srv.Serve(ln); serveErr != nil && !errors.Is(serveErr, http.ErrServerClosed) && logger != nil { + logger.Warn("pprof listener stopped", "addr", addr, "error", serveErr) + } + }() + return ln.Addr().String(), func() { _ = srv.Close() }, nil +} + +// MemoryLimitOutcome says what ApplyMemoryLimit did, for the startup log. +type MemoryLimitOutcome struct { + // Source is "env" (GOMEMLIMIT already set), "cgroup" (derived), or + // "none" (no limit found; runtime default of math.MaxInt64 kept). + Source string + // CgroupLimit is the raw cgroup limit in bytes when one was found. + CgroupLimit int64 + // Limit is the effective GOMEMLIMIT in bytes. + Limit int64 +} + +// ApplyMemoryLimit sets the Go soft memory limit to MemoryLimitPercent of +// the cgroup memory limit unless GOMEMLIMIT is already set (the runtime has +// then applied it) or no finite cgroup limit exists. getenv and readFile are +// injectable for tests; nil means os.Getenv / os.ReadFile. +func ApplyMemoryLimit(getenv func(string) string, readFile func(string) ([]byte, error)) MemoryLimitOutcome { + if getenv == nil { + getenv = os.Getenv + } + if readFile == nil { + readFile = os.ReadFile + } + if strings.TrimSpace(getenv(memoryLimitEnv)) != "" { + return MemoryLimitOutcome{Source: "env", Limit: debug.SetMemoryLimit(-1)} + } + cg, ok := CgroupMemoryLimit(readFile) + if !ok { + return MemoryLimitOutcome{Source: "none", Limit: debug.SetMemoryLimit(-1)} + } + limit := DeriveMemoryLimit(cg) + debug.SetMemoryLimit(limit) + return MemoryLimitOutcome{Source: "cgroup", CgroupLimit: cg, Limit: limit} +} + +// DeriveMemoryLimit is MemoryLimitPercent of the cgroup limit. +func DeriveMemoryLimit(cgroupLimit int64) int64 { + return cgroupLimit / 100 * MemoryLimitPercent +} + +// CgroupMemoryLimit reads the container memory limit from cgroup v2, then +// v1. ok is false when neither file exists or the cgroup is unlimited. +func CgroupMemoryLimit(readFile func(string) ([]byte, error)) (limit int64, ok bool) { + for _, path := range []string{cgroupV2MemoryMax, cgroupV1MemoryLimit} { + data, err := readFile(path) + if err != nil { + continue + } + raw := strings.TrimSpace(string(data)) + if raw == "max" || raw == "" { + return 0, false + } + n, err := strconv.ParseInt(raw, 10, 64) + if err != nil || n <= 0 || n >= cgroupV1NoLimitFloor { + return 0, false + } + return n, true + } + return 0, false +} diff --git a/src/pkg/runtimeobs/runtimeobs_test.go b/src/pkg/runtimeobs/runtimeobs_test.go new file mode 100644 index 0000000000..a8e21f5eca --- /dev/null +++ b/src/pkg/runtimeobs/runtimeobs_test.go @@ -0,0 +1,172 @@ +package runtimeobs + +import ( + "bytes" + "context" + "errors" + "io" + "log/slog" + "net/http" + "runtime/debug" + "strings" + "testing" + "time" +) + +func TestParseThreads(t *testing.T) { + status := []byte("Name:\thive\nState:\tS (sleeping)\nThreads:\t156\nSigQ:\t0/1\n") + if got := ParseThreads(status); got != 156 { + t.Fatalf("ParseThreads = %d, want 156", got) + } + if got := ParseThreads([]byte("Name:\thive\n")); got != 0 { + t.Fatalf("ParseThreads without field = %d, want 0", got) + } +} + +func TestTakeIsPopulated(t *testing.T) { + s := Take() + if s.Goroutines < 1 { + t.Fatalf("Goroutines = %d, want >= 1", s.Goroutines) + } + if s.TotalSys == 0 || s.NextGC == 0 { + t.Fatalf("runtime/metrics samples missing: %+v", s) + } +} + +func TestStartLoggerEmitsPerTick(t *testing.T) { + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, nil)) + tick := make(chan time.Time) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + StartLogger(ctx, logger, tick) + for i := 0; i < 3; i++ { + tick <- time.Time{} + } + cancel() + // The send above only returns once the goroutine received, so three + // records have been written (the handler writes synchronously). + deadline := time.Now().Add(2 * time.Second) + for strings.Count(buf.String(), "mem: runtime snapshot") < 3 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + out := buf.String() + if n := strings.Count(out, "mem: runtime snapshot"); n != 3 { + t.Fatalf("got %d mem lines, want 3:\n%s", n, out) + } + for _, key := range []string{"heap_inuse=", "heap_objects=", "goroutines=", "threads=", "next_gc="} { + if !strings.Contains(out, key) { + t.Errorf("mem line lacks %s:\n%s", key, out) + } + } +} + +func TestIsLoopbackAddr(t *testing.T) { + for addr, want := range map[string]bool{ + "127.0.0.1:6060": true, + "localhost:6060": true, + "[::1]:6060": true, + "127.9.9.9:1": true, + "0.0.0.0:6060": false, + ":6060": false, + "10.0.0.1:6060": false, + "127.0.0.1": false, + } { + if got := IsLoopbackAddr(addr); got != want { + t.Errorf("IsLoopbackAddr(%q) = %v, want %v", addr, got, want) + } + } +} + +func TestStartPprofRejectsNonLoopback(t *testing.T) { + if _, _, err := StartPprof("0.0.0.0:0", nil); !errors.Is(err, ErrNotLoopback) { + t.Fatalf("err = %v, want ErrNotLoopback", err) + } + if _, stop, err := StartPprof("", nil); err != nil || stop == nil { + t.Fatalf("empty addr: err=%v stop-nil=%v", err, stop == nil) + } +} + +func TestStartPprofServesIndexAndRuntime(t *testing.T) { + bound, stop, err := StartPprof("127.0.0.1:0", nil) + if err != nil { + t.Fatal(err) + } + defer stop() + for _, path := range []string{"/debug/pprof/", "/debug/pprof/goroutine?debug=1", "/debug/runtime"} { + resp, err := http.Get("http://" + bound + path) + if err != nil { + t.Fatalf("GET %s: %v", path, err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s: status %d", path, resp.StatusCode) + } + if path == "/debug/runtime" && !strings.Contains(string(body), "goroutines=") { + t.Fatalf("/debug/runtime body: %s", body) + } + } +} + +func TestCgroupMemoryLimit(t *testing.T) { + files := func(m map[string]string) func(string) ([]byte, error) { + return func(p string) ([]byte, error) { + if v, ok := m[p]; ok { + return []byte(v), nil + } + return nil, errors.New("missing") + } + } + cases := []struct { + name string + files map[string]string + want int64 + ok bool + }{ + {"v2 limit", map[string]string{cgroupV2MemoryMax: "42949672960\n"}, 42949672960, true}, + {"v2 max", map[string]string{cgroupV2MemoryMax: "max\n"}, 0, false}, + {"v1 limit", map[string]string{cgroupV1MemoryLimit: "25769803776\n"}, 25769803776, true}, + {"v1 unlimited", map[string]string{cgroupV1MemoryLimit: "9223372036854771712\n"}, 0, false}, + {"none", map[string]string{}, 0, false}, + {"garbage", map[string]string{cgroupV2MemoryMax: "lots\n"}, 0, false}, + } + for _, tc := range cases { + got, ok := CgroupMemoryLimit(files(tc.files)) + if got != tc.want || ok != tc.ok { + t.Errorf("%s: got (%d,%v) want (%d,%v)", tc.name, got, ok, tc.want, tc.ok) + } + } +} + +func TestApplyMemoryLimit(t *testing.T) { + prev := debug.SetMemoryLimit(-1) + t.Cleanup(func() { debug.SetMemoryLimit(prev) }) + + const cg = int64(40 << 30) + readV2 := func(p string) ([]byte, error) { + if p == cgroupV2MemoryMax { + return []byte("42949672960"), nil + } + return nil, errors.New("missing") + } + out := ApplyMemoryLimit(func(string) string { return "" }, readV2) + want := DeriveMemoryLimit(cg) + if out.Source != "cgroup" || out.Limit != want || debug.SetMemoryLimit(-1) != want { + t.Fatalf("cgroup path: %+v (runtime limit %d, want %d)", out, debug.SetMemoryLimit(-1), want) + } + if want >= cg || want < cg*80/100 { + t.Fatalf("derived limit %d is not ~%d%% of %d", want, MemoryLimitPercent, cg) + } + + debug.SetMemoryLimit(prev) + out = ApplyMemoryLimit(func(string) string { return "16GiB" }, readV2) + if out.Source != "env" || debug.SetMemoryLimit(-1) != prev { + t.Fatalf("env path must not touch the runtime limit: %+v", out) + } + + out = ApplyMemoryLimit(func(string) string { return "" }, func(string) ([]byte, error) { return nil, errors.New("missing") }) + if out.Source != "none" || debug.SetMemoryLimit(-1) != prev { + t.Fatalf("no-cgroup path must not touch the runtime limit: %+v", out) + } +} From 45259be8b493aad86b8c7dc5cf532b8ff23b2481 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 16:13:05 -0400 Subject: [PATCH 2/6] runtime: satisfy CI ratchets for pprof/memstats - health body test decodes the new runtime object - settings parity allowlist: debug.pprof_addr is an operator knob - dashboard import ratchet: allow pkg/runtimeobs - agent runtime golden: Config gained the debug block - runtimeobs logger test waits on the record instead of sleeping - api-reference citations re-pinned after the server.go import shift Signed-off-by: Andy Anderson --- .../testutil/dashboard_import_allowlist.txt | 1 + .../config/testdata/agent_runtime_golden.json | 1 + src/pkg/dashboard/server.go | 2 +- src/pkg/dashboard/server_test.go | 9 +++++- .../dashboard/settings_parity_allowlist.txt | 1 + src/pkg/runtimeobs/runtimeobs_test.go | 29 ++++++++++++++----- 6 files changed, 34 insertions(+), 9 deletions(-) diff --git a/src/internal/testutil/dashboard_import_allowlist.txt b/src/internal/testutil/dashboard_import_allowlist.txt index bd257e6647..ac334ce4b8 100644 --- a/src/internal/testutil/dashboard_import_allowlist.txt +++ b/src/internal/testutil/dashboard_import_allowlist.txt @@ -54,6 +54,7 @@ github.com/hivecommons/hive/pkg/resolve github.com/hivecommons/hive/pkg/retro github.com/hivecommons/hive/pkg/review github.com/hivecommons/hive/pkg/rotation +github.com/hivecommons/hive/pkg/runtimeobs github.com/hivecommons/hive/pkg/scheduler github.com/hivecommons/hive/pkg/spokebackup github.com/hivecommons/hive/pkg/stageengine diff --git a/src/pkg/config/testdata/agent_runtime_golden.json b/src/pkg/config/testdata/agent_runtime_golden.json index 09dd9530d7..3d31e6a2aa 100644 --- a/src/pkg/config/testdata/agent_runtime_golden.json +++ b/src/pkg/config/testdata/agent_runtime_golden.json @@ -615,6 +615,7 @@ }, "otel": {}, "Tracing": {}, + "debug": {}, "tool_approval": { "enabled": true, "rules": [ diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index d6a6a23008..5d26805530 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -8,7 +8,6 @@ import ( "embed" "encoding/json" "fmt" - "github.com/hivecommons/hive/pkg/runtimeobs" "html" "io" "io/fs" @@ -35,6 +34,7 @@ import ( spoke "github.com/hivecommons/hive/pkg/hub/spoke" "github.com/hivecommons/hive/pkg/planning" "github.com/hivecommons/hive/pkg/review/autoscale" + "github.com/hivecommons/hive/pkg/runtimeobs" "github.com/hivecommons/hive/pkg/stageengine" "github.com/hivecommons/hive/pkg/stagelabels" "github.com/hivecommons/hive/pkg/timeline" diff --git a/src/pkg/dashboard/server_test.go b/src/pkg/dashboard/server_test.go index afd2ca0d11..522f7bfa06 100644 --- a/src/pkg/dashboard/server_test.go +++ b/src/pkg/dashboard/server_test.go @@ -117,13 +117,20 @@ func TestHandleHealth_Body(t *testing.T) { rec := httptest.NewRecorder() s.handleHealth(rec, httptest.NewRequest(http.MethodGet, "/api/health", nil)) - var payload map[string]string + var payload map[string]any if err := json.NewDecoder(rec.Body).Decode(&payload); err != nil { t.Fatalf("decode error: %v", err) } if payload["status"] != "ok" { t.Errorf(`expected {"status":"ok"}, got %v`, payload) } + rt, ok := payload["runtime"].(map[string]any) + if !ok { + t.Fatalf("expected runtime object in health body, got %v", payload["runtime"]) + } + if g, _ := rt["goroutines"].(float64); g < 1 { + t.Errorf("runtime.goroutines = %v, want >= 1", rt["goroutines"]) + } } func TestHandleStatus_NilStatus_StatusCode(t *testing.T) { diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 06bd0ed9b4..be7e8cce68 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -160,6 +160,7 @@ compliance.frameworks[] # TODO #11854 contribute.token_idle_max # TODO #11854 contribute.token_max_age # TODO #11854 convergence.mutation.max_writers_per_target # boundary capacity tuning; yaml-only (#14645) +debug.pprof_addr # loopback-only pprof bind address, operator/debug knob, not a Settings control (#14681) deployment.runtime # set via HIVE_DEPLOYMENT_RUNTIME env var or auto-detected, not a Settings control (#14072) duplicate_sweep.bot_authors[] # TODO #11854 evidence.complexity.enabled # TODO #11854 diff --git a/src/pkg/runtimeobs/runtimeobs_test.go b/src/pkg/runtimeobs/runtimeobs_test.go index a8e21f5eca..1b429f762f 100644 --- a/src/pkg/runtimeobs/runtimeobs_test.go +++ b/src/pkg/runtimeobs/runtimeobs_test.go @@ -35,21 +35,23 @@ func TestTakeIsPopulated(t *testing.T) { func TestStartLoggerEmitsPerTick(t *testing.T) { var buf bytes.Buffer - logger := slog.New(slog.NewTextHandler(&buf, nil)) + written := make(chan struct{}, 8) + logger := slog.New(&signallingHandler{Handler: slog.NewTextHandler(&buf, nil), done: written}) tick := make(chan time.Time) ctx, cancel := context.WithCancel(context.Background()) defer cancel() StartLogger(ctx, logger, tick) for i := 0; i < 3; i++ { tick <- time.Time{} + // Each tick is handled synchronously; wait for its record before the + // next so the buffer is complete when the goroutine is stopped. + select { + case <-written: + case <-time.After(5 * time.Second): + t.Fatalf("tick %d: no log record written", i) + } } cancel() - // The send above only returns once the goroutine received, so three - // records have been written (the handler writes synchronously). - deadline := time.Now().Add(2 * time.Second) - for strings.Count(buf.String(), "mem: runtime snapshot") < 3 && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } out := buf.String() if n := strings.Count(out, "mem: runtime snapshot"); n != 3 { t.Fatalf("got %d mem lines, want 3:\n%s", n, out) @@ -170,3 +172,16 @@ func TestApplyMemoryLimit(t *testing.T) { t.Fatalf("no-cgroup path must not touch the runtime limit: %+v", out) } } + +// signallingHandler sends on done after every record so tests can wait on the +// observable write instead of sleeping. +type signallingHandler struct { + slog.Handler + done chan<- struct{} +} + +func (h *signallingHandler) Handle(ctx context.Context, r slog.Record) error { + err := h.Handler.Handle(ctx, r) + h.done <- struct{}{} + return err +} From d9a6edf3796f2c158b278bab48f7bc378503c69f Mon Sep 17 00:00:00 2001 From: clubanderson <407614+clubanderson@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:14:36 +0000 Subject: [PATCH 3/6] changelog: fragment for #14682 Generated by changelog-fragment-guard from the PR title (#5675). Edit the wording freely, or replace this file with the no-changelog label if the change is not user-visible. Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- changelog.d/changed-14682.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/changed-14682.md diff --git a/changelog.d/changed-14682.md b/changelog.d/changed-14682.md new file mode 100644 index 0000000000..3e3e311018 --- /dev/null +++ b/changelog.d/changed-14682.md @@ -0,0 +1 @@ +- runtime: loopback pprof, periodic memstats and cgroup-aware GOMEMLIMIT (#14682) From 4481844bd9f6ad678459fec0352a8aa3c0034e05 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 16:24:49 -0400 Subject: [PATCH 4/6] runtimeobs: cover the default logger, disabled/occupied pprof and no-/proc paths Signed-off-by: Andy Anderson --- src/pkg/runtimeobs/runtimeobs.go | 7 +-- src/pkg/runtimeobs/runtimeobs_test.go | 64 +++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/pkg/runtimeobs/runtimeobs.go b/src/pkg/runtimeobs/runtimeobs.go index d3e8b0550b..a9e62ff927 100644 --- a/src/pkg/runtimeobs/runtimeobs.go +++ b/src/pkg/runtimeobs/runtimeobs.go @@ -50,11 +50,12 @@ const ( // pprofReadHeaderTimeout bounds a slow client on the debug listener; the // handlers themselves stream for as long as the profile asks (seconds=). pprofReadHeaderTimeout = 10 * time.Second - - // procSelfStatus is where the OS thread count comes from on Linux. - procSelfStatus = "/proc/self/status" ) +// procSelfStatus is where the OS thread count comes from on Linux; a +// variable so tests can point it at a missing file. +var procSelfStatus = "/proc/self/status" + // Snapshot is one reading of the process-level runtime counters. Every field // comes from runtime/metrics (no stop-the-world) except Threads, which is // read from /proc on Linux and is 0 where that is unavailable. diff --git a/src/pkg/runtimeobs/runtimeobs_test.go b/src/pkg/runtimeobs/runtimeobs_test.go index 1b429f762f..c30e42ae1d 100644 --- a/src/pkg/runtimeobs/runtimeobs_test.go +++ b/src/pkg/runtimeobs/runtimeobs_test.go @@ -6,7 +6,10 @@ import ( "errors" "io" "log/slog" + "net" "net/http" + "path/filepath" + "runtime" "runtime/debug" "strings" "testing" @@ -185,3 +188,64 @@ func (h *signallingHandler) Handle(ctx context.Context, r slog.Record) error { h.done <- struct{}{} return err } + +func TestStartLoggerDefaultsAndStopsOnCancel(t *testing.T) { + before := runtime.NumGoroutine() + ctx, cancel := context.WithCancel(context.Background()) + StartLogger(ctx, nil, nil) + cancel() + deadline := time.After(5 * time.Second) + for runtime.NumGoroutine() > before { + select { + case <-deadline: + t.Fatalf("logger goroutine did not stop: %d goroutines, baseline %d", runtime.NumGoroutine(), before) + default: + runtime.Gosched() + } + } +} + +func TestStartPprofEmptyAddrIsDisabled(t *testing.T) { + bound, stop, err := StartPprof("", nil) + if err != nil || bound != "" || stop == nil { + t.Fatalf("StartPprof(\"\") = %q, %v, stop nil=%v", bound, err, stop == nil) + } + stop() +} + +func TestStartPprofReportsListenError(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Skip("no loopback listener:", err) + } + defer ln.Close() + if _, _, err := StartPprof(ln.Addr().String(), nil); err == nil { + t.Fatal("StartPprof on an occupied port must fail") + } +} + +func TestThreadCountWithoutProcStatus(t *testing.T) { + saved := procSelfStatus + procSelfStatus = filepath.Join(t.TempDir(), "missing") + t.Cleanup(func() { procSelfStatus = saved }) + if got := threadCount(); got != 0 { + t.Fatalf("threadCount without /proc = %d, want 0", got) + } + if s := Take(); s.Threads != 0 { + t.Fatalf("Take().Threads without /proc = %d, want 0", s.Threads) + } +} + +func TestApplyMemoryLimitUsesProcessDefaults(t *testing.T) { + saved := debug.SetMemoryLimit(-1) + t.Cleanup(func() { debug.SetMemoryLimit(saved) }) + out := ApplyMemoryLimit(nil, nil) + switch out.Source { + case "env", "none", "cgroup": + default: + t.Fatalf("unexpected source %q: %+v", out.Source, out) + } + if out.Limit <= 0 { + t.Fatalf("effective limit must be positive: %+v", out) + } +} From 58d61965eaa0b04ca41038ec43ee3f308b8bcae3 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 16:25:35 -0400 Subject: [PATCH 5/6] changelog: single bulleted fragment for the runtime diagnostics Signed-off-by: Andy Anderson --- changelog.d/added-runtime-pprof-memstats.md | 2 +- changelog.d/changed-14682.md | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 changelog.d/changed-14682.md diff --git a/changelog.d/added-runtime-pprof-memstats.md b/changelog.d/added-runtime-pprof-memstats.md index 1e25c322fc..f31d552a10 100644 --- a/changelog.d/added-runtime-pprof-memstats.md +++ b/changelog.d/added-runtime-pprof-memstats.md @@ -1 +1 @@ -Loopback-only `net/http/pprof` listener (`debug.pprof_addr`, default `127.0.0.1:6060`, `off` to disable), a `mem: runtime snapshot` INFO line every 60 s (heap in-use/live/objects, next GC, goroutines, OS threads, GOMEMLIMIT), the same counters under `runtime` in `GET /api/health`, and a cgroup-aware GOMEMLIMIT default (85% of the container memory limit when the env var is unset). Refs #14681. +- Loopback-only `net/http/pprof` listener (`debug.pprof_addr`, default `127.0.0.1:6060`, `off` to disable), a `mem: runtime snapshot` INFO line every 60 s (heap in-use/live/objects, next GC, goroutines, OS threads, GOMEMLIMIT), the same counters under `runtime` in `GET /api/health`, and a cgroup-aware GOMEMLIMIT default (85% of the container memory limit when the env var is unset) ([#14681](https://github.com/hivecommons/hive/issues/14681)). diff --git a/changelog.d/changed-14682.md b/changelog.d/changed-14682.md deleted file mode 100644 index 3e3e311018..0000000000 --- a/changelog.d/changed-14682.md +++ /dev/null @@ -1 +0,0 @@ -- runtime: loopback pprof, periodic memstats and cgroup-aware GOMEMLIMIT (#14682) From 53a9fc6e80ee8d0802133009a22f409d32e9373f Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 27 Oct 2026 21:07:49 -0400 Subject: [PATCH 6/6] docs: re-pin api-reference citations after the server.go import shift Signed-off-by: Andy Anderson --- src/docs/api-reference.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 08c74f9c68..ce798b8162 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -13,17 +13,17 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)) | `pkg/dashboard/api.go:42` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:43` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:64` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1505` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1506` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1507` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1513` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N · $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376) | `pkg/dashboard/server.go:1518` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1519` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1520` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1521` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1522` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1523` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1524` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1506` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1507` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1508` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1514` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced}` is the governor strip's `running N · $X this hour` figure; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376) | `pkg/dashboard/server.go:1519` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1520` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1521` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1522` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1523` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1524` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1525` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | @@ -66,7 +66,7 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:44` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1585` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1586` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:77` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:78` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:79` | @@ -94,9 +94,9 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1525` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1526` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1527` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1526` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1527` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1528` | ## Configuration @@ -593,7 +593,7 @@ always resolved server-side from the validated token. | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:275` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:276` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:277` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1532` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1533` | ## Hub SaaS