Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-memory-growth-dashboard-stage-tracker.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Stopped the unbounded memory, thread and connection growth in the hive process that OOM-killed hosted hives ([#14681](https://github.com/hivecommons/hive/issues/14681)): `/api/stage-tracker` held its cache mutex across the whole tracker build (one review-evidence lookup per open PR, each a glob of the repo's evidence directory on NFS β€” tens of seconds), and every webhook delivery and every status publish invalidated that cache through the same mutex, so webhook handlers, the governor's publish and every dashboard poll queued behind back-to-back rebuilds that never drained. Invalidation is now a flag flip that never waits, at most one rebuild runs at a time outside the lock, concurrent polls get the previous body as `source: stale`, a poll whose client went away stops waiting, and the evidence-directory sibling listing is memoised per repo directory for 60 s instead of re-read per PR.
34 changes: 17 additions & 17 deletions src/docs/api-reference.md

Large diffs are not rendered by default.

9 changes: 8 additions & 1 deletion src/docs/operator-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -1222,7 +1222,14 @@ v5 and v6 spokes) tracking one repo all stay on webhook mode.
- Every status publish (cadence or wake) invalidates the `/api/stage-tracker`
cache and sends a `stage-tracker` event on `/api/events`, so the dashboard's
activity monitor refetches at once instead of waiting out
`dashboard.stage_tracker.cache_ttl_s` plus its poll.
`dashboard.stage_tracker.cache_ttl_s` plus its poll. Invalidation only
marks the cached body stale: the publish, and every webhook below, returns
at once even while a tracker rebuild is running. The next poll starts
**one** rebuild outside any lock; polls that land during it get the
previous body with `source: stale` and `max-age=0` rather than queueing,
so a slow build (one sentinel evidence read per open PR, on NFS in hosted
hives) can never back up webhook handlers, the governor's publish, or an
unbounded line of dashboard requests (hivecommons/hive#14681).
- **The activity monitor row moves without waiting for a cycle.** A delivery
that names a PR or issue this hive tracks (`pull_request*`,
`check_suite`/`check_run`, `status`, `push` to a PR branch, `issue_comment`,
Expand Down
9 changes: 9 additions & 0 deletions src/pkg/dashboard/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,15 @@ type Server struct {
// the status snapshot, served until its TTL lapses (stage_tracker.go).
stageTrackerCache stageTrackerCache
stageTrackerCacheMu sync.Mutex
// stageTrackerBuild is the one rebuild in flight, nil when none; it is
// guarded by stageTrackerCacheMu, which is never held across the build.
stageTrackerBuild *stageTrackerBuild
// stageTrackerGen counts invalidations so a build can tell whether the
// status it rendered was superseded while it ran.
stageTrackerGen uint64
// stageTrackerBuildFn replaces buildStageTracker in tests that need a
// build they can hold open; nil in production.
stageTrackerBuildFn func(status *StatusPayload, store *stagelabels.Store, now time.Time) StageTrackerResponse
// evidenceHB is the background-refreshed evidence aggregate the hub
// heartbeat reads (evidence_heartbeat.go); zero value is ready to use.
evidenceHB evidenceHeartbeatCache
Expand Down
149 changes: 127 additions & 22 deletions src/pkg/dashboard/stage_tracker.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
package dashboard

import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
Expand Down Expand Up @@ -1545,8 +1546,29 @@
body []byte
etag string
generatedAt time.Time
// stale is set by invalidateStageTrackerCache: the body is still served
// while the next request rebuilds, so an invalidation never blocks and a
// reader never waits behind a build it did not start.
stale bool
}

// stageTrackerBuild is the single in-flight /api/stage-tracker rebuild. done
// is closed when the cache has been updated (or the build failed).
type stageTrackerBuild struct {
done chan struct{}
}

// stageTrackerSource values for the X-Hive-Stage-Tracker-Source header and
// the payload's source field.
const (
stageTrackerSourceFreshName = "fresh"
stageTrackerSourceCacheName = "cache"
// stageTrackerSourceStaleName marks a body served from the cache while a
// rebuild is in flight or after an invalidation: still the last complete
// tracker, just older than the status it will be rebuilt from.
stageTrackerSourceStaleName = "stale"
)

func (s *Server) stageTrackerCacheTTL() time.Duration {
if s != nil && s.deps != nil && s.deps.Config != nil {
return s.deps.Config.StageTrackerCacheTTL()
Expand All @@ -1555,22 +1577,94 @@
}

// stageTrackerPayload returns the cached body when it is younger than the
// TTL, else builds, encodes and caches a new one. fresh reports a rebuild.
// TTL and not invalidated, else builds, encodes and caches a new one. fresh
// reports a rebuild by this call.
//
// The build runs OUTSIDE stageTrackerCacheMu and at most one runs at a time:
// the mutex only guards the cached fields, so invalidateStageTrackerCache
// (every webhook, every status publish) returns at once instead of queueing
// behind a build that walks the evidence tree on NFS. Requests that arrive
// while a build is in flight are served the previous body as "stale"; only a
// request with no body to fall back on waits, and only until the in-flight
// build finishes or its own context ends. Before this, the mutex was held
// across the build, so every webhook handler, the governor's publish and
// every tracker poll queued behind it in FIFO order β€” and because each
// publish and webhook re-invalidated the cache, the queue never drained
// (hivecommons/hive#14681: unbounded blocked handlers, connections and
// threads).
func (s *Server) stageTrackerPayload(status *StatusPayload, now time.Time) (body []byte, etag string, generatedAt time.Time, fresh bool) {
var source string
body, etag, generatedAt, source = s.stageTrackerPayloadSource(context.Background(), status, now)
return body, etag, generatedAt, source == stageTrackerSourceFreshName
}

// stageTrackerPayloadSource is stageTrackerPayload reporting where the body
// came from (stageTrackerSource*Name; "" with a nil body when the build
// failed or ctx ended while waiting on an in-flight build).
func (s *Server) stageTrackerPayloadSource(ctx context.Context, status *StatusPayload, now time.Time) (body []byte, etag string, generatedAt time.Time, source string) {
ttl := s.stageTrackerCacheTTL()
s.stageTrackerCacheMu.Lock()
defer s.stageTrackerCacheMu.Unlock()
s.stageTrackerServed = true
c := &s.stageTrackerCache
if len(c.body) > 0 && now.Sub(c.generatedAt) < ttl {
return c.body, c.etag, c.generatedAt, false
c := s.stageTrackerCache
if len(c.body) > 0 && !c.stale && now.Sub(c.generatedAt) < ttl {
s.stageTrackerCacheMu.Unlock()
return c.body, c.etag, c.generatedAt, stageTrackerSourceCacheName
}
if inflight := s.stageTrackerBuild; inflight != nil {
s.stageTrackerCacheMu.Unlock()
if len(c.body) > 0 {
return c.body, c.etag, c.generatedAt, stageTrackerSourceStaleName
}
select {
case <-inflight.done:
case <-ctx.Done():
return nil, "", now, ""
}
s.stageTrackerCacheMu.Lock()
c = s.stageTrackerCache
s.stageTrackerCacheMu.Unlock()
if len(c.body) == 0 {
return nil, "", now, ""
}
return c.body, c.etag, c.generatedAt, stageTrackerSourceCacheName
}
inflight := &stageTrackerBuild{done: make(chan struct{})}
s.stageTrackerBuild = inflight
gen := s.stageTrackerGen
s.stageTrackerCacheMu.Unlock()

built, ok := s.encodeStageTracker(status, now, ttl)

s.stageTrackerCacheMu.Lock()
if ok {
// An invalidation that landed during the build means newer status
// exists; keep serving this body but let the next request rebuild.
built.stale = s.stageTrackerGen != gen
s.stageTrackerCache = built
}
s.stageTrackerBuild = nil
close(inflight.done)
s.stageTrackerCacheMu.Unlock()
if !ok {
return nil, "", now, ""
}
resp := s.buildStageTracker(status, s.stageStoreOrShared(), now)
return built.body, built.etag, built.generatedAt, stageTrackerSourceFreshName
}

// encodeStageTracker builds and encodes the tracker for status. It holds no
// server lock: this is the slow part (one evidence read per PR, on NFS in
// hosted hives).
func (s *Server) encodeStageTracker(status *StatusPayload, now time.Time, ttl time.Duration) (stageTrackerCache, bool) {
build := s.buildStageTracker
if s.stageTrackerBuildFn != nil {
build = s.stageTrackerBuildFn
}
resp := build(status, s.stageStoreOrShared(), now)
resp.CacheTTLSeconds = int64(ttl / time.Second)
// The encoded body carries source=cache: that is what every later hit
// serves verbatim, and what the ETag validates. The building request
// alone rewrites the marker to fresh on its way out (handleStageTracker).
resp.Source = "cache"
resp.Source = stageTrackerSourceCacheName
// The ETag hashes the body with generated_at zeroed: the timestamp moves
// on every rebuild, so including it would turn an unchanged tracker into
// a 200 each time (the client revalidates with this ETag and gets a 304).
Expand All @@ -1579,26 +1673,29 @@
hashResp.GeneratedAt = time.Time{}
if err := json.NewEncoder(&hashBuf).Encode(hashResp); err != nil {
slogWarnOverviewEncode(err)
return nil, "", now, true
return stageTrackerCache{}, false
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(resp); err != nil {
slogWarnOverviewEncode(err)
return nil, "", now, true
return stageTrackerCache{}, false
}
c.body, c.etag, c.generatedAt = buf.Bytes(), webstatic.ETagFor(hashBuf.Bytes()), now
return c.body, c.etag, c.generatedAt, true
return stageTrackerCache{body: buf.Bytes(), etag: webstatic.ETagFor(hashBuf.Bytes()), generatedAt: now}, true
}

// invalidateStageTrackerCache forgets the memoised payload. UpdateStatusIfFresh
// calls it on every status publish so tracker freshness follows the eval
// cycle rather than the cache TTL alone. It reports whether the tracker has
// ever been served: only then is an SSE nudge worth sending, which also keeps
// the /api/events stream free of extra frames for clients that never asked
// for the tracker.
// invalidateStageTrackerCache marks the memoised payload stale.
// UpdateStatusIfFresh calls it on every status publish, and every webhook
// re-read that changed a row calls it too, so tracker freshness follows the
// eval cycle and the webhook stream rather than the cache TTL alone. It only
// flips a flag under the cache mutex β€” it never waits for a build in
// progress β€” so callers on the webhook and publish paths return at once. It
// reports whether the tracker has ever been served: only then is an SSE
// nudge worth sending, which also keeps the /api/events stream free of extra
// frames for clients that never asked for the tracker.
func (s *Server) invalidateStageTrackerCache() (served bool) {
s.stageTrackerCacheMu.Lock()
s.stageTrackerCache = stageTrackerCache{}
s.stageTrackerCache.stale = true
s.stageTrackerGen++
served = s.stageTrackerServed
s.stageTrackerCacheMu.Unlock()
return served
Expand All @@ -1613,6 +1710,7 @@

var stageTrackerSourceCache = []byte(`"source":"cache"`)
var stageTrackerSourceFresh = []byte(`"source":"fresh"`)
var stageTrackerSourceStale = []byte(`"source":"stale"`)

// handleStageTracker serves GET /api/stage-tracker from the memoised payload.
func (s *Server) handleStageTracker(w http.ResponseWriter, r *http.Request) {
Expand All @@ -1625,18 +1723,22 @@
return
}
now := time.Now().UTC()
body, etag, generatedAt, fresh := s.stageTrackerPayload(status, now)
body, etag, generatedAt, source := s.stageTrackerPayloadSource(r.Context(), status, now)
if body == nil {
if r.Context().Err() != nil {
jsonError(w, "stage tracker build still in progress", http.StatusServiceUnavailable)
return
}
jsonError(w, "stage tracker unavailable", http.StatusInternalServerError)
return
}
remaining := s.stageTrackerCacheTTL() - now.Sub(generatedAt)
if remaining < 0 {
if remaining < 0 || source == stageTrackerSourceStaleName {
remaining = 0
}
w.Header().Set("ETag", etag)
w.Header().Set("Cache-Control", "private, max-age="+strconv.FormatInt(int64(remaining/time.Second), 10))
w.Header().Set("X-Hive-Stage-Tracker-Source", map[bool]string{true: "fresh", false: "cache"}[fresh])
w.Header().Set("X-Hive-Stage-Tracker-Source", source)
w.Header().Set("X-Hive-Stage-Tracker-Generated-At", generatedAt.UTC().Format(time.RFC3339Nano))
w.Header().Set("Last-Modified", generatedAt.UTC().Format(http.TimeFormat))
if match := r.Header.Get("If-None-Match"); match != "" && strings.Contains(match, etag) {
Expand All @@ -1645,8 +1747,11 @@
return
}
github.RecordLocalServe(stageTrackerRESTCaller, r.URL.Path, http.StatusOK)
if fresh {
switch source {
case stageTrackerSourceFreshName:
body = bytes.Replace(body, stageTrackerSourceCache, stageTrackerSourceFresh, 1)
case stageTrackerSourceStaleName:
body = bytes.Replace(body, stageTrackerSourceCache, stageTrackerSourceStale, 1)
}
if _, err := w.Write(body); err != nil {
slogWarnOverviewEncode(err)
Expand Down
Loading
Loading