diff --git a/changelog.d/fixed-heartbeat-app-keys-held-bound.md b/changelog.d/fixed-heartbeat-app-keys-held-bound.md new file mode 100644 index 0000000000..d01c87e64d --- /dev/null +++ b/changelog.d/fixed-heartbeat-app-keys-held-bound.md @@ -0,0 +1 @@ +- Hub heartbeat collect no longer times out when the PVC scan behind `app_keys_held` stalls: the scan is bounded to 2s and falls back to the last completed result (#18484). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 71ab7af0e0..c0cc4ad0d9 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -4947,7 +4947,7 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { // Report the fingerprint of every ADDITIONAL per-app-id key already // on the PVC, so the hub delivers the fleet's other App keys once // and then stops re-sending them. - GitHubAppKeysHeld: spoke.TimedSection("app_keys_held", appKeys.HeldFingerprints), + GitHubAppKeysHeld: spoke.TimedSection("app_keys_held", func() map[string]string { return appKeys.HeldFingerprintsWithin(2 * time.Second) }), // Component reach counters (#3993, phase 2a of #3973): per // (component, running commit) span counts aggregated in-process, // exporter or not (D2) — the heartbeat is the only channel that diff --git a/src/pkg/appkey/appkey.go b/src/pkg/appkey/appkey.go index 9a57a80fc0..573737b2d9 100644 --- a/src/pkg/appkey/appkey.go +++ b/src/pkg/appkey/appkey.go @@ -67,6 +67,11 @@ type Resolver struct { fpMu sync.Mutex fpCache map[string]heldFingerprint + + // boundMu guards lastHeld and inflight for HeldFingerprintsWithin. + boundMu sync.Mutex + lastHeld map[string]string + inflight chan struct{} } // heldFingerprint memoizes a key file's fingerprint against the size and mtime @@ -179,6 +184,41 @@ func (r *Resolver) HeldFingerprints() map[string]string { return held } +// HeldFingerprintsWithin is HeldFingerprints with a wall-clock bound, for the +// heartbeat hot path. The scan stats and reads files on the PVC, which can +// stall for many seconds on a slow volume and eat the whole heartbeat collect +// budget (#18484). The scan runs in a single-flight background goroutine; if it +// does not finish within timeout, the last completed result is returned (nil +// before the first completion) and the scan keeps going to refresh the cache +// for the next beat. +func (r *Resolver) HeldFingerprintsWithin(timeout time.Duration) map[string]string { + r.boundMu.Lock() + done := r.inflight + if done == nil { + done = make(chan struct{}) + r.inflight = done + go func() { + held := r.HeldFingerprints() + r.boundMu.Lock() + r.lastHeld = held + r.inflight = nil + r.boundMu.Unlock() + close(done) + }() + } + r.boundMu.Unlock() + + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case <-done: + case <-timer.C: + } + r.boundMu.Lock() + defer r.boundMu.Unlock() + return r.lastHeld +} + // writePerAppIDKey persists a hub-delivered per-app-id key to its PVC file // atomically (temp file in the same dir, then rename) with a restrictive 0600 // mode from creation, so a spoke can never sign with a half-written key. Returns diff --git a/src/pkg/appkey/held_within_unix_test.go b/src/pkg/appkey/held_within_unix_test.go new file mode 100644 index 0000000000..d55d3a0985 --- /dev/null +++ b/src/pkg/appkey/held_within_unix_test.go @@ -0,0 +1,35 @@ +//go:build unix + +package appkey + +import ( + "os" + "path/filepath" + "syscall" + "testing" + "time" +) + +// TestHeldFingerprintsWithinBoundsSlowScan: a scan that stalls (slow PVC) must +// not hold the heartbeat collect past the bound (#18484). +func TestHeldFingerprintsWithinBoundsSlowScan(t *testing.T) { + dir := t.TempDir() + // A FIFO named like a key file blocks the scan's read until opened for write. + fifo := filepath.Join(dir, "gh-app-key-42.pem") + if err := syscall.Mkfifo(fifo, 0o600); err != nil { + t.Skipf("mkfifo unavailable: %v", err) + } + r := &Resolver{DataDir: dir} + start := time.Now() + got := r.HeldFingerprintsWithin(100 * time.Millisecond) + if el := time.Since(start); el > 2*time.Second { + t.Fatalf("HeldFingerprintsWithin blocked %v on a stalled scan", el) + } + if got != nil { + t.Fatalf("no completed scan yet, want nil, got %v", got) + } + // Release the stalled scan so the goroutine exits. + if f, err := os.OpenFile(fifo, os.O_WRONLY, 0); err == nil { + f.Close() + } +}