Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-heartbeat-app-keys-held-bound.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Hub heartbeat collect no longer times out when the PVC scan behind `app_keys_held` stalls: the scan is bounded to 2s and falls back to the last completed result (#18484).
2 changes: 1 addition & 1 deletion src/cmd/hive/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4947,7 +4947,7 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) {
// Report the fingerprint of every ADDITIONAL per-app-id key already
// on the PVC, so the hub delivers the fleet's other App keys once
// and then stops re-sending them.
GitHubAppKeysHeld: spoke.TimedSection("app_keys_held", appKeys.HeldFingerprints),
GitHubAppKeysHeld: spoke.TimedSection("app_keys_held", func() map[string]string { return appKeys.HeldFingerprintsWithin(2 * time.Second) }),
// Component reach counters (#3993, phase 2a of #3973): per
// (component, running commit) span counts aggregated in-process,
// exporter or not (D2) — the heartbeat is the only channel that
Expand Down
40 changes: 40 additions & 0 deletions src/pkg/appkey/appkey.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,11 @@ type Resolver struct {

fpMu sync.Mutex
fpCache map[string]heldFingerprint

// boundMu guards lastHeld and inflight for HeldFingerprintsWithin.
boundMu sync.Mutex
lastHeld map[string]string
inflight chan struct{}
}

// heldFingerprint memoizes a key file's fingerprint against the size and mtime
Expand Down Expand Up @@ -179,6 +184,41 @@ func (r *Resolver) HeldFingerprints() map[string]string {
return held
}

// HeldFingerprintsWithin is HeldFingerprints with a wall-clock bound, for the
// heartbeat hot path. The scan stats and reads files on the PVC, which can
// stall for many seconds on a slow volume and eat the whole heartbeat collect
// budget (#18484). The scan runs in a single-flight background goroutine; if it
// does not finish within timeout, the last completed result is returned (nil
// before the first completion) and the scan keeps going to refresh the cache
// for the next beat.
func (r *Resolver) HeldFingerprintsWithin(timeout time.Duration) map[string]string {
r.boundMu.Lock()
done := r.inflight
if done == nil {
done = make(chan struct{})
r.inflight = done
go func() {
held := r.HeldFingerprints()
r.boundMu.Lock()
r.lastHeld = held
r.inflight = nil
r.boundMu.Unlock()
close(done)
}()
}
r.boundMu.Unlock()

timer := time.NewTimer(timeout)
defer timer.Stop()
select {
case <-done:
case <-timer.C:
}
r.boundMu.Lock()
defer r.boundMu.Unlock()
return r.lastHeld
}

// writePerAppIDKey persists a hub-delivered per-app-id key to its PVC file
// atomically (temp file in the same dir, then rename) with a restrictive 0600
// mode from creation, so a spoke can never sign with a half-written key. Returns
Expand Down
35 changes: 35 additions & 0 deletions src/pkg/appkey/held_within_unix_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
//go:build unix

package appkey

import (
"os"
"path/filepath"
"syscall"
"testing"
"time"
)

// TestHeldFingerprintsWithinBoundsSlowScan: a scan that stalls (slow PVC) must
// not hold the heartbeat collect past the bound (#18484).
func TestHeldFingerprintsWithinBoundsSlowScan(t *testing.T) {
dir := t.TempDir()
// A FIFO named like a key file blocks the scan's read until opened for write.
fifo := filepath.Join(dir, "gh-app-key-42.pem")
if err := syscall.Mkfifo(fifo, 0o600); err != nil {
t.Skipf("mkfifo unavailable: %v", err)
}
r := &Resolver{DataDir: dir}
start := time.Now()
got := r.HeldFingerprintsWithin(100 * time.Millisecond)
if el := time.Since(start); el > 2*time.Second {
t.Fatalf("HeldFingerprintsWithin blocked %v on a stalled scan", el)
}
if got != nil {
t.Fatalf("no completed scan yet, want nil, got %v", got)
}
// Release the stalled scan so the goroutine exits.
if f, err := os.OpenFile(fifo, os.O_WRONLY, 0); err == nil {
f.Close()
}
}
Loading