Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-16010-hub-digest-forget-race.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- A heartbeat racing a hive delete can no longer re-insert an activity digest after the hive was forgotten; the digest is now recorded and purged under the same registry lock ([#16010](https://github.com/hivecommons/hive/issues/16010)).
42 changes: 42 additions & 0 deletions src/pkg/hub/activity_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"runtime"
"strings"
"testing"
"time"
Expand Down Expand Up @@ -519,3 +520,44 @@ func TestActivityStoreForget(t *testing.T) {
var nilStore *activityStore
nilStore.forget("hive-a")
}

// TestRemoveRegistryEntryForgetsDigestUnderRegistryLock pins the #16010 fix:
// the digest purge runs inside s.mu, the same critical section in which a
// heartbeat records, so a racing beat cannot re-insert a digest after the
// forget. With the activity store held busy, the delete must be observed
// holding s.mu while it waits to forget.
func TestRemoveRegistryEntryForgetsDigestUnderRegistryLock(t *testing.T) {
cleanup := helperSetupTempDirs(t)
defer cleanup()

s := pullOnlyTestServer(t)
s.registry.Hives = []RegistryEntry{{ID: "doomed"}}
s.activity.record("doomed", activityDigest("o/r", 1), activityT0)

s.activity.mu.Lock()
done := make(chan struct{})
go func() {
defer close(done)
s.removeRegistryEntry("doomed", "tester")
}()

held := false
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if s.mu.TryLock() {
s.mu.Unlock()
runtime.Gosched()
continue
}
held = true
break
}
s.activity.mu.Unlock()
<-done
if !held {
t.Fatal("removeRegistryEntry forgot the digest outside s.mu; a racing heartbeat could re-insert it")
}
if got := s.activity.snapshot([]string{"doomed"}); len(got) != 0 {
t.Errorf("digest survived removal: %v", got)
}
}
21 changes: 13 additions & 8 deletions src/pkg/hub/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -2730,11 +2730,12 @@ func (s *HubServer) handleHeartbeat(w http.ResponseWriter, r *http.Request) {
s.registry.Hives = append(s.registry.Hives, entry)
}
s.registry.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
s.mu.Unlock()

// Activity digest (#15951): stored after s.mu is released (leaf lock),
// only once the beat is authenticated and accepted into the registry.
// Activity digest (#15951): stored only once the beat is authenticated and
// accepted into the registry. Recorded while still holding s.mu so a
// concurrent delete (which forgets under s.mu) cannot leave a stale digest
// behind (#16010). activityStore.mu is a leaf lock, so taking it here is safe.
activityAck := s.activity.record(payload.HiveID, payload.Activity, time.Now().UTC())
s.mu.Unlock()

// Timeline: record whatever changed on this beat. Nothing is written when
// nothing changed, which is the common case — at ~2 min per beat and 42
Expand Down Expand Up @@ -4280,8 +4281,6 @@ func (s *HubServer) removeRegistryEntry(id, by string) {
// with it. An uncollectible hive is by definition never armed, so this is
// the ONLY path that can ever retire its entry (see forgetUncollectibleUpgrade).
s.forgetUncollectibleUpgrade(id)
// Memory-only activity digests must not outlive the hive (#16002).
s.activity.forget(id)
s.mu.Lock()
removed := false
for i, h := range s.registry.Hives {
Expand All @@ -4291,6 +4290,10 @@ func (s *HubServer) removeRegistryEntry(id, by string) {
break
}
}
// Memory-only activity digests must not outlive the hive (#16002). Forgotten
// under s.mu so a heartbeat (which records under s.mu too) cannot slip a
// digest in between the removal and the purge (#16010).
s.activity.forget(id)
s.mu.Unlock()
if !removed {
return
Expand Down Expand Up @@ -4346,8 +4349,6 @@ func (s *HubServer) handleRegistryDelete(w http.ResponseWriter, r *http.Request)
}
id := r.PathValue("id")
s.forgetUncollectibleUpgrade(id)
// Memory-only activity digests must not outlive the hive (#16002).
s.activity.forget(id)
s.mu.Lock()
removed := false
for i, h := range s.registry.Hives {
Expand All @@ -4357,6 +4358,10 @@ func (s *HubServer) handleRegistryDelete(w http.ResponseWriter, r *http.Request)
break
}
}
// Memory-only activity digests must not outlive the hive (#16002). Forgotten
// under s.mu so a heartbeat (which records under s.mu too) cannot slip a
// digest in between the removal and the purge (#16010).
s.activity.forget(id)
s.mu.Unlock()
if removed {
// Synchronous flush for the same durability reason as removeRegistryEntry.
Expand Down
Loading