diff --git a/changelog.d/fixed-16010-hub-digest-forget-race.md b/changelog.d/fixed-16010-hub-digest-forget-race.md new file mode 100644 index 0000000000..37524788f3 --- /dev/null +++ b/changelog.d/fixed-16010-hub-digest-forget-race.md @@ -0,0 +1 @@ +- A heartbeat racing a hive delete can no longer re-insert an activity digest after the hive was forgotten; the digest is now recorded and purged under the same registry lock ([#16010](https://github.com/hivecommons/hive/issues/16010)). diff --git a/src/pkg/hub/activity_test.go b/src/pkg/hub/activity_test.go index d7de543655..e7132db121 100644 --- a/src/pkg/hub/activity_test.go +++ b/src/pkg/hub/activity_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "runtime" "strings" "testing" "time" @@ -519,3 +520,44 @@ func TestActivityStoreForget(t *testing.T) { var nilStore *activityStore nilStore.forget("hive-a") } + +// TestRemoveRegistryEntryForgetsDigestUnderRegistryLock pins the #16010 fix: +// the digest purge runs inside s.mu, the same critical section in which a +// heartbeat records, so a racing beat cannot re-insert a digest after the +// forget. With the activity store held busy, the delete must be observed +// holding s.mu while it waits to forget. +func TestRemoveRegistryEntryForgetsDigestUnderRegistryLock(t *testing.T) { + cleanup := helperSetupTempDirs(t) + defer cleanup() + + s := pullOnlyTestServer(t) + s.registry.Hives = []RegistryEntry{{ID: "doomed"}} + s.activity.record("doomed", activityDigest("o/r", 1), activityT0) + + s.activity.mu.Lock() + done := make(chan struct{}) + go func() { + defer close(done) + s.removeRegistryEntry("doomed", "tester") + }() + + held := false + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if s.mu.TryLock() { + s.mu.Unlock() + runtime.Gosched() + continue + } + held = true + break + } + s.activity.mu.Unlock() + <-done + if !held { + t.Fatal("removeRegistryEntry forgot the digest outside s.mu; a racing heartbeat could re-insert it") + } + if got := s.activity.snapshot([]string{"doomed"}); len(got) != 0 { + t.Errorf("digest survived removal: %v", got) + } +} diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index 0d2686b9ed..df7fd2f9c4 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -2730,11 +2730,12 @@ func (s *HubServer) handleHeartbeat(w http.ResponseWriter, r *http.Request) { s.registry.Hives = append(s.registry.Hives, entry) } s.registry.UpdatedAt = time.Now().UTC().Format(time.RFC3339) - s.mu.Unlock() - - // Activity digest (#15951): stored after s.mu is released (leaf lock), - // only once the beat is authenticated and accepted into the registry. + // Activity digest (#15951): stored only once the beat is authenticated and + // accepted into the registry. Recorded while still holding s.mu so a + // concurrent delete (which forgets under s.mu) cannot leave a stale digest + // behind (#16010). activityStore.mu is a leaf lock, so taking it here is safe. activityAck := s.activity.record(payload.HiveID, payload.Activity, time.Now().UTC()) + s.mu.Unlock() // Timeline: record whatever changed on this beat. Nothing is written when // nothing changed, which is the common case — at ~2 min per beat and 42 @@ -4280,8 +4281,6 @@ func (s *HubServer) removeRegistryEntry(id, by string) { // with it. An uncollectible hive is by definition never armed, so this is // the ONLY path that can ever retire its entry (see forgetUncollectibleUpgrade). s.forgetUncollectibleUpgrade(id) - // Memory-only activity digests must not outlive the hive (#16002). - s.activity.forget(id) s.mu.Lock() removed := false for i, h := range s.registry.Hives { @@ -4291,6 +4290,10 @@ func (s *HubServer) removeRegistryEntry(id, by string) { break } } + // Memory-only activity digests must not outlive the hive (#16002). Forgotten + // under s.mu so a heartbeat (which records under s.mu too) cannot slip a + // digest in between the removal and the purge (#16010). + s.activity.forget(id) s.mu.Unlock() if !removed { return @@ -4346,8 +4349,6 @@ func (s *HubServer) handleRegistryDelete(w http.ResponseWriter, r *http.Request) } id := r.PathValue("id") s.forgetUncollectibleUpgrade(id) - // Memory-only activity digests must not outlive the hive (#16002). - s.activity.forget(id) s.mu.Lock() removed := false for i, h := range s.registry.Hives { @@ -4357,6 +4358,10 @@ func (s *HubServer) handleRegistryDelete(w http.ResponseWriter, r *http.Request) break } } + // Memory-only activity digests must not outlive the hive (#16002). Forgotten + // under s.mu so a heartbeat (which records under s.mu too) cannot slip a + // digest in between the removal and the purge (#16010). + s.activity.forget(id) s.mu.Unlock() if removed { // Synchronous flush for the same durability reason as removeRegistryEntry.