diff --git a/changelog.d/fixed-15779-reconcile-starvation.md b/changelog.d/fixed-15779-reconcile-starvation.md new file mode 100644 index 0000000000..8a3e81610e --- /dev/null +++ b/changelog.d/fixed-15779-reconcile-starvation.md @@ -0,0 +1 @@ +- Fixed the reconcile (enumerate) pass being starved by wake-driven cached cycles: a wake whose cached snapshot is older than the effective eval interval now enumerates, and the loop runs an owed reconcile pass regardless of which channel woke it, so the snapshot can no longer age for an hour while the ticker tick loses to webhook, verdict and re-kick wakes. The stage tracker's "since" for an issue the blocked sweep just unblocked now restarts at the lift instead of counting from issue creation ([#15779](https://github.com/hivecommons/hive/issues/15779)). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index d5ad531f8e..f51a1ffaca 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -5878,8 +5878,22 @@ func (b *boot) runLoopWith(deps runLoopDeps) { if err := waitForgeStateReady(b.ctx, b.forgeState.Load(), b.logger, forgeStateReadyLogInterval); err != nil { return } + loopNow := deps.now + if loopNow == nil { + loopNow = time.Now + } b.evalCycles++ deps.runEval(b, nil, evalSourceEnumerate) + // lastEnumerateAt is when the last reconcile pass started. Wake-driven + // cycles plan from the cached snapshot, and a busy spoke can be woken + // every few minutes for hours; the reconcile cadence must not depend on + // the ticker tick winning the select against those wakes + // (hivecommons/hive#15779). reconcileOverdue is the one check both the + // wake paths and the loop top consult. + lastEnumerateAt := loopNow() + reconcileOverdue := func(now time.Time) bool { + return lastEvalInterval > 0 && now.Sub(lastEnumerateAt) >= lastEvalInterval + } deps.runRotation(b) if b.wd != nil { b.wd.Tick(b.ctx) @@ -5894,6 +5908,15 @@ func (b *boot) runLoopWith(deps runLoopDeps) { // gates, planning from the stored snapshot — already patched by the // delivery's targeted re-read — so a wake costs no listing. tick := func(source evalCycleSource) { + if now := loopNow(); source == evalSourceCached && reconcileOverdue(now) { + b.logger.Info("cached snapshot older than eval interval; enumerating", + "snapshot_age", now.Sub(lastEnumerateAt).Round(time.Second).String(), + "eval_interval_s", int(lastEvalInterval.Seconds())) + source = evalSourceEnumerate + } + if source == evalSourceEnumerate { + lastEnumerateAt = loopNow() + } b.cfgReloadMu.Lock() restarted := deps.restartCrashed(b.ctx, b.agentMgr) for _, name := range restarted { @@ -6095,12 +6118,18 @@ func (b *boot) runLoopWith(deps runLoopDeps) { "agent", who, "next_kick", next.UTC().Format(time.RFC3339), "idle_for", idleFor.String(), "was_armed_for", wakeArmedFor.UTC().Format(time.RFC3339)) lastWake = now - // Keep this cycle lean: planning and delivery without sweeps. - b.cfgReloadMu.Lock() - b.evalCycles++ - deps.runEval(b, nil, evalSourceCached) - deps.persist(b) - b.cfgReloadMu.Unlock() + if reconcileOverdue(now) { + // The reconcile pass is owed; run the full tick instead of + // the lean cached cycle so it is not starved by re-kicks. + tick(evalSourceCached) + } else { + // Keep this cycle lean: planning and delivery without sweeps. + b.cfgReloadMu.Lock() + b.evalCycles++ + deps.runEval(b, nil, evalSourceCached) + deps.persist(b) + b.cfgReloadMu.Unlock() + } after, afterWho := b.gov.NextContinuousKickAt() if backoff, stalled := wakeHold.observe(next, after, who, afterWho, wakeNow()); stalled { b.logger.Info("continuous re-kick wake: eval cycle left the due re-kick unchanged (agent gated outside the governor); backing off", @@ -6112,6 +6141,23 @@ func (b *boot) runLoopWith(deps runLoopDeps) { for { // Serve due kicks between cycles even when other channels stay ready. serveContinuousWake() + // A reconcile pass owed for a full interval runs here regardless of + // which channel woke the loop, so a ticker tick that was consumed, + // dropped or outraced by wake channels cannot starve enumeration. + if now := loopNow(); b.ctx.Err() == nil && reconcileOverdue(now) { + b.logger.Info("eval interval elapsed without a reconcile pass; enumerating", + "since_last_enumerate", now.Sub(lastEnumerateAt).Round(time.Second).String(), + "eval_interval_s", int(lastEvalInterval.Seconds())) + tick(evalSourceEnumerate) + armContinuousWake() + armWebhookHealthRecheck() + // The cadence tick for this interval, if it is already pending, + // would only repeat the pass that just ran. + select { + case <-ticker.Chan(): + default: + } + } select { case <-b.ctx.Done(): b.logger.Info("shutting down, persisting state") diff --git a/src/cmd/hive/reconcile_starvation_test.go b/src/cmd/hive/reconcile_starvation_test.go new file mode 100644 index 0000000000..cb43358a8c --- /dev/null +++ b/src/cmd/hive/reconcile_starvation_test.go @@ -0,0 +1,112 @@ +package main + +import ( + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/governor" +) + +// loopClock is a fake clock the test advances from its own goroutine while +// the loop reads it through deps.now. +type loopClock struct { + mu sync.Mutex + t time.Time +} + +func (c *loopClock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +func (c *loopClock) advance(d time.Duration) { + c.mu.Lock() + c.t = c.t.Add(d) + c.mu.Unlock() +} + +// Wake-driven cached cycles arriving more often than the eval interval +// must not starve the reconcile pass: once the cached snapshot is older +// than the interval, the next wake enumerates even if the ticker never +// fires (hivecommons/hive#15779). +func TestRunLoopWakeEnumeratesWhenSnapshotOlderThanInterval(t *testing.T) { + b, log, cancel := newLoopBoot(t, 900, 0) + defer cancel() + b.reviewWake = make(chan struct{}, 1) + clock := &loopClock{t: time.Now()} + gov := newFakeTicker() + h := &loopHarness{startupOK: true, pending: []*fakeTicker{gov}, persisted: make(chan struct{}, 16)} + deps := h.deps() + deps.now = clock.now + done := runLoopAsync(b, deps) + <-h.persisted // initial reconcile pass + + // Four wakes 5 minutes apart against a 900s interval: the first two + // plan from the cache, the third lands 15m after the last reconcile + // and must enumerate; the fourth is cached again. + for i := 0; i < 4; i++ { + clock.advance(5 * time.Minute) + b.reviewWake <- struct{}{} + <-h.persisted + } + cancel() + waitDone(t, done) + + want := []evalCycleSource{evalSourceEnumerate, evalSourceCached, evalSourceCached, evalSourceEnumerate, evalSourceCached} + if len(h.sources) != len(want) { + t.Fatalf("sources = %v, want %v", h.sources, want) + } + for i := range want { + if h.sources[i] != want[i] { + t.Fatalf("sources = %v, want %v", h.sources, want) + } + } + if !strings.Contains(log.String(), "cached snapshot older than eval interval; enumerating") { + t.Fatalf("missing upgrade log line:\n%s", log.String()) + } +} + +// A ticker tick that is lost (never delivered) must not stop the reconcile +// pass: any loop iteration that finds a full interval elapsed since the +// last reconcile runs one, here on a not-yet-due continuous wake timer. +func TestRunLoopTopRunsOverdueReconcileWithoutTickerTick(t *testing.T) { + b, log, cancel := newLoopBoot(t, 900, 0) + defer cancel() + b.cfg.Governor.Modes = map[string]config.ModeConfig{"idle": {Cadences: map[string]config.Cadence{"alpha": "continuous"}}} + b.gov = governor.New(b.cfg.Governor, b.cfg.EnabledAgents(), b.logger) + b.gov.Evaluate(0, 0, 0, 0) + clock := &loopClock{t: time.Now()} + b.gov.SeedContinuousNextKickForTest("alpha", clock.now().Add(24*time.Hour)) + gov := newFakeTicker() + wake := newFakeTicker() + h := &loopHarness{startupOK: true, pending: []*fakeTicker{gov}, persisted: make(chan struct{}, 16)} + deps := h.deps() + deps.now = clock.now + deps.newWakeTimer = func(time.Duration) loopTicker { return wake } + done := runLoopAsync(b, deps) + <-h.persisted + + // Poll wake inside the interval: nothing runs, only a re-arm. + clock.advance(10 * time.Minute) + wake.fire() + // Poll wake after the interval elapsed with no ticker tick delivered. + clock.advance(6 * time.Minute) + wake.fire() + <-h.persisted + cancel() + waitDone(t, done) + + if len(h.sources) != 2 || h.sources[1] != evalSourceEnumerate { + t.Fatalf("sources = %v, want initial + loop-top reconcile pass", h.sources) + } + if h.sweeps != 2 { + t.Fatalf("sweeps = %d, want the owed reconcile to run as a full tick", h.sweeps) + } + if !strings.Contains(log.String(), "eval interval elapsed without a reconcile pass; enumerating") { + t.Fatalf("missing loop-top log line:\n%s", log.String()) + } +} diff --git a/src/cmd/hive/webhook_wake_test.go b/src/cmd/hive/webhook_wake_test.go index 945b774d4d..ee9be04979 100644 --- a/src/cmd/hive/webhook_wake_test.go +++ b/src/cmd/hive/webhook_wake_test.go @@ -312,6 +312,7 @@ func TestRunLoopHealthRecheckResetsTickerWhenWebhooksLapse(t *testing.T) { clock.Advance(121 * time.Second) github.RecordGitHubActivity(clock.Now()) // a poll saw a change webhooks missed health.fire() + <-h.persisted // the loop top's overdue reconcile pass (asserted below) cancel() waitDone(t, done) @@ -324,7 +325,13 @@ func TestRunLoopHealthRecheckResetsTickerWhenWebhooksLapse(t *testing.T) { if !strings.Contains(log.String(), "webhooks degraded; polling cadence restored") { t.Fatalf("log = %s", log.String()) } - if len(h.evals) != 1 { - t.Fatalf("health re-check must not run an eval cycle, evals = %d", len(h.evals)) + // The re-check itself runs no eval cycle; the one extra cycle is the + // loop top's owed reconcile pass, because 121s have passed since the + // last one against the restored 60s cadence (hivecommons/hive#15779). + if len(h.sources) != 2 || h.sources[1] != evalSourceEnumerate { + t.Fatalf("sources = %v, want first cycle + one overdue reconcile pass", h.sources) + } + if !strings.Contains(log.String(), "eval interval elapsed without a reconcile pass; enumerating") { + t.Fatalf("missing overdue reconcile log: %s", log.String()) } } diff --git a/src/pkg/dashboard/stage_tracker_build.go b/src/pkg/dashboard/stage_tracker_build.go index c7e366de7b..8aa170777c 100644 --- a/src/pkg/dashboard/stage_tracker_build.go +++ b/src/pkg/dashboard/stage_tracker_build.go @@ -82,7 +82,14 @@ func (s *Server) buildStageTracker(status *StatusPayload, store *stagelabels.Sto if store == nil { return stagelabels.Record{}, false } - return store.Get(stagelabels.Key(repo, number)) + rec, ok := store.Get(stagelabels.Key(repo, number)) + if ok && rec.Stage == stagelabels.None { + // A stage-less record only remembers the blocked sweep's lift + // (hivecommons/hive#15779): the row is still unstaged, but its + // since-time starts at the lift rather than at creation. + return stagelabels.Record{UnblockedAt: rec.UnblockedAt}, false + } + return rec, ok } presence, maintainerPolicy := s.stageTrackerPresence(), s.stageTrackerMaintainerPolicy() fixWork := s.stageTrackerFixWorkLookup() @@ -113,7 +120,7 @@ func (s *Server) buildStageTracker(status *StatusPayload, store *stagelabels.Sto } if hasRec && rec.Stage != stagelabels.None { stage = rec.Stage - t := rec.EnteredAt + t := rec.StageSinceAt() it.StageSince = &t it.Reason = rec.Reason stageTrackerRegression(it, rec) @@ -316,6 +323,7 @@ func (s *Server) buildStageTracker(status *StatusPayload, store *stagelabels.Sto finish(&it, issue.Labels, stage, rec, ok, false) stageTrackerBlockedBy(&it, issue.OpenBlockers()) stageTrackerBlockedOnPR(&it, issue.Body, bucketReason, status.GitHubBaseURL, prState) + stageTrackerSinceUnblock(&it, rec, ok) advisory := stageTrackerAdvisoryFor(issue.Labels, issue.Body, advisoryHint) if advisory == nil { applyStageTrackerNew(&it, stageTrackerNewState(it, ok, issueFilter, status.Governor, now)) @@ -938,3 +946,14 @@ func stageTrackerParkedOnDecision(it StageTrackerItem) bool { lower := strings.ToLower(strings.TrimSpace(base)) return lower == stageengine.StageReasonVerdictRequiresHuman || lower == stageengine.StageReasonDecisionRequested } + +// stageTrackerSinceUnblock ages an unstaged, no-longer-blocked issue from +// the blocked sweep's lift rather than from creation +// (hivecommons/hive#15779). A row blocked again keeps its own age. +func stageTrackerSinceUnblock(it *StageTrackerItem, rec stagelabels.Record, hasRec bool) { + if hasRec || rec.UnblockedAt.IsZero() || it.BlockedOnPR != nil || len(it.BlockedOn) > 0 || slices.Contains(it.Modifiers, modBlocked) { + return + } + t := rec.UnblockedAt + it.StageSince = &t +} diff --git a/src/pkg/dashboard/stage_tracker_unblocked_since_test.go b/src/pkg/dashboard/stage_tracker_unblocked_since_test.go new file mode 100644 index 0000000000..9048dabcfd --- /dev/null +++ b/src/pkg/dashboard/stage_tracker_unblocked_since_test.go @@ -0,0 +1,61 @@ +package dashboard + +import ( + "testing" + "time" + + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +// An issue the blocked sweep just unblocked has no stage record yet; its +// row must age from the lift, not from issue creation +// (hivecommons/hive#15779). +func TestStageTrackerNewRowAgesFromUnblock(t *testing.T) { + now := time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC) + s, _ := stageTrackerFixture(t, now) + clock := now + store := stagelabels.NewStore("", func() time.Time { return clock }) + s.stageStore = store + created := now.Add(-17 * time.Hour) + lifted := now.Add(-20 * time.Minute) + s.statusMu.Lock() + s.status.Repos[0].ActionableIssues = append(s.status.Repos[0].ActionableIssues, + github.Issue{Number: 15517, Title: "Was blocked", Author: "pat", AuthorIsHuman: true, CreatedAt: created, UpdatedAt: created}) + s.statusMu.Unlock() + + resp := s.buildStageTracker(s.status, store, now) + it := trackerItem(t, resp, "issue", 15517) + if it.StageSlug != StageTrackerNewSlug || it.StageSince == nil || !it.StageSince.Equal(created) { + t.Fatalf("before the lift: slug %q since %v, want 000-new since creation %v", it.StageSlug, it.StageSince, created) + } + + store.NoteUnblocked(stagelabels.Key("acme/console", 15517), lifted) + resp = s.buildStageTracker(s.status, store, now) + it = trackerItem(t, resp, "issue", 15517) + if it.StageSlug != StageTrackerNewSlug || it.Stage != 0 || it.New == nil { + t.Fatalf("after the lift the row must still be 000-new: stage %d slug %q new %+v", it.Stage, it.StageSlug, it.New) + } + if it.StageSince == nil || !it.StageSince.Equal(lifted) { + t.Fatalf("after the lift since = %v, want the lift %v", it.StageSince, lifted) + } + + // The stage-label pass that follows may still decide None for the + // issue (SyncIssueStageLabels → ForgetStage); the lift survives it. + store.ForgetStage(stagelabels.Key("acme/console", 15517)) + resp = s.buildStageTracker(s.status, store, now) + it = trackerItem(t, resp, "issue", 15517) + if it.StageSlug != StageTrackerNewSlug || it.StageSince == nil || !it.StageSince.Equal(lifted) { + t.Fatalf("after a None pass: slug %q since %v, want 000-new since the lift %v", it.StageSlug, it.StageSince, lifted) + } + + // The next stage-label pass stages it: since is the stage entry. + staged := now.Add(-5 * time.Minute) + clock = staged + store.Observe(stagelabels.Key("acme/console", 15517), stagelabels.Actionable, "actionable") + resp = s.buildStageTracker(s.status, store, now) + it = trackerItem(t, resp, "issue", 15517) + if it.Stage != stagelabels.Actionable.Number() || it.StageSince == nil || !it.StageSince.Equal(staged) { + t.Fatalf("after staging: stage %d since %v, want 010 since %v", it.Stage, it.StageSince, staged) + } +} diff --git a/src/pkg/github/issue_stage_labels.go b/src/pkg/github/issue_stage_labels.go index 49e86aa014..677ed29b6c 100644 --- a/src/pkg/github/issue_stage_labels.go +++ b/src/pkg/github/issue_stage_labels.go @@ -477,7 +477,7 @@ func SyncIssueStageLabelsWith(ctx context.Context, client *Client, result *Actio var want stagelabels.Desired var tr stagelabels.Transition if stage == stagelabels.None { - store.Forget(key) + store.ForgetStage(key) } else { tr = store.ObserveProgress(key, stage, reason, d.Progress) noteIssueCoveringPR(client, store, key, in, stage) diff --git a/src/pkg/github/item_refresh_hook.go b/src/pkg/github/item_refresh_hook.go index fd828c819c..cb4da9f828 100644 --- a/src/pkg/github/item_refresh_hook.go +++ b/src/pkg/github/item_refresh_hook.go @@ -3,6 +3,9 @@ package github import ( "context" "sync" + "time" + + "github.com/hivecommons/hive/pkg/stagelabels" ) // ItemRefreshHook is told, with the configured repo name and the item number, @@ -47,6 +50,8 @@ func (c *Client) unblockIssue(ctx context.Context, repo string, number int) erro if err := c.RemoveLabel(ctx, repo, number, BlockedLabel); err != nil { return err } + // The row's since-time restarts at the lift (hivecommons/hive#15779). + stagelabels.Shared().NoteUnblocked(stagelabels.Key(repo, number), time.Now()) notifyItemRefresh(repo, number, false) return nil } diff --git a/src/pkg/stagelabels/store.go b/src/pkg/stagelabels/store.go index 9b8db788d9..049502e366 100644 --- a/src/pkg/stagelabels/store.go +++ b/src/pkg/stagelabels/store.go @@ -58,6 +58,12 @@ type Record struct { // read as a fresh note (hivecommons/hive#15729). Zero on records // written before it existed; NoteChangedAt falls back to EnteredAt. ReasonChangedAt time.Time `json:"reason_changed_at,omitempty"` + // UnblockedAt is when the blocked sweep last lifted the item's block + // (hivecommons/hive#15779). A blocked issue has no stage, so the lift + // leaves a stage-less record carrying only this time until the next + // stage-label pass stages the issue; the dashboard ages the row from + // StageSinceAt so "since" restarts at the lift, not at issue creation. + UnblockedAt time.Time `json:"unblocked_at,omitempty"` // Author is the item's author login, lower-cased, as the lane saw it // (hivecommons/hive#12502). It lets the dashboard answer "which of this // contributor's PRs are returned to them" from the store alone, with no @@ -649,8 +655,7 @@ func (s *Store) ObserveProgress(key string, stage Stage, reason string, progress tr := Transition{Prev: prev.Stage, Next: stage} if stage == None { if had { - delete(s.items, key) - s.dirty = true + s.forgetStageLocked(key, prev) tr.Changed = prev.Stage != None } return tr @@ -746,6 +751,33 @@ func (r Record) NoteChangedAt() time.Time { return r.EnteredAt } +// StageSinceAt is when the row's since-time starts: stage entry, or the +// later block lift when the sweep unblocked the item after it entered the +// stage (or before any stage was recorded). +func (r Record) StageSinceAt() time.Time { + if r.UnblockedAt.After(r.EnteredAt) { + return r.UnblockedAt + } + return r.EnteredAt +} + +// NoteUnblocked records that the blocked sweep lifted key's block at now. +// A blocked issue carries no stage record (its stage is None), so a +// stage-less record is created to hold the time; the next ObserveProgress +// keeps UnblockedAt while it stages the issue, and a None observation +// drops the record as usual. +func (s *Store) NoteUnblocked(key string, now time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + r := s.items[key] + r.UnblockedAt = now + if r.LastSeen.IsZero() { + r.LastSeen = now + } + s.items[key] = r + s.dirty = true +} + // NoteActivity folds this cycle's activity inputs into key's record and // reports whether anything a person could have done changed since the last // observation. The first observation of a record seeds LastActivityAt @@ -1949,7 +1981,7 @@ func (s *Store) SetCoveringPR(key, prKey string) { s.dirty = true } -// Forget drops key (closed/merged item). +// Forget drops key (closed/merged item), lift record included. func (s *Store) Forget(key string) { s.mu.Lock() defer s.mu.Unlock() @@ -1959,6 +1991,29 @@ func (s *Store) Forget(key string) { } } +// ForgetStage drops key's stage for an item that is still open but has no +// stage (a blocked issue, or one outside the hive's buckets). Unlike +// Forget it keeps the blocked sweep's lift, so the stage-label pass that +// runs right after an unblock does not reset the row's since-time back to +// creation (hivecommons/hive#15779). +func (s *Store) ForgetStage(key string) { + s.mu.Lock() + defer s.mu.Unlock() + if r, ok := s.items[key]; ok { + s.forgetStageLocked(key, r) + } +} + +// forgetStageLocked is ForgetStage with s.mu held and the record in hand. +func (s *Store) forgetStageLocked(key string, r Record) { + if r.UnblockedAt.IsZero() { + delete(s.items, key) + } else { + s.items[key] = Record{UnblockedAt: r.UnblockedAt, LastSeen: s.now()} + } + s.dirty = true +} + // Prune drops records not seen for maxAge (closed items the sweep stopped // listing). Returns how many were dropped. Author-wide trust-promotion // records (TrustPromotionKey) are per contributor, not per item, and diff --git a/src/pkg/stagelabels/unblocked_test.go b/src/pkg/stagelabels/unblocked_test.go new file mode 100644 index 0000000000..6e9351e1da --- /dev/null +++ b/src/pkg/stagelabels/unblocked_test.go @@ -0,0 +1,61 @@ +package stagelabels + +import ( + "testing" + "time" +) + +// NoteUnblocked keeps the lift on a stage-less record, StageSinceAt reads +// it as the since-time, and the next staging observation keeps it while +// stage entry (later) takes over (hivecommons/hive#15779). +func TestNoteUnblockedAgesRecordFromLift(t *testing.T) { + clock := time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC) + s := NewStore("", func() time.Time { return clock }) + key := Key("o/r", 15517) + lifted := clock.Add(-20 * time.Minute) + + s.NoteUnblocked(key, lifted) + rec, ok := s.Get(key) + if !ok || rec.Stage != None || !rec.UnblockedAt.Equal(lifted) || !rec.StageSinceAt().Equal(lifted) { + t.Fatalf("after lift: ok=%v rec=%+v since=%v, want stage-less record since the lift", ok, rec, rec.StageSinceAt()) + } + + tr := s.Observe(key, Actionable, "actionable") + if !tr.Changed || tr.Prev != None { + t.Fatalf("staging after lift transition = %+v, want a change from None", tr) + } + rec, _ = s.Get(key) + if !rec.UnblockedAt.Equal(lifted) || !rec.StageSinceAt().Equal(clock) { + t.Fatalf("after staging: rec=%+v since=%v, want stage entry %v", rec, rec.StageSinceAt(), clock) + } + + // A lift after stage entry moves the since-time forward again. + later := clock.Add(time.Hour) + s.NoteUnblocked(key, later) + rec, _ = s.Get(key) + if !rec.StageSinceAt().Equal(later) { + t.Fatalf("since after a later lift = %v, want %v", rec.StageSinceAt(), later) + } + // The stage-label pass deciding None for a still-open issue (the + // row the symptom describes) keeps the lift; only Forget — the + // closed/merged path — drops it. + s.Observe(key, None, "") + rec, ok = s.Get(key) + if !ok || rec.Stage != None || !rec.UnblockedAt.Equal(later) || rec.Reason != "" { + t.Fatalf("after a None observation: ok=%v rec=%+v, want a lift-only record", ok, rec) + } + s.ForgetStage(key) + if rec, ok = s.Get(key); !ok || !rec.UnblockedAt.Equal(later) { + t.Fatalf("after ForgetStage: ok=%v rec=%+v, want the lift kept", ok, rec) + } + s.Forget(key) + if _, ok := s.Get(key); ok { + t.Fatal("Forget must drop the record, lift included") + } + s.ForgetStage(Key("o/r", 1)) + s.Observe(Key("o/r", 2), Actionable, "actionable") + s.ForgetStage(Key("o/r", 2)) + if _, ok := s.Get(Key("o/r", 2)); ok { + t.Fatal("ForgetStage without a lift must drop the record") + } +}