diff --git a/changelog.d/fixed-bounded-terminal-pr-enrichment.md b/changelog.d/fixed-bounded-terminal-pr-enrichment.md new file mode 100644 index 0000000000..bfc1e0cbac --- /dev/null +++ b/changelog.d/fixed-bounded-terminal-pr-enrichment.md @@ -0,0 +1 @@ +- Keep historical terminal-PR accounting off current-work claim evaluation with cancellable bounded review batches, retained retries, and idempotent timeline persistence; preserve synchronous fail-closed claim safety and durable first observations. diff --git a/src/cmd/hive/github_client_config.go b/src/cmd/hive/github_client_config.go index 252d039746..56593b527f 100644 --- a/src/cmd/hive/github_client_config.go +++ b/src/cmd/hive/github_client_config.go @@ -402,6 +402,7 @@ func (b *boot) applyGitHubClientDashboardHooks(client *github.Client) { client.SetPRReviewObservedHook(func(obs github.PRReviewObservation) { recordObservedReview(b.dashSrv, obs) }) + client.StartPRTerminalEnrichment(b.ctx) client.SetPROpenedHook(func(agentName, repo string, number int, url string) { b.dashSrv.LinearAgentPROpened(agentName, repo, number, url) // Same typed hook feeds the lifecycle timeline: the watcher fires diff --git a/src/cmd/hive/github_client_enrichment_test.go b/src/cmd/hive/github_client_enrichment_test.go new file mode 100644 index 0000000000..2d43c832b7 --- /dev/null +++ b/src/cmd/hive/github_client_enrichment_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/github" +) + +func TestAdoptGitHubClientRetiresHistoricalAccountingWorker(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + previous, next := &github.Client{}, &github.Client{} + done := previous.StartPRTerminalEnrichment(ctx) + b := &boot{ghClient: previous} + b.adoptGitHubClient(next, nil) + const stopTimeout = 5 * time.Second + select { + case <-done: + case <-time.After(stopTimeout): + t.Fatal("client rebuild left old historical accounting worker running") + } + if ctx.Err() != nil || b.currentGitHubClient() != next { + t.Fatal("retiring historical accounting disturbed boot context/client adoption") + } +} diff --git a/src/cmd/hive/github_client_provider.go b/src/cmd/hive/github_client_provider.go index 3c36263c2d..afdd6e5de3 100644 --- a/src/cmd/hive/github_client_provider.go +++ b/src/cmd/hive/github_client_provider.go @@ -71,6 +71,9 @@ func (b *boot) adoptGitHubClient(newClient *github.Client, newAppAuth *github.Ap if b == nil || newClient == nil { return } + if previous := b.currentGitHubClient(); previous != nil && previous != newClient { + previous.StopPRTerminalEnrichment() + } b.ghClient = newClient b.appAuth = newAppAuth // Provider consumers (metrics and fleet-stats collectors, the scheduler's diff --git a/src/cmd/hive/lifecyclewire.go b/src/cmd/hive/lifecyclewire.go index 8c582ab27b..a9af4b801e 100644 --- a/src/cmd/hive/lifecyclewire.go +++ b/src/cmd/hive/lifecyclewire.go @@ -107,7 +107,12 @@ func recordMergedOnLinkedIssues(store *timeline.Store, prRef, agent string, at i for k, v := range attrs { copyAttrs[k] = v } - store.Record(timeline.Event{IssueRef: issue, Kind: timeline.KindMerged, Agent: agent, At: at, Attrs: copyAttrs}) + event := timeline.Event{IssueRef: issue, Kind: timeline.KindMerged, Agent: agent, At: at, Attrs: copyAttrs} + if at > 0 { + store.RecordObservation(event) + } else { + store.Record(event) + } } } @@ -133,7 +138,7 @@ func recordObservedMerge(rec lifecycleRecorder, org, repo string, number int, ob if !observedAt.IsZero() { event.At = observedAt.UnixMilli() } - store.Record(event) + store.RecordObservation(event) recordMergedOnLinkedIssues(store, ref, "", event.At, attrs) } @@ -163,13 +168,18 @@ func recordPROpenedForIssuesAt(rec lifecycleRecorder, org, agent, repo string, n if n <= 0 || n == number { continue } - store.Record(timeline.Event{ + event := timeline.Event{ IssueRef: timelineRef(org, repo, n), Kind: timeline.KindPROpened, Agent: agent, At: at, Attrs: map[string]string{timeline.AttrPRRef: prRef, "url": url}, - }) + } + if at > 0 { + store.RecordObservation(event) + } else { + store.Record(event) + } } } @@ -322,7 +332,7 @@ func backfillMergedPROnIssues(rec lifecycleRecorder, org string, obs github.PRTe if obs.Actor != "" { attrs["actor"] = obs.Actor } - store.Record(timeline.Event{IssueRef: ref, Kind: timeline.KindMerged, At: mergedAt, Attrs: attrs}) + store.RecordObservation(timeline.Event{IssueRef: ref, Kind: timeline.KindMerged, At: mergedAt, Attrs: attrs}) } return len(needMerged) } diff --git a/src/docs/audit-log.md b/src/docs/audit-log.md index e6466bba07..f2ae973405 100644 --- a/src/docs/audit-log.md +++ b/src/docs/audit-log.md @@ -15,6 +15,35 @@ what `gh` command) lives in the ## Format +### Historical PR observations and dispatch + +The closed-PR settle scan still lists current claims and records claim/churn +history synchronously. A listing failure remains non-authoritative; deferred +accounting never means a claim is clear. Only historical merge actor lookup, +terminal audit/timeline observations, and terminal review reads run in the +client's single lifecycle-bound accounting worker. + +The worker coalesces pending work by repository/PR and attempts at most eight +observations per batch, with a ten-second cancellable API budget and a +ten-second wake cadence. Failed work retains exponential retry backoff rather +than being silently discarded. These are accounting bounds, not a promise +that an uninterruptible filesystem sink finishes within ten seconds. A slow +sink occupies the accounting worker, not the claim evaluation call stack. +The `historical PR enrichment batch` log reports processed, failed, deferred, +duration, budget, and limit. + +Pending accounting is in memory; after a restart or client credential rebuild +the existing 72-hour terminal settle scan replays recent observations. +This does not extend that historical +recovery window or make the accounting queue an ownership authority. +Timeline first sightings and corrected precious metadata still wait for atomic +persistence. Identical replay does not increment counts or rewrite the file; +dirty state after a failed save is retried. Atomic writers are serialized, but +disk I/O no longer holds the journey reader mutex, and an older snapshot cannot +acknowledge a concurrent unsaved update. New precious writers may still wait +for an earlier disk writer, preserving their durability rather than reporting +false completion. + One JSON object per line (JSONL), append-only. Written by `pkg/dashboard/audit.go`. diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index eb5d85539a..7e3236bc35 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -346,6 +346,7 @@ type Client struct { // prreviews_observed.go. prReviewObservedHook atomic.Pointer[PRReviewObservedHook] prTerminalReviews prTerminalReviewCache + prTerminalEnrichment terminalEnrichmentState prRepoPolicyGate atomic.Pointer[PRRepoPolicyGate] // agentStartHook is told when an agent's own relay request shows it // working an issue (#10527). See agent_start_hook.go. diff --git a/src/pkg/github/pr_terminal_enrichment.go b/src/pkg/github/pr_terminal_enrichment.go new file mode 100644 index 0000000000..e93c1b6254 --- /dev/null +++ b/src/pkg/github/pr_terminal_enrichment.go @@ -0,0 +1,215 @@ +package github + +import ( + "context" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + + gh "github.com/google/go-github/v72/github" +) + +const ( + terminalEnrichmentInterval = 10 * time.Second + terminalEnrichmentBudget = 10 * time.Second + terminalEnrichmentBatch = 8 +) + +type terminalEnrichmentTask struct { + owner, repoName string + observation PRTerminalObservation + updatedAt time.Time + identity HiveIdentity + mergedBy string + sequence uint64 + revision uint64 +} + +func (t terminalEnrichmentTask) key() string { + return strings.ToLower(t.owner+"/"+t.repoName) + "#" + strconv.Itoa(t.observation.Number) +} + +func (t terminalEnrichmentTask) sameObservation(other terminalEnrichmentTask) bool { + a, b := t.observation, other.observation + return t.updatedAt.Equal(other.updatedAt) && t.identity == other.identity && t.mergedBy == other.mergedBy && + a.Repo == b.Repo && a.Number == b.Number && a.State == b.State && a.ObservedAt.Equal(b.ObservedAt) && + a.Actor == b.Actor && a.Attribution == b.Attribution && a.CreatedAt.Equal(b.CreatedAt) && + a.URL == b.URL && slices.Equal(a.Issues, b.Issues) +} + +// Only observational accounting is deferred. Claim listings, churn history, +// ledger persistence and fail-closed dispatch filtering never read this queue. +// Pending tasks survive failed batches and coalesce by PR, not by scan. On a +// process restart the existing terminal listing replays the settle window; +// timeline sinks must therefore remain idempotent and persist first sightings. +type terminalEnrichmentState struct { + mu sync.Mutex + started bool + cancel context.CancelFunc + done chan struct{} + wake chan struct{} + pending map[string]terminalEnrichmentTask + sequence uint64 + retries retryTracker +} + +// StartPRTerminalEnrichment isolates historical accounting from current-work +// evaluation. It starts at most one worker per client, using the same lifecycle +// context as the client's request watchers. Unstarted clients retain the +// synchronous observation API (including standalone callers and tests). +func (c *Client) StartPRTerminalEnrichment(ctx context.Context) <-chan struct{} { + if c == nil || ctx == nil { + done := make(chan struct{}) + close(done) + return done + } + s := &c.prTerminalEnrichment + s.mu.Lock() + defer s.mu.Unlock() + if s.started { + return s.done + } + s.started = true + ctx, s.cancel = context.WithCancel(ctx) + s.done = make(chan struct{}) + s.wake = make(chan struct{}, 1) + s.pending = make(map[string]terminalEnrichmentTask) + go func() { + defer close(s.done) + ticker := time.NewTicker(terminalEnrichmentInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + case <-s.wake: + } + if ctx.Err() != nil { + return + } + c.enrichTerminalPRBatch(ctx, terminalEnrichmentBudget, terminalEnrichmentBatch) + } + }() + return s.done +} + +// StopPRTerminalEnrichment retires a replaced client's worker without waiting +// on an uninterruptible accounting sink. The replacement's settle scan replays +// recent pending history, just as on process restart. +func (c *Client) StopPRTerminalEnrichment() { + if c == nil { + return + } + s := &c.prTerminalEnrichment + s.mu.Lock() + defer s.mu.Unlock() + if s.cancel != nil { + s.cancel() + } +} + +func (c *Client) observeTerminalPR(ctx context.Context, task terminalEnrichmentTask) { + s := &c.prTerminalEnrichment + s.mu.Lock() + if !s.started { + s.mu.Unlock() + c.enrichTerminalPR(ctx, task) + return + } + key := task.key() + old, exists := s.pending[key] + if exists && task.sameObservation(old) { + s.mu.Unlock() + return + } + if !exists || !task.updatedAt.Before(old.updatedAt) { + s.sequence++ + task.revision = s.sequence + if exists { + task.sequence = old.sequence + } else { + task.sequence = s.sequence + } + s.pending[key] = task + } + // Wake only on an empty-to-nonempty transition. A listing must not + // repeatedly wake a failed/backed-off batch or bypass its bounded cadence. + if !exists && len(s.pending) == 1 { + select { + case s.wake <- struct{}{}: + default: + } + } + s.mu.Unlock() +} + +func (c *Client) enrichTerminalPR(ctx context.Context, task terminalEnrichmentTask) bool { + if ctx.Err() != nil { + return false + } + obs := task.observation + if obs.State == PRStateMerged { + pr := &gh.PullRequest{Number: gh.Ptr(obs.Number), MergedBy: &gh.User{Login: gh.Ptr(task.mergedBy)}} + obs.Actor = c.prTerminalMergedBy(ctx, task.owner, task.repoName, pr) + obs.Attribution = prTerminalMergeAttribution(obs.Actor, task.identity, c.appBotLogin) + } + if ctx.Err() != nil { + return false + } + c.emitPRTerminalObserved(obs) + return c.emitTerminalPRReviews(ctx, task.owner, task.repoName, obs.Repo, obs.Number, task.updatedAt, task.identity) +} + +func (c *Client) enrichTerminalPRBatch(ctx context.Context, budget time.Duration, limit int) { + started := time.Now() + ctx, cancel := context.WithTimeout(ctx, budget) + defer cancel() + s := &c.prTerminalEnrichment + s.mu.Lock() + tasks := make([]terminalEnrichmentTask, 0, len(s.pending)) + for _, task := range s.pending { + tasks = append(tasks, task) + } + s.mu.Unlock() + sort.Slice(tasks, func(i, j int) bool { return tasks[i].sequence < tasks[j].sequence }) + processed, failed := 0, 0 + for _, task := range tasks { + key := task.key() + if ctx.Err() != nil || processed+failed >= limit { + break + } + if !s.retries.allows(key, time.Now()) { + continue + } + ok := c.enrichTerminalPR(ctx, task) + if ok { + processed++ + s.retries.clear(key) + s.mu.Lock() + current := s.pending[key] + // A scan may have enqueued a newer terminal state/review while + // this request was in flight. Never acknowledge that newer work. + if current.revision == task.revision { + delete(s.pending, key) + } + s.mu.Unlock() + } else { + failed++ + // Accounting is not discarded at the request-file quarantine + // horizon: a later successful batch must still record it. + s.retries.noteFailure(key, time.Now()) + } + } + s.mu.Lock() + pending := len(s.pending) + s.mu.Unlock() + if c.logger != nil && (processed+failed > 0) { + c.logger.Info("historical PR enrichment batch", + "processed", processed, "failed", failed, "deferred", pending, + "duration", time.Since(started), "budget", budget, "limit", limit) + } +} diff --git a/src/pkg/github/pr_terminal_enrichment_test.go b/src/pkg/github/pr_terminal_enrichment_test.go new file mode 100644 index 0000000000..35faf82063 --- /dev/null +++ b/src/pkg/github/pr_terminal_enrichment_test.go @@ -0,0 +1,298 @@ +package github + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +const terminalEnrichmentTestTimeout = 5 * time.Second + +func TestTerminalEnrichmentIdenticalInFlightReplayDoesNotStarveNextTask(t *testing.T) { + c := &Client{} + deferTerminalTasks(c) + first, next := historicalTask(1, time.Now()), historicalTask(2, time.Now()) + c.observeTerminalPR(context.Background(), first) + c.observeTerminalPR(context.Background(), next) + var seen []int + c.SetPRTerminalObservedHook(func(obs PRTerminalObservation) { + seen = append(seen, obs.Number) + if obs.Number == first.observation.Number { + c.observeTerminalPR(context.Background(), first) + } + }) + c.enrichTerminalPRBatch(context.Background(), terminalEnrichmentTestTimeout, 1) + if _, exists := c.prTerminalEnrichment.pending[first.key()]; exists { + t.Fatal("identical in-flight replay prevented acknowledgment") + } + c.enrichTerminalPRBatch(context.Background(), terminalEnrichmentTestTimeout, 1) + if len(seen) != 2 || seen[1] != next.observation.Number || len(c.prTerminalEnrichment.pending) != 0 { + t.Fatalf("identical old replay starved subsequent history: %v", seen) + } +} + +func TestTerminalEnrichmentReplacedClientStopsWithoutRootCancellation(t *testing.T) { + c := &Client{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := c.StartPRTerminalEnrichment(ctx) + c.StopPRTerminalEnrichment() + waitTerminalEnrichment(t, done) + if ctx.Err() != nil { + t.Fatal("retiring old client cancelled shared boot context") + } +} + +func TestTerminalEnrichmentCoalescesWithoutOverwritingNewerHistory(t *testing.T) { + c := &Client{} + deferTerminalTasks(c) + at := time.Now() + current := historicalTask(1, at) + current.observation.URL = "current" + older := historicalTask(1, at.Add(-time.Hour)) + older.observation.URL = "obsolete" + c.observeTerminalPR(context.Background(), current) + c.observeTerminalPR(context.Background(), older) + var seen []PRTerminalObservation + c.SetPRTerminalObservedHook(func(obs PRTerminalObservation) { seen = append(seen, obs) }) + c.enrichTerminalPRBatch(context.Background(), terminalEnrichmentTestTimeout, terminalEnrichmentBatch) + if len(seen) != 1 || seen[0].URL != "current" { + t.Fatalf("older scan replaced current historical evidence: %+v", seen) + } +} + +func waitTerminalEnrichment(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(terminalEnrichmentTestTimeout): + t.Fatal("terminal enrichment did not reach synchronization point") + } +} + +func deferTerminalTasks(c *Client) { + c.prTerminalEnrichment.started = true + c.prTerminalEnrichment.pending = make(map[string]terminalEnrichmentTask) + c.prTerminalEnrichment.wake = make(chan struct{}, 1) +} + +func historicalTask(number int, at time.Time) terminalEnrichmentTask { + return terminalEnrichmentTask{ + owner: "torch-spyre", repoName: "spyre-inference", updatedAt: at, + observation: PRTerminalObservation{ + Repo: "spyre-inference", Number: number, State: PRStateClosed, ObservedAt: at, + }, + } +} + +func TestTerminalEnrichmentSlowReviewsDoNotBlockClaimSafety(t *testing.T) { + entered := make(chan struct{}) + var once sync.Once + var failListing atomic.Bool + now := time.Now().UTC().Truncate(time.Second) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if strings.HasSuffix(r.URL.Path, "/reviews") { + once.Do(func() { close(entered) }) + <-r.Context().Done() + return + } + if failListing.Load() { + http.Error(w, "listing unavailable", http.StatusForbidden) + return + } + if r.URL.Query().Get("state") == "closed" { + _ = json.NewEncoder(w).Encode([]map[string]any{ + mergedPR(500, "kubestellar-hive[bot]", "done", "Fixes #200", now, now), + }) + } else { + _ = json.NewEncoder(w).Encode([]map[string]any{ + pr(423, "kubestellar-hive[bot]", "current work", "Fixes #100"), + }) + } + })) + defer srv.Close() + c := NewClientForTest(srv.URL, "torch-spyre", []string{"spyre-inference"}, testLogger()) + c.SetPRReviewObservedHook(func(PRReviewObservation) {}) + ctx, cancel := context.WithCancel(context.Background()) + done := c.StartPRTerminalEnrichment(ctx) + defer func() { cancel(); waitTerminalEnrichment(t, done) }() + if c.StartPRTerminalEnrichment(ctx) != done { + t.Fatal("starting twice created another worker") + } + identity := HiveIdentity{AppLogin: "kubestellar-hive[bot]"} + scan, err := c.FetchClaimScan(ctx, identity) + if err != nil || len(scan.Claims) != 2 { + t.Fatalf("initial authoritative claims: %+v, %v", scan, err) + } + waitTerminalEnrichment(t, entered) + // Historical HTTP remains blocked; another current-work scan must finish. + currentCtx, stop := context.WithTimeout(ctx, terminalEnrichmentTestTimeout) + defer stop() + repeated, err := c.FetchClaimScan(currentCtx, identity) + if err != nil || len(repeated.Claims) != 2 { + t.Fatalf("current claims waited for historical reviews: %+v, %v", repeated, err) + } + ledger := NewClaimLedger("", testLogger()) + ledger.ReconcileScan(scan, true, nil) + failListing.Store(true) + partial, err := c.FetchClaimScan(currentCtx, identity) + if err == nil { + t.Fatal("failed authoritative listing reported success") + } + ledger.ReconcileScan(partial, false, nil) + if claims := ledger.Claims(); len(claims) != 2 { + t.Fatalf("historical isolation released claims on failed listing: %+v", claims) + } +} + +func TestTerminalEnrichmentSlowAccountingDoesNotBlockClaimScan(t *testing.T) { + now := time.Now() + srv := mergedClaimServer(t, + []map[string]any{pr(423, "kubestellar-hive[bot]", "current", "Fixes #100")}, + []map[string]any{mergedPR(500, "kubestellar-hive[bot]", "done", "Fixes #200", now, now)}) + c := NewClientForTest(srv.URL, "torch-spyre", []string{"spyre-inference"}, testLogger()) + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + c.SetPRTerminalObservedHook(func(PRTerminalObservation) { + once.Do(func() { close(entered) }) + <-release + }) + ctx, cancel := context.WithCancel(context.Background()) + done := c.StartPRTerminalEnrichment(ctx) + defer func() { close(release); cancel(); waitTerminalEnrichment(t, done) }() + identity := HiveIdentity{AppLogin: "kubestellar-hive[bot]"} + if _, err := c.FetchClaimScan(ctx, identity); err != nil { + t.Fatal(err) + } + waitTerminalEnrichment(t, entered) + currentCtx, stop := context.WithTimeout(ctx, terminalEnrichmentTestTimeout) + defer stop() + scan, err := c.FetchClaimScan(currentCtx, identity) + if err != nil || len(scan.Claims) != 2 { + t.Fatalf("slow accounting blocked current claim servicing: %+v, %v", scan, err) + } +} + +func TestTerminalEnrichmentBatchLimitAndInFlightRevision(t *testing.T) { + c := &Client{} + deferTerminalTasks(c) + at := time.Now() + first, next := historicalTask(1, at), historicalTask(2, at) + c.observeTerminalPR(context.Background(), first) + c.observeTerminalPR(context.Background(), next) + var seen []int + c.SetPRTerminalObservedHook(func(obs PRTerminalObservation) { + seen = append(seen, obs.Number) + if len(seen) == 1 { + // Even an equal updated_at can carry corrected metadata. + newer := first + newer.observation.URL = "corrected" + c.observeTerminalPR(context.Background(), newer) + } + }) + c.enrichTerminalPRBatch(context.Background(), terminalEnrichmentTestTimeout, 1) + if len(seen) != 1 || seen[0] != 1 || len(c.prTerminalEnrichment.pending) != 2 { + t.Fatalf("batch cap or in-flight update lost: seen=%v pending=%v", seen, c.prTerminalEnrichment.pending) + } + c.enrichTerminalPRBatch(context.Background(), terminalEnrichmentTestTimeout, terminalEnrichmentBatch) + if len(c.prTerminalEnrichment.pending) != 0 || len(seen) != 3 || seen[2] != 2 { + t.Fatalf("deferred work did not drain fairly: seen=%v pending=%v", seen, c.prTerminalEnrichment.pending) + } + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + c.observeTerminalPR(context.Background(), first) + c.enrichTerminalPRBatch(cancelled, terminalEnrichmentTestTimeout, terminalEnrichmentBatch) + if len(seen) != 3 || len(c.prTerminalEnrichment.pending) != 1 { + t.Fatal("cancelled batch consumed accounting") + } +} + +func TestTerminalEnrichmentFailedReviewsRetryWithoutBlockingHealthyTasks(t *testing.T) { + var failed atomic.Bool + failed.Store(true) + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + if strings.Contains(r.URL.Path, "/1/") && failed.Load() { + http.Error(w, "temporary", http.StatusForbidden) + return + } + _, _ = w.Write([]byte("[]")) + })) + defer srv.Close() + c := NewClientForTest(srv.URL, "torch-spyre", []string{"spyre-inference"}, testLogger()) + deferTerminalTasks(c) + c.SetPRReviewObservedHook(func(PRReviewObservation) {}) + first := historicalTask(1, time.Now()) + c.observeTerminalPR(context.Background(), first) + c.observeTerminalPR(context.Background(), historicalTask(2, first.updatedAt)) + c.enrichTerminalPRBatch(context.Background(), terminalEnrichmentTestTimeout, terminalEnrichmentBatch) + if len(c.prTerminalEnrichment.pending) != 1 { + t.Fatal("failed accounting disappeared or healthy accounting was blocked") + } + before := requests.Load() + c.enrichTerminalPRBatch(context.Background(), terminalEnrichmentTestTimeout, terminalEnrichmentBatch) + if requests.Load() != before { + t.Fatal("failed historical request bypassed retry backoff") + } + failed.Store(false) + c.prTerminalEnrichment.retries.clear(first.key()) + c.enrichTerminalPRBatch(context.Background(), terminalEnrichmentTestTimeout, terminalEnrichmentBatch) + if len(c.prTerminalEnrichment.pending) != 0 { + t.Fatal("successful retry did not retain and acknowledge historical work") + } +} + +func TestTerminalEnrichmentDeadlineRetainsUnfinishedAccounting(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-r.Context().Done() + })) + defer srv.Close() + c := NewClientForTest(srv.URL, "torch-spyre", []string{"spyre-inference"}, testLogger()) + deferTerminalTasks(c) + c.SetPRReviewObservedHook(func(PRReviewObservation) {}) + c.observeTerminalPR(context.Background(), historicalTask(1, time.Now())) + // The production budget is ten seconds; this fixture uses a short deadline, + // not a sleep, so CI does not spend the real servicing interval waiting. + const budget = 20 * time.Millisecond + c.enrichTerminalPRBatch(context.Background(), budget, terminalEnrichmentBatch) + if len(c.prTerminalEnrichment.pending) != 1 { + t.Fatal("deadline lost the unfinished historical observation") + } +} + +func TestTerminalEnrichmentRestartReplaysUnprocessedRecentHistory(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + srv := mergedClaimServer(t, nil, []map[string]any{ + mergedPR(500, "kubestellar-hive[bot]", "done", "Fixes #200", now, now), + }) + identity := HiveIdentity{AppLogin: "kubestellar-hive[bot]"} + before := NewClientForTest(srv.URL, "torch-spyre", []string{"spyre-inference"}, testLogger()) + deferTerminalTasks(before) + if _, err := before.FetchClaimScan(context.Background(), identity); err != nil { + t.Fatal(err) + } + if len(before.prTerminalEnrichment.pending) != 1 { + t.Fatal("fixture did not leave historical accounting outstanding") + } + after := NewClientForTest(srv.URL, "torch-spyre", []string{"spyre-inference"}, testLogger()) + deferTerminalTasks(after) + var observed []PRTerminalObservation + after.SetPRTerminalObservedHook(func(obs PRTerminalObservation) { observed = append(observed, obs) }) + scan, err := after.FetchClaimScan(context.Background(), identity) + if err != nil || len(scan.Claims) != 1 { + t.Fatalf("restart lost authoritative merged claim: %+v, %v", scan, err) + } + after.enrichTerminalPRBatch(context.Background(), terminalEnrichmentTestTimeout, terminalEnrichmentBatch) + if len(observed) != 1 || !observed[0].ObservedAt.Equal(now) { + t.Fatalf("restart did not replay the real historical timestamp: %+v", observed) + } +} diff --git a/src/pkg/github/prclaims.go b/src/pkg/github/prclaims.go index 32e6b3ccc5..0b649fdd95 100644 --- a/src/pkg/github/prclaims.go +++ b/src/pkg/github/prclaims.go @@ -1080,13 +1080,16 @@ func (c *Client) FetchClaimScan(ctx context.Context, identity HiveIdentity) (Cla closedAt = pr.GetUpdatedAt().Time } if !closedAt.Before(mergedCutoff) { - c.emitPRTerminalObserved(PRTerminalObservation{ - Repo: repo, - Number: pr.GetNumber(), - State: PRStateClosed, - ObservedAt: closedAt, + c.observeTerminalPR(ctx, terminalEnrichmentTask{ + owner: owner, repoName: repoName, identity: identity, + updatedAt: pr.GetUpdatedAt().Time, + observation: PRTerminalObservation{ + Repo: repo, + Number: pr.GetNumber(), + State: PRStateClosed, + ObservedAt: closedAt, + }, }) - c.emitTerminalPRReviews(ctx, owner, repoName, repo, pr.GetNumber(), pr.GetUpdatedAt().Time, identity) history = append(history, prHistoryFromClaims(claimsFromPR(pr, repo, identity, now), PRStateClosed, now)...) } @@ -1095,21 +1098,21 @@ func (c *Client) FetchClaimScan(ctx context.Context, identity HiveIdentity) (Cla if mergedAt.Before(mergedCutoff) { continue } - mergedBy := c.prTerminalMergedBy(ctx, owner, repoName, pr) - c.emitPRTerminalObserved(PRTerminalObservation{ - Repo: repo, - Number: pr.GetNumber(), - State: PRStateMerged, - ObservedAt: mergedAt, - Actor: mergedBy, - Attribution: prTerminalMergeAttribution(mergedBy, identity, c.appBotLogin), - CreatedAt: pr.GetCreatedAt().Time, - URL: pr.GetHTMLURL(), - Issues: mergedPRClosingIssues(pr, repo, owner+"/"+repoName), + c.observeTerminalPR(ctx, terminalEnrichmentTask{ + owner: owner, repoName: repoName, identity: identity, + updatedAt: pr.GetUpdatedAt().Time, mergedBy: safeGetLogin(pr.GetMergedBy()), + observation: PRTerminalObservation{ + Repo: repo, + Number: pr.GetNumber(), + State: PRStateMerged, + ObservedAt: mergedAt, + CreatedAt: pr.GetCreatedAt().Time, + URL: pr.GetHTMLURL(), + Issues: mergedPRClosingIssues(pr, repo, owner+"/"+repoName), + }, }) // #16116: other people's reviews reach the throughput card's // Reviewed row the way their merges reach Merged·Closed. - c.emitTerminalPRReviews(ctx, owner, repoName, repo, pr.GetNumber(), pr.GetUpdatedAt().Time, identity) prClaims := claimsFromPR(pr, repo, identity, now) history = append(history, prHistoryFromClaims(prClaims, PRStateMerged, now)...) for _, claim := range prClaims { diff --git a/src/pkg/github/prreviews_observed.go b/src/pkg/github/prreviews_observed.go index ab50b633ac..757e753088 100644 --- a/src/pkg/github/prreviews_observed.go +++ b/src/pkg/github/prreviews_observed.go @@ -70,10 +70,10 @@ func (c *Client) prReviewObserved() PRReviewObservedHook { // next scan. The dashboard sink de-duplicates // repeats by review ID. A failed read emits nothing, is // not cached, and does not fail the scan: the next scan retries it. -func (c *Client) emitTerminalPRReviews(ctx context.Context, owner, repoName, repo string, number int, updatedAt time.Time, identity HiveIdentity) { +func (c *Client) emitTerminalPRReviews(ctx context.Context, owner, repoName, repo string, number int, updatedAt time.Time, identity HiveIdentity) bool { hook := c.prReviewObserved() if hook == nil || number <= 0 { - return + return true } key := strings.ToLower(owner+"/"+repoName) + "#" + strconv.Itoa(number) now := time.Now() @@ -82,14 +82,14 @@ func (c *Client) emitTerminalPRReviews(ctx context.Context, owner, repoName, rep for _, r := range reviews { hook(r) } - return + return true } fresh, err := c.listTerminalPRReviews(ctx, owner, repoName, repo, number, identity) if err != nil { if c.logger != nil { c.logger.Debug("terminal PR reviews lookup failed", "repo", owner+"/"+repoName, "number", number, "error", err) } - return + return false } seen := c.prTerminalReviews.seen(key, now) c.prTerminalReviews.put(key, updatedAt, fresh, now) @@ -98,6 +98,7 @@ func (c *Client) emitTerminalPRReviews(ctx context.Context, owner, repoName, rep hook(r) } } + return true } // listTerminalPRReviews pages through a PR's reviews and keeps the submitted diff --git a/src/pkg/timeline/observation_test.go b/src/pkg/timeline/observation_test.go new file mode 100644 index 0000000000..5cab14d725 --- /dev/null +++ b/src/pkg/timeline/observation_test.go @@ -0,0 +1,188 @@ +package timeline + +import ( + "errors" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" +) + +const observationTestTimeout = 5 * time.Second + +func TestRecordObservationKeepsDistinctPRsAndTimeCorrections(t *testing.T) { + s := NewStore() + const ref = "org/repo#1" + first := Event{IssueRef: ref, Kind: KindMerged, At: 1234, Attrs: map[string]string{AttrPRRef: "org/repo#2"}} + s.RecordObservation(first) + other := first + other.Attrs = map[string]string{AttrPRRef: "org/repo#3"} + s.RecordObservation(other) + earlier := other + earlier.At-- + s.RecordObservation(earlier) + later := other + later.At++ + s.RecordObservation(later) + j, _ := s.Journey(ref) + st := j.Stages[KindMerged] + if st.Count != 4 || st.FirstAt != earlier.At || st.LastAt != later.At || st.Attrs[AttrPRRef] != "org/repo#3" { + t.Fatalf("distinct linked PR or corrected timestamp was deduplicated: %+v", st) + } +} + +func waitObservation(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(observationTestTimeout): + t.Fatal("timeline observation did not reach synchronization point") + } +} + +func TestRecordObservationPersistsFirstAndCorrectionsNotReplays(t *testing.T) { + path := filepath.Join(t.TempDir(), "timeline.json") + s := NewStore() + if err := s.EnablePersistence(path, nil); err != nil { + t.Fatal(err) + } + writes := 0 + s.writeFile = func(path string, data []byte, subject string) error { + writes++ + return writeFileAtomic(path, data, subject) + } + e := Event{IssueRef: "org/repo#1", Kind: KindMerged, At: 1234, Attrs: map[string]string{"actor": "unknown"}} + s.RecordObservation(e) + s.RecordObservation(e) + j, ok := s.Journey(e.IssueRef) + if !ok || j.Stages[KindMerged].Count != 1 || writes != 1 { + t.Fatalf("identical observation counted/wrote again: journey=%+v writes=%d", j, writes) + } + e.Attrs = map[string]string{"actor": "human"} + s.RecordObservation(e) + if writes != 2 { + t.Fatalf("corrected precious metadata was not durable: writes=%d", writes) + } + restarted := NewStore() + if err := restarted.EnablePersistence(path, nil); err != nil { + t.Fatal(err) + } + restarted.writeFile = func(string, []byte, string) error { + t.Error("unchanged restart replay persisted again") + return nil + } + restarted.RecordObservation(e) + j, _ = restarted.Journey(e.IssueRef) + if st := j.Stages[KindMerged]; st.Count != 2 || st.Attrs["actor"] != "human" || st.FirstAt != e.At { + t.Fatalf("first/corrected observation did not survive restart: %+v", st) + } + s.Record(e) + j, _ = s.Journey(e.IssueRef) + if j.Stages[KindMerged].Count != 3 || writes != 3 { + t.Fatal("ordinary Record lost its repeated-event semantics") + } +} + +func TestRecordObservationFailedPersistenceRetriesUnchangedEvidence(t *testing.T) { + s := NewStore() + if err := s.EnablePersistence(filepath.Join(t.TempDir(), "timeline.json"), nil); err != nil { + t.Fatal(err) + } + writes := 0 + s.writeFile = func(path string, data []byte, subject string) error { + writes++ + if writes == 1 { + return errors.New("injected historical write failure") + } + return writeFileAtomic(path, data, subject) + } + e := Event{IssueRef: "org/repo#1", Kind: KindMerged, At: 1234} + s.RecordObservation(e) + if !s.dirty { + t.Fatal("failed precious observation was acknowledged") + } + s.RecordObservation(e) + j, _ := s.Journey(e.IssueRef) + if writes != 2 || s.dirty || j.Stages[KindMerged].Count != 1 { + t.Fatalf("unchanged replay did not retry dirty persistence: writes=%d dirty=%v journey=%+v", writes, s.dirty, j) + } +} + +func TestSlowHistoricalPersistenceDoesNotHoldJourneyLockOrLoseConcurrentState(t *testing.T) { + s := NewStore() + path := filepath.Join(t.TempDir(), "timeline.json") + if err := s.EnablePersistence(path, nil); err != nil { + t.Fatal(err) + } + entered, release, recorded := make(chan struct{}), make(chan struct{}), make(chan struct{}) + defer func() { close(release); waitObservation(t, recorded) }() + var writes atomic.Int32 + s.writeFile = func(path string, data []byte, subject string) error { + if writes.Add(1) == 1 { + close(entered) + <-release + } + return writeFileAtomic(path, data, subject) + } + // Throttled non-precious current-work updates should not need a disk + // barrier. Precious historical events still force their first write. + s.lastPersist = time.Now() + historical := Event{IssueRef: "org/repo#1", Kind: KindMerged, At: 1234} + go func() { s.RecordObservation(historical); close(recorded) }() + waitObservation(t, entered) + current := Event{IssueRef: "org/repo#2", Kind: KindEnumerated, At: 5678} + serviced := make(chan struct{}) + go func() { + s.Record(current) + _, _ = s.Journey(historical.IssueRef) + close(serviced) + }() + waitObservation(t, serviced) + // Releasing the writer happens in the defer, after proving a blocked + // fsync/rename cannot hold the in-memory read/current-update mutex. +} + +func TestRecordObservationDoesNotAcknowledgeConcurrentUnsavedState(t *testing.T) { + s := NewStore() + if err := s.EnablePersistence(filepath.Join(t.TempDir(), "timeline.json"), nil); err != nil { + t.Fatal(err) + } + entered, release, recorded := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + defer func() { unblock(); waitObservation(t, recorded) }() + s.writeFile = func(path string, data []byte, subject string) error { + close(entered) + <-release + return writeFileAtomic(path, data, subject) + } + s.lastPersist = time.Now() + historical := Event{IssueRef: "org/repo#1", Kind: KindMerged, At: 1234} + go func() { s.RecordObservation(historical); close(recorded) }() + waitObservation(t, entered) + current := Event{IssueRef: "org/repo#2", Kind: KindEnumerated, At: 5678} + updated := make(chan struct{}) + go func() { s.Record(current); close(updated) }() + select { + case <-updated: + case <-time.After(observationTestTimeout): + unblock() + waitObservation(t, recorded) + t.Fatal("current throttled state blocked on historical persistence") + } + unblock() + waitObservation(t, recorded) + if !s.dirty { + t.Fatal("older historical snapshot acknowledged concurrent unsaved work") + } + s.writeFile = nil + s.RecordObservation(historical) + reloaded := NewStore() + if err := reloaded.EnablePersistence(s.path, nil); err != nil { + t.Fatal(err) + } + if _, ok := reloaded.Journey(current.IssueRef); !ok { + t.Fatal("retry lost the concurrent current-work journey") + } +} diff --git a/src/pkg/timeline/timeline.go b/src/pkg/timeline/timeline.go index ce9fbfa7c2..310b61c15b 100644 --- a/src/pkg/timeline/timeline.go +++ b/src/pkg/timeline/timeline.go @@ -238,9 +238,10 @@ type persistedTimeline struct { // // The zero value is NOT ready for use — construct with NewStore. type Store struct { - mu sync.RWMutex - journeys map[string]*Journey - cap int + mu sync.RWMutex + persistMu sync.Mutex + journeys map[string]*Journey + cap int // persistence (all guarded by mu) path string @@ -248,6 +249,9 @@ type Store struct { dirty bool lastPersist time.Time lastErrLog time.Time + // writeFile is nil in production; tests can block the atomic writer + // without making filesystem latency a scheduler-dependent assertion. + writeFile func(string, []byte, string) error } // NewStore returns a Store retaining up to MaxJourneys journeys. @@ -324,6 +328,18 @@ func (s *Store) EnablePersistence(path string, logger *slog.Logger) error { // (<= 0) it is stamped with the current time. Events with an empty IssueRef // or Kind carry no journey identity and are dropped. Safe for concurrent use. func (s *Store) Record(e Event) { + s.record(e, false) +} + +// RecordObservation folds immutable historical evidence into a journey without +// counting or persisting an unchanged replay. New stages, earlier/later times +// and metadata corrections retain Record's immediate precious-stage durability. +// A replay also retries dirty state left by a failed write. +func (s *Store) RecordObservation(e Event) { + s.record(e, e.At > 0) +} + +func (s *Store) record(e Event, observation bool) { if s == nil || e.IssueRef == "" || e.Kind == "" { return } @@ -336,6 +352,10 @@ func (s *Store) Record(e Event) { defer s.mu.Unlock() j := s.journeys[e.IssueRef] + if observation && observationUnchanged(j, e, at) { + s.persistMaybeLocked(e.Kind) + return + } if j == nil { s.enforceCapForOneMoreLocked() j = &Journey{ @@ -383,6 +403,23 @@ func (s *Store) Record(e Event) { s.persistMaybeLocked(e.Kind) } +func observationUnchanged(j *Journey, e Event, at int64) bool { + if j == nil { + return false + } + st := j.Stages[e.Kind] + if st == nil || at < st.FirstAt || at > st.LastAt || + (e.Agent != "" && (e.Agent != st.Agent || e.Agent != j.Agent)) { + return false + } + for k, v := range e.Attrs { + if old, ok := st.Attrs[k]; !ok || old != v { + return false + } + } + return true +} + // deriveCurrent computes a journey's current stage: merged is sticky-terminal; // blocked holds only while no progress stage fired at-or-after it; otherwise // the furthest progress stage reached. A journey with only unknown kinds falls @@ -937,6 +974,16 @@ func (s *Store) persistMaybeLocked(kind Kind) { if !force && time.Since(s.lastPersist) < persistMinInterval { return } + // Never hold the journey lock while waiting for another disk writer. + // Historical accounting may be slow, but readers/current-work status + // must still see the in-memory journeys. Writers remain serialized. + s.mu.Unlock() + s.persistMu.Lock() + s.mu.Lock() + defer s.persistMu.Unlock() + if !s.dirty || (!force && time.Since(s.lastPersist) < persistMinInterval) { + return + } if err := s.persistLocked(); err != nil { if time.Since(s.lastErrLog) >= persistErrLogInterval { s.lastErrLog = time.Now() @@ -947,7 +994,6 @@ func (s *Store) persistMaybeLocked(kind Kind) { } return } - s.dirty = false s.lastPersist = time.Now() } @@ -966,7 +1012,21 @@ func (s *Store) persistLocked() error { if err != nil { return fmt.Errorf("marshaling lifecycle timeline: %w", err) } - return writeFileAtomic(s.path, data, "lifecycle timeline") + path := s.path + write := s.writeFile + if write == nil { + write = writeFileAtomic + } + // Clear only this snapshot's dirty bit. Record may set it again while + // I/O is in flight; that newer state must not be acknowledged by this save. + s.dirty = false + s.mu.Unlock() + err = write(path, data, "lifecycle timeline") + s.mu.Lock() + if err != nil { + s.dirty = true + } + return err } // writeFileAtomic commits data to path with the repo's atomic-persist idiom