Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-bounded-terminal-pr-enrichment.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Keep historical terminal-PR accounting off current-work claim evaluation with cancellable bounded review batches, retained retries, and idempotent timeline persistence; preserve synchronous fail-closed claim safety and durable first observations.
1 change: 1 addition & 0 deletions src/cmd/hive/github_client_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,7 @@ func (b *boot) applyGitHubClientDashboardHooks(client *github.Client) {
client.SetPRReviewObservedHook(func(obs github.PRReviewObservation) {
recordObservedReview(b.dashSrv, obs)
})
client.StartPRTerminalEnrichment(b.ctx)
client.SetPROpenedHook(func(agentName, repo string, number int, url string) {
b.dashSrv.LinearAgentPROpened(agentName, repo, number, url)
// Same typed hook feeds the lifecycle timeline: the watcher fires
Expand Down
27 changes: 27 additions & 0 deletions src/cmd/hive/github_client_enrichment_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package main

import (
"context"
"testing"
"time"

"github.com/hivecommons/hive/pkg/github"
)

func TestAdoptGitHubClientRetiresHistoricalAccountingWorker(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
previous, next := &github.Client{}, &github.Client{}
done := previous.StartPRTerminalEnrichment(ctx)
b := &boot{ghClient: previous}
b.adoptGitHubClient(next, nil)
const stopTimeout = 5 * time.Second
select {
case <-done:
case <-time.After(stopTimeout):
t.Fatal("client rebuild left old historical accounting worker running")
}
if ctx.Err() != nil || b.currentGitHubClient() != next {
t.Fatal("retiring historical accounting disturbed boot context/client adoption")
}
}
3 changes: 3 additions & 0 deletions src/cmd/hive/github_client_provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ func (b *boot) adoptGitHubClient(newClient *github.Client, newAppAuth *github.Ap
if b == nil || newClient == nil {
return
}
if previous := b.currentGitHubClient(); previous != nil && previous != newClient {
previous.StopPRTerminalEnrichment()
}
b.ghClient = newClient
b.appAuth = newAppAuth
// Provider consumers (metrics and fleet-stats collectors, the scheduler's
Expand Down
20 changes: 15 additions & 5 deletions src/cmd/hive/lifecyclewire.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,12 @@ func recordMergedOnLinkedIssues(store *timeline.Store, prRef, agent string, at i
for k, v := range attrs {
copyAttrs[k] = v
}
store.Record(timeline.Event{IssueRef: issue, Kind: timeline.KindMerged, Agent: agent, At: at, Attrs: copyAttrs})
event := timeline.Event{IssueRef: issue, Kind: timeline.KindMerged, Agent: agent, At: at, Attrs: copyAttrs}
if at > 0 {
store.RecordObservation(event)
} else {
store.Record(event)
}
}
}

Expand All @@ -133,7 +138,7 @@ func recordObservedMerge(rec lifecycleRecorder, org, repo string, number int, ob
if !observedAt.IsZero() {
event.At = observedAt.UnixMilli()
}
store.Record(event)
store.RecordObservation(event)
recordMergedOnLinkedIssues(store, ref, "", event.At, attrs)
}

Expand Down Expand Up @@ -163,13 +168,18 @@ func recordPROpenedForIssuesAt(rec lifecycleRecorder, org, agent, repo string, n
if n <= 0 || n == number {
continue
}
store.Record(timeline.Event{
event := timeline.Event{
IssueRef: timelineRef(org, repo, n),
Kind: timeline.KindPROpened,
Agent: agent,
At: at,
Attrs: map[string]string{timeline.AttrPRRef: prRef, "url": url},
})
}
if at > 0 {
store.RecordObservation(event)
} else {
store.Record(event)
}
}
}

Expand Down Expand Up @@ -322,7 +332,7 @@ func backfillMergedPROnIssues(rec lifecycleRecorder, org string, obs github.PRTe
if obs.Actor != "" {
attrs["actor"] = obs.Actor
}
store.Record(timeline.Event{IssueRef: ref, Kind: timeline.KindMerged, At: mergedAt, Attrs: attrs})
store.RecordObservation(timeline.Event{IssueRef: ref, Kind: timeline.KindMerged, At: mergedAt, Attrs: attrs})
}
return len(needMerged)
}
Expand Down
29 changes: 29 additions & 0 deletions src/docs/audit-log.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,35 @@ what `gh` command) lives in the

## Format

### Historical PR observations and dispatch

The closed-PR settle scan still lists current claims and records claim/churn
history synchronously. A listing failure remains non-authoritative; deferred
accounting never means a claim is clear. Only historical merge actor lookup,
terminal audit/timeline observations, and terminal review reads run in the
client's single lifecycle-bound accounting worker.

The worker coalesces pending work by repository/PR and attempts at most eight
observations per batch, with a ten-second cancellable API budget and a
ten-second wake cadence. Failed work retains exponential retry backoff rather
than being silently discarded. These are accounting bounds, not a promise
that an uninterruptible filesystem sink finishes within ten seconds. A slow
sink occupies the accounting worker, not the claim evaluation call stack.
The `historical PR enrichment batch` log reports processed, failed, deferred,
duration, budget, and limit.

Pending accounting is in memory; after a restart or client credential rebuild
the existing 72-hour terminal settle scan replays recent observations.
This does not extend that historical
recovery window or make the accounting queue an ownership authority.
Timeline first sightings and corrected precious metadata still wait for atomic
persistence. Identical replay does not increment counts or rewrite the file;
dirty state after a failed save is retried. Atomic writers are serialized, but
disk I/O no longer holds the journey reader mutex, and an older snapshot cannot
acknowledge a concurrent unsaved update. New precious writers may still wait
for an earlier disk writer, preserving their durability rather than reporting
false completion.

One JSON object per line (JSONL), append-only. Written by
`pkg/dashboard/audit.go`.

Expand Down
1 change: 1 addition & 0 deletions src/pkg/github/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,7 @@ type Client struct {
// prreviews_observed.go.
prReviewObservedHook atomic.Pointer[PRReviewObservedHook]
prTerminalReviews prTerminalReviewCache
prTerminalEnrichment terminalEnrichmentState
prRepoPolicyGate atomic.Pointer[PRRepoPolicyGate]
// agentStartHook is told when an agent's own relay request shows it
// working an issue (#10527). See agent_start_hook.go.
Expand Down
215 changes: 215 additions & 0 deletions src/pkg/github/pr_terminal_enrichment.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
package github

import (
"context"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"

gh "github.com/google/go-github/v72/github"
)

const (
terminalEnrichmentInterval = 10 * time.Second
terminalEnrichmentBudget = 10 * time.Second
terminalEnrichmentBatch = 8
)

type terminalEnrichmentTask struct {
owner, repoName string
observation PRTerminalObservation
updatedAt time.Time
identity HiveIdentity
mergedBy string
sequence uint64
revision uint64
}

func (t terminalEnrichmentTask) key() string {
return strings.ToLower(t.owner+"/"+t.repoName) + "#" + strconv.Itoa(t.observation.Number)
}

func (t terminalEnrichmentTask) sameObservation(other terminalEnrichmentTask) bool {
a, b := t.observation, other.observation
return t.updatedAt.Equal(other.updatedAt) && t.identity == other.identity && t.mergedBy == other.mergedBy &&
a.Repo == b.Repo && a.Number == b.Number && a.State == b.State && a.ObservedAt.Equal(b.ObservedAt) &&
a.Actor == b.Actor && a.Attribution == b.Attribution && a.CreatedAt.Equal(b.CreatedAt) &&
a.URL == b.URL && slices.Equal(a.Issues, b.Issues)
}

// Only observational accounting is deferred. Claim listings, churn history,
// ledger persistence and fail-closed dispatch filtering never read this queue.
// Pending tasks survive failed batches and coalesce by PR, not by scan. On a
// process restart the existing terminal listing replays the settle window;
// timeline sinks must therefore remain idempotent and persist first sightings.
type terminalEnrichmentState struct {
mu sync.Mutex
started bool
cancel context.CancelFunc
done chan struct{}
wake chan struct{}
pending map[string]terminalEnrichmentTask
sequence uint64
retries retryTracker
}

// StartPRTerminalEnrichment isolates historical accounting from current-work
// evaluation. It starts at most one worker per client, using the same lifecycle
// context as the client's request watchers. Unstarted clients retain the
// synchronous observation API (including standalone callers and tests).
func (c *Client) StartPRTerminalEnrichment(ctx context.Context) <-chan struct{} {
if c == nil || ctx == nil {
done := make(chan struct{})
close(done)
return done
}
s := &c.prTerminalEnrichment
s.mu.Lock()
defer s.mu.Unlock()
if s.started {
return s.done
}
s.started = true
ctx, s.cancel = context.WithCancel(ctx)
s.done = make(chan struct{})
s.wake = make(chan struct{}, 1)
s.pending = make(map[string]terminalEnrichmentTask)
go func() {
defer close(s.done)
ticker := time.NewTicker(terminalEnrichmentInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
case <-s.wake:
}
if ctx.Err() != nil {
return
}
c.enrichTerminalPRBatch(ctx, terminalEnrichmentBudget, terminalEnrichmentBatch)
}
}()
return s.done
}

// StopPRTerminalEnrichment retires a replaced client's worker without waiting
// on an uninterruptible accounting sink. The replacement's settle scan replays
// recent pending history, just as on process restart.
func (c *Client) StopPRTerminalEnrichment() {
if c == nil {
return
}
s := &c.prTerminalEnrichment
s.mu.Lock()
defer s.mu.Unlock()
if s.cancel != nil {
s.cancel()
}
}

func (c *Client) observeTerminalPR(ctx context.Context, task terminalEnrichmentTask) {
s := &c.prTerminalEnrichment
s.mu.Lock()
if !s.started {
s.mu.Unlock()
c.enrichTerminalPR(ctx, task)
return
}
key := task.key()
old, exists := s.pending[key]
if exists && task.sameObservation(old) {
s.mu.Unlock()
return
}
if !exists || !task.updatedAt.Before(old.updatedAt) {
s.sequence++
task.revision = s.sequence
if exists {
task.sequence = old.sequence
} else {
task.sequence = s.sequence
}
s.pending[key] = task
}
// Wake only on an empty-to-nonempty transition. A listing must not
// repeatedly wake a failed/backed-off batch or bypass its bounded cadence.
if !exists && len(s.pending) == 1 {
select {
case s.wake <- struct{}{}:
default:
}
}
s.mu.Unlock()
}

func (c *Client) enrichTerminalPR(ctx context.Context, task terminalEnrichmentTask) bool {
if ctx.Err() != nil {
return false
}
obs := task.observation
if obs.State == PRStateMerged {
pr := &gh.PullRequest{Number: gh.Ptr(obs.Number), MergedBy: &gh.User{Login: gh.Ptr(task.mergedBy)}}
obs.Actor = c.prTerminalMergedBy(ctx, task.owner, task.repoName, pr)
obs.Attribution = prTerminalMergeAttribution(obs.Actor, task.identity, c.appBotLogin)
}
if ctx.Err() != nil {
return false
}
c.emitPRTerminalObserved(obs)
return c.emitTerminalPRReviews(ctx, task.owner, task.repoName, obs.Repo, obs.Number, task.updatedAt, task.identity)
}

func (c *Client) enrichTerminalPRBatch(ctx context.Context, budget time.Duration, limit int) {
started := time.Now()
ctx, cancel := context.WithTimeout(ctx, budget)
defer cancel()
s := &c.prTerminalEnrichment
s.mu.Lock()
tasks := make([]terminalEnrichmentTask, 0, len(s.pending))
for _, task := range s.pending {
tasks = append(tasks, task)
}
s.mu.Unlock()
sort.Slice(tasks, func(i, j int) bool { return tasks[i].sequence < tasks[j].sequence })
processed, failed := 0, 0
for _, task := range tasks {
key := task.key()
if ctx.Err() != nil || processed+failed >= limit {
break
}
if !s.retries.allows(key, time.Now()) {
continue
}
ok := c.enrichTerminalPR(ctx, task)
if ok {
processed++
s.retries.clear(key)
s.mu.Lock()
current := s.pending[key]
// A scan may have enqueued a newer terminal state/review while
// this request was in flight. Never acknowledge that newer work.
if current.revision == task.revision {
delete(s.pending, key)
}
s.mu.Unlock()
} else {
failed++
// Accounting is not discarded at the request-file quarantine
// horizon: a later successful batch must still record it.
s.retries.noteFailure(key, time.Now())
}
}
s.mu.Lock()
pending := len(s.pending)
s.mu.Unlock()
if c.logger != nil && (processed+failed > 0) {
c.logger.Info("historical PR enrichment batch",
"processed", processed, "failed", failed, "deferred", pending,
"duration", time.Since(started), "budget", budget, "limit", limit)
}
}
Loading
Loading