diff --git a/changelog.d/fixed-16292-transcript-scan-overrun.md b/changelog.d/fixed-16292-transcript-scan-overrun.md new file mode 100644 index 0000000000..97119aec27 --- /dev/null +++ b/changelog.d/fixed-16292-transcript-scan-overrun.md @@ -0,0 +1 @@ +- The copilot transcript activity scan overrun warning is now debounced: it fires once after three consecutive passes longer than the scan interval and re-arms only after a pass under 80% of it, so a marginal overrun on a many-agent hive no longer logs a WARN every ~25s and trips the fleet report ([#16292](https://github.com/hivecommons/hive/issues/16292)). diff --git a/src/crux/agent/busy_visibility.go b/src/crux/agent/busy_visibility.go index 3be89cf659..add112ea11 100644 --- a/src/crux/agent/busy_visibility.go +++ b/src/crux/agent/busy_visibility.go @@ -407,14 +407,14 @@ func (m *Manager) StartTranscriptActivityRefresh(ctx context.Context) { // tick; scans never overlap because they run on this goroutine alone. func (m *Manager) runTranscriptActivityRefresh(ctx context.Context, ticks <-chan time.Time, now func() time.Time) { caches := map[string]*transcriptSessionCache{} - slow := false - m.refreshTranscriptActivity(caches, now, &slow) + pace := &transcriptScanPace{} + m.refreshTranscriptActivity(caches, now, pace) for { select { case <-ctx.Done(): return case <-ticks: - m.refreshTranscriptActivity(caches, now, &slow) + m.refreshTranscriptActivity(caches, now, pace) } } } @@ -422,9 +422,9 @@ func (m *Manager) runTranscriptActivityRefresh(ctx context.Context, ticks <-chan // refreshTranscriptActivity runs one scan pass: pick the stale agents under // a short read lock, stat their sessions with no lock held, then publish // the results under a short write lock. Returns how many agents were -// scanned. slow tracks the previous pass's verdict so a walk that outlives -// the scan interval is logged on the transition, not every tick. -func (m *Manager) refreshTranscriptActivity(caches map[string]*transcriptSessionCache, now func() time.Time, slow *bool) int { +// scanned. pace debounces the overrun warning: it fires once after +// several consecutive slow passes, and re-arms only after a clearly fast one. +func (m *Manager) refreshTranscriptActivity(caches map[string]*transcriptSessionCache, now func() time.Time, pace *transcriptScanPace) int { started := now() m.mu.RLock() scans := m.pendingTranscriptActivityScansLocked(started) @@ -445,16 +445,10 @@ func (m *Manager) refreshTranscriptActivity(caches map[string]*transcriptSession delete(caches, name) } } - if took := now().Sub(started); took > transcriptLivenessScanInterval { - if slow != nil && !*slow && m.logger != nil { - m.logger.Warn("copilot transcript activity scan outlived its interval; agent liveness reads may lag", - "took", took.Round(time.Millisecond).String(), "agents", len(scans), "interval", transcriptLivenessScanInterval.String()) - } - if slow != nil { - *slow = true - } - } else if slow != nil { - *slow = false + took := now().Sub(started) + if pace != nil && pace.observe(took, transcriptLivenessScanInterval) && m.logger != nil { + m.logger.Warn("copilot transcript activity scan outlived its interval; agent liveness reads may lag", + "took", took.Round(time.Millisecond).String(), "agents", len(scans), "interval", transcriptLivenessScanInterval.String()) } return len(results) } diff --git a/src/crux/agent/transcript_scan_pace.go b/src/crux/agent/transcript_scan_pace.go new file mode 100644 index 0000000000..3fa29bf76c --- /dev/null +++ b/src/crux/agent/transcript_scan_pace.go @@ -0,0 +1,40 @@ +package agent + +import "time" + +const ( + // transcriptScanSlowPassesToWarn is how many consecutive passes must + // outlive the scan interval before the overrun is logged. + transcriptScanSlowPassesToWarn = 3 + // transcriptScanRecoverPercent is the share of the scan interval a pass + // must finish under to end a slow episode. The gap between this and 100% + // keeps a marginal overrun from flapping the episode on and off. + transcriptScanRecoverPercent = 80 +) + +// transcriptScanPace tracks scan pass durations so the overrun warning fires +// once per slow episode instead of on every fast-to-slow flip. +type transcriptScanPace struct { + slowRun int + warned bool +} + +// observe records one pass and reports whether to log the overrun now. +func (p *transcriptScanPace) observe(took, interval time.Duration) bool { + switch { + case took > interval: + p.slowRun++ + if p.slowRun >= transcriptScanSlowPassesToWarn && !p.warned { + p.warned = true + return true + } + case took*100 < interval*transcriptScanRecoverPercent: + p.slowRun = 0 + p.warned = false + default: + // Marginal pass: breaks the consecutive-slow streak without + // re-arming the warning, so flapping around the interval stays quiet. + p.slowRun = 0 + } + return false +} diff --git a/src/crux/agent/transcript_scan_pace_test.go b/src/crux/agent/transcript_scan_pace_test.go new file mode 100644 index 0000000000..5abbea0075 --- /dev/null +++ b/src/crux/agent/transcript_scan_pace_test.go @@ -0,0 +1,49 @@ +package agent + +import ( + "bytes" + "log/slog" + "strings" + "testing" + "time" +) + +func TestTranscriptScanOverrunWarningHysteresis(t *testing.T) { + cases := []struct { + name string + takes []time.Duration + want int + }{ + {"marginal flapping", []time.Duration{10700 * time.Millisecond, 9500 * time.Millisecond, 10700 * time.Millisecond, 9500 * time.Millisecond, 10700 * time.Millisecond, 9500 * time.Millisecond, 10700 * time.Millisecond, 9500 * time.Millisecond}, 0}, + {"sustained overrun warns once", []time.Duration{11 * time.Second, 11 * time.Second, 11 * time.Second, 11 * time.Second, 11 * time.Second}, 1}, + {"two slow passes stay quiet", []time.Duration{11 * time.Second, 11 * time.Second, 2 * time.Second}, 0}, + {"clear fast pass re-arms", []time.Duration{11 * time.Second, 11 * time.Second, 11 * time.Second, 2 * time.Second, 11 * time.Second, 11 * time.Second, 11 * time.Second}, 2}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + m, _ := transcriptRefreshTestManager(t) + m.logger = slog.New(slog.NewTextHandler(&buf, nil)) + base := time.Date(2026, 11, 2, 12, 0, 0, 0, time.UTC) + caches := map[string]*transcriptSessionCache{} + pace := &transcriptScanPace{} + for i, took := range tc.takes { + start := base.Add(time.Duration(i) * time.Minute) + calls := 0 + clock := func() time.Time { + calls++ + if calls == 1 { + return start + } + return start.Add(took) + } + if n := m.refreshTranscriptActivity(caches, clock, pace); n != 1 { + t.Fatalf("pass %d scanned %d agents, want 1", i, n) + } + } + if got := strings.Count(buf.String(), "transcript activity scan outlived its interval"); got != tc.want { + t.Fatalf("warnings = %d, want %d\n%s", got, tc.want, buf.String()) + } + }) + } +} diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 806d596738..da5bd9b132 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -159,7 +159,7 @@ title is `[selfreport] : `. | `proxy_plain_forward_refused` | the hive proxy refused a plain-HTTP forward to a non-globally-routable destination (`proxy: plain HTTP forward refused`, `crux/proxy/github_proxy_connect.go`) — a client ignored its proxy bypass for a private address (e.g. the Docker bridge gateway). Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: fix the client's `NO_PROXY` or target address; the refusal is counted as a proxy violation | low | | `sha_poll_commit_message_unavailable` | the SHA poller's commit-message fetch got a non-200 from GitHub (typically a 403 rate limit or token scope; `SHA poll: commit message fetch non-200`, `pkg/hub/saas_sha_poller.go`). The poller backs off per SHA and keeps serving the SHA without a message. The WARN itself is deduped to once per hour per (SHA, status) (`warnGitHubFetchOnce`), so one stuck commit logs one line and never trips this class; it is reported only when ≥5 distinct SHA/status pairs fail within 10 min (a broad outage such as an exhausted rate limit or a revoked token). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `tmux_session_check_failed` | the agent's tmux session `has-session` check failed (`tmux has-session failed`, `crux/agent/manager.go`) — typically the session was lost or the tmux server is unhealthy, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | -| `transcript_scan_slow` | a Copilot transcript activity scan took longer than its interval (`transcript activity scan outlived its interval`, `crux/agent/busy_visibility.go`) — agent liveness reads may lag. Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: check the agent's transcript/session volume; it is bounded by concurrency, so repeated hits indicate slow disk or very large session directories | low | +| `transcript_scan_slow` | a Copilot transcript activity scan took longer than its interval (`transcript activity scan outlived its interval`, `crux/agent/busy_visibility.go`) — agent liveness reads may lag. The WARN itself is debounced at the source (`transcriptScanPace`, `crux/agent/transcript_scan_pace.go`): it fires once per slow episode, only after 3 consecutive passes outlive the interval, and re-arms only after a pass finishes under 80% of the interval, so one sustained overrun logs one line and never trips this class; it is reported only when ≥5 distinct slow→fast→slow episodes occur within 10 min. Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: check the agent's transcript/session volume; it is bounded by concurrency, so repeated episodes indicate slow disk or very large session directories | low | | `kick_freshness_withheld` | a kick was withheld because every issue/PR ref failed the pre-kick freshness check (`kick withheld: every ref failed the freshness check`, `cmd/hive/kick_freshness.go`). The kick is retried on the next cycle; reported only when the same withholding repeats ≥5 times in 10 min (refs persistently stale or unreadable). Deferred API-budget refusals log at INFO and never trip it. Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `unclassified_error_repeated` | **generic catch-all**: the same WARN/ERROR log line (numbers, hashes, paths and quoted strings collapsed) repeated ≥5 times in 10 min. Fed by a `slog.Handler` tee (`pkg/fleetreport/slogfeed`) on the hive's root logger, so a repeating defect is noticed even with no hook site. Skips benign lines (read/keepalive i/o timeouts, WARNs whose `error` is a GitHub rate-limit refusal or a 502/503/504 gateway failure — the caller retries next tick, context cancellation, the permissions watcher's `fixed …` self-healing lines) and any message that already matches another class's keywords. Clears automatically after 20 quiet minutes (2× the window), so a "cleared" comment follows once the symptom stops | low | diff --git a/src/docs/notifications.md b/src/docs/notifications.md index c611aec27c..a905bdcb16 100644 --- a/src/docs/notifications.md +++ b/src/docs/notifications.md @@ -414,6 +414,7 @@ without a class. Review submissions and review requests are always counted as | `github.(*Client).PostNoteOrFold` | `status` | passthrough wrapper; attributed to its caller | | `github.(*Client).SweepLegacyFleetReportIssues` | `status` | | | `github.(*Client).UpsertIssueMarkerComment` | `status` | passthrough wrapper; attributed to its caller | +| `github.(*clientStatusForge).CreateComment` | `status` | passthrough wrapper (prstatus.Write status-comment creation); attributed to its caller | | `github.(*Client).closeDecomposedParent` | `status` | | | `github.(*Client).closeIssueForMergedPRClaimWithIssue` | `status` | | | `github.(*Client).closeSupersededPRDecision` | `status` | | diff --git a/src/pkg/github/notification_write_sites.go b/src/pkg/github/notification_write_sites.go index b764528475..b0b42313f6 100644 --- a/src/pkg/github/notification_write_sites.go +++ b/src/pkg/github/notification_write_sites.go @@ -27,6 +27,7 @@ var notificationWriteSites = map[string]NotificationWriteSite{ "main.(*repoForgeRouter).CreateIssueComment": {Class: NotificationClassAttention, Passthrough: true}, "main.liveTriageCommenter.CreateIssueComment": {Class: NotificationClassAttention, Passthrough: true}, "github.(*Client).UpsertIssueMarkerComment": {Class: NotificationClassStatus, Passthrough: true}, + "github.(*clientStatusForge).CreateComment": {Class: NotificationClassStatus, Passthrough: true}, // status: bookkeeping readable on demand. "dashboard.(*ContributeWSHub).markAlreadyDoneIssue": {Class: NotificationClassStatus}, diff --git a/src/pkg/writeback/testdata/direct-writes-baseline.txt b/src/pkg/writeback/testdata/direct-writes-baseline.txt index 670a7f79ba..fd121804ef 100644 --- a/src/pkg/writeback/testdata/direct-writes-baseline.txt +++ b/src/pkg/writeback/testdata/direct-writes-baseline.txt @@ -15,7 +15,7 @@ raw pkg/github/automerge/automerge_held_runs.go 3 raw pkg/github/automerge/automerge_merge_commit.go 1 raw pkg/github/automerge/automerge_merge_forward.go 1 raw pkg/github/automerge/automerge_stage_human.go 1 -raw pkg/github/automerge/automerge_stage_labels.go 3 +raw pkg/github/automerge/automerge_stage_labels.go 5 raw pkg/github/automerge/automerge_sweep_queued.go 3 raw pkg/github/automerge/automerge_sweep_trusted.go 4 raw pkg/github/automerge/automerge_unauthorized.go 3 @@ -38,6 +38,7 @@ raw pkg/github/pr_contributor_link.go 1 raw pkg/github/pr_issue_link.go 2 raw pkg/github/pr_level_hold.go 5 raw pkg/github/pr_signed_reconcile.go 1 +raw pkg/github/pr_status_comment.go 2 raw pkg/github/pr_supersession_autoclose.go 3 raw pkg/github/pr_supersession_resolve.go 1 raw pkg/github/pr_supersession_sweep.go 2 @@ -106,7 +107,7 @@ wrap pkg/github/pr_duplicate_guard.go 1 wrap pkg/github/pr_level_hold.go 1 wrap pkg/github/pr_reporter_trust_release.go 1 wrap pkg/github/pr_request_watcher.go 3 -wrap pkg/github/pr_status_comment.go 1 +wrap pkg/github/pr_status_comment.go 2 wrap pkg/github/pr_supersession_autoclose.go 1 wrap pkg/github/pr_supersession_decision.go 1 wrap pkg/github/pr_supersession_resolve.go 6