From fd16d219dc5e3edea84e31266aeeae3a42a33ffc Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:18:53 -0500 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=90=9B=20fix(agent):=20add=20hysteres?= =?UTF-8?q?is=20to=20the=20transcript=20scan=20overrun=20warning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../fixed-16292-transcript-scan-overrun.md | 1 + src/crux/agent/busy_visibility.go | 25 ++++------ src/crux/agent/transcript_scan_pace.go | 36 ++++++++++++++ src/crux/agent/transcript_scan_pace_test.go | 49 +++++++++++++++++++ 4 files changed, 95 insertions(+), 16 deletions(-) create mode 100644 changelog.d/fixed-16292-transcript-scan-overrun.md create mode 100644 src/crux/agent/transcript_scan_pace.go create mode 100644 src/crux/agent/transcript_scan_pace_test.go diff --git a/changelog.d/fixed-16292-transcript-scan-overrun.md b/changelog.d/fixed-16292-transcript-scan-overrun.md new file mode 100644 index 0000000000..97119aec27 --- /dev/null +++ b/changelog.d/fixed-16292-transcript-scan-overrun.md @@ -0,0 +1 @@ +- The copilot transcript activity scan overrun warning is now debounced: it fires once after three consecutive passes longer than the scan interval and re-arms only after a pass under 80% of it, so a marginal overrun on a many-agent hive no longer logs a WARN every ~25s and trips the fleet report ([#16292](https://github.com/hivecommons/hive/issues/16292)). diff --git a/src/crux/agent/busy_visibility.go b/src/crux/agent/busy_visibility.go index 3be89cf659..cddc9b289f 100644 --- a/src/crux/agent/busy_visibility.go +++ b/src/crux/agent/busy_visibility.go @@ -407,14 +407,14 @@ func (m *Manager) StartTranscriptActivityRefresh(ctx context.Context) { // tick; scans never overlap because they run on this goroutine alone. func (m *Manager) runTranscriptActivityRefresh(ctx context.Context, ticks <-chan time.Time, now func() time.Time) { caches := map[string]*transcriptSessionCache{} - slow := false - m.refreshTranscriptActivity(caches, now, &slow) + slow := &transcriptScanPace{} + m.refreshTranscriptActivity(caches, now, slow) for { select { case <-ctx.Done(): return case <-ticks: - m.refreshTranscriptActivity(caches, now, &slow) + m.refreshTranscriptActivity(caches, now, slow) } } } @@ -422,9 +422,9 @@ func (m *Manager) runTranscriptActivityRefresh(ctx context.Context, ticks <-chan // refreshTranscriptActivity runs one scan pass: pick the stale agents under // a short read lock, stat their sessions with no lock held, then publish // the results under a short write lock. Returns how many agents were -// scanned. slow tracks the previous pass's verdict so a walk that outlives -// the scan interval is logged on the transition, not every tick. -func (m *Manager) refreshTranscriptActivity(caches map[string]*transcriptSessionCache, now func() time.Time, slow *bool) int { +// scanned. slow debounces the overrun warning: it fires once after +// several consecutive slow passes, and re-arms only after a clearly fast one. +func (m *Manager) refreshTranscriptActivity(caches map[string]*transcriptSessionCache, now func() time.Time, slow *transcriptScanPace) int { started := now() m.mu.RLock() scans := m.pendingTranscriptActivityScansLocked(started) @@ -445,16 +445,9 @@ func (m *Manager) refreshTranscriptActivity(caches map[string]*transcriptSession delete(caches, name) } } - if took := now().Sub(started); took > transcriptLivenessScanInterval { - if slow != nil && !*slow && m.logger != nil { - m.logger.Warn("copilot transcript activity scan outlived its interval; agent liveness reads may lag", - "took", took.Round(time.Millisecond).String(), "agents", len(scans), "interval", transcriptLivenessScanInterval.String()) - } - if slow != nil { - *slow = true - } - } else if slow != nil { - *slow = false + if slow != nil && slow.observe(now().Sub(started), transcriptLivenessScanInterval) && m.logger != nil { + m.logger.Warn("copilot transcript activity scan outlived its interval; agent liveness reads may lag", + "took", now().Sub(started).Round(time.Millisecond).String(), "agents", len(scans), "interval", transcriptLivenessScanInterval.String()) } return len(results) } diff --git a/src/crux/agent/transcript_scan_pace.go b/src/crux/agent/transcript_scan_pace.go new file mode 100644 index 0000000000..7db858302e --- /dev/null +++ b/src/crux/agent/transcript_scan_pace.go @@ -0,0 +1,36 @@ +package agent + +import "time" + +const ( + // transcriptScanSlowPassesToWarn is how many consecutive passes must + // outlive the scan interval before the overrun is logged. + transcriptScanSlowPassesToWarn = 3 + // transcriptScanRecoverPercent is the share of the scan interval a pass + // must finish under to end a slow episode. The gap between this and 100% + // keeps a marginal overrun from flapping the episode on and off. + transcriptScanRecoverPercent = 80 +) + +// transcriptScanPace tracks scan pass durations so the overrun warning fires +// once per slow episode instead of on every fast-to-slow flip. +type transcriptScanPace struct { + slowRun int + warned bool +} + +// observe records one pass and reports whether to log the overrun now. +func (p *transcriptScanPace) observe(took, interval time.Duration) bool { + switch { + case took > interval: + p.slowRun++ + if p.slowRun >= transcriptScanSlowPassesToWarn && !p.warned { + p.warned = true + return true + } + case took*100 < interval*transcriptScanRecoverPercent: + p.slowRun = 0 + p.warned = false + } + return false +} diff --git a/src/crux/agent/transcript_scan_pace_test.go b/src/crux/agent/transcript_scan_pace_test.go new file mode 100644 index 0000000000..5abbea0075 --- /dev/null +++ b/src/crux/agent/transcript_scan_pace_test.go @@ -0,0 +1,49 @@ +package agent + +import ( + "bytes" + "log/slog" + "strings" + "testing" + "time" +) + +func TestTranscriptScanOverrunWarningHysteresis(t *testing.T) { + cases := []struct { + name string + takes []time.Duration + want int + }{ + {"marginal flapping", []time.Duration{10700 * time.Millisecond, 9500 * time.Millisecond, 10700 * time.Millisecond, 9500 * time.Millisecond, 10700 * time.Millisecond, 9500 * time.Millisecond, 10700 * time.Millisecond, 9500 * time.Millisecond}, 0}, + {"sustained overrun warns once", []time.Duration{11 * time.Second, 11 * time.Second, 11 * time.Second, 11 * time.Second, 11 * time.Second}, 1}, + {"two slow passes stay quiet", []time.Duration{11 * time.Second, 11 * time.Second, 2 * time.Second}, 0}, + {"clear fast pass re-arms", []time.Duration{11 * time.Second, 11 * time.Second, 11 * time.Second, 2 * time.Second, 11 * time.Second, 11 * time.Second, 11 * time.Second}, 2}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + m, _ := transcriptRefreshTestManager(t) + m.logger = slog.New(slog.NewTextHandler(&buf, nil)) + base := time.Date(2026, 11, 2, 12, 0, 0, 0, time.UTC) + caches := map[string]*transcriptSessionCache{} + pace := &transcriptScanPace{} + for i, took := range tc.takes { + start := base.Add(time.Duration(i) * time.Minute) + calls := 0 + clock := func() time.Time { + calls++ + if calls == 1 { + return start + } + return start.Add(took) + } + if n := m.refreshTranscriptActivity(caches, clock, pace); n != 1 { + t.Fatalf("pass %d scanned %d agents, want 1", i, n) + } + } + if got := strings.Count(buf.String(), "transcript activity scan outlived its interval"); got != tc.want { + t.Fatalf("warnings = %d, want %d\n%s", got, tc.want, buf.String()) + } + }) + } +} From 7f35ec4eacdff219bcd076e7900d99481994731b Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:39:33 -0500 Subject: [PATCH 2/4] fix(agent): marginal scan passes break the slow streak without re-arming Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/crux/agent/transcript_scan_pace.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/crux/agent/transcript_scan_pace.go b/src/crux/agent/transcript_scan_pace.go index 7db858302e..3fa29bf76c 100644 --- a/src/crux/agent/transcript_scan_pace.go +++ b/src/crux/agent/transcript_scan_pace.go @@ -31,6 +31,10 @@ func (p *transcriptScanPace) observe(took, interval time.Duration) bool { case took*100 < interval*transcriptScanRecoverPercent: p.slowRun = 0 p.warned = false + default: + // Marginal pass: breaks the consecutive-slow streak without + // re-arming the warning, so flapping around the interval stays quiet. + p.slowRun = 0 } return false } From 03d4bff9b2dfea7fb73c6b4ec304e52616c9e55b Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 18:36:15 -0500 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=93=9D=20docs(agent):=20document=20tr?= =?UTF-8?q?anscript=5Fscan=5Fslow=20debounce;=20hoist=20took=20in=20refres?= =?UTF-8?q?hTranscriptActivity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review findings on #16310: the fleet-report class table now describes the once-per-episode / 3-consecutive / 80% re-arm behavior so operators can interpret the ≥5-in-10-min threshold. Compute took once so the judged and logged durations match, and rename the pace parameter to match its type and the tests. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/crux/agent/busy_visibility.go | 15 ++++++++------- src/docs/fleet-report.md | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/crux/agent/busy_visibility.go b/src/crux/agent/busy_visibility.go index cddc9b289f..add112ea11 100644 --- a/src/crux/agent/busy_visibility.go +++ b/src/crux/agent/busy_visibility.go @@ -407,14 +407,14 @@ func (m *Manager) StartTranscriptActivityRefresh(ctx context.Context) { // tick; scans never overlap because they run on this goroutine alone. func (m *Manager) runTranscriptActivityRefresh(ctx context.Context, ticks <-chan time.Time, now func() time.Time) { caches := map[string]*transcriptSessionCache{} - slow := &transcriptScanPace{} - m.refreshTranscriptActivity(caches, now, slow) + pace := &transcriptScanPace{} + m.refreshTranscriptActivity(caches, now, pace) for { select { case <-ctx.Done(): return case <-ticks: - m.refreshTranscriptActivity(caches, now, slow) + m.refreshTranscriptActivity(caches, now, pace) } } } @@ -422,9 +422,9 @@ func (m *Manager) runTranscriptActivityRefresh(ctx context.Context, ticks <-chan // refreshTranscriptActivity runs one scan pass: pick the stale agents under // a short read lock, stat their sessions with no lock held, then publish // the results under a short write lock. Returns how many agents were -// scanned. slow debounces the overrun warning: it fires once after +// scanned. pace debounces the overrun warning: it fires once after // several consecutive slow passes, and re-arms only after a clearly fast one. -func (m *Manager) refreshTranscriptActivity(caches map[string]*transcriptSessionCache, now func() time.Time, slow *transcriptScanPace) int { +func (m *Manager) refreshTranscriptActivity(caches map[string]*transcriptSessionCache, now func() time.Time, pace *transcriptScanPace) int { started := now() m.mu.RLock() scans := m.pendingTranscriptActivityScansLocked(started) @@ -445,9 +445,10 @@ func (m *Manager) refreshTranscriptActivity(caches map[string]*transcriptSession delete(caches, name) } } - if slow != nil && slow.observe(now().Sub(started), transcriptLivenessScanInterval) && m.logger != nil { + took := now().Sub(started) + if pace != nil && pace.observe(took, transcriptLivenessScanInterval) && m.logger != nil { m.logger.Warn("copilot transcript activity scan outlived its interval; agent liveness reads may lag", - "took", now().Sub(started).Round(time.Millisecond).String(), "agents", len(scans), "interval", transcriptLivenessScanInterval.String()) + "took", took.Round(time.Millisecond).String(), "agents", len(scans), "interval", transcriptLivenessScanInterval.String()) } return len(results) } diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 806d596738..da5bd9b132 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -159,7 +159,7 @@ title is `[selfreport] : `. | `proxy_plain_forward_refused` | the hive proxy refused a plain-HTTP forward to a non-globally-routable destination (`proxy: plain HTTP forward refused`, `crux/proxy/github_proxy_connect.go`) — a client ignored its proxy bypass for a private address (e.g. the Docker bridge gateway). Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: fix the client's `NO_PROXY` or target address; the refusal is counted as a proxy violation | low | | `sha_poll_commit_message_unavailable` | the SHA poller's commit-message fetch got a non-200 from GitHub (typically a 403 rate limit or token scope; `SHA poll: commit message fetch non-200`, `pkg/hub/saas_sha_poller.go`). The poller backs off per SHA and keeps serving the SHA without a message. The WARN itself is deduped to once per hour per (SHA, status) (`warnGitHubFetchOnce`), so one stuck commit logs one line and never trips this class; it is reported only when ≥5 distinct SHA/status pairs fail within 10 min (a broad outage such as an exhausted rate limit or a revoked token). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `tmux_session_check_failed` | the agent's tmux session `has-session` check failed (`tmux has-session failed`, `crux/agent/manager.go`) — typically the session was lost or the tmux server is unhealthy, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | -| `transcript_scan_slow` | a Copilot transcript activity scan took longer than its interval (`transcript activity scan outlived its interval`, `crux/agent/busy_visibility.go`) — agent liveness reads may lag. Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: check the agent's transcript/session volume; it is bounded by concurrency, so repeated hits indicate slow disk or very large session directories | low | +| `transcript_scan_slow` | a Copilot transcript activity scan took longer than its interval (`transcript activity scan outlived its interval`, `crux/agent/busy_visibility.go`) — agent liveness reads may lag. The WARN itself is debounced at the source (`transcriptScanPace`, `crux/agent/transcript_scan_pace.go`): it fires once per slow episode, only after 3 consecutive passes outlive the interval, and re-arms only after a pass finishes under 80% of the interval, so one sustained overrun logs one line and never trips this class; it is reported only when ≥5 distinct slow→fast→slow episodes occur within 10 min. Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: check the agent's transcript/session volume; it is bounded by concurrency, so repeated episodes indicate slow disk or very large session directories | low | | `kick_freshness_withheld` | a kick was withheld because every issue/PR ref failed the pre-kick freshness check (`kick withheld: every ref failed the freshness check`, `cmd/hive/kick_freshness.go`). The kick is retried on the next cycle; reported only when the same withholding repeats ≥5 times in 10 min (refs persistently stale or unreadable). Deferred API-budget refusals log at INFO and never trip it. Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `unclassified_error_repeated` | **generic catch-all**: the same WARN/ERROR log line (numbers, hashes, paths and quoted strings collapsed) repeated ≥5 times in 10 min. Fed by a `slog.Handler` tee (`pkg/fleetreport/slogfeed`) on the hive's root logger, so a repeating defect is noticed even with no hook site. Skips benign lines (read/keepalive i/o timeouts, WARNs whose `error` is a GitHub rate-limit refusal or a 502/503/504 gateway failure — the caller retries next tick, context cancellation, the permissions watcher's `fixed …` self-healing lines) and any message that already matches another class's keywords. Clears automatically after 20 quiet minutes (2× the window), so a "cleared" comment follows once the symptom stops | low | From 2a454c494caaf221de7678394d54205f9458068e Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:40:07 -0500 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=A7=AA=20test(ci):=20reconcile=20writ?= =?UTF-8?q?e=20ratchets=20with=20the=20prstatus=20status=20writer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v5 went red after #16319 and #16321 landed together: the prstatus forge adapters added raw/wrapper comment writes in pr_status_comment.go and automerge_stage_labels.go that TestDirectWriteRatchet had no baseline for, and clientStatusForge.CreateComment is a new CreateIssueComment call site TestNotificationWriteSitesInventory requires a class tag for. Record the moved status-comment writes in the ratchet baseline, and inventory clientStatusForge.CreateComment as a status passthrough (attributed to the UpsertPRStatusSection / PostNoteOrFold caller), with the matching row in docs/notifications.md. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/docs/notifications.md | 1 + src/pkg/github/notification_write_sites.go | 1 + src/pkg/writeback/testdata/direct-writes-baseline.txt | 5 +++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/docs/notifications.md b/src/docs/notifications.md index c611aec27c..a905bdcb16 100644 --- a/src/docs/notifications.md +++ b/src/docs/notifications.md @@ -414,6 +414,7 @@ without a class. Review submissions and review requests are always counted as | `github.(*Client).PostNoteOrFold` | `status` | passthrough wrapper; attributed to its caller | | `github.(*Client).SweepLegacyFleetReportIssues` | `status` | | | `github.(*Client).UpsertIssueMarkerComment` | `status` | passthrough wrapper; attributed to its caller | +| `github.(*clientStatusForge).CreateComment` | `status` | passthrough wrapper (prstatus.Write status-comment creation); attributed to its caller | | `github.(*Client).closeDecomposedParent` | `status` | | | `github.(*Client).closeIssueForMergedPRClaimWithIssue` | `status` | | | `github.(*Client).closeSupersededPRDecision` | `status` | | diff --git a/src/pkg/github/notification_write_sites.go b/src/pkg/github/notification_write_sites.go index b764528475..b0b42313f6 100644 --- a/src/pkg/github/notification_write_sites.go +++ b/src/pkg/github/notification_write_sites.go @@ -27,6 +27,7 @@ var notificationWriteSites = map[string]NotificationWriteSite{ "main.(*repoForgeRouter).CreateIssueComment": {Class: NotificationClassAttention, Passthrough: true}, "main.liveTriageCommenter.CreateIssueComment": {Class: NotificationClassAttention, Passthrough: true}, "github.(*Client).UpsertIssueMarkerComment": {Class: NotificationClassStatus, Passthrough: true}, + "github.(*clientStatusForge).CreateComment": {Class: NotificationClassStatus, Passthrough: true}, // status: bookkeeping readable on demand. "dashboard.(*ContributeWSHub).markAlreadyDoneIssue": {Class: NotificationClassStatus}, diff --git a/src/pkg/writeback/testdata/direct-writes-baseline.txt b/src/pkg/writeback/testdata/direct-writes-baseline.txt index 670a7f79ba..fd121804ef 100644 --- a/src/pkg/writeback/testdata/direct-writes-baseline.txt +++ b/src/pkg/writeback/testdata/direct-writes-baseline.txt @@ -15,7 +15,7 @@ raw pkg/github/automerge/automerge_held_runs.go 3 raw pkg/github/automerge/automerge_merge_commit.go 1 raw pkg/github/automerge/automerge_merge_forward.go 1 raw pkg/github/automerge/automerge_stage_human.go 1 -raw pkg/github/automerge/automerge_stage_labels.go 3 +raw pkg/github/automerge/automerge_stage_labels.go 5 raw pkg/github/automerge/automerge_sweep_queued.go 3 raw pkg/github/automerge/automerge_sweep_trusted.go 4 raw pkg/github/automerge/automerge_unauthorized.go 3 @@ -38,6 +38,7 @@ raw pkg/github/pr_contributor_link.go 1 raw pkg/github/pr_issue_link.go 2 raw pkg/github/pr_level_hold.go 5 raw pkg/github/pr_signed_reconcile.go 1 +raw pkg/github/pr_status_comment.go 2 raw pkg/github/pr_supersession_autoclose.go 3 raw pkg/github/pr_supersession_resolve.go 1 raw pkg/github/pr_supersession_sweep.go 2 @@ -106,7 +107,7 @@ wrap pkg/github/pr_duplicate_guard.go 1 wrap pkg/github/pr_level_hold.go 1 wrap pkg/github/pr_reporter_trust_release.go 1 wrap pkg/github/pr_request_watcher.go 3 -wrap pkg/github/pr_status_comment.go 1 +wrap pkg/github/pr_status_comment.go 2 wrap pkg/github/pr_supersession_autoclose.go 1 wrap pkg/github/pr_supersession_decision.go 1 wrap pkg/github/pr_supersession_resolve.go 6