Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-16292-transcript-scan-overrun.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- The copilot transcript activity scan overrun warning is now debounced: it fires once after three consecutive passes longer than the scan interval and re-arms only after a pass under 80% of it, so a marginal overrun on a many-agent hive no longer logs a WARN every ~25s and trips the fleet report ([#16292](https://github.com/hivecommons/hive/issues/16292)).
26 changes: 10 additions & 16 deletions src/crux/agent/busy_visibility.go
Original file line number Diff line number Diff line change
Expand Up @@ -407,24 +407,24 @@ func (m *Manager) StartTranscriptActivityRefresh(ctx context.Context) {
// tick; scans never overlap because they run on this goroutine alone.
func (m *Manager) runTranscriptActivityRefresh(ctx context.Context, ticks <-chan time.Time, now func() time.Time) {
caches := map[string]*transcriptSessionCache{}
slow := false
m.refreshTranscriptActivity(caches, now, &slow)
pace := &transcriptScanPace{}
m.refreshTranscriptActivity(caches, now, pace)
for {
select {
case <-ctx.Done():
return
case <-ticks:
m.refreshTranscriptActivity(caches, now, &slow)
m.refreshTranscriptActivity(caches, now, pace)
}
}
}

// refreshTranscriptActivity runs one scan pass: pick the stale agents under
// a short read lock, stat their sessions with no lock held, then publish
// the results under a short write lock. Returns how many agents were
// scanned. slow tracks the previous pass's verdict so a walk that outlives
// the scan interval is logged on the transition, not every tick.
func (m *Manager) refreshTranscriptActivity(caches map[string]*transcriptSessionCache, now func() time.Time, slow *bool) int {
// scanned. pace debounces the overrun warning: it fires once after
// several consecutive slow passes, and re-arms only after a clearly fast one.
func (m *Manager) refreshTranscriptActivity(caches map[string]*transcriptSessionCache, now func() time.Time, pace *transcriptScanPace) int {
started := now()
m.mu.RLock()
scans := m.pendingTranscriptActivityScansLocked(started)
Expand All @@ -445,16 +445,10 @@ func (m *Manager) refreshTranscriptActivity(caches map[string]*transcriptSession
delete(caches, name)
}
}
if took := now().Sub(started); took > transcriptLivenessScanInterval {
if slow != nil && !*slow && m.logger != nil {
m.logger.Warn("copilot transcript activity scan outlived its interval; agent liveness reads may lag",
"took", took.Round(time.Millisecond).String(), "agents", len(scans), "interval", transcriptLivenessScanInterval.String())
}
if slow != nil {
*slow = true
}
} else if slow != nil {
*slow = false
took := now().Sub(started)
if pace != nil && pace.observe(took, transcriptLivenessScanInterval) && m.logger != nil {
m.logger.Warn("copilot transcript activity scan outlived its interval; agent liveness reads may lag",
"took", took.Round(time.Millisecond).String(), "agents", len(scans), "interval", transcriptLivenessScanInterval.String())
}
return len(results)
}
Expand Down
40 changes: 40 additions & 0 deletions src/crux/agent/transcript_scan_pace.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package agent

import "time"

const (
// transcriptScanSlowPassesToWarn is how many consecutive passes must
// outlive the scan interval before the overrun is logged.
transcriptScanSlowPassesToWarn = 3
// transcriptScanRecoverPercent is the share of the scan interval a pass
// must finish under to end a slow episode. The gap between this and 100%
// keeps a marginal overrun from flapping the episode on and off.
transcriptScanRecoverPercent = 80
)

// transcriptScanPace tracks scan pass durations so the overrun warning fires
// once per slow episode instead of on every fast-to-slow flip.
type transcriptScanPace struct {
slowRun int
warned bool
}

// observe records one pass and reports whether to log the overrun now.
func (p *transcriptScanPace) observe(took, interval time.Duration) bool {
switch {
case took > interval:
p.slowRun++
if p.slowRun >= transcriptScanSlowPassesToWarn && !p.warned {
p.warned = true
return true
}
case took*100 < interval*transcriptScanRecoverPercent:
p.slowRun = 0
p.warned = false
default:
// Marginal pass: breaks the consecutive-slow streak without
// re-arming the warning, so flapping around the interval stays quiet.
p.slowRun = 0
}
return false
}
49 changes: 49 additions & 0 deletions src/crux/agent/transcript_scan_pace_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package agent

import (
"bytes"
"log/slog"
"strings"
"testing"
"time"
)

func TestTranscriptScanOverrunWarningHysteresis(t *testing.T) {
cases := []struct {
name string
takes []time.Duration
want int
}{
{"marginal flapping", []time.Duration{10700 * time.Millisecond, 9500 * time.Millisecond, 10700 * time.Millisecond, 9500 * time.Millisecond, 10700 * time.Millisecond, 9500 * time.Millisecond, 10700 * time.Millisecond, 9500 * time.Millisecond}, 0},
{"sustained overrun warns once", []time.Duration{11 * time.Second, 11 * time.Second, 11 * time.Second, 11 * time.Second, 11 * time.Second}, 1},
{"two slow passes stay quiet", []time.Duration{11 * time.Second, 11 * time.Second, 2 * time.Second}, 0},
{"clear fast pass re-arms", []time.Duration{11 * time.Second, 11 * time.Second, 11 * time.Second, 2 * time.Second, 11 * time.Second, 11 * time.Second, 11 * time.Second}, 2},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
var buf bytes.Buffer
m, _ := transcriptRefreshTestManager(t)
m.logger = slog.New(slog.NewTextHandler(&buf, nil))
base := time.Date(2026, 11, 2, 12, 0, 0, 0, time.UTC)
caches := map[string]*transcriptSessionCache{}
pace := &transcriptScanPace{}
for i, took := range tc.takes {
start := base.Add(time.Duration(i) * time.Minute)
calls := 0
clock := func() time.Time {
calls++
if calls == 1 {
return start
}
return start.Add(took)
}
if n := m.refreshTranscriptActivity(caches, clock, pace); n != 1 {
t.Fatalf("pass %d scanned %d agents, want 1", i, n)
}
}
if got := strings.Count(buf.String(), "transcript activity scan outlived its interval"); got != tc.want {
t.Fatalf("warnings = %d, want %d\n%s", got, tc.want, buf.String())
}
})
}
}
2 changes: 1 addition & 1 deletion src/docs/fleet-report.md
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ title is `[selfreport] <class>: <short message>`.
| `proxy_plain_forward_refused` | the hive proxy refused a plain-HTTP forward to a non-globally-routable destination (`proxy: plain HTTP forward refused`, `crux/proxy/github_proxy_connect.go`) — a client ignored its proxy bypass for a private address (e.g. the Docker bridge gateway). Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: fix the client's `NO_PROXY` or target address; the refusal is counted as a proxy violation | low |
| `sha_poll_commit_message_unavailable` | the SHA poller's commit-message fetch got a non-200 from GitHub (typically a 403 rate limit or token scope; `SHA poll: commit message fetch non-200`, `pkg/hub/saas_sha_poller.go`). The poller backs off per SHA and keeps serving the SHA without a message. The WARN itself is deduped to once per hour per (SHA, status) (`warnGitHubFetchOnce`), so one stuck commit logs one line and never trips this class; it is reported only when ≥5 distinct SHA/status pairs fail within 10 min (a broad outage such as an exhausted rate limit or a revoked token). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low |
| `tmux_session_check_failed` | the agent's tmux session `has-session` check failed (`tmux has-session failed`, `crux/agent/manager.go`) — typically the session was lost or the tmux server is unhealthy, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium |
| `transcript_scan_slow` | a Copilot transcript activity scan took longer than its interval (`transcript activity scan outlived its interval`, `crux/agent/busy_visibility.go`) — agent liveness reads may lag. Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: check the agent's transcript/session volume; it is bounded by concurrency, so repeated hits indicate slow disk or very large session directories | low |
| `transcript_scan_slow` | a Copilot transcript activity scan took longer than its interval (`transcript activity scan outlived its interval`, `crux/agent/busy_visibility.go`) — agent liveness reads may lag. The WARN itself is debounced at the source (`transcriptScanPace`, `crux/agent/transcript_scan_pace.go`): it fires once per slow episode, only after 3 consecutive passes outlive the interval, and re-arms only after a pass finishes under 80% of the interval, so one sustained overrun logs one line and never trips this class; it is reported only when ≥5 distinct slow→fast→slow episodes occur within 10 min. Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: check the agent's transcript/session volume; it is bounded by concurrency, so repeated episodes indicate slow disk or very large session directories | low |
| `kick_freshness_withheld` | a kick was withheld because every issue/PR ref failed the pre-kick freshness check (`kick withheld: every ref failed the freshness check`, `cmd/hive/kick_freshness.go`). The kick is retried on the next cycle; reported only when the same withholding repeats ≥5 times in 10 min (refs persistently stale or unreadable). Deferred API-budget refusals log at INFO and never trip it. Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low |
| `unclassified_error_repeated` | **generic catch-all**: the same WARN/ERROR log line (numbers, hashes, paths and quoted strings collapsed) repeated ≥5 times in 10 min. Fed by a `slog.Handler` tee (`pkg/fleetreport/slogfeed`) on the hive's root logger, so a repeating defect is noticed even with no hook site. Skips benign lines (read/keepalive i/o timeouts, WARNs whose `error` is a GitHub rate-limit refusal or a 502/503/504 gateway failure — the caller retries next tick, context cancellation, the permissions watcher's `fixed …` self-healing lines) and any message that already matches another class's keywords. Clears automatically after 20 quiet minutes (2× the window), so a "cleared" comment follows once the symptom stops | low |

Expand Down
1 change: 1 addition & 0 deletions src/docs/notifications.md
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,7 @@ without a class. Review submissions and review requests are always counted as
| `github.(*Client).PostNoteOrFold` | `status` | passthrough wrapper; attributed to its caller |
| `github.(*Client).SweepLegacyFleetReportIssues` | `status` | |
| `github.(*Client).UpsertIssueMarkerComment` | `status` | passthrough wrapper; attributed to its caller |
| `github.(*clientStatusForge).CreateComment` | `status` | passthrough wrapper (prstatus.Write status-comment creation); attributed to its caller |
| `github.(*Client).closeDecomposedParent` | `status` | |
| `github.(*Client).closeIssueForMergedPRClaimWithIssue` | `status` | |
| `github.(*Client).closeSupersededPRDecision` | `status` | |
Expand Down
1 change: 1 addition & 0 deletions src/pkg/github/notification_write_sites.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ var notificationWriteSites = map[string]NotificationWriteSite{
"main.(*repoForgeRouter).CreateIssueComment": {Class: NotificationClassAttention, Passthrough: true},
"main.liveTriageCommenter.CreateIssueComment": {Class: NotificationClassAttention, Passthrough: true},
"github.(*Client).UpsertIssueMarkerComment": {Class: NotificationClassStatus, Passthrough: true},
"github.(*clientStatusForge).CreateComment": {Class: NotificationClassStatus, Passthrough: true},

// status: bookkeeping readable on demand.
"dashboard.(*ContributeWSHub).markAlreadyDoneIssue": {Class: NotificationClassStatus},
Expand Down
5 changes: 3 additions & 2 deletions src/pkg/writeback/testdata/direct-writes-baseline.txt
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ raw pkg/github/automerge/automerge_held_runs.go 3
raw pkg/github/automerge/automerge_merge_commit.go 1
raw pkg/github/automerge/automerge_merge_forward.go 1
raw pkg/github/automerge/automerge_stage_human.go 1
raw pkg/github/automerge/automerge_stage_labels.go 3
raw pkg/github/automerge/automerge_stage_labels.go 5
raw pkg/github/automerge/automerge_sweep_queued.go 3
raw pkg/github/automerge/automerge_sweep_trusted.go 4
raw pkg/github/automerge/automerge_unauthorized.go 3
Expand All @@ -38,6 +38,7 @@ raw pkg/github/pr_contributor_link.go 1
raw pkg/github/pr_issue_link.go 2
raw pkg/github/pr_level_hold.go 5
raw pkg/github/pr_signed_reconcile.go 1
raw pkg/github/pr_status_comment.go 2
raw pkg/github/pr_supersession_autoclose.go 3
raw pkg/github/pr_supersession_resolve.go 1
raw pkg/github/pr_supersession_sweep.go 2
Expand Down Expand Up @@ -106,7 +107,7 @@ wrap pkg/github/pr_duplicate_guard.go 1
wrap pkg/github/pr_level_hold.go 1
wrap pkg/github/pr_reporter_trust_release.go 1
wrap pkg/github/pr_request_watcher.go 3
wrap pkg/github/pr_status_comment.go 1
wrap pkg/github/pr_status_comment.go 2
wrap pkg/github/pr_supersession_autoclose.go 1
wrap pkg/github/pr_supersession_decision.go 1
wrap pkg/github/pr_supersession_resolve.go 6
Expand Down
Loading