diff --git a/bin/contributor-relay.js b/bin/contributor-relay.js index 06fc9a475a..a97755c688 100755 --- a/bin/contributor-relay.js +++ b/bin/contributor-relay.js @@ -1715,6 +1715,10 @@ function makeHub(url, token, disabled = false, contributorId = '', repoScope = n // #16462/#16463: set by standby_ack, cleared whenever the connection it // was granted on goes away. standbyAcked: false, + // What this hub's CURRENT connection was last told about the quota hold + // (quota_hold, capability quota_hold_v1). Reset on every auth_ok: a fresh + // connection on the hub starts out not held. + quotaHoldSent: false, // Set by handleTLSHostMismatch when the dialed host is not in the served // certificate; parks the hub (no reconnect loop) until contributor.env // changes. relocationAttempted bounds the hub-registry re-resolution to @@ -1852,6 +1856,41 @@ function writeRelayStatus() { if (temporary) { try { fs.unlinkSync(temporary); } catch (_) {} } console.error(`WARNING: could not write relay status file ${RELAY_STATUS_FILE}: ${e.message}`); } + // Every quota hold transition already ends here (it changes relayReadiness), + // so this is the one place the hub is told — see reportQuotaHold. + reportQuotaHoldToHubs(); +} + +// ── Quota hold report (quota_hold_v1) ──────────────────────────────────────── +// While the quota guard (#6951) or the provider quota hold (#6541) is holding, +// this relay withholds `ready` and declines pushed work, but its socket stays +// up and heartbeats. A hub counting connected relays as contributor-first +// cover (RoleCoverage) could not tell, so it withheld the resident agent's +// kicks for a lane nobody was working. Tell each hub that advertises +// quota_hold_v1 when the hold starts and ends; a hub without the token never +// sees the frame, and keeps today's reading. +function relayQuotaHeld() { + return contributorQuotaPaused || quotaHoldActive(); +} + +function relayQuotaHoldReason() { + return contributorQuotaPaused ? 'quota_guard' : 'provider_quota'; +} + +function reportQuotaHold(hub) { + if (!hub || !hub.authenticated || hub.disabled) return; + if (!Array.isArray(hub.serverCapabilities) || !hub.serverCapabilities.includes('quota_hold_v1')) return; + const held = relayQuotaHeld(); + if (held === hub.quotaHoldSent) return; + if (!hub.ws || hub.ws.readyState !== WebSocket.OPEN) return; + hub.quotaHoldSent = held; + sendTo(hub, held + ? { type: 'quota_hold', seq: nextSeq(), state: 'held', reason: relayQuotaHoldReason() } + : { type: 'quota_hold', seq: nextSeq(), state: 'released' }); +} + +function reportQuotaHoldToHubs() { + for (const hub of hubs) reportQuotaHold(hub); } let progressInterval = null; @@ -9637,6 +9676,10 @@ function handleMessage(data, hub) { hub.authFailed = false; hub.connectionId = msg.connection_id || ''; hub.serverCapabilities = Array.isArray(msg.server_capabilities) ? msg.server_capabilities.slice() : []; + // A fresh connection starts out not held on the hub; restate a live hold + // before anything else so contributor-first cover is right from the start. + hub.quotaHoldSent = false; + reportQuotaHold(hub); declareStandby(hub); hub.scopeSent = false; negotiateRepoScope(hub); diff --git a/bin/contributor-relay.test.js b/bin/contributor-relay.test.js index e49142aa2a..210d95f12c 100644 --- a/bin/contributor-relay.test.js +++ b/bin/contributor-relay.test.js @@ -17020,3 +17020,86 @@ test('#16463 standby acknowledgement does not outlive the connection it was gran relay.setCurrentTask(null); teardown(relay); } }); + +// --------------------------------------------------------------------------- +// quota_hold_v1: a held relay tells the hub, so it stops counting as +// contributor-first cover. Observed on 2026-11-08: a relay acting as scanner +// was held by its quota guard (weekly window 15% left, reserve 20%) while the +// hub kept reporting it as covering the lane, and nobody worked it. +// --------------------------------------------------------------------------- + +function quotaHoldFrames(sent) { + return sent.filter(m => m.type === 'quota_hold'); +} + +test('quota_hold_v1: the quota guard hold is reported, and released before the relay asks again', () => { + const p = tmp6951(JSON.stringify({ state: 'available', limits: [{ id: 'weekly', kind: 'weekly', pct_remaining: 15 }] })); + const relay = loadRelay({ env: { HIVE_CONTRIBUTOR_QUOTA_READING_FILE: p } }); + const warn = console.warn; console.warn = () => {}; + const log = console.log; console.log = () => {}; + try { + relay.setCliReady(true); + const sent = []; + relay.setWs({ readyState: 1, send: s => sent.push(JSON.parse(s)) }); + relay.handleMessage(JSON.stringify({ type: 'auth_ok', contributor_id: 'c1', trust_tier: 'contributor', server_capabilities: ['quota_hold_v1'] })); + assert.strictEqual(sent.some(m => m.type === 'ready'), false, 'precondition: the guard withholds ready at 15% < 20%'); + assert.deepStrictEqual(quotaHoldFrames(sent).map(m => [m.state, m.reason]), [['held', 'quota_guard']], + `the hub must hear the hold once: ${JSON.stringify(sent.map(m => m.type))}`); + + relay.retryContributorQuota(); + assert.strictEqual(quotaHoldFrames(sent).length, 1, 'a retry that is still held must not repeat the frame'); + + fs.writeFileSync(p, JSON.stringify({ state: 'available', limits: [{ id: 'weekly', kind: 'weekly', pct_remaining: 90 }] })); + relay.retryContributorQuota(); + const frames = quotaHoldFrames(sent); + assert.deepStrictEqual(frames.map(m => m.state), ['held', 'released'], 'the release must be reported'); + const releasedAt = sent.indexOf(frames[1]); + const readyAt = sent.findIndex(m => m.type === 'ready'); + assert.ok(readyAt > releasedAt, `released must reach the hub before the next ready: ${JSON.stringify(sent.map(m => m.type + (m.state ? ':' + m.state : '')))}`); + } finally { console.warn = warn; console.log = log; fs.unlinkSync(p); teardown(relay); } +}); + +test('quota_hold_v1: a hub that does not advertise the token never sees the frame', () => { + const p = tmp6951(JSON.stringify({ state: 'available', limits: [{ id: 'weekly', kind: 'weekly', pct_remaining: 15 }] })); + const relay = loadRelay({ env: { HIVE_CONTRIBUTOR_QUOTA_READING_FILE: p } }); + const warn = console.warn; console.warn = () => {}; + const log = console.log; console.log = () => {}; + try { + relay.setCliReady(true); + const sent = []; + relay.setWs({ readyState: 1, send: s => sent.push(JSON.parse(s)) }); + relay.handleMessage(JSON.stringify({ type: 'auth_ok', contributor_id: 'c1', trust_tier: 'contributor', server_capabilities: ['standby_v1'] })); + relay.enterQuotaHold({ line: 'quota exhausted', resetMs: 60 * 60 * 1000 }); + fs.writeFileSync(p, JSON.stringify({ state: 'available', limits: [{ id: 'weekly', kind: 'weekly', pct_remaining: 90 }] })); + relay.retryContributorQuota(); + relay.releaseQuotaHold('test'); + assert.strictEqual(relay.getContributorQuotaPaused(), false, 'precondition: both holds came and went'); + assert.deepStrictEqual(quotaHoldFrames(sent), [], 'an old hub must see exactly today\'s traffic'); + } finally { console.warn = warn; console.log = log; fs.unlinkSync(p); teardown(relay); } +}); + +test('quota_hold_v1: the provider quota hold is reported, restated on reconnect, and released', () => { + const relay = loadRelay({}); + const warn = console.warn; console.warn = () => {}; + const log = console.log; console.log = () => {}; + try { + relay.setCliReady(true); + const sent = []; + relay.setWs({ readyState: 1, send: s => sent.push(JSON.parse(s)) }); + const authOK = JSON.stringify({ type: 'auth_ok', contributor_id: 'c1', trust_tier: 'contributor', server_capabilities: ['quota_hold_v1'] }); + relay.handleMessage(authOK); + relay.enterQuotaHold({ line: 'quota exhausted', resetMs: 60 * 60 * 1000 }); + assert.deepStrictEqual(quotaHoldFrames(sent).map(m => [m.state, m.reason]), [['held', 'provider_quota']]); + + // A reconnect is a new connection the hub knows nothing about: restate. + relay.handleMessage(authOK); + assert.deepStrictEqual(quotaHoldFrames(sent).map(m => m.state), ['held', 'held'], + 'auth_ok during a hold must tell the new connection, once'); + + relay.releaseQuotaHold('test'); + const frames = quotaHoldFrames(sent); + assert.deepStrictEqual(frames.map(m => m.state), ['held', 'held', 'released']); + const readyAt = sent.findIndex((m, i) => i > sent.indexOf(frames[2]) && m.type === 'ready'); + assert.ok(readyAt > 0, `release must still re-advertise after telling the hub: ${JSON.stringify(sent.map(m => m.type))}`); + } finally { console.warn = warn; console.log = log; teardown(relay); } +}); diff --git a/changelog.d/fixed-relay-hold-not-cover.md b/changelog.d/fixed-relay-hold-not-cover.md new file mode 100644 index 0000000000..d64814b262 --- /dev/null +++ b/changelog.d/fixed-relay-hold-not-cover.md @@ -0,0 +1,2 @@ +- Contributor-first lanes are no longer left unworked by a relay that is connected but not taking work. A relay in standby mode, or one whose quota guard or provider quota hold is holding all work, used to count as covering its role, so the governor withheld the resident agent's kicks while nobody worked the lane (seen on 2026-11-08: a scanner relay held at 15% of its weekly window kept scanner "covered"). The hub now skips standby relays, and relays tell a hub that advertises the new `quota_hold_v1` capability when a quota hold starts and ends; older relays and older hubs behave as before. +- Switching `contributor_first` on for a lane a relay already covers now logs "contributor relay covering a contributor-first lane — withholding the resident agent's kicks" once. The line was written only when the governor first saw the cover, and it records cover for every role whether or not the flag is set, so the usual order (relay connected, then the owner flips the General-tab toggle) left no log line at all. diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index bb878179e6..41fc034ead 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -1325,9 +1325,8 @@ covering. Standby (above) is the opposite order on purpose — it offers paused *budget* work to volunteers and never fires before the budget is gone. `contributor_first` inverts the order for one lane. While at least one -contributor relay is **acting as that role and connected** — fresh heartbeat, -not in a failure-streak pause — the governor **withholds the resident agent's -kicks**, at the same gates where it withholds them for an exhausted budget +contributor relay is **acting as that role and connected, and taking work**, +the governor **withholds the resident agent's kicks**, at the same gates where it withholds them for an exhausted budget (cadence kicks, continuous re-kicks, CEL event kicks and crash-recovery resume kicks). The contributor is the lane's primary worker and spends their own inference; the resident agent is the fallback. Once no relay has covered the @@ -1336,6 +1335,21 @@ cadence. The grace exists because a relay reconnect is routine (the hub keeps a dropped relay's lease for thirty minutes for the same reason); without it the resident agent would flap on and off with every socket drop. +A relay counts as covering the role only when all of these hold: + +- its heartbeat is fresh; +- it is not in a failure-streak pause; +- it is not in standby mode (`just contribute-standby`). A standby relay never + asks for ordinary work; it only takes what you dispatch to it by hand; +- it has not reported a quota hold. A relay whose quota guard is holding stops + asking for work but stays connected. A relay new enough to know about this + tells the hub (`quota_hold_v1`, see + [the contributor relay guide](contributor-relay.md#a-held-relay-tells-the-hub-quota_hold_v1)); + an older relay is still counted while held; +- contributions are not suspended (`hub.contribute_suspended`). + +A relay that is busy on a task still counts. + ```yaml agents: quality: diff --git a/src/docs/contributor-relay.md b/src/docs/contributor-relay.md index 751ed3b70b..5bd13f040f 100644 --- a/src/docs/contributor-relay.md +++ b/src/docs/contributor-relay.md @@ -1962,6 +1962,36 @@ Six behaviours are worth knowing: left to the maintainer; taking it would strand exactly the hosts just listed. See "How the reading reaches the guard" for the marker mechanics. +### A held relay tells the hub (`quota_hold_v1`) + +While the quota guard or the provider quota hold is holding, the relay stays +connected and keeps answering heartbeats; it just stops sending `ready` and +declines anything pushed to it. Before this, the hub could not tell such a relay +from one that was working, so a relay acting as a role still counted as +[contributor-first](agent-configuration.md#contributor-first-lanes-contributor_first) +cover and the resident agent's kicks were withheld for a lane nobody was +working. + +A hub that advertises `quota_hold_v1` in `auth_ok.server_capabilities` now hears +about it: + +```jsonc +{ "type": "quota_hold", "seq": 51, "state": "held", "reason": "quota_guard" } // or "provider_quota" +{ "type": "quota_hold", "seq": 52, "state": "released" } +``` + +- The relay sends `held` when either hold starts and `released` when both have + ended, before the `ready` that follows a release. It sends each transition + once per connection, and restates a live hold right after `auth_ok`, because + a new connection starts out not held on the hub. +- The hub records it on the connection and uses it only for contributor-first + coverage. It changes no lease, assignment or standby state. +- A hub that does not advertise the token never receives the frame. A relay + that never sends it is read as not held. Either way the behaviour is exactly + what it was before. + +The expired-CLI-login hold is not reported this way yet. + ### Provider adapter sources The `src/pkg/rotation` probers normalize each provider's usage into the reading diff --git a/src/pkg/dashboard/contribute_coverage.go b/src/pkg/dashboard/contribute_coverage.go index bf7331b4de..25c17d074b 100644 --- a/src/pkg/dashboard/contribute_coverage.go +++ b/src/pkg/dashboard/contribute_coverage.go @@ -22,13 +22,24 @@ import ( // silently cannot keep the resident agent idle; // - it is not in a failure-streak pause: a relay whose runtime keeps dying // is connected but is not doing the lane's work; +// - it has not declared standby (standby_declare, accepted or not). A +// standby relay stays connected but never asks for ordinary work; it only +// takes what an owner dispatches to it by hand, so it drains nothing; +// - it has not reported a quota hold (quota_hold, capability +// quota_hold_v1). A relay whose quota guard is holding withholds `ready` +// and declines pushed work, so it is connected and covering nothing. +// Observed on 2026-11-08: a relay held at 15% of its weekly window kept +// the scanner lane "covered" while nobody worked it. A relay too old to +// send quota_hold is still counted, as before; // - the operator has not suspended contributions (hub.contribute_suspended). // The suspend switch refuses work to every relay while their sockets stay // up, so a connected relay is then covering nothing; reporting it as // coverage would turn "pause contributors" into "pause the lane". // // A relay that is busy on a task still counts — it is draining the lane, -// which is the point. Whether the relay is ALLOWED the role (trust tier, +// which is the point. Standby and quota hold do not wait for the task to end: +// both mean the relay will not take the lane's NEXT item, which is what the +// governor is deciding whether to withhold. Whether the relay is ALLOWED the role (trust tier, // grants, the delegatable list) was settled when the hub set c.role; nothing // is re-derived here. func (h *ContributeWSHub) RoleCoverage(now time.Time) map[string]governor.ContributorCoverage { @@ -51,8 +62,9 @@ func (h *ContributeWSHub) RoleCoverage(now time.Time) map[string]governor.Contri if c.profile != nil { login = strings.TrimSpace(c.profile.GitHubUsername) } + notTakingWork := c.standbyDeclared || len(c.standby) > 0 || c.quotaHeld c.mu.Unlock() - if role == "" || !fresh || login == "" { + if role == "" || !fresh || login == "" || notTakingWork { continue } candidates = append(candidates, candidate{role: role, login: login, identity: identityOf(c)}) diff --git a/src/pkg/dashboard/contribute_coverage_test.go b/src/pkg/dashboard/contribute_coverage_test.go index 551d3140ef..37d099185f 100644 --- a/src/pkg/dashboard/contribute_coverage_test.go +++ b/src/pkg/dashboard/contribute_coverage_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/hivecommons/hive/pkg/governor" + standbypkg "github.com/hivecommons/hive/pkg/standby" ) func coverageConn(login, role string, pong time.Time) *ContributorConnection { @@ -113,3 +114,103 @@ func TestRoleCoverage_SuspendedContributionsCoverNothing(t *testing.T) { t.Fatalf("contributions resumed: RoleCoverage = %+v, want quality covered by alice", got) } } + +// A relay in standby mode stays connected but never asks for ordinary work: +// it takes only what an owner dispatches to it by hand. It must not count as +// contributor-first cover, or the resident agent is withheld for a lane that +// nobody is draining. Driven through the real standby_declare and +// standby_release handlers, so the flag RoleCoverage reads is the one the +// protocol sets. +func TestRoleCoverage_StandbyRelayDoesNotCover(t *testing.T) { + hub, _ := covK2Hub(t) + now := time.Now() + bob := coverageConn("bob", "quality", now) + bob.ws, _ = wsPipe(t) + hub.mu.Lock() + hub.connections["alice"] = coverageConn("alice", "quality", now) + hub.connections["bob"] = bob + hub.mu.Unlock() + session := &wsSession{h: hub, contributor: bob} + + session.handleStandbyDeclare(WSMessage{Type: "standby_declare", Standby: &WSStandby{Lanes: []string{"quality"}}}) + want := map[string]governor.ContributorCoverage{"quality": {Contributors: []string{"alice"}}} + if got := hub.RoleCoverage(now); !reflect.DeepEqual(got, want) { + t.Fatalf("standby relay counted as cover: RoleCoverage = %+v, want %+v", got, want) + } + + session.handleStandbyRelease(WSMessage{Type: "standby_release"}) + want = map[string]governor.ContributorCoverage{"quality": {Contributors: []string{"alice", "bob"}}} + if got := hub.RoleCoverage(now); !reflect.DeepEqual(got, want) { + t.Fatalf("after standby_release: RoleCoverage = %+v, want %+v", got, want) + } +} + +// The live case from 2026-11-08 on hive-wild-mole: the only relay acting as +// scanner had its quota guard holding all work (weekly window 15% left, +// reserve 20%), yet the governor withheld scanner's kicks because the hub +// still reported the relay as covering. A held relay covers nothing; once it +// reports the hold released it covers again. +func TestRoleCoverage_QuotaHeldRelayDoesNotCover(t *testing.T) { + hub, _ := covK2Hub(t) + now := time.Now() + relay := coverageConn("Danathar", "scanner", now) + hub.mu.Lock() + hub.connections["relay"] = relay + hub.mu.Unlock() + session := &wsSession{h: hub, contributor: relay} + + want := map[string]governor.ContributorCoverage{"scanner": {Contributors: []string{"Danathar"}}} + if got := hub.RoleCoverage(now); !reflect.DeepEqual(got, want) { + t.Fatalf("precondition: RoleCoverage = %+v, want %+v", got, want) + } + + session.handleQuotaHold(WSMessage{Type: "quota_hold", State: "held", Reason: "quota_guard"}) + if got := hub.RoleCoverage(now); got != nil { + t.Fatalf("quota-held relay counted as cover: RoleCoverage = %+v, want nil", got) + } + + session.handleQuotaHold(WSMessage{Type: "quota_hold", State: "released"}) + if got := hub.RoleCoverage(now); !reflect.DeepEqual(got, want) { + t.Fatalf("after the hold was released: RoleCoverage = %+v, want %+v", got, want) + } +} + +// The relay sends quota_hold only to a hub that advertises the token, so the +// hub must advertise it or the fix never reaches the wire. +func TestServerCapabilities_AdvertisesQuotaHold(t *testing.T) { + if !testContainsString(serverCapabilities(), capQuotaHoldV1) { + t.Fatalf("serverCapabilities() = %v, missing %q", serverCapabilities(), capQuotaHoldV1) + } +} + +// A suspended standby configuration has every lane of its declaration +// rejected, so nothing is recorded in conn.standby — but the relay is still in +// standby mode and still never asks for ordinary work. It must not cover. +func TestRoleCoverage_RejectedStandbyDeclarationDoesNotCover(t *testing.T) { + hub, _ := covK2Hub(t) + now := time.Now() + bob := coverageConn("bob", "quality", now) + bob.ws, _ = wsPipe(t) + key := standbyOutcomeKey("bob", standbyConfigFromConnection(bob)) + hub.completedMu.Lock() + for n := 1; n <= standbypkg.DefaultSuspendThreshold; n++ { + hub.standbyOutcomes = append(hub.standbyOutcomes, standbyOutcomeRecord{ + Key: key, Lane: "quality", Repo: "org/r", Number: n, Kind: standbypkg.OutcomeClosedUnmerged, + }) + } + hub.completedMu.Unlock() + if suspended, _ := hub.standbySuspended(key); !suspended { + t.Fatal("setup: bob's standby configuration must be suspended") + } + hub.mu.Lock() + hub.connections["bob"] = bob + hub.mu.Unlock() + + (&wsSession{h: hub, contributor: bob}).handleStandbyDeclare(WSMessage{Type: "standby_declare"}) + if len(bob.standby) != 0 { + t.Fatalf("setup: a suspended declaration must record no lane, got %+v", bob.standby) + } + if got := hub.RoleCoverage(now); got != nil { + t.Fatalf("rejected standby relay counted as cover: RoleCoverage = %+v, want nil", got) + } +} diff --git a/src/pkg/dashboard/contribute_protocol.go b/src/pkg/dashboard/contribute_protocol.go index a99a63d6dd..08b284e9dd 100644 --- a/src/pkg/dashboard/contribute_protocol.go +++ b/src/pkg/dashboard/contribute_protocol.go @@ -91,6 +91,13 @@ const ( // and records per-connection standby availability. Purely additive: relays // that never advertise or send standby frames see today's traffic. capStandbyV1 = "standby_v1" + // capQuotaHoldV1: the hub accepts quota_hold messages (State "held" or + // "released") telling it the relay's quota guard has started or stopped + // withholding `ready`. A held connection does not count as contributor-first + // cover (RoleCoverage). Purely additive: a relay that never sends it is + // read as not held, exactly as before, and a relay only sends it to a hub + // that advertises this token. + capQuotaHoldV1 = "quota_hold_v1" // capRunStage: the relay can receive stage-scoped run assignments. Unlike // plain task metadata, a stage is load-bearing for per-stage generations, so @@ -144,6 +151,7 @@ func serverCapabilities() []string { capExtExecClaudeManagedAgents, capFixPRInterrupt, capRepoScope, + capQuotaHoldV1, } } diff --git a/src/pkg/dashboard/contribute_ws.go b/src/pkg/dashboard/contribute_ws.go index 37cdae5118..b7e908a2ad 100644 --- a/src/pkg/dashboard/contribute_ws.go +++ b/src/pkg/dashboard/contribute_ws.go @@ -116,11 +116,23 @@ type ContributorConnection struct { advisorModel string advisorEffort string standby map[string]StandbyConnectionState - role string // empty = task-driven mode, "scanner"/"reviewer"/etc. = role mode - clientRole string // relay-requested HIVE_AGENT_ROLE; owner assignment may override it - assignedRole string // owner-selected role; "none" forces general work - connectedAt time.Time - currentTask *WSTaskAssign + // standbyDeclared is set by ANY standby_declare on this connection, + // accepted or rejected, and cleared when standby_release leaves no lane + // recorded. A relay in standby mode never asks for ordinary work, even + // when every lane it declared was rejected (a suspended standby key), so + // RoleCoverage must not count it as covering its role either way. + standbyDeclared bool + // quotaHeld is the relay's own report (quota_hold, capability + // quota_hold_v1) that its contributor quota guard or provider quota hold + // is withholding `ready`: the socket stays up and heartbeats, but the + // relay asks for no work. RoleCoverage skips such a connection. A relay + // that never sends quota_hold leaves it false, which is today's reading. + quotaHeld bool + role string // empty = task-driven mode, "scanner"/"reviewer"/etc. = role mode + clientRole string // relay-requested HIVE_AGENT_ROLE; owner assignment may override it + assignedRole string // owner-selected role; "none" forces general work + connectedAt time.Time + currentTask *WSTaskAssign // fixTasks are the fix-pr tasks in flight on this connection, keyed by // task id (hivecommons/hive#12436). They never replace currentTask: a // spawned fix runs beside the primary task and a preempting one parks it. @@ -2571,6 +2583,8 @@ func (h *ContributeWSHub) HandleWS(w http.ResponseWriter, r *http.Request) { s.handleStandbyDeclare(msg) case "standby_release": s.handleStandbyRelease(msg) + case "quota_hold": + s.handleQuotaHold(msg) case "knowledge_state": s.handleKnowledgeState(msg) case "pong": @@ -3179,6 +3193,9 @@ func (s *wsSession) handleStandbyDeclare(msg WSMessage) { config := standbyConfigFromConnection(s.contributor) key := standbyOutcomeKey(s.contributor.profile.GitHubUsername, config) if suspended, _ := s.h.standbySuspended(key); suspended { + s.contributor.mu.Lock() + s.contributor.standbyDeclared = true + s.contributor.mu.Unlock() for _, lane := range lanes { rejected = append(rejected, WSStandbyLane{Lane: lane, Reason: string(standbypkg.ReasonSuspended)}) } @@ -3186,6 +3203,7 @@ func (s *wsSession) handleStandbyDeclare(msg WSMessage) { return } s.contributor.mu.Lock() + s.contributor.standbyDeclared = true if s.contributor.standby == nil { s.contributor.standby = map[string]StandbyConnectionState{} } @@ -3230,10 +3248,42 @@ func (s *wsSession) handleStandbyRelease(msg WSMessage) { delete(s.contributor.standby, lane) } } + if len(s.contributor.standby) == 0 { + s.contributor.standbyDeclared = false + } s.contributor.mu.Unlock() _ = s.contributor.send(WSMessage{Type: "standby_ack", Seq: msg.Seq}) } +// handleQuotaHold records the relay's report that its quota guard has started +// or stopped holding all work (capability quota_hold_v1). The relay sends +// State "held" when it starts withholding `ready` for quota and "released" +// when it is about to ask again; any other value reads as released, which is +// what a hub that never heard the message assumes. The flag only feeds +// RoleCoverage: it changes no lease, no assignment and no standby state, and +// the relay still declines a pushed assignment on its own while held. +func (s *wsSession) handleQuotaHold(msg WSMessage) { + if s.contributor == nil { + return + } + held := strings.EqualFold(strings.TrimSpace(msg.State), "held") + s.contributor.mu.Lock() + changed := s.contributor.quotaHeld != held + s.contributor.quotaHeld = held + s.contributor.mu.Unlock() + if changed { + reason := strings.TrimSpace(msg.Reason) + if len(reason) > 64 { + reason = reason[:64] + } + s.h.logger.Info("[contribute-ws] contributor relay quota hold changed", + "id", s.contributor.connID, + "username", contributorUsername(s.contributor), + "held", held, + "reason", reason) + } +} + func normalizeStandbyLanes(lanes []string) []string { if len(lanes) == 0 { return []string{"all"} diff --git a/src/pkg/governor/contributor_first.go b/src/pkg/governor/contributor_first.go index ae661b5623..8a922bcc38 100644 --- a/src/pkg/governor/contributor_first.go +++ b/src/pkg/governor/contributor_first.go @@ -20,8 +20,9 @@ import ( // paused BUDGET work to volunteers and never fires before the budget is gone. // // contributor_first inverts the order for one lane. While at least one -// connected, non-stale, non-failure-paused relay is acting as the role (and -// the operator has not suspended contributions), the +// connected, non-stale, non-failure-paused relay is acting as the role and +// taking work (not in standby, not quota-held; see RoleCoverage in +// pkg/dashboard), and the operator has not suspended contributions, the // governor WITHHOLDS the resident agent's kicks — at exactly the gates where it // withholds them for an exhausted budget. The relay is the primary worker; the // resident agent is the fallback, and it resumes on its normal cadence once no @@ -77,6 +78,14 @@ type coverageRecord struct { contributors []string since time.Time lastSeen time.Time + // announced is whether the "covering a contributor-first lane" line has + // been logged for this cover. Coverage is recorded whether or not the + // lane is flagged, so the usual order — a relay already covering, then the + // owner switching contributor_first on — finds the record already known; + // keying the line on a NEW record left that switch with no log line at + // all. Cleared while the lane is not flagged, so switching the flag off + // and on again announces again. + announced bool } // SetContributorCoverage replaces the governor's view of which roles connected @@ -109,11 +118,19 @@ func (g *Governor) SetContributorCoverage(coverage map[string]ContributorCoverag rec, known := g.coverage[role] if !known { rec = coverageRecord{since: now} - if g.contributorFirstLocked(role) && g.logger != nil { + } + switch flagged := g.contributorFirstLocked(role); { + case !flagged: + rec.announced = false + case !rec.announced: + // A new cover on a flagged lane, or the flag just switched on + // for a lane a relay was already covering. + rec.announced = true + if g.logger != nil { g.logger.Info("contributor relay covering a contributor-first lane — withholding the resident agent's kicks", "agent", role, "contributors", strings.Join(logins, ",")) } - } else if len(rec.contributors) == 0 && g.contributorFirstLocked(role) && g.logger != nil { + case len(rec.contributors) == 0 && g.logger != nil: // Back within the grace window: coverage never really ended. g.logger.Info("contributor relay returned within grace — resident agent stays withheld", "agent", role, "contributors", strings.Join(logins, ",")) diff --git a/src/pkg/governor/contributor_first_test.go b/src/pkg/governor/contributor_first_test.go index 216a1bb3f0..6898dbe8c1 100644 --- a/src/pkg/governor/contributor_first_test.go +++ b/src/pkg/governor/contributor_first_test.go @@ -1,7 +1,9 @@ package governor import ( + "bytes" "log/slog" + "strings" "testing" "time" @@ -334,3 +336,42 @@ func TestContributorFirst_MixedCaseAgentNameKeepsItsGrace(t *testing.T) { t.Fatalf("GraceUntil = %v, want %v", g.GetState().CoveredLanes["quality"].GraceUntil, want) } } + +// The General-tab toggle's usual order: a relay is already acting as the role +// (the governor records coverage for every role, flagged or not), THEN the +// owner switches contributor_first on. The covering line must appear once, +// on the first evaluation that sees the flag, and not again on later ticks. +// Seen on hive-wild-mole on 2026-11-08: coverage since 12:30:45Z, flag saved +// 12:31:18Z, and no covering line in the hive log. +func TestContributorFirst_FlagSwitchedOnForCoveredLaneLogsOnce(t *testing.T) { + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})) + cfg, agents := standardConfig("quality", "scanner") + g := New(cfg, agents, logger) + const line = "contributor relay covering a contributor-first lane" + scanner := map[string]ContributorCoverage{"scanner": {Contributors: []string{"Danathar"}}} + + g.SetContributorCoverage(scanner) + if n := strings.Count(buf.String(), line); n != 0 { + t.Fatalf("flag off: covering line logged %d times, want 0\n%s", n, buf.String()) + } + + flagged := make(map[string]config.AgentConfig, len(agents)) + for name, ac := range agents { + flagged[name] = ac + } + sc := flagged["scanner"] + sc.ContributorFirst = true + flagged["scanner"] = sc + g.UpdateAgents(flagged) + + for i := 0; i < 3; i++ { + g.SetContributorCoverage(scanner) + } + if n := strings.Count(buf.String(), line); n != 1 { + t.Fatalf("flag switched on for a covered lane: covering line logged %d times, want exactly 1\n%s", n, buf.String()) + } + if !strings.Contains(buf.String(), "agent=scanner") { + t.Fatalf("covering line must name the lane\n%s", buf.String()) + } +}