From cfe210da9a0363f72d693c5b04eb4ce3d9e7c9d2 Mon Sep 17 00:00:00 2001 From: Danathar Date: Mon, 9 Nov 2026 09:48:09 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20[v6]=20contributor-first:=20a=20?= =?UTF-8?q?relay=20busy=20for=20another=20hive=20does=20not=20cover=20this?= =?UTF-8?q?=20hive's=20lane?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A relay subscribed to several hives works one task at a time. While that task belonged to hive B, hive A still saw a connected relay holding its "Acting as" role with a fresh heartbeat and no refusal, so RoleCoverage counted it as contributor-first cover and A's resident agent stayed withheld while nobody worked A's lane. The relay now tells every other hub that advertises the new busy_elsewhere_v1 capability when it starts and finishes another hive's task (state busy/clear plus the task's start time; no hive, repo or issue). It restates on auth_ok and clears on complete, fail and revoke, since all of those end in writeRelayStatus. The hub records the flag per connection, clears it on the relay's next ready, and RoleCoverage skips a busy-elsewhere connection that holds no task or fix-pr task from this hub. No wall-clock expiry: the flag dies with the connection, and a stale flag can only fall back to the resident agent working the lane. Old relays and old hubs keep today's behaviour. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BQYnPGFJx81c8JXR3o1sg1 Signed-off-by: Danathar --- bin/contributor-relay.js | 44 ++++++++ bin/contributor-relay.test.js | 79 ++++++++++++++ .../fixed-busy-elsewhere-relay-not-cover.md | 1 + src/docs/agent-configuration.md | 8 ++ src/docs/contributor-relay.md | 40 +++++++ src/docs/design/agent-state-inventory.md | 2 +- src/pkg/dashboard/contribute_coverage.go | 30 +++++- src/pkg/dashboard/contribute_coverage_test.go | 101 ++++++++++++++++++ src/pkg/dashboard/contribute_protocol.go | 9 ++ src/pkg/dashboard/contribute_ws.go | 58 ++++++++++ src/pkg/governor/contributor_first.go | 4 +- 11 files changed, 371 insertions(+), 5 deletions(-) create mode 100644 changelog.d/fixed-busy-elsewhere-relay-not-cover.md diff --git a/bin/contributor-relay.js b/bin/contributor-relay.js index 80fc208d33..01d000550d 100755 --- a/bin/contributor-relay.js +++ b/bin/contributor-relay.js @@ -1776,6 +1776,10 @@ function makeHub(url, token, disabled = false, contributorId = '', repoScope = n // (quota_hold, capability quota_hold_v1). Reset on every auth_ok: a fresh // connection on the hub starts out not held. quotaHoldSent: false, + // What this hub's CURRENT connection was last told about this relay + // working ANOTHER hive's task (busy_elsewhere, capability + // busy_elsewhere_v1). Reset on every auth_ok, like quotaHoldSent. + busyElsewhereSent: false, // Set by handleTLSHostMismatch when the dialed host is not in the served // certificate; parks the hub (no reconnect loop) until contributor.env // changes. relocationAttempted bounds the hub-registry re-resolution to @@ -1916,6 +1920,9 @@ function writeRelayStatus() { // Every quota hold transition already ends here (it changes relayReadiness), // so this is the one place the hub is told — see reportQuotaHold. reportQuotaHoldToHubs(); + // Every currentTask transition (assign, complete, fail, revoke) ends here + // too, so the same holds for busy_elsewhere — see reportBusyElsewhere. + reportBusyElsewhereToHubs(); } // ── Quota hold report (quota_hold_v1) ──────────────────────────────────────── @@ -1950,6 +1957,39 @@ function reportQuotaHoldToHubs() { for (const hub of hubs) reportQuotaHold(hub); } +// ── Busy on another hive (busy_elsewhere_v1) ───────────────────────────────── +// A relay subscribed to several hives works one primary task at a time. While +// that task belongs to hive B, hive A still sees a connected relay that +// heartbeats, holds its "Acting as" role and has been refused nothing, so A's +// contributor-first gate (RoleCoverage) counted it as covering A's lane and +// withheld A's resident agent while nobody worked that lane. Tell every OTHER +// hub that advertises busy_elsewhere_v1 when such a task starts and when it +// ends; a hub without the token never sees the frame, and keeps today's +// reading. The frame carries only when the other task started — never which +// hive, repository or issue — so one hive learns nothing about another's work. +// The local review cycle belongs to no hub and is not reported. +function busyElsewhereFor(hub) { + if (!currentTask || isLocalOnlyTask(currentTask)) return false; + const owner = currentTask._hub; + return !!owner && owner !== hub; +} + +function reportBusyElsewhere(hub) { + if (!hub || !hub.authenticated || hub.disabled) return; + if (!Array.isArray(hub.serverCapabilities) || !hub.serverCapabilities.includes('busy_elsewhere_v1')) return; + const busy = busyElsewhereFor(hub); + if (busy === hub.busyElsewhereSent) return; + if (!hub.ws || hub.ws.readyState !== WebSocket.OPEN) return; + hub.busyElsewhereSent = busy; + sendTo(hub, busy + ? { type: 'busy_elsewhere', seq: nextSeq(), state: 'busy', since: currentTaskStartedAt || new Date().toISOString() } + : { type: 'busy_elsewhere', seq: nextSeq(), state: 'clear' }); +} + +function reportBusyElsewhereToHubs() { + for (const hub of hubs) reportBusyElsewhere(hub); +} + let progressInterval = null; let verdictWatchInterval = null; let verdictFastPathLine = null; @@ -9810,6 +9850,10 @@ function handleMessage(data, hub) { // before anything else so contributor-first cover is right from the start. hub.quotaHoldSent = false; reportQuotaHold(hub); + // Same for busy_elsewhere: a new connection starts out not busy on the + // hub, so restate another hive's running task to it. + hub.busyElsewhereSent = false; + reportBusyElsewhere(hub); declareStandby(hub); hub.scopeSent = false; negotiateRepoScope(hub); diff --git a/bin/contributor-relay.test.js b/bin/contributor-relay.test.js index 02a6903518..e5ce79755a 100644 --- a/bin/contributor-relay.test.js +++ b/bin/contributor-relay.test.js @@ -17338,6 +17338,85 @@ test('#16463 standby acknowledgement does not outlive the connection it was gran } }); +// --------------------------------------------------------------------------- +// busy_elsewhere_v1: a relay working one hive's task tells its OTHER hives, so +// they stop counting it as contributor-first cover. Observed on 2026-11-09: a +// relay set "Acting as: scanner" on projectbluefin worked hivecommons tasks +// from 14:25Z to 14:32Z and sent projectbluefin no ready. +// --------------------------------------------------------------------------- + +function busyElsewhereFrames(sent) { + return sent.filter(m => m.type === 'busy_elsewhere'); +} + +function busyAuth(relay, hub, capabilities = ['busy_elsewhere_v1']) { + relay.handleMessage(JSON.stringify({ type: 'auth_ok', contributor_id: 'c1', trust_tier: 'contributor', server_capabilities: capabilities }), hub); +} + +// Rotate the polling slot to hive B (as a task_unavailable from A would) and +// take B's task through the real assignment handler. +function takeTaskFromHubB(relay, hubs, taskId) { + relay.sendReadyForNextTask('unavailable'); + relay.handleMessage(JSON.stringify({ type: 'task_assign', task_id: taskId, task_gen: 1, kind: 'issue', repo: 'org/b', number: 7, title: 'work on hive B' }), hubs[1]); + assert.strictEqual(relay.getCurrentTask() && relay.getCurrentTask().task_id, taskId, 'precondition: the relay took hive B\'s task'); +} + +test('busy_elsewhere_v1: another hive\'s task is reported to the other hive, restated on reconnect, and cleared when it finishes', () => { + const relay = loadRelay({ env: MULTI_HUB_ENV }); + const log = console.log; const warn = console.warn; + console.log = () => {}; console.warn = () => {}; + try { + const { hubs, sentA, sentB } = attachHubSinks(relay); + busyAuth(relay, hubs[0]); busyAuth(relay, hubs[1]); + assert.deepStrictEqual(busyElsewhereFrames([...sentA, ...sentB]), [], 'an idle relay is busy nowhere'); + + takeTaskFromHubB(relay, hubs, 'b-1'); + const frames = busyElsewhereFrames(sentA); + assert.deepStrictEqual(frames.map(m => m.state), ['busy'], 'hive A must hear about hive B\'s task once'); + assert.deepStrictEqual(Object.keys(frames[0]).sort(), ['seq', 'since', 'state', 'type'], + 'the frame names no hive, repository or task: one hive learns nothing about another\'s work'); + assert.ok(!Number.isNaN(Date.parse(frames[0].since)), `since must be a timestamp: ${frames[0].since}`); + assert.deepStrictEqual(busyElsewhereFrames(sentB), [], 'the hive that owns the task is never told it is busy elsewhere'); + + // A reconnect to hive A is a new connection that knows nothing: restate. + busyAuth(relay, hubs[0]); + assert.deepStrictEqual(busyElsewhereFrames(sentA).map(m => m.state), ['busy', 'busy']); + + relay.finishCurrentTask({ completionSignal: 'verdict', summary: 'done', tmuxLines: [], prURL: null, noWork: null }); + assert.deepStrictEqual(busyElsewhereFrames(sentA).map(m => m.state), ['busy', 'busy', 'clear'], + 'finishing hive B\'s task must clear the report on hive A'); + assert.deepStrictEqual(busyElsewhereFrames(sentB), []); + } finally { console.log = log; console.warn = warn; relay.setCurrentTask(null); teardown(relay); } +}); + +test('busy_elsewhere_v1: a revoked task clears the report', () => { + const relay = loadRelay({ env: MULTI_HUB_ENV }); + const log = console.log; const warn = console.warn; + console.log = () => {}; console.warn = () => {}; + try { + const { hubs, sentA } = attachHubSinks(relay); + busyAuth(relay, hubs[0]); busyAuth(relay, hubs[1]); + takeTaskFromHubB(relay, hubs, 'b-2'); + relay.handleMessage(JSON.stringify({ type: 'task_revoke', task_id: 'b-2', reason: 'operator stop' }), hubs[1]); + assert.strictEqual(relay.getCurrentTask(), null, 'precondition: the revoke ended the task'); + assert.deepStrictEqual(busyElsewhereFrames(sentA).map(m => m.state), ['busy', 'clear']); + } finally { console.log = log; console.warn = warn; relay.setCurrentTask(null); teardown(relay); } +}); + +test('busy_elsewhere_v1: a hive that does not advertise the token never sees the frame', () => { + const relay = loadRelay({ env: MULTI_HUB_ENV }); + const log = console.log; const warn = console.warn; + console.log = () => {}; console.warn = () => {}; + try { + const { hubs, sentA, sentB } = attachHubSinks(relay); + busyAuth(relay, hubs[0], ['quota_hold_v1']); busyAuth(relay, hubs[1]); + takeTaskFromHubB(relay, hubs, 'b-3'); + busyAuth(relay, hubs[0], ['quota_hold_v1']); + relay.finishCurrentTask({ completionSignal: 'verdict', summary: 'done', tmuxLines: [], prURL: null, noWork: null }); + assert.deepStrictEqual(busyElsewhereFrames([...sentA, ...sentB]), [], 'an old hub must see exactly today\'s traffic'); + } finally { console.log = log; console.warn = warn; relay.setCurrentTask(null); teardown(relay); } +}); + // --------------------------------------------------------------------------- // quota_hold_v1: a held relay tells the hub, so it stops counting as // contributor-first cover. Observed on 2026-11-08: a relay acting as scanner diff --git a/changelog.d/fixed-busy-elsewhere-relay-not-cover.md b/changelog.d/fixed-busy-elsewhere-relay-not-cover.md new file mode 100644 index 0000000000..74c2b06636 --- /dev/null +++ b/changelog.d/fixed-busy-elsewhere-relay-not-cover.md @@ -0,0 +1 @@ +- Contributor-first lanes are no longer left unworked by a relay that is busy on another hive. A relay subscribed to several hives works one task at a time, and while that task belonged to another hive it still looked to this one like a connected relay holding its role, so the governor kept the resident agent's kicks withheld while nobody worked the lane (seen on 2026-11-09: a relay set "Acting as: scanner" on one hive worked another hive's tasks for seven minutes and sent the first hive no request for work). Relays now tell every other hive that advertises the new `busy_elsewhere_v1` capability when they start and finish another hive's task, and the hub leaves such a relay out of coverage until the task ends or the relay asks it for work again. The frame names no hive, repository or issue. Older relays and older hubs behave as before. diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index 5d5ec4cbd8..4d097ec910 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -1394,6 +1394,14 @@ A relay counts as covering the role only when all of these hold: tells the hub (`quota_hold_v1`, see [the contributor relay guide](contributor-relay.md#a-held-relay-tells-the-hub-quota_hold_v1)); an older relay is still counted while held; +- it is not busy on another hive. A relay subscribed to several hives works + one task at a time; while that task belongs to another hive it stays + connected here but asks this hive for nothing. A relay new enough to know + about this tells every other hive it is subscribed to (`busy_elsewhere_v1`, see + [the contributor relay guide](contributor-relay.md#a-relay-busy-on-another-hive-tells-the-hub-busy_elsewhere_v1)) + and clears it when that task ends; its next request for work here clears it + too. A relay that this hive has also handed a task or a fix-pr task still + counts, and an older relay is still counted while busy elsewhere; - it is not in a failure-streak pause; - it has not used up its hourly or daily task allowance (`hub.tier_limits..max_per_hour` / `max_per_day` for the diff --git a/src/docs/contributor-relay.md b/src/docs/contributor-relay.md index de062c3cbd..ce93ff4263 100644 --- a/src/docs/contributor-relay.md +++ b/src/docs/contributor-relay.md @@ -2021,6 +2021,46 @@ about it: The expired-CLI-login hold is not reported this way yet. +### A relay busy on another hive tells the hub (`busy_elsewhere_v1`) + +A relay subscribed to several hives works one task at a time. While that task +belongs to hive B, hive A still sees a connected relay that answers heartbeats, +holds its "Acting as" role and has been refused nothing. Before this, a relay +acting as a role on A kept counting as +[contributor-first](agent-configuration.md#contributor-first-lanes-contributor_first) +cover there for as long as it worked B's tasks, so A's resident agent was held +back while nobody worked A's lane. + +Every other hub that advertises `busy_elsewhere_v1` in +`auth_ok.server_capabilities` now hears about it: + +```jsonc +{ "type": "busy_elsewhere", "seq": 61, "state": "busy", "since": "2026-11-09T14:25:03Z" } +{ "type": "busy_elsewhere", "seq": 62, "state": "clear" } +``` + +- The relay sends `busy` when it starts a task from another hive and `clear` + when that task completes, fails or is revoked. It sends each transition once + per connection, and restates a running task right after `auth_ok`, because a + new connection starts out not busy on the hub. The hive that owns the task + is never sent the frame. +- The frame says only when the other task started. It never names the other + hive, repository or issue. +- The hub records it on the connection and uses it only for contributor-first + coverage. It changes no lease, assignment or standby state. A relay this hub + has also handed a task or a fix-pr task still counts. +- There is no timer on the hub. The flag goes away with the connection (a + socket close or a relay restart), and the relay's next `ready` to this hub + clears it, since a relay asks for work only when it holds none. A timer + would start counting a relay that is still busy on a long task again; a flag + left set by mistake only means the resident agent works the lane, which is + what happens without `contributor_first`. +- A hub that does not advertise the token never receives the frame. A relay + that never sends it is read as not busy elsewhere. Either way the behaviour + is exactly what it was before. + +The local PR-review cycle belongs to no hive and is not reported. + ### Provider adapter sources The `src/pkg/rotation` probers normalize each provider's usage into the reading diff --git a/src/docs/design/agent-state-inventory.md b/src/docs/design/agent-state-inventory.md index 75819f97a8..147743a109 100644 --- a/src/docs/design/agent-state-inventory.md +++ b/src/docs/design/agent-state-inventory.md @@ -180,7 +180,7 @@ key generations (`/data/saas/hub-generations.json`). | # | State | Where (v5) | On a spoke (dashboard) pod roll | Verdict | |---|-------|------------|--------------------------------|---------| -| 27 | Completed-task ledger + per-task cooldowns + PR URLs | `completedTasks` / `completedTaskCooldown` / `completedTaskPRURL` (`contribute_ws.go:975`), persisted to `/data/contributors/completed-tasks.json` | Survives; was moved to the PVC ledger through the #3792 → #3989 lineage after re-offer loops | **fixed** | +| 27 | Completed-task ledger + per-task cooldowns + PR URLs | `completedTasks` / `completedTaskCooldown` / `completedTaskPRURL` (`contribute_ws.go:985`), persisted to `/data/contributors/completed-tasks.json` | Survives; was moved to the PVC ledger through the #3792 → #3989 lineage after re-offer loops | **fixed** | | 28 | Failure cooldowns + quarantine counters | `failedTasks` / `consecutiveFailures` (`:373`, `:381`), persisted to `/data/contributors/failed-tasks.json` | Survives (#2435 livelock fix, made durable with the ledger) | **fixed** | | 29 | No-PR completion streaks | `noPRStreaks` (`:394`), `/data/contributors/no-pr-streaks.json` | Survives (#3980; geometric backoff must outlive the completion entry it escalates) | **fixed** | | 30 | `no_work_needed` verdicts | `noWorkVerdicts` (`:416`), `/data/contributors/no-work-verdicts.json` | Survives (#3987/#3997) — "persisted in the same PVC-backed ledger dir as the cooldowns so a pod restart does not forget the verdict" | **fixed** | diff --git a/src/pkg/dashboard/contribute_coverage.go b/src/pkg/dashboard/contribute_coverage.go index ee19abdf91..a473e32506 100644 --- a/src/pkg/dashboard/contribute_coverage.go +++ b/src/pkg/dashboard/contribute_coverage.go @@ -34,6 +34,15 @@ import ( // Observed on 2026-11-08: a relay held at 15% of its weekly window kept // the scanner lane "covered" while nobody worked it. A relay too old to // send quota_hold is still counted, as before; +// - it has not reported that it is busy on another hive (busy_elsewhere, +// capability busy_elsewhere_v1) while holding no task from this one. A +// relay subscribed to several hives works one task at a time; while that +// task belongs to another hive it heartbeats and keeps its role here but +// asks this hub for nothing. Observed on 2026-11-09: a relay set "Acting +// as: scanner" on one hive spent the next seven minutes on another hive's +// tasks and sent the first hive no ready. Its next ready here clears the +// report (see busyElsewhereLocked). A relay too old to send busy_elsewhere +// is still counted, as before; // - it is not in a failure-streak pause: a relay whose runtime keeps dying // is connected but is not doing the lane's work; // - its identity has not used up its hourly or daily task allowance @@ -60,7 +69,9 @@ import ( // since. Standby, quota hold, a used-up allowance and a closed tier do not // wait for the task to end: each means the relay will not take the lane's NEXT // item, which is what the governor is deciding whether to withhold. (Holding -// max_concurrent tasks is different: that relay is busy, and counts.) Whether +// max_concurrent tasks is different: that relay is busy, and counts. So is a +// relay busy on another hive that this hub has also handed a task or a fix-pr +// task: it is working this lane too.) Whether // the relay is ALLOWED the role (trust tier, grants, the delegatable list) was // settled when the hub set c.role; nothing is re-derived here. func (h *ContributeWSHub) RoleCoverage(now time.Time) map[string]governor.ContributorCoverage { @@ -84,7 +95,7 @@ func (h *ContributeWSHub) RoleCoverage(now time.Time) map[string]governor.Contri login = strings.TrimSpace(c.profile.GitHubUsername) tier = c.profile.TrustTier } - notTakingWork := c.standbyDeclared || len(c.standby) > 0 || c.quotaHeld + notTakingWork := c.standbyDeclared || len(c.standby) > 0 || c.quotaHeld || c.busyElsewhereLocked() queueEmpty := c.noWorkForWindowLocked(now) c.mu.Unlock() if role == "" || !fresh || login == "" || notTakingWork || queueEmpty { @@ -190,6 +201,21 @@ func (c *ContributorConnection) noWorkForWindowLocked(now time.Time) bool { return now.Sub(c.lastIdleReasonSince) >= coverageNoWorkWindow } +// busyElsewhereLocked reports whether c has said it is working another hive's +// task (busy_elsewhere) and this hub has handed it no task of its own, primary +// or fix-pr. The caller holds c.mu. +// +// There is no wall-clock expiry. The flag is per connection, so a socket close +// or a relay restart drops it with the connection; the relay clears it when the +// other task ends, and the relay's next ready here clears it too (handleReady). +// A timeout would only re-count a relay that is still busy on a long task — +// the bug this check exists to stop — while a flag left set by mistake errs +// the safe way: the resident agent works the lane, as it would without +// contributor_first. +func (c *ContributorConnection) busyElsewhereLocked() bool { + return c.busyElsewhere && c.currentTask == nil && len(c.fixTasks) == 0 +} + // contributionsSuspended mirrors the task-selection gate in // contribute_select.go: with hub.contribute_suspended set, no relay is handed // work, whatever role it holds. diff --git a/src/pkg/dashboard/contribute_coverage_test.go b/src/pkg/dashboard/contribute_coverage_test.go index cc5b53ea64..cc7102b652 100644 --- a/src/pkg/dashboard/contribute_coverage_test.go +++ b/src/pkg/dashboard/contribute_coverage_test.go @@ -614,3 +614,104 @@ func TestRoleCoverage_FixTaskDispatchEndsNoWorkRun(t *testing.T) { t.Fatalf("relay after its fix settled: RoleCoverage = %+v, want %+v", got, scannerAlice) } } + +// A relay subscribed to several hives works one task at a time. Observed on +// 2026-11-09: a relay set "Acting as: scanner" on projectbluefin spent +// 14:25Z-14:32Z on hivecommons tasks and sent projectbluefin no ready. To that +// hub it was a connected, role-holding relay with a fresh heartbeat that had +// never been refused anything, which is exactly what the first assertion +// below counts as cover. A relay that never sends busy_elsewhere (an older +// one) is still read that way; one that reports the other hive's task covers +// nothing until it reports the task finished. +func TestRoleCoverage_BusyElsewhereRelayDoesNotCover(t *testing.T) { + hub, _ := covK2Hub(t) + now := time.Now() + relay := coverageConn("Danathar", "scanner", now) + hub.mu.Lock() + hub.connections["relay"] = relay + hub.mu.Unlock() + session := &wsSession{h: hub, contributor: relay} + + want := map[string]governor.ContributorCoverage{"scanner": {Contributors: []string{"Danathar"}}} + if got := hub.RoleCoverage(now); !reflect.DeepEqual(got, want) { + t.Fatalf("a role-holding relay that has sent no ready and no report: RoleCoverage = %+v, want %+v (today's reading)", got, want) + } + + session.handleBusyElsewhere(WSMessage{Type: "busy_elsewhere", State: "busy", Since: now.UTC().Format(time.RFC3339)}) + if got := hub.RoleCoverage(now); got != nil { + t.Fatalf("relay busy on another hive counted as cover: RoleCoverage = %+v, want nil", got) + } + + session.handleBusyElsewhere(WSMessage{Type: "busy_elsewhere", State: "clear"}) + if got := hub.RoleCoverage(now); !reflect.DeepEqual(got, want) { + t.Fatalf("after the other hive's task ended: RoleCoverage = %+v, want %+v", got, want) + } +} + +// A relay busy on another hive that this hub has ALSO handed work (a primary +// task, or a fix-pr task) is working this lane too, and counts. +func TestRoleCoverage_BusyElsewhereWithThisHubsTaskStillCovers(t *testing.T) { + hub, _ := covK2Hub(t) + now := time.Now() + relay := coverageConn("Danathar", "scanner", now) + hub.mu.Lock() + hub.connections["relay"] = relay + hub.mu.Unlock() + (&wsSession{h: hub, contributor: relay}).handleBusyElsewhere(WSMessage{Type: "busy_elsewhere", State: "busy"}) + want := map[string]governor.ContributorCoverage{"scanner": {Contributors: []string{"Danathar"}}} + + relay.mu.Lock() + relay.currentTask = &WSTaskAssign{TaskID: "t-1", Repo: "myorg/repo1", Number: 7} + relay.mu.Unlock() + if got := hub.RoleCoverage(now); !reflect.DeepEqual(got, want) { + t.Fatalf("busy elsewhere but holding this hub's task: RoleCoverage = %+v, want %+v", got, want) + } + + relay.mu.Lock() + relay.currentTask = nil + relay.fixTasks = map[string]*WSTaskAssign{"fix-1": {TaskID: "fix-1", Repo: "myorg/repo1", Number: 8}} + relay.mu.Unlock() + if got := hub.RoleCoverage(now); !reflect.DeepEqual(got, want) { + t.Fatalf("busy elsewhere but holding this hub's fix-pr task: RoleCoverage = %+v, want %+v", got, want) + } + + relay.mu.Lock() + relay.fixTasks = nil + relay.mu.Unlock() + if got := hub.RoleCoverage(now); got != nil { + t.Fatalf("busy elsewhere with nothing from this hub: RoleCoverage = %+v, want nil", got) + } +} + +// The hub keeps no wall-clock expiry on busy_elsewhere; instead the relay's +// next ready here clears it, because a relay asks only when it holds no task. +// So a lost "clear" frame cannot keep a relay that is back asking this hub out +// of coverage. Driven through the real ready handler. +func TestRoleCoverage_ReadyClearsBusyElsewhere(t *testing.T) { + hub, _, relay, sess, read := noWorkRelay(t) + sess.handleBusyElsewhere(WSMessage{Type: "busy_elsewhere", State: "busy"}) + if got := coverageAt(hub, relay, time.Now()); got != nil { + t.Fatalf("setup: busy-elsewhere relay must not cover, got %+v", got) + } + + if f := sendReady(t, sess, read); f["type"] != "task_unavailable" { + t.Fatalf("setup: expected task_unavailable on an empty queue, got %v", f) + } + relay.mu.Lock() + busy := relay.busyElsewhere + relay.mu.Unlock() + if busy { + t.Fatal("a ready must clear busy_elsewhere") + } + if got := coverageAt(hub, relay, idleSince(relay).Add(30*time.Second)); !reflect.DeepEqual(got, scannerAlice) { + t.Fatalf("relay asking this hub again: RoleCoverage = %+v, want %+v", got, scannerAlice) + } +} + +// The relay sends busy_elsewhere only to a hub that advertises the token, so +// the hub must advertise it or the fix never reaches the wire. +func TestServerCapabilities_AdvertisesBusyElsewhere(t *testing.T) { + if !testContainsString(serverCapabilities(), capBusyElsewhereV1) { + t.Fatalf("serverCapabilities() = %v, missing %q", serverCapabilities(), capBusyElsewhereV1) + } +} diff --git a/src/pkg/dashboard/contribute_protocol.go b/src/pkg/dashboard/contribute_protocol.go index 08b284e9dd..fe0c720d07 100644 --- a/src/pkg/dashboard/contribute_protocol.go +++ b/src/pkg/dashboard/contribute_protocol.go @@ -98,6 +98,14 @@ const ( // read as not held, exactly as before, and a relay only sends it to a hub // that advertises this token. capQuotaHoldV1 = "quota_hold_v1" + // capBusyElsewhereV1: the hub accepts busy_elsewhere messages (State + // "busy" or "clear") telling it the relay is working ANOTHER hive's task. + // A relay subscribed to several hives works one task at a time, so while + // it is busy elsewhere it does not count as contributor-first cover here + // (RoleCoverage). Purely additive: a relay that never sends it is read as + // not busy elsewhere, exactly as before, and a relay only sends it to a hub + // that advertises this token. + capBusyElsewhereV1 = "busy_elsewhere_v1" // capRunStage: the relay can receive stage-scoped run assignments. Unlike // plain task metadata, a stage is load-bearing for per-stage generations, so @@ -152,6 +160,7 @@ func serverCapabilities() []string { capFixPRInterrupt, capRepoScope, capQuotaHoldV1, + capBusyElsewhereV1, } } diff --git a/src/pkg/dashboard/contribute_ws.go b/src/pkg/dashboard/contribute_ws.go index 658f544127..34311169f6 100644 --- a/src/pkg/dashboard/contribute_ws.go +++ b/src/pkg/dashboard/contribute_ws.go @@ -122,6 +122,15 @@ type ContributorConnection struct { // when every lane it declared was rejected (a suspended standby key), so // RoleCoverage must not count it as covering its role either way. standbyDeclared bool + // busyElsewhere is the relay's own report (busy_elsewhere, capability + // busy_elsewhere_v1) that it is working a task from ANOTHER hive it is + // subscribed to. Such a relay heartbeats and keeps its role here but asks + // this hub for nothing until that task ends, so RoleCoverage skips it + // unless this hub has handed it work too. Set and cleared by the relay; + // also cleared by the relay's next `ready` here (a relay asks only when it + // holds no task), and it dies with the connection. A relay that never sends + // busy_elsewhere leaves it false, which is today's reading. + busyElsewhere bool // quotaHeld is the relay's own report (quota_hold, capability // quota_hold_v1) that its contributor quota guard or provider quota hold // is withholding `ready`: the socket stays up and heartbeats, but the @@ -559,6 +568,7 @@ type WSMessage struct { Permissions []string `json:"permissions,omitempty"` Reason string `json:"reason,omitempty"` State string `json:"state,omitempty"` + Since string `json:"since,omitempty"` ContributeNeedsDecisionLabel *string `json:"contribute_needs_decision_label,omitempty"` OperatorMessage *ContributorOperatorMessage `json:"operator_message,omitempty"` // FailureKind is the OPTIONAL, client-declared cause of a task_failed @@ -2595,6 +2605,8 @@ func (h *ContributeWSHub) HandleWS(w http.ResponseWriter, r *http.Request) { s.handleStandbyRelease(msg) case "quota_hold": s.handleQuotaHold(msg) + case "busy_elsewhere": + s.handleBusyElsewhere(msg) case "knowledge_state": s.handleKnowledgeState(msg) case "pong": @@ -3294,6 +3306,43 @@ func (s *wsSession) handleQuotaHold(msg WSMessage) { } } +// handleBusyElsewhere records the relay's report that it has started or +// finished working a task from another hive it is subscribed to (capability +// busy_elsewhere_v1). The relay sends State "busy" when such a task starts and +// "clear" when it ends; any other value reads as clear, which is what a hub +// that never heard the message assumes. The flag only feeds RoleCoverage: it +// changes no lease, assignment or standby state here. +func (s *wsSession) handleBusyElsewhere(msg WSMessage) { + if s.contributor == nil { + return + } + busy := strings.EqualFold(strings.TrimSpace(msg.State), "busy") + s.contributor.mu.Lock() + changed := s.contributor.busyElsewhere != busy + s.contributor.busyElsewhere = busy + s.contributor.mu.Unlock() + if changed { + s.logBusyElsewhereChange(busy, msg.Since, "relay") + } +} + +// logBusyElsewhereChange is the one log line for a busy_elsewhere transition. +// since is client-supplied, so it is bounded like quota_hold's reason; source +// names who cleared it ("relay" for the relay's own frame, "ready" when the +// relay asked this hub for work). +func (s *wsSession) logBusyElsewhereChange(busy bool, since, source string) { + since = strings.TrimSpace(since) + if len(since) > 40 { + since = since[:40] + } + s.h.logger.Info("[contribute-ws] contributor relay busy on another hive changed", + "id", s.contributor.connID, + "username", contributorUsername(s.contributor), + "busy_elsewhere", busy, + "since", since, + "source", source) +} + func normalizeStandbyLanes(lanes []string) []string { if len(lanes) == 0 { return []string{"all"} @@ -3688,6 +3737,12 @@ func (s *wsSession) handleReady(msg WSMessage) (stop bool) { // #12805: the scope version this ready was sent under; selection only // proceeds when it matches the acknowledged scope (0 = not stated). s.contributor.readyScopeVersion = msg.ScopeVersion + // A relay asks for work only when it holds no task, so a ready also says + // it is no longer busy on another hive (busy_elsewhere_v1). Clearing here + // means a lost "clear" frame cannot keep the relay out of coverage once it + // is back asking this hub. + wasBusyElsewhere := s.contributor.busyElsewhere + s.contributor.busyElsewhere = false abandoned := s.contributor.currentTask // #7317: captured under the same lock as currentTask so the run record // below can report the task's real wall-clock duration. On the session @@ -3707,6 +3762,9 @@ func (s *wsSession) handleReady(msg WSMessage) (stop bool) { s.contributor.credentialDelivered = false s.contributor.pendingExternalTask = nil s.contributor.mu.Unlock() + if wasBusyElsewhere { + s.logBusyElsewhereChange(false, "", "ready") + } if abandoned != nil { // C4: the relay explicitly gave up this task, so revoke its // server-issued lease — a later task_progress for it must not resurrect diff --git a/src/pkg/governor/contributor_first.go b/src/pkg/governor/contributor_first.go index 0d815f3905..fb00a9f588 100644 --- a/src/pkg/governor/contributor_first.go +++ b/src/pkg/governor/contributor_first.go @@ -22,8 +22,8 @@ import ( // contributor_first inverts the order for one lane. While at least one // connected, non-stale, non-failure-paused relay is acting as the role and // taking work (not in standby, not quota-held, not at its hourly or daily -// task limit, not revoked or in a disabled tier, not idle on an empty queue; -// see RoleCoverage in pkg/dashboard), and the operator has not suspended contributions, the +// task limit, not revoked or in a disabled tier, not idle on an empty queue, +// not busy on another hive; see RoleCoverage in pkg/dashboard), and the operator has not suspended contributions, the // governor WITHHOLDS the resident agent's kicks — at exactly the gates where it // withholds them for an exhausted budget. The relay is the primary worker; the // resident agent is the fallback, and it resumes on its normal cadence once no