Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions bin/contributor-relay.js
Original file line number Diff line number Diff line change
Expand Up @@ -1715,6 +1715,10 @@ function makeHub(url, token, disabled = false, contributorId = '', repoScope = n
// #16462/#16463: set by standby_ack, cleared whenever the connection it
// was granted on goes away.
standbyAcked: false,
// What this hub's CURRENT connection was last told about the quota hold
// (quota_hold, capability quota_hold_v1). Reset on every auth_ok: a fresh
// connection on the hub starts out not held.
quotaHoldSent: false,
// Set by handleTLSHostMismatch when the dialed host is not in the served
// certificate; parks the hub (no reconnect loop) until contributor.env
// changes. relocationAttempted bounds the hub-registry re-resolution to
Expand Down Expand Up @@ -1852,6 +1856,41 @@ function writeRelayStatus() {
if (temporary) { try { fs.unlinkSync(temporary); } catch (_) {} }
console.error(`WARNING: could not write relay status file ${RELAY_STATUS_FILE}: ${e.message}`);
}
// Every quota hold transition already ends here (it changes relayReadiness),
// so this is the one place the hub is told — see reportQuotaHold.
reportQuotaHoldToHubs();
}

// ── Quota hold report (quota_hold_v1) ────────────────────────────────────────
// While the quota guard (#6951) or the provider quota hold (#6541) is holding,
// this relay withholds `ready` and declines pushed work, but its socket stays
// up and heartbeats. A hub counting connected relays as contributor-first
// cover (RoleCoverage) could not tell, so it withheld the resident agent's
// kicks for a lane nobody was working. Tell each hub that advertises
// quota_hold_v1 when the hold starts and ends; a hub without the token never
// sees the frame, and keeps today's reading.
function relayQuotaHeld() {
return contributorQuotaPaused || quotaHoldActive();
}

function relayQuotaHoldReason() {
return contributorQuotaPaused ? 'quota_guard' : 'provider_quota';
}

function reportQuotaHold(hub) {
if (!hub || !hub.authenticated || hub.disabled) return;
if (!Array.isArray(hub.serverCapabilities) || !hub.serverCapabilities.includes('quota_hold_v1')) return;
const held = relayQuotaHeld();
if (held === hub.quotaHoldSent) return;
if (!hub.ws || hub.ws.readyState !== WebSocket.OPEN) return;
hub.quotaHoldSent = held;
sendTo(hub, held
? { type: 'quota_hold', seq: nextSeq(), state: 'held', reason: relayQuotaHoldReason() }
: { type: 'quota_hold', seq: nextSeq(), state: 'released' });
}

function reportQuotaHoldToHubs() {
for (const hub of hubs) reportQuotaHold(hub);
}

let progressInterval = null;
Expand Down Expand Up @@ -9637,6 +9676,10 @@ function handleMessage(data, hub) {
hub.authFailed = false;
hub.connectionId = msg.connection_id || '';
hub.serverCapabilities = Array.isArray(msg.server_capabilities) ? msg.server_capabilities.slice() : [];
// A fresh connection starts out not held on the hub; restate a live hold
// before anything else so contributor-first cover is right from the start.
hub.quotaHoldSent = false;
reportQuotaHold(hub);
declareStandby(hub);
hub.scopeSent = false;
negotiateRepoScope(hub);
Expand Down
83 changes: 83 additions & 0 deletions bin/contributor-relay.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -17020,3 +17020,86 @@ test('#16463 standby acknowledgement does not outlive the connection it was gran
relay.setCurrentTask(null); teardown(relay);
}
});

// ---------------------------------------------------------------------------
// quota_hold_v1: a held relay tells the hub, so it stops counting as
// contributor-first cover. Observed on 2026-11-08: a relay acting as scanner
// was held by its quota guard (weekly window 15% left, reserve 20%) while the
// hub kept reporting it as covering the lane, and nobody worked it.
// ---------------------------------------------------------------------------

function quotaHoldFrames(sent) {
return sent.filter(m => m.type === 'quota_hold');
}

test('quota_hold_v1: the quota guard hold is reported, and released before the relay asks again', () => {
const p = tmp6951(JSON.stringify({ state: 'available', limits: [{ id: 'weekly', kind: 'weekly', pct_remaining: 15 }] }));
const relay = loadRelay({ env: { HIVE_CONTRIBUTOR_QUOTA_READING_FILE: p } });
const warn = console.warn; console.warn = () => {};
const log = console.log; console.log = () => {};
try {
relay.setCliReady(true);
const sent = [];
relay.setWs({ readyState: 1, send: s => sent.push(JSON.parse(s)) });
relay.handleMessage(JSON.stringify({ type: 'auth_ok', contributor_id: 'c1', trust_tier: 'contributor', server_capabilities: ['quota_hold_v1'] }));
assert.strictEqual(sent.some(m => m.type === 'ready'), false, 'precondition: the guard withholds ready at 15% < 20%');
assert.deepStrictEqual(quotaHoldFrames(sent).map(m => [m.state, m.reason]), [['held', 'quota_guard']],
`the hub must hear the hold once: ${JSON.stringify(sent.map(m => m.type))}`);

relay.retryContributorQuota();
assert.strictEqual(quotaHoldFrames(sent).length, 1, 'a retry that is still held must not repeat the frame');

fs.writeFileSync(p, JSON.stringify({ state: 'available', limits: [{ id: 'weekly', kind: 'weekly', pct_remaining: 90 }] }));
relay.retryContributorQuota();
const frames = quotaHoldFrames(sent);
assert.deepStrictEqual(frames.map(m => m.state), ['held', 'released'], 'the release must be reported');
const releasedAt = sent.indexOf(frames[1]);
const readyAt = sent.findIndex(m => m.type === 'ready');
assert.ok(readyAt > releasedAt, `released must reach the hub before the next ready: ${JSON.stringify(sent.map(m => m.type + (m.state ? ':' + m.state : '')))}`);
} finally { console.warn = warn; console.log = log; fs.unlinkSync(p); teardown(relay); }
});

test('quota_hold_v1: a hub that does not advertise the token never sees the frame', () => {
const p = tmp6951(JSON.stringify({ state: 'available', limits: [{ id: 'weekly', kind: 'weekly', pct_remaining: 15 }] }));
const relay = loadRelay({ env: { HIVE_CONTRIBUTOR_QUOTA_READING_FILE: p } });
const warn = console.warn; console.warn = () => {};
const log = console.log; console.log = () => {};
try {
relay.setCliReady(true);
const sent = [];
relay.setWs({ readyState: 1, send: s => sent.push(JSON.parse(s)) });
relay.handleMessage(JSON.stringify({ type: 'auth_ok', contributor_id: 'c1', trust_tier: 'contributor', server_capabilities: ['standby_v1'] }));
relay.enterQuotaHold({ line: 'quota exhausted', resetMs: 60 * 60 * 1000 });
fs.writeFileSync(p, JSON.stringify({ state: 'available', limits: [{ id: 'weekly', kind: 'weekly', pct_remaining: 90 }] }));
relay.retryContributorQuota();
relay.releaseQuotaHold('test');
assert.strictEqual(relay.getContributorQuotaPaused(), false, 'precondition: both holds came and went');
assert.deepStrictEqual(quotaHoldFrames(sent), [], 'an old hub must see exactly today\'s traffic');
} finally { console.warn = warn; console.log = log; fs.unlinkSync(p); teardown(relay); }
});

test('quota_hold_v1: the provider quota hold is reported, restated on reconnect, and released', () => {
const relay = loadRelay({});
const warn = console.warn; console.warn = () => {};
const log = console.log; console.log = () => {};
try {
relay.setCliReady(true);
const sent = [];
relay.setWs({ readyState: 1, send: s => sent.push(JSON.parse(s)) });
const authOK = JSON.stringify({ type: 'auth_ok', contributor_id: 'c1', trust_tier: 'contributor', server_capabilities: ['quota_hold_v1'] });
relay.handleMessage(authOK);
relay.enterQuotaHold({ line: 'quota exhausted', resetMs: 60 * 60 * 1000 });
assert.deepStrictEqual(quotaHoldFrames(sent).map(m => [m.state, m.reason]), [['held', 'provider_quota']]);

// A reconnect is a new connection the hub knows nothing about: restate.
relay.handleMessage(authOK);
assert.deepStrictEqual(quotaHoldFrames(sent).map(m => m.state), ['held', 'held'],
'auth_ok during a hold must tell the new connection, once');

relay.releaseQuotaHold('test');
const frames = quotaHoldFrames(sent);
assert.deepStrictEqual(frames.map(m => m.state), ['held', 'held', 'released']);
const readyAt = sent.findIndex((m, i) => i > sent.indexOf(frames[2]) && m.type === 'ready');
assert.ok(readyAt > 0, `release must still re-advertise after telling the hub: ${JSON.stringify(sent.map(m => m.type))}`);
} finally { console.warn = warn; console.log = log; teardown(relay); }
});
2 changes: 2 additions & 0 deletions changelog.d/fixed-relay-hold-not-cover.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
- Contributor-first lanes are no longer left unworked by a relay that is connected but not taking work. A relay in standby mode, or one whose quota guard or provider quota hold is holding all work, used to count as covering its role, so the governor withheld the resident agent's kicks while nobody worked the lane (seen on 2026-11-08: a scanner relay held at 15% of its weekly window kept scanner "covered"). The hub now skips standby relays, and relays tell a hub that advertises the new `quota_hold_v1` capability when a quota hold starts and ends; older relays and older hubs behave as before.
- Switching `contributor_first` on for a lane a relay already covers now logs "contributor relay covering a contributor-first lane — withholding the resident agent's kicks" once. The line was written only when the governor first saw the cover, and it records cover for every role whether or not the flag is set, so the usual order (relay connected, then the owner flips the General-tab toggle) left no log line at all.
20 changes: 17 additions & 3 deletions src/docs/agent-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -1325,9 +1325,8 @@ covering. Standby (above) is the opposite order on purpose — it offers paused
*budget* work to volunteers and never fires before the budget is gone.

`contributor_first` inverts the order for one lane. While at least one
contributor relay is **acting as that role and connected** — fresh heartbeat,
not in a failure-streak pause — the governor **withholds the resident agent's
kicks**, at the same gates where it withholds them for an exhausted budget
contributor relay is **acting as that role and connected, and taking work**,
the governor **withholds the resident agent's kicks**, at the same gates where it withholds them for an exhausted budget
(cadence kicks, continuous re-kicks, CEL event kicks and crash-recovery resume
kicks). The contributor is the lane's primary worker and spends their own
inference; the resident agent is the fallback. Once no relay has covered the
Expand All @@ -1336,6 +1335,21 @@ cadence. The grace exists because a relay reconnect is routine (the hub keeps
a dropped relay's lease for thirty minutes for the same reason); without it
the resident agent would flap on and off with every socket drop.

A relay counts as covering the role only when all of these hold:

- its heartbeat is fresh;
- it is not in a failure-streak pause;
- it is not in standby mode (`just contribute-standby`). A standby relay never
asks for ordinary work; it only takes what you dispatch to it by hand;
- it has not reported a quota hold. A relay whose quota guard is holding stops
asking for work but stays connected. A relay new enough to know about this
tells the hub (`quota_hold_v1`, see
[the contributor relay guide](contributor-relay.md#a-held-relay-tells-the-hub-quota_hold_v1));
an older relay is still counted while held;
- contributions are not suspended (`hub.contribute_suspended`).

A relay that is busy on a task still counts.

```yaml
agents:
quality:
Expand Down
30 changes: 30 additions & 0 deletions src/docs/contributor-relay.md
Original file line number Diff line number Diff line change
Expand Up @@ -1962,6 +1962,36 @@ Six behaviours are worth knowing:
left to the maintainer; taking it would strand exactly the hosts just listed.
See "How the reading reaches the guard" for the marker mechanics.

### A held relay tells the hub (`quota_hold_v1`)

While the quota guard or the provider quota hold is holding, the relay stays
connected and keeps answering heartbeats; it just stops sending `ready` and
declines anything pushed to it. Before this, the hub could not tell such a relay
from one that was working, so a relay acting as a role still counted as
[contributor-first](agent-configuration.md#contributor-first-lanes-contributor_first)
cover and the resident agent's kicks were withheld for a lane nobody was
working.

A hub that advertises `quota_hold_v1` in `auth_ok.server_capabilities` now hears
about it:

```jsonc
{ "type": "quota_hold", "seq": 51, "state": "held", "reason": "quota_guard" } // or "provider_quota"
{ "type": "quota_hold", "seq": 52, "state": "released" }
```

- The relay sends `held` when either hold starts and `released` when both have
ended, before the `ready` that follows a release. It sends each transition
once per connection, and restates a live hold right after `auth_ok`, because
a new connection starts out not held on the hub.
- The hub records it on the connection and uses it only for contributor-first
coverage. It changes no lease, assignment or standby state.
- A hub that does not advertise the token never receives the frame. A relay
that never sends it is read as not held. Either way the behaviour is exactly
what it was before.

The expired-CLI-login hold is not reported this way yet.

### Provider adapter sources

The `src/pkg/rotation` probers normalize each provider's usage into the reading
Expand Down
16 changes: 14 additions & 2 deletions src/pkg/dashboard/contribute_coverage.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,24 @@ import (
// silently cannot keep the resident agent idle;
// - it is not in a failure-streak pause: a relay whose runtime keeps dying
// is connected but is not doing the lane's work;
// - it has not declared standby (standby_declare, accepted or not). A
// standby relay stays connected but never asks for ordinary work; it only
// takes what an owner dispatches to it by hand, so it drains nothing;
// - it has not reported a quota hold (quota_hold, capability
// quota_hold_v1). A relay whose quota guard is holding withholds `ready`
// and declines pushed work, so it is connected and covering nothing.
// Observed on 2026-11-08: a relay held at 15% of its weekly window kept
// the scanner lane "covered" while nobody worked it. A relay too old to
// send quota_hold is still counted, as before;
// - the operator has not suspended contributions (hub.contribute_suspended).
// The suspend switch refuses work to every relay while their sockets stay
// up, so a connected relay is then covering nothing; reporting it as
// coverage would turn "pause contributors" into "pause the lane".
//
// A relay that is busy on a task still counts — it is draining the lane,
// which is the point. Whether the relay is ALLOWED the role (trust tier,
// which is the point. Standby and quota hold do not wait for the task to end:
// both mean the relay will not take the lane's NEXT item, which is what the
// governor is deciding whether to withhold. Whether the relay is ALLOWED the role (trust tier,
// grants, the delegatable list) was settled when the hub set c.role; nothing
// is re-derived here.
func (h *ContributeWSHub) RoleCoverage(now time.Time) map[string]governor.ContributorCoverage {
Expand All @@ -51,8 +62,9 @@ func (h *ContributeWSHub) RoleCoverage(now time.Time) map[string]governor.Contri
if c.profile != nil {
login = strings.TrimSpace(c.profile.GitHubUsername)
}
notTakingWork := c.standbyDeclared || len(c.standby) > 0 || c.quotaHeld
c.mu.Unlock()
if role == "" || !fresh || login == "" {
if role == "" || !fresh || login == "" || notTakingWork {
continue
}
candidates = append(candidates, candidate{role: role, login: login, identity: identityOf(c)})
Expand Down
Loading
Loading