Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions bin/contributor-agent.sh
Original file line number Diff line number Diff line change
Expand Up @@ -834,6 +834,12 @@ echo ""
cleanup() {
echo "Shutting down..."
kill "$RELAY_PID" 2>/dev/null || true
# Wait for the relay before ending the container. This script is PID 1, so
# its exit SIGKILLs everything left, and a relay stopped just after an
# agent's verdict is still writing that task's task_complete. The relay bounds
# that wait itself, inside the runtime's stop timeout. tmux goes after it,
# because the relay's own shutdown still stops the agent in that session.
wait "$RELAY_PID" 2>/dev/null || true
# The publisher starts and stops with the contributor (#10299): a survivor
# would keep refreshing this pool's presence marker and tell the next relay a
# reading is coming that nothing will write.
Expand Down
121 changes: 118 additions & 3 deletions bin/contributor-relay.js
Original file line number Diff line number Diff line change
Expand Up @@ -2373,6 +2373,11 @@ function requestWork(hub, trigger, expectedConnectGeneration = null) {
logSkippedWorkRequest(hub, trigger, 'standby mode');
return false;
}
// Ahead of startDeferredFix below: a relay that is exiting starts nothing.
if (shuttingDown) {
logSkippedWorkRequest(hub, trigger, 'the relay is shutting down');
return false;
}
if (!hubScopeReady(hub)) return false;
const conn = readyConnLabel(hub);
if (expectedConnectGeneration !== null && hub.connectGeneration !== expectedConnectGeneration) {
Expand Down Expand Up @@ -2428,6 +2433,71 @@ function scheduleReadyRetry(hub, trigger) {
}, TASK_UNAVAILABLE_RETRY_MS);
}

// ── Shutdown with a finished task still on the wire ─────────────────────────
//
// Observed live (2026-11-08): a relay logged a task's HIVE_VERDICT and the
// no_work_needed it implied, a `docker stop` arrived about a second later, and
// the hub never got task_complete — it booked "abandoned: connection lost" and
// re-offered the issue, which was worked again 13 minutes later.
//
// The relay is synchronous from "Task ... completed" to the send: the label
// calls are execSync and stopping the agent (quitLiveCLI) sleeps for at least
// three seconds, all on the event loop. A signal that lands in that window is
// therefore handled only AFTER the send, but the old handler then called
// process.exit(0) at once, and process.exit does not wait for a socket write
// still queued behind it. So a finished task's frame is the thing to wait for:
// SHUTDOWN_HELD_FRAMES are sent with a write callback and tracked until it
// fires, and the signal handler holds the exit while any are outstanding.
//
// The wait is bounded below the container runtime's default 10 s stop timeout
// (docker and podman both SIGKILL after it). The handler can only run once the
// synchronous finish has returned, which already spent about 3 s of that, so
// 5 s keeps the whole shutdown inside the 10 s; a healthy socket drains in
// milliseconds, so the deadline only matters for a wedged one.
const SHUTDOWN_HELD_FRAMES = new Set(['task_complete', 'task_failed']);
const SHUTDOWN_FLUSH_DEADLINE_MS = Number(process.env.HIVE_RELAY_SHUTDOWN_FLUSH_MS) || 5000;
let shuttingDown = false;
const unflushedHeldFrames = new Set();

function sendHeldFrame(hub, msg, payload) {
let settle;
const entry = {
type: msg.type,
taskID: msg.task_id,
hubURL: hub.url,
flushed: new Promise(resolve => { settle = resolve; }),
};
// Added BEFORE the send, so a callback that fires synchronously still finds
// (and removes) it.
unflushedHeldFrames.add(entry);
const done = (err) => {
if (!unflushedHeldFrames.delete(entry)) return;
if (err) console.error(`The ${entry.type} frame for ${entry.taskID || '(no task)'} did not reach ${entry.hubURL || 'the hub'}: ${err.message || err}`);
settle();
};
try {
hub.ws.send(payload, done);
} catch (e) {
done(e);
throw e;
}
}

function waitForHeldFrames(deadlineAt) {
if (unflushedHeldFrames.size === 0) return Promise.resolve(true);
const remaining = deadlineAt - Date.now();
if (remaining <= 0) return Promise.resolve(false);
let timer = null;
const expired = new Promise(resolve => { timer = setTimeout(() => resolve('deadline'), remaining); });
const drained = Promise.all(Array.from(unflushedHeldFrames, e => e.flushed));
return Promise.race([drained, expired]).then(outcome => {
clearTimeout(timer);
if (outcome === 'deadline') return unflushedHeldFrames.size === 0;
// A decline sent during the wait is held too; go round again for it.
return waitForHeldFrames(deadlineAt);
});
}

function sendTo(hub, msg) {
// #5715: a local-only task (the synthetic pr-review cycle) has no
// server-issued lease, so an ownership frame naming it can only ever be
Expand All @@ -2453,6 +2523,8 @@ function sendTo(hub, msg) {
if (hub.repoScope !== null || hub.scopeSent) msg = { ...msg, scope_version: hub.scopeVersion };
}
if (msg && msg.type === 'ready' && quotaHoldActive()) return;
// A relay on its way out must not ask for work it will never run.
if (msg && msg.type === 'ready' && shuttingDown) return;
// #7996: same choke point for an expired CLI login — see enterLoginHold.
if (msg && msg.type === 'ready' && loginHoldActive()) return;
if (msg && msg.type === 'ready') {
Expand Down Expand Up @@ -2484,7 +2556,8 @@ function sendTo(hub, msg) {
// answer it with a close nobody can attribute.
console.error(`Frame ${msg.type} is still ${utf8Bytes(payload)} bytes after trimming (limit ${budget}) — the hub may close the connection with code 1009`);
}
hub.ws.send(payload);
if (msg && SHUTDOWN_HELD_FRAMES.has(msg.type)) sendHeldFrame(hub, msg, payload);
else hub.ws.send(payload);
// #7732: recorded only for a frame that actually left, so a `ready`
// dropped on a closed socket does not look like an open question.
if (msg && msg.type === 'ready') hub.readyOutstanding = true;
Expand Down Expand Up @@ -9810,6 +9883,14 @@ function handleMessage(data, hub) {
sendTo(hub, { type: 'task_failed', seq: nextSeq(), task_id: msg.task_id, reason: 'Hive disabled by contributor', failure_kind: 'environment' });
break;
}
// An assignment that answers the `ready` a just-finished task sent, and
// lands while the relay waits for that task's frame to leave: hand it
// straight back rather than start work the exit would abandon.
if (shuttingDown) {
console.log(`Declining ${taskKey(msg)} from ${hub.url} — the relay is shutting down`);
sendTo(hub, { type: 'task_failed', seq: nextSeq(), task_id: msg.task_id, reason: 'contributor relay is shutting down', failure_kind: 'environment' });
break;
}
// A standby assignment is pushed by the hub that acknowledged this relay's
// standby; the relay is not polling, so that hub is never the active slot.
const standbyPush = !!msg.standby_lane && hub.standbyAcked === true;
Expand Down Expand Up @@ -10316,8 +10397,37 @@ function cleanup() {
}
}

process.on('SIGTERM', () => { cleanup(); process.exit(0); });
process.on('SIGINT', () => { cleanup(); process.exit(0); });
// Overridable in tests only, so the handler can be driven in-process.
let exitProcess = (code) => process.exit(code);

// SIGTERM/SIGINT. With no finished task's frame still on the wire this is the
// old immediate cleanup-and-exit. With one (see SHUTDOWN_HELD_FRAMES), wait
// for its write callback, bounded by SHUTDOWN_FLUSH_DEADLINE_MS, then exit.
// shuttingDown is set first, so nothing asks for or accepts work meanwhile. A
// second signal during the wait exits at once.
function handleShutdownSignal(signal) {
if (shuttingDown) {
console.warn(`${signal} again while waiting to report a finished task — exiting now`);
cleanup();
exitProcess(0);
return;
}
shuttingDown = true;
if (unflushedHeldFrames.size === 0) {
cleanup();
exitProcess(0);
return;
}
const what = Array.from(unflushedHeldFrames, e => `${e.type} for ${e.taskID || '(no task)'} to ${e.hubURL || 'the hub'}`).join(', ');
console.log(`${signal}: waiting up to ${Math.round(SHUTDOWN_FLUSH_DEADLINE_MS / 1000)}s for ${what} to be written before exiting`);
waitForHeldFrames(Date.now() + SHUTDOWN_FLUSH_DEADLINE_MS).then(drained => {
if (!drained) console.warn(`${signal}: exiting with a finished task's frame still unwritten after ${SHUTDOWN_FLUSH_DEADLINE_MS}ms — the hub will book the task from the dropped connection`);
try { cleanup(); } finally { exitProcess(0); }
});
}

process.on('SIGTERM', () => handleShutdownSignal('SIGTERM'));
process.on('SIGINT', () => handleShutdownSignal('SIGINT'));
process.on('SIGUSR1', handleProfileSwitchSignal);

// Last-resort backstop (kubestellar/hive#5655): the scoped token must never
Expand Down Expand Up @@ -10653,6 +10763,11 @@ if (process.env.HIVE_RELAY_TEST_MODE === '1') {
blockingPromptKey,
getCLIState,
setWs: (w) => { hubs[0].ws = w; },
handleShutdownSignal,
isShuttingDown: () => shuttingDown,
unflushedHeldFrameCount: () => unflushedHeldFrames.size,
SHUTDOWN_FLUSH_DEADLINE_MS,
__setExitProcess: (fn) => { exitProcess = fn; },
getHubs: () => hubs,
getReadyRetryArmed: (hub = hubs[0]) => !!(hub && hub.readyRetryTimer),
// Peer-protocol compatibility (kubestellar/hive#2547). Exported so the
Expand Down
181 changes: 181 additions & 0 deletions bin/contributor-relay.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -10819,6 +10819,187 @@ test('#5655 a crash exit (uncaught exception) still takes the scoped token with
} finally { cleanupTmp(); }
});

// ---------------------------------------------------------------------------
// A stop that lands just after a task's verdict must not lose the verdict.
//
// Observed live 2026-11-08: the relay logged "Task ... completed —
// signal=verdict (HIVE_VERDICT: no_work_needed)", a `docker stop` arrived about
// a second later, and the hub never got task_complete. It booked the run
// "abandoned: connection lost with the task still held" and the issue was
// worked again 13 minutes later.
//
// The finish is synchronous (label calls are execSync, stopping the agent
// sleeps), so a signal that arrives during it is handled only after
// task_complete was handed to the socket — and the handler then called
// process.exit(0) with the write still queued. The child below reproduces that
// order with a real signal: SIGTERM is raised, then the task is finished in
// the same synchronous turn, and the fake socket completes its write a moment
// later, the way a real one does.
// ---------------------------------------------------------------------------

const VERDICT_SHUTDOWN_CHILD_DRIVER = `
'use strict';
const Module = require('module');
const origLoad = Module._load;
class FakeWebSocket {
static get OPEN() { return 1; }
constructor() { this.readyState = 1; }
on() {} close() {} ping() {} terminate() {} removeAllListeners() {}
send(payload, cb) {
const frame = JSON.parse(payload);
console.log('QUEUED ' + frame.type);
setTimeout(() => {
console.log('WRITTEN ' + frame.type);
if (typeof cb === 'function') cb();
}, Number(process.env.FAKE_WS_WRITE_MS));
}
}
Module._load = function (request) {
if (request === 'ws') return FakeWebSocket;
return origLoad.apply(this, arguments);
};
const relay = require(process.env.RELAY_UNDER_TEST);
Module._load = origLoad;
relay.setWs(new FakeWebSocket());
relay.setCurrentTask({ task_id: 'ct-Danathar/arch-bootc-627-1', task_gen: 1, kind: 'issue', repo: 'Danathar/arch-bootc', number: 627 });
const t0 = Date.now();
process.on('exit', () => console.log('EXITED_AFTER_MS ' + (Date.now() - t0)));
// Keep the loop alive: the process must end through the signal handler.
setInterval(() => {}, 1000);
process.kill(process.pid, 'SIGTERM');
if (process.env.SHUTDOWN_CASE === 'verdict') {
relay.finishCurrentTask({
completionSignal: 'verdict',
summary: 'Agent returned to idle (reported no_work_needed)',
tmuxLines: ['HIVE_VERDICT: no_work_needed — the requested change is not needed'],
prURL: null,
noWork: { verdict: 'no_work_needed', reason: 'the requested change is not needed' },
});
} else {
// Mid-task, nothing finished: only an ordinary progress frame is queued.
relay.sendTo(relay.getHubs()[0], { type: 'task_progress', seq: 1, task_id: 'ct-Danathar/arch-bootc-627-1', task_gen: 1, status: 'working' });
}
`;

function runVerdictShutdownChild(shutdownCase, writeMs) {
const scratchRoot = path.join(__dirname, '..', '.relay-test-tmp');
fs.mkdirSync(scratchRoot, { recursive: true });
const tmpDir = fs.mkdtempSync(path.join(scratchRoot, 'relay-verdict-shutdown-'));
const res = spawnSync(process.execPath, ['-e', VERDICT_SHUTDOWN_CHILD_DRIVER], {
env: {
...process.env,
HIVE_RELAY_TEST_MODE: '1',
CONTRIBUTOR_MODE: 'headless',
AGENT_BACKEND: 'claude',
HIVE_REGISTRATION_TOKEN: 'test-token',
HIVE_HUB: '',
HIVE_WORKSPACE_DIR: '',
RELAY_UNDER_TEST: RELAY_PATH,
SHUTDOWN_CASE: shutdownCase,
FAKE_WS_WRITE_MS: String(writeMs),
HIVE_CONTRIBUTOR_CONFIG_DIR: tmpDir,
HIVE_RELAY_PID_FILE: path.join(tmpDir, 'contributor-relay.pid'),
HIVE_RELAY_STATUS_FILE: path.join(tmpDir, 'relay-status.json'),
HIVE_HUBS_SEEN_FILE: path.join(tmpDir, 'hubs-seen.json'),
HIVE_RELAY_ACTIVITY_FILE: path.join(tmpDir, 'relay-activity.jsonl'),
HIVE_RELAY_AUDIT_DIR: path.join(tmpDir, 'relay-audit'),
HIVE_GH_TOKEN_CACHE: path.join(tmpDir, 'gh-token.cache'),
HIVE_TASK_FILE: path.join(tmpDir, 'contributor-task.json'),
HIVE_HEADLESS_STATUS_FILE: path.join(tmpDir, 'headless-status.json'),
},
encoding: 'utf8',
timeout: 30000,
});
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch (_) {}
const lines = String(res.stdout || '').split('\n');
const exitedAfter = Number((lines.find(l => l.startsWith('EXITED_AFTER_MS ')) || '').split(' ')[1]);
return { res, lines, exitedAfter };
}

test('SIGTERM during a verdict finish waits for task_complete to be written before exiting', () => {
const { res, lines, exitedAfter } = runVerdictShutdownChild('verdict', 300);
const out = `stdout:\n${res.stdout}\nstderr:\n${res.stderr}`;
assert.strictEqual(res.status, 0, `the handler must still exit 0 (${out})`);
const queued = lines.indexOf('QUEUED task_complete');
const written = lines.indexOf('WRITTEN task_complete');
const exited = lines.findIndex(l => l.startsWith('EXITED_AFTER_MS '));
assert.ok(queued >= 0, `the finish must hand task_complete to the socket (${out})`);
assert.ok(written > queued && exited > written,
`the relay exited before the task_complete write completed — the hub would book the finished task abandoned (${out})`);
assert.ok(exitedAfter < 5000, `the wait must end when the write does, not at the deadline (exited after ${exitedAfter}ms)`);
const waiting = lines.findIndex(l => /^SIGTERM: waiting up to \d+s for task_complete for ct-Danathar\/arch-bootc-627-1/.test(l));
assert.ok(waiting >= 0, `the wait must say what it is waiting for (${out})`);
assert.ok(!lines.slice(waiting).some(l => l === 'QUEUED ready'),
`no ready may go out once shutdown has started (${out})`);
});

test('SIGTERM with no finished task to report still exits at once', () => {
const { res, lines, exitedAfter } = runVerdictShutdownChild('progress', 3000);
const out = `stdout:\n${res.stdout}\nstderr:\n${res.stderr}`;
assert.strictEqual(res.status, 0, `the handler must exit 0 (${out})`);
assert.ok(lines.includes('QUEUED task_progress'), `precondition: a progress frame was queued (${out})`);
assert.ok(!lines.includes('WRITTEN task_progress'),
`a progress frame must not hold the exit — only a finished task's frame does (${out})`);
assert.ok(exitedAfter < 1500, `an idle shutdown must stay immediate (exited after ${exitedAfter}ms)`);
});

// The same handler driven in-process, for the parts a child cannot time
// cheaply: the deadline, work offered during the wait, and a second signal.
function loadRelayWithStuckSocket(env = {}) {
const relay = loadRelay({ backend: 'claude', env });
const sent = [];
// A socket whose writes never complete: the send callback is never called.
relay.setWs({ readyState: 1, send: (payload) => { sent.push(JSON.parse(payload)); } });
const exits = [];
relay.__setExitProcess((code) => { exits.push(code); });
relay.setCurrentTask({ task_id: 'ct-foo/bar-834-1', task_gen: 1, kind: 'issue', repo: 'foo/bar', number: 834 });
relay.finishCurrentTask({ completionSignal: 'verdict', summary: 'done', tmuxLines: [], prURL: null, noWork: null });
return { relay, sent, exits };
}

test('SIGTERM shutdown gives up on an unwritten task_complete at the deadline, and takes no work meanwhile', async () => {
const log = console.log; const warn = console.warn;
console.log = () => {}; console.warn = () => {};
const { relay, sent, exits } = loadRelayWithStuckSocket({ HIVE_RELAY_SHUTDOWN_FLUSH_MS: '80' });
try {
assert.ok(sent.some(m => m.type === 'task_complete'), 'precondition: the finish handed task_complete to the socket');
assert.strictEqual(relay.unflushedHeldFrameCount(), 1, 'precondition: its write has not completed');
relay.handleShutdownSignal('SIGTERM');
assert.strictEqual(relay.isShuttingDown(), true);
assert.deepStrictEqual(exits, [], 'a finished task still on the wire holds the exit');
const before = sent.length;
relay.requestWork(relay.getHubs()[0], 'test_after_sigterm');
relay.handleMessage(JSON.stringify({ type: 'task_assign', task_id: 'ct-foo/bar-900-1', kind: 'issue', repo: 'foo/bar', number: 900, title: 'late offer', prompt: 'p' }), relay.getHubs()[0]);
const after = sent.slice(before);
assert.ok(!after.some(m => m.type === 'ready'), `no ready during shutdown: ${JSON.stringify(after)}`);
assert.ok(after.some(m => m.type === 'task_failed' && m.task_id === 'ct-foo/bar-900-1' && /shutting down/.test(m.reason)),
`an assignment during shutdown is handed back: ${JSON.stringify(after)}`);
assert.strictEqual(relay.getCurrentTask(), null, 'the late assignment must not become the current task');
await new Promise(r => setTimeout(r, 250));
assert.deepStrictEqual(exits, [0], 'the wait is bounded: the relay exits once the deadline passes');
} finally { console.log = log; console.warn = warn; teardown(relay); }
});

test('a second signal during the shutdown wait exits at once', () => {
const log = console.log; const warn = console.warn;
console.log = () => {}; console.warn = () => {};
const { relay, exits } = loadRelayWithStuckSocket({ HIVE_RELAY_SHUTDOWN_FLUSH_MS: '80' });
try {
relay.handleShutdownSignal('SIGTERM');
assert.deepStrictEqual(exits, [], 'precondition: the first signal waits');
relay.handleShutdownSignal('SIGINT');
assert.deepStrictEqual(exits, [0], 'the second signal must not wait');
} finally { console.log = log; console.warn = warn; teardown(relay); }
});

test('the shutdown deadline stays below the 10 s container stop timeout', () => {
const relay = loadRelay({ backend: 'claude' });
try {
assert.ok(relay.SHUTDOWN_FLUSH_DEADLINE_MS > 0 && relay.SHUTDOWN_FLUSH_DEADLINE_MS < 10000,
`docker and podman SIGKILL 10 s after the stop signal; got ${relay.SHUTDOWN_FLUSH_DEADLINE_MS}ms`);
} finally { teardown(relay); }
});

// kubestellar/hive#5650 — the relay lost track of a live claude after the first
// task of a session, then booked the NEXT task complete off the previous one's
// transcript.
Expand Down
Loading
Loading