diff --git a/services/ensemble-service.ts b/services/ensemble-service.ts index 7892a79..f48dad4 100644 --- a/services/ensemble-service.ts +++ b/services/ensemble-service.ts @@ -55,6 +55,10 @@ const COMPLETION_SIGNAL_WINDOW_MS = 180_000 // unaffected: a team that is really done still ends within seconds. const SINGLE_SIGNAL_IDLE_THRESHOLD_MS = 480_000 const TWO_SIGNAL_IDLE_THRESHOLD_MS = 300_000 +/** A sign-off is short; long analysis prose that mentions "done" is not one. */ +const MAX_COMPLETION_SIGNAL_LENGTH = 400 +/** Machinery an agent can discuss without meaning "I am finished". */ +const ORCHESTRATION_VOCABULARY = /---STATUS|COLLAB_DONE|team-say|team-read|DONE protocol|\bsentinel\b/i const MIN_MESSAGES_BEFORE_AUTO_DISBAND = 10 // Explicit sentinel: when every active agent sends this exact marker as a full // message, the team auto-disbands immediately — no idle wait, no minimum @@ -127,6 +131,14 @@ interface CompletionSignal { */ function isCompletionStatement(content: string): boolean { if (CLOSING_PATTERNS.some(pattern => pattern.test(content))) return true + // Below this line only the heuristic path remains, so the shape of the message + // matters. A sign-off is short: a long analysis that happens to contain "done" + // is an agent working, not an agent finishing. + if (content.length > MAX_COMPLETION_SIGNAL_LENGTH) return false + // And an agent describing the orchestration is not reporting on itself. Seen on + // a review task pointed at collab-poll.sh: both agents quoted the script's own + // ---STATUS:{ACTIVE,QUIET,DONE,WAITING} sentinel, which ended the run. + if (ORCHESTRATION_VOCABULARY.test(content)) return false if (!COMPLETION_PATTERNS.some(pattern => pattern.test(content))) return false return !CONTINUATION_PATTERNS.some(pattern => pattern.test(content)) } @@ -271,6 +283,22 @@ class EnsembleService { private shouldAutoDisband(team: EnsembleTeam): boolean { const messages = getMessages(team.id) + // `ensemble steer` exists so a user can redirect a running team, so a sentinel or + // sign-off from before that redirect no longer describes the work: an agent that + // had already finished is not finished with the new instruction. + // + // This gates completion EVIDENCE only. The message count and the idle clock still + // read the whole run: filtering those too resets the min-message floor on every + // steer, and a team that finishes shortly after being steered — in words rather + // than the sentinel — then never meets the floor again and stays active forever. + const lastUserMessageMs = messages + .filter(message => message.from === 'user' && message.timestamp) + .reduce((latest, message) => Math.max(latest, new Date(message.timestamp).getTime() || 0), 0) + const isAfterLastRedirect = (message: EnsembleMessage): boolean => { + if (!lastUserMessageMs) return true + const ts = message.timestamp ? new Date(message.timestamp).getTime() : NaN + return Number.isNaN(ts) || ts >= lastUserMessageMs + } const nonEnsembleMessages = messages.filter(message => message.from !== 'ensemble') const lastMessage = nonEnsembleMessages[nonEnsembleMessages.length - 1] if (!lastMessage) return false @@ -283,7 +311,7 @@ class EnsembleService { const activeNames = new Set(team.agents.filter(a => a.status === 'active').map(a => a.name)) const sentinelSenders = new Set( messages - .filter(m => activeNames.has(m.from) && m.content.trim() === EXPLICIT_DONE_SENTINEL) + .filter(m => activeNames.has(m.from) && isAfterLastRedirect(m) && m.content.trim() === EXPLICIT_DONE_SENTINEL) .map(m => m.from), ) if (activeNames.size >= 2 && sentinelSenders.size >= activeNames.size) return true @@ -303,7 +331,8 @@ class EnsembleService { const idleForMs = Date.now() - lastTimestamp const activeAgentNames = new Set(activeAgents.map(agent => agent.name)) const completionSignals = messages - .filter(message => activeAgentNames.has(message.from) && this.hasCompletionSignal(message.content)) + .filter(message => activeAgentNames.has(message.from) && isAfterLastRedirect(message) + && this.hasCompletionSignal(message.content)) .map(message => ({ agentName: message.from, timestamp: message.timestamp ? new Date(message.timestamp).getTime() : NaN, @@ -317,7 +346,7 @@ class EnsembleService { // third agent mid-task. The exact sentinel above is the fast path; these // patterns are only a safety net for teams that go quiet without sending it. if (idleForMs <= TWO_SIGNAL_IDLE_THRESHOLD_MS) return false - if (this.hasTwoRecentCompletionSignals(completionSignals)) return true + if (this.hasRecentCompletionSignalsFromAll(completionSignals, activeAgentNames.size)) return true if (idleForMs <= SINGLE_SIGNAL_IDLE_THRESHOLD_MS) return false return completionSignals.length >= 1 } @@ -326,11 +355,22 @@ class EnsembleService { return isCompletionStatement(content) } - private hasTwoRecentCompletionSignals(signals: CompletionSignal[]): boolean { - for (let i = 0; i < signals.length; i++) { - for (let j = i + 1; j < signals.length; j++) { - if (signals[j].timestamp - signals[i].timestamp > COMPLETION_SIGNAL_WINDOW_MS) break - if (signals[i].agentName !== signals[j].agentName) return true + /** + * True when EVERY active agent produced a completion signal inside one + * COMPLETION_SIGNAL_WINDOW_MS window — the same bar the sentinel path uses. + * Two agents agreeing is not the team agreeing: in a trio that ends the run + * for the third, which is what the comment above this call warns about. + */ + private hasRecentCompletionSignalsFromAll( + signals: CompletionSignal[], activeAgentCount: number, + ): boolean { + if (activeAgentCount < 2) return false + for (let start = 0; start < signals.length; start++) { + const seen = new Set([signals[start].agentName]) + for (let end = start + 1; end < signals.length; end++) { + if (signals[end].timestamp - signals[start].timestamp > COMPLETION_SIGNAL_WINDOW_MS) break + seen.add(signals[end].agentName) + if (seen.size >= activeAgentCount) return true } } return false @@ -1225,4 +1265,6 @@ export const __testing = { SINGLE_SIGNAL_IDLE_THRESHOLD_MS, COMPLETION_PATTERNS, CONTINUATION_PATTERNS, + MAX_COMPLETION_SIGNAL_LENGTH, + ORCHESTRATION_VOCABULARY, } diff --git a/tests/ensemble.test.ts b/tests/ensemble.test.ts index e8469af..782270b 100644 --- a/tests/ensemble.test.ts +++ b/tests/ensemble.test.ts @@ -69,6 +69,23 @@ function fillerMessages(count: number, teamId = 'team-1'): EnsembleMessage[] { ) } +/** + * Filler older than the messages a wording-path test cares about. The wording path + * only fires while idle sits between TWO_SIGNAL_IDLE_THRESHOLD_MS and + * SINGLE_SIGNAL_IDLE_THRESHOLD_MS, so the signals under test must be the newest + * messages in the feed. + */ +function olderFiller(count: number, teamId = 'team-1'): EnsembleMessage[] { + return Array.from({ length: count }, (_, i) => + makeMessage({ + teamId, + from: i % 2 === 0 ? 'codex-1' : 'claude-2', + content: `analysis step ${i}`, + timestamp: `2026-03-18T11:50:${String(i).padStart(2, '0')}.000Z`, + }), + ) +} + /** A three-agent team, for checks that must not assume a pair. */ function makeTrioTeam(): EnsembleTeam { return makeTeam({ @@ -318,6 +335,116 @@ describe('shouldAutoDisband() — tested via checkIdleTeams()', () => { expect(appendedMessages.some(m => m.content.includes('Auto-disband'))).toBe(false) }) + it('does NOT auto-disband a trio when only two of three used completion wording', async () => { + // The sentinel path already requires every agent. The wording path did not, so a + // quiet trio could still lose its third agent to the other two agreeing. + const team = makeTrioTeam() + const messages: EnsembleMessage[] = [ + ...olderFiller(8), + makeMessage({ from: 'codex-1', teamId: 'team-1', content: 'Ik ben klaar', timestamp: '2026-03-18T11:58:00.000Z' }), + makeMessage({ from: 'claude-2', teamId: 'team-1', content: 'Ik ben ook klaar', timestamp: '2026-03-18T11:58:30.000Z' }), + ] + + const { mod, appendedMessages } = await setupServiceWithMocks(team, messages) + await mod.checkIdleTeams() + + expect(appendedMessages.some(m => m.content.includes('Auto-disband'))).toBe(false) + }) + + it('auto-disbands a trio once all three used completion wording', async () => { + const team = makeTrioTeam() + const messages: EnsembleMessage[] = [ + ...olderFiller(8), + makeMessage({ from: 'codex-1', teamId: 'team-1', content: 'Ik ben klaar', timestamp: '2026-03-18T11:58:00.000Z' }), + makeMessage({ from: 'claude-2', teamId: 'team-1', content: 'Ik ben ook klaar', timestamp: '2026-03-18T11:58:20.000Z' }), + makeMessage({ from: 'grok-3', teamId: 'team-1', content: 'Ik ben klaar', timestamp: '2026-03-18T11:58:40.000Z' }), + ] + + const { mod, appendedMessages } = await setupServiceWithMocks(team, messages) + await mod.checkIdleTeams() + + expect(appendedMessages.some(m => m.content.includes('Auto-disband'))).toBe(true) + }) + + it('still ends a team that finishes shortly after being steered', async () => { + // The redirect cutoff gates completion evidence, not the message count or the + // idle clock. Filtering those too resets the min-message floor on every steer, so + // a team that wraps up in words rather than the sentinel right after a redirect + // never meets the floor again and stays active forever. + const team = makeTeam() + const messages: EnsembleMessage[] = [ + ...Array.from({ length: 12 }, (_, i) => makeMessage({ + teamId: 'team-1', + from: i % 2 === 0 ? 'codex-1' : 'claude-2', + content: `analysis step ${i}`, + timestamp: `2026-03-18T11:30:${String(i).padStart(2, '0')}.000Z`, + })), + makeMessage({ from: 'user', teamId: 'team-1', content: 'also check the retry path', timestamp: '2026-03-18T11:50:00.000Z' }), + makeMessage({ from: 'codex-1', teamId: 'team-1', content: 'ack: checking the retry path', timestamp: '2026-03-18T11:50:10.000Z' }), + makeMessage({ from: 'codex-1', teamId: 'team-1', content: 'Ik ben klaar', timestamp: '2026-03-18T11:52:00.000Z' }), + makeMessage({ from: 'claude-2', teamId: 'team-1', content: 'Ik ben ook klaar', timestamp: '2026-03-18T11:52:30.000Z' }), + ] + + const { mod, appendedMessages } = await setupServiceWithMocks(team, messages) + await mod.checkIdleTeams() + + expect(appendedMessages.some(m => m.content.includes('Auto-disband'))).toBe(true) + }) + + it('still ends a single-agent team through the idle path', async () => { + // Both the sentinel path and the all-agents wording path require two or more + // agents, so a solo team can only end via the long-idle fallback. Locking that in: + // a solo run must not become unendable. + const team = makeTeam({ + agents: [{ agentId: 'a1', name: 'codex-1', program: 'codex', role: 'lead', hostId: 'local', status: 'active' }], + }) + const messages: EnsembleMessage[] = [ + ...Array.from({ length: 12 }, (_, i) => makeMessage({ + teamId: 'team-1', from: 'codex-1', content: `analysis step ${i}`, + timestamp: `2026-03-18T11:30:${String(i).padStart(2, '0')}.000Z`, + })), + makeMessage({ from: 'codex-1', teamId: 'team-1', content: 'Ik ben klaar', timestamp: '2026-03-18T11:50:00.000Z' }), + ] + + const { mod, appendedMessages } = await setupServiceWithMocks(team, messages) + await mod.checkIdleTeams() + + expect(appendedMessages.some(m => m.content.includes('Auto-disband'))).toBe(true) + }) + + it('voids done sentinels sent before the user steered the team', async () => { + // An agent that had already signalled is not finished with the new instruction: + // the team must not disband while it answers. + const team = makeTeam() + const messages: EnsembleMessage[] = [ + ...fillerMessages(8), + makeMessage({ from: 'codex-1', teamId: 'team-1', content: '<>', timestamp: '2026-03-18T12:04:20.000Z' }), + makeMessage({ from: 'claude-2', teamId: 'team-1', content: '<>', timestamp: '2026-03-18T12:04:30.000Z' }), + makeMessage({ from: 'user', teamId: 'team-1', content: 'One more thing: check the retry path', timestamp: '2026-03-18T12:04:45.000Z' }), + makeMessage({ from: 'claude-2', teamId: 'team-1', content: 'ack: checking the retry path now', timestamp: '2026-03-18T12:04:50.000Z' }), + ] + + const { mod, appendedMessages } = await setupServiceWithMocks(team, messages) + await mod.checkIdleTeams() + + expect(appendedMessages.some(m => m.content.includes('Auto-disband'))).toBe(false) + }) + + it('still disbands on sentinels sent after the user steered the team', async () => { + const team = makeTeam() + const messages: EnsembleMessage[] = [ + ...fillerMessages(8), + makeMessage({ from: 'user', teamId: 'team-1', content: 'wrap it up please', timestamp: '2026-03-18T12:04:20.000Z' }), + makeMessage({ from: 'codex-1', teamId: 'team-1', content: '<>', timestamp: '2026-03-18T12:04:40.000Z' }), + makeMessage({ from: 'claude-2', teamId: 'team-1', content: '<>', timestamp: '2026-03-18T12:04:50.000Z' }), + ] + + const { mod, appendedMessages } = await setupServiceWithMocks(team, messages) + await mod.checkIdleTeams() + + expect(appendedMessages.some(m => m.content.includes('Auto-disband'))).toBe(true) + }) + it('auto-disbands when two different agents send completion signals and the team went quiet', async () => { const team = makeTeam() const messages: EnsembleMessage[] = [ diff --git a/tests/premature-disband.test.ts b/tests/premature-disband.test.ts index 8aa70d8..c243855 100644 --- a/tests/premature-disband.test.ts +++ b/tests/premature-disband.test.ts @@ -60,3 +60,32 @@ it('idle thresholds leave room for an agent that is reading', () => { ) }) }) + +// A message about the orchestration is not a message about the agent. Seen on a +// review task pointed at scripts/collab-poll.sh: both agents quoted its own +// ---STATUS:{ACTIVE,QUIET,DONE,WAITING} sentinel, and the run ended mid-analysis. +describe('describing the machinery is not finishing', () => { + it('quoting the DONE sentinel of a script under review is not an ending', () => { + assert.strictEqual( + hasCompletionSignal('collab-poll.sh emits TSV lines terminated by a ---STATUS:{ACTIVE,QUIET,DONE,WAITING} sentinel'), + false, + ) + }) + + it('talking about the DONE protocol is not an ending', () => { + assert.strictEqual(hasCompletionSignal('I will follow the DONE protocol once we agree'), false) + }) + + it('announcing a team-say is not an ending', () => { + assert.strictEqual(hasCompletionSignal('I will send the done sentinel via team-say when you agree'), false) + }) + + it('a long analysis that merely contains finishing words is not an ending', () => { + const analysis = 'Exchange 2: the two scripts disagree about what finished means, and the cursor is done differently in each. '.padEnd(600, 'x') + assert.strictEqual(hasCompletionSignal(analysis), false) + }) + + it('but a short genuine sign-off still counts', () => { + assert.strictEqual(hasCompletionSignal('Ik ben klaar'), true) + }) +})