diff --git a/ds-bot/src/host/bot-tasks.js b/ds-bot/src/host/bot-tasks.js index ff427e8d9..59fc06235 100644 --- a/ds-bot/src/host/bot-tasks.js +++ b/ds-bot/src/host/bot-tasks.js @@ -527,6 +527,8 @@ export function install(rt) { } } + const hasInboxInput = agent => Boolean(agent?.inbox?.nextStep?.length || agent?.inbox?.nextTurn?.length) + // Settles a task whose turn ended while the plugin was reloading (durable log). async function reconcileEndedTask(sessionId, ended) { if (rt.readOnly?.() !== undefined) return false @@ -550,6 +552,11 @@ export function install(rt) { .join('\n\n') .trim() const failure = reason?.kind === 'error' ? turnFailure(reason.error) : undefined + if (reason?.kind !== 'error' && reason?.kind !== 'aborted') { + const agent = await rt.resolveAgent(sessionId).catch(() => undefined) + // A child work can already be closed while its reply still waits for replay. + if (hasInboxInput(agent) || rt.unreadOf?.(sessionId)?.length > 0) return false + } await handleTurnEnd(bot, task, reason, failure, said) return true } @@ -571,8 +578,7 @@ export function install(rt) { if ((rt.runningJobs?.(sid)?.length ?? 0) > 0) continue if ((rt.unreadOf?.(sid)?.length ?? 0) > 0) continue const agent = await rt.resolveAgent(sid).catch(() => undefined) - const pendingInbox = (agent?.inbox?.nextStep?.length ?? 0) + (agent?.inbox?.nextTurn?.length ?? 0) - if (agent?.status === 'running' || pendingInbox > 0) continue + if (agent?.status === 'running' || hasInboxInput(agent)) continue if (hasOpenOutgoingWorks(bot, task)) { if (task.status !== 'waiting_work') { task.status = 'waiting_work' diff --git a/ds-bot/src/host/delivery.js b/ds-bot/src/host/delivery.js index 0c507a2a4..f7fe96327 100644 --- a/ds-bot/src/host/delivery.js +++ b/ds-bot/src/host/delivery.js @@ -3,6 +3,7 @@ import { randomUUID } from 'node:crypto' import { createUserMessage } from '@deepseek-ai/dsh-llm' import { HISTORY_TIMEOUT_MS, LONG_HEAD, MESSAGE_INLINE_MAX, READ_LINE_MAX, SOURCE_KIND, TEXT_KEPT, TEXT_SHOW_MAX } from './constants.js' import { headOf, textOf } from './text.js' +import { sameConversation } from './threads.js' export function install(rt) { const { ctx, state } = rt @@ -57,19 +58,63 @@ export function install(rt) { .map(entry => ({ ...entry, delivered: true }))) // A message shows the text its delivery carried: the last record of its id written before // a delivery mark. A merged request whose replacement never went out keeps the text that did. - function pairMessages(a, b) { + function pairRecords(a, b) { const written = new Map() const shown = new Map() + const marks = [] for (const record of rt.ledger.records(pairBook(a, b))) { if (record.event === 'delivered') { if (written.has(record.id)) shown.set(record.id, written.get(record.id)) + marks.push(record) } else if (record.event === undefined) { written.set(record.id, record) if (record.delivered === true) shown.set(record.id, record) } } - return [...shown.values()] + return { written, shown, marks } } + const pairMessages = (a, b) => [...pairRecords(a, b).shown.values()] + + function deliveredReplyOf(fromId, toId, replyTo) { + if (typeof fromId !== 'string' || fromId === '' || typeof replyTo !== 'string' || replyTo === '') return undefined + if (typeof toId === 'string' && toId !== '') { + if (rt.botOf(fromId) === undefined || rt.botOf(toId) === undefined) { + return (state.exchanges ?? []).findLast(entry => entry.from === fromId && entry.to === toId && entry.role === 'reply' && entry.replyTo === replyTo) + } + return pairMessages(fromId, toId).findLast(record => record.from === fromId && record.to === toId && record.role === 'reply' && record.replyTo === replyTo) + } + if (rt.botOf(fromId) !== undefined) { + for (const otherId of Object.keys(state.bots ?? {})) { + if (otherId === fromId) continue + const found = pairMessages(fromId, otherId).findLast(record => record.from === fromId && record.to === otherId && record.role === 'reply' && record.replyTo === replyTo) + if (found !== undefined) return found + } + } + return (state.exchanges ?? []).findLast(entry => entry.from === fromId && entry.role === 'reply' && entry.replyTo === replyTo) + } + + const wasReplyDelivered = (fromId, toId, replyTo) => deliveredReplyOf(fromId, toId, replyTo) !== undefined + + function deliveredFailuresOf(fromId, toId, requestId, targetSessionId, parentWork) { + if (typeof fromId !== 'string' || typeof toId !== 'string' || typeof requestId !== 'string' || requestId === '') return [] + if (rt.botOf(fromId) === undefined || rt.botOf(toId) === undefined) return [] + const wantedWork = typeof parentWork === 'string' && parentWork !== '' ? parentWork : undefined + const { written, marks } = pairRecords(fromId, toId) + const matches = [] + for (const mark of marks) { + if (mark.failureRequestId !== requestId) continue + const markWork = typeof mark.work === 'string' && mark.work !== '' ? mark.work : undefined + if (markWork !== wantedWork) continue + const message = written.get(mark.id) + if (message?.from === fromId && message?.to === toId && sameConversation(state, toId, mark.sessionId, targetSessionId)) { + matches.push({ id: mark.id, replyTo: message.replyTo, work: markWork, sessionId: mark.sessionId }) + } + } + return matches + } + + const wasFailureRequestDelivered = (fromId, toId, requestId, targetSessionId, parentWork) => + deliveredFailuresOf(fromId, toId, requestId, targetSessionId, parentWork).length > 0 // `sessionId` defaults to the recipient's own chat; group turns name its group Session. // `deferred`: a request the Bot already read mid-turn, handed back to answer in a @@ -85,7 +130,7 @@ export function install(rt) { // result that came back goes on with. `merged`: a request that takes in an earlier // one its recipient had not read, under that one's exchange. `fromPart`: the Session // of the sending Bot, which a reply comes back to when that is a background part. - async function deliver({ fromId, fromName: senderName, toId, sessionId, text, role, replyTo, roomId, audience, note, exchangeId = randomUUID(), steer = false, deferred = false, resumed = false, retried = false, resent = false, merged = false, abandoned = false, recordExchange = true, askedBy, fromRoomId, fromPart, images = [], files = [], length, work, taskId, imOrigin }) { + async function deliver({ fromId, fromName: senderName, toId, sessionId, text, role, replyTo, roomId, audience, note, exchangeId = randomUUID(), steer = false, deferred = false, resumed = false, retried = false, resent = false, merged = false, abandoned = false, recordExchange = true, askedBy, fromRoomId, fromPart, images = [], files = [], length, work, taskId, imOrigin, failureRequestId }) { const from = rt.botOf(fromId) const to = rt.botOf(toId) if (to === undefined) throw new Error(`Unknown Bot ${toId}`) @@ -139,7 +184,14 @@ export function install(rt) { else agent.followup(message) await ctx.sessions.flush(agent.session) if (book !== undefined) { - rt.ledger.append(book, { event: 'delivered', id: exchangeId, sessionId: targetSessionId, time: Date.now() }) + rt.ledger.append(book, { + event: 'delivered', + id: exchangeId, + sessionId: targetSessionId, + time: Date.now(), + ...(typeof failureRequestId === 'string' && failureRequestId !== '' ? { failureRequestId } : {}), + ...(typeof work === 'string' && work !== '' ? { work } : {}), + }) // The message went out either way; a mark that is lost only hides it from the dialog. await rt.ledger.written(book).catch(() => {}) } else if (recordExchange && EXCHANGE_ROLES.has(role) && !deferred && !resumed && !retried) { @@ -199,8 +251,8 @@ export function install(rt) { } } - // The text of a stored file, up to what the dialogs show. - async function fileText(file) { + // The text of a stored file, limited to what the dialogs show by default. + async function fileText(file, maxLength = TEXT_SHOW_MAX) { const attachments = ctx.get?.('attachments') if (attachments === undefined) return undefined const decoder = new TextDecoder() @@ -208,7 +260,7 @@ export function install(rt) { try { for await (const chunk of attachments.readFileStream(file, AbortSignal.timeout(HISTORY_TIMEOUT_MS))) { text += decoder.decode(chunk, { stream: true }) - if (text.length > TEXT_SHOW_MAX) return { text: headOf(text, TEXT_SHOW_MAX), whole: false } + if (text.length > maxLength) return { text: headOf(text, maxLength), whole: false } } } catch (error) { rt.warn('ds-bot: file %s not read: %s', file?.name, error) @@ -231,5 +283,5 @@ export function install(rt) { return { ...record, ledger: true } } - Object.assign(rt, { resolveAgent, deliver, exchangeText, attachLong, fileText, exchangesOf, exchangeOf }) + Object.assign(rt, { resolveAgent, deliver, exchangeText, attachLong, fileText, exchangesOf, exchangeOf, deliveredReplyOf, wasReplyDelivered, deliveredFailuresOf, wasFailureRequestDelivered }) } diff --git a/ds-bot/src/host/handoffs.js b/ds-bot/src/host/handoffs.js index 5455a2b83..cbdd0d527 100644 --- a/ds-bot/src/host/handoffs.js +++ b/ds-bot/src/host/handoffs.js @@ -197,6 +197,7 @@ export function install(rt) { if (work === undefined) return const { parent } = work delete works()[id] + for (const child of Object.values(works())) if (child.parent === id) delete child.parent const above = works()[parent] if (above !== undefined) above.open = above.open.filter(other => other !== id) for (const result of work.results ?? []) whole.delete(result.id) diff --git a/ds-bot/src/host/resume.js b/ds-bot/src/host/resume.js index 26601ac82..7655e1a93 100644 --- a/ds-bot/src/host/resume.js +++ b/ds-bot/src/host/resume.js @@ -16,7 +16,7 @@ import { join } from 'node:path' import { HISTORY_TIMEOUT_MS, MESSAGE_INLINE_MAX, SOURCE_KIND, STEER_CUT } from './constants.js' import { answerSender, cutOffText, failureLine, turnFailure } from './turns.js' import { textOf } from './text.js' -import { threadCurrentOfPart } from './threads.js' +import { threadCurrentOfPart, threadOfPart } from './threads.js' // Long enough for DSH to finish opening its workspaces after a start. export const RESUME_DELAY_MS = 3000 @@ -319,26 +319,79 @@ export function install(rt) { return Array.isArray(seen?.events) ? seen.events : undefined } + const isTerminalReason = reason => reason !== undefined && reason.kind !== 'interrupted' + && !(reason.kind === 'aborted' && reason.reason?.kind === 'disposed') + + const whereFor = (asker) => { + const sender = asker?.senderSessionId + if (sender === undefined) return { sender: undefined, back: undefined, where: undefined } + const back = rt.roomOf(asker.senderRoomId)?.sessions?.[sender] + // The answer would have gone to the part of the asking Bot's chat the request came + // from, which is not its foreground chat when that part is a background one. + const where = back ?? (asker.senderPart === undefined ? undefined : threadCurrentOfPart(rt.botOf(sender), asker.senderPart)) ?? rt.chatOf(sender) + return { sender, back, where } + } + + async function deliveredAnswer(botId, sender, workId) { + const delivered = rt.deliveredReplyOf?.(botId, sender, workId) + if (delivered === undefined) { + return { answered: rt.deliveredReplyOf === undefined && sender !== undefined && rt.wasReplyDelivered?.(botId, sender, workId) === true } + } + const file = delivered.files?.[0] + const stored = file === undefined ? undefined : await rt.fileText(file, delivered.length) + if (stored?.whole !== true) return { answered: true, text: delivered.text } + // Delivery warnings are prepended to the preview and included in its declared length. + const prefixLength = (delivered.length ?? stored.text.length) - stored.text.length + const prefix = prefixLength > 0 ? delivered.text.slice(0, prefixLength) : '' + return { answered: true, text: prefix + stored.text } + } + + async function alreadyAnswered(botId, sender, where, back, roomId, workId) { + const delivered = await deliveredAnswer(botId, sender, workId) + if (delivered.answered) return delivered + if (where === undefined) return { answered: false } + const sent = await logOf(where) + if (sent !== undefined && answeredIn(sent, workId)) return { answered: true } + const segments = back === undefined + ? (threadOfPart(rt.botOf(sender), where)?.segments ?? []) + : (rt.roomOf(roomId)?.segments?.[sender] ?? []) + for (const segment of [...segments].reverse()) { + if (segment?.sessionId === undefined || segment.sessionId === where) continue + const earlier = await logOf(segment.sessionId) + if (earlier !== undefined && answeredIn(earlier, workId)) return { answered: true } + } + return { answered: false } + } + // A turn that ended while DS Bot was not watching it: its answer may exist and never // have gone to the Bot that waits for it. The Session's durable log says what it did; // the work it was for is settled once, the way a finished turn settles it. - async function reconcileEnded(sessionId, record, ended) { + async function reconcileEnded(sessionId, record, ended, prefetchedEvents) { const trigger = record.trigger if (trigger === undefined) return const workId = rt.workOf(trigger) - if (!rt.ownsWork(record.botId, workId)) return + const ownsPrimary = rt.ownsWork(record.botId, workId) + const hasOwnedWork = ownsPrimary || Object.values(rt.state?.works ?? {}).some(work => work.to === record.botId) + if (!hasOwnedWork) return const exchangeId = trigger.exchangeId - const asker = rt.workAsker(workId) - const sender = asker?.senderSessionId - if (exchangeId === undefined || sender === undefined || rt.botOf(sender) === undefined) return - const name = rt.botOf(rt.selfOf(sessionId))?.name ?? 'The other Bot' + if (exchangeId === undefined) return + const primaryAsker = ownsPrimary ? rt.workAsker(workId) : undefined + const primarySender = primaryAsker?.senderSessionId + const canSettlePrimary = ownsPrimary && primarySender !== undefined && rt.botOf(primarySender) !== undefined + const name = rt.botOf(rt.selfOf(sessionId))?.name ?? rt.botOf(record.botId)?.name ?? 'The other Bot' // DSH stopped that turn as it shut down: it is picked up at the next start. if (ended.reason?.kind === 'aborted' && ended.reason.reason?.kind === 'disposed') { update(sessionId, () => record) return } - const events = await logOf(sessionId) + const events = prefetchedEvents ?? await logOf(sessionId) if (events === undefined) { + if (!canSettlePrimary) return + const delivered = await deliveredAnswer(record.botId, primarySender, workId) + if (delivered.answered) { + rt.closeWork?.(workId, name, delivered.text || rt.answerText(workId, '', name)) + return + } // The record of that work cannot be read: whether it answered cannot be // established. The sender is told that, and the work is not run again. const why = 'the record of that work could not be read' @@ -352,7 +405,7 @@ export function install(rt) { where: room === undefined ? 'own chat' : `group "${room.name}"`, sessionId, }) - answerSender(rt, sessionId, asker, text) + answerSender(rt, sessionId, primaryAsker, text) return } const turn = turnOf(events, exchangeId) @@ -360,16 +413,16 @@ export function install(rt) { // waiting for it keeps what it had. if (turn === undefined) return const said = turn.texts.join('\n\n') - // The answer went up in another run: the work closes with what it said, and - // nothing is sent twice. - const back = rt.roomOf(asker.senderRoomId)?.sessions?.[sender] - // The answer would have gone to the part of the asking Bot's chat the request came - // from, which is not its foreground chat when that part is a background one. - const where = back ?? (asker.senderPart === undefined ? undefined : threadCurrentOfPart(rt.botOf(sender), asker.senderPart)) ?? rt.chatOf(sender) - const sent = await logOf(where) - if (sent !== undefined && answeredIn(sent, workId)) { - rt.closeWork?.(workId, name, rt.answerText(workId, said, name)) - return + let primarySettled = !canSettlePrimary + if (canSettlePrimary) { + // The answer went up in another run (even before a part switch): the work closes + // with what it said, and nothing is sent twice. + const { sender, back, where } = whereFor(primaryAsker) + const answered = await alreadyAnswered(record.botId, sender, where, back, primaryAsker.senderRoomId, workId) + if (answered.answered) { + rt.closeWork?.(workId, name, answered.text || rt.answerText(workId, said, name)) + primarySettled = true + } } const reason = turn.reason // A turn that went on to a fresh part goes on there; it is not settled here. @@ -377,20 +430,48 @@ export function install(rt) { // The turn was cut off before it finished: the work is picked up again, as after // a restart that cut it off. if (reason === undefined || reason.kind === 'interrupted') { - await resumeTurn(sessionId, record) + if (!primarySettled) await resumeTurn(sessionId, record) return } - if (reason.kind === 'error') { - const failure = turnFailure(reason.error) - answerSender(rt, sessionId, asker, `${name} could not answer your message: its model call failed (${failureLine(failure)}). Tell the user if it matters; do not retry right away.`) - return + const failure = reason.kind === 'error' ? turnFailure(reason.error) : undefined + const dedupKey = failure !== undefined + ? `reconcile:${sessionId}:${turn.turn ?? ''}:error:${failure.requestId ?? failure.code}` + : reason.kind === 'aborted' + ? `reconcile:${sessionId}:${turn.turn ?? ''}:aborted` + : undefined + const settleFailureOrAbort = (asker) => { + if (failure !== undefined) { + answerSender(rt, sessionId, asker, `${name} could not answer your message: its model call failed (${failureLine(failure)}). Tell the user if it matters; do not retry right away.`, undefined, { failure, dedupKey }) + } else { + // The turn was stopped before it could answer: the sender is told, as it is told + // when DS Bot watches the turn end. + answerSender(rt, sessionId, asker, `${name} was stopped before it could answer your message.`, undefined, { dedupKey }) + } } - if (reason.kind === 'aborted') { - // The turn was stopped before it could answer: the sender is told, as it is told - // when DS Bot watches the turn end. - answerSender(rt, sessionId, asker, `${name} was stopped before it could answer your message.`) + if (reason.kind === 'error' || reason.kind === 'aborted') { + if (!primarySettled) settleFailureOrAbort(primaryAsker) + // Steered requests that joined the same failed or aborted turn also owe their own + // callers settlement, so distinct parent works cannot stall across a restart. + const seen = new Set([workId, exchangeId]) + for (const took of turn.took) { + if (took?.kind !== SOURCE_KIND || (took.role !== 'request' && took.role !== 'stopped') || took.audience === 'user') continue + const joinedId = took.exchangeId + if (typeof joinedId !== 'string' || joinedId === '' || seen.has(joinedId)) continue + seen.add(joinedId) + if (!rt.ownsWork(record.botId, joinedId)) continue + const joinedAsker = rt.workAsker(joinedId) + const { sender, back, where } = whereFor(joinedAsker) + if (sender === undefined || rt.botOf(sender) === undefined) continue + const answered = await alreadyAnswered(record.botId, sender, where, back, joinedAsker.senderRoomId, joinedId) + if (answered.answered) { + rt.closeWork?.(joinedId, name, answered.text || rt.answerText(joinedId, '', name)) + continue + } + settleFailureOrAbort(joinedAsker) + } return } + if (primarySettled) return // A turn cut off at the output limit is not a finish: the answer says so, and the // diagnostics record it, with the route the Session's log holds for it. const cutOff = reason?.kind === 'max-tokens' @@ -422,7 +503,51 @@ export function install(rt) { } const text = rt.answerText(workId, answer, name) const long = text.length > MESSAGE_INLINE_MAX ? await rt.attachLong(text, `Answer from ${name}.txt`) : undefined - answerSender(rt, sessionId, asker, text, long) + answerSender(rt, sessionId, primaryAsker, text, long) + } + + // Checks whether the work or reply-expecting trigger a stale inflight turn names has + // already been answered in the pair ledger or the asker's session log. For a + // continuation turn (`role: 'reply' | 'report' | 'resume'`), the owed answer is the + // upstream work and its asker, never the child reply that triggered the turn. + async function settleDeliveredTrigger(record, trigger) { + const workId = rt.workOf(trigger) + if (workId !== undefined) { + const owns = rt.ownsWork(record.botId, workId) + const asker = owns ? rt.workAsker(workId) : undefined + const sender = asker?.senderSessionId + ?? (trigger.role === 'request' && trigger.audience !== 'user' ? trigger.senderSessionId : undefined) + const name = rt.botOf(record.botId)?.name ?? 'The other Bot' + if (owns && asker !== undefined && sender !== undefined && rt.botOf(sender) !== undefined) { + const { back, where } = whereFor(asker) + const answered = await alreadyAnswered(record.botId, sender, where, back, asker.senderRoomId, workId) + if (answered.answered) { + rt.closeWork?.(workId, name, answered.text || rt.answerText(workId, '', name)) + return true + } + return false + } + const delivered = await deliveredAnswer(record.botId, sender, workId) + if (delivered.answered) { + if (owns) rt.closeWork?.(workId, name, delivered.text || rt.answerText(workId, '', name)) + return true + } + return false + } + if (trigger.role === 'stopped' && trigger.exchangeId !== undefined && trigger.senderSessionId !== undefined) { + return rt.wasReplyDelivered?.(record.botId, trigger.senderSessionId, trigger.exchangeId) === true + } + return false + } + + async function settleEnded(sessionId, record, ended, events) { + const task = rt.taskOwners?.has(sessionId) + if (!task || isTerminalReason(ended.reason)) { + await reconcileEnded(sessionId, record, ended, events).catch(error => rt.warn('ds-bot: settling the work of %s failed: %s', sessionId, error)) + } + if (task) { + await rt.reconcileEndedTask?.(sessionId, ended).catch(error => rt.warn('ds-bot: settling task %s failed: %s', sessionId, error)) + } } async function resumeAll() { @@ -453,20 +578,29 @@ export function install(rt) { if (first?.type === 'turn/end') { // The turn ended while DS Bot was not watching it: what it did is settled from // the Session's durable log. - if (rt.taskOwners?.has(sessionId)) { - await rt.reconcileEndedTask?.(sessionId, first).catch(error => rt.warn('ds-bot: settling task %s failed: %s', sessionId, error)) - } else { - await reconcileEnded(sessionId, record, first).catch(error => rt.warn('ds-bot: settling the work of %s failed: %s', sessionId, error)) - } + await settleEnded(sessionId, record, first) continue } // Still running: noted again, as if it had started here. if (first === undefined && (await rt.resolveAgent(sessionId).catch(() => undefined))?.status === 'running') { update(sessionId, () => record) keepTaskSessions.add(sessionId) - } else { - cut.push([sessionId, record]) + continue + } + // If inflight.json still held a turn whose durable log already shows a finished + // turn/end, or whose owed reply is already recorded in the pair ledger, settle it + // instead of re-running via resumeTurn. + if (first === undefined && record.roomId === undefined && record.trigger !== undefined) { + const trigger = record.trigger + const events = trigger.exchangeId !== undefined ? await logOf(sessionId) : undefined + const turn = events === undefined ? undefined : turnOf(events, trigger.exchangeId) + if (isTerminalReason(turn?.reason)) { + await settleEnded(sessionId, record, { reason: turn.reason }, events) + continue + } + if (await settleDeliveredTrigger(record, trigger)) continue } + cut.push([sessionId, record]) } firstTurn = undefined for (const [sessionId, list] of Object.entries(dropped)) { diff --git a/ds-bot/src/host/threads.js b/ds-bot/src/host/threads.js index 811b2680a..aa5183ccb 100644 --- a/ds-bot/src/host/threads.js +++ b/ds-bot/src/host/threads.js @@ -126,6 +126,27 @@ export const successorOfMovedPart = (bot, freshId) => { return kind === 'open' || kind === 'task' ? freshId : undefined } +// Whether two Session ids belong to the same Bot conversation thread or group membership. +// Stays stable across part switches and open-part focus swaps, while isolating distinct +// parallel chats, background tasks and groups. +export function sameConversation(state, botId, sessionA, sessionB) { + if (sessionA === sessionB) return true + if (sessionA === undefined || sessionB === undefined || botId === undefined) return false + for (const room of Object.values(state?.rooms ?? {})) { + const current = room?.sessions?.[botId] + const segments = room?.segments?.[botId] ?? [] + const hasA = current === sessionA || segments.some(segment => segment?.sessionId === sessionA) + const hasB = current === sessionB || segments.some(segment => segment?.sessionId === sessionB) + if (hasA || hasB) return hasA && hasB + } + const bot = state?.bots?.[botId] + if (bot === undefined) return false + const threadA = threadOfPart(bot, sessionA) + const threadB = threadOfPart(bot, sessionB) + if (threadA === undefined || threadB === undefined) return false + return threadA.thread === threadB.thread && threadA.index === threadB.index +} + // Every live thread's current session (foreground, user-opened parallel parts, and active // tasks only). Settled tasks are excluded so folder/engine/model sweeps do not wake them. export const ownChatSessions = bot => [ diff --git a/ds-bot/src/host/turns.js b/ds-bot/src/host/turns.js index c96738953..25e695114 100644 --- a/ds-bot/src/host/turns.js +++ b/ds-bot/src/host/turns.js @@ -8,7 +8,7 @@ import { HANDOFF_ACK, LATE_ANSWER, MESSAGE_INLINE_MAX, REFERENCE_KIND, REPLY_KIN import { cleanText } from './diagnostics.js' import { absoluteLinks } from './links.js' import { splitRef, textOf } from './text.js' -import { threadCurrentOfPart } from './threads.js' +import { sameConversation, threadCurrentOfPart } from './threads.js' import { viaOf } from '../shared/im-platforms.js' import { reminderText } from './schedules.js' import { usageChunk } from './usage.js' @@ -51,10 +51,60 @@ export function turnFailure(error) { export const failureLine = failure => `${failure.code}${failure.status ? ` ${failure.status}` : ''}${failure.message ? `: ${failure.message.slice(0, 200)}` : ''}` +const MAX_REPLY_GUARDS = 1000 +const replyGuards = new WeakMap() + +const guardsOf = (rt) => { + let guards = replyGuards.get(rt) + if (guards === undefined) { + guards = { exchanges: new Set(), failureRequests: [], turnFailures: [] } + replyGuards.set(rt, guards) + } + return guards +} + +const pushLimited = (list, item) => { + list.push(item) + if (list.length > MAX_REPLY_GUARDS) list.shift() +} + +const removeItem = (list, item) => { + const at = list.indexOf(item) + if (at !== -1) list.splice(at, 1) +} + +// The live parent work a child work hangs under, matching closeWork's lookup. +const liveParentOf = (rt, exchangeId) => { + if (typeof exchangeId !== 'string' || exchangeId === '') return undefined + const works = rt.state?.works + const parent = works?.[exchangeId]?.parent + return parent !== undefined && parent !== exchangeId && works?.[parent] !== undefined ? parent : undefined +} + +// Coalescing a second child's failure into an earlier delivery for the same parent work +// is safe only while that earlier delivery is still pending in parent.open; once the +// caller has already consumed it and parked the parent, suppressing this child would +// leave the parent idle with no continuation. +const parentCoveredBy = (rt, parentWork, primaryExchangeId) => { + if (parentWork === undefined) return true + if (typeof primaryExchangeId !== 'string' || primaryExchangeId === '') return false + return rt.state?.works?.[parentWork]?.open?.includes(primaryExchangeId) === true +} + +function dropSuppressedWork(rt, exchangeId, name, text) { + if (typeof exchangeId !== 'string' || exchangeId === '') return + if (typeof rt.dropWork === 'function') { + rt.dropWork(exchangeId) + return + } + const parent = rt.closeWork?.(exchangeId, name, text) + if (parent !== undefined) rt.takenWork?.(parent, exchangeId) +} + // The reply a sender waits for, sent the way a finished turn sends it: the work it was // for closes, and the answer goes up to the Bot that asked. `id` is the Session the // answer comes from; `source` is the trigger the turn answered. -export function answerSender(rt, id, source, text, long) { +export function answerSender(rt, id, source, text, long, options = {}) { const back = rt.roomOf(source.senderRoomId)?.sessions?.[source.senderSessionId] // The asking Bot may have written from a background part of its own chat: its reply // goes back to that part's thread, not to the part the user is reading. When the part @@ -62,8 +112,55 @@ export function answerSender(rt, id, source, text, long) { const asker = source.senderPart === undefined ? undefined : rt.botOf(source.senderSessionId) const part = back === undefined && asker !== undefined ? threadCurrentOfPart(asker, source.senderPart) : undefined const target = back ?? part - const name = rt.botOf(rt.selfOf(id))?.name ?? 'The other Bot' + const fromId = rt.selfOf(id) + const toId = source.senderSessionId + const targetSessionId = target ?? rt.chatOf?.(toId) ?? toId + const name = rt.botOf(fromId)?.name ?? 'The other Bot' + const guards = guardsOf(rt) + const exchangeKey = typeof source.exchangeId === 'string' && source.exchangeId !== '' + ? `${fromId ?? ''}\0${toId ?? ''}\0${source.exchangeId}` + : undefined + const deliveredReply = rt.deliveredReplyOf?.(fromId, toId, source.exchangeId) + const alreadySent = deliveredReply !== undefined + || (rt.deliveredReplyOf === undefined && rt.wasReplyDelivered?.(fromId, toId, source.exchangeId) === true) + if ((exchangeKey !== undefined && guards.exchanges.has(exchangeKey)) || alreadySent) { + if (typeof source.exchangeId === 'string' && source.exchangeId !== '') { + rt.closeWork?.(source.exchangeId, name, deliveredReply?.text || text) + } + return false + } + const failureRequestId = typeof options?.failure?.requestId === 'string' && options.failure.requestId !== '' + ? options.failure.requestId + : undefined + const dedupKey = typeof options?.dedupKey === 'string' && options.dedupKey !== '' ? options.dedupKey : undefined + const ownParent = liveParentOf(rt, source.exchangeId) + const matchSameScope = entry => entry.fromId === fromId && entry.toId === toId + && entry.work === ownParent + && sameConversation(rt.state, toId, entry.targetSessionId, targetSessionId) + && parentCoveredBy(rt, ownParent, entry.exchangeId) + const matchedReq = failureRequestId === undefined ? undefined + : guards.failureRequests.findLast(entry => entry.requestId === failureRequestId && matchSameScope(entry)) + const matchedTurn = dedupKey === undefined ? undefined + : guards.turnFailures.findLast(entry => entry.dedupKey === dedupKey && matchSameScope(entry)) + const matchedInMemory = matchedReq !== undefined || matchedTurn !== undefined + const matchedLedger = matchedInMemory || failureRequestId === undefined ? undefined + : (rt.deliveredFailuresOf?.(fromId, toId, failureRequestId, targetSessionId, ownParent) ?? []) + .findLast(entry => parentCoveredBy(rt, ownParent, entry.replyTo)) + if (matchedInMemory || matchedLedger !== undefined) { + dropSuppressedWork(rt, source.exchangeId, name, text) + return false + } const work = rt.closeWork?.(source.exchangeId, name, text) + if (exchangeKey !== undefined) { + guards.exchanges.add(exchangeKey) + if (guards.exchanges.size > MAX_REPLY_GUARDS) guards.exchanges.delete(guards.exchanges.values().next().value) + } + const failureEntry = failureRequestId === undefined ? undefined + : { fromId, toId, targetSessionId, requestId: failureRequestId, work, exchangeId: source.exchangeId } + if (failureEntry !== undefined) pushLimited(guards.failureRequests, failureEntry) + const turnEntry = dedupKey === undefined ? undefined + : { fromId, toId, targetSessionId, dedupKey, work, exchangeId: source.exchangeId } + if (turnEntry !== undefined) pushLimited(guards.turnFailures, turnEntry) // A reply that arrives after the user gave up the wait is a late answer: it closes // nothing, opens nothing, and says on it that the wait was already over. const late = work === undefined ? rt.abandonedWork?.(source.exchangeId) : undefined @@ -72,8 +169,8 @@ export function answerSender(rt, id, source, text, long) { const length = long === undefined ? undefined : late === undefined ? long.length : (long.length ?? text.length) + marker.length const exchangeId = randomUUID() void rt.deliver({ - fromId: rt.selfOf(id), - toId: source.senderSessionId, + fromId, + toId, ...(target ? { sessionId: target } : {}), text: body, ...(length === undefined ? {} : { files: long.files, length }), @@ -81,6 +178,7 @@ export function answerSender(rt, id, source, text, long) { replyTo: source.exchangeId, exchangeId, ...(work ? { work } : {}), + ...(failureRequestId === undefined ? {} : { failureRequestId }), }).catch((error) => { rt.warn('ds-bot: reply delivery failed: %s', error) // The reply never reached the asker: its note goes — the notice below carries @@ -91,16 +189,23 @@ export function answerSender(rt, id, source, text, long) { rt.answerMissed?.(source.exchangeId) const said = `${name} could not send its reply to you (${failureLine(turnFailure(error))}); what it had answered:\n` void rt.deliver({ - fromId: rt.selfOf(id), - toId: source.senderSessionId, + fromId, + toId, ...(target ? { sessionId: target } : {}), text: `${said}${long?.text ?? text}`, ...(long?.length === undefined ? {} : { files: long.files, length: said.length + long.length }), role: 'reply', replyTo: source.exchangeId, ...(work ? { work } : {}), - }).catch(again => rt.warn('ds-bot: reply failure notice not delivered: %s', again)) + ...(failureRequestId === undefined ? {} : { failureRequestId }), + }).catch((again) => { + rt.warn('ds-bot: reply failure notice not delivered: %s', again) + if (exchangeKey !== undefined) guards.exchanges.delete(exchangeKey) + if (failureEntry !== undefined) removeItem(guards.failureRequests, failureEntry) + if (turnEntry !== undefined) removeItem(guards.turnFailures, turnEntry) + }) }) + return true } // A group's own Session never reaches a model, so the snapshot DSH prepared there @@ -513,7 +618,8 @@ export function install(rt) { // reply in its Session there while it is still a member. A sender that asked for it // while working on another Bot's request goes on with that work when it reads it. // `long`: the text as attachLong sends it, when it went as a file. - const tellSender = (id, source, text, long) => answerSender(rt, id, source, text, long) + let turnEndSeq = 0 + const tellSender = (id, source, text, long, options) => answerSender(rt, id, source, text, long, options) // Runs before a deleted Bot leaves the team. Its Sessions stop with it, and the // turn/end that follows is no longer the team's, so their activity and turn records go @@ -1300,15 +1406,21 @@ export function install(rt) { // Whoever the turn answers gets the same three-way rule: a failure notice, // the turn's text (or a stopped-before-answer line), or nothing. const name = rt.botOf(rt.selfOf(id))?.name ?? 'The other Bot' + const turnEndToken = ++turnEndSeq + const turnDedupKey = failure !== undefined + ? `${turnEndToken}:error:${failure.requestId ?? failure.code}` + : reason?.kind === 'aborted' + ? `${turnEndToken}:aborted` + : undefined const answer = (source, reply) => { if (!expectsReply(source) || answersUser(source)) return if (failure !== undefined) { // The sender was told a reply comes later; without this it would wait forever. - tellSender(id, source, `${name} could not answer your message: its model call failed (${failureLine(failure)}). Tell the user if it matters; do not retry right away.`) + tellSender(id, source, `${name} could not answer your message: its model call failed (${failureLine(failure)}). Tell the user if it matters; do not retry right away.`, undefined, { failure, dedupKey: turnDedupKey }) } else if (reply !== '' || reason?.kind === 'aborted') { // A turn stopped before it could answer still owes the sender a line, // or it would wait forever for a reply that never comes. - tellSender(id, source, reply !== '' ? reply : `${name} was stopped before it could answer your message.`) + tellSender(id, source, reply !== '' ? reply : `${name} was stopped before it could answer your message.`, undefined, reply === '' && turnDedupKey !== undefined ? { dedupKey: turnDedupKey } : undefined) } } // Work a Bot asked for, also when what came back for it opens the turn: while diff --git a/ds-bot/test/bot-tasks.test.js b/ds-bot/test/bot-tasks.test.js index 957b07ec3..3bdf97520 100644 --- a/ds-bot/test/bot-tasks.test.js +++ b/ds-bot/test/bot-tasks.test.js @@ -511,7 +511,10 @@ test('reconcileTasksAfterRestart marks orphaned idle tasks interrupted while pre assert.match(interruptReport.text, /restarted/) // reconcileEndedTask settles a task whose turn ended during plugin reload using durable log. - env.agentMap.get(tInflight.sessionId).session.events.push( + const inflightAgent = env.agentMap.get(tInflight.sessionId) + const firstMsg = inflightAgent.followups[0] + await env.emit(tInflight.sessionId, 'user/message', { source: firstMsg.source, content: firstMsg.content }) + inflightAgent.session.events.push( { type: 'turn/start', seq: 10, data: { turn: 1 } }, { type: 'assistant/message', seq: 11, data: { message: { role: 'assistant', content: [{ type: 'text', text: 'Reload finished work.' }] } } }, ) diff --git a/ds-bot/test/delivery-dedup.test.js b/ds-bot/test/delivery-dedup.test.js new file mode 100644 index 000000000..2a6992f21 --- /dev/null +++ b/ds-bot/test/delivery-dedup.test.js @@ -0,0 +1,683 @@ +import assert from 'node:assert/strict' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'node:test' +import { setTimeout as realDelay } from 'node:timers/promises' +import { Context } from '@deepseek-ai/cordis' +import { LlmAdapter } from '@deepseek-ai/dsh-llm' +import { SessionId } from '@deepseek-ai/dsh-session' +import { mountAgentLoopTestDependencies, mountAgentLoopTestHarness } from '@deepseek-ai/dsh-agent-loop-testkit' +import { createRuntime } from '../index.js' +import { answerSender } from '../src/host/turns.js' +import { fakeAttachments, fakeContext } from './fake-ctx.js' + +const settle = async (check, ms = 4000) => { + const until = Date.now() + ms + while (!check() && Date.now() < until) await realDelay(10) + assert.ok(check(), 'condition not reached') +} + +const textOf = message => (message?.content ?? []).map(block => block.text ?? '').join('\n') + +const baseSeed = () => ({ + version: 1, + mainBotIds: ['a'], + bots: { + a: { id: 'a', name: 'Chief', role: 'Boss', sessionId: 'a', createdAt: 1 }, + b: { id: 'b', name: 'Writer', role: 'Write', sessionId: 'b', createdAt: 2 }, + }, + rooms: {}, + agents: {}, + exchanges: [], + exchangeTurns: {}, + questions: {}, + prefs: {}, + revision: 0, +}) + +async function createLoopRig(t, seed = baseSeed()) { + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-loop-')) + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + + const root = new Context() + await mountAgentLoopTestDependencies(root) + + const stepGate = Promise.withResolvers() + const modes = { b: 'two-step-fail', requestId: 'req-dup-1319', callCount: 0 } + + root.tools.register({ + name: 'wait_step', + description: 'Pause mid-turn so another Bot message can steer in', + parameters: {}, + output: { schema: { type: 'object' }, render: (_args, value) => [{ type: 'text', text: JSON.stringify(value) }] }, + async execute() { + await stepGate.promise + return { ok: true } + }, + }) + + class TestAdapter extends LlmAdapter { + async resolveModel(provider, id) { + return { provider, id, name: id, inputModalities: ['text'], context: { contextWindow: 100000 } } + } + async *stream(options) { + modes.callCount += 1 + if (options.sessionId === 'b') { + const last = options.messages.at(-1) + const hasToolResult = last?.role === 'tool' + if (!hasToolResult && modes.b === 'two-step-fail') { + const args = '{}' + yield { type: 'block-start', index: 0, blockType: 'tool-call' } + yield { type: 'tool-call-delta', index: 0, id: `call_${modes.callCount}`, name: 'wait_step', argumentsDelta: args } + yield { type: 'block-end', index: 0, block: { type: 'tool-call', id: `call_${modes.callCount}`, name: 'wait_step', arguments: args } } + yield { type: 'finish', reason: { kind: 'tool-calls' } } + return + } + yield { + type: 'finish', + reason: { + kind: 'error', + failure: { + code: 'INVALID_REQUEST', + status: 400, + message: `invalid tool payload (request_id: ${modes.requestId})`, + requestId: modes.requestId, + }, + }, + } + return + } + const reply = `Handled ${options.messages.length}` + yield { type: 'block-start', index: 0, blockType: 'text' } + yield { type: 'text-delta', index: 0, text: reply } + yield { type: 'block-end', index: 0, block: { type: 'text', text: reply } } + yield { type: 'finish', reason: { kind: 'stop' } } + } + } + root.llm.registerAdapter(['dedup-test'], new TestAdapter()) + await mountAgentLoopTestHarness(root) + + const agents = new Map() + for (const id of Object.keys(seed.bots)) { + const bot = seed.bots[id] + const sessions = [bot.sessionId, ...(bot.openParts ?? []).map(part => part.sessionId)] + for (const sessionId of sessions) { + agents.set(sessionId, await root.agentLoop.create(SessionId(sessionId), { provider: 'dedup-test', model: 'test' })) + } + } + + const deliveries = [] + const ctx = fakeContext({ + services: { + agents: root.agents, + llm: root.llm, + sessions: root.sessions, + tools: root.tools, + workspaceRegistry: { get: () => undefined, resolveByPath: async () => undefined, create: async path => ({ id: 'w1', path }), archiveSession: async () => {} }, + sessionController: { + resolveAgent: async sessionId => ({ agent: agents.get(sessionId) }), + create: async () => ({ sessionId: 's-unused' }), + selectModel: async () => {}, + page: async () => ({ records: [], hasMore: false }), + inspect: async sessionId => ({ events: agents.get(sessionId)?.session.snapshotEvents() ?? [] }), + }, + sessionTitle: { rename: async () => {} }, + }, + }) + Object.assign(ctx, { + agents: root.agents, + llm: root.llm, + sessions: root.sessions, + tools: root.tools, + workspaceRegistry: ctx.get('workspaceRegistry'), + sessionController: ctx.get('sessionController'), + sessionTitle: ctx.get('sessionTitle'), + }) + root.on('session/event', (session, event) => { + if (event.type === 'user/message' && event.data?.source?.kind === 'bot') { + deliveries.push({ sessionId: session.id, source: event.data.source, text: textOf({ content: event.data.content }) }) + } + for (const listener of ctx.listeners.get('session/event') ?? []) { + if (listener.length >= 2) listener(session, event) + } + }) + root.on('agent/inbox/discarded', (payload) => { + for (const listener of ctx.listeners.get('agent/inbox/discarded') ?? []) listener(payload) + }) + + const rt = createRuntime(ctx, { home }) + // Exercise the in-turn steering path, not busy-foreground task spawning. + t.mock.method(rt, 'shouldMoveForeground', () => false) + t.after(async () => { + stepGate.resolve() + await ctx.dispose() + await root.fiber.dispose() + rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }) + }) + await rt.load() + const tool = (name, args, sessionId) => rt.teamTools.find(def => def.name === name).execute(args, { agent: agents.get(sessionId) }) + return { home, rt, agents, tool, stepGate, modes, deliveries } +} + +async function createRestartHost(t, home, logs, services) { + const ctx = fakeContext({ services }) + const agents = new Map() + const agentOf = (sessionId) => { + if (!agents.has(sessionId)) { + const calls = [] + const inbox = { nextStep: [], nextTurn: [], remove: () => false } + agents.set(sessionId, { + session: { id: sessionId }, + calls, + inbox, + status: 'idle', + followup(message) { calls.push(message) }, + steer(message) { calls.push(message) }, + cancel() {}, + }) + } + return agents.get(sessionId) + } + Object.assign(ctx, { + llm: { listProviders: () => [], listModels: async () => [], resolveModelInfo: async () => ({}) }, + workspaceRegistry: { get: () => undefined, resolveByPath: async () => undefined, create: async path => ({ id: 'w1', path }), archiveSession: async () => {} }, + sessionController: { + create: async () => ({ sessionId: 's1' }), + resolveAgent: async sessionId => ({ agent: agentOf(sessionId) }), + selectModel: async () => {}, + inspect: async sessionId => ({ events: logs.get(sessionId) ?? [] }), + }, + sessionTitle: { rename: async () => {} }, + sessions: { flush: async () => {} }, + }) + const rt = createRuntime(ctx, { home }) + let seq = 0 + const emit = (id, type, data) => { + const session = agents.get(id)?.session ?? { id } + for (const listener of ctx.listeners.get('session/event') ?? []) { + if (listener.length >= 2) listener(session, { type, seq: ++seq, data }) + } + } + await rt.load() + const host = { ctx, rt, agentOf, emit, stop: () => ctx.dispose() } + return host +} + +test('a failed turn with steered follow-ups pushes only one failure notice per request_id and settles open works', async (t) => { + const { home, rt, agents, tool, stepGate, modes, deliveries } = await createLoopRig(t) + + // Chief sends an initial task to Writer; Writer starts working and pauses in wait_step. + await tool('message_bot', { to: 'Writer', message: 'Run task 1' }, 'a') + await settle(() => modes.callCount >= 1 && rt.live.get('b')?.tool === 'wait_step') + + // While Writer is mid-turn, Chief steers a follow-up update (default when: 'now'). + await tool('message_bot', { to: 'Writer', message: 'Also include check 2' }, 'a') + await settle(() => agents.get('b').inbox.nextStep.length === 1) + + // Let wait_step finish so Writer claims the steered follow-up and hits INVALID_REQUEST. + stepGate.resolve() + await agents.get('b').whenIdle() + await settle(() => deliveries.some(entry => entry.sessionId === 'a' && entry.source.role === 'reply')) + await agents.get('a').whenIdle() + await realDelay(30) + + const failureRepliesToChief = deliveries.filter(entry => entry.sessionId === 'a' && entry.source.role === 'reply' && entry.text.includes('req-dup-1319')) + assert.equal(failureRepliesToChief.length, 1, 'Chief must receive the failure notice for req-dup-1319 only once') + const state = JSON.parse(readFileSync(join(home, 'state.json'), 'utf8')) + assert.deepEqual(Object.keys(state.works ?? {}), [], 'all works bound to the failed turn must be closed') + + // A subsequent turn that fails with a new request_id still notifies Chief. + modes.b = 'fail-once' + modes.requestId = 'req-new-456' + await tool('message_bot', { to: 'Writer', message: 'Retry with clean args' }, 'a') + await agents.get('b').whenIdle() + await settle(() => deliveries.some(entry => entry.sessionId === 'a' && entry.source.role === 'reply' && entry.text.includes('req-new-456'))) + await agents.get('a').whenIdle() + const newReplies = deliveries.filter(entry => entry.sessionId === 'a' && entry.source.role === 'reply' && entry.text.includes('req-new-456')) + assert.equal(newReplies.length, 1) +}) + +test('a failed turn with steered requests from two openParts notifies each conversation once per request_id', async (t) => { + const seed = baseSeed() + seed.bots.a.openParts = [{ sessionId: 'a-bg', segments: [] }] + const { rt, agents, tool, stepGate, modes, deliveries } = await createLoopRig(t, seed) + + await tool('message_bot', { to: 'Writer', message: 'From fg 1' }, 'a') + await settle(() => modes.callCount >= 1 && rt.live.get('b')?.tool === 'wait_step') + + // Same fg thread steers another request, and a separate bg openPart thread also steers one. + await tool('message_bot', { to: 'Writer', message: 'From fg 2' }, 'a') + await tool('message_bot', { to: 'Writer', message: 'From bg 1' }, 'a-bg') + await settle(() => agents.get('b').inbox.nextStep.length === 2) + + stepGate.resolve() + await agents.get('b').whenIdle() + await settle(() => deliveries.some(entry => entry.sessionId === 'a' && entry.source.role === 'reply') && deliveries.some(entry => entry.sessionId === 'a-bg' && entry.source.role === 'reply')) + await Promise.all([agents.get('a').whenIdle(), agents.get('a-bg').whenIdle()]) + await realDelay(30) + + const fgFailures = deliveries.filter(entry => entry.sessionId === 'a' && entry.source.role === 'reply' && entry.text.includes('req-dup-1319')) + const bgFailures = deliveries.filter(entry => entry.sessionId === 'a-bg' && entry.source.role === 'reply' && entry.text.includes('req-dup-1319')) + assert.equal(fgFailures.length, 1, 'foreground conversation receives one notice') + assert.equal(bgFailures.length, 1, 'independent background openPart receives one notice') +}) + +test('answerSender is idempotent per exchangeId and per failure requestId while preserving late replies after abandonWork', async (t) => { + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-answer-')) + const seed = baseSeed() + seed.works = { + 'ex-1': { from: 'a', to: 'b', open: [], results: [], at: 1 }, + 'ex-parent': { from: 'a', to: 'b', open: ['ex-child-1', 'ex-child-2'], results: [], at: 1 }, + 'ex-child-1': { from: 'a', to: 'b', parent: 'ex-parent', open: [], results: [], at: 2 }, + 'ex-child-2': { from: 'a', to: 'b', parent: 'ex-parent', open: [], results: [], at: 3 }, + 'ex-abandoned-parent': { from: 'a', to: 'a', open: ['ex-late'], results: [], at: 1 }, + 'ex-late': { from: 'a', to: 'b', parent: 'ex-abandoned-parent', open: [], results: [], at: 2 }, + } + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + const host = await createRestartHost(t, home, new Map()) + t.after(async () => { + await host.stop() + rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }) + }) + + const { rt } = host + const src1 = { kind: 'bot', role: 'reply', exchangeId: 'ex-1', senderSessionId: 'b' } + const req1 = { kind: 'bot', role: 'request', exchangeId: 'ex-1', senderSessionId: 'a' } + answerSender(rt, 'b', req1, 'Final report verbatim') + answerSender(rt, 'b', req1, 'Final report verbatim') + await settle(() => host.agentOf('a').calls.length >= 1) + await realDelay(30) + assert.equal(host.agentOf('a').calls.length, 1, 'same exchangeId must only deliver once') + + // Failure deduplication by requestId also cleans up duplicate child open work entries. + const child1 = { kind: 'bot', role: 'request', exchangeId: 'ex-child-1', senderSessionId: 'a' } + const child2 = { kind: 'bot', role: 'request', exchangeId: 'ex-child-2', senderSessionId: 'a' } + const failure = { code: 'INVALID_REQUEST', status: 400, requestId: 'req-same-99', message: 'bad arg' } + answerSender(rt, 'b', child1, 'Writer failed (req-same-99)', undefined, { failure }) + answerSender(rt, 'b', child2, 'Writer failed (req-same-99)', undefined, { failure }) + await settle(() => host.agentOf('a').calls.length >= 2) + await realDelay(30) + assert.equal(host.agentOf('a').calls.length, 2, 'same requestId failure must only deliver once') + assert.equal(rt.state.works['ex-child-2'], undefined, 'suppressed duplicate work must be removed') + assert.deepEqual(rt.state.works['ex-parent'].open, ['ex-child-1'], 'suppressed duplicate must not stay stuck in parent.open') + + // abandonWork + late answer still delivers once, then a replay is blocked. + rt.abandonWork('ex-late', 'user') + await settle(() => host.agentOf('a').calls.length >= 3) + const lateSource = { kind: 'bot', role: 'request', exchangeId: 'ex-late', senderSessionId: 'a' } + answerSender(rt, 'b', lateSource, 'Actual late completion') + await settle(() => host.agentOf('a').calls.length >= 4) + answerSender(rt, 'b', lateSource, 'Actual late completion') + await realDelay(30) + assert.equal(host.agentOf('a').calls.length, 4, 'late reply arrives once and replay is blocked') + assert.equal(src1.exchangeId, 'ex-1') +}) + +test('reconcileEnded does not duplicate replies or request_id failures after the asker moves to a fresh part', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }) + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-reconcile-')) + const logs = new Map() + const seed = baseSeed() + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + + const first = await createRestartHost(t, home, logs) + t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 })) + + const request = { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'ex-part-switch', senderName: 'Chief', senderSessionId: 'a' } + first.rt.openWork('ex-part-switch', { from: 'a', to: 'b' }) + first.emit('b', 'turn/start', { turn: 1 }) + first.emit('b', 'user/message', { content: [{ type: 'text', text: '[Message from Chief]\nCheck' }], source: request }) + first.emit('b', 'assistant/message', { message: { content: [{ type: 'text', text: 'The plan is ready.' }] } }) + + // Deliver the reply to part 'a', Chief consumes it, then Chief advances to part 'a-2'. + await first.rt.deliver({ fromId: 'b', toId: 'a', sessionId: 'a', text: 'The plan is ready.', role: 'reply', replyTo: 'ex-part-switch', exchangeId: 'reply-1' }) + const deliveredReply = first.agentOf('a').calls[0] + first.emit('a', 'user/message', { content: deliveredReply.content, source: deliveredReply.source }) + first.rt.state.bots.a.segments = [{ sessionId: 'a', endedAt: 10 }] + first.rt.state.bots.a.sessionId = 'a-2' + await first.rt.save({ required: true }) + + // Leave inflight.turns.b active as if plugin reload/restart happened before Writer's turn/end persisted, + // while Writer's durable log holds the completed turn and Chief's new part 'a-2' has no record of it. + logs.set('b', [ + { type: 'turn/start', seq: 1, data: { turn: 1 } }, + { type: 'user/message', seq: 2, data: { source: request, content: [{ type: 'text', text: '[Message from Chief]\nCheck' }] } }, + { type: 'assistant/message', seq: 3, data: { message: { content: [{ type: 'text', text: 'The plan is ready.' }] } } }, + { type: 'turn/end', seq: 4, data: { turn: 1, reason: { kind: 'completed' } } }, + ]) + logs.set('a-2', []) + await first.stop() + + const second = await createRestartHost(t, home, logs) + second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } }) + t.mock.timers.tick(3000) + await new Promise(resolve => setImmediate(resolve)) + await realDelay(30) + + assert.equal(second.agentOf('a-2').calls.length, 0, 'reply already in pair ledger must not be re-delivered after part switch') + assert.equal(second.agentOf('a').calls.length, 0) + assert.equal(second.agentOf('b').calls.length, 0) + await second.stop() +}) + +test('resumeAll settles already-ended turns from disk without re-running or duplicating replies', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }) + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-resume-')) + t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 })) + + const seed = baseSeed() + seed.works = { 'ex-ended': { from: 'a', to: 'b', open: [], results: [], at: 1 } } + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + const logs = new Map() + const request = { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'ex-ended', senderName: 'Chief', senderSessionId: 'a' } + logs.set('b', [ + { type: 'turn/start', seq: 1, data: { turn: 1 } }, + { type: 'user/message', seq: 2, data: { source: request, content: [{ type: 'text', text: '[Message from Chief]\nFinish' }] } }, + { type: 'assistant/message', seq: 3, data: { message: { content: [{ type: 'text', text: 'Already finished answer.' }] } } }, + { type: 'turn/end', seq: 4, data: { turn: 1, reason: { kind: 'completed' } } }, + ]) + logs.set('a', []) + + // Pre-seed inflight.turns as left behind if inflight.json did not clear on crash. + writeFileSync(join(home, 'inflight.json'), JSON.stringify({ + turns: { b: { botId: 'b', since: 1, trigger: { role: 'request', exchangeId: 'ex-ended', senderSessionId: 'a' } } }, + queued: {}, + })) + + const first = await createRestartHost(t, home, logs) + t.mock.timers.tick(3000) + await settle(() => first.agentOf('a').calls.length >= 1 || first.agentOf('b').calls.length >= 1) + assert.equal(first.agentOf('b').calls.length, 0, 'completed turn on disk must not be re-run with resumeTurn') + assert.equal(first.agentOf('a').calls.length, 1, 'completed turn reply must settle to Chief once') + assert.match(textOf(first.agentOf('a').calls[0]), /Already finished answer\./) + await first.stop() + + // Second crash before inflight clears: now reply is already in pair ledger, so nothing is re-sent. + writeFileSync(join(home, 'inflight.json'), JSON.stringify({ + turns: { b: { botId: 'b', since: 1, trigger: { role: 'request', exchangeId: 'ex-ended', senderSessionId: 'a' } } }, + queued: {}, + })) + const second = await createRestartHost(t, home, logs) + t.mock.timers.tick(3000) + await new Promise(resolve => setImmediate(resolve)) + await realDelay(30) + assert.equal(second.agentOf('b').calls.length, 0) + assert.equal(second.agentOf('a').calls.length, 0, 'already-delivered reply must not be sent again') + await second.stop() +}) + +async function checkAttachmentRecovery(t, prefix) { + t.mock.timers.enable({ apis: ['setTimeout'] }) + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-attachment-')) + t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 })) + const services = { attachments: fakeAttachments() } + const seed = baseSeed() + seed.works = { + parent: { from: 'user', to: 'a', open: ['child'], results: [], at: 1 }, + child: { from: 'a', to: 'b', parent: 'parent', open: [], results: [], at: 1 }, + } + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + const logs = new Map() + const first = await createRestartHost(t, home, logs, services) + const full = '结论\n'.repeat(70_000) + 'End of report.' + const attachment = await first.rt.attachLong(full, 'Answer from Writer.txt') + await first.rt.deliver({ + fromId: 'b', toId: 'a', ...attachment, role: 'reply', replyTo: 'child', + text: prefix + attachment.text, length: prefix.length + attachment.length, + }) + first.emit('a', 'user/message', { source: first.agentOf('a').calls[0].source }) + await first.stop() + + const request = { kind: 'bot', role: 'request', exchangeId: 'child', senderSessionId: 'a' } + logs.set('b', [ + { type: 'turn/start', data: { turn: 1 } }, + { type: 'user/message', data: { source: request } }, + { type: 'turn/end', data: { turn: 1, reason: { kind: 'completed' } } }, + ]) + writeFileSync(join(home, 'inflight.json'), JSON.stringify({ + turns: { b: { botId: 'b', since: 1, trigger: request } }, + queued: {}, + })) + const restarted = await createRestartHost(t, home, logs, services) + t.mock.timers.tick(3000) + await settle(() => restarted.rt.state.works.child === undefined) + assert.ok(restarted.rt.answerText('parent', '', 'Chief').endsWith(prefix + full), 'parent keeps the entire reply, not its preview or a dialog-sized prefix') + assert.equal(restarted.agentOf('a').calls.length, 0) + assert.equal(restarted.agentOf('b').calls.length, 0) + await restarted.stop() +} + +test('restart keeps the full attachment-backed reply in the parent work without re-delivery', t => checkAttachmentRecovery(t, '')) + +test('restart keeps the fallback notice with the full attachment-backed reply', t => checkAttachmentRecovery(t, 'Writer could not send its reply to you (offline); what it had answered:\n')) + +test('restart settles a completed peer handoff task from disk and replies to the asking Bot', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }) + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-task-')) + t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 })) + const seed = baseSeed() + seed.bots.b.tasks = [{ id: 'task-1', sessionId: 'b-task', title: 'Peer request', origin: 'bot-handoff', status: 'running', exchangeId: 'child' }] + seed.works = { child: { from: 'a', to: 'b', open: [], results: [], at: 1 } } + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + const request = { kind: 'bot', role: 'request', exchangeId: 'child', senderSessionId: 'a' } + const logs = new Map([['b-task', [ + { type: 'turn/start', data: { turn: 1 } }, + { type: 'user/message', data: { source: request } }, + { type: 'assistant/message', data: { message: { content: [{ type: 'text', text: 'Peer task finished.' }] } } }, + { type: 'turn/end', data: { turn: 1, reason: { kind: 'completed' } } }, + ]]]) + writeFileSync(join(home, 'inflight.json'), JSON.stringify({ + turns: { 'b-task': { botId: 'b', since: 1, trigger: request } }, + queued: {}, + })) + const host = await createRestartHost(t, home, logs) + t.mock.timers.tick(3000) + await settle(() => host.rt.state.bots.b.tasks[0].status === 'completed' && host.agentOf('a').calls.length > 0) + assert.equal(host.agentOf('a').calls.length, 1) + assert.match(textOf(host.agentOf('a').calls[0]), /Peer task finished\./) + assert.equal(host.agentOf('b-task').calls.length, 0) + assert.equal(host.agentOf('b').calls.length, 0) + await host.stop() +}) + +async function checkUnreadTaskReply(t, inInbox) { + t.mock.timers.enable({ apis: ['setTimeout'] }) + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-task-queued-')) + const seed = baseSeed() + seed.bots.c = { id: 'c', name: 'Reviewer', role: 'Reviewer', sessionId: 'c', createdAt: 3 } + seed.bots.b.tasks = [{ id: 'task-1', sessionId: 'b-task', title: 'Peer request', origin: 'bot-handoff', status: 'waiting_work', exchangeId: 'request' }] + seed.works = { + request: { from: 'a', to: 'b', open: ['review'], results: [{ id: 'review', name: 'Reviewer', text: 'Review result.' }], at: 1 }, + } + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + const request = { kind: 'bot', role: 'request', exchangeId: 'request', senderSessionId: 'a' } + const reply = { fromId: 'c', toId: 'b', sessionId: 'b-task', role: 'reply', replyTo: 'review', work: 'request', exchangeId: 'review-reply', text: 'Review result.' } + const logs = new Map([['b-task', [ + { type: 'turn/start', data: { turn: 1 } }, + { type: 'user/message', data: { source: request } }, + { type: 'assistant/message', data: { message: { content: [{ type: 'text', text: 'Waiting for review.' }] } } }, + { type: 'turn/end', data: { turn: 1, reason: { kind: 'completed' } } }, + ]]]) + writeFileSync(join(home, 'inflight.json'), JSON.stringify({ + turns: { 'b-task': { botId: 'b', since: 1, trigger: request } }, + queued: inInbox ? {} : { 'b-task': [{ at: 2, args: reply }] }, + })) + const host = await createRestartHost(t, home, logs) + t.after(async () => { + await host.stop() + rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }) + }) + if (inInbox) host.agentOf('b-task').inbox.nextTurn.push({ source: { kind: 'bot', exchangeId: reply.exchangeId } }) + const reconciliation = t.mock.method(host.rt, 'reconcileTasksAfterRestart') + t.mock.timers.tick(3000) + await settle(() => reconciliation.mock.callCount() > 0) + await reconciliation.mock.calls[0].result + assert.deepEqual(host.agentOf('b-task').calls.map(({ source }) => source.exchangeId), inInbox ? [] : ['review-reply']) + assert.equal(host.agentOf('b').calls.length, 0) + assert.equal(host.agentOf('a').calls.length, 0) + assert.equal(host.rt.state.bots.b.tasks[0].status, 'waiting_work') + assert.deepEqual(host.rt.state.works.request.open, ['review']) +} + +test('restart replays an unread child reply in its background task without rerunning the earlier turn', t => checkUnreadTaskReply(t, false)) + +test('restart keeps a task active when a child reply is still in the agent inbox', t => checkUnreadTaskReply(t, true)) + +test('answerSender keeps separate continuations for distinct parent works sharing a failed turn', async (t) => { + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-parents-')) + const seed = baseSeed() + seed.bots.c = { id: 'c', name: 'Lead', role: 'Lead', sessionId: 'c', createdAt: 3 } + seed.works = { + 'ex-no-parent': { from: 'a', to: 'b', open: [], results: [], at: 1 }, + 'parent-1': { from: 'c', to: 'a', open: ['child-p1'], results: [], at: 1 }, + 'child-p1': { from: 'a', to: 'b', parent: 'parent-1', open: [], results: [], at: 2 }, + 'parent-2': { from: 'c', to: 'a', open: ['child-p2'], results: [], at: 1 }, + 'child-p2': { from: 'a', to: 'b', parent: 'parent-2', open: [], results: [], at: 3 }, + 'parent-consumed': { from: 'c', to: 'a', open: ['child-c1', 'child-c2'], results: [], at: 1 }, + 'child-c1': { from: 'a', to: 'b', parent: 'parent-consumed', open: [], results: [], at: 2 }, + 'child-c2': { from: 'a', to: 'b', parent: 'parent-consumed', open: [], results: [], at: 3 }, + } + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + const host = await createRestartHost(t, home, new Map()) + t.after(async () => { + await host.stop() + rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }) + }) + + const { rt } = host + const failure = { code: 'INVALID_REQUEST', status: 400, requestId: 'req-shared-parents', message: 'bad' } + const dedupKey = 'turn:b:1:error:req-shared-parents' + + // Unparented work followed by parent-1 and parent-2 in the same failed turn: each distinct + // parent work must receive its own continuation carrying `work`. + assert.equal(answerSender(rt, 'b', { kind: 'bot', role: 'request', exchangeId: 'ex-no-parent', senderSessionId: 'a' }, 'Writer failed', undefined, { failure, dedupKey }), true) + assert.equal(answerSender(rt, 'b', { kind: 'bot', role: 'request', exchangeId: 'child-p1', senderSessionId: 'a' }, 'Writer failed', undefined, { failure, dedupKey }), true) + assert.equal(answerSender(rt, 'b', { kind: 'bot', role: 'request', exchangeId: 'child-p2', senderSessionId: 'a' }, 'Writer failed', undefined, { failure, dedupKey }), true) + await settle(() => host.agentOf('a').calls.length >= 3) + assert.deepEqual( + host.agentOf('a').calls.map(msg => [msg.source.replyTo, msg.source.work]), + [['ex-no-parent', undefined], ['child-p1', 'parent-1'], ['child-p2', 'parent-2']], + 'distinct parent works must never be suppressed by another parent or unparented failure', + ) + + // Same parent work, but once the first failed child was already consumed out of parent.open, + // a second child's failure must still deliver so the parked parent is woken up. + const sameParentFailure = { code: 'INVALID_REQUEST', status: 400, requestId: 'req-consumed-parent', message: 'bad' } + assert.equal(answerSender(rt, 'b', { kind: 'bot', role: 'request', exchangeId: 'child-c1', senderSessionId: 'a' }, 'Writer failed', undefined, { failure: sameParentFailure }), true) + await settle(() => host.agentOf('a').calls.length >= 4) + rt.takenWork('parent-consumed', 'child-c1') + assert.equal(answerSender(rt, 'b', { kind: 'bot', role: 'request', exchangeId: 'child-c2', senderSessionId: 'a' }, 'Writer failed', undefined, { failure: sameParentFailure }), true) + await settle(() => host.agentOf('a').calls.length >= 5) + assert.deepEqual( + [host.agentOf('a').calls[4].source.replyTo, host.agentOf('a').calls[4].source.work], + ['child-c2', 'parent-consumed'], + ) +}) + +test('resumeAll checks the upstream work and asker for role=reply continuation turns', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }) + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-reply-resume-')) + t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 })) + + const seed = baseSeed() + seed.bots.c = { id: 'c', name: 'Lead', role: 'Lead', sessionId: 'c', createdAt: 3 } + seed.works = { + 'up-work': { from: 'c', to: 'a', open: [], results: [{ id: 'sub-req', from: 'Writer', text: 'sub done' }], at: 1 }, + } + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + const logs = new Map([['a', []], ['b', []], ['c', []]]) + + // First boot: Writer already delivered the child reply (`sub-reply` -> `sub-req`) to Chief, + // and Chief was interrupted mid-continuation (`role: 'reply'`) before answering Lead (`up-work`). + // resumeAll must NOT mistake the child reply for an already-delivered upstream answer. + const seedHost = await createRestartHost(t, home, logs) + await seedHost.rt.deliver({ + fromId: 'b', toId: 'a', sessionId: 'a', text: 'sub done', + role: 'reply', replyTo: 'sub-req', exchangeId: 'sub-reply', work: 'up-work', + }) + await seedHost.stop() + + writeFileSync(join(home, 'inflight.json'), JSON.stringify({ + turns: { + a: { + botId: 'a', + since: 1, + trigger: { role: 'reply', exchangeId: 'sub-reply', work: 'up-work', senderSessionId: 'b' }, + }, + }, + queued: {}, + })) + const first = await createRestartHost(t, home, logs) + t.mock.timers.tick(3000) + await settle(() => first.agentOf('a').calls.length >= 1) + assert.equal(first.agentOf('a').calls[0].source.role, 'resume') + assert.equal(first.agentOf('a').calls[0].source.work, 'up-work', 'unanswered upstream work must resume') + + // Now Chief delivers the upstream answer to Lead (`up-work`), then crashes before inflight clears. + await first.rt.deliver({ + fromId: 'a', toId: 'c', sessionId: 'c', text: 'upstream final answer', + role: 'reply', replyTo: 'up-work', exchangeId: 'up-reply', + }) + await first.stop() + + writeFileSync(join(home, 'inflight.json'), JSON.stringify({ + turns: { + a: { + botId: 'a', + since: 2, + trigger: { role: 'reply', exchangeId: 'sub-reply', work: 'up-work', senderSessionId: 'b' }, + }, + }, + queued: {}, + })) + const second = await createRestartHost(t, home, logs) + t.mock.timers.tick(3000) + await new Promise(resolve => setImmediate(resolve)) + await realDelay(30) + assert.equal(second.agentOf('a').calls.length, 0, 'already-answered upstream work must not resume again') + assert.equal(second.agentOf('c').calls.length, 0, 'already-delivered upstream reply must not duplicate') + assert.equal(second.rt.state.works['up-work'], undefined, 'upstream work is closed once confirmed delivered') + await second.stop() +}) + +test('reconcileEnded settles distinct parent works that steered into the same failed turn', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }) + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-reconcile-steered-')) + t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 })) + + const seed = baseSeed() + seed.bots.c = { id: 'c', name: 'Lead', role: 'Lead', sessionId: 'c', createdAt: 3 } + seed.works = { + 'p1': { from: 'c', to: 'a', open: ['req-1'], results: [], at: 1 }, + 'p2': { from: 'c', to: 'a', open: ['req-2'], results: [], at: 1 }, + 'req-1': { from: 'a', to: 'b', parent: 'p1', open: [], results: [], at: 2 }, + 'req-2': { from: 'a', to: 'b', parent: 'p2', open: [], results: [], at: 3 }, + } + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + const req1 = { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'req-1', senderName: 'Chief', senderSessionId: 'a' } + const req2 = { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'req-2', senderName: 'Chief', senderSessionId: 'a' } + const logs = new Map([ + ['a', []], + ['b', [ + { type: 'turn/start', seq: 1, data: { turn: 1 } }, + { type: 'user/message', seq: 2, data: { source: req1, content: [{ type: 'text', text: 'first' }] } }, + { type: 'user/message', seq: 3, data: { source: req2, content: [{ type: 'text', text: 'second' }] } }, + { type: 'turn/end', seq: 4, data: { turn: 1, reason: { kind: 'error', error: { code: 'INVALID_REQUEST', status: 400, requestId: 'req-rec-1', message: 'bad' } } } }, + ]], + ]) + writeFileSync(join(home, 'inflight.json'), JSON.stringify({ + turns: { b: { botId: 'b', since: 1, trigger: { role: 'request', exchangeId: 'req-1', senderSessionId: 'a' } } }, + queued: {}, + })) + + const host = await createRestartHost(t, home, logs) + t.mock.timers.tick(3000) + await settle(() => host.agentOf('a').calls.length >= 2) + assert.deepEqual( + host.agentOf('a').calls.map(msg => [msg.source.replyTo, msg.source.work]), + [['req-1', 'p1'], ['req-2', 'p2']], + 'reconcileEnded must notify every distinct parent work steered into the failed turn', + ) + await host.stop() +})