From 06f63c9cc9bf3a47cca818bcf4cb24995cf724fc Mon Sep 17 00:00:00 2001 From: Fei2411 <279467036+Fei2411@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:48:21 -0800 Subject: [PATCH 1/4] Deduplicate member reply and failure notice delivery across steered follow-ups and part switches When a member Bot turn was joined mid-turn by steered message_bot follow-ups and failed (e.g. INVALID_REQUEST with a request_id), turn/end called answerSender once for the primary request and once per joined request, pushing duplicate failure notices with the same request_id into the sender's nextTurn queue. In long conversations that switched parts (or reconciled ended/inflight turns on reload), reconcileEnded only checked the sender's current session log rather than earlier thread segments or the pair delivery ledger, and resumeAll did not inspect whether the session log or pair ledger already recorded a terminal turn before re-running. - Record failureRequestId on delivered pair-ledger entries and expose wasReplyDelivered / wasFailureRequestDelivered helpers in delivery.js - Add sameConversation in threads.js to compare sender conversation identity across thread part switches while keeping parallel openParts and rooms isolated - Make answerSender idempotent per (from, to, exchangeId), per (from, to, conversation, failureRequestId), and per turn/end pass, while settling open works on suppressed duplicates so parent handoffs never hang - Check the pair delivery ledger and earlier thread segments in reconcileEnded and check terminal session logs + pair ledger in resumeAll before re-delivering or re-running --- ds-bot/src/host/delivery.js | 40 ++- ds-bot/src/host/resume.js | 58 ++++- ds-bot/src/host/threads.js | 21 ++ ds-bot/src/host/turns.js | 107 +++++++- ds-bot/test/delivery-dedup.test.js | 406 +++++++++++++++++++++++++++++ 5 files changed, 608 insertions(+), 24 deletions(-) create mode 100644 ds-bot/test/delivery-dedup.test.js diff --git a/ds-bot/src/host/delivery.js b/ds-bot/src/host/delivery.js index 3253c3ca5f..795ad60a4e 100644 --- a/ds-bot/src/host/delivery.js +++ b/ds-bot/src/host/delivery.js @@ -3,6 +3,7 @@ import { randomUUID } from 'node:crypto' import { createUserMessage } from '@deepseek-ai/dsh-llm' import { HISTORY_TIMEOUT_MS, LONG_HEAD, MESSAGE_INLINE_MAX, READ_LINE_MAX, SOURCE_KIND, TEXT_KEPT, TEXT_SHOW_MAX } from './constants.js' import { headOf, textOf } from './text.js' +import { sameConversation } from './threads.js' export function install(rt) { const { ctx, state } = rt @@ -53,18 +54,41 @@ export function install(rt) { .map(entry => ({ ...entry, delivered: true }))) // A message shows the text its delivery carried: the last record of its id written before // a delivery mark. A merged request whose replacement never went out keeps the text that did. - function pairMessages(a, b) { + function pairRecords(a, b) { const written = new Map() const shown = new Map() + const marks = [] for (const record of rt.ledger.records(pairBook(a, b))) { if (record.event === 'delivered') { if (written.has(record.id)) shown.set(record.id, written.get(record.id)) + marks.push(record) } else if (record.event === undefined) { written.set(record.id, record) if (record.delivered === true) shown.set(record.id, record) } } - return [...shown.values()] + return { written, shown, marks } + } + const pairMessages = (a, b) => [...pairRecords(a, b).shown.values()] + + function wasReplyDelivered(fromId, toId, replyTo) { + if (typeof fromId !== 'string' || typeof toId !== 'string' || typeof replyTo !== 'string' || replyTo === '') return false + if (rt.botOf(fromId) === undefined || rt.botOf(toId) === undefined) { + return (state.exchanges ?? []).some(entry => entry.from === fromId && entry.to === toId && entry.role === 'reply' && entry.replyTo === replyTo) + } + return pairMessages(fromId, toId).some(record => record.from === fromId && record.to === toId && record.role === 'reply' && record.replyTo === replyTo) + } + + function wasFailureRequestDelivered(fromId, toId, requestId, targetSessionId) { + if (typeof fromId !== 'string' || typeof toId !== 'string' || typeof requestId !== 'string' || requestId === '') return false + if (rt.botOf(fromId) === undefined || rt.botOf(toId) === undefined) return false + const { written, marks } = pairRecords(fromId, toId) + for (const mark of marks) { + if (mark.failureRequestId !== requestId) continue + const message = written.get(mark.id) + if (message?.from === fromId && message?.to === toId && sameConversation(state, toId, mark.sessionId, targetSessionId)) return true + } + return false } // `sessionId` defaults to the recipient's own chat; group turns name its group Session. @@ -81,7 +105,7 @@ export function install(rt) { // result that came back goes on with. `merged`: a request that takes in an earlier // one its recipient had not read, under that one's exchange. `fromPart`: the Session // of the sending Bot, which a reply comes back to when that is a background part. - async function deliver({ fromId, fromName: senderName, toId, sessionId, text, role, replyTo, roomId, audience, note, exchangeId = randomUUID(), steer = false, deferred = false, resumed = false, retried = false, resent = false, merged = false, abandoned = false, recordExchange = true, askedBy, fromRoomId, fromPart, images = [], files = [], length, work }) { + async function deliver({ fromId, fromName: senderName, toId, sessionId, text, role, replyTo, roomId, audience, note, exchangeId = randomUUID(), steer = false, deferred = false, resumed = false, retried = false, resent = false, merged = false, abandoned = false, recordExchange = true, askedBy, fromRoomId, fromPart, images = [], files = [], length, work, failureRequestId }) { const from = rt.botOf(fromId) const to = rt.botOf(toId) if (to === undefined) throw new Error(`Unknown Bot ${toId}`) @@ -133,7 +157,13 @@ export function install(rt) { else agent.followup(message) await ctx.sessions.flush(agent.session) if (book !== undefined) { - rt.ledger.append(book, { event: 'delivered', id: exchangeId, sessionId: targetSessionId, time: Date.now() }) + rt.ledger.append(book, { + event: 'delivered', + id: exchangeId, + sessionId: targetSessionId, + time: Date.now(), + ...(typeof failureRequestId === 'string' && failureRequestId !== '' ? { failureRequestId } : {}), + }) // The message went out either way; a mark that is lost only hides it from the dialog. await rt.ledger.written(book).catch(() => {}) } else if (recordExchange && EXCHANGE_ROLES.has(role) && !deferred && !resumed && !retried) { @@ -225,5 +255,5 @@ export function install(rt) { return { ...record, ledger: true } } - Object.assign(rt, { resolveAgent, deliver, exchangeText, attachLong, fileText, exchangesOf, exchangeOf }) + Object.assign(rt, { resolveAgent, deliver, exchangeText, attachLong, fileText, exchangesOf, exchangeOf, wasReplyDelivered, wasFailureRequestDelivered }) } diff --git a/ds-bot/src/host/resume.js b/ds-bot/src/host/resume.js index 492ac98a87..526486df35 100644 --- a/ds-bot/src/host/resume.js +++ b/ds-bot/src/host/resume.js @@ -16,7 +16,7 @@ import { join } from 'node:path' import { HISTORY_TIMEOUT_MS, MESSAGE_INLINE_MAX, SOURCE_KIND, STEER_CUT } from './constants.js' import { answerSender, cutOffText, failureLine, turnFailure } from './turns.js' import { textOf } from './text.js' -import { threadCurrentOfPart } from './threads.js' +import { threadCurrentOfPart, threadOfPart } from './threads.js' // Long enough for DSH to finish opening its workspaces after a start. export const RESUME_DELAY_MS = 3000 @@ -311,6 +311,24 @@ export function install(rt) { return Array.isArray(seen?.events) ? seen.events : undefined } + const isTerminalReason = reason => reason !== undefined && reason.kind !== 'interrupted' + && !(reason.kind === 'aborted' && reason.reason?.kind === 'disposed') + + async function alreadyAnswered(botId, sender, where, back, workId) { + if (rt.wasReplyDelivered?.(botId, sender, workId) === true) return true + const sent = await logOf(where) + if (sent !== undefined && answeredIn(sent, workId)) return true + if (back === undefined) { + const segments = threadOfPart(rt.botOf(sender), where)?.segments ?? [] + for (const segment of [...segments].reverse()) { + if (segment?.sessionId === undefined || segment.sessionId === where) continue + const earlier = await logOf(segment.sessionId) + if (earlier !== undefined && answeredIn(earlier, workId)) return true + } + } + return false + } + // A turn that ended while DS Bot was not watching it: its answer may exist and never // have gone to the Bot that waits for it. The Session's durable log says what it did; // the work it was for is settled once, the way a finished turn settles it. @@ -331,6 +349,10 @@ export function install(rt) { } const events = await logOf(sessionId) if (events === undefined) { + if (rt.wasReplyDelivered?.(record.botId, sender, workId) === true) { + rt.closeWork?.(workId, name, rt.answerText(workId, '', name)) + return + } // The record of that work cannot be read: whether it answered cannot be // established. The sender is told that, and the work is not run again. const why = 'the record of that work could not be read' @@ -352,14 +374,13 @@ export function install(rt) { // waiting for it keeps what it had. if (turn === undefined) return const said = turn.texts.join('\n\n') - // The answer went up in another run: the work closes with what it said, and - // nothing is sent twice. + // The answer went up in another run (even before a part switch): the work closes + // with what it said, and nothing is sent twice. const back = rt.roomOf(asker.senderRoomId)?.sessions?.[sender] // The answer would have gone to the part of the asking Bot's chat the request came // from, which is not its foreground chat when that part is a background one. const where = back ?? (asker.senderPart === undefined ? undefined : threadCurrentOfPart(rt.botOf(sender), asker.senderPart)) ?? rt.chatOf(sender) - const sent = await logOf(where) - if (sent !== undefined && answeredIn(sent, workId)) { + if (await alreadyAnswered(record.botId, sender, where, back, workId)) { rt.closeWork?.(workId, name, rt.answerText(workId, said, name)) return } @@ -374,7 +395,7 @@ export function install(rt) { } if (reason.kind === 'error') { const failure = turnFailure(reason.error) - answerSender(rt, sessionId, asker, `${name} could not answer your message: its model call failed (${failureLine(failure)}). Tell the user if it matters; do not retry right away.`) + answerSender(rt, sessionId, asker, `${name} could not answer your message: its model call failed (${failureLine(failure)}). Tell the user if it matters; do not retry right away.`, undefined, { failure }) return } if (reason.kind === 'aborted') { @@ -448,8 +469,29 @@ export function install(rt) { continue } // Still running: noted again, as if it had started here. - if (first === undefined && (await rt.resolveAgent(sessionId).catch(() => undefined))?.status === 'running') update(sessionId, () => record) - else cut.push([sessionId, record]) + if (first === undefined && (await rt.resolveAgent(sessionId).catch(() => undefined))?.status === 'running') { + update(sessionId, () => record) + continue + } + // If inflight.json still held a turn whose durable log already shows a finished + // turn/end, settle from the log instead of re-running the request via resumeTurn. + if (first === undefined && record.roomId === undefined && record.trigger?.exchangeId !== undefined) { + const events = await logOf(sessionId) + const turn = events === undefined ? undefined : turnOf(events, record.trigger.exchangeId) + if (isTerminalReason(turn?.reason)) { + await reconcileEnded(sessionId, record, { reason: turn.reason }).catch(error => rt.warn('ds-bot: settling the work of %s failed: %s', sessionId, error)) + continue + } + if (turn === undefined && record.trigger.senderSessionId !== undefined + && rt.wasReplyDelivered?.(record.botId, record.trigger.senderSessionId, record.trigger.exchangeId) === true) { + const workId = rt.workOf(record.trigger) + if (rt.ownsWork(record.botId, workId)) { + rt.closeWork?.(workId, rt.botOf(record.botId)?.name ?? 'The other Bot', '') + } + continue + } + } + cut.push([sessionId, record]) } firstTurn = undefined for (const [sessionId, list] of Object.entries(dropped)) { diff --git a/ds-bot/src/host/threads.js b/ds-bot/src/host/threads.js index 73ad1677e5..4a668f3b4e 100644 --- a/ds-bot/src/host/threads.js +++ b/ds-bot/src/host/threads.js @@ -92,6 +92,27 @@ export const openSessions = bot => (bot?.openParts ?? []).flatMap(part => [part. // ran and turned this one into a background thread. export const successorOfMovedPart = (bot, freshId) => (threadOf(bot, freshId)?.thread === 'open' ? freshId : undefined) +// Whether two Session ids belong to the same Bot conversation thread or group membership. +// Stays stable across part switches and open-part focus swaps, while isolating distinct +// parallel openParts threads and distinct groups. +export function sameConversation(state, botId, sessionA, sessionB) { + if (sessionA === sessionB) return true + if (sessionA === undefined || sessionB === undefined || botId === undefined) return false + for (const room of Object.values(state?.rooms ?? {})) { + const current = room?.sessions?.[botId] + const segments = room?.segments?.[botId] ?? [] + const hasA = current === sessionA || segments.some(segment => segment?.sessionId === sessionA) + const hasB = current === sessionB || segments.some(segment => segment?.sessionId === sessionB) + if (hasA || hasB) return hasA && hasB + } + const bot = state?.bots?.[botId] + if (bot === undefined) return false + const threadA = threadOfPart(bot, sessionA) + const threadB = threadOfPart(bot, sessionB) + if (threadA === undefined || threadB === undefined) return false + return threadA.thread === threadB.thread && threadA.index === threadB.index +} + // Every live thread's current session. A change that must reach all of a Bot's own // chats (its run mode) walks these, so no thread can be left on the other side. export const ownChatSessions = bot => [currentSession(bot), ...(bot?.openParts ?? []).map(part => part.sessionId)].filter((id, index, list) => id !== undefined && list.indexOf(id) === index) diff --git a/ds-bot/src/host/turns.js b/ds-bot/src/host/turns.js index bf1ec726d1..b0099eb686 100644 --- a/ds-bot/src/host/turns.js +++ b/ds-bot/src/host/turns.js @@ -8,7 +8,7 @@ import { HANDOFF_ACK, LATE_ANSWER, MESSAGE_INLINE_MAX, REFERENCE_KIND, REPLY_KIN import { cleanText } from './diagnostics.js' import { absoluteLinks } from './links.js' import { splitRef, textOf } from './text.js' -import { threadCurrentOfPart } from './threads.js' +import { sameConversation, threadCurrentOfPart } from './threads.js' import { viaOf } from '../shared/im-platforms.js' import { reminderText } from './schedules.js' import { usageChunk } from './usage.js' @@ -51,10 +51,43 @@ export function turnFailure(error) { export const failureLine = failure => `${failure.code}${failure.status ? ` ${failure.status}` : ''}${failure.message ? `: ${failure.message.slice(0, 200)}` : ''}` +const MAX_REPLY_GUARDS = 1000 +const replyGuards = new WeakMap() + +const guardsOf = (rt) => { + let guards = replyGuards.get(rt) + if (guards === undefined) { + guards = { exchanges: new Set(), failureRequests: [], turnFailures: [] } + replyGuards.set(rt, guards) + } + return guards +} + +const pushLimited = (list, item) => { + list.push(item) + if (list.length > MAX_REPLY_GUARDS) list.shift() +} + +const removeItem = (list, item) => { + const at = list.indexOf(item) + if (at !== -1) list.splice(at, 1) +} + +function settleSuppressedWork(rt, exchangeId, name, text, primaryWork) { + if (typeof exchangeId !== 'string' || exchangeId === '') return + const ownParent = rt.state?.works?.[exchangeId]?.parent + if (ownParent !== undefined && ownParent === primaryWork && typeof rt.dropWork === 'function') { + rt.dropWork(exchangeId) + return + } + const parent = rt.closeWork?.(exchangeId, name, text) + if (parent !== undefined) rt.takenWork?.(parent, exchangeId) +} + // The reply a sender waits for, sent the way a finished turn sends it: the work it was // for closes, and the answer goes up to the Bot that asked. `id` is the Session the // answer comes from; `source` is the trigger the turn answered. -export function answerSender(rt, id, source, text, long) { +export function answerSender(rt, id, source, text, long, options = {}) { const back = rt.roomOf(source.senderRoomId)?.sessions?.[source.senderSessionId] // The asking Bot may have written from a background part of its own chat: its reply // goes back to that part's thread, not to the part the user is reading. When the part @@ -62,8 +95,45 @@ export function answerSender(rt, id, source, text, long) { const asker = source.senderPart === undefined ? undefined : rt.botOf(source.senderSessionId) const part = back === undefined && asker !== undefined ? threadCurrentOfPart(asker, source.senderPart) : undefined const target = back ?? part - const name = rt.botOf(rt.selfOf(id))?.name ?? 'The other Bot' + const fromId = rt.selfOf(id) + const toId = source.senderSessionId + const targetSessionId = target ?? rt.chatOf?.(toId) ?? toId + const name = rt.botOf(fromId)?.name ?? 'The other Bot' + const guards = guardsOf(rt) + const exchangeKey = typeof source.exchangeId === 'string' && source.exchangeId !== '' + ? `${fromId ?? ''}\0${toId ?? ''}\0${source.exchangeId}` + : undefined + if ((exchangeKey !== undefined && guards.exchanges.has(exchangeKey)) + || rt.wasReplyDelivered?.(fromId, toId, source.exchangeId) === true) { + settleSuppressedWork(rt, source.exchangeId, name, text, undefined) + return false + } + const failureRequestId = typeof options?.failure?.requestId === 'string' && options.failure.requestId !== '' + ? options.failure.requestId + : undefined + const dedupKey = typeof options?.dedupKey === 'string' && options.dedupKey !== '' ? options.dedupKey : undefined + const matchSameTarget = entry => entry.fromId === fromId && entry.toId === toId + && sameConversation(rt.state, toId, entry.targetSessionId, targetSessionId) + const matchedReq = failureRequestId === undefined ? undefined + : guards.failureRequests.find(entry => entry.requestId === failureRequestId && matchSameTarget(entry)) + const matchedTurn = dedupKey === undefined ? undefined + : guards.turnFailures.find(entry => entry.dedupKey === dedupKey && matchSameTarget(entry)) + if (matchedReq !== undefined || matchedTurn !== undefined + || (failureRequestId !== undefined && rt.wasFailureRequestDelivered?.(fromId, toId, failureRequestId, targetSessionId) === true)) { + settleSuppressedWork(rt, source.exchangeId, name, text, (matchedReq ?? matchedTurn)?.work) + return false + } const work = rt.closeWork?.(source.exchangeId, name, text) + if (exchangeKey !== undefined) { + guards.exchanges.add(exchangeKey) + if (guards.exchanges.size > MAX_REPLY_GUARDS) guards.exchanges.delete(guards.exchanges.values().next().value) + } + const failureEntry = failureRequestId === undefined ? undefined + : { fromId, toId, targetSessionId, requestId: failureRequestId, work } + if (failureEntry !== undefined) pushLimited(guards.failureRequests, failureEntry) + const turnEntry = dedupKey === undefined ? undefined + : { fromId, toId, targetSessionId, dedupKey, work } + if (turnEntry !== undefined) pushLimited(guards.turnFailures, turnEntry) // A reply that arrives after the user gave up the wait is a late answer: it closes // nothing, opens nothing, and says on it that the wait was already over. const late = work === undefined ? rt.abandonedWork?.(source.exchangeId) : undefined @@ -72,8 +142,8 @@ export function answerSender(rt, id, source, text, long) { const length = long === undefined ? undefined : late === undefined ? long.length : (long.length ?? text.length) + marker.length const exchangeId = randomUUID() void rt.deliver({ - fromId: rt.selfOf(id), - toId: source.senderSessionId, + fromId, + toId, ...(target ? { sessionId: target } : {}), text: body, ...(length === undefined ? {} : { files: long.files, length }), @@ -81,6 +151,7 @@ export function answerSender(rt, id, source, text, long) { replyTo: source.exchangeId, exchangeId, ...(work ? { work } : {}), + ...(failureRequestId === undefined ? {} : { failureRequestId }), }).catch((error) => { rt.warn('ds-bot: reply delivery failed: %s', error) // The reply never reached the asker: its note goes — the notice below carries @@ -91,16 +162,23 @@ export function answerSender(rt, id, source, text, long) { rt.answerMissed?.(source.exchangeId) const said = `${name} could not send its reply to you (${failureLine(turnFailure(error))}); what it had answered:\n` void rt.deliver({ - fromId: rt.selfOf(id), - toId: source.senderSessionId, + fromId, + toId, ...(target ? { sessionId: target } : {}), text: `${said}${long?.text ?? text}`, ...(long?.length === undefined ? {} : { files: long.files, length: said.length + long.length }), role: 'reply', replyTo: source.exchangeId, ...(work ? { work } : {}), - }).catch(again => rt.warn('ds-bot: reply failure notice not delivered: %s', again)) + ...(failureRequestId === undefined ? {} : { failureRequestId }), + }).catch((again) => { + rt.warn('ds-bot: reply failure notice not delivered: %s', again) + if (exchangeKey !== undefined) guards.exchanges.delete(exchangeKey) + if (failureEntry !== undefined) removeItem(guards.failureRequests, failureEntry) + if (turnEntry !== undefined) removeItem(guards.turnFailures, turnEntry) + }) }) + return true } // A group's own Session never reaches a model, so the snapshot DSH prepared there @@ -491,7 +569,8 @@ export function install(rt) { // reply in its Session there while it is still a member. A sender that asked for it // while working on another Bot's request goes on with that work when it reads it. // `long`: the text as attachLong sends it, when it went as a file. - const tellSender = (id, source, text, long) => answerSender(rt, id, source, text, long) + let turnEndSeq = 0 + const tellSender = (id, source, text, long, options) => answerSender(rt, id, source, text, long, options) // Runs before a deleted Bot leaves the team. Its Sessions stop with it, and the // turn/end that follows is no longer the team's, so their activity and turn records go @@ -1245,15 +1324,21 @@ export function install(rt) { // Whoever the turn answers gets the same three-way rule: a failure notice, // the turn's text (or a stopped-before-answer line), or nothing. const name = rt.botOf(rt.selfOf(id))?.name ?? 'The other Bot' + const turnEndToken = ++turnEndSeq + const turnDedupKey = failure !== undefined + ? `${turnEndToken}:error:${failure.requestId ?? failure.code}` + : reason?.kind === 'aborted' + ? `${turnEndToken}:aborted` + : undefined const answer = (source, reply) => { if (!expectsReply(source) || answersUser(source)) return if (failure !== undefined) { // The sender was told a reply comes later; without this it would wait forever. - tellSender(id, source, `${name} could not answer your message: its model call failed (${failureLine(failure)}). Tell the user if it matters; do not retry right away.`) + tellSender(id, source, `${name} could not answer your message: its model call failed (${failureLine(failure)}). Tell the user if it matters; do not retry right away.`, undefined, { failure, dedupKey: turnDedupKey }) } else if (reply !== '' || reason?.kind === 'aborted') { // A turn stopped before it could answer still owes the sender a line, // or it would wait forever for a reply that never comes. - tellSender(id, source, reply !== '' ? reply : `${name} was stopped before it could answer your message.`) + tellSender(id, source, reply !== '' ? reply : `${name} was stopped before it could answer your message.`, undefined, reply === '' && turnDedupKey !== undefined ? { dedupKey: turnDedupKey } : undefined) } } // Work a Bot asked for, also when what came back for it opens the turn: while diff --git a/ds-bot/test/delivery-dedup.test.js b/ds-bot/test/delivery-dedup.test.js new file mode 100644 index 0000000000..4f58079b16 --- /dev/null +++ b/ds-bot/test/delivery-dedup.test.js @@ -0,0 +1,406 @@ +import assert from 'node:assert/strict' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'node:test' +import { setTimeout as realDelay } from 'node:timers/promises' +import { Context } from '@deepseek-ai/cordis' +import { LlmAdapter } from '@deepseek-ai/dsh-llm' +import { SessionId } from '@deepseek-ai/dsh-session' +import { mountAgentLoopTestDependencies, mountAgentLoopTestHarness } from '@deepseek-ai/dsh-agent-loop-testkit' +import { createRuntime } from '../index.js' +import { answerSender } from '../src/host/turns.js' +import { fakeContext } from './fake-ctx.js' + +const settle = async (check, ms = 4000) => { + const until = Date.now() + ms + while (!check() && Date.now() < until) await realDelay(10) + assert.ok(check(), 'condition not reached') +} + +const textOf = message => (message?.content ?? []).map(block => block.text ?? '').join('\n') + +const baseSeed = () => ({ + version: 1, + mainBotIds: ['a'], + bots: { + a: { id: 'a', name: 'Chief', role: 'Boss', sessionId: 'a', createdAt: 1 }, + b: { id: 'b', name: 'Writer', role: 'Write', sessionId: 'b', createdAt: 2 }, + }, + rooms: {}, + agents: {}, + exchanges: [], + exchangeTurns: {}, + questions: {}, + prefs: {}, + revision: 0, +}) + +async function createLoopRig(t, seed = baseSeed()) { + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-loop-')) + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + + const root = new Context() + await mountAgentLoopTestDependencies(root) + + const stepGate = Promise.withResolvers() + const modes = { b: 'two-step-fail', requestId: 'req-dup-1319', callCount: 0 } + + root.tools.register({ + name: 'wait_step', + description: 'Pause mid-turn so another Bot message can steer in', + parameters: {}, + output: { schema: { type: 'object' }, render: (_args, value) => [{ type: 'text', text: JSON.stringify(value) }] }, + async execute() { + await stepGate.promise + return { ok: true } + }, + }) + + class TestAdapter extends LlmAdapter { + async resolveModel(provider, id) { + return { provider, id, name: id, inputModalities: ['text'], context: { contextWindow: 100000 } } + } + async *stream(options) { + modes.callCount += 1 + if (options.sessionId === 'b') { + const last = options.messages.at(-1) + const hasToolResult = last?.role === 'tool' + if (!hasToolResult && modes.b === 'two-step-fail') { + const args = '{}' + yield { type: 'block-start', index: 0, blockType: 'tool-call' } + yield { type: 'tool-call-delta', index: 0, id: `call_${modes.callCount}`, name: 'wait_step', argumentsDelta: args } + yield { type: 'block-end', index: 0, block: { type: 'tool-call', id: `call_${modes.callCount}`, name: 'wait_step', arguments: args } } + yield { type: 'finish', reason: { kind: 'tool-calls' } } + return + } + yield { + type: 'finish', + reason: { + kind: 'error', + failure: { + code: 'INVALID_REQUEST', + status: 400, + message: `invalid tool payload (request_id: ${modes.requestId})`, + requestId: modes.requestId, + }, + }, + } + return + } + const reply = `Handled ${options.messages.length}` + yield { type: 'block-start', index: 0, blockType: 'text' } + yield { type: 'text-delta', index: 0, text: reply } + yield { type: 'block-end', index: 0, block: { type: 'text', text: reply } } + yield { type: 'finish', reason: { kind: 'stop' } } + } + } + root.llm.registerAdapter(['dedup-test'], new TestAdapter()) + await mountAgentLoopTestHarness(root) + + const agents = new Map() + for (const id of Object.keys(seed.bots)) { + const bot = seed.bots[id] + const sessions = [bot.sessionId, ...(bot.openParts ?? []).map(part => part.sessionId)] + for (const sessionId of sessions) { + agents.set(sessionId, await root.agentLoop.create(SessionId(sessionId), { provider: 'dedup-test', model: 'test' })) + } + } + + const deliveries = [] + const ctx = fakeContext({ + services: { + agents: root.agents, + llm: root.llm, + sessions: root.sessions, + tools: root.tools, + workspaceRegistry: { get: () => undefined, resolveByPath: async () => undefined, create: async path => ({ id: 'w1', path }), archiveSession: async () => {} }, + sessionController: { + resolveAgent: async sessionId => ({ agent: agents.get(sessionId) }), + create: async () => ({ sessionId: 's-unused' }), + selectModel: async () => {}, + page: async () => ({ records: [], hasMore: false }), + inspect: async sessionId => ({ events: agents.get(sessionId)?.session.snapshotEvents() ?? [] }), + }, + sessionTitle: { rename: async () => {} }, + }, + }) + Object.assign(ctx, { + agents: root.agents, + llm: root.llm, + sessions: root.sessions, + tools: root.tools, + workspaceRegistry: ctx.get('workspaceRegistry'), + sessionController: ctx.get('sessionController'), + sessionTitle: ctx.get('sessionTitle'), + }) + root.on('session/event', (session, event) => { + if (event.type === 'user/message' && event.data?.source?.kind === 'bot') { + deliveries.push({ sessionId: session.id, source: event.data.source, text: textOf({ content: event.data.content }) }) + } + for (const listener of ctx.listeners.get('session/event') ?? []) { + if (listener.length >= 2) listener(session, event) + } + }) + root.on('agent/inbox/discarded', (payload) => { + for (const listener of ctx.listeners.get('agent/inbox/discarded') ?? []) listener(payload) + }) + + const rt = createRuntime(ctx, { home }) + t.after(async () => { + stepGate.resolve() + await ctx.dispose() + await root.fiber.dispose() + rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }) + }) + await rt.load() + const tool = (name, args, sessionId) => rt.teamTools.find(def => def.name === name).execute(args, { agent: agents.get(sessionId) }) + return { home, rt, agents, tool, stepGate, modes, deliveries } +} + +async function createRestartHost(t, home, logs) { + const ctx = fakeContext() + const agents = new Map() + const agentOf = (sessionId) => { + if (!agents.has(sessionId)) { + const calls = [] + const inbox = { nextStep: [], nextTurn: [], remove: () => false } + agents.set(sessionId, { + session: { id: sessionId }, + calls, + inbox, + status: 'idle', + followup(message) { calls.push(message) }, + steer(message) { calls.push(message) }, + cancel() {}, + }) + } + return agents.get(sessionId) + } + Object.assign(ctx, { + llm: { listProviders: () => [], listModels: async () => [], resolveModelInfo: async () => ({}) }, + workspaceRegistry: { get: () => undefined, resolveByPath: async () => undefined, create: async path => ({ id: 'w1', path }), archiveSession: async () => {} }, + sessionController: { + create: async () => ({ sessionId: 's1' }), + resolveAgent: async sessionId => ({ agent: agentOf(sessionId) }), + selectModel: async () => {}, + inspect: async sessionId => ({ events: logs.get(sessionId) ?? [] }), + }, + sessionTitle: { rename: async () => {} }, + sessions: { flush: async () => {} }, + }) + const rt = createRuntime(ctx, { home }) + let seq = 0 + const emit = (id, type, data) => { + const session = agents.get(id)?.session ?? { id } + for (const listener of ctx.listeners.get('session/event') ?? []) { + if (listener.length >= 2) listener(session, { type, seq: ++seq, data }) + } + } + await rt.load() + const host = { ctx, rt, agentOf, emit, stop: () => ctx.dispose() } + return host +} + +test('a failed turn with steered follow-ups pushes only one failure notice per request_id and settles open works', async (t) => { + const { home, rt, agents, tool, stepGate, modes, deliveries } = await createLoopRig(t) + + // Chief sends an initial task to Writer; Writer starts working and pauses in wait_step. + await tool('message_bot', { to: 'Writer', message: 'Run task 1' }, 'a') + await settle(() => modes.callCount >= 1 && rt.live.get('b')?.tool === 'wait_step') + + // While Writer is mid-turn, Chief steers a follow-up update (default when: 'now'). + await tool('message_bot', { to: 'Writer', message: 'Also include check 2' }, 'a') + await settle(() => agents.get('b').inbox.nextStep.length === 1) + + // Let wait_step finish so Writer claims the steered follow-up and hits INVALID_REQUEST. + stepGate.resolve() + await agents.get('b').whenIdle() + await settle(() => deliveries.some(entry => entry.sessionId === 'a' && entry.source.role === 'reply')) + await agents.get('a').whenIdle() + await realDelay(30) + + const failureRepliesToChief = deliveries.filter(entry => entry.sessionId === 'a' && entry.source.role === 'reply' && entry.text.includes('req-dup-1319')) + assert.equal(failureRepliesToChief.length, 1, 'Chief must receive the failure notice for req-dup-1319 only once') + const state = JSON.parse(readFileSync(join(home, 'state.json'), 'utf8')) + assert.deepEqual(Object.keys(state.works ?? {}), [], 'all works bound to the failed turn must be closed') + + // A subsequent turn that fails with a new request_id still notifies Chief. + modes.b = 'fail-once' + modes.requestId = 'req-new-456' + await tool('message_bot', { to: 'Writer', message: 'Retry with clean args' }, 'a') + await agents.get('b').whenIdle() + await settle(() => deliveries.some(entry => entry.sessionId === 'a' && entry.source.role === 'reply' && entry.text.includes('req-new-456'))) + await agents.get('a').whenIdle() + const newReplies = deliveries.filter(entry => entry.sessionId === 'a' && entry.source.role === 'reply' && entry.text.includes('req-new-456')) + assert.equal(newReplies.length, 1) +}) + +test('a failed turn with steered requests from two openParts notifies each conversation once per request_id', async (t) => { + const seed = baseSeed() + seed.bots.a.openParts = [{ sessionId: 'a-bg', segments: [] }] + const { rt, agents, tool, stepGate, modes, deliveries } = await createLoopRig(t, seed) + + await tool('message_bot', { to: 'Writer', message: 'From fg 1' }, 'a') + await settle(() => modes.callCount >= 1 && rt.live.get('b')?.tool === 'wait_step') + + // Same fg thread steers another request, and a separate bg openPart thread also steers one. + await tool('message_bot', { to: 'Writer', message: 'From fg 2' }, 'a') + await tool('message_bot', { to: 'Writer', message: 'From bg 1' }, 'a-bg') + await settle(() => agents.get('b').inbox.nextStep.length === 2) + + stepGate.resolve() + await agents.get('b').whenIdle() + await settle(() => deliveries.some(entry => entry.sessionId === 'a' && entry.source.role === 'reply') && deliveries.some(entry => entry.sessionId === 'a-bg' && entry.source.role === 'reply')) + await Promise.all([agents.get('a').whenIdle(), agents.get('a-bg').whenIdle()]) + await realDelay(30) + + const fgFailures = deliveries.filter(entry => entry.sessionId === 'a' && entry.source.role === 'reply' && entry.text.includes('req-dup-1319')) + const bgFailures = deliveries.filter(entry => entry.sessionId === 'a-bg' && entry.source.role === 'reply' && entry.text.includes('req-dup-1319')) + assert.equal(fgFailures.length, 1, 'foreground conversation receives one notice') + assert.equal(bgFailures.length, 1, 'independent background openPart receives one notice') +}) + +test('answerSender is idempotent per exchangeId and per failure requestId while preserving late replies after abandonWork', async (t) => { + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-answer-')) + const seed = baseSeed() + seed.works = { + 'ex-1': { from: 'a', to: 'b', open: [], results: [], at: 1 }, + 'ex-parent': { from: 'a', to: 'b', open: ['ex-child-1', 'ex-child-2'], results: [], at: 1 }, + 'ex-child-1': { from: 'a', to: 'b', parent: 'ex-parent', open: [], results: [], at: 2 }, + 'ex-child-2': { from: 'a', to: 'b', parent: 'ex-parent', open: [], results: [], at: 3 }, + 'ex-abandoned-parent': { from: 'a', to: 'a', open: ['ex-late'], results: [], at: 1 }, + 'ex-late': { from: 'a', to: 'b', parent: 'ex-abandoned-parent', open: [], results: [], at: 2 }, + } + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + const host = await createRestartHost(t, home, new Map()) + t.after(async () => { + await host.stop() + rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }) + }) + + const { rt } = host + const src1 = { kind: 'bot', role: 'reply', exchangeId: 'ex-1', senderSessionId: 'b' } + const req1 = { kind: 'bot', role: 'request', exchangeId: 'ex-1', senderSessionId: 'a' } + answerSender(rt, 'b', req1, 'Final report verbatim') + answerSender(rt, 'b', req1, 'Final report verbatim') + await settle(() => host.agentOf('a').calls.length >= 1) + await realDelay(30) + assert.equal(host.agentOf('a').calls.length, 1, 'same exchangeId must only deliver once') + + // Failure deduplication by requestId also cleans up duplicate child open work entries. + const child1 = { kind: 'bot', role: 'request', exchangeId: 'ex-child-1', senderSessionId: 'a' } + const child2 = { kind: 'bot', role: 'request', exchangeId: 'ex-child-2', senderSessionId: 'a' } + const failure = { code: 'INVALID_REQUEST', status: 400, requestId: 'req-same-99', message: 'bad arg' } + answerSender(rt, 'b', child1, 'Writer failed (req-same-99)', undefined, { failure }) + answerSender(rt, 'b', child2, 'Writer failed (req-same-99)', undefined, { failure }) + await settle(() => host.agentOf('a').calls.length >= 2) + await realDelay(30) + assert.equal(host.agentOf('a').calls.length, 2, 'same requestId failure must only deliver once') + assert.equal(rt.state.works['ex-child-2'], undefined, 'suppressed duplicate work must be removed') + assert.deepEqual(rt.state.works['ex-parent'].open, ['ex-child-1'], 'suppressed duplicate must not stay stuck in parent.open') + + // abandonWork + late answer still delivers once, then a replay is blocked. + rt.abandonWork('ex-late', 'user') + await settle(() => host.agentOf('a').calls.length >= 3) + const lateSource = { kind: 'bot', role: 'request', exchangeId: 'ex-late', senderSessionId: 'a' } + answerSender(rt, 'b', lateSource, 'Actual late completion') + await settle(() => host.agentOf('a').calls.length >= 4) + answerSender(rt, 'b', lateSource, 'Actual late completion') + await realDelay(30) + assert.equal(host.agentOf('a').calls.length, 4, 'late reply arrives once and replay is blocked') + assert.equal(src1.exchangeId, 'ex-1') +}) + +test('reconcileEnded does not duplicate replies or request_id failures after the asker moves to a fresh part', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }) + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-reconcile-')) + const logs = new Map() + const seed = baseSeed() + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + + const first = await createRestartHost(t, home, logs) + t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 })) + + const request = { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'ex-part-switch', senderName: 'Chief', senderSessionId: 'a' } + first.rt.openWork('ex-part-switch', { from: 'a', to: 'b' }) + first.emit('b', 'turn/start', { turn: 1 }) + first.emit('b', 'user/message', { content: [{ type: 'text', text: '[Message from Chief]\nCheck' }], source: request }) + first.emit('b', 'assistant/message', { message: { content: [{ type: 'text', text: 'The plan is ready.' }] } }) + + // Deliver the reply to part 'a', Chief consumes it, then Chief advances to part 'a-2'. + await first.rt.deliver({ fromId: 'b', toId: 'a', sessionId: 'a', text: 'The plan is ready.', role: 'reply', replyTo: 'ex-part-switch', exchangeId: 'reply-1' }) + const deliveredReply = first.agentOf('a').calls[0] + first.emit('a', 'user/message', { content: deliveredReply.content, source: deliveredReply.source }) + first.rt.state.bots.a.segments = [{ sessionId: 'a', endedAt: 10 }] + first.rt.state.bots.a.sessionId = 'a-2' + await first.rt.save({ required: true }) + + // Leave inflight.turns.b active as if plugin reload/restart happened before Writer's turn/end persisted, + // while Writer's durable log holds the completed turn and Chief's new part 'a-2' has no record of it. + logs.set('b', [ + { type: 'turn/start', seq: 1, data: { turn: 1 } }, + { type: 'user/message', seq: 2, data: { source: request, content: [{ type: 'text', text: '[Message from Chief]\nCheck' }] } }, + { type: 'assistant/message', seq: 3, data: { message: { content: [{ type: 'text', text: 'The plan is ready.' }] } } }, + { type: 'turn/end', seq: 4, data: { turn: 1, reason: { kind: 'completed' } } }, + ]) + logs.set('a-2', []) + await first.stop() + + const second = await createRestartHost(t, home, logs) + second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } }) + t.mock.timers.tick(3000) + await new Promise(resolve => setImmediate(resolve)) + await realDelay(30) + + assert.equal(second.agentOf('a-2').calls.length, 0, 'reply already in pair ledger must not be re-delivered after part switch') + assert.equal(second.agentOf('a').calls.length, 0) + assert.equal(second.agentOf('b').calls.length, 0) + await second.stop() +}) + +test('resumeAll settles already-ended turns from disk without re-running or duplicating replies', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }) + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-resume-')) + t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 })) + + const seed = baseSeed() + seed.works = { 'ex-ended': { from: 'a', to: 'b', open: [], results: [], at: 1 } } + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + const logs = new Map() + const request = { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'ex-ended', senderName: 'Chief', senderSessionId: 'a' } + logs.set('b', [ + { type: 'turn/start', seq: 1, data: { turn: 1 } }, + { type: 'user/message', seq: 2, data: { source: request, content: [{ type: 'text', text: '[Message from Chief]\nFinish' }] } }, + { type: 'assistant/message', seq: 3, data: { message: { content: [{ type: 'text', text: 'Already finished answer.' }] } } }, + { type: 'turn/end', seq: 4, data: { turn: 1, reason: { kind: 'completed' } } }, + ]) + logs.set('a', []) + + // Pre-seed inflight.turns as left behind if inflight.json did not clear on crash. + writeFileSync(join(home, 'inflight.json'), JSON.stringify({ + turns: { b: { botId: 'b', since: 1, trigger: { role: 'request', exchangeId: 'ex-ended', senderSessionId: 'a' } } }, + queued: {}, + })) + + const first = await createRestartHost(t, home, logs) + t.mock.timers.tick(3000) + await settle(() => first.agentOf('a').calls.length >= 1 || first.agentOf('b').calls.length >= 1) + assert.equal(first.agentOf('b').calls.length, 0, 'completed turn on disk must not be re-run with resumeTurn') + assert.equal(first.agentOf('a').calls.length, 1, 'completed turn reply must settle to Chief once') + assert.match(textOf(first.agentOf('a').calls[0]), /Already finished answer\./) + await first.stop() + + // Second crash before inflight clears: now reply is already in pair ledger, so nothing is re-sent. + writeFileSync(join(home, 'inflight.json'), JSON.stringify({ + turns: { b: { botId: 'b', since: 1, trigger: { role: 'request', exchangeId: 'ex-ended', senderSessionId: 'a' } } }, + queued: {}, + })) + const second = await createRestartHost(t, home, logs) + t.mock.timers.tick(3000) + await new Promise(resolve => setImmediate(resolve)) + await realDelay(30) + assert.equal(second.agentOf('b').calls.length, 0) + assert.equal(second.agentOf('a').calls.length, 0, 'already-delivered reply must not be sent again') + await second.stop() +}) From ab557d3621f159e0ec05d7b6d255941d2105ed11 Mon Sep 17 00:00:00 2001 From: Fei2411 <279467036+Fei2411@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:51:13 -0800 Subject: [PATCH 2/4] Scope failure deduplication by parent work and fix reply-continuation resume Prevent distinct parent works from stalling when steered into the same failed turn, and fix restart reconciliation for continuation turns: - Scope in-memory and ledger failure deduplication in answerSender by parent work and require the primary child to still be pending in parent.open before coalescing a second child's failure. - Settle steered child requests from distinct parent works in reconcileEnded as well. - In resumeAll, resolve role=reply continuation triggers via rt.workOf(trigger) and the upstream asker rather than the child reply's exchangeId and senderSessionId. - Clear stale child parent pointers in dropWork. Fixes #1319. --- ds-bot/src/host/delivery.js | 44 +++++-- ds-bot/src/host/handoffs.js | 1 + ds-bot/src/host/resume.js | 181 ++++++++++++++++++++--------- ds-bot/src/host/turns.js | 55 ++++++--- ds-bot/test/delivery-dedup.test.js | 158 +++++++++++++++++++++++++ 5 files changed, 361 insertions(+), 78 deletions(-) diff --git a/ds-bot/src/host/delivery.js b/ds-bot/src/host/delivery.js index 795ad60a4e..e3bc4b059b 100644 --- a/ds-bot/src/host/delivery.js +++ b/ds-bot/src/host/delivery.js @@ -71,26 +71,47 @@ export function install(rt) { } const pairMessages = (a, b) => [...pairRecords(a, b).shown.values()] - function wasReplyDelivered(fromId, toId, replyTo) { - if (typeof fromId !== 'string' || typeof toId !== 'string' || typeof replyTo !== 'string' || replyTo === '') return false - if (rt.botOf(fromId) === undefined || rt.botOf(toId) === undefined) { - return (state.exchanges ?? []).some(entry => entry.from === fromId && entry.to === toId && entry.role === 'reply' && entry.replyTo === replyTo) + function deliveredReplyOf(fromId, toId, replyTo) { + if (typeof fromId !== 'string' || fromId === '' || typeof replyTo !== 'string' || replyTo === '') return undefined + if (typeof toId === 'string' && toId !== '') { + if (rt.botOf(fromId) === undefined || rt.botOf(toId) === undefined) { + return (state.exchanges ?? []).findLast(entry => entry.from === fromId && entry.to === toId && entry.role === 'reply' && entry.replyTo === replyTo) + } + return pairMessages(fromId, toId).findLast(record => record.from === fromId && record.to === toId && record.role === 'reply' && record.replyTo === replyTo) + } + if (rt.botOf(fromId) !== undefined) { + for (const otherId of Object.keys(state.bots ?? {})) { + if (otherId === fromId) continue + const found = pairMessages(fromId, otherId).findLast(record => record.from === fromId && record.to === otherId && record.role === 'reply' && record.replyTo === replyTo) + if (found !== undefined) return found + } } - return pairMessages(fromId, toId).some(record => record.from === fromId && record.to === toId && record.role === 'reply' && record.replyTo === replyTo) + return (state.exchanges ?? []).findLast(entry => entry.from === fromId && entry.role === 'reply' && entry.replyTo === replyTo) } - function wasFailureRequestDelivered(fromId, toId, requestId, targetSessionId) { - if (typeof fromId !== 'string' || typeof toId !== 'string' || typeof requestId !== 'string' || requestId === '') return false - if (rt.botOf(fromId) === undefined || rt.botOf(toId) === undefined) return false + const wasReplyDelivered = (fromId, toId, replyTo) => deliveredReplyOf(fromId, toId, replyTo) !== undefined + + function deliveredFailuresOf(fromId, toId, requestId, targetSessionId, parentWork) { + if (typeof fromId !== 'string' || typeof toId !== 'string' || typeof requestId !== 'string' || requestId === '') return [] + if (rt.botOf(fromId) === undefined || rt.botOf(toId) === undefined) return [] + const wantedWork = typeof parentWork === 'string' && parentWork !== '' ? parentWork : undefined const { written, marks } = pairRecords(fromId, toId) + const matches = [] for (const mark of marks) { if (mark.failureRequestId !== requestId) continue + const markWork = typeof mark.work === 'string' && mark.work !== '' ? mark.work : undefined + if (markWork !== wantedWork) continue const message = written.get(mark.id) - if (message?.from === fromId && message?.to === toId && sameConversation(state, toId, mark.sessionId, targetSessionId)) return true + if (message?.from === fromId && message?.to === toId && sameConversation(state, toId, mark.sessionId, targetSessionId)) { + matches.push({ id: mark.id, replyTo: message.replyTo, work: markWork, sessionId: mark.sessionId }) + } } - return false + return matches } + const wasFailureRequestDelivered = (fromId, toId, requestId, targetSessionId, parentWork) => + deliveredFailuresOf(fromId, toId, requestId, targetSessionId, parentWork).length > 0 + // `sessionId` defaults to the recipient's own chat; group turns name its group Session. // `deferred`: a request the Bot already read mid-turn, handed back to answer in a // turn of its own; its exchange was recorded when it first arrived. `resumed` and @@ -163,6 +184,7 @@ export function install(rt) { sessionId: targetSessionId, time: Date.now(), ...(typeof failureRequestId === 'string' && failureRequestId !== '' ? { failureRequestId } : {}), + ...(typeof work === 'string' && work !== '' ? { work } : {}), }) // The message went out either way; a mark that is lost only hides it from the dialog. await rt.ledger.written(book).catch(() => {}) @@ -255,5 +277,5 @@ export function install(rt) { return { ...record, ledger: true } } - Object.assign(rt, { resolveAgent, deliver, exchangeText, attachLong, fileText, exchangesOf, exchangeOf, wasReplyDelivered, wasFailureRequestDelivered }) + Object.assign(rt, { resolveAgent, deliver, exchangeText, attachLong, fileText, exchangesOf, exchangeOf, deliveredReplyOf, wasReplyDelivered, deliveredFailuresOf, wasFailureRequestDelivered }) } diff --git a/ds-bot/src/host/handoffs.js b/ds-bot/src/host/handoffs.js index 7a4b3d999f..a78f30ae33 100644 --- a/ds-bot/src/host/handoffs.js +++ b/ds-bot/src/host/handoffs.js @@ -197,6 +197,7 @@ export function install(rt) { if (work === undefined) return const { parent } = work delete works()[id] + for (const child of Object.values(works())) if (child.parent === id) delete child.parent const above = works()[parent] if (above !== undefined) above.open = above.open.filter(other => other !== id) for (const result of work.results ?? []) whole.delete(result.id) diff --git a/ds-bot/src/host/resume.js b/ds-bot/src/host/resume.js index 526486df35..37a9d3de02 100644 --- a/ds-bot/src/host/resume.js +++ b/ds-bot/src/host/resume.js @@ -314,43 +314,61 @@ export function install(rt) { const isTerminalReason = reason => reason !== undefined && reason.kind !== 'interrupted' && !(reason.kind === 'aborted' && reason.reason?.kind === 'disposed') - async function alreadyAnswered(botId, sender, where, back, workId) { - if (rt.wasReplyDelivered?.(botId, sender, workId) === true) return true + const whereFor = (asker) => { + const sender = asker?.senderSessionId + if (sender === undefined) return { sender: undefined, back: undefined, where: undefined } + const back = rt.roomOf(asker.senderRoomId)?.sessions?.[sender] + // The answer would have gone to the part of the asking Bot's chat the request came + // from, which is not its foreground chat when that part is a background one. + const where = back ?? (asker.senderPart === undefined ? undefined : threadCurrentOfPart(rt.botOf(sender), asker.senderPart)) ?? rt.chatOf(sender) + return { sender, back, where } + } + + async function alreadyAnswered(botId, sender, where, back, roomId, workId) { + const delivered = rt.deliveredReplyOf?.(botId, sender, workId) + if (delivered !== undefined) return { answered: true, text: delivered.text } + if (rt.deliveredReplyOf === undefined && rt.wasReplyDelivered?.(botId, sender, workId) === true) return { answered: true } + if (where === undefined) return { answered: false } const sent = await logOf(where) - if (sent !== undefined && answeredIn(sent, workId)) return true - if (back === undefined) { - const segments = threadOfPart(rt.botOf(sender), where)?.segments ?? [] - for (const segment of [...segments].reverse()) { - if (segment?.sessionId === undefined || segment.sessionId === where) continue - const earlier = await logOf(segment.sessionId) - if (earlier !== undefined && answeredIn(earlier, workId)) return true - } + if (sent !== undefined && answeredIn(sent, workId)) return { answered: true } + const segments = back === undefined + ? (threadOfPart(rt.botOf(sender), where)?.segments ?? []) + : (rt.roomOf(roomId)?.segments?.[sender] ?? []) + for (const segment of [...segments].reverse()) { + if (segment?.sessionId === undefined || segment.sessionId === where) continue + const earlier = await logOf(segment.sessionId) + if (earlier !== undefined && answeredIn(earlier, workId)) return { answered: true } } - return false + return { answered: false } } // A turn that ended while DS Bot was not watching it: its answer may exist and never // have gone to the Bot that waits for it. The Session's durable log says what it did; // the work it was for is settled once, the way a finished turn settles it. - async function reconcileEnded(sessionId, record, ended) { + async function reconcileEnded(sessionId, record, ended, prefetchedEvents) { const trigger = record.trigger if (trigger === undefined) return const workId = rt.workOf(trigger) - if (!rt.ownsWork(record.botId, workId)) return + const ownsPrimary = rt.ownsWork(record.botId, workId) + const hasOwnedWork = ownsPrimary || Object.values(rt.state?.works ?? {}).some(work => work.to === record.botId) + if (!hasOwnedWork) return const exchangeId = trigger.exchangeId - const asker = rt.workAsker(workId) - const sender = asker?.senderSessionId - if (exchangeId === undefined || sender === undefined || rt.botOf(sender) === undefined) return - const name = rt.botOf(rt.selfOf(sessionId))?.name ?? 'The other Bot' + if (exchangeId === undefined) return + const primaryAsker = ownsPrimary ? rt.workAsker(workId) : undefined + const primarySender = primaryAsker?.senderSessionId + const canSettlePrimary = ownsPrimary && primarySender !== undefined && rt.botOf(primarySender) !== undefined + const name = rt.botOf(rt.selfOf(sessionId))?.name ?? rt.botOf(record.botId)?.name ?? 'The other Bot' // DSH stopped that turn as it shut down: it is picked up at the next start. if (ended.reason?.kind === 'aborted' && ended.reason.reason?.kind === 'disposed') { update(sessionId, () => record) return } - const events = await logOf(sessionId) + const events = prefetchedEvents ?? await logOf(sessionId) if (events === undefined) { - if (rt.wasReplyDelivered?.(record.botId, sender, workId) === true) { - rt.closeWork?.(workId, name, rt.answerText(workId, '', name)) + if (!canSettlePrimary) return + const delivered = rt.deliveredReplyOf?.(record.botId, primarySender, workId) + if (delivered !== undefined || (rt.deliveredReplyOf === undefined && rt.wasReplyDelivered?.(record.botId, primarySender, workId) === true)) { + rt.closeWork?.(workId, name, delivered?.text || rt.answerText(workId, '', name)) return } // The record of that work cannot be read: whether it answered cannot be @@ -366,7 +384,7 @@ export function install(rt) { where: room === undefined ? 'own chat' : `group "${room.name}"`, sessionId, }) - answerSender(rt, sessionId, asker, text) + answerSender(rt, sessionId, primaryAsker, text) return } const turn = turnOf(events, exchangeId) @@ -374,15 +392,16 @@ export function install(rt) { // waiting for it keeps what it had. if (turn === undefined) return const said = turn.texts.join('\n\n') - // The answer went up in another run (even before a part switch): the work closes - // with what it said, and nothing is sent twice. - const back = rt.roomOf(asker.senderRoomId)?.sessions?.[sender] - // The answer would have gone to the part of the asking Bot's chat the request came - // from, which is not its foreground chat when that part is a background one. - const where = back ?? (asker.senderPart === undefined ? undefined : threadCurrentOfPart(rt.botOf(sender), asker.senderPart)) ?? rt.chatOf(sender) - if (await alreadyAnswered(record.botId, sender, where, back, workId)) { - rt.closeWork?.(workId, name, rt.answerText(workId, said, name)) - return + let primarySettled = !canSettlePrimary + if (canSettlePrimary) { + // The answer went up in another run (even before a part switch): the work closes + // with what it said, and nothing is sent twice. + const { sender, back, where } = whereFor(primaryAsker) + const answered = await alreadyAnswered(record.botId, sender, where, back, primaryAsker.senderRoomId, workId) + if (answered.answered) { + rt.closeWork?.(workId, name, answered.text || rt.answerText(workId, said, name)) + primarySettled = true + } } const reason = turn.reason // A turn that went on to a fresh part goes on there; it is not settled here. @@ -390,20 +409,48 @@ export function install(rt) { // The turn was cut off before it finished: the work is picked up again, as after // a restart that cut it off. if (reason === undefined || reason.kind === 'interrupted') { - await resumeTurn(sessionId, record) + if (!primarySettled) await resumeTurn(sessionId, record) return } - if (reason.kind === 'error') { - const failure = turnFailure(reason.error) - answerSender(rt, sessionId, asker, `${name} could not answer your message: its model call failed (${failureLine(failure)}). Tell the user if it matters; do not retry right away.`, undefined, { failure }) - return + const failure = reason.kind === 'error' ? turnFailure(reason.error) : undefined + const dedupKey = failure !== undefined + ? `reconcile:${sessionId}:${turn.turn ?? ''}:error:${failure.requestId ?? failure.code}` + : reason.kind === 'aborted' + ? `reconcile:${sessionId}:${turn.turn ?? ''}:aborted` + : undefined + const settleFailureOrAbort = (asker) => { + if (failure !== undefined) { + answerSender(rt, sessionId, asker, `${name} could not answer your message: its model call failed (${failureLine(failure)}). Tell the user if it matters; do not retry right away.`, undefined, { failure, dedupKey }) + } else { + // The turn was stopped before it could answer: the sender is told, as it is told + // when DS Bot watches the turn end. + answerSender(rt, sessionId, asker, `${name} was stopped before it could answer your message.`, undefined, { dedupKey }) + } } - if (reason.kind === 'aborted') { - // The turn was stopped before it could answer: the sender is told, as it is told - // when DS Bot watches the turn end. - answerSender(rt, sessionId, asker, `${name} was stopped before it could answer your message.`) + if (reason.kind === 'error' || reason.kind === 'aborted') { + if (!primarySettled) settleFailureOrAbort(primaryAsker) + // Steered requests that joined the same failed or aborted turn also owe their own + // callers settlement, so distinct parent works cannot stall across a restart. + const seen = new Set([workId, exchangeId]) + for (const took of turn.took) { + if (took?.kind !== SOURCE_KIND || (took.role !== 'request' && took.role !== 'stopped') || took.audience === 'user') continue + const joinedId = took.exchangeId + if (typeof joinedId !== 'string' || joinedId === '' || seen.has(joinedId)) continue + seen.add(joinedId) + if (!rt.ownsWork(record.botId, joinedId)) continue + const joinedAsker = rt.workAsker(joinedId) + const { sender, back, where } = whereFor(joinedAsker) + if (sender === undefined || rt.botOf(sender) === undefined) continue + const answered = await alreadyAnswered(record.botId, sender, where, back, joinedAsker.senderRoomId, joinedId) + if (answered.answered) { + rt.closeWork?.(joinedId, name, answered.text || rt.answerText(joinedId, '', name)) + continue + } + settleFailureOrAbort(joinedAsker) + } return } + if (primarySettled) return // A turn cut off at the output limit is not a finish: the answer says so, and the // diagnostics record it, with the route the Session's log holds for it. const cutOff = reason?.kind === 'max-tokens' @@ -435,7 +482,40 @@ export function install(rt) { } const text = rt.answerText(workId, answer, name) const long = text.length > MESSAGE_INLINE_MAX ? await rt.attachLong(text, `Answer from ${name}.txt`) : undefined - answerSender(rt, sessionId, asker, text, long) + answerSender(rt, sessionId, primaryAsker, text, long) + } + + // Checks whether the work or reply-expecting trigger a stale inflight turn names has + // already been answered in the pair ledger or the asker's session log. For a + // continuation turn (`role: 'reply' | 'report' | 'resume'`), the owed answer is the + // upstream work and its asker, never the child reply that triggered the turn. + async function settleDeliveredTrigger(record, trigger) { + const workId = rt.workOf(trigger) + if (workId !== undefined) { + const owns = rt.ownsWork(record.botId, workId) + const asker = owns ? rt.workAsker(workId) : undefined + const sender = asker?.senderSessionId + ?? (trigger.role === 'request' && trigger.audience !== 'user' ? trigger.senderSessionId : undefined) + const name = rt.botOf(record.botId)?.name ?? 'The other Bot' + const delivered = rt.deliveredReplyOf?.(record.botId, sender, workId) + if (delivered !== undefined || (rt.deliveredReplyOf === undefined && sender !== undefined && rt.wasReplyDelivered?.(record.botId, sender, workId) === true)) { + if (owns) rt.closeWork?.(workId, name, delivered?.text || rt.answerText(workId, '', name)) + return true + } + if (owns && asker !== undefined && sender !== undefined && rt.botOf(sender) !== undefined) { + const { back, where } = whereFor(asker) + const answered = await alreadyAnswered(record.botId, sender, where, back, asker.senderRoomId, workId) + if (answered.answered) { + rt.closeWork?.(workId, name, answered.text || rt.answerText(workId, '', name)) + return true + } + } + return false + } + if (trigger.role === 'stopped' && trigger.exchangeId !== undefined && trigger.senderSessionId !== undefined) { + return rt.wasReplyDelivered?.(record.botId, trigger.senderSessionId, trigger.exchangeId) === true + } + return false } async function resumeAll() { @@ -474,22 +554,17 @@ export function install(rt) { continue } // If inflight.json still held a turn whose durable log already shows a finished - // turn/end, settle from the log instead of re-running the request via resumeTurn. - if (first === undefined && record.roomId === undefined && record.trigger?.exchangeId !== undefined) { - const events = await logOf(sessionId) - const turn = events === undefined ? undefined : turnOf(events, record.trigger.exchangeId) + // turn/end, or whose owed reply is already recorded in the pair ledger, settle it + // instead of re-running via resumeTurn. + if (first === undefined && record.roomId === undefined && record.trigger !== undefined) { + const trigger = record.trigger + const events = trigger.exchangeId !== undefined ? await logOf(sessionId) : undefined + const turn = events === undefined ? undefined : turnOf(events, trigger.exchangeId) if (isTerminalReason(turn?.reason)) { - await reconcileEnded(sessionId, record, { reason: turn.reason }).catch(error => rt.warn('ds-bot: settling the work of %s failed: %s', sessionId, error)) - continue - } - if (turn === undefined && record.trigger.senderSessionId !== undefined - && rt.wasReplyDelivered?.(record.botId, record.trigger.senderSessionId, record.trigger.exchangeId) === true) { - const workId = rt.workOf(record.trigger) - if (rt.ownsWork(record.botId, workId)) { - rt.closeWork?.(workId, rt.botOf(record.botId)?.name ?? 'The other Bot', '') - } + await reconcileEnded(sessionId, record, { reason: turn.reason }, events).catch(error => rt.warn('ds-bot: settling the work of %s failed: %s', sessionId, error)) continue } + if (await settleDeliveredTrigger(record, trigger)) continue } cut.push([sessionId, record]) } diff --git a/ds-bot/src/host/turns.js b/ds-bot/src/host/turns.js index 81af6e7a79..c8a5ac64c1 100644 --- a/ds-bot/src/host/turns.js +++ b/ds-bot/src/host/turns.js @@ -73,10 +73,27 @@ const removeItem = (list, item) => { if (at !== -1) list.splice(at, 1) } -function settleSuppressedWork(rt, exchangeId, name, text, primaryWork) { +// The live parent work a child work hangs under, matching closeWork's lookup. +const liveParentOf = (rt, exchangeId) => { + if (typeof exchangeId !== 'string' || exchangeId === '') return undefined + const works = rt.state?.works + const parent = works?.[exchangeId]?.parent + return parent !== undefined && parent !== exchangeId && works?.[parent] !== undefined ? parent : undefined +} + +// Coalescing a second child's failure into an earlier delivery for the same parent work +// is safe only while that earlier delivery is still pending in parent.open; once the +// caller has already consumed it and parked the parent, suppressing this child would +// leave the parent idle with no continuation. +const parentCoveredBy = (rt, parentWork, primaryExchangeId) => { + if (parentWork === undefined) return true + if (typeof primaryExchangeId !== 'string' || primaryExchangeId === '') return false + return rt.state?.works?.[parentWork]?.open?.includes(primaryExchangeId) === true +} + +function dropSuppressedWork(rt, exchangeId, name, text) { if (typeof exchangeId !== 'string' || exchangeId === '') return - const ownParent = rt.state?.works?.[exchangeId]?.parent - if (ownParent !== undefined && ownParent === primaryWork && typeof rt.dropWork === 'function') { + if (typeof rt.dropWork === 'function') { rt.dropWork(exchangeId) return } @@ -103,24 +120,34 @@ export function answerSender(rt, id, source, text, long, options = {}) { const exchangeKey = typeof source.exchangeId === 'string' && source.exchangeId !== '' ? `${fromId ?? ''}\0${toId ?? ''}\0${source.exchangeId}` : undefined - if ((exchangeKey !== undefined && guards.exchanges.has(exchangeKey)) - || rt.wasReplyDelivered?.(fromId, toId, source.exchangeId) === true) { - settleSuppressedWork(rt, source.exchangeId, name, text, undefined) + const deliveredReply = rt.deliveredReplyOf?.(fromId, toId, source.exchangeId) + const alreadySent = deliveredReply !== undefined + || (rt.deliveredReplyOf === undefined && rt.wasReplyDelivered?.(fromId, toId, source.exchangeId) === true) + if ((exchangeKey !== undefined && guards.exchanges.has(exchangeKey)) || alreadySent) { + if (typeof source.exchangeId === 'string' && source.exchangeId !== '') { + rt.closeWork?.(source.exchangeId, name, deliveredReply?.text || text) + } return false } const failureRequestId = typeof options?.failure?.requestId === 'string' && options.failure.requestId !== '' ? options.failure.requestId : undefined const dedupKey = typeof options?.dedupKey === 'string' && options.dedupKey !== '' ? options.dedupKey : undefined - const matchSameTarget = entry => entry.fromId === fromId && entry.toId === toId + const ownParent = liveParentOf(rt, source.exchangeId) + const matchSameScope = entry => entry.fromId === fromId && entry.toId === toId + && entry.work === ownParent && sameConversation(rt.state, toId, entry.targetSessionId, targetSessionId) + && parentCoveredBy(rt, ownParent, entry.exchangeId) const matchedReq = failureRequestId === undefined ? undefined - : guards.failureRequests.find(entry => entry.requestId === failureRequestId && matchSameTarget(entry)) + : guards.failureRequests.findLast(entry => entry.requestId === failureRequestId && matchSameScope(entry)) const matchedTurn = dedupKey === undefined ? undefined - : guards.turnFailures.find(entry => entry.dedupKey === dedupKey && matchSameTarget(entry)) - if (matchedReq !== undefined || matchedTurn !== undefined - || (failureRequestId !== undefined && rt.wasFailureRequestDelivered?.(fromId, toId, failureRequestId, targetSessionId) === true)) { - settleSuppressedWork(rt, source.exchangeId, name, text, (matchedReq ?? matchedTurn)?.work) + : guards.turnFailures.findLast(entry => entry.dedupKey === dedupKey && matchSameScope(entry)) + const matchedInMemory = matchedReq !== undefined || matchedTurn !== undefined + const matchedLedger = matchedInMemory || failureRequestId === undefined ? undefined + : (rt.deliveredFailuresOf?.(fromId, toId, failureRequestId, targetSessionId, ownParent) ?? []) + .findLast(entry => parentCoveredBy(rt, ownParent, entry.replyTo)) + if (matchedInMemory || matchedLedger !== undefined) { + dropSuppressedWork(rt, source.exchangeId, name, text) return false } const work = rt.closeWork?.(source.exchangeId, name, text) @@ -129,10 +156,10 @@ export function answerSender(rt, id, source, text, long, options = {}) { if (guards.exchanges.size > MAX_REPLY_GUARDS) guards.exchanges.delete(guards.exchanges.values().next().value) } const failureEntry = failureRequestId === undefined ? undefined - : { fromId, toId, targetSessionId, requestId: failureRequestId, work } + : { fromId, toId, targetSessionId, requestId: failureRequestId, work, exchangeId: source.exchangeId } if (failureEntry !== undefined) pushLimited(guards.failureRequests, failureEntry) const turnEntry = dedupKey === undefined ? undefined - : { fromId, toId, targetSessionId, dedupKey, work } + : { fromId, toId, targetSessionId, dedupKey, work, exchangeId: source.exchangeId } if (turnEntry !== undefined) pushLimited(guards.turnFailures, turnEntry) // A reply that arrives after the user gave up the wait is a late answer: it closes // nothing, opens nothing, and says on it that the wait was already over. diff --git a/ds-bot/test/delivery-dedup.test.js b/ds-bot/test/delivery-dedup.test.js index 4f58079b16..f1a5eb6186 100644 --- a/ds-bot/test/delivery-dedup.test.js +++ b/ds-bot/test/delivery-dedup.test.js @@ -404,3 +404,161 @@ test('resumeAll settles already-ended turns from disk without re-running or dupl assert.equal(second.agentOf('a').calls.length, 0, 'already-delivered reply must not be sent again') await second.stop() }) + +test('answerSender keeps separate continuations for distinct parent works sharing a failed turn', async (t) => { + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-parents-')) + const seed = baseSeed() + seed.bots.c = { id: 'c', name: 'Lead', role: 'Lead', sessionId: 'c', createdAt: 3 } + seed.works = { + 'ex-no-parent': { from: 'a', to: 'b', open: [], results: [], at: 1 }, + 'parent-1': { from: 'c', to: 'a', open: ['child-p1'], results: [], at: 1 }, + 'child-p1': { from: 'a', to: 'b', parent: 'parent-1', open: [], results: [], at: 2 }, + 'parent-2': { from: 'c', to: 'a', open: ['child-p2'], results: [], at: 1 }, + 'child-p2': { from: 'a', to: 'b', parent: 'parent-2', open: [], results: [], at: 3 }, + 'parent-consumed': { from: 'c', to: 'a', open: ['child-c1', 'child-c2'], results: [], at: 1 }, + 'child-c1': { from: 'a', to: 'b', parent: 'parent-consumed', open: [], results: [], at: 2 }, + 'child-c2': { from: 'a', to: 'b', parent: 'parent-consumed', open: [], results: [], at: 3 }, + } + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + const host = await createRestartHost(t, home, new Map()) + t.after(async () => { + await host.stop() + rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }) + }) + + const { rt } = host + const failure = { code: 'INVALID_REQUEST', status: 400, requestId: 'req-shared-parents', message: 'bad' } + const dedupKey = 'turn:b:1:error:req-shared-parents' + + // Unparented work followed by parent-1 and parent-2 in the same failed turn: each distinct + // parent work must receive its own continuation carrying `work`. + assert.equal(answerSender(rt, 'b', { kind: 'bot', role: 'request', exchangeId: 'ex-no-parent', senderSessionId: 'a' }, 'Writer failed', undefined, { failure, dedupKey }), true) + assert.equal(answerSender(rt, 'b', { kind: 'bot', role: 'request', exchangeId: 'child-p1', senderSessionId: 'a' }, 'Writer failed', undefined, { failure, dedupKey }), true) + assert.equal(answerSender(rt, 'b', { kind: 'bot', role: 'request', exchangeId: 'child-p2', senderSessionId: 'a' }, 'Writer failed', undefined, { failure, dedupKey }), true) + await settle(() => host.agentOf('a').calls.length >= 3) + assert.deepEqual( + host.agentOf('a').calls.map(msg => [msg.source.replyTo, msg.source.work]), + [['ex-no-parent', undefined], ['child-p1', 'parent-1'], ['child-p2', 'parent-2']], + 'distinct parent works must never be suppressed by another parent or unparented failure', + ) + + // Same parent work, but once the first failed child was already consumed out of parent.open, + // a second child's failure must still deliver so the parked parent is woken up. + const sameParentFailure = { code: 'INVALID_REQUEST', status: 400, requestId: 'req-consumed-parent', message: 'bad' } + assert.equal(answerSender(rt, 'b', { kind: 'bot', role: 'request', exchangeId: 'child-c1', senderSessionId: 'a' }, 'Writer failed', undefined, { failure: sameParentFailure }), true) + await settle(() => host.agentOf('a').calls.length >= 4) + rt.takenWork('parent-consumed', 'child-c1') + assert.equal(answerSender(rt, 'b', { kind: 'bot', role: 'request', exchangeId: 'child-c2', senderSessionId: 'a' }, 'Writer failed', undefined, { failure: sameParentFailure }), true) + await settle(() => host.agentOf('a').calls.length >= 5) + assert.deepEqual( + [host.agentOf('a').calls[4].source.replyTo, host.agentOf('a').calls[4].source.work], + ['child-c2', 'parent-consumed'], + ) +}) + +test('resumeAll checks the upstream work and asker for role=reply continuation turns', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }) + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-reply-resume-')) + t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 })) + + const seed = baseSeed() + seed.bots.c = { id: 'c', name: 'Lead', role: 'Lead', sessionId: 'c', createdAt: 3 } + seed.works = { + 'up-work': { from: 'c', to: 'a', open: [], results: [{ id: 'sub-req', from: 'Writer', text: 'sub done' }], at: 1 }, + } + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + const logs = new Map([['a', []], ['b', []], ['c', []]]) + + // First boot: Writer already delivered the child reply (`sub-reply` -> `sub-req`) to Chief, + // and Chief was interrupted mid-continuation (`role: 'reply'`) before answering Lead (`up-work`). + // resumeAll must NOT mistake the child reply for an already-delivered upstream answer. + const seedHost = await createRestartHost(t, home, logs) + await seedHost.rt.deliver({ + fromId: 'b', toId: 'a', sessionId: 'a', text: 'sub done', + role: 'reply', replyTo: 'sub-req', exchangeId: 'sub-reply', work: 'up-work', + }) + await seedHost.stop() + + writeFileSync(join(home, 'inflight.json'), JSON.stringify({ + turns: { + a: { + botId: 'a', + since: 1, + trigger: { role: 'reply', exchangeId: 'sub-reply', work: 'up-work', senderSessionId: 'b' }, + }, + }, + queued: {}, + })) + const first = await createRestartHost(t, home, logs) + t.mock.timers.tick(3000) + await settle(() => first.agentOf('a').calls.length >= 1) + assert.equal(first.agentOf('a').calls[0].source.role, 'resume') + assert.equal(first.agentOf('a').calls[0].source.work, 'up-work', 'unanswered upstream work must resume') + + // Now Chief delivers the upstream answer to Lead (`up-work`), then crashes before inflight clears. + await first.rt.deliver({ + fromId: 'a', toId: 'c', sessionId: 'c', text: 'upstream final answer', + role: 'reply', replyTo: 'up-work', exchangeId: 'up-reply', + }) + await first.stop() + + writeFileSync(join(home, 'inflight.json'), JSON.stringify({ + turns: { + a: { + botId: 'a', + since: 2, + trigger: { role: 'reply', exchangeId: 'sub-reply', work: 'up-work', senderSessionId: 'b' }, + }, + }, + queued: {}, + })) + const second = await createRestartHost(t, home, logs) + t.mock.timers.tick(3000) + await new Promise(resolve => setImmediate(resolve)) + await realDelay(30) + assert.equal(second.agentOf('a').calls.length, 0, 'already-answered upstream work must not resume again') + assert.equal(second.agentOf('c').calls.length, 0, 'already-delivered upstream reply must not duplicate') + assert.equal(second.rt.state.works['up-work'], undefined, 'upstream work is closed once confirmed delivered') + await second.stop() +}) + +test('reconcileEnded settles distinct parent works that steered into the same failed turn', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }) + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-reconcile-steered-')) + t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 })) + + const seed = baseSeed() + seed.bots.c = { id: 'c', name: 'Lead', role: 'Lead', sessionId: 'c', createdAt: 3 } + seed.works = { + 'p1': { from: 'c', to: 'a', open: ['req-1'], results: [], at: 1 }, + 'p2': { from: 'c', to: 'a', open: ['req-2'], results: [], at: 1 }, + 'req-1': { from: 'a', to: 'b', parent: 'p1', open: [], results: [], at: 2 }, + 'req-2': { from: 'a', to: 'b', parent: 'p2', open: [], results: [], at: 3 }, + } + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + const req1 = { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'req-1', senderName: 'Chief', senderSessionId: 'a' } + const req2 = { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'req-2', senderName: 'Chief', senderSessionId: 'a' } + const logs = new Map([ + ['a', []], + ['b', [ + { type: 'turn/start', seq: 1, data: { turn: 1 } }, + { type: 'user/message', seq: 2, data: { source: req1, content: [{ type: 'text', text: 'first' }] } }, + { type: 'user/message', seq: 3, data: { source: req2, content: [{ type: 'text', text: 'second' }] } }, + { type: 'turn/end', seq: 4, data: { turn: 1, reason: { kind: 'error', error: { code: 'INVALID_REQUEST', status: 400, requestId: 'req-rec-1', message: 'bad' } } } }, + ]], + ]) + writeFileSync(join(home, 'inflight.json'), JSON.stringify({ + turns: { b: { botId: 'b', since: 1, trigger: { role: 'request', exchangeId: 'req-1', senderSessionId: 'a' } } }, + queued: {}, + })) + + const host = await createRestartHost(t, home, logs) + t.mock.timers.tick(3000) + await settle(() => host.agentOf('a').calls.length >= 2) + assert.deepEqual( + host.agentOf('a').calls.map(msg => [msg.source.replyTo, msg.source.work]), + [['req-1', 'p1'], ['req-2', 'p2']], + 'reconcileEnded must notify every distinct parent work steered into the failed turn', + ) + await host.stop() +}) From cbc9e8bbe67a45ad89ca6e1be4e2505aaf004d14 Mon Sep 17 00:00:00 2001 From: lulu feizhu <279467036+Fei2411@users.noreply.github.com> Date: Fri, 20 Nov 2026 18:54:09 -0800 Subject: [PATCH 3/4] Keep delivery warnings with recovered long replies Recovery reads the original attachment, but delivery may add a warning only to its preview. Restore that prefix without duplicating the preview. --- ds-bot/src/host/resume.js | 6 +++++- ds-bot/test/delivery-dedup.test.js | 15 +++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/ds-bot/src/host/resume.js b/ds-bot/src/host/resume.js index 2c01d075cc..7655e1a933 100644 --- a/ds-bot/src/host/resume.js +++ b/ds-bot/src/host/resume.js @@ -339,7 +339,11 @@ export function install(rt) { } const file = delivered.files?.[0] const stored = file === undefined ? undefined : await rt.fileText(file, delivered.length) - return { answered: true, text: stored?.whole === true ? stored.text : delivered.text } + if (stored?.whole !== true) return { answered: true, text: delivered.text } + // Delivery warnings are prepended to the preview and included in its declared length. + const prefixLength = (delivered.length ?? stored.text.length) - stored.text.length + const prefix = prefixLength > 0 ? delivered.text.slice(0, prefixLength) : '' + return { answered: true, text: prefix + stored.text } } async function alreadyAnswered(botId, sender, where, back, roomId, workId) { diff --git a/ds-bot/test/delivery-dedup.test.js b/ds-bot/test/delivery-dedup.test.js index a5a3a6b6f3..0b9f883269 100644 --- a/ds-bot/test/delivery-dedup.test.js +++ b/ds-bot/test/delivery-dedup.test.js @@ -407,7 +407,7 @@ test('resumeAll settles already-ended turns from disk without re-running or dupl await second.stop() }) -test('restart keeps the full attachment-backed reply in the parent work without re-delivery', async (t) => { +async function checkAttachmentRecovery(t, prefix) { t.mock.timers.enable({ apis: ['setTimeout'] }) const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-attachment-')) t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 })) @@ -422,7 +422,10 @@ test('restart keeps the full attachment-backed reply in the parent work without const first = await createRestartHost(t, home, logs, services) const full = '结论\n'.repeat(70_000) + 'End of report.' const attachment = await first.rt.attachLong(full, 'Answer from Writer.txt') - await first.rt.deliver({ fromId: 'b', toId: 'a', ...attachment, role: 'reply', replyTo: 'child' }) + await first.rt.deliver({ + fromId: 'b', toId: 'a', ...attachment, role: 'reply', replyTo: 'child', + text: prefix + attachment.text, length: prefix.length + attachment.length, + }) first.emit('a', 'user/message', { source: first.agentOf('a').calls[0].source }) await first.stop() @@ -439,11 +442,15 @@ test('restart keeps the full attachment-backed reply in the parent work without const restarted = await createRestartHost(t, home, logs, services) t.mock.timers.tick(3000) await settle(() => restarted.rt.state.works.child === undefined) - assert.ok(restarted.rt.answerText('parent', '', 'Chief').endsWith(full), 'parent keeps the entire reply, not its preview or a dialog-sized prefix') + assert.ok(restarted.rt.answerText('parent', '', 'Chief').endsWith(prefix + full), 'parent keeps the entire reply, not its preview or a dialog-sized prefix') assert.equal(restarted.agentOf('a').calls.length, 0) assert.equal(restarted.agentOf('b').calls.length, 0) await restarted.stop() -}) +} + +test('restart keeps the full attachment-backed reply in the parent work without re-delivery', t => checkAttachmentRecovery(t, '')) + +test('restart keeps the fallback notice with the full attachment-backed reply', t => checkAttachmentRecovery(t, 'Writer could not send its reply to you (offline); what it had answered:\n')) test('restart settles a completed peer handoff task from disk and replies to the asking Bot', async (t) => { t.mock.timers.enable({ apis: ['setTimeout'] }) From 61b5fbe3f6f13aba4533087cb2d98d3018324a71 Mon Sep 17 00:00:00 2001 From: lulu feizhu <279467036+Fei2411@users.noreply.github.com> Date: Fri, 20 Nov 2026 18:54:09 -0800 Subject: [PATCH 4/4] Keep recovered tasks active while child replies are queued Do not finalize a recovered task until its queued input can reach that task rather than the foreground chat. --- ds-bot/src/host/bot-tasks.js | 10 +++++-- ds-bot/test/bot-tasks.test.js | 5 +++- ds-bot/test/delivery-dedup.test.js | 43 ++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/ds-bot/src/host/bot-tasks.js b/ds-bot/src/host/bot-tasks.js index ff427e8d90..59fc062359 100644 --- a/ds-bot/src/host/bot-tasks.js +++ b/ds-bot/src/host/bot-tasks.js @@ -527,6 +527,8 @@ export function install(rt) { } } + const hasInboxInput = agent => Boolean(agent?.inbox?.nextStep?.length || agent?.inbox?.nextTurn?.length) + // Settles a task whose turn ended while the plugin was reloading (durable log). async function reconcileEndedTask(sessionId, ended) { if (rt.readOnly?.() !== undefined) return false @@ -550,6 +552,11 @@ export function install(rt) { .join('\n\n') .trim() const failure = reason?.kind === 'error' ? turnFailure(reason.error) : undefined + if (reason?.kind !== 'error' && reason?.kind !== 'aborted') { + const agent = await rt.resolveAgent(sessionId).catch(() => undefined) + // A child work can already be closed while its reply still waits for replay. + if (hasInboxInput(agent) || rt.unreadOf?.(sessionId)?.length > 0) return false + } await handleTurnEnd(bot, task, reason, failure, said) return true } @@ -571,8 +578,7 @@ export function install(rt) { if ((rt.runningJobs?.(sid)?.length ?? 0) > 0) continue if ((rt.unreadOf?.(sid)?.length ?? 0) > 0) continue const agent = await rt.resolveAgent(sid).catch(() => undefined) - const pendingInbox = (agent?.inbox?.nextStep?.length ?? 0) + (agent?.inbox?.nextTurn?.length ?? 0) - if (agent?.status === 'running' || pendingInbox > 0) continue + if (agent?.status === 'running' || hasInboxInput(agent)) continue if (hasOpenOutgoingWorks(bot, task)) { if (task.status !== 'waiting_work') { task.status = 'waiting_work' diff --git a/ds-bot/test/bot-tasks.test.js b/ds-bot/test/bot-tasks.test.js index 957b07ec3e..3bdf97520c 100644 --- a/ds-bot/test/bot-tasks.test.js +++ b/ds-bot/test/bot-tasks.test.js @@ -511,7 +511,10 @@ test('reconcileTasksAfterRestart marks orphaned idle tasks interrupted while pre assert.match(interruptReport.text, /restarted/) // reconcileEndedTask settles a task whose turn ended during plugin reload using durable log. - env.agentMap.get(tInflight.sessionId).session.events.push( + const inflightAgent = env.agentMap.get(tInflight.sessionId) + const firstMsg = inflightAgent.followups[0] + await env.emit(tInflight.sessionId, 'user/message', { source: firstMsg.source, content: firstMsg.content }) + inflightAgent.session.events.push( { type: 'turn/start', seq: 10, data: { turn: 1 } }, { type: 'assistant/message', seq: 11, data: { message: { role: 'assistant', content: [{ type: 'text', text: 'Reload finished work.' }] } } }, ) diff --git a/ds-bot/test/delivery-dedup.test.js b/ds-bot/test/delivery-dedup.test.js index 0b9f883269..2a6992f215 100644 --- a/ds-bot/test/delivery-dedup.test.js +++ b/ds-bot/test/delivery-dedup.test.js @@ -481,6 +481,49 @@ test('restart settles a completed peer handoff task from disk and replies to the await host.stop() }) +async function checkUnreadTaskReply(t, inInbox) { + t.mock.timers.enable({ apis: ['setTimeout'] }) + const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-task-queued-')) + const seed = baseSeed() + seed.bots.c = { id: 'c', name: 'Reviewer', role: 'Reviewer', sessionId: 'c', createdAt: 3 } + seed.bots.b.tasks = [{ id: 'task-1', sessionId: 'b-task', title: 'Peer request', origin: 'bot-handoff', status: 'waiting_work', exchangeId: 'request' }] + seed.works = { + request: { from: 'a', to: 'b', open: ['review'], results: [{ id: 'review', name: 'Reviewer', text: 'Review result.' }], at: 1 }, + } + writeFileSync(join(home, 'state.json'), JSON.stringify(seed)) + const request = { kind: 'bot', role: 'request', exchangeId: 'request', senderSessionId: 'a' } + const reply = { fromId: 'c', toId: 'b', sessionId: 'b-task', role: 'reply', replyTo: 'review', work: 'request', exchangeId: 'review-reply', text: 'Review result.' } + const logs = new Map([['b-task', [ + { type: 'turn/start', data: { turn: 1 } }, + { type: 'user/message', data: { source: request } }, + { type: 'assistant/message', data: { message: { content: [{ type: 'text', text: 'Waiting for review.' }] } } }, + { type: 'turn/end', data: { turn: 1, reason: { kind: 'completed' } } }, + ]]]) + writeFileSync(join(home, 'inflight.json'), JSON.stringify({ + turns: { 'b-task': { botId: 'b', since: 1, trigger: request } }, + queued: inInbox ? {} : { 'b-task': [{ at: 2, args: reply }] }, + })) + const host = await createRestartHost(t, home, logs) + t.after(async () => { + await host.stop() + rmSync(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }) + }) + if (inInbox) host.agentOf('b-task').inbox.nextTurn.push({ source: { kind: 'bot', exchangeId: reply.exchangeId } }) + const reconciliation = t.mock.method(host.rt, 'reconcileTasksAfterRestart') + t.mock.timers.tick(3000) + await settle(() => reconciliation.mock.callCount() > 0) + await reconciliation.mock.calls[0].result + assert.deepEqual(host.agentOf('b-task').calls.map(({ source }) => source.exchangeId), inInbox ? [] : ['review-reply']) + assert.equal(host.agentOf('b').calls.length, 0) + assert.equal(host.agentOf('a').calls.length, 0) + assert.equal(host.rt.state.bots.b.tasks[0].status, 'waiting_work') + assert.deepEqual(host.rt.state.works.request.open, ['review']) +} + +test('restart replays an unread child reply in its background task without rerunning the earlier turn', t => checkUnreadTaskReply(t, false)) + +test('restart keeps a task active when a child reply is still in the agent inbox', t => checkUnreadTaskReply(t, true)) + test('answerSender keeps separate continuations for distinct parent works sharing a failed turn', async (t) => { const home = mkdtempSync(join(tmpdir(), 'ds-bot-dedup-parents-')) const seed = baseSeed()