From 0169a2049696a986ee0f1f1c119ece1c6c900032 Mon Sep 17 00:00:00 2001 From: antra-tess Date: Fri, 18 Sep 2026 00:58:13 -0700 Subject: [PATCH] feat(guard): add durable tool result withholding and recovery Preserve full original output in Chronicle while keeping pending results out of speculative compression. On a provider refusal, withhold the latest batch and retry inference once without repeating tool execution. Co-Authored-By: Codex (GPT-6) --- README.md | 5 + changelog.d/tool-result-guard.added.md | 6 + docs/tool-result-guard.md | 81 ++++++ src/agent.ts | 53 +++- src/framework.ts | 157 ++++++++-- src/tool-result-guard.ts | 132 +++++++++ src/types/agent.ts | 6 + test/tool-result-guard.test.ts | 384 +++++++++++++++++++++++++ 8 files changed, 800 insertions(+), 24 deletions(-) create mode 100644 changelog.d/tool-result-guard.added.md create mode 100644 docs/tool-result-guard.md create mode 100644 src/tool-result-guard.ts create mode 100644 test/tool-result-guard.test.ts diff --git a/README.md b/README.md index 92fe15c..853793e 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,11 @@ await server.start(); An agent wraps an LLM identity: model, system prompt, context strategy, and tool permissions. Multiple agents can coexist, each with independent context and inference state. +Agents can opt into the [tool result guard](docs/tool-result-guard.md) through +`agent_settings` with `{"action":"update","tool_result_guard":true}`. The +setting persists across restarts; withheld results remain recoverable in +Chronicle's audit history. + ```typescript { name: 'researcher', diff --git a/changelog.d/tool-result-guard.added.md b/changelog.d/tool-result-guard.added.md new file mode 100644 index 0000000..5bee684 --- /dev/null +++ b/changelog.d/tool-result-guard.added.md @@ -0,0 +1,6 @@ +- Add the opt-in, durable `agent_settings.tool_result_guard` setting (recipe: + `toolResultGuard`). On a provider refusal after tool output, withhold the + latest result batch and retry inference once without rerunning tools or + automatically rewinding older messages. Full originals remain in an + append-only Chronicle audit log; pending output stays out of speculative + compression, and disabling the guard does not restore withheld results. diff --git a/docs/tool-result-guard.md b/docs/tool-result-guard.md new file mode 100644 index 0000000..007df57 --- /dev/null +++ b/docs/tool-result-guard.md @@ -0,0 +1,81 @@ +# Tool result guard + +The tool result guard is off by default. An agent can enable it through +`agent_settings`: + +```json +{"action":"update","tool_result_guard":true} +``` + +Use `{"action":"get"}` to inspect the effective boolean and its source. +The setting persists across turns and process restarts. Set it to `false` +to disable it, or explicitly reset `tool_result_guard` to restore the recipe +default. Recipes can set `AgentConfig.toolResultGuard: true`. +Disabling the setting never restores previously withheld output. + +## Behavior + +The guard applies to the batch of results returned together by the agent's +latest tool round, including text, images, and errors. It recognizes a +structured provider `stopReason: 'refusal'`, not keywords in output, ordinary +provider errors, or natural-language refusal text. + +On that signal, every result in the batch is withheld. Tool calls, result IDs, +and error flags remain intact; each entire payload becomes: + +> Tool result withheld by the guard. The tool has already executed. + +The refused attempt's partial assistant output is discarded. Inference is +retried once on the same model, within the same logical turn; executed tools +are never automatically run again. Refusal details stay in operational logs, +outside the agent-facing notice and setting description. + +The guard takes precedence over Membrane's unchanged-input refusal retries +for a pending batch and its recovery attempt. A second refusal stops this +recovery without automatic rewind of older exchanges or human messages. +Explicit operator `/unstick` remains a separate action. A later clean tool +round can stage a new batch with its own single recovery allowance. + +Normal successful rounds admit the preceding results to memory. This works +for framework yielding streams (including ephemeral agents and context-budget +restarts) and the backward-compatible direct `Agent.runInference` API. + +## Chronicle and memory + +Withholding is non-destructive. Before submitting a guarded batch, the host +appends a `staged` record to the Chronicle append-log state +`framework/tool-result-guard`. It contains the full `originals` (including +pre-truncation data, error strings, and image bytes), the serialized history +`content`, and the `wireResults`. A `linked` record connects its `batchId` to +the context message's `messageId`; later `accepted` or `withheld` records +record the outcome. No guard operation deletes or overwrites these records. +Payload fields larger than 10 KB use Chronicle blobs (`{blobId}`), following +the inference log convention; resolve them with `store.getBlob(blobId)` and +parse the JSON. The append-log snapshots retain the blob references. + +The context manager initially receives only placeholders, so speculative +compression cannot incorporate output that is subsequently withheld. Raw +pending output goes directly to the provider. A clean following response +promotes the history payload through Chronicle's versioned message-edit API. +On a refusal, the placeholders remain. The audit slot is not a context or +compression source. + +If the process stops before acceptance, placeholders remain after restart; +the full pending originals are still available in the audit. This is +deliberately conservative: an interrupted submission does not establish that +the output was accepted. Similarly, if context compilation omits a pending +exchange, its unsubmitted payload is not admitted to memory. + +For example, an operator can inspect records without changing the agent's +view: + +```ts +const store = framework.getStore(); +const records = store.getStateJson('framework/tool-result-guard'); +// For large logs, use getStateLen/getStateItemJson instead of loading all. +const historical = store.getStateJsonAt('framework/tool-result-guard', sequence); +``` + +This is reactive recovery, not pre-submission screening: the provider sees +the original batch once before returning the signal. The setting is not +retroactive and does not rewrite results already accepted into memory. diff --git a/src/agent.ts b/src/agent.ts index 0855aaf..f9508e2 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -2,6 +2,7 @@ import type { Membrane, NormalizedMessage, NormalizedRequest, ContentBlock, Yiel import { isAbortedResponse } from '@animalabs/membrane'; import { createHash } from 'node:crypto'; import type { CacheWireReceipt, KvUnifiedRequestHooks } from './kv-unified-wire.js'; +import { ToolResultGuard, TOOL_RESULT_GUARD_NOTICE } from './tool-result-guard.js'; import { toolResultDataToHistoryString, truncateForHistory, @@ -82,6 +83,7 @@ export class Agent { readonly thinking: AgentConfig['thinking']; /** Refusal auto-rewind policy (see AgentConfig.refusalHandling). */ readonly refusalHandling: AgentConfig['refusalHandling']; + readonly toolResultGuard: ToolResultGuard; /** Prose delivery mode (see AgentConfig.proseRouting). Default 'locus'. */ readonly proseRouting: 'locus' | 'explicit' | 'hybrid' | 'disabled'; /** Exact whole-response known-tool wrapper containment (default off). */ @@ -143,6 +145,7 @@ export class Agent { this.temperature = config.temperature; this.thinking = config.thinking; this.refusalHandling = config.refusalHandling; + this.toolResultGuard = new ToolResultGuard(config.name, contextManager, config.toolResultGuard); this.proseRouting = config.proseRouting ?? 'locus'; this.toolWrapperProseGuard = config.toolWrapperProseGuard ?? false; this.cacheTtl = config.cacheTtl ?? '1h'; @@ -582,17 +585,28 @@ export class Agent { // Filter tools to only allowed ones const tools = availableTools.filter((t) => this.canUseTool(t.name)); + // The direct Agent API uses the same admission rules as framework-driven + // streams. Stage before compiling so compression only sees placeholders. + const guardedResults = this._state.status === 'ready' && this.toolResultGuard.enabled; + if (guardedResults && this._state.status === 'ready') { + const content = this.buildToolResultMessages(this._state.toolResults)[0].content; + const wireResults = content.flatMap((block) => block.type === 'tool_result' + // buildToolResultMessages serializes every payload to a string. + ? [{ toolUseId: block.toolUseId, content: block.content as string, isError: block.isError }] : []); + this.toolResultGuard.storeResults(content, wireResults, this._state.toolResults); + } + // Compile context (with optional injections) const { messages, systemInjections } = await this.compileWithInjections(budget, injections); // If we have pending tool results, add them - if (this._state.status === 'ready') { + if (this._state.status === 'ready' && !guardedResults) { const toolResultMessages = this.buildToolResultMessages(this._state.toolResults); messages.push(...toolResultMessages); } const request: NormalizedRequest = { - messages, + messages: this.toolResultGuard.prepareRequest(messages, true), system: this.buildSystemPrompt(systemInjections), config: { model: this.model, @@ -648,6 +662,7 @@ export class Agent { } catch (error) { // On error, go back to idle this._state = { status: 'idle' }; + this.toolResultGuard.recovering = false; throw error; } } @@ -773,7 +788,7 @@ export class Agent { } return { - messages, + messages: this.toolResultGuard.prepareRequest(messages), system: this.buildSystemPrompt(systemInjections), config: { model: this.model, @@ -830,6 +845,7 @@ export class Agent { this.lastStreamOutputTokens = 0; const request = await this.buildActivationRequest(availableTools, injections, budget); + request.messages = this.toolResultGuard.prepareRequest(request.messages, true); const receiptAware = (this.contextManager as unknown as { getStrategy?: () => unknown }) .getStrategy?.() as { @@ -854,6 +870,7 @@ export class Agent { }; } + const agent = this; const stream = this.membrane.streamYielding(request, { emitTokens: true, emitBlocks: false, @@ -863,7 +880,13 @@ export class Agent { // (cache-warm), where a framework-level requeue would recompile and // land a different window. The framework's driveStream handles the // resulting `retrying` event by discarding the abandoned attempt. - ...(this.refusalHandling?.retries ? { refusalRetries: this.refusalHandling.retries } : {}), + // Membrane reads this per physical round, including after settings + // tools run. A guarded result must reach the host on its FIRST refusal; + // never spend retries resubmitting unchanged guarded content. + get refusalRetries() { + return agent.toolResultGuard.hasPending || agent.toolResultGuard.recovering + ? 0 : agent.refusalHandling?.retries ?? 0; + }, }); this._state = { status: 'streaming', stream }; @@ -1041,9 +1064,21 @@ export class Agent { request: NormalizedRequest, signal?: AbortSignal ): Promise { - const response = await this.membrane.stream(request, { signal }); + let response = await this.membrane.stream(request, { signal }); + if (!isAbortedResponse(response) && response.stopReason === 'refusal') { + const ids = this.toolResultGuard.withhold('unknown'); + if (ids) { + const withheld = new Set(ids); + request = { ...request, messages: request.messages.map((message) => ({ + ...message, content: message.content.map((block) => block.type === 'tool_result' && withheld.has(block.toolUseId) + ? { type: 'tool_result', toolUseId: block.toolUseId, content: TOOL_RESULT_GUARD_NOTICE, isError: block.isError } : block), + })) }; + response = await this.membrane.stream(request, { signal }); + } + } if (isAbortedResponse(response)) { + this.toolResultGuard.recovering = false; const partialContent = response.partialContent ?? []; const { toolCalls, speechContent } = this.extractToolCallsAndSpeech(partialContent); return { @@ -1056,10 +1091,14 @@ export class Agent { }; } - const { toolCalls, speechContent } = this.extractToolCallsAndSpeech(response.content); + const guardedRefusal = response.stopReason === 'refusal' && this.toolResultGuard.recovering; + if (response.stopReason !== 'refusal') this.toolResultGuard.accept(); + this.toolResultGuard.recovering = false; + const content = guardedRefusal ? [] : response.content; + const { toolCalls, speechContent } = this.extractToolCallsAndSpeech(content); // Add assistant response to context - this.contextManager.addMessage(this.name, response.content); + if (!guardedRefusal) this.contextManager.addMessage(this.name, content); return { toolCalls, diff --git a/src/framework.ts b/src/framework.ts index 3bad576..a9c54c8 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -436,7 +436,8 @@ function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): } function isTurnContinuation(reason: string): boolean { - return reason === 'context_budget_restart' || reason === 'tool_results_ready'; + return reason === 'context_budget_restart' || reason === 'tool_results_ready' + || reason === 'tool_result_guard_retry'; } const CONVERSATION_ROUTER_STATE_ID = 'framework/conversation-router'; const INFERENCE_LOG_ID = 'framework/inference-log'; @@ -2332,6 +2333,11 @@ export class AgentFramework { ext.keys.forEach((k) => taken.add(k)); result.set('_framework', ext); } + { + const ext = this.toolResultGuardSettingsExtension(); + ext.keys.forEach((key) => taken.add(key)); + result.set('_toolResultGuard', ext); + } for (const module of this.moduleRegistry.getAllModules()) { const ext = module.getAgentSettingsExtension?.(); if (!ext) continue; @@ -2402,6 +2408,56 @@ export class AgentFramework { }; } + private toolResultGuardSettingsExtension(): AgentSettingsExtension { + const get = (name: string): Record => { + const guard = this.agents.get(name)?.toolResultGuard; + return { + tool_result_guard: guard?.enabled ?? false, + tool_result_guard_source: guard?.settingOverride !== undefined ? 'runtime_override' : 'recipe_default', + }; + }; + const set = (name: string, enabled: boolean | undefined): Record => { + const agent = this.agents.get(name); + if (!agent) throw new Error(`Unknown agent: ${name}`); + const data = this.store.getStateJson(FRAMEWORK_STATE_ID); + const state = (data && typeof data === 'object' ? data : {}) as Record; + const values = { ...((state.toolResultGuards as Record | undefined) ?? {}) }; + if (enabled === undefined) delete values[name]; + else values[name] = enabled; + state.toolResultGuards = values; + this.store.setStateJson(FRAMEWORK_STATE_ID, state); + agent.toolResultGuard.setOverride(enabled); + return get(name); + }; + return { + properties: { + tool_result_guard: { + type: 'boolean', + description: 'Enable the tool result guard. It can withhold a batch of tool output and retry inference once. ' + + 'Tools are not re-executed. Persists across turns and restarts until explicitly changed. ' + + 'Disabling affects future results only; previously withheld results stay withheld.', + }, + }, + keys: ['tool_result_guard'], + get, + update: (name, patch) => { + if (typeof patch.tool_result_guard !== 'boolean') throw new Error('tool_result_guard must be a boolean'); + return set(name, patch.tool_result_guard); + }, + reset: (name) => set(name, undefined), + }; + } + + private restoreToolResultGuardSetting(agent: Agent): void { + const enabled = (this.store.getStateJson(FRAMEWORK_STATE_ID) as { + toolResultGuards?: Record; + } | null)?.toolResultGuards?.[agent.name]; + if (enabled !== undefined && typeof enabled !== 'boolean') { + throw new Error(`Invalid persisted tool_result_guard for ${agent.name}: expected a boolean`); + } + agent.toolResultGuard.setOverride(enabled); + } + private buildThinkTool( policy: SameRoundThinkTextPolicy, ): import('./types/index.js').ToolDefinition { @@ -3783,6 +3839,7 @@ export class AgentFramework { }); const agent = new Agent(config, contextManager, this.membrane); + this.restoreToolResultGuardSetting(agent); this.ephemeralCandidates.set(agent, contextManager); const cleanup = () => { @@ -6087,6 +6144,7 @@ export class AgentFramework { }); const agent = new Agent(config, contextManager, this.membrane); + this.restoreToolResultGuardSetting(agent); const restoredSettings = this.readAgentRuntimeSettings(config.name); if (restoredSettings) { agent.restoreRuntimeSettings( @@ -6161,6 +6219,7 @@ export class AgentFramework { allowedTools: [...SUBCONSCIOUS_TOOL_NAMES, 'think', 'skip_reply', 'end_turn'], }; const agent = new Agent(agentConfig, contextManager, this.membrane); + this.restoreToolResultGuardSetting(agent); this.agents.set(name, agent); this.agentConfigs.set(name, agentConfig); this.subconsciousAgentName = name; @@ -6282,7 +6341,10 @@ export class AgentFramework { // divergence breaks the compile prefix). const { blocks: toolResultContent, spilled } = await this.buildStoredToolResultContent(agent.name, currentState.toolResults, maxChars); - agent.getContextManager().addMessage('user', toolResultContent); + const membraneResults = currentState.toolResults.map(tc => + this.toMembraneToolResult(tc.id, tc.result, maxChars, spilled.get(tc.id)) + ); + agent.toolResultGuard.storeResults(toolResultContent, membraneResults, currentState.toolResults); // Flush any messages that were deferred while this turn was in // flight. Route to the PRIMARY agent — deferred messages are @@ -6520,9 +6582,7 @@ export class AgentFramework { // Mid-turn messages collected above ride along as injected user // messages (membrane ≥0.5.72) — appended after the tool_result // envelope so the next round of THIS turn hears them. - const membraneResults = currentState.toolResults.map(tc => - this.toMembraneToolResult(tc.id, tc.result, maxChars, spilled.get(tc.id)) - ); + agent.toolResultGuard.markSubmitted(); currentState.stream.provideToolResults( membraneResults, midTurnInjections.length > 0 ? { injectedMessages: midTurnInjections } : undefined, @@ -6933,6 +6993,7 @@ export class AgentFramework { const config: AgentConfig = { ...templateConfig, name, strategy: undefined }; const agent = new Agent(config, contextManager, this.membrane); + this.restoreToolResultGuardSetting(agent); this.agents.set(name, agent); this.agentConfigs.set(name, config); this.conversationAgentHomes.set(name, channelId); @@ -8074,6 +8135,8 @@ export class AgentFramework { turnToken: number, ownsProviderGate: boolean, ): Promise { + const continuingTurn = trigger?.reason === 'context_budget_restart' + || trigger?.reason === 'tool_result_guard_retry'; // Flush messages deferred during the PREVIOUS turn — before the // checkpoint, the locus announcement, and the compile — so a turn started // by a queued wake actually CONTAINS the message that woke it. (2026-07-31 @@ -8090,7 +8153,7 @@ export class AgentFramework { // products — undoing the turn must not destroy them. if ( attempt === 0 && - trigger?.reason !== 'context_budget_restart' && + !continuingTurn && this.deferredMessages.length > 0 ) { { @@ -8117,7 +8180,7 @@ export class AgentFramework { } // Record turn checkpoint before inference (only on first attempt, not retries) - if (attempt === 0) { + if (attempt === 0 && trigger?.reason !== 'tool_result_guard_retry') { this.recordTurnCheckpoint(agent.name); this.redoStacks.delete(agent.name); // new work invalidates redo } @@ -8135,7 +8198,7 @@ export class AgentFramework { // after the restart fell through to the hours-old defaultPublishChannel. // Keep the turn's trigger channel across restarts; only real new turns // reset it. - if (trigger?.reason !== 'context_budget_restart') { + if (!continuingTurn) { if (trigger?.channelId) { this.activeTriggerChannels.set(agent.name, trigger.channelId); } else { @@ -8155,12 +8218,12 @@ export class AgentFramework { // BEFORE this turn compiles, so the agent always knows where its voice // goes (announce-on-change only — no per-turn chatter, append-only for // KV stability). - if (trigger?.reason === 'context_budget_restart') { + if (continuingTurn) { const previousLogicalToolState = this.logicalTurnToolCalls.get(agent); this.logicalTurnToolCalls.set(agent, { turnToken, count: previousLogicalToolState?.count ?? 0 }); } - if (trigger?.reason !== 'context_budget_restart') { + if (!continuingTurn) { if (attempt === 0) this.maybePrimeProseMode(agent); const previousLogicalToolState = this.logicalTurnToolCalls.get(agent); if (attempt === 0) { @@ -8205,7 +8268,7 @@ export class AgentFramework { }); // A budget restart continues the same logical inference window. The // predecessor keeps EventGate liveness until its successor terminates. - if (trigger?.reason !== 'context_budget_restart') { + if (!continuingTurn) { this.eventGate?.onInferenceStarted(agent.name); } this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), startedAt: Date.now() }); @@ -8567,7 +8630,8 @@ export class AgentFramework { // turn's prose is bound for, without re-deriving routing. channelId: typingChannel ?? undefined, }); - if (proseStream && event.meta.type === 'text') { + if (proseStream && event.meta.type === 'text' + && !agent.toolResultGuard.hasPending && !agent.toolResultGuard.recovering) { emitOutgoing(proseStream.feed(event.content)); } break; @@ -8618,6 +8682,9 @@ export class AgentFramework { } case 'tool-calls': { + // A tool-call response is a clean physical round: admit its + // preceding results before storing/dispatching this new round. + agent.toolResultGuard.accept(); adoptInjectedRound(); hadToolCalls = true; this.recordLogicalTurnToolCalls(agent, myTurnToken ?? -1, event.calls.length); @@ -8773,7 +8840,60 @@ export class AgentFramework { case 'complete': { adoptInjectedRound(); const durationMs = Date.now() - startTime; - const response = event.response; + let response = event.response; + const guardRefusal = response.stopReason === 'refusal' + && (agent.toolResultGuard.hasPending || agent.toolResultGuard.recovering); + if (guardRefusal) { + const category = (response.raw?.response as { + stop_details?: { category?: string }; + } | undefined)?.stop_details?.category ?? 'unknown'; + const withheld = agent.toolResultGuard.withhold(category); + // Nothing from the abandoned physical attempt is published or + // persisted as assistant speech, including a terminal refusal + // after the single recovery retry. + proseStream?.reset(); + if (withheld) { + const usage = response.details?.usage ?? response.usage; + const tokenUsage = usage ? { + input: usage.inputTokens, output: usage.outputTokens, + cacheCreation: usage.cacheCreationTokens, cacheRead: usage.cacheReadTokens, + } : undefined; + this.noteRefusal(agent.name, category, tokenUsage); + this.logInference({ + timestamp: startTime, agentName: agent.name, requestId, + success: false, error: 'Tool output withheld by the guard', + request: compiledRequest ?? {}, response, durationMs, tokenUsage, stopReason: 'refusal', + }); + console.error(`[tool-result-guard] agent=${agent.name} withheld ${withheld.length} result(s); retrying inference`); + this.emitTrace({ + type: 'inference:stream_restarted', agentName: agent.name, + reason: 'tool_result_guard', inputTokens: agent.lastStreamInputTokens, + budget: agent.maxStreamTokens, + }); + await turnSpeechChain; + if (this.agents.get(agent.name) !== agent || agent.streamId !== myStreamId) { + generationLost = true; + lifecyclePhase = 'aborted'; + return; + } + agent.reset(); + preserveEventGateForSuccessor = true; + lifecyclePhase = 'aborted'; + // Restart inference inside this logical turn. No tool is + // executed again and no settle/checkpoint/locus reset occurs. + await this.startAgentStream(agent, { + ...trigger, agentName: agent.name, reason: 'tool_result_guard_retry', + source: trigger?.source ?? 'framework', timestamp: Date.now(), + }, attempt); + return; + } + const lastResult = response.content.reduce((last, block, index) => + block.type === 'tool_result' ? index : last, -1); + response = { ...response, content: response.content.slice(0, lastResult + 1), rawAssistantText: '' }; + agent.toolResultGuard.recovering = false; + } else if (response.stopReason !== 'refusal') { + agent.toolResultGuard.accept(); + } // If the agent is still waiting_for_tools when 'complete' fires // (shouldn't happen after incomplete-tool-call fix, but guard anyway), @@ -8802,12 +8922,14 @@ export class AgentFramework { // one door a giant blob still walks through). const readyState = agent.state as AgentState; if (readyState.status === 'ready') { - const { blocks: toolResultContent } = await this.buildStoredToolResultContent( + const cap = this.resolveToolResultInlineCap(agent).cap; + const { blocks: toolResultContent, spilled } = await this.buildStoredToolResultContent( agent.name, readyState.toolResults, - this.resolveToolResultInlineCap(agent).cap, + cap, ); - agent.getContextManager().addMessage('user', toolResultContent); + agent.toolResultGuard.storeResults(toolResultContent, readyState.toolResults.map((tc) => + this.toMembraneToolResult(tc.id, tc.result, cap, spilled.get(tc.id))), readyState.toolResults); } } @@ -9021,7 +9143,7 @@ export class AgentFramework { }, () => this.finishUnstick(agent.name, false, category), ); - } else if (rh?.autoRewind) { + } else if (rh?.autoRewind && !guardRefusal) { const cap = Math.max(1, rh.maxRewinds ?? 3); const used = this.refusalRewinds.get(agent.name) ?? 0; doRewind( @@ -9642,6 +9764,7 @@ export class AgentFramework { // (typing still stops, compression still runs — matching the observed // wedge). onInferenceEnded is idempotent, so a redundant call is safe. if (ownsPhysicalStream && !preserveEventGateForSuccessor) { + agent.toolResultGuard.recovering = false; this.eventGate?.onInferenceEnded(agent.name); } if (!generationLost && ownsPhysicalStream) { diff --git a/src/tool-result-guard.ts b/src/tool-result-guard.ts new file mode 100644 index 0000000..fd27933 --- /dev/null +++ b/src/tool-result-guard.ts @@ -0,0 +1,132 @@ +import { randomUUID } from 'node:crypto'; +import type { ContextManager, MessageId } from '@animalabs/context-manager'; +import type { ContentBlock, NormalizedMessage, ToolResult } from '@animalabs/membrane'; +import type { CompletedToolCall } from './types/index.js'; +import { isStateExistsError } from './module-registry.js'; + +export const TOOL_RESULT_GUARD_NOTICE = 'Tool result withheld by the guard. The tool has already executed.'; +export const TOOL_RESULT_GUARD_AUDIT_STATE = 'framework/tool-result-guard'; + +interface PendingBatch { + id: string; + messageId: MessageId; + content: ContentBlock[]; + wireResults: ToolResult[]; + submitted: boolean; +} + +/** + * Admission of newly returned tool output to durable model-facing memory. + * + * Raw output is appended to a separate Chronicle audit slot BEFORE the + * placeholder enters the context manager. In particular, onNewMessage and + * speculative compression never see unaccepted output. Acceptance edits the + * placeholder through CM's versioned edit API; withholding only appends an + * audit event. Neither operation erases the original output or its blobs. + */ +export class ToolResultGuard { + private pending: PendingBatch | undefined; + private registered = false; + private override: boolean | undefined; + /** True until a recovery produces a clean response/new tool round. */ + recovering = false; + + constructor( + private readonly agentName: string, + private readonly cm: ContextManager, + private readonly configured = false, + ) {} + + get enabled(): boolean { return this.override ?? this.configured; } + setOverride(value: boolean | undefined): void { this.override = value; } + get settingOverride(): boolean | undefined { return this.override; } + get hasPending(): boolean { return this.pending !== undefined; } + + private append(record: Record): void { + const store = this.cm.getStore(); + if (!this.registered) { + try { + store.registerState({ id: TOOL_RESULT_GUARD_AUDIT_STATE, strategy: 'append_log' }); + } catch (error) { + if (!isStateExistsError(error)) throw error; + } + this.registered = true; + } + store.appendToStateJson(TOOL_RESULT_GUARD_AUDIT_STATE, { + agentName: this.agentName, timestamp: Date.now(), ...record, + }); + } + + private archive(value: unknown): unknown { + const json = JSON.stringify(value); + // Match inference-log storage: large payloads (especially images and + // pre-spill output) must not be copied into every append-log snapshot. + return json.length > 10_000 + ? { blobId: this.cm.getStore().storeBlob(Buffer.from(json), 'application/json') } + : JSON.parse(json); + } + + storeResults(content: ContentBlock[], wireResults: ToolResult[], originals: CompletedToolCall[]): MessageId { + if (!this.enabled) return this.cm.addMessage('user', content); + if (this.pending) throw new Error('Tool result guard already has a pending batch'); + const id = randomUUID(); + // Includes full pre-truncation/error/image payloads, not just the wire + // preview. This slot is audit data, never a context/compression source. + this.append({ type: 'staged', batchId: id, + originals: this.archive(originals), content: this.archive(content), wireResults: this.archive(wireResults) }); + const withheld: ContentBlock[] = content.map((block) => block.type === 'tool_result' + ? { type: 'tool_result', toolUseId: block.toolUseId, content: TOOL_RESULT_GUARD_NOTICE, isError: block.isError } + : block); + const messageId = this.cm.addMessage('user', withheld); + this.pending = { id, messageId, content, wireResults, submitted: false }; + this.append({ type: 'linked', batchId: id, messageId }); + return messageId; + } + + /** Live continuation submitted directly through provideToolResults. */ + markSubmitted(): void { if (this.pending) this.pending.submitted = true; } + + /** A budget/error restart compiles placeholders; restore pending output + * only in this provider request, never in the strategy's view. */ + prepareRequest(messages: NormalizedMessage[], recordSubmission = false): NormalizedMessage[] { + const pending = this.pending; + if (!pending) return messages; + const byId = new Map(pending.wireResults.map((result) => [result.toolUseId, result])); + const present = new Set(messages.flatMap((message) => message.content + .filter((block) => block.type === 'tool_result' && byId.has(block.toolUseId)) + .map((block) => (block as ContentBlock & { toolUseId: string }).toolUseId))); + // A strategy may have folded the entire exchange away. Do not release + // content that was never submitted. Its originals remain in the audit. + const submitted = present.size === byId.size; + if (recordSubmission) pending.submitted = submitted; + if (!submitted) return messages; + return messages.map((message) => ({ ...message, content: message.content.map((block) => { + const result = block.type === 'tool_result' ? byId.get(block.toolUseId) : undefined; + return result ? { type: 'tool_result', toolUseId: result.toolUseId, content: result.content, isError: result.isError } : block; + }) })); + } + + /** A clean physical response accepts precisely the last submitted batch. */ + accept(): void { + const pending = this.pending; + if (pending) { + this.append({ type: pending.submitted ? 'accepted' : 'withheld', batchId: pending.id, messageId: pending.messageId }); + if (pending.submitted) this.cm.editMessage(pending.messageId, pending.content); + this.pending = undefined; + } + this.recovering = false; + } + + /** At most one recovery per batch; no scanning/deleting older history. */ + withhold(category: string): string[] | null { + const pending = this.pending; + if (!pending?.submitted) return null; + const ids = pending.wireResults.map((result) => result.toolUseId); + // Even a failed outcome-log write must never re-arm rejected output for + // a later submission. Its originals were archived before admission. + this.pending = undefined; + this.recovering = true; + this.append({ type: 'withheld', batchId: pending.id, messageId: pending.messageId, toolUseIds: ids, category }); + return ids; + } +} diff --git a/src/types/agent.ts b/src/types/agent.ts index b5cd747..e7009cc 100644 --- a/src/types/agent.ts +++ b/src/types/agent.ts @@ -170,6 +170,12 @@ export interface AgentConfig { announceHumanTurns?: boolean; }; + /** Opt-in tool-output admission and one guarded retry after a provider + * refusal. Agents can persistently override this via agent_settings + * tool_result_guard. Full originals remain in Chronicle's audit history. + * Default false. Disabling does not restore previously withheld output. */ + toolResultGuard?: boolean; + /** * How the agent's PLAIN PROSE (non-tool output) reaches channels. * - 'locus' (default): host-inferred — the turn-frozen locus machinery. diff --git a/test/tool-result-guard.test.ts b/test/tool-result-guard.test.ts new file mode 100644 index 0000000..4398596 --- /dev/null +++ b/test/tool-result-guard.test.ts @@ -0,0 +1,384 @@ +import { afterEach, test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { PassthroughStrategy, type StoredMessage, type StrategyContext } from '@animalabs/context-manager'; +import type { ContentBlock, Membrane, NormalizedRequest, NormalizedResponse, YieldingStreamOptions } from '@animalabs/membrane'; +import { Membrane as RealMembrane, NativeFormatter, type ProviderAdapter, type ProviderRequest, + type ProviderResponse, type StreamCallbacks } from '@animalabs/membrane'; +import { AgentFramework, type AgentConfig, type AgentSettingsExtension, type Module, type ModuleContext, + type ToolCall, type ToolResult, type ProcessEvent, type ProcessState } from '../src/index.js'; +import { TOOL_RESULT_GUARD_AUDIT_STATE, TOOL_RESULT_GUARD_NOTICE } from '../src/tool-result-guard.js'; +import { MockYieldingStream, createMockResponse } from './helpers/mock-membrane.js'; + +const dirs: string[] = []; +afterEach(() => { while (dirs.length) rmSync(dirs.pop()!, { recursive: true, force: true }); }); +const answer = () => createMockResponse([{ type: 'text', text: 'continued' }]); +const refused = () => ({ + ...createMockResponse([{ type: 'text', text: 'discard-this-partial-output' }], 'refusal'), + raw: { request: {}, response: { stop_details: { category: 'test-category' } } }, +}) as NormalizedResponse; +const calls = (...ids: string[]) => createMockResponse(ids.map((id) => ({ + type: 'tool_use', id, name: 'test--read', input: {}, +})), 'tool_use'); + +class ScriptMembrane { + requests: NormalizedRequest[] = []; + streams: MockYieldingStream[] = []; + retriesAtSubmission: number[] = []; + onSubmit?: () => void; + constructor(readonly scripts: NormalizedResponse[][]) {} + streamYielding(request: NormalizedRequest, options: YieldingStreamOptions = {}) { + this.requests.push(structuredClone({ ...request, onCacheWireReceipt: undefined })); + const script = this.scripts.shift(); + assert.ok(script, 'unexpected extra inference'); + const stream = new MockYieldingStream(script); + const provide = stream.provideToolResults.bind(stream); + stream.provideToolResults = (...args) => { + this.retriesAtSubmission.push(options.refusalRetries ?? 0); + this.onSubmit?.(); + provide(...args); + }; + this.streams.push(stream); + return stream; + } + asMembrane() { return this as unknown as Membrane; } +} + +class ReadModule implements Module { + readonly name = 'test'; + readonly calls: string[] = []; + readonly speeches: string[] = []; + constructor(readonly results: Record = {}) {} + async start(ctx: ModuleContext) { ctx.registerSpeechHandler('*'); } + async stop() {} + getTools() { return [{ name: 'read', description: 'Read a result', inputSchema: { type: 'object' as const, properties: {} } }]; } + async handleToolCall(call: ToolCall): Promise { + this.calls.push(call.id); + return this.results[call.id] ?? { success: true, data: `payload-${call.id}` }; + } + async onProcess(event: ProcessEvent, _state: ProcessState) { + return event.type === 'external-message' + ? { addMessages: [{ participant: 'user', content: [{ type: 'text' as const, text: String(event.content) }] }], requestInference: true } + : {}; + } + async onAgentSpeech(_name: string, content: ContentBlock[]) { + this.speeches.push(...content.flatMap((block) => block.type === 'text' ? [block.text] : [])); + } +} + +class IngressObserver extends PassthroughStrategy { + snapshots: string[] = []; + async onNewMessage(_message: StoredMessage, ctx: StrategyContext) { + this.snapshots.push(JSON.stringify(ctx.messageStore.getAll())); + } +} + +async function harness(scripts: NormalizedResponse[][], config: Partial = {}, results?: Record) { + const dir = mkdtempSync(join(tmpdir(), 'af-tool-result-guard-')); dirs.push(dir); + const membrane = new ScriptMembrane(scripts); + const module = new ReadModule(results); + const base = { storePath: join(dir, 'store'), membrane: membrane.asMembrane(), + agents: [{ name: 'assistant', model: 'test', systemPrompt: 'system', ...config }], modules: [module], syncIntervalMs: 0 }; + const framework = await AgentFramework.create(base); + const run = async () => { + framework.pushEvent({ type: 'external-message', source: 'test', content: 'read', metadata: {} }); + await framework.runUntilIdle(); + }; + return { framework, membrane, module, base, run }; +} + +function extension(framework: AgentFramework): AgentSettingsExtension { + const extensions = (framework as unknown as { + collectAgentSettingsExtensions(): Map; + }).collectAgentSettingsExtensions(); + return [...extensions.values()].find((ext) => ext.keys.includes('tool_result_guard'))!; +} + +function toolResults(framework: AgentFramework) { + return framework.getAgent('assistant')!.getContextManager().getAllMessages() + .flatMap((message) => message.content.filter((block) => block.type === 'tool_result')); +} + +test('default off retains existing refusal behavior and original tool output', async () => { + const h = await harness([[calls('one'), refused()]]); + try { + await h.run(); + assert.equal(h.membrane.requests.length, 1); + assert.match(JSON.stringify(toolResults(h.framework)), /payload-one/); + assert.equal(extension(h.framework).get('assistant').tool_result_guard, false); + } finally { await h.framework.stop(); } +}); + +test('withholds the entire latest batch, retries inference once, and keeps originals in Chronicle', async () => { + const strategy = new IngressObserver(); + const image = Buffer.from('original-image-bytes').toString('base64'); + const h = await harness([[calls('text', 'image', 'error'), refused()], [answer()]], + { toolResultGuard: true, strategy, refusalHandling: { retries: 3 } }, { + text: { success: true, data: 'original-text-payload' }, + image: { success: true, data: [{ type: 'image', mimeType: 'image/png', data: image }] }, + error: { success: false, isError: true, error: 'original-error-payload' }, + }); + let stagedSequence = 0; + let originalStopped = false; + h.membrane.onSubmit = () => { stagedSequence = h.framework.getStore().currentSequence(); }; + try { + await h.run(); + assert.deepEqual(h.module.calls.sort(), ['error', 'image', 'text']); + assert.equal(h.membrane.requests.length, 2); + assert.deepEqual(h.membrane.retriesAtSubmission, [0], 'first refusal must reach guard before plain retries'); + const retry = h.membrane.requests[1]; + assert.equal(retry.config.model, 'test'); + assert.doesNotMatch(JSON.stringify(retry), /original-text-payload|original-error-payload|discard-this-partial-output|test-category/); + assert.ok(!JSON.stringify(retry).includes(image)); + assert.equal(retry.messages.flatMap((m) => m.content).filter((b) => b.type === 'tool_use').length, 3); + const guarded = toolResults(h.framework); + assert.equal(guarded.length, 3); + assert.ok(guarded.every((block) => block.content === TOOL_RESULT_GUARD_NOTICE)); + assert.equal(guarded.find((b) => b.toolUseId === 'error')?.isError, true); + assert.deepEqual(h.module.speeches, ['continued']); + assert.ok(strategy.snapshots.every((s) => !s.includes('original-text-payload') && !s.includes(image)), + 'background strategy ingress must never see withheld payloads'); + + const store = h.framework.getStore(); + const audit = store.getStateJson(TOOL_RESULT_GUARD_AUDIT_STATE) as Array>; + assert.match(JSON.stringify(audit[0]), /original-text-payload|original-error-payload/); + assert.ok(JSON.stringify(audit[0]).includes(image)); + assert.equal(audit.at(-1)?.type, 'withheld'); + const historical = store.getStateJsonAt(TOOL_RESULT_GUARD_AUDIT_STATE, stagedSequence) as unknown[]; + assert.deepEqual(audit[0], historical[0], 'redaction only appends; original Chronicle record is unchanged'); + const original = structuredClone(audit[0]); + extension(h.framework).update('assistant', { tool_result_guard: false }); + assert.ok(toolResults(h.framework).every((block) => block.content === TOOL_RESULT_GUARD_NOTICE)); + await h.framework.stop(); + originalStopped = true; + const restarted = await AgentFramework.create(h.base); + try { + assert.equal(extension(restarted).get('assistant').tool_result_guard, false, 'explicit disable persists'); + assert.ok(toolResults(restarted).every((block) => block.content === TOOL_RESULT_GUARD_NOTICE)); + assert.deepEqual((restarted.getStore().getStateJson(TOOL_RESULT_GUARD_AUDIT_STATE) as unknown[])[0], original); + const preview = await restarted.previewActivation('assistant'); + assert.doesNotMatch(JSON.stringify(preview), /original-text-payload|original-error-payload/); + } finally { await restarted.stop(); } + } finally { if (!originalStopped) await h.framework.stop(); } +}); + +test('successful physical rounds admit results; refusal affects only the newest batch', async () => { + const h = await harness([[calls('accepted'), calls('withheld'), refused()], [answer()]], { toolResultGuard: true }); + try { + await h.run(); + const results = toolResults(h.framework); + assert.match(String(results.find((b) => b.toolUseId === 'accepted')?.content), /payload-accepted/); + assert.equal(results.find((b) => b.toolUseId === 'withheld')?.content, TOOL_RESULT_GUARD_NOTICE); + assert.deepEqual(h.module.calls, ['accepted', 'withheld']); + assert.equal(h.framework.getAgent('assistant')!.toolResultGuard.enabled, true); + } finally { await h.framework.stop(); } +}); + +test('a second refusal stops recovery without auto-rewinding older or human messages', async () => { + const h = await harness([[calls('one'), refused()], [refused()]], + { toolResultGuard: true, refusalHandling: { autoRewind: true, retries: 3 } }); + try { + await h.run(); + assert.equal(h.membrane.requests.length, 2); + assert.deepEqual(h.module.calls, ['one']); + assert.deepEqual(h.module.speeches, []); + const messages = h.framework.getAgent('assistant')!.getContextManager().getAllMessages(); + assert.ok(messages.some((m) => m.content.some((b) => b.type === 'text' && b.text === 'read'))); + assert.doesNotMatch(JSON.stringify(messages), /discard-this-partial-output|refusal-rewind/); + assert.equal(toolResults(h.framework)[0].content, TOOL_RESULT_GUARD_NOTICE); + } finally { await h.framework.stop(); } +}); + +test('durable typed agent setting can be enabled, disabled, and explicitly reset', async () => { + const h = await harness([]); + let originalStopped = false; + try { + const ext = extension(h.framework); + for (const value of ['true', 1, null, {}]) { + assert.throws(() => ext.update('assistant', { tool_result_guard: value }), /must be a boolean/); + } + ext.update('assistant', { tool_result_guard: true }); + assert.equal(ext.get('assistant').tool_result_guard, true); + await h.framework.stop(); + originalStopped = true; + const restarted = await AgentFramework.create(h.base); + try { + const restored = extension(restarted); + assert.equal(restored.get('assistant').tool_result_guard, true); + assert.equal(restored.get('assistant').tool_result_guard_source, 'runtime_override'); + restored.reset!('assistant'); + assert.equal(restored.get('assistant').tool_result_guard, false); + const tool = restarted.getAllTools().find((t) => t.name === 'agent_settings')!; + assert.equal((tool.inputSchema.properties as Record).tool_result_guard.type, 'boolean'); + assert.doesNotMatch(JSON.stringify(tool), /classifier/i); + } finally { await restarted.stop(); } + } finally { if (!originalStopped) await h.framework.stop(); } +}); + +test('a normal successful response releases the pending output into versioned history', async () => { + const h = await harness([[calls('one'), answer()]], { toolResultGuard: true }); + try { + await h.run(); + assert.equal(h.membrane.requests.length, 1); + assert.match(String(toolResults(h.framework)[0].content), /payload-one/); + const records = h.framework.getStore().getStateJson(TOOL_RESULT_GUARD_AUDIT_STATE) as Array>; + assert.equal(records.at(-1)?.type, 'accepted'); + assert.equal(h.framework.getAgent('assistant')!.toolResultGuard.enabled, true); + } finally { await h.framework.stop(); } +}); + +test('refusal without a new tool result does not invoke tool guard recovery', async () => { + const h = await harness([[refused()]], { toolResultGuard: true }); + try { await h.run(); assert.equal(h.membrane.requests.length, 1); } + finally { await h.framework.stop(); } +}); + +test('budget restart submits staged originals, then recovers without re-executing tools', async () => { + const h = await harness([[calls('one')], [refused()], [answer()]], { toolResultGuard: true, maxStreamTokens: 1 }); + try { + await h.run(); + assert.equal(h.membrane.requests.length, 3); + assert.match(JSON.stringify(h.membrane.requests[1]), /payload-one/); + assert.doesNotMatch(JSON.stringify(h.membrane.requests[2]), /payload-one/); + assert.deepEqual(h.module.calls, ['one']); + assert.equal(toolResults(h.framework)[0].content, TOOL_RESULT_GUARD_NOTICE); + } finally { await h.framework.stop(); } +}); + +test('ephemeral run settles only after recovery and counts each tool once', async () => { + const h = await harness([[calls('one'), refused()], [answer()]]); + try { + const created = await h.framework.createEphemeralAgent({ + name: 'ephemeral', model: 'test', systemPrompt: 'system', toolResultGuard: true, + }); + created.contextManager.addMessage('user', [{ type: 'text', text: 'read' }]); + const completion = h.framework.runEphemeralToCompletion(created.agent, created.contextManager); + h.framework.start(); + const result = await completion; + assert.deepEqual(result, { speech: 'continued', toolCallsCount: 1 }); + assert.deepEqual(h.module.calls, ['one']); + assert.equal(h.membrane.requests.length, 2); + } finally { await h.framework.stop(); } +}); + +test('quiesced scheduler admits a queued guard recovery as a continuation', async () => { + const h = await harness([[answer()]], { toolResultGuard: true }); + try { + await h.framework.quiesce(); + h.framework.getAgent('assistant')!.getContextManager().addMessage('user', [{ + type: 'text', text: TOOL_RESULT_GUARD_NOTICE, + }]); + // A recovery can be requeued while waiting for provider admission. It + // must finish the held turn even after the host stops admitting new work. + (h.framework as unknown as { pendingRequests: Array> }).pendingRequests.push({ + agentName: 'assistant', reason: 'tool_result_guard_retry', source: 'framework', timestamp: Date.now(), + }); + await h.framework.runUntilIdle(); + assert.equal(h.membrane.requests.length, 1); + assert.equal(h.framework.getHostModeStatus().quiesced, true); + } finally { await h.framework.stop(); } +}); + +test('native Membrane observes the first refusal even when guard is enabled by a tool mid-stream', async () => { + const h = await harness([]); + await h.framework.stop(); + const requests: ProviderRequest[] = []; + const adapter: ProviderAdapter = { + name: 'test', usageCacheConvention: 'cache-excluded', supportsModel: () => true, + complete: async () => { throw new Error('unexpected complete'); }, + stream: async (request: ProviderRequest, callbacks: StreamCallbacks): Promise => { + requests.push(structuredClone(request)); + const index = requests.length; + assert.ok(index <= 3, 'must not retry unchanged refused input'); + const content = index === 1 ? [ + { type: 'tool_use', id: 'enable', name: 'agent_settings', input: { action: 'update', tool_result_guard: true } }, + { type: 'tool_use', id: 'one', name: 'test--read', input: {} }, + ] : [{ type: 'text', text: index === 2 ? 'discard-native-partial' : 'continued' }]; + if (index > 1) callbacks.onChunk?.(index === 2 ? 'discard-native-partial' : 'continued'); + return { + content, stopReason: index === 1 ? 'tool_use' : index === 2 ? 'refusal' : 'end_turn', + usage: { inputTokens: 20, outputTokens: 5 }, model: 'test', + raw: { response: { stop_details: { category: 'test-category' } } }, + } as ProviderResponse; + }, + }; + const framework = await AgentFramework.create({ ...h.base, + agents: [{ name: 'assistant', model: 'test', systemPrompt: 'system', refusalHandling: { retries: 4 } }], + membrane: new RealMembrane(adapter, { formatter: new NativeFormatter() }), + }); + const routed: string[] = []; + const outgoing: string[] = []; + (framework as unknown as { channelRegistry: unknown }).channelRegistry = new Proxy({ + resolveLocus: () => 'world:test', + routeSpeech: async (_agent: string, speech: string) => { + routed.push(speech); return { delivered: true, channelId: 'world:test' }; + }, + sendOutgoingChunk: (_channel: string, _agent: string, _id: string, _index: number, delta: string) => { outgoing.push(delta); }, + getDefaultPublishChannel: () => null, isChannelOpen: () => true, + getDescriptor: () => undefined, getChannelTools: () => [], + }, { get: (target, key: string) => key in target ? (target as Record)[key] : () => undefined }); + try { + framework.pushEvent({ type: 'external-message', source: 'test', content: 'read', metadata: {} }); + await framework.runUntilIdle(); + assert.equal(requests.length, 3); + assert.match(JSON.stringify(requests[1]), /payload-one/); + assert.doesNotMatch(JSON.stringify(requests[2]), /payload-one|discard-native-partial|test-category/); + assert.match(JSON.stringify(requests[2]), /Tool result withheld by the guard/); + assert.deepEqual(h.module.calls, ['one']); + assert.equal(extension(framework).get('assistant').tool_result_guard, true); + assert.deepEqual(h.module.speeches, ['continued']); + assert.deepEqual(routed, ['continued']); + assert.doesNotMatch(outgoing.join(''), /discard-native-partial/); + } finally { await framework.stop(); } +}); + +test('backward-compatible direct Agent inference also guards tool results', async () => { + const h = await harness([], { toolResultGuard: true }); + const responses = [calls('one'), refused(), answer()]; + const requests: NormalizedRequest[] = []; + (h.membrane as unknown as { stream: (request: NormalizedRequest) => Promise }).stream = async (request) => { + requests.push(structuredClone(request)); + const response = responses.shift(); + assert.ok(response); + return response; + }; + try { + const agent = h.framework.getAgent('assistant')!; + agent.getContextManager().addMessage('user', [{ type: 'text', text: 'read' }]); + const first = await agent.runInference(h.framework.getAllTools()); + assert.equal(first.toolCalls.length, 1); + agent.provideToolResult('one', { success: true, data: 'direct-original' }); + const final = await agent.runInference(h.framework.getAllTools()); + assert.deepEqual(final.speechContent, [{ type: 'text', text: 'continued' }]); + assert.equal(requests.length, 3); + assert.match(JSON.stringify(requests[1]), /direct-original/); + assert.doesNotMatch(JSON.stringify(requests[2]), /direct-original|discard-this/); + assert.equal(toolResults(h.framework)[0].content, TOOL_RESULT_GUARD_NOTICE); + } finally { await h.framework.stop(); } +}); + +test('full oversized output survives withholding and reopening as a Chronicle blob', async () => { + const original = 'original-large-'.repeat(8_000) + 'end-of-original'; + const h = await harness([[calls('large'), refused()], [answer()]], { toolResultGuard: true }, { + large: { success: true, data: original }, + }); + let originalStopped = false; + try { + await h.run(); + const audit = h.framework.getStore().getStateJson(TOOL_RESULT_GUARD_AUDIT_STATE) as Array<{ + originals: { blobId: string }; + }>; + const blobId = audit[0].originals.blobId; + assert.equal(typeof blobId, 'string'); + const blob = h.framework.getStore().getBlob(blobId)!; + assert.equal(JSON.parse(blob.toString())[0].result.data, original, 'keep pre-truncation bytes'); + await h.framework.stop(); originalStopped = true; + const restarted = await AgentFramework.create(h.base); + try { + assert.deepEqual(restarted.getStore().getBlob(blobId), blob); + assert.equal(toolResults(restarted)[0].content, TOOL_RESULT_GUARD_NOTICE); + } finally { await restarted.stop(); } + } finally { if (!originalStopped) await h.framework.stop(); } +});