diff --git a/changelog.d/403-truncation-image-slots.fixed.md b/changelog.d/403-truncation-image-slots.fixed.md new file mode 100644 index 000000000..0f23518ab --- /dev/null +++ b/changelog.d/403-truncation-image-slots.fixed.md @@ -0,0 +1,8 @@ +- A tool result stored over its inline cap no longer keeps every line shaped like a reference stub from past the cut (#403). Every `[ref_…] …` line past the cut was added back after the truncation notice, whether or not it named an image and with no limit, so a text result listing channel attachments was stored far over the cap: a 25,980-character result at a cap of 2,000 was stored at 25,168 characters. The lines past the cut were kept so that `save_recent_image`'s index counts every image the resident saw, and now the stored copy keeps what the live copy showed. + - A result with no inline image keeps nothing past its cut. Its live copy is the stored text, so the resident never saw what lies past the cut either. A truncated text result no longer lists the image stubs past the cut: the spill file holds them, and without a writable workspace they aren't kept, like the rest of the text past the cut. Such a reference's record is still in the registry, so running its tool again brings its stub back. + - A result carrying an inline image went to the model as blocks: every image and every reference's stub, whatever the text's length, and each text block cut at the cap on its own. Past its cut it keeps the image slots those blocks showed, placeholders and image-typed stubs, the newest 64 of them, the most images the tool-image ledger holds bytes for. A slot in the part of a text block that the live copy cut away isn't kept. When more than 64 fell past the cut, a note says how many were left out, and `save_recent_image` doesn't count those. + - History's own truncation, which cuts a script's tool results and those on the direct Agent API, keeps nothing past its cut either: each is read as its text. + - The stored copy's cut, history's own truncation, and the live copy's cut of a text block beside an image never leave part of a slot before their notice. A cut that would split a placeholder moves back to the placeholder's start, and one that would split a stub-shaped line moves back to the line's start when at most 677 characters of the line lie before the cut, since that part could still read as an image; a longer part can't, and stays. The live copy and the stored copy cut alike, so the stored copy counts exactly the slots the live copy showed. + - `save_recent_image` counts a reference stub as an image by the type its stored text shows, as it did after a restart or an eviction, and no longer by the registry's record while one exists. A stub counts the same whether or not the process still holds its record, and a reference issued again with testimony of another type no longer changes how an earlier stub counts. + - A save in the same batch as a long result counts it as its stored copy will be cut, and an error result, or one with no data, which history stores as its error text, counts that text's slots before it is stored as it does after. + - A slot is no longer than its writer writes it: a stub-shaped line longer than the longest stub the framework writes (677 characters), or a placeholder whose type or subtype is over 127 characters or whose size's number, ledger nonce or sequence is over 16 characters, is quoted text and counts nowhere. A tool image whose type or subtype is over 127 characters, the most RFC 6838 allows, is labelled `application/octet-stream`. A placeholder stored before with such a type no longer counts; only a server that sent one could have produced it. diff --git a/src/framework.ts b/src/framework.ts index 6d854a393..4eaf72802 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1,6 +1,6 @@ import { ToolPresentation, presentationTools, isPresentationTool, renderCatalogue, type PresentationSnapshot } from "./tool-presentation.js"; import { dirname, join } from 'node:path'; -import { INLINE_WITHHELD_TEXT, REFERENCE_LIMITS, classifyBlock, isInlineContradiction, isReferenceExpired, referenceRegistry, referenceStubOrNull, referenceTypeLabel, sanitizeLabel, unfetchableReason } from './mcpl/references.js'; +import { INLINE_WITHHELD_TEXT, REFERENCE_LIMITS, classifyBlock, isInlineContradiction, isReferenceExpired, referenceRegistry, referenceStubOrNull, sanitizeLabel, unfetchableReason } from './mcpl/references.js'; import { ReferenceFetcher, DEFAULT_FETCH_MAX_BYTES, EAGER_FETCH_TIMEOUT_MS } from './mcpl/reference-fetcher.js'; import { appendFileSync, closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, @@ -110,15 +110,19 @@ import { toolResultDataToHistoryString, truncateForHistory, DEFAULT_TOOL_RESULT_INLINE_MAX_CHARS, + type HistoryBlockSpan, } from './tool-result-history.js'; import { TOOL_IMAGE_REF_RE, ToolImageLedger, - formatPreservedImageSlots, + cutStoredToolResult, + formatKeptImageSlots, parseImagePlaceholders, sha256Hex, - splitPreservingImageSlots, + storedImageSlots, + toolResultCutAt, type ParsedImagePlaceholder, + type ShownRanges, } from './tool-image-ledger.js'; import { randomUUID, createHash } from 'node:crypto'; import { PyRunner, buildInjectedTools, formatLimit } from './code-execution/py-runner.js'; @@ -15909,23 +15913,9 @@ export class AgentFramework { }> { const spilled = new Map(); const dateLabel = new Date().toISOString().slice(0, 10); - const ledger = this.toolImageLedgerFor(agentName); for (const tc of toolResults) { - // Retain every image the result carries BEFORE the placeholder is - // written (issue #104): the placeholder embeds the ref, so the image - // stays reachable by provenance after the bytes leave the wire. - const raw = tc.result.isError - ? tc.result.error ?? 'Unknown error' - : nonErrorResultText(tc.result, undefined, { - imageRef: (blockIndex, image) => ledger.retain({ - toolCallId: tc.id, - toolName: tc.name, - blockIndex, - data: image.data, - mediaType: image.mimeType, - }).ref, - }); - spilled.set(tc.id, await this.spillOrTruncate(raw, maxChars, `${dateLabel}-${tc.id}`)); + const { text, shown } = this.storedToolResultText(agentName, tc, maxChars); + spilled.set(tc.id, await this.spillOrTruncate(text, maxChars, `${dateLabel}-${tc.id}`, shown)); } const blocks: ContentBlock[] = toolResults.map(tc => ({ type: 'tool_result' as const, @@ -15939,6 +15929,47 @@ export class AgentFramework { return { blocks, spilled }; } + /** + * A tool result's text as its stored copy is written before any cut, and + * what its live copy showed beside that text (see cutStoredToolResult). An + * error result is its error text. Any other is serialized by + * nonErrorResultText, which retains each inline image BEFORE its + * placeholder is written (issue #104): the placeholder embeds the ref, so + * the image stays reachable by provenance after the bytes leave the wire. + * + * The live copy is the stored text itself unless the result is a content + * array carrying an inline image, which tryNativeToolResultContent sends + * as native blocks. It takes the same arrays the serializer takes block by + * block, and shows each image and each reference block's stub whole and + * each text block up to where it cuts that block at the cap (`maxChars`, + * toolResultCutAt), so those are the ranges of the text it showed. + */ + private storedToolResultText( + agentName: string, + tc: CompletedToolCall, + maxChars: number | undefined, + ): { text: string; shown: ShownRanges | null } { + if (tc.result.isError) return { text: tc.result.error ?? 'Unknown error', shown: null }; + const ledger = this.toolImageLedgerFor(agentName); + let blocks = null as HistoryBlockSpan[] | null; + const text = nonErrorResultText(tc.result, undefined, { + imageRef: (blockIndex, image) => ledger.retain({ + toolCallId: tc.id, + toolName: tc.name, + blockIndex, + data: image.data, + mediaType: image.mimeType, + }).ref, + onBlocks: (spans) => { blocks = spans; }, + }); + if (!blocks?.some((block) => block.kind === 'image')) return { text, shown: null }; + return { + text, + shown: blocks.map(({ kind, start, end }): [number, number] => + [start, kind === 'text' && maxChars ? start + toolResultCutAt(text.slice(start, end), maxChars) : end]), + }; + } + /** Where a tool result labelled `label` is written: `/tool-results/