diff --git a/.changeset/mcp-fold-unchanged-subtrees.md b/.changeset/mcp-fold-unchanged-subtrees.md new file mode 100644 index 000000000..c3969fb52 --- /dev/null +++ b/.changeset/mcp-fold-unchanged-subtrees.md @@ -0,0 +1,5 @@ +--- +"e2e": patch +--- + +When an action changes most of the screen, large subtrees that did not change are shown as one line instead of being repeated. diff --git a/packages/core/src/agent/default-agent.ts b/packages/core/src/agent/default-agent.ts index 2cd709564..5548dcd8f 100644 --- a/packages/core/src/agent/default-agent.ts +++ b/packages/core/src/agent/default-agent.ts @@ -16,7 +16,7 @@ import type { ExecutorObservation, ExecutorObserveOptions, ReplayedPrefix, StepE import { interactiveNodeCount } from './observation.ts'; import { testTitleLine } from './prompts.ts'; import { grammarTools } from './primitives.ts'; -import { ScreenPresenter } from './screen-update.ts'; +import { boundPresenter, presenterFor, type ScreenPresenter } from './screen-update.ts'; import { compactScreenHistory, compactScreenshotHistory } from './transcript-compaction.ts'; import { createToolLoopExecutor, type ToolLoopHelpers } from './tool-loop.ts'; import { toolAppliesTo, withToolContext } from './tool.ts'; @@ -51,18 +51,6 @@ const ASSERT_RULES = `You are judging exactly one assertion against the current - Each scroll result contains a fresh observation. When screenshots are shown, each result includes fresh pixels too. - Conclude passed when the collected evidence establishes the assertion, failed with ASSERTION_FAILED when it contradicts the assertion, and failed with ASSERTION_INCONCLUSIVE when the available evidence or remaining budget cannot establish an answer.`; -/** One presenter per dispatched step, shared by the opening prompt and the tools that follow it. */ -const presenters = new WeakMap(); - -function presenterFor(context: StepExecutorContext): ScreenPresenter { - let presenter = presenters.get(context); - if (presenter === undefined) { - presenter = new ScreenPresenter(); - presenters.set(context, presenter); - } - return presenter; -} - /** What the built-in agent is built from: an agents entry's `system` and its validated `tools`. */ export interface BuiltInAgentOptions { /** Appended to the built-in execution rules; read by act and exploring assert loops. */ @@ -90,6 +78,11 @@ function createActLoop(options: BuiltInAgentOptions): StepExecutor { prepareMessages: (messages) => compactScreenshotHistory(compactScreenHistory(messages)), tools: (context, helpers) => loopTools(context, helpers, options.tools), buildPrompt: async (context) => { + // The run's presenter is read before the first await: another run on + // this context may bind its own presenter while this one waits on + // observe, and the opening prompt must read this run's, not the other + // run's. + const presenter = boundPresenter(context) ?? presenterFor(context); // The opening look is tree-only, unless no listed interactive node // exists at all: then the opening prompt carries a screenshot, because the tree describes a canvas, a game, // or a semantics-free native screen too poorly to act on, and the model @@ -102,7 +95,7 @@ function createActLoop(options: BuiltInAgentOptions): StepExecutor { const extra: string[] = []; if (context.replayedPrefix !== undefined) extra.push(formatReplayedPrefix(context.replayedPrefix)); if (context.ledger !== '') extra.push(`Previously completed steps:\n${context.ledger}`); - return openingPrompt(context, `Execute this test step: ${context.step.instruction}`, extra, observation); + return openingPrompt(context, `Execute this test step: ${context.step.instruction}`, extra, observation, presenter); }, }); } @@ -118,8 +111,11 @@ function createAssertionLoop(options: BuiltInAgentOptions): StepExecutor { system: rules(ASSERT_RULES, options.system), inconclusiveOnOverflow: true, tools: (context, helpers) => loopTools(context, helpers, options.tools), - buildPrompt: async (context) => - openingPrompt(context, `Judge whether this assertion holds: ${context.step.instruction}`, [], await context.observe()), + buildPrompt: async (context) => { + // The run's presenter is read before the first await, as the act loop does. + const presenter = boundPresenter(context) ?? presenterFor(context); + return openingPrompt(context, `Judge whether this assertion holds: ${context.step.instruction}`, [], await context.observe(), presenter); + }, }); return { ...loop, runStep: (context) => loop.runStep(withVision(context)) }; } @@ -155,7 +151,7 @@ function loopTools( ): ToolSet { return { ...guardedTools(helpers, projectTools(context, tools)), - ...grammarTools(context, { guard: helpers.guard, screen: presenterFor(context) }), + ...grammarTools(context, { guard: helpers.guard }), }; } @@ -165,6 +161,7 @@ function openingPrompt( lead: string, extra: readonly string[], observation: ExecutorObservation, + presenter: ScreenPresenter, ): string | ModelMessage[] { const title = testTitleLine(context.attempt.titlePath); const parts = [ @@ -173,7 +170,7 @@ function openingPrompt( ...(context.step.params === undefined ? [] : [`Step parameters:\n${JSON.stringify(context.step.params)}`]), ...extra, ]; - const opening = presenterFor(context).open(observation); + const opening = presenter.open(observation); if (typeof opening === 'string') return [...parts, opening].join('\n\n'); return [ { diff --git a/packages/core/src/agent/primitives.ts b/packages/core/src/agent/primitives.ts index 003371796..318c5271f 100644 --- a/packages/core/src/agent/primitives.ts +++ b/packages/core/src/agent/primitives.ts @@ -24,7 +24,7 @@ import { cacheTokenFields, readCost } from './model/sdk.ts'; import { OperationQueue } from './operation-queue.ts'; import { imagePointToViewport, POINT_VERBS } from './point-tap.ts'; import { SAMPLE_NAMES, SAMPLE_PREFIX } from './sample-files.ts'; -import { ScreenPresenter, type ScreenOutput, type ScreenUpdateOptions } from './screen-update.ts'; +import { ScreenPresenter, boundPresenter, type ScreenOutput, type ScreenUpdateOptions } from './screen-update.ts'; /** Codes the model may pick when concluding; runtime codes are runtime-assigned. */ const MODEL_ERROR_CODES = [ @@ -227,9 +227,9 @@ interface GrammarToolSetup extends GrammarToolOptions { /** * Renders screens for the model and remembers what it has seen, so every * result after the first reports the changes rather than the whole tree. - * Shared with the opening prompt by the built-in agent and with the session - * by `e2e mcp`; a fresh one per step otherwise, so the first look - * (`observe`) returns the whole screen. + * Passed by `e2e mcp` to share with the session; without one, the loop + * run's presenter when built inside a loop, else a fresh presenter of its + * own. */ readonly screen?: ScreenPresenter; /** @@ -898,7 +898,7 @@ function createPointTools( /** * What every verb runs through: the loop guard, the - * step's presenter, one queue for the tool bodies (an action and its look + * loop run's presenter, one queue for the tool bodies (an action and its look * at the result run together, so a batched turn gets one coherent result * per action), and the two moves each verb makes: present the screen after * an action, and perform an action then present. A failed action still @@ -912,7 +912,10 @@ function createPointTools( */ function verbKit(context: StepExecutorContext, options: GrammarToolSetup) { const guard = options.guard ?? ((body: () => Promise) => body()); - const screen: ScreenPresenter = options.screen ?? new ScreenPresenter(); + // No presenter of its own: the loop run's when built inside one, else a + // fresh presenter per toolset, so tools used outside a loop never share + // screen state with a run or with each other. + const screen: ScreenPresenter = options.screen ?? boundPresenter(context) ?? new ScreenPresenter(); const queue = new OperationQueue(); const inOrder = (body: () => Promise): Promise => queue.run(body); /** The screen after an action: the changes since the one the model holds and, once the step shows pixels, a fresh screenshot. */ diff --git a/packages/core/src/agent/screen-update.ts b/packages/core/src/agent/screen-update.ts index d46cc9550..a62c5a0df 100644 --- a/packages/core/src/agent/screen-update.ts +++ b/packages/core/src/agent/screen-update.ts @@ -6,7 +6,13 @@ * the lines that appeared, the lines that changed, the lines that went away. * Node ids are stable for as long as an element exists, so a line the model * saw earlier still names the same element, and a diff is a complete update - * rather than a hint. A screen that changed mostly goes out whole again. + * rather than a hint. A screen that changed mostly goes out whole again, + * with large subtrees the model already holds folded to one line each, + * naming the revision that last listed them in full, so a persistent sidebar + * is not repeated on every page transition. A fold line never names a + * revision that only reported the subtree as a diff, or that itself folded + * it: the presenter remembers, per subtree, the revision that last listed + * it whole and what it read then. * * A screen may carry its pixels too. The presenter is where a screenshot * joins the text: it notes the image's size and coordinate space under the @@ -19,7 +25,8 @@ */ import type { ToolResultPart } from 'ai'; -import type { ExecutorObservation, ExecutorPixels } from './executor.ts'; +import type { ExecutorObservation, ExecutorPixels, StepExecutorContext } from './executor.ts'; +import { sha256Hex } from '../internal/ids.ts'; import { interactiveNodeCount, pixelDigest } from './observation.ts'; import type { VisionDegradation } from '../run/steps.ts'; @@ -29,9 +36,18 @@ const MAX_DIFF_LINES = 60; /** A diff touching more than this share of the new screen goes out whole. */ const MAX_DIFF_SHARE = 0.5; +/** A subtree folds only when it holds at least this many nodes below its root. */ +const MIN_FOLD_NODES = 10; + /** Every full screen is introduced by this phrase; the transcript elision looks for it. */ export const FULL_SCREEN_PATTERN = /Current screen \(revision /; +/** A full screen's heading with its revision: `Current screen (revision b1, ...`; the transcript elision reads it. */ +export const SCREEN_REVISION_PATTERN = /Current screen \(revision ([^,)]+),/; + +/** A fold line's dependency: `(12 nodes inside, unchanged since revision b1)`; the transcript elision looks for it. No `/g`, so a later `.test()` or `.exec()` call is not stateful; a caller that needs every match builds its own global copy. */ +export const FOLD_DEPENDENCY_PATTERN = /\(\d+ nodes inside, unchanged since revision ([^)]+)\)/; + /** The screen as the model last received it, indexed for comparison. */ interface ShownScreen { readonly revision: string; @@ -50,6 +66,13 @@ interface ShownScreen { /** A node line whose role is the on-screen keyboard, as device trees list it. */ const KEYBOARD_LINE = /^#\S+ keyboard\b/; +/** A subtree as the model last saw it whole: the revision that listed it in full, and what it read then. */ +interface FullyShownSubtree { + readonly revision: string; + /** SHA-256 of the subtree's node lines, focus stripped, in document order. */ + readonly digest: string; +} + /** * What the model needs to hear when an action closed the keyboard: on a * touch screen the tap that closes it is often spent on closing it, so the @@ -116,6 +139,8 @@ export class ScreenPresenter { private screenshotDigest: string | undefined; /** True once the engine said it captures no screenshots at all. */ private pixelsDenied = false; + /** The revision that last listed each subtree in full, by its root node id, and what it read then. */ + private fullyShown = new Map(); /** True once a screenshot went to the model in this step. */ get showingPixels(): boolean { @@ -130,7 +155,63 @@ export class ScreenPresenter { /** The step's first screen, whole. */ initial(observation: ExecutorObservation): string { this.shown = observation.treeUnavailable === true ? undefined : indexScreen(observation); - return renderFull(observation); + return this.renderWhole(observation); + } + + /** + * A whole screen with nothing to fold against, and the record that it + * listed every node in full at its revision. A truncated listing records + * nothing: nodes past the cut were never shown. + */ + private renderWhole(observation: ExecutorObservation): string { + const rendered = renderFull(observation); + this.noteListed(rendered.listed, rendered.folded); + return rendered.text; + } + + /** + * Forgets every subtree the model last saw whole. The overflow retry cuts + * the screens those records name, so the next full render lists everything + * in full instead of folding against text the model no longer holds. + */ + clearFoldRecords(): void { + this.fullyShown.clear(); + } + + /** + * Records the subtrees a full render listed in full or folded, and drops + * every other record. Called with no map by a render that listed nothing + * in full, a diff or an unchanged screen, which only prunes: a subtree + * the model last saw whole stays on record until a node in it leaves the + * screen. A full render replaces the record with exactly what it showed: + * a subtree it neither listed nor folded is not on record afterwards, so + * a later fold can never name a screen this render did not show. A + * truncated render lists and folds nothing, so it clears the record + * entirely: the screens the old records named may already be elided from + * the transcript, and a fold naming one would point at content the model + * no longer holds. + */ + private noteListed(listed?: ReadonlyMap, folded?: ReadonlySet): void { + const shown = this.shown; + if (shown === undefined) { + this.fullyShown.clear(); + return; + } + if (listed === undefined) { + for (const id of this.fullyShown.keys()) { + if (!shown.byId.has(id)) this.fullyShown.delete(id); + } + return; + } + const next = new Map(); + for (const id of folded ?? []) { + const kept = this.fullyShown.get(id); + if (kept !== undefined) next.set(id, kept); + } + for (const [id, digest] of listed) { + next.set(id, { revision: shown.revision, digest }); + } + this.fullyShown = next; } /** @@ -150,11 +231,11 @@ export class ScreenPresenter { const previous = this.shown; const next = indexScreen(observation); this.shown = next; - if (previous === undefined) return `${lead}${renderFull(observation)}`; + if (previous === undefined) return `${lead}${this.renderWhole(observation)}`; const closed = options.keyboardNote === false || options.lead === undefined ? '' : keyboardClosedNote(previous, next); const tabs = tabNote(previous.tabs, next.tabs); // Another tab is another document: its ids say nothing about the tab the model held. - if (tabs?.moved === true) return `${lead}${tabs.note} ${renderFull(observation)}${closed}`; + if (tabs?.moved === true) return `${lead}${tabs.note} ${this.renderWhole(observation)}${closed}`; // A tab opened is the action's effect, so an unchanged screen blames no control. const expectChange = options.expectChange === true && tabs === undefined; if (tabs !== undefined) lead = `${lead}${tabs.note} `; @@ -162,14 +243,18 @@ export class ScreenPresenter { const changes = diff.length; // A truncated screen is never called unchanged: what it left out is unknown. if (changes === 0 && !observation.truncated) { + this.noteListed(); return `${lead}${renderUnchanged(previous.revision, observation, expectChange, pixels)}${closed}`; } if (changes >= MAX_DIFF_LINES || changes > MAX_DIFF_SHARE * next.order.length) { - return `${lead}The screen changed substantially since revision ${previous.revision}. ${renderFull(observation)}${closed}`; + const rendered = renderFull(observation, this.fullyShown); + this.noteListed(rendered.listed, rendered.folded); + return `${lead}The screen changed substantially since revision ${previous.revision}. ${rendered.text}${closed}`; } const assurance = observation.truncated ? 'The screen listing is truncated: nodes past the cut are not listed and none is reported removed; every listed node keeps the id you have.' : 'Every node not listed as removed is still on screen under the id you have.'; + this.noteListed(); return [ `${lead}Screen changes since revision ${previous.revision} (now revision ${observation.revision}${describeLocation(observation)}, ${String(next.nodes)} nodes): ${describeCounts(diff)}. ${assurance}`, ...diff, @@ -238,6 +323,61 @@ export class ScreenPresenter { } } +/** + * One presenter per loop run, stacked on the step's context for the run, so + * the presenter state matches that run's transcript alone: two loops run + * against one step context each see their own first screen, and a run nested + * inside another run's tool reads its own. Tools built outside a run read + * the top binding while one is in progress, else use a presenter of their + * own. Keyed by the step's context object, so a run's presenter dies with it. + */ +const presenterStacks = new WeakMap(); + +/** + * The loop run's presenter, or a fresh presenter of its own when no run + * bound one: the opening prompt inside a run reads the run's presenter, and + * a prompt built outside a run never shares screen state with one. + */ +export function presenterFor(context: StepExecutorContext): ScreenPresenter { + return boundPresenter(context) ?? new ScreenPresenter(); +} + +/** The presenter the innermost loop run bound to the step's context, if one bound one: reads only, never creates. */ +export function boundPresenter(context: StepExecutorContext): ScreenPresenter | undefined { + return presenterStacks.get(context)?.at(-1); +} + +/** + * Binds a fresh presenter to the step's context, pushed onto the stack above + * whatever runs already bound, and returns it. A loop run binds one at the + * start of its run, so each run starts from a screen its own model has seen, + * and removes its own presenter when the run ends. `presenterFor` stays the + * fallback for the opening prompt inside a run. + */ +export function bindPresenter(context: StepExecutorContext): ScreenPresenter { + const presenter = new ScreenPresenter(); + const stack = presenterStacks.get(context); + if (stack === undefined) presenterStacks.set(context, [presenter]); + else stack.push(presenter); + return presenter; +} + +/** + * Removes the loop run's own presenter from the stack when the run ends, by + * identity: a run that ends while another run is still bound above it leaves + * that run's presenter in place, and a run that ends after an overlapping + * run never puts back a presenter whose run is over. Tools built outside a + * loop then find no binding and use a presenter of their own. + */ +export function restorePresenter(context: StepExecutorContext, presenter: ScreenPresenter): void { + const stack = presenterStacks.get(context); + if (stack === undefined) return; + const index = stack.lastIndexOf(presenter); + if (index === -1) return; + stack.splice(index, 1); + if (stack.length === 0) presenterStacks.delete(context); +} + /** The line under a screenshot: its pixel size and the coordinate space the point verbs read. */ function screenshotNote(pixels: Pick): string { return `Screenshot attached: ${String(pixels.width)} by ${String(pixels.height)} pixels${ @@ -266,13 +406,215 @@ function describeCounts(diff: readonly string[]): string { return counts === '' ? 'no listed node changed' : counts; } -/** One full screen, headed so the elision can find it later. */ -function renderFull(observation: ExecutorObservation): string { +/** A rendered full screen, with the subtrees it listed in full and what each read, and the subtrees it folded. */ +interface RenderedFullScreen { + readonly text: string; + /** Node ids whose subtree the rendering listed in full, each with the digest of its node lines. */ + readonly listed: ReadonlyMap; + /** Node ids whose subtree the rendering folded, keeping the record of the revision that last listed it in full. */ + readonly folded: ReadonlySet; +} + +/** One full screen, headed so the elision can find it later. A screen that + * follows one the model holds folds its large unchanged subtrees: the root + * line stays, with the count of the nodes inside and the revision that last + * listed them in full, and the nodes inside are left out. The first screen of a + * step has nothing to fold against, and a truncated listing never folds, + * since the nodes past the cut are unknown. */ +function renderFull( + observation: ExecutorObservation, + fullyShown?: ReadonlyMap, +): RenderedFullScreen { if (observation.treeUnavailable === true) { - return `Current screen (revision ${observation.revision}${describeLocation(observation)}, semantic capture unavailable):\n${observation.text}`; + return { + text: `Current screen (revision ${observation.revision}${describeLocation(observation)}, semantic capture unavailable):\n${observation.text}`, + listed: new Map(), + folded: new Set(), + }; } const nodes = indexScreen(observation).nodes; - return `Current screen (revision ${observation.revision}${describeLocation(observation)}, ${String(nodes)} nodes):\n${observation.text}`; + const head = `Current screen (revision ${observation.revision}${describeLocation(observation)}, ${String(nodes)} nodes):\n`; + if (fullyShown === undefined || observation.truncated) { + const entries = treeEntries(observation); + return { + text: `${head}${observation.text}`, + listed: observation.truncated ? new Map() : subtreeDigests(entries, spanEnds(entries)), + folded: new Set(), + }; + } + const folded = foldUnchangedSubtrees(observation, fullyShown); + return { text: `${head}${folded.text}`, listed: folded.listed, folded: folded.folded }; +} + +/** A node line of a screen listing, with its depth and id. */ +interface TreeEntry { + readonly raw: string; + readonly indent: number; + readonly id: string | undefined; +} + +/** A screen listing split into lines, each with its depth and node id. */ +function treeEntries(observation: ExecutorObservation): TreeEntry[] { + return observation.text.split('\n').map((raw) => ({ + raw, + indent: indentOf(raw), + id: lineId(raw), + })); +} + +/** + * The end of every node's span: the first line after it that is not one of + * its descendants, a marker line with no node id or a line at its depth or + * above. A marker line closes every open span: no subtree reaches past one. + */ +function spanEnds(entries: readonly TreeEntry[]): number[] { + const ends = Array.from({ length: entries.length }, () => entries.length); + const open: number[] = []; + for (let index = 0; index < entries.length; index += 1) { + const entry = entries[index]!; + if (entry.id === undefined) { + while (open.length > 0) ends[open.pop()!] = index; + continue; + } + while (open.length > 0 && entries[open[open.length - 1]!]!.indent >= entry.indent) { + ends[open.pop()!] = index; + } + open.push(index); + } + return ends; +} + +/** + * The digest of every subtree large enough to fold: its node lines, focus + * stripped and unindented, in document order. Focus moves with every action, + * so it is left out the way the fold's own comparison leaves it out. Lines + * are joined with a separator no node line carries, so two different subtrees + * cannot digest alike by regrouping lines. A smaller subtree never folds, so + * it needs no digest: it is listed in full whenever a screen goes out whole, + * which puts it on record at that revision if it later grows large enough. + */ +function subtreeDigests(entries: readonly TreeEntry[], ends: readonly number[]): Map { + const digests = new Map(); + for (let index = 0; index < entries.length; index += 1) { + const entry = entries[index]!; + if (entry.id === undefined) continue; + if (ends[index]! - index - 1 < MIN_FOLD_NODES) continue; + const lines: string[] = []; + for (let inner = index; inner < ends[index]!; inner += 1) { + lines.push(withoutFocus(entries[inner]!.raw).trimStart()); + } + digests.set(entry.id, sha256Hex(lines.join('\n'))); + } + return digests; +} + +/** Leading spaces of one listing line: its depth in the tree. */ +function indentOf(line: string): number { + return line.length - line.trimStart().length; +} + +/** True when the line's state bracket names the focused node. */ +function hasFocus(line: string): boolean { + const bracket = stateBracket(line); + return bracket !== undefined && bracket.states.split(' ').includes('focused'); +} + +/** + * The trailing state bracket of a line, with the line before it, or + * undefined when the line carries none. The states bracket is always the + * last token of a line. + */ +function stateBracket(line: string): { readonly states: string; readonly before: string } | undefined { + const match = / \[([^\]]*)\]$/.exec(line); + return match === null ? undefined : { states: match[1]!, before: line.slice(0, match.index) }; +} + +/** + * The screen listing with every large unchanged subtree replaced by its root + * line plus a summary, and the subtrees the listing left in full. A subtree + * folds only when it is not the document root, holds enough nodes to be + * worth the hiding, carries no focused node, and still reads as it did at + * the revision the fold line names: the digest covers its node ids and lines, + * so no comparison with the screen before it is needed. A change update in + * between may have listed it only in part, or an earlier screen may itself + * have folded it, so the last observation's revision is no proof the model + * holds it whole. A subtree that fails the digest check is listed in full + * instead, which puts it on record at this revision. + */ +function foldUnchangedSubtrees( + observation: ExecutorObservation, + fullyShown: ReadonlyMap, +): { readonly text: string; readonly listed: ReadonlyMap; readonly folded: ReadonlySet } { + const entries = treeEntries(observation); + const ends = spanEnds(entries); + const digests = subtreeDigests(entries, ends); + const foldedSpans: Array<{ readonly start: number; readonly end: number }> = []; + const foldedIds = new Set(); + const out: string[] = []; + let index = 0; + while (index < entries.length) { + const entry = entries[index]!; + if (entry.id === undefined) { + out.push(entry.raw); + index += 1; + continue; + } + const end = ends[index]!; + const inside = end - index - 1; + const shown = fullyShown.get(entry.id); + if ( + entry.indent > 0 && inside >= MIN_FOLD_NODES && + shown !== undefined && digests.get(entry.id) === shown.digest && + !subtreeHoldsFocus(entries, index, end) + ) { + out.push(`${entry.raw} (${String(inside)} nodes inside, unchanged since revision ${shown.revision})`); + foldedSpans.push({ start: index, end }); + foldedIds.add(entry.id); + index = end; + } else { + out.push(entry.raw); + index += 1; + } + } + return { text: out.join('\n'), listed: fullyListedSubtrees(entries, ends, digests, foldedSpans), folded: foldedIds }; +} + +/** True when a node in the span holds focus: the fold would hide where focus is. */ +function subtreeHoldsFocus(entries: readonly TreeEntry[], start: number, end: number): boolean { + for (let index = start; index < end; index += 1) { + if (hasFocus(entries[index]!.raw)) return true; + } + return false; +} + +/** + * The subtrees a folded listing left in full: every node whose span holds no + * line a fold hid. A folded root's own span is not fully listed, its hidden + * nodes are not, and neither is a subtree holding a folded one: the fold + * line it would name later has to be a screen that showed all of it. + */ +function fullyListedSubtrees( + entries: readonly TreeEntry[], + ends: readonly number[], + digests: ReadonlyMap, + foldedSpans: readonly { readonly start: number; readonly end: number }[], +): ReadonlyMap { + const hidden = Array.from({ length: entries.length }, () => 0); + for (const span of foldedSpans) { + for (let index = span.start + 1; index < span.end; index += 1) hidden[index] = 1; + } + const prefix = Array.from({ length: entries.length + 1 }, () => 0); + for (let index = 0; index < entries.length; index += 1) prefix[index + 1] = prefix[index]! + hidden[index]!; + const listed = new Map(); + for (let index = 0; index < entries.length; index += 1) { + const entry = entries[index]!; + if (entry.id === undefined) continue; + if (prefix[ends[index]!]! - prefix[index]! !== 0) continue; + const digest = digests.get(entry.id); + if (digest === undefined) continue; + listed.set(entry.id, digest); + } + return listed; } /** @@ -348,16 +690,15 @@ function lineId(line: string): string | undefined { } /** - * The line without its focus state. Focus moves with every action and on its + * The node lines with focus stripped. Focus moves with every action and on its * own, so a diff that reported it would drown the changes that matter and - * make a control that did nothing look like it did something. The states - * bracket is always the last token of a line. The full screen keeps focus. + * make a control that did nothing look like it did something. The full screen keeps focus. */ function withoutFocus(line: string): string { - return line.replace(/ \[([^\]]*)\]$/, (_match, states: string) => { - const kept = states.split(' ').filter((state) => state !== 'focused'); - return kept.length === 0 ? '' : ` [${kept.join(' ')}]`; - }); + const bracket = stateBracket(line); + if (bracket === undefined) return line; + const kept = bracket.states.split(' ').filter((state) => state !== 'focused'); + return kept.length === 0 ? bracket.before : `${bracket.before} [${kept.join(' ')}]`; } function indexScreen(observation: ExecutorObservation): ShownScreen { diff --git a/packages/core/src/agent/tool-loop.ts b/packages/core/src/agent/tool-loop.ts index c82999f71..5d88be793 100644 --- a/packages/core/src/agent/tool-loop.ts +++ b/packages/core/src/agent/tool-loop.ts @@ -28,7 +28,7 @@ import { } from './model/tool-choice.ts'; import { withStallGuard } from './model/stall.ts'; import { providerHints, type ProviderHints, type ProviderModelRef } from './model/provider-hints.ts'; -import { isScreenOutput, toolResultTexts } from './screen-update.ts'; +import { bindPresenter, isScreenOutput, restorePresenter, toolResultTexts, type ScreenPresenter } from './screen-update.ts'; import { shapeOf } from './message-shape.ts'; import { compactScreenHistory } from './transcript-compaction.ts'; import { AgentError, isAgentError } from './error.ts'; @@ -257,6 +257,8 @@ class LoopRun { * verdict, and the note completes its message. */ private overflowNote: string | undefined; + /** The presenter this run bound for its context, so a shrink clears this run's folds, never another run's. */ + private presenter: ScreenPresenter | undefined; /** How this step asks for tool calls; flips to `auto` when the model rejects a forced choice. */ private toolChoice: ToolChoiceMode; /** The last turn that ran, for continuing after a reply without tool calls. */ @@ -305,6 +307,22 @@ class LoopRun { } async run(): Promise { + // A fresh presenter per run, removed after it: the transcript belongs to + // this loop, not the step context. Removed by identity, so a run that ends + // while another run on this context is still going leaves that run's + // presenter in place and never puts back one whose run is over. + const presenter = bindPresenter(this.context); + this.presenter = presenter; + try { + return await this.runLoop(); + } finally { + restorePresenter(this.context, presenter); + this.presenter = undefined; + } + } + + /** The loop itself: tools, opening prompt, and model turns until a verdict. */ + private async runLoop(): Promise { const helpers = this.helpers(); const tools: ToolSet = { ...this.options.tools(this.context, helpers), @@ -565,6 +583,8 @@ class LoopRun { } this.overflowNote = ' again after the history was shrunk once'; this.turnOffset = this.turnsUsed; + // The retry may have cut the screens this run's folds name, so the next screen renders in full. + this.presenter?.clearFoldRecords(); this.note(`context overflow before turn ${String(this.turnsUsed + 1)}: history shrunk from ${String(before)} to ${String(after)} chars, retrying once`); return shrunk; } diff --git a/packages/core/src/agent/transcript-compaction.ts b/packages/core/src/agent/transcript-compaction.ts index 0f8ecba73..6b63b3427 100644 --- a/packages/core/src/agent/transcript-compaction.ts +++ b/packages/core/src/agent/transcript-compaction.ts @@ -11,7 +11,7 @@ */ import type { FilePart, ModelMessage, TextPart, ToolResultPart, UserContent } from 'ai'; -import { FULL_SCREEN_PATTERN, toolResultTexts } from './screen-update.ts'; +import { FOLD_DEPENDENCY_PATTERN, FULL_SCREEN_PATTERN, SCREEN_REVISION_PATTERN, toolResultTexts } from './screen-update.ts'; /** A part of a user message; a string message counts as one text part. */ type UserPart = Exclude[number]; @@ -86,9 +86,22 @@ function rewriteOldestParts(messages: ModelMessage[], count: number, rewrite: Pa * How many of the newest full screens stay verbatim in the transcript. One: * a whole new screen means the page changed mostly, and the screen it * replaced is dead weight on every later turn. Change updates never elide. + * A screen a surviving screen's fold lines depend on stays too: a folded + * line names the revision that last listed its subtree in full, and that + * screen is the only place the model can still read it. */ const FULL_SCREEN_PRESERVE_COUNT = 1; +/** The revision a full screen's heading carries, or undefined when it carries none. */ +function screenRevision(text: string): string | undefined { + return SCREEN_REVISION_PATTERN.exec(text)?.[1]; +} + +/** The revisions a full screen's fold lines depend on. The exported pattern carries no `/g`, so the global copy for `matchAll` is built here. */ +function foldDependencies(text: string): string[] { + return [...text.matchAll(new RegExp(FOLD_DEPENDENCY_PATTERN, 'g'))].map((match) => match[1]!); +} + /** * Bytes of superseded full screens a transcript carries before they are * elided. Re-reading a stale screen from the cache costs a tenth of sending @@ -104,23 +117,56 @@ export interface CompactScreenHistoryOptions { /** * Elides full screens the transcript no longer needs: every full screen but - * the newest few is reduced to its lead and a notice, in the opening prompt - * and in tool results alike, once the stale screens together outgrow the - * budget. Diffs are never touched. Returns the input array unchanged when - * nothing qualifies, so the caller can skip the override. + * the newest is reduced to its lead and a notice, in the opening prompt + * and in tool results alike, once the screens that may be elided together + * outgrow the budget. Diffs are never touched. A screen a surviving screen's + * fold lines depend on is never elided, with the screens it in turn depends + * on: the fold line names the revision that last listed its subtree in full, + * and eliding it would leave the model a pointer to content it no longer + * holds. Returns the input array unchanged when nothing qualifies, so the + * caller can skip the override. */ export function compactScreenHistory( messages: ModelMessage[], options: CompactScreenHistoryOptions = {}, ): ModelMessage[] { - const screens = transcriptParts(messages).map(fullScreenText).filter((text) => text !== undefined); - const stale = screens.length - FULL_SCREEN_PRESERVE_COUNT; - if (stale <= 0) return messages; - const staleBytes = screens.slice(0, stale).reduce((bytes, text) => bytes + Buffer.byteLength(text, 'utf8'), 0); + const screens = transcriptParts(messages) + .map(fullScreenText) + .filter((text) => text !== undefined) + .map((text) => ({ text, revision: screenRevision(text) })); + const preserved = new Set(); + for (let at = Math.max(0, screens.length - FULL_SCREEN_PRESERVE_COUNT); at < screens.length; at += 1) { + preserved.add(at); + } + // Follow each preserved screen's fold lines to the screens they depend on, + // transitively: a preserved screen's fold may name a screen that itself + // folded another subtree, and only that screen lists it in full. + const queue = [...preserved]; + while (queue.length > 0) { + const index = queue.pop()!; + for (const dependency of foldDependencies(screens[index]!.text)) { + screens.forEach((screen, at) => { + if (screen.revision === dependency && !preserved.has(at)) { + preserved.add(at); + queue.push(at); + } + }); + } + } + const elidable = screens.filter((_, index) => !preserved.has(index)); + if (elidable.length === 0) return messages; + const staleBytes = elidable.reduce((bytes, screen) => bytes + Buffer.byteLength(screen.text, 'utf8'), 0); if (staleBytes <= (options.keepStaleBytes ?? KEEP_STALE_SCREEN_BYTES)) return messages; - return rewriteOldestParts(messages, stale, { - user: (part) => (part.type === 'text' && isFullScreen(part.text) ? { ...part, text: elideScreen(part.text) } : undefined), - tool: elideResultScreen, + const preservedRevisions = new Set( + [...preserved].map((index) => screens[index]!.revision).filter((revision) => revision !== undefined), + ); + const keep = (text: string): boolean => { + const revision = screenRevision(text); + return revision !== undefined && preservedRevisions.has(revision); + }; + return rewriteOldestParts(messages, elidable.length, { + user: (part) => (part.type === 'text' && isFullScreen(part.text) && !keep(part.text) ? { ...part, text: elideScreen(part.text) } : undefined), + tool: (part) => elideResultScreen(part, keep), }); } @@ -132,19 +178,21 @@ function fullScreenText(part: UserPart | ToolPart): string | undefined { /** * The tool result with its full screen elided, or undefined when it carries - * none. A result with a screenshot attached loses only the screen text and - * keeps its other items: the image, or the notice that replaced it. + * none or the screen is one a surviving screen depends on. A result with a + * screenshot attached loses only the screen text and keeps its other items: + * the image, or the notice that replaced it. */ -function elideResultScreen(part: ToolPart): ToolPart | undefined { +function elideResultScreen(part: ToolPart, keep: (text: string) => boolean): ToolPart | undefined { if (part.type !== 'tool-result') return undefined; const { output } = part; if (output.type === 'text') { - return isFullScreen(output.value) ? { ...part, output: { ...output, value: elideScreen(output.value) } } : undefined; + return isFullScreen(output.value) && !keep(output.value) ? { ...part, output: { ...output, value: elideScreen(output.value) } } : undefined; } if (output.type !== 'content' || !output.value.some(isScreenItem)) return undefined; + if (output.value.every((item) => !isScreenItem(item) || keep(item.text))) return undefined; return { ...part, - output: { ...output, value: output.value.map((item) => (isScreenItem(item) ? { ...item, text: elideScreen(item.text) } : item)) }, + output: { ...output, value: output.value.map((item) => (isScreenItem(item) && !keep(item.text) ? { ...item, text: elideScreen(item.text) } : item)) }, }; } diff --git a/packages/e2e/tests/unit/agent-screen-update.test.ts b/packages/e2e/tests/unit/agent-screen-update.test.ts index 8038b7b81..facc4e9fd 100644 --- a/packages/e2e/tests/unit/agent-screen-update.test.ts +++ b/packages/e2e/tests/unit/agent-screen-update.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from 'vitest'; +import type { ModelMessage, ToolResultPart } from 'ai'; import type { ExecutorObservation } from '@e2e-dev/core/lib/agent/executor'; -import { ScreenPresenter } from '@e2e-dev/core/lib/agent/screen-update'; +import { FOLD_DEPENDENCY_PATTERN, ScreenPresenter } from '@e2e-dev/core/lib/agent/screen-update'; +import { compactScreenHistory } from '@e2e-dev/core/lib/agent/transcript-compaction'; function screen(revision: string, lines: readonly string[], extra: Partial = {}): ExecutorObservation { return { @@ -21,6 +23,22 @@ const HOME = [ ' #n6 textbox "Email"', ]; +/** One tool result carrying a rendered screen as its text. */ +function toolResult(id: string, text: string): ModelMessage { + return { + role: 'tool', + content: [{ type: 'tool-result', toolCallId: id, toolName: 'tap', output: { type: 'text', value: text } }], + }; +} + +/** The text of a tool message's one text result. */ +function toolText(message: ModelMessage): string { + const [part] = message.content as ToolResultPart[]; + const output = part!.output; + if (output.type !== 'text') throw new Error('expected a text tool result'); + return output.value; +} + describe('ScreenPresenter', () => { it('retires semantic history through repeated unavailable captures and shows recovery whole', () => { const presenter = new ScreenPresenter(); @@ -236,6 +254,339 @@ describe('ScreenPresenter', () => { }); }); +describe('ScreenPresenter folds large unchanged subtrees in the full-screen fallback', () => { + /** A sidebar of `count` links under a navigation node, ids #n3 and up. */ + function sidebar(count: number): string[] { + return [ + ' #n2 navigation "Sidebar"', + ...Array.from({ length: count }, (_, i) => ` #n${String(i + 3)} link "Item ${String(i + 1)}"`), + ]; + } + + const MAIN_LIST = [ + ' #n20 main "Order list"', + ' #n21 heading "Orders"', + ' #n22 button "Order 1"', + ' #n23 button "Order 2"', + ]; + + const MAIN_DETAILS = [ + ' #n20 main "Order details"', + ' #n30 heading "Order 42"', + ' #n31 text "Total $12"', + ' #n32 text "Placed yesterday"', + ' #n33 button "Refund"', + ' #n34 button "Ship"', + ' #n35 button "Print"', + ' #n36 button "Archive"', + ' #n37 link "Back to orders"', + ]; + + const MAIN_SEARCH = [ + ' #n40 main "Order search"', + ' #n41 heading "Find an order"', + ' #n42 textbox "Order number"', + ' #n43 button "Search"', + ' #n44 text "No order selected"', + ' #n45 button "Clear"', + ' #n46 link "All orders"', + ' #n47 text "Search by number or customer"', + ' #n48 button "Help"', + ]; + + function orderScreen(main: readonly string[], sidebarCount: number): string[] { + return ['#n1 document "Orders"', ...sidebar(sidebarCount), ...main]; + } + + it('folds a large unchanged sidebar, and lists the changed main in full', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const text = presenter.update(screen('b2', orderScreen(MAIN_DETAILS, 12), { path: '/orders' }), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b1.'); + expect(text).toContain('Current screen (revision b2, path /orders, 23 nodes):'); + expect(text).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); + expect(text).not.toContain('#n3 link "Item 1"'); + expect(text).not.toContain('#n14 link "Item 12"'); + expect(text).toContain(' #n30 heading "Order 42"'); + expect(text).toContain(' #n37 link "Back to orders"'); + }); + + it('keeps a small unchanged subtree listed in full', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 3))); + const text = presenter.update(screen('b2', orderScreen(MAIN_DETAILS, 3)), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b1.'); + expect(text).not.toContain('nodes inside, unchanged since revision'); + expect(text).toContain(' #n3 link "Item 1"'); + expect(text).toContain(' #n30 heading "Order 42"'); + }); + + it('folds an unchanged group inside a changed sidebar, and lists the changed sibling', () => { + const presenter = new ScreenPresenter(); + const before = [ + '#n1 document "Orders"', + ' #n2 navigation "Sidebar"', + ' #n3 list "Sections"', + ...Array.from({ length: 12 }, (_, i) => ` #n${String(i + 40)} link "Section ${String(i + 1)}"`), + ' #n60 link "Settings"', + ...MAIN_LIST, + ]; + presenter.initial(screen('b1', before)); + const after = before.map((line) => (line.includes('#n60 ') ? ' #n60 link "Settings (2)"' : line)); + after.splice(after.indexOf(' #n20 main "Order list"'), 4, ...MAIN_DETAILS); + const text = presenter.update(screen('b2', after), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b1.'); + expect(text).toContain(' #n3 list "Sections" (12 nodes inside, unchanged since revision b1)'); + expect(text).not.toContain('#n40 link "Section 1"'); + expect(text).toContain(' #n60 link "Settings (2)"'); + expect(text).not.toContain('navigation "Sidebar" ('); + }); + + it('does not fold a subtree that lost a node, even when the rest is unchanged', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const after = orderScreen(MAIN_DETAILS, 12).filter((line) => !line.includes('#n5 ')); + const text = presenter.update(screen('b2', after), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b1.'); + expect(text).not.toContain('nodes inside, unchanged since revision'); + expect(text).toContain(' #n3 link "Item 1"'); + }); + + it('does not fold a subtree holding the focused node', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const after = orderScreen(MAIN_DETAILS, 12).map((line) => + (line.includes('#n8 ') ? ' #n8 link "Item 6" [focused]' : line), + ); + const text = presenter.update(screen('b2', after), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b1.'); + expect(text).not.toContain('nodes inside, unchanged since revision'); + expect(text).toContain('#n8 link "Item 6" [focused]'); + }); + + it('does not fold on a truncated listing, whose cut nodes are unknown', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + // Twelve more added nodes, so the change still trips the whole-screen fallback + // even though a truncated listing reports no removals. + const extra = Array.from({ length: 12 }, (_, i) => ` #n${String(i + 40)} button "Extra ${String(i + 1)}"`); + const after = [...orderScreen(MAIN_DETAILS, 12), ...extra]; + const text = presenter.update(screen('b2', after, { truncated: true }), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b1.'); + expect(text).not.toContain('nodes inside, unchanged since revision'); + expect(text).toContain(' #n3 link "Item 1"'); + }); + + it('does not let a fold swallow lines past a marker line inside the subtree', () => { + const presenter = new ScreenPresenter(); + // A marker line with no node id sits inside the sidebar, between its + // eleventh and twelfth links: it closes the sidebar's span there. + const sidebarWithMarker = [ + ' #n2 navigation "Sidebar"', + ...Array.from({ length: 11 }, (_, i) => ` #n${String(i + 3)} link "Item ${String(i + 1)}"`), + ' [End of visible items]', + ' #n14 link "Item 12"', + ]; + presenter.initial(screen('b1', ['#n1 document "Orders"', ...sidebarWithMarker, ...MAIN_LIST])); + const text = presenter.update( + screen('b2', ['#n1 document "Orders"', ...sidebarWithMarker, ...MAIN_DETAILS]), + { lead: 'Tapped #n22.' }, + ); + expect(text).toContain('The screen changed substantially since revision b1.'); + // The fold covers only the eleven links before the marker. + expect(text).toContain(' #n2 navigation "Sidebar" (11 nodes inside, unchanged since revision b1)'); + expect(text).not.toContain('#n3 link "Item 1"'); + // The marker and the link past it are still listed in full. + expect(text).toContain(' [End of visible items]'); + expect(text).toContain(' #n14 link "Item 12"'); + }); + + it('sends the first screen whole, with no folding', () => { + const presenter = new ScreenPresenter(); + const text = presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + expect(text).toContain('Current screen (revision b1, 18 nodes):'); + expect(text).not.toContain('nodes inside, unchanged since revision'); + expect(text).toContain(' #n3 link "Item 1"'); + expect(text).toContain(' #n14 link "Item 12"'); + }); + + it('names the revision that last listed the subtree in full, not an intervening diff', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + // The sidebar is not listed at b2: the diff names only the changed heading. + const changed = orderScreen(MAIN_LIST, 12).map((line) => + (line.includes('#n21 ') ? ' #n21 heading "Orders (2)"' : line), + ); + const diff = presenter.update(screen('b2', changed)); + expect(diff).toContain('changed #n21 heading "Orders (2)"'); + expect(diff).not.toContain('Current screen'); + const text = presenter.update(screen('b3', orderScreen(MAIN_DETAILS, 12)), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b2.'); + expect(text).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); + expect(text).not.toContain('unchanged since revision b2'); + }); + + it('names the original revision when the previous screen itself folded the subtree', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const first = presenter.update(screen('b2', orderScreen(MAIN_DETAILS, 12)), { lead: 'Tapped #n22.' }); + expect(first).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); + const second = presenter.update(screen('b3', orderScreen(MAIN_SEARCH, 12)), { lead: 'Tapped #n43.' }); + expect(second).toContain('The screen changed substantially since revision b2.'); + expect(second).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); + expect(second).not.toContain('unchanged since revision b2'); + }); + + it('lists a changed subtree in full again, then folds it against that listing', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const renamed = orderScreen(MAIN_LIST, 12).map((line) => + (line.includes('#n5 ') ? ' #n5 link "Item 3 (new)"' : line), + ); + expect(presenter.update(screen('b2', renamed))).toContain('changed #n5 link "Item 3 (new)"'); + // The sidebar changed at b2, so b3 may not fold it against b1: the model + // holds the old item text there, not the new one. + const rename = (lines: readonly string[]): string[] => + lines.map((line) => (line.includes('#n5 ') ? ' #n5 link "Item 3 (new)"' : line)); + const relisted = presenter.update(screen('b3', rename(orderScreen(MAIN_DETAILS, 12))), { lead: 'Tapped #n22.' }); + expect(relisted).toContain('The screen changed substantially since revision b2.'); + expect(relisted).not.toContain('nodes inside, unchanged since revision'); + expect(relisted).toContain(' #n5 link "Item 3 (new)"'); + const folded = presenter.update(screen('b4', rename(orderScreen(MAIN_SEARCH, 12))), { lead: 'Tapped #n43.' }); + expect(folded).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b3)'); + expect(folded).not.toContain('unchanged since revision b1'); + }); + + it('still folds against the original revision when a change was undone', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const renamed = orderScreen(MAIN_LIST, 12).map((line) => + (line.includes('#n5 ') ? ' #n5 link "Item 3 (new)"' : line), + ); + presenter.update(screen('b2', renamed)); + presenter.update(screen('b3', orderScreen(MAIN_LIST, 12))); + const text = presenter.update(screen('b4', orderScreen(MAIN_DETAILS, 12)), { lead: 'Tapped #n22.' }); + expect(text).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); + }); + + it('folds against the original listing when an intervening change was undone, even though the previous screen differs', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const renamed = orderScreen(MAIN_LIST, 12).map((line) => + (line.includes('#n5 ') ? ' #n5 link "Item 3 (new)"' : line), + ); + expect(presenter.update(screen('b2', renamed))).toContain('changed #n5 link "Item 3 (new)"'); + // b3 changes most of the screen and puts the sidebar back as b1 listed + // it: the fold names b1, since the digest is what the model holds, not + // the screen before it. + const text = presenter.update(screen('b3', orderScreen(MAIN_DETAILS, 12)), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b2.'); + expect(text).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); + }); + + it('a truncated full screen clears the fold record, so the next fallback lists the subtree in full instead of naming an elided screen', () => { + const presenter = new ScreenPresenter(); + const b1 = presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const b2 = presenter.update(screen('b2', orderScreen(MAIN_DETAILS, 12)), { lead: 'Tapped #n22.' }); + expect(b2).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); + // b3 is a truncated full screen: it records nothing and folds nothing. + const extra = Array.from({ length: 12 }, (_, i) => ` #n${String(i + 60)} button "Extra ${String(i + 1)}"`); + const b3 = presenter.update(screen('b3', [...orderScreen(MAIN_SEARCH, 12), ...extra], { truncated: true }), { lead: 'Tapped #n43.' }); + expect(b3).toContain('The screen changed substantially since revision b2.'); + expect(b3).not.toContain('nodes inside, unchanged since revision'); + // By the time b4 renders, compaction has already elided b1 and b2: the + // truncated screen carries no fold, so nothing preserves them. + const messages = [b1, b2, b3].map((text, index) => toolResult(`c${String(index)}`, text)); + const compacted = compactScreenHistory(messages, { keepStaleBytes: 0 }); + const b1Text = toolText(compacted[0]!); + expect(b1Text).toContain('[earlier screen elided'); + expect(b1Text).not.toContain('#n3 link "Item 1"'); + // The record b3 cleared means b4 cannot fold against b1: the sidebar is + // listed in full again instead of naming a screen the model no longer holds. + const b4 = presenter.update(screen('b4', orderScreen(MAIN_DETAILS, 12)), { lead: 'Tapped #n22.' }); + expect(b4).toContain('The screen changed substantially since revision b3.'); + expect(b4).not.toContain('nodes inside, unchanged since revision'); + expect(b4).toContain(' #n3 link "Item 1"'); + expect(b4).toContain(' #n14 link "Item 12"'); + }); + + it('lists the subtree in full again after the fold records are cleared', () => { + const presenter = new ScreenPresenter(); + presenter.initial(screen('b1', orderScreen(MAIN_LIST, 12))); + const folded = presenter.update(screen('b2', orderScreen(MAIN_DETAILS, 12)), { lead: 'Tapped #n22.' }); + expect(folded).toContain(' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'); + presenter.clearFoldRecords(); + const text = presenter.update(screen('b3', orderScreen(MAIN_SEARCH, 12)), { lead: 'Tapped #n43.' }); + expect(text).toContain('The screen changed substantially since revision b2.'); + expect(text).not.toContain('nodes inside, unchanged since revision'); + expect(text).toContain(' #n3 link "Item 1"'); + expect(text).toContain(' #n14 link "Item 12"'); + }); + + it('drops the fold record when a subtree leaves the screen, so a re-added subtree is listed in full', () => { + const presenter = new ScreenPresenter(); + const extras = [ + ' #n50 group "Extras"', + ...Array.from({ length: 24 }, (_, i) => ` #n${String(60 + i)} button "Extra ${String(i + 1)}"`), + ]; + const wide = (main: readonly string[], sidebarCount: number): string[] => + ['#n1 document "Orders"', ...sidebar(sidebarCount), ...main, ...extras]; + presenter.initial(screen('b1', wide(MAIN_LIST, 12))); + // The sidebar leaves in a change update: 13 removed lines of 43 nodes. + const removed = presenter.update(screen('b2', ['#n1 document "Orders"', ...MAIN_LIST, ...extras])); + expect(removed).toContain('removed #n2 navigation "Sidebar"'); + expect(removed).not.toContain('Current screen'); + const readded = presenter.update(screen('b3', wide(MAIN_LIST, 12))); + expect(readded).toContain('added #n2 navigation "Sidebar"'); + expect(readded).not.toContain('Current screen'); + // The record was dropped when the sidebar left, so the fallback lists it + // in full even though it reads exactly as b1 listed it. + const renamedExtras = extras.map((line, index) => + (index > 0 ? line.replace(/Extra (\d+)/, 'Extra $1 (new)') : line), + ); + const text = presenter.update(screen('b4', ['#n1 document "Orders"', ...sidebar(12), ...MAIN_DETAILS, ...renamedExtras]), { lead: 'Tapped #n22.' }); + expect(text).toContain('The screen changed substantially since revision b3.'); + expect(text).not.toContain('nodes inside, unchanged since revision'); + expect(text).toContain(' #n3 link "Item 1"'); + }); + + it('exports a fold dependency pattern a repeated .test() can reuse', () => { + const line = ' #n2 navigation "Sidebar" (12 nodes inside, unchanged since revision b1)'; + expect(FOLD_DEPENDENCY_PATTERN.test(line)).toBe(true); + expect(FOLD_DEPENDENCY_PATTERN.test(line)).toBe(true); + expect(FOLD_DEPENDENCY_PATTERN.exec(line)?.[1]).toBe('b1'); + expect(FOLD_DEPENDENCY_PATTERN.exec(line)?.[1]).toBe('b1'); + }); + + it('never folds a top-level root, even one holding enough unchanged nodes', () => { + const presenter = new ScreenPresenter(); + // A second top-level root with its leaves directly under it: the root + // itself is the only node that could fold, and its indent is 0. + const second = [ + '#n50 document "Second"', + ...Array.from({ length: 12 }, (_, i) => ` #n${String(60 + i)} link "Second ${String(i + 1)}"`), + ]; + const first = [ + '#n1 document "First"', + ' #n2 heading "First"', + ' #n3 button "First 1"', + ' #n4 button "First 2"', + ' #n5 button "First 3"', + ]; + presenter.initial(screen('b1', [...first, ...second])); + const changedFirst = [ + '#n1 document "First (new)"', + ' #n2 heading "First changed"', + ...Array.from({ length: 9 }, (_, i) => ` #n${String(70 + i)} button "New ${String(i + 1)}"`), + ]; + const text = presenter.update(screen('b2', [...changedFirst, ...second]), { lead: 'Tapped #n3.' }); + expect(text).toContain('The screen changed substantially since revision b1.'); + expect(text).not.toContain('nodes inside, unchanged since revision'); + expect(text).toContain('#n50 document "Second"'); + expect(text).toContain(' #n60 link "Second 1"'); + }); +}); + describe('ScreenPresenter and the on-screen keyboard', () => { const KEYBOARD = [ ' #n20 keyboard "Padding-Left"', diff --git a/packages/e2e/tests/unit/tool-loop-overflow.test.ts b/packages/e2e/tests/unit/tool-loop-overflow.test.ts index 9f2f0d0f3..1ee6d0363 100644 --- a/packages/e2e/tests/unit/tool-loop-overflow.test.ts +++ b/packages/e2e/tests/unit/tool-loop-overflow.test.ts @@ -1,10 +1,12 @@ import { describe, expect, it } from 'vitest'; import { z } from 'zod'; +import { createGrammarTools } from '@e2e-dev/core/lib/agent/primitives'; import { createToolLoopExecutor } from '@e2e-dev/core/lib/agent/tool-loop'; import { projectTools } from '@e2e-dev/core/lib/agent/default-agent'; +import type { StepExecutor, StepExecutorContext } from '@e2e-dev/core/lib/agent/executor'; import { defineTool } from '@e2e-dev/core/lib/agent/tool'; import { resolveConfig } from '../../src/config/resolve-run.ts'; -import { defineEngine, type EngineHandle } from '@e2e-dev/core/lib/engine/index'; +import { defineEngine, type EngineHandle, type SemanticNode } from '@e2e-dev/core/lib/engine/index'; import { createEngineSession } from '@e2e-dev/core/lib/engine/session'; import { Deadline } from '@e2e-dev/core/lib/internal/time'; import { AttemptBudget } from '@e2e-dev/core/lib/run/budget'; @@ -75,11 +77,47 @@ const changingBigScreen = (variant: () => number) => ), }); +/** A dense page whose sidebar sits at the end, as the review's repro: about 900 nodes, the sidebar last, so the overflow clip cuts it off the screen a later fold would name. The rows change per model turn, never per capture: the observation feed re-captures until the screen settles, and a variant that moved on every capture would never settle. */ +const foldingBigScreen = (variant: () => number) => + defineEngine({ + name: 'fake', + version: '1', + spiVersion: 1, + observe: async () => { + const rows = Array.from({ length: 880 }, (_, i) => ({ + ref: { id: `n${String(i)}`, revision: '' }, + role: 'text', + name: `v${String(variant())} row ${String(i)} ${'x'.repeat(20)}`, + })); + const sidebar: SemanticNode = { + ref: { id: 'n900', revision: '' }, + role: 'navigation', + name: 'Sidebar', + children: Array.from({ length: 12 }, (_, i) => ({ + ref: { id: `n${String(901 + i)}`, revision: '' }, + role: 'link', + name: `Item ${String(i + 1)}`, + })), + }; + return snapshot([...rows, sidebar]); + }, + }); + const OVERFLOW = 'prompt is too long: 300000 tokens > 200000 maximum'; /** A read-only project tool, so the loop has a first turn to spend before the refusal. */ const peek = defineTool({ inputSchema: z.object({}), execute: async () => 'looked' }, { mutates: false }); +/** The text of one direct `observe` call on a toolset built from the context, outside any loop. */ +async function observeDirectly(context: StepExecutorContext): Promise { + const tools = createGrammarTools(context); + const observe = tools.observe; + if (observe?.execute === undefined) throw new Error('no observe tool'); + // The ToolSet type erases each tool's input to never; observe takes no input. + const output = await observe.execute({} as never, { toolCallId: 'observe', messages: [], context: undefined }); + return typeof output === 'string' ? output : output.text; +} + describe('tool loop context overflow', () => { it.each(['act', 'assert'] as const)('shrinks overflow history for a %s loop unless inconclusiveOnOverflow is set', async kind => { const model = installFakeLoopModel(({ turn }) => { @@ -132,6 +170,314 @@ describe('tool loop context overflow', () => { expect(step.metrics?.modelCalls).toBe(3); }); + it('lists the next screen in full after the overflow retry cut the screen its folds named', async () => { + let variant = 0; + const model = installFakeLoopModel(({ turn }) => { + variant = turn; + if (turn === 1) return [{ toolName: 'observe', input: {} }]; + if (turn === 2) throw new Error(OVERFLOW); + if (turn === 3) return [{ toolName: 'observe', input: {} }]; + return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'read the sidebar' } }]; + }); + const { fixtures, steps } = runtime(foldingBigScreen(() => variant), { agents: { default: { model } } }); + + await fixtures.agent.act('read the big output'); + + expect(loopCalls).toHaveLength(4); + // The second look folded the sidebar against the opening screen. The + // revision it names is whatever the settling captures minted, so the + // test reads it off the fold line instead of assuming b1. + const folded = loopCalls[1]!.lastToolResult; + const dependency = /\(\d+ nodes inside, unchanged since revision (b\d+)\)/.exec(folded)?.[1]; + expect(dependency).toMatch(/^b\d+$/); + // The retry cut the opening screen's tail, sidebar included, and the + // second screen's fold line with it. + expect(loopCalls[2]!.prompt).toContain("more characters cut: the request exceeded the model's context window"); + expect(loopCalls[2]!.prompt).not.toContain('link "Item 1"'); + // The next look lists the sidebar in full instead of folding against a + // screen the model no longer holds whole. + const next = loopCalls[3]!.lastToolResult; + expect(next).toMatch(/Current screen \(revision b\d+/); + expect(next).toContain('#n901 link "Item 1"'); + expect(next).not.toContain('unchanged since revision'); + const step = steps.all()[0]!; + expect(step.status).toBe('passed'); + expect(step.metrics?.modelCalls).toBe(3); + }); + + it('lists the next screen in full after the overflow retry cut the screen its folds named, for a custom loop built from createGrammarTools', async () => { + let variant = 0; + const model = installFakeLoopModel(({ turn }) => { + variant = turn; + if (turn === 1) return [{ toolName: 'observe', input: {} }]; + if (turn === 2) throw new Error(OVERFLOW); + if (turn === 3) return [{ toolName: 'observe', input: {} }]; + return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'read the sidebar' } }]; + }); + // The documented custom brain: the loop chassis plus the grammar tools, + // with no presenter of its own. Its first observe lists the sidebar in + // full, so the second look must not fold against the cut screen. + const executor = createToolLoopExecutor({ + name: 'custom-loop', + tools: (context) => createGrammarTools(context), + buildPrompt: () => 'read the big output', + }); + const { fixtures, steps } = runtime(foldingBigScreen(() => variant), { agents: { default: { executor, model } } }); + + await fixtures.agent.act('read the big output'); + + expect(loopCalls).toHaveLength(4); + // The first look listed the sidebar in full, at whatever revision the + // settling captures minted. + const first = loopCalls[1]!.lastToolResult; + expect(first).toContain('#n901 link "Item 1"'); + expect(first).not.toContain('unchanged since revision'); + // The retry cut the first screen's tail, sidebar included: the opening + // prompt carries no screen here, so the cut shows in the observe result + // the retried request carries. + expect(loopCalls[2]!.lastToolResult).toContain("more characters cut: the request exceeded the model's context window"); + expect(loopCalls[2]!.lastToolResult).not.toContain('link "Item 1"'); + // The next look lists the sidebar in full instead of folding against a + // screen the model no longer holds whole. + const next = loopCalls[3]!.lastToolResult; + expect(next).toMatch(/Current screen \(revision b\d+/); + expect(next).toContain('#n901 link "Item 1"'); + expect(next).not.toContain('unchanged since revision'); + const step = steps.all()[0]!; + expect(step.status).toBe('passed'); + expect(step.metrics?.modelCalls).toBe(3); + }); + + it('gives a second custom loop sharing one step context its own first screen', async () => { + const model = installFakeLoopModel(({ turn }) => { + if (turn === 1) return [{ toolName: 'observe', input: {} }]; + if (turn === 2) return [{ toolName: 'observe', input: {} }]; + if (turn === 3) return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'loop A read the sidebar' } }]; + if (turn === 4) return [{ toolName: 'observe', input: {} }]; + return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'loop B read the sidebar' } }]; + }); + // The review's repro: one custom runStep that runs two loops against the + // same step context, each built from createGrammarTools. The screen never + // changes, so a presenter shared across the loops would answer loop B's + // first look with the screen loop A already showed. + const loop = () => + createToolLoopExecutor({ + name: 'custom-loop', + tools: (context) => createGrammarTools(context), + buildPrompt: () => 'read the big output', + }); + const executor: StepExecutor = { + name: 'two-loops', + runStep: async (context) => { + await loop().runStep(context); + return loop().runStep(context); + }, + }; + const { fixtures, steps } = runtime(foldingBigScreen(() => 0), { agents: { default: { executor, model } } }); + + await fixtures.agent.act('read the big output'); + + expect(loopCalls).toHaveLength(5); + // Loop A's second look saw the screen it already holds, so the shared + // presenter has state loop B never saw. + const second = loopCalls[2]!.lastToolResult; + expect(second).toContain('Screen unchanged since revision'); + // Loop B's first look lists the sidebar in full, as its own first screen, + // instead of reporting the first loop's screen unchanged. + const loopBFirst = loopCalls[4]!.lastToolResult; + expect(loopBFirst).toMatch(/Current screen \(revision b\d+/); + expect(loopBFirst).toContain('#n901 link "Item 1"'); + expect(loopBFirst).not.toContain('unchanged since revision'); + const step = steps.all()[0]!; + expect(step.status).toBe('passed'); + }); + + it('gives tools built after two concurrent runs on one context their own first screen', async () => { + let after: string | undefined; + // Content-scripted, not turn-scripted: the two runs interleave, so no + // absolute turn belongs to either loop. + const model = installFakeLoopModel((call) => { + if (call.lastToolResult === '') return [{ toolName: 'observe', input: {} }]; + return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'loop read the sidebar' } }]; + }); + const loop = () => + createToolLoopExecutor({ + name: 'custom-loop', + tools: (context) => createGrammarTools(context), + buildPrompt: () => 'read the big output', + }); + // The review's repro: one custom runStep that runs two loops against the + // same step context at once, then builds grammar tools from it. The + // screen never changes, so a presenter left bound by either run would + // answer the later look with the screen that run already showed. + const executor: StepExecutor = { + name: 'two-concurrent-loops', + runStep: async (context) => { + const [first] = await Promise.all([loop().runStep(context), loop().runStep(context)]); + after = await observeDirectly(context); + return first!; + }, + }; + const { fixtures, steps } = runtime(foldingBigScreen(() => 0), { agents: { default: { executor, model } } }); + + await fixtures.agent.act('read the big output'); + + expect(loopCalls).toHaveLength(4); + // Each loop's first look lists the sidebar in full, as its own first screen. + for (const call of loopCalls) { + if (call.lastToolResult === '') continue; + expect(call.lastToolResult).toMatch(/Current screen \(revision b\d+/); + expect(call.lastToolResult).toContain('#n901 link "Item 1"'); + expect(call.lastToolResult).not.toContain('unchanged since revision'); + } + // The tools built after both runs get their own first screen, instead of + // reporting the screen the run that ended last already showed. + expect(after).toMatch(/Current screen \(revision b\d+/); + expect(after).toContain('#n901 link "Item 1"'); + expect(after).not.toContain('unchanged since revision'); + const step = steps.all()[0]!; + expect(step.status).toBe('passed'); + }); + + it('gives a loop nested inside another loop\'s tool its own first screen, and the outer loop keeps its own', async () => { + let outerContext: StepExecutorContext | undefined; + const inner = () => + createToolLoopExecutor({ + name: 'inner-loop', + tools: (context) => createGrammarTools(context), + buildPrompt: () => 'read the big output', + }); + // A project tool that runs a second loop against the same step context, + // as a custom executor that delegates part of a step would. + const nest = defineTool( + { + inputSchema: z.object({}), + execute: async () => { + const verdict = await inner().runStep(outerContext!); + return `nested loop ${verdict.status}`; + }, + }, + { mutates: false }, + ); + const model = installFakeLoopModel((call) => { + // The inner loop offers no `nest` tool, so the tool names tell the + // loops apart whatever order their turns interleave in. + if (call.toolNames.includes('nest')) { + if (call.lastToolResult === '') return [{ toolName: 'nest', input: {} }]; + if (call.lastToolResult.includes('nested loop passed')) return [{ toolName: 'observe', input: {} }]; + return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'outer read the sidebar after the nested run' } }]; + } + if (call.lastToolResult === '') return [{ toolName: 'observe', input: {} }]; + return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'inner read the sidebar' } }]; + }); + const outer = createToolLoopExecutor({ + name: 'outer-loop', + tools: (context, helpers) => ({ + ...projectTools(context, { nest }), + ...createGrammarTools(context, { guard: helpers.guard }), + }), + buildPrompt: () => 'read the big output', + }); + const executor: StepExecutor = { + name: 'nested-loops', + runStep: async (context) => { + outerContext = context; + return outer.runStep(context); + }, + }; + const { fixtures, steps } = runtime(foldingBigScreen(() => 0), { agents: { default: { executor, model } } }); + + await fixtures.agent.act('read the big output'); + + expect(loopCalls).toHaveLength(5); + // The inner loop's first look and the outer loop's look after the nested + // run each list the sidebar in full, as their own first screens: the + // nested run never touched the outer presenter's screen, and the outer + // look never reuses the inner run's. A run that reused the outer + // binding instead of binding fresh would answer the outer look with the + // screen the inner run already showed. + for (const call of loopCalls) { + if (call.lastToolResult === '' || call.lastToolResult.includes('nested loop passed')) continue; + expect(call.lastToolResult).toMatch(/Current screen \(revision b\d+/); + expect(call.lastToolResult).toContain('#n901 link "Item 1"'); + expect(call.lastToolResult).not.toContain('unchanged since revision'); + } + const step = steps.all()[0]!; + expect(step.status).toBe('passed'); + }); + + it('gives each toolset built outside a loop its own first screen', async () => { + let first: string | undefined; + let second: string | undefined; + // The review's repro: a custom runStep builds createGrammarTools twice, + // once per model call, and calls observe on each. The screen never + // changes, so a presenter shared across the toolsets would answer the + // second look with the screen the first toolset already showed. + const executor: StepExecutor = { + name: 'two-toolsets', + runStep: async (context) => { + first = await observeDirectly(context); + second = await observeDirectly(context); + return { status: 'passed', summary: 'read the sidebar twice' }; + }, + }; + const model = installFakeLoopModel(() => [{ toolName: 'complete_step', input: { status: 'passed', summary: 'unused' } }]); + const { fixtures, steps } = runtime(foldingBigScreen(() => 0), { agents: { default: { executor, model } } }); + + await fixtures.agent.act('read the big output'); + + expect(loopCalls).toHaveLength(0); + // Each toolset's first look lists the sidebar in full, as its own first + // screen, instead of reporting the other toolset's screen unchanged. + for (const look of [first, second]) { + expect(look).toMatch(/Current screen \(revision b\d+/); + expect(look).toContain('#n901 link "Item 1"'); + expect(look).not.toContain('unchanged since revision'); + } + const step = steps.all()[0]!; + expect(step.status).toBe('passed'); + }); + + it('gives tools built after a loop run their own first screen', async () => { + let after: string | undefined; + const model = installFakeLoopModel(({ turn }) => { + if (turn === 1) return [{ toolName: 'observe', input: {} }]; + return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'loop read the sidebar' } }]; + }); + // The review's second repro: a loop run, then createGrammarTools built + // from the same step context and observed. The screen never changes, so + // a presenter shared with the run would answer with the screen the + // loop's model already holds. + const loop = () => + createToolLoopExecutor({ + name: 'custom-loop', + tools: (context) => createGrammarTools(context), + buildPrompt: () => 'read the big output', + }); + const executor: StepExecutor = { + name: 'loop-then-tools', + runStep: async (context) => { + const verdict = await loop().runStep(context); + after = await observeDirectly(context); + return verdict; + }, + }; + const { fixtures, steps } = runtime(foldingBigScreen(() => 0), { agents: { default: { executor, model } } }); + + await fixtures.agent.act('read the big output'); + + expect(loopCalls).toHaveLength(2); + // The loop's first look listed the sidebar in full. + expect(loopCalls[1]!.lastToolResult).toContain('#n901 link "Item 1"'); + expect(loopCalls[1]!.lastToolResult).not.toContain('unchanged since revision'); + // The tools built after the run get their own first screen. + expect(after).toMatch(/Current screen \(revision b\d+/); + expect(after).toContain('#n901 link "Item 1"'); + expect(after).not.toContain('unchanged since revision'); + const step = steps.all()[0]!; + expect(step.status).toBe('passed'); + }); + it('reports CONTEXT_OVERFLOW when the shrunk request is refused again', async () => { const model = installFakeLoopModel(({ turn }) => { if (turn === 1) return [{ toolName: 'peek', input: {} }]; diff --git a/packages/e2e/tests/unit/transcript-compaction.test.ts b/packages/e2e/tests/unit/transcript-compaction.test.ts index a66d45f4d..1f0a3ed30 100644 --- a/packages/e2e/tests/unit/transcript-compaction.test.ts +++ b/packages/e2e/tests/unit/transcript-compaction.test.ts @@ -53,6 +53,16 @@ function toolOutput(message: ModelMessage): ToolResultPart['output'] { return part!.output; } +/** The full-screen text a tool message's result carries. */ +function screenText(message: ModelMessage): string { + const output = toolOutput(message); + if (output.type === 'text') return output.value; + if (output.type !== 'content') throw new Error('expected a tool result carrying a full screen'); + const textItem = output.value.find((item) => item.type === 'text'); + if (textItem === undefined) throw new Error('expected a tool result carrying a full screen'); + return textItem.text; +} + /** Characters of text a transcript carries, the text beside a screenshot included; the overflow retry's measure. */ function textChars(messages: readonly ModelMessage[]): number { return messages.reduce((total, message) => { @@ -67,6 +77,21 @@ function textChars(messages: readonly ModelMessage[]): number { }, 0); } +/** A full screen whose subtree folds against an earlier revision. */ +function foldedScreenResult(id: string, revision: string, dependsOn: string, subtree = 'Sidebar'): ModelMessage { + return { + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId: id, + toolName: 'tap', + output: { type: 'text', value: `Tapped #n3.\n\nThe screen changed substantially since revision b0. Current screen (revision ${revision}, 14 nodes):\n#n1 document\n #n2 navigation "${subtree}" (12 nodes inside, unchanged since revision ${dependsOn})\n #n20 main "Orders"` }, + }, + ], + }; +} + function changesResult(id: string): ModelMessage { return { role: 'tool', @@ -153,6 +178,105 @@ describe('compactScreenHistory', () => { expect(compacted[2]).toEqual(messages[2]); }); + it('keeps a full screen a surviving folded screen depends on', () => { + const messages = [ + opening, + changesResult('c1'), + fullScreenResult('c2', 'b2'), + foldedScreenResult('c4', 'b4', 'b0'), + ]; + const compacted = compactScreenHistory(messages, { keepStaleBytes: 0 }); + expect(compacted).not.toBe(messages); + // The opening screen is the only place the folded sidebar is listed in + // full, so it stays verbatim; the middle screen, which nothing names, + // is elided. + expect(compacted[0]).toEqual(messages[0]); + expect(compacted[1]).toEqual(messages[1]); + expect(toolOutput(compacted[2]!)).toMatchObject({ type: 'text', value: expect.stringContaining('[earlier screen elided') }); + expect(compacted[3]).toEqual(messages[3]); + }); + + it('keeps a depended-on screen that arrived as a plain-text tool result', () => { + const messages = [ + opening, + fullScreenResult('c1', 'b1'), + fullScreenResult('c2', 'b2'), + fullScreenResult('c3', 'b2a'), + foldedScreenResult('c4', 'b3', 'b2'), + ]; + const compacted = compactScreenHistory(messages, { keepStaleBytes: 0 }); + expect(compacted).not.toBe(messages); + // The sidebar was last listed in full by the b2 text result, so b3's + // fold names b2 and that result stays verbatim. The screen after b2 + // keeps the rewrite budget alive past it, so the keep-check on the text + // result is actually exercised: without it, b2 would be elided and b2a + // would survive instead. + expect(compacted[0]!.content).toContain('[earlier screen elided'); + expect(toolOutput(compacted[1]!)).toMatchObject({ type: 'text', value: expect.stringContaining('[earlier screen elided') }); + expect(compacted[2]).toBe(messages[2]); + expect(toolOutput(compacted[3]!)).toMatchObject({ type: 'text', value: expect.stringContaining('[earlier screen elided') }); + expect(compacted[4]).toBe(messages[4]); + }); + + it('elides a screen no surviving fold depends on, even when it is the oldest', () => { + const messages = [ + opening, + changesResult('c1'), + fullScreenResult('c2', 'b2'), + foldedScreenResult('c4', 'b4', 'b2'), + ]; + const compacted = compactScreenHistory(messages, { keepStaleBytes: 0 }); + expect(compacted).not.toBe(messages); + expect(compacted[0]!.content).toContain('[earlier screen elided'); + expect(compacted[2]).toEqual(messages[2]); + expect(compacted[3]).toEqual(messages[3]); + }); + + it('keeps a dependency screen transitively, through a screen that itself folded', () => { + const messages = [ + opening, + fullScreenResult('c1', 'bx'), + foldedScreenResult('c2', 'b1', 'b0', 'Footer'), + foldedScreenResult('c3', 'b2', 'b1'), + ]; + const compacted = compactScreenHistory(messages, { keepStaleBytes: 0 }); + expect(compacted).not.toBe(messages); + // The newest screen names b1, and b1 names b0: both stay, so every fold + // line still points at a screen the model holds. Only the screen no + // fold names is elided. + expect(compacted[0]).toEqual(messages[0]); + expect(toolOutput(compacted[1]!)).toMatchObject({ type: 'text', value: expect.stringContaining('[earlier screen elided') }); + expect(compacted[2]).toEqual(messages[2]); + expect(compacted[3]).toEqual(messages[3]); + }); + + it('leaves a screenshot result whose screens are all kept untouched, without spending the elision budget', () => { + const messages = [ + pixelScreenResult('c1', 'b1'), + fullScreenResult('c2', 'b2'), + fullScreenResult('c3', 'b3'), + foldedScreenResult('c4', 'b4', 'b1'), + ]; + // Only b2 and b3 may be elided: b1 stays as the fold dependency of b4, + // and b4 stays as the newest screen. A budget that exactly covers the + // elidable screens leaves the transcript alone only when the kept + // screens spend none of it; counting b1 and b4 would push the total + // over budget and elide b2 and b3. + const elidableBytes = [messages[1]!, messages[2]!].reduce( + (bytes, message) => bytes + Buffer.byteLength(screenText(message), 'utf8'), + 0, + ); + expect(compactScreenHistory(messages, { keepStaleBytes: elidableBytes })).toBe(messages); + const compacted = compactScreenHistory(messages, { keepStaleBytes: 0 }); + expect(compacted).not.toBe(messages); + // The opening screenshot result carries only the kept b1 screen, so it + // is returned untouched while the elidable screens go. + expect(compacted[0]).toBe(messages[0]); + expect(toolOutput(compacted[1]!)).toMatchObject({ type: 'text', value: expect.stringContaining('[earlier screen elided') }); + expect(toolOutput(compacted[2]!)).toMatchObject({ type: 'text', value: expect.stringContaining('[earlier screen elided') }); + expect(compacted[3]).toBe(messages[3]); + }); + it('shrinks a pixel-mode history on the budget-free pass the overflow retry runs after the cache-friendly pass kept it', () => { const messages = [opening, pixelScreenResult('c1', 'b1', 40), changesResult('c2'), fullScreenResult('c3', 'b3'), pixelScreenResult('c4', 'b4', 40)]; const kept = compactScreenHistory(messages); diff --git a/plugins/e2e/skills/e2e/references/agent.md b/plugins/e2e/skills/e2e/references/agent.md index c564dbb33..d5f951647 100644 --- a/plugins/e2e/skills/e2e/references/agent.md +++ b/plugins/e2e/skills/e2e/references/agent.md @@ -355,7 +355,11 @@ summaries, and your context; never raw HTML, cookies, headers, environment values, or a `Secret`'s value; password fields masked. The first screen of a step arrives whole; later action results report what changed, keyed by node ids stable while an element exists, or the whole screen when -most changed. Pixels arrive through `vision` on a judgment or the act loop's +most changed, with large subtrees unchanged since the screen that last +listed them in full folded to one line each, naming the node count and that +revision. `observe` returns what changed since the last screen, +not the whole tree; only the MCP `observe` returns the complete tree every +time. Pixels arrive through `vision` on a judgment or the act loop's `screenshot` and point tools, with secure fields masked. When the browser engine's tree capture times out, the model gets a screenshot and a warning, a judgment needs `vision: true`, and no control may be inferred diff --git a/skills/e2e/references/agent.md b/skills/e2e/references/agent.md index c564dbb33..d5f951647 100644 --- a/skills/e2e/references/agent.md +++ b/skills/e2e/references/agent.md @@ -355,7 +355,11 @@ summaries, and your context; never raw HTML, cookies, headers, environment values, or a `Secret`'s value; password fields masked. The first screen of a step arrives whole; later action results report what changed, keyed by node ids stable while an element exists, or the whole screen when -most changed. Pixels arrive through `vision` on a judgment or the act loop's +most changed, with large subtrees unchanged since the screen that last +listed them in full folded to one line each, naming the node count and that +revision. `observe` returns what changed since the last screen, +not the whole tree; only the MCP `observe` returns the complete tree every +time. Pixels arrive through `vision` on a judgment or the act loop's `screenshot` and point tools, with secure fields masked. When the browser engine's tree capture times out, the model gets a screenshot and a warning, a judgment needs `vision: true`, and no control may be inferred